mod document;
mod entities;
mod tokenizer;
mod tree_builder;
pub use document::{Attribute, Children, Document, Node, NodeId, NodeKind};
pub use tokenizer::Position;
use tokenizer::Tokenizer;
use tree_builder::TreeBuilder;
pub fn parse(input: &str) -> Document {
let mut tokenizer = Tokenizer::new(input);
let mut tree_builder = TreeBuilder::new();
while let Some(token) = tokenizer.next() {
if let Some(state) = tree_builder.process_token(&token.kind, token.position) {
tokenizer.switch_to(state);
}
tokenizer.set_in_foreign_content(tree_builder.is_in_foreign_content());
}
tree_builder.stop_parsing();
tree_builder.into_document()
}
#[cfg(test)]
mod tests {
use super::parse;
use crate::document::{Document, NodeId, NodeKind};
use crate::tree_builder::{HTML_NAMESPACE, MATHML_NAMESPACE, SVG_NAMESPACE};
fn body_of(document: &Document) -> NodeId {
let root = document.root();
let html = document
.children(root)
.find(|&node| matches!(document.node(node).kind, NodeKind::Element { .. }))
.unwrap();
document.children(html).nth(1).unwrap()
}
#[test]
fn parses_a_minimal_document_into_the_expected_tree_shape() {
let document = parse(
"<!DOCTYPE html><html><head><title>Hi</title></head><body><p>Hello</p></body></html>",
);
let root = document.root();
let root_children: Vec<_> = document.children(root).collect();
assert_eq!(root_children.len(), 2);
assert_eq!(
document.node(root_children[0]).kind,
NodeKind::Doctype {
name: Some("html".to_owned()),
public_identifier: Some(String::new()),
system_identifier: Some(String::new()),
}
);
let html = root_children[1];
let html_children: Vec<_> = document.children(html).collect();
assert_eq!(html_children.len(), 2);
let (head, body) = (html_children[0], html_children[1]);
let title = document.children(head).next().unwrap();
let NodeKind::Element { name, .. } = &document.node(title).kind else {
unreachable!()
};
assert_eq!(name, "title");
let title_text = document.children(title).next().unwrap();
assert_eq!(
document.node(title_text).kind,
NodeKind::Text {
content: "Hi".to_owned()
}
);
let p = document.children(body).next().unwrap();
let NodeKind::Element { name, .. } = &document.node(p).kind else {
unreachable!()
};
assert_eq!(name, "p");
let p_text = document.children(p).next().unwrap();
assert_eq!(
document.node(p_text).kind,
NodeKind::Text {
content: "Hello".to_owned()
}
);
}
#[test]
fn parses_implied_html_head_body_when_missing() {
let document = parse("<p>Hello</p>");
let root = document.root();
assert_eq!(document.children(root).count(), 1);
let html = document.children(root).next().unwrap();
let html_children: Vec<_> = document.children(html).collect();
assert_eq!(html_children.len(), 2);
let body = html_children[1];
let p = document.children(body).next().unwrap();
let NodeKind::Element { name, .. } = &document.node(p).kind else {
unreachable!()
};
assert_eq!(name, "p");
}
#[test]
fn rcdata_element_content_is_not_parsed_as_markup() {
let document = parse("<title><b>not bold</b></title>");
let root = document.root();
let html = document.children(root).next().unwrap();
let head = document.children(html).next().unwrap();
let title = document.children(head).next().unwrap();
let text = document.children(title).next().unwrap();
assert_eq!(
document.node(text).kind,
NodeKind::Text {
content: "<b>not bold</b>".to_owned()
}
);
}
#[test]
fn parse_syncs_in_foreign_content_for_cdata_sections() {
let document = parse("<svg><![CDATA[hello]]></svg>");
let root = document.root();
let html = document.children(root).next().unwrap();
let body = document.children(html).nth(1).unwrap();
let svg = document.children(body).next().unwrap();
let content = document.children(svg).next().unwrap();
assert_eq!(
document.node(content).kind,
NodeKind::Text {
content: "hello".to_owned()
}
);
}
#[test]
fn cdata_outside_foreign_content_becomes_a_bogus_comment() {
let document = parse("<p><![CDATA[hello]]></p>");
let root = document.root();
let html = document.children(root).next().unwrap();
let body = document.children(html).nth(1).unwrap();
let p = document.children(body).next().unwrap();
let content = document.children(p).next().unwrap();
assert_eq!(
document.node(content).kind,
NodeKind::Comment {
content: "[CDATA[hello]]".to_owned()
}
);
}
#[test]
fn optional_end_tags_produce_sibling_li_elements() {
let document = parse("<ul><li>a<li>b</ul>");
let body = body_of(&document);
let ul = document.children(body).next().unwrap();
let items: Vec<_> = document.children(ul).collect();
assert_eq!(items.len(), 2);
for (&li, expected_text) in items.iter().zip(["a", "b"]) {
let NodeKind::Element { name, .. } = &document.node(li).kind else {
unreachable!()
};
assert_eq!(name, "li");
let text = document.children(li).next().unwrap();
assert_eq!(
document.node(text).kind,
NodeKind::Text {
content: expected_text.to_owned()
}
);
}
}
#[test]
fn svg_element_keeps_svg_namespace_end_to_end() {
let document = parse("<svg><circle/></svg>");
let body = body_of(&document);
let svg = document.children(body).next().unwrap();
assert_eq!(
document.node(svg).kind,
NodeKind::Element {
name: "svg".to_owned(),
namespace: Some(SVG_NAMESPACE.to_owned()),
attributes: vec![],
}
);
let circle = document.children(svg).next().unwrap();
assert_eq!(
document.node(circle).kind,
NodeKind::Element {
name: "circle".to_owned(),
namespace: Some(SVG_NAMESPACE.to_owned()),
attributes: vec![],
}
);
}
#[test]
fn mathml_element_keeps_mathml_namespace_end_to_end() {
let document = parse("<math><mi>x</mi></math>");
let body = body_of(&document);
let math = document.children(body).next().unwrap();
assert_eq!(
document.node(math).kind,
NodeKind::Element {
name: "math".to_owned(),
namespace: Some(MATHML_NAMESPACE.to_owned()),
attributes: vec![],
}
);
let mi = document.children(math).next().unwrap();
assert_eq!(
document.node(mi).kind,
NodeKind::Element {
name: "mi".to_owned(),
namespace: Some(MATHML_NAMESPACE.to_owned()),
attributes: vec![],
}
);
let text = document.children(mi).next().unwrap();
assert_eq!(
document.node(text).kind,
NodeKind::Text {
content: "x".to_owned()
}
);
}
#[test]
fn script_content_is_not_tokenized_as_markup() {
let document = parse("<script>1 < 2;</script>");
let root = document.root();
let html = document.children(root).next().unwrap();
let head = document.children(html).next().unwrap();
let script = document.children(head).next().unwrap();
let NodeKind::Element { name, .. } = &document.node(script).kind else {
unreachable!()
};
assert_eq!(name, "script");
let text = document.children(script).next().unwrap();
assert_eq!(
document.node(text).kind,
NodeKind::Text {
content: "1 < 2;".to_owned()
}
);
}
#[test]
fn style_content_is_not_tokenized_as_markup() {
let document = parse("<style>a{color:red}</style>");
let root = document.root();
let html = document.children(root).next().unwrap();
let head = document.children(html).next().unwrap();
let style = document.children(head).next().unwrap();
let NodeKind::Element { name, .. } = &document.node(style).kind else {
unreachable!()
};
assert_eq!(name, "style");
let text = document.children(style).next().unwrap();
assert_eq!(
document.node(text).kind,
NodeKind::Text {
content: "a{color:red}".to_owned()
}
);
}
#[test]
fn named_character_references_resolve_to_decoded_text() {
let document = parse("<p>& ©</p>");
let body = body_of(&document);
let p = document.children(body).next().unwrap();
let text = document.children(p).next().unwrap();
assert_eq!(
document.node(text).kind,
NodeKind::Text {
content: "& \u{a9}".to_owned()
}
);
}
#[test]
fn custom_element_gets_html_namespace_like_any_plain_element() {
let document = parse("<my-widget>hi</my-widget>");
let body = body_of(&document);
let widget = document.children(body).next().unwrap();
assert_eq!(
document.node(widget).kind,
NodeKind::Element {
name: "my-widget".to_owned(),
namespace: Some(HTML_NAMESPACE.to_owned()),
attributes: vec![],
}
);
let text = document.children(widget).next().unwrap();
assert_eq!(
document.node(text).kind,
NodeKind::Text {
content: "hi".to_owned()
}
);
}
#[test]
fn xml_lang_attribute_stays_a_literal_unnamespaced_attribute_name() {
let document = parse(r#"<p xml:lang="de">hi</p>"#);
let body = body_of(&document);
let p = document.children(body).next().unwrap();
let NodeKind::Element { attributes, .. } = &document.node(p).kind else {
unreachable!()
};
assert_eq!(attributes.len(), 1);
assert_eq!(attributes[0].name, "xml:lang");
assert_eq!(attributes[0].value, "de");
assert_eq!(attributes[0].namespace, None);
}
#[test]
fn table_without_tbody_or_tr_gets_them_synthesized() {
let document = parse("<table><td>x</td></table>");
let body = body_of(&document);
let table = document.children(body).next().unwrap();
let tbody = document.children(table).next().unwrap();
let NodeKind::Element { name, .. } = &document.node(tbody).kind else {
unreachable!()
};
assert_eq!(name, "tbody");
let tr = document.children(tbody).next().unwrap();
let NodeKind::Element { name, .. } = &document.node(tr).kind else {
unreachable!()
};
assert_eq!(name, "tr");
let td = document.children(tr).next().unwrap();
let NodeKind::Element { name, .. } = &document.node(td).kind else {
unreachable!()
};
assert_eq!(name, "td");
let text = document.children(td).next().unwrap();
assert_eq!(
document.node(text).kind,
NodeKind::Text {
content: "x".to_owned()
}
);
}
#[test]
fn adoption_agency_spec_example_misnested_b_i_tags() {
let document = parse("<p>1<b>2<i>3</b>4</i>5</p>");
let body = body_of(&document);
let p = document.children(body).next().unwrap();
let p_children: Vec<_> = document.children(p).collect();
assert_eq!(p_children.len(), 4);
assert_eq!(
document.node(p_children[0]).kind,
NodeKind::Text {
content: "1".to_owned()
}
);
let b = p_children[1];
let NodeKind::Element { name, .. } = &document.node(b).kind else {
unreachable!()
};
assert_eq!(name, "b");
let b_children: Vec<_> = document.children(b).collect();
assert_eq!(b_children.len(), 2);
assert_eq!(
document.node(b_children[0]).kind,
NodeKind::Text {
content: "2".to_owned()
}
);
let inner_i = b_children[1];
let NodeKind::Element { name, .. } = &document.node(inner_i).kind else {
unreachable!()
};
assert_eq!(name, "i");
let inner_i_text = document.children(inner_i).next().unwrap();
assert_eq!(
document.node(inner_i_text).kind,
NodeKind::Text {
content: "3".to_owned()
}
);
let outer_i = p_children[2];
let NodeKind::Element { name, .. } = &document.node(outer_i).kind else {
unreachable!()
};
assert_eq!(name, "i");
let outer_i_text = document.children(outer_i).next().unwrap();
assert_eq!(
document.node(outer_i_text).kind,
NodeKind::Text {
content: "4".to_owned()
}
);
assert_eq!(
document.node(p_children[3]).kind,
NodeKind::Text {
content: "5".to_owned()
}
);
}
#[test]
fn adoption_agency_spec_example_indirectly_nests_two_a_elements_via_table_misnesting() {
let document = parse(r#"<a href="a">a<table><a href="b">b</table>x"#);
let body = body_of(&document);
let body_children: Vec<_> = document.children(body).collect();
assert_eq!(body_children.len(), 2);
let a1 = body_children[0];
let a1_children: Vec<_> = document.children(a1).collect();
assert_eq!(a1_children.len(), 3);
assert_eq!(
document.node(a1_children[0]).kind,
NodeKind::Text {
content: "a".to_owned()
}
);
let a2 = a1_children[1];
let NodeKind::Element { name, .. } = &document.node(a2).kind else {
unreachable!()
};
assert_eq!(name, "a");
let a2_text = document.children(a2).next().unwrap();
assert_eq!(
document.node(a2_text).kind,
NodeKind::Text {
content: "b".to_owned()
}
);
let table = a1_children[2];
let NodeKind::Element { name, .. } = &document.node(table).kind else {
unreachable!()
};
assert_eq!(name, "table");
assert_eq!(document.children(table).count(), 0);
let a3 = body_children[1];
let NodeKind::Element { name, .. } = &document.node(a3).kind else {
unreachable!()
};
assert_eq!(name, "a");
let a3_text = document.children(a3).next().unwrap();
assert_eq!(
document.node(a3_text).kind,
NodeKind::Text {
content: "x".to_owned()
}
);
}
#[test]
fn quirks_mode_changes_whether_table_closes_an_open_p_element() {
let no_quirks = parse("<!DOCTYPE html><p><table></table>");
let body = body_of(&no_quirks);
let children: Vec<_> = no_quirks.children(body).collect();
assert_eq!(children.len(), 2);
let NodeKind::Element { name, .. } = &no_quirks.node(children[0]).kind else {
unreachable!()
};
assert_eq!(name, "p");
assert_eq!(no_quirks.children(children[0]).count(), 0);
let NodeKind::Element { name, .. } = &no_quirks.node(children[1]).kind else {
unreachable!()
};
assert_eq!(name, "table");
let quirks = parse("<p><table></table>"); let body = body_of(&quirks);
let children: Vec<_> = quirks.children(body).collect();
assert_eq!(children.len(), 1);
let NodeKind::Element { name, .. } = &quirks.node(children[0]).kind else {
unreachable!()
};
assert_eq!(name, "p");
let p_children: Vec<_> = quirks.children(children[0]).collect();
assert_eq!(p_children.len(), 1);
let NodeKind::Element { name, .. } = &quirks.node(p_children[0]).kind else {
unreachable!()
};
assert_eq!(name, "table");
}
#[test]
fn end_tag_that_walks_out_of_foreign_content_does_not_loop_forever() {
let document = parse("<a><svg></a>");
let body = body_of(&document);
assert_eq!(document.children(body).count(), 1);
}
#[test]
fn template_end_tag_resets_a_stale_insertion_mode() {
let document = parse("<table><thead><template><td></template></table>");
let body = body_of(&document);
assert_eq!(document.children(body).count(), 1);
}
#[test]
fn frameset_document_replaces_body_and_ignores_stray_text() {
let document = parse("<!DOCTYPE html><frameset>test");
let root = document.root();
let root_children: Vec<_> = document.children(root).collect();
assert_eq!(root_children.len(), 2);
assert_eq!(
document.node(root_children[0]).kind,
NodeKind::Doctype {
name: Some("html".to_owned()),
public_identifier: Some(String::new()),
system_identifier: Some(String::new()),
}
);
let html = root_children[1];
let html_children: Vec<_> = document.children(html).collect();
assert_eq!(html_children.len(), 2);
let NodeKind::Element { name, .. } = &document.node(html_children[0]).kind else {
unreachable!()
};
assert_eq!(name, "head");
let frameset = html_children[1];
let NodeKind::Element { name, .. } = &document.node(frameset).kind else {
unreachable!()
};
assert_eq!(name, "frameset");
assert_eq!(document.children(frameset).count(), 0);
}
#[test]
fn template_content_is_a_separate_fragment_from_the_template_element() {
let document = parse("<body><template>Hello</template>");
let body = body_of(&document);
let template = document.children(body).next().unwrap();
let NodeKind::Element { name, .. } = &document.node(template).kind else {
unreachable!()
};
assert_eq!(name, "template");
let template_children: Vec<_> = document.children(template).collect();
assert_eq!(template_children.len(), 1);
let content = template_children[0];
assert_eq!(document.node(content).kind, NodeKind::DocumentFragment);
let content_children: Vec<_> = document.children(content).collect();
assert_eq!(content_children.len(), 1);
assert_eq!(
document.node(content_children[0]).kind,
NodeKind::Text {
content: "Hello".to_owned()
}
);
}
#[test]
fn selected_option_content_is_mirrored_into_selectedcontent() {
let document =
parse("<select><button><selectedcontent></button><option>X<option selected>Y");
let body = body_of(&document);
let select = document.children(body).next().unwrap();
let button = document.children(select).next().unwrap();
let selectedcontent = document.children(button).next().unwrap();
let selectedcontent_children: Vec<_> = document.children(selectedcontent).collect();
assert_eq!(selectedcontent_children.len(), 1);
assert_eq!(
document.node(selectedcontent_children[0]).kind,
NodeKind::Text {
content: "Y".to_owned()
}
);
}
}