1mod document;
10mod entities;
11mod tokenizer;
12mod tree_builder;
13
14pub use document::{Attribute, Children, Document, Node, NodeId, NodeKind};
15pub use tokenizer::Position;
16
17use tokenizer::Tokenizer;
18use tree_builder::TreeBuilder;
19
20pub fn parse(input: &str) -> Document {
36 let mut tokenizer = Tokenizer::new(input);
37 let mut tree_builder = TreeBuilder::new();
38 while let Some(token) = tokenizer.next() {
39 if let Some(state) = tree_builder.process_token(&token.kind, token.position) {
40 tokenizer.switch_to(state);
41 }
42 tokenizer.set_in_foreign_content(tree_builder.is_in_foreign_content());
43 }
44 tree_builder.stop_parsing();
45 tree_builder.into_document()
46}
47
48#[cfg(test)]
49mod tests {
50 use super::parse;
51 use crate::document::{Document, NodeId, NodeKind};
52 use crate::tree_builder::{HTML_NAMESPACE, MATHML_NAMESPACE, SVG_NAMESPACE};
53
54 fn body_of(document: &Document) -> NodeId {
57 let root = document.root();
58 let html = document
62 .children(root)
63 .find(|&node| matches!(document.node(node).kind, NodeKind::Element { .. }))
64 .unwrap();
65 document.children(html).nth(1).unwrap()
66 }
67
68 #[test]
69 fn parses_a_minimal_document_into_the_expected_tree_shape() {
70 let document = parse(
71 "<!DOCTYPE html><html><head><title>Hi</title></head><body><p>Hello</p></body></html>",
72 );
73
74 let root = document.root();
75 let root_children: Vec<_> = document.children(root).collect();
76 assert_eq!(root_children.len(), 2);
77 assert_eq!(
78 document.node(root_children[0]).kind,
79 NodeKind::Doctype {
80 name: Some("html".to_owned()),
81 public_identifier: Some(String::new()),
82 system_identifier: Some(String::new()),
83 }
84 );
85
86 let html = root_children[1];
87 let html_children: Vec<_> = document.children(html).collect();
88 assert_eq!(html_children.len(), 2);
89 let (head, body) = (html_children[0], html_children[1]);
90
91 let title = document.children(head).next().unwrap();
92 let NodeKind::Element { name, .. } = &document.node(title).kind else {
93 unreachable!()
94 };
95 assert_eq!(name, "title");
96 let title_text = document.children(title).next().unwrap();
97 assert_eq!(
98 document.node(title_text).kind,
99 NodeKind::Text {
100 content: "Hi".to_owned()
101 }
102 );
103
104 let p = document.children(body).next().unwrap();
105 let NodeKind::Element { name, .. } = &document.node(p).kind else {
106 unreachable!()
107 };
108 assert_eq!(name, "p");
109 let p_text = document.children(p).next().unwrap();
110 assert_eq!(
111 document.node(p_text).kind,
112 NodeKind::Text {
113 content: "Hello".to_owned()
114 }
115 );
116 }
117
118 #[test]
119 fn parses_implied_html_head_body_when_missing() {
120 let document = parse("<p>Hello</p>");
121
122 let root = document.root();
123 assert_eq!(document.children(root).count(), 1);
124 let html = document.children(root).next().unwrap();
125 let html_children: Vec<_> = document.children(html).collect();
126 assert_eq!(html_children.len(), 2);
127 let body = html_children[1];
128
129 let p = document.children(body).next().unwrap();
130 let NodeKind::Element { name, .. } = &document.node(p).kind else {
131 unreachable!()
132 };
133 assert_eq!(name, "p");
134 }
135
136 #[test]
137 fn rcdata_element_content_is_not_parsed_as_markup() {
138 let document = parse("<title><b>not bold</b></title>");
139
140 let root = document.root();
141 let html = document.children(root).next().unwrap();
142 let head = document.children(html).next().unwrap();
143 let title = document.children(head).next().unwrap();
144 let text = document.children(title).next().unwrap();
145 assert_eq!(
146 document.node(text).kind,
147 NodeKind::Text {
148 content: "<b>not bold</b>".to_owned()
149 }
150 );
151 }
152
153 #[test]
154 fn parse_syncs_in_foreign_content_for_cdata_sections() {
155 let document = parse("<svg><![CDATA[hello]]></svg>");
160
161 let root = document.root();
162 let html = document.children(root).next().unwrap();
163 let body = document.children(html).nth(1).unwrap();
164 let svg = document.children(body).next().unwrap();
165 let content = document.children(svg).next().unwrap();
166 assert_eq!(
167 document.node(content).kind,
168 NodeKind::Text {
169 content: "hello".to_owned()
170 }
171 );
172 }
173
174 #[test]
175 fn cdata_outside_foreign_content_becomes_a_bogus_comment() {
176 let document = parse("<p><![CDATA[hello]]></p>");
177
178 let root = document.root();
179 let html = document.children(root).next().unwrap();
180 let body = document.children(html).nth(1).unwrap();
181 let p = document.children(body).next().unwrap();
182 let content = document.children(p).next().unwrap();
183 assert_eq!(
184 document.node(content).kind,
185 NodeKind::Comment {
186 content: "[CDATA[hello]]".to_owned()
187 }
188 );
189 }
190
191 #[test]
200 fn optional_end_tags_produce_sibling_li_elements() {
201 let document = parse("<ul><li>a<li>b</ul>");
203 let body = body_of(&document);
204 let ul = document.children(body).next().unwrap();
205 let items: Vec<_> = document.children(ul).collect();
206 assert_eq!(items.len(), 2);
207 for (&li, expected_text) in items.iter().zip(["a", "b"]) {
208 let NodeKind::Element { name, .. } = &document.node(li).kind else {
209 unreachable!()
210 };
211 assert_eq!(name, "li");
212 let text = document.children(li).next().unwrap();
213 assert_eq!(
214 document.node(text).kind,
215 NodeKind::Text {
216 content: expected_text.to_owned()
217 }
218 );
219 }
220 }
221
222 #[test]
223 fn svg_element_keeps_svg_namespace_end_to_end() {
224 let document = parse("<svg><circle/></svg>");
226 let body = body_of(&document);
227 let svg = document.children(body).next().unwrap();
228 assert_eq!(
229 document.node(svg).kind,
230 NodeKind::Element {
231 name: "svg".to_owned(),
232 namespace: Some(SVG_NAMESPACE.to_owned()),
233 attributes: vec![],
234 }
235 );
236 let circle = document.children(svg).next().unwrap();
237 assert_eq!(
238 document.node(circle).kind,
239 NodeKind::Element {
240 name: "circle".to_owned(),
241 namespace: Some(SVG_NAMESPACE.to_owned()),
242 attributes: vec![],
243 }
244 );
245 }
246
247 #[test]
248 fn mathml_element_keeps_mathml_namespace_end_to_end() {
249 let document = parse("<math><mi>x</mi></math>");
251 let body = body_of(&document);
252 let math = document.children(body).next().unwrap();
253 assert_eq!(
254 document.node(math).kind,
255 NodeKind::Element {
256 name: "math".to_owned(),
257 namespace: Some(MATHML_NAMESPACE.to_owned()),
258 attributes: vec![],
259 }
260 );
261 let mi = document.children(math).next().unwrap();
262 assert_eq!(
263 document.node(mi).kind,
264 NodeKind::Element {
265 name: "mi".to_owned(),
266 namespace: Some(MATHML_NAMESPACE.to_owned()),
267 attributes: vec![],
268 }
269 );
270 let text = document.children(mi).next().unwrap();
271 assert_eq!(
272 document.node(text).kind,
273 NodeKind::Text {
274 content: "x".to_owned()
275 }
276 );
277 }
278
279 #[test]
280 fn script_content_is_not_tokenized_as_markup() {
281 let document = parse("<script>1 < 2;</script>");
283 let root = document.root();
284 let html = document.children(root).next().unwrap();
285 let head = document.children(html).next().unwrap();
286 let script = document.children(head).next().unwrap();
287 let NodeKind::Element { name, .. } = &document.node(script).kind else {
288 unreachable!()
289 };
290 assert_eq!(name, "script");
291 let text = document.children(script).next().unwrap();
292 assert_eq!(
293 document.node(text).kind,
294 NodeKind::Text {
295 content: "1 < 2;".to_owned()
296 }
297 );
298 }
299
300 #[test]
301 fn style_content_is_not_tokenized_as_markup() {
302 let document = parse("<style>a{color:red}</style>");
304 let root = document.root();
305 let html = document.children(root).next().unwrap();
306 let head = document.children(html).next().unwrap();
307 let style = document.children(head).next().unwrap();
308 let NodeKind::Element { name, .. } = &document.node(style).kind else {
309 unreachable!()
310 };
311 assert_eq!(name, "style");
312 let text = document.children(style).next().unwrap();
313 assert_eq!(
314 document.node(text).kind,
315 NodeKind::Text {
316 content: "a{color:red}".to_owned()
317 }
318 );
319 }
320
321 #[test]
322 fn named_character_references_resolve_to_decoded_text() {
323 let document = parse("<p>& ©</p>");
325 let body = body_of(&document);
326 let p = document.children(body).next().unwrap();
327 let text = document.children(p).next().unwrap();
328 assert_eq!(
329 document.node(text).kind,
330 NodeKind::Text {
331 content: "& \u{a9}".to_owned()
332 }
333 );
334 }
335
336 #[test]
337 fn custom_element_gets_html_namespace_like_any_plain_element() {
338 let document = parse("<my-widget>hi</my-widget>");
340 let body = body_of(&document);
341 let widget = document.children(body).next().unwrap();
342 assert_eq!(
343 document.node(widget).kind,
344 NodeKind::Element {
345 name: "my-widget".to_owned(),
346 namespace: Some(HTML_NAMESPACE.to_owned()),
347 attributes: vec![],
348 }
349 );
350 let text = document.children(widget).next().unwrap();
351 assert_eq!(
352 document.node(text).kind,
353 NodeKind::Text {
354 content: "hi".to_owned()
355 }
356 );
357 }
358
359 #[test]
360 fn xml_lang_attribute_stays_a_literal_unnamespaced_attribute_name() {
361 let document = parse(r#"<p xml:lang="de">hi</p>"#);
370 let body = body_of(&document);
371 let p = document.children(body).next().unwrap();
372 let NodeKind::Element { attributes, .. } = &document.node(p).kind else {
373 unreachable!()
374 };
375 assert_eq!(attributes.len(), 1);
376 assert_eq!(attributes[0].name, "xml:lang");
377 assert_eq!(attributes[0].value, "de");
378 assert_eq!(attributes[0].namespace, None);
379 }
380
381 #[test]
382 fn table_without_tbody_or_tr_gets_them_synthesized() {
383 let document = parse("<table><td>x</td></table>");
386 let body = body_of(&document);
387 let table = document.children(body).next().unwrap();
388 let tbody = document.children(table).next().unwrap();
389 let NodeKind::Element { name, .. } = &document.node(tbody).kind else {
390 unreachable!()
391 };
392 assert_eq!(name, "tbody");
393 let tr = document.children(tbody).next().unwrap();
394 let NodeKind::Element { name, .. } = &document.node(tr).kind else {
395 unreachable!()
396 };
397 assert_eq!(name, "tr");
398 let td = document.children(tr).next().unwrap();
399 let NodeKind::Element { name, .. } = &document.node(td).kind else {
400 unreachable!()
401 };
402 assert_eq!(name, "td");
403 let text = document.children(td).next().unwrap();
404 assert_eq!(
405 document.node(text).kind,
406 NodeKind::Text {
407 content: "x".to_owned()
408 }
409 );
410 }
411
412 #[test]
413 fn adoption_agency_spec_example_misnested_b_i_tags() {
414 let document = parse("<p>1<b>2<i>3</b>4</i>5</p>");
420 let body = body_of(&document);
421 let p = document.children(body).next().unwrap();
422 let p_children: Vec<_> = document.children(p).collect();
423 assert_eq!(p_children.len(), 4);
424 assert_eq!(
425 document.node(p_children[0]).kind,
426 NodeKind::Text {
427 content: "1".to_owned()
428 }
429 );
430
431 let b = p_children[1];
432 let NodeKind::Element { name, .. } = &document.node(b).kind else {
433 unreachable!()
434 };
435 assert_eq!(name, "b");
436 let b_children: Vec<_> = document.children(b).collect();
437 assert_eq!(b_children.len(), 2);
438 assert_eq!(
439 document.node(b_children[0]).kind,
440 NodeKind::Text {
441 content: "2".to_owned()
442 }
443 );
444 let inner_i = b_children[1];
445 let NodeKind::Element { name, .. } = &document.node(inner_i).kind else {
446 unreachable!()
447 };
448 assert_eq!(name, "i");
449 let inner_i_text = document.children(inner_i).next().unwrap();
450 assert_eq!(
451 document.node(inner_i_text).kind,
452 NodeKind::Text {
453 content: "3".to_owned()
454 }
455 );
456
457 let outer_i = p_children[2];
458 let NodeKind::Element { name, .. } = &document.node(outer_i).kind else {
459 unreachable!()
460 };
461 assert_eq!(name, "i");
462 let outer_i_text = document.children(outer_i).next().unwrap();
463 assert_eq!(
464 document.node(outer_i_text).kind,
465 NodeKind::Text {
466 content: "4".to_owned()
467 }
468 );
469
470 assert_eq!(
473 document.node(p_children[3]).kind,
474 NodeKind::Text {
475 content: "5".to_owned()
476 }
477 );
478 }
479
480 #[test]
481 fn adoption_agency_spec_example_indirectly_nests_two_a_elements_via_table_misnesting() {
482 let document = parse(r#"<a href="a">a<table><a href="b">b</table>x"#);
490 let body = body_of(&document);
491 let body_children: Vec<_> = document.children(body).collect();
492 assert_eq!(body_children.len(), 2);
493
494 let a1 = body_children[0];
495 let a1_children: Vec<_> = document.children(a1).collect();
496 assert_eq!(a1_children.len(), 3);
497 assert_eq!(
498 document.node(a1_children[0]).kind,
499 NodeKind::Text {
500 content: "a".to_owned()
501 }
502 );
503 let a2 = a1_children[1];
504 let NodeKind::Element { name, .. } = &document.node(a2).kind else {
505 unreachable!()
506 };
507 assert_eq!(name, "a");
508 let a2_text = document.children(a2).next().unwrap();
509 assert_eq!(
510 document.node(a2_text).kind,
511 NodeKind::Text {
512 content: "b".to_owned()
513 }
514 );
515 let table = a1_children[2];
516 let NodeKind::Element { name, .. } = &document.node(table).kind else {
517 unreachable!()
518 };
519 assert_eq!(name, "table");
520 assert_eq!(document.children(table).count(), 0);
521
522 let a3 = body_children[1];
526 let NodeKind::Element { name, .. } = &document.node(a3).kind else {
527 unreachable!()
528 };
529 assert_eq!(name, "a");
530 let a3_text = document.children(a3).next().unwrap();
531 assert_eq!(
532 document.node(a3_text).kind,
533 NodeKind::Text {
534 content: "x".to_owned()
535 }
536 );
537 }
538
539 #[test]
540 fn quirks_mode_changes_whether_table_closes_an_open_p_element() {
541 let no_quirks = parse("<!DOCTYPE html><p><table></table>");
546 let body = body_of(&no_quirks);
547 let children: Vec<_> = no_quirks.children(body).collect();
548 assert_eq!(children.len(), 2);
549 let NodeKind::Element { name, .. } = &no_quirks.node(children[0]).kind else {
550 unreachable!()
551 };
552 assert_eq!(name, "p");
553 assert_eq!(no_quirks.children(children[0]).count(), 0);
554 let NodeKind::Element { name, .. } = &no_quirks.node(children[1]).kind else {
555 unreachable!()
556 };
557 assert_eq!(name, "table");
558
559 let quirks = parse("<p><table></table>"); let body = body_of(&quirks);
561 let children: Vec<_> = quirks.children(body).collect();
562 assert_eq!(children.len(), 1);
563 let NodeKind::Element { name, .. } = &quirks.node(children[0]).kind else {
564 unreachable!()
565 };
566 assert_eq!(name, "p");
567 let p_children: Vec<_> = quirks.children(children[0]).collect();
568 assert_eq!(p_children.len(), 1);
569 let NodeKind::Element { name, .. } = &quirks.node(p_children[0]).kind else {
570 unreachable!()
571 };
572 assert_eq!(name, "table");
573 }
574
575 #[test]
583 fn end_tag_that_walks_out_of_foreign_content_does_not_loop_forever() {
584 let document = parse("<a><svg></a>");
593 let body = body_of(&document);
594 assert_eq!(document.children(body).count(), 1);
595 }
596
597 #[test]
598 fn template_end_tag_resets_a_stale_insertion_mode() {
599 let document = parse("<table><thead><template><td></template></table>");
610 let body = body_of(&document);
611 assert_eq!(document.children(body).count(), 1);
612 }
613
614 #[test]
615 fn frameset_document_replaces_body_and_ignores_stray_text() {
616 let document = parse("<!DOCTYPE html><frameset>test");
621
622 let root = document.root();
623 let root_children: Vec<_> = document.children(root).collect();
624 assert_eq!(root_children.len(), 2);
625 assert_eq!(
626 document.node(root_children[0]).kind,
627 NodeKind::Doctype {
628 name: Some("html".to_owned()),
629 public_identifier: Some(String::new()),
630 system_identifier: Some(String::new()),
631 }
632 );
633 let html = root_children[1];
634
635 let html_children: Vec<_> = document.children(html).collect();
636 assert_eq!(html_children.len(), 2);
637 let NodeKind::Element { name, .. } = &document.node(html_children[0]).kind else {
638 unreachable!()
639 };
640 assert_eq!(name, "head");
641 let frameset = html_children[1];
642 let NodeKind::Element { name, .. } = &document.node(frameset).kind else {
643 unreachable!()
644 };
645 assert_eq!(name, "frameset");
646 assert_eq!(document.children(frameset).count(), 0);
647 }
648
649 #[test]
650 fn template_content_is_a_separate_fragment_from_the_template_element() {
651 let document = parse("<body><template>Hello</template>");
655 let body = body_of(&document);
656
657 let template = document.children(body).next().unwrap();
658 let NodeKind::Element { name, .. } = &document.node(template).kind else {
659 unreachable!()
660 };
661 assert_eq!(name, "template");
662
663 let template_children: Vec<_> = document.children(template).collect();
664 assert_eq!(template_children.len(), 1);
665 let content = template_children[0];
666 assert_eq!(document.node(content).kind, NodeKind::DocumentFragment);
667
668 let content_children: Vec<_> = document.children(content).collect();
669 assert_eq!(content_children.len(), 1);
670 assert_eq!(
671 document.node(content_children[0]).kind,
672 NodeKind::Text {
673 content: "Hello".to_owned()
674 }
675 );
676 }
677
678 #[test]
679 fn selected_option_content_is_mirrored_into_selectedcontent() {
680 let document =
685 parse("<select><button><selectedcontent></button><option>X<option selected>Y");
686 let body = body_of(&document);
687
688 let select = document.children(body).next().unwrap();
689 let button = document.children(select).next().unwrap();
690 let selectedcontent = document.children(button).next().unwrap();
691 let selectedcontent_children: Vec<_> = document.children(selectedcontent).collect();
692 assert_eq!(selectedcontent_children.len(), 1);
693 assert_eq!(
694 document.node(selectedcontent_children[0]).kind,
695 NodeKind::Text {
696 content: "Y".to_owned()
697 }
698 );
699 }
700}