1use std::collections::VecDeque;
10
11use crate::entities;
12
13#[derive(Debug, Clone, Copy, PartialEq, Eq)]
17pub struct Position {
18 pub line: u32,
20 pub column: u32,
22 pub byte_offset: usize,
24}
25
26#[derive(Debug, Clone, PartialEq, Eq)]
28pub(crate) struct Attribute {
29 pub(crate) name: String,
30 pub(crate) value: String,
31}
32
33#[derive(Debug, Clone, PartialEq, Eq)]
37pub(crate) struct TagToken {
38 pub(crate) name: String,
39 pub(crate) self_closing: bool,
40 pub(crate) attributes: Vec<Attribute>,
41}
42
43#[derive(Debug, Clone, Default, PartialEq, Eq)]
45pub(crate) struct DoctypeToken {
46 pub(crate) name: Option<String>,
47 pub(crate) public_identifier: Option<String>,
48 pub(crate) system_identifier: Option<String>,
49 pub(crate) force_quirks: bool,
50}
51
52#[derive(Debug, Clone, PartialEq, Eq)]
58pub(crate) struct ProcessingInstructionToken {
59 pub(crate) target: String,
60 pub(crate) data: String,
61}
62
63#[derive(Debug, Clone, PartialEq, Eq)]
65pub(crate) enum TokenKind {
66 Doctype(DoctypeToken),
67 StartTag(TagToken),
68 EndTag(TagToken),
69 Comment(String),
70 ProcessingInstruction(ProcessingInstructionToken),
71 Character(char),
75 Eof,
76}
77
78#[derive(Debug, Clone, PartialEq, Eq)]
80pub(crate) struct Token {
81 pub(crate) kind: TokenKind,
82 pub(crate) position: Position,
83}
84
85#[derive(Debug, Clone, Copy, PartialEq, Eq)]
89enum State {
90 Data,
91 TagOpen,
92 EndTagOpen,
93 TagName,
94 BeforeAttributeName,
95 AttributeName,
96 AfterAttributeName,
97 BeforeAttributeValue,
98 AttributeValueDoubleQuoted,
99 AttributeValueSingleQuoted,
100 AttributeValueUnquoted,
101 AfterAttributeValueQuoted,
102 SelfClosingStartTag,
103 CharacterReference,
104 NamedCharacterReference,
105 AmbiguousAmpersand,
106 NumericCharacterReference,
107 HexadecimalCharacterReferenceStart,
108 HexadecimalCharacterReference,
109 DecimalCharacterReference,
110 NumericCharacterReferenceEnd,
111 MarkupDeclarationOpen,
112 BogusComment,
113 CommentStart,
114 CommentStartDash,
115 Comment,
116 CommentLessThanSign,
117 CommentLessThanSignBang,
118 CommentLessThanSignBangDash,
119 CommentLessThanSignBangDashDash,
120 CommentEndDash,
121 CommentEnd,
122 CommentEndBang,
123 Doctype,
124 BeforeDoctypeName,
125 DoctypeName,
126 AfterDoctypeName,
127 AfterDoctypePublicKeyword,
128 BeforeDoctypePublicIdentifier,
129 DoctypePublicIdentifierDoubleQuoted,
130 DoctypePublicIdentifierSingleQuoted,
131 AfterDoctypePublicIdentifier,
132 BetweenDoctypePublicAndSystemIdentifiers,
133 AfterDoctypeSystemKeyword,
134 BeforeDoctypeSystemIdentifier,
135 DoctypeSystemIdentifierDoubleQuoted,
136 DoctypeSystemIdentifierSingleQuoted,
137 AfterDoctypeSystemIdentifier,
138 BogusDoctype,
139 ProcessingInstructionOpen,
140 ProcessingInstructionTarget,
141 AfterProcessingInstructionTarget,
142 ProcessingInstructionData,
143 ProcessingInstructionQuestionable,
144 RcData,
145 RcDataLessThanSign,
146 RcDataEndTagOpen,
147 RcDataEndTagName,
148 RawText,
149 RawTextLessThanSign,
150 RawTextEndTagOpen,
151 RawTextEndTagName,
152 PlainText,
153 ScriptData,
154 ScriptDataLessThanSign,
155 ScriptDataEndTagOpen,
156 ScriptDataEndTagName,
157 ScriptDataEscapeStart,
158 ScriptDataEscapeStartDash,
159 ScriptDataEscaped,
160 ScriptDataEscapedDash,
161 ScriptDataEscapedDashDash,
162 ScriptDataEscapedLessThanSign,
163 ScriptDataEscapedEndTagOpen,
164 ScriptDataEscapedEndTagName,
165 ScriptDataDoubleEscapeStart,
166 ScriptDataDoubleEscaped,
167 ScriptDataDoubleEscapedDash,
168 ScriptDataDoubleEscapedDashDash,
169 ScriptDataDoubleEscapedLessThanSign,
170 ScriptDataDoubleEscapeEnd,
171 CdataSection,
172 CdataSectionBracket,
173 CdataSectionEnd,
174}
175
176#[derive(Debug, Clone, Copy, PartialEq, Eq)]
195pub(crate) enum ExternalState {
196 RcData,
197 RawText,
198 ScriptData,
199 PlainText,
200}
201
202#[derive(Debug, Clone, Copy, PartialEq, Eq)]
207enum DoctypeIdentifierKind {
208 Public,
209 System,
210}
211
212#[derive(Debug, Clone, Copy, PartialEq, Eq)]
218enum AttributeValueTarget {
219 Index(usize),
220 Discarded,
221}
222
223pub(crate) struct Tokenizer {
228 chars: Vec<char>,
229 positions: Vec<Position>,
230 index: usize,
231 saved: Option<(Option<char>, Position)>,
234 state: State,
235 current_tag: Option<TagToken>,
236 current_tag_is_end: bool,
237 current_tag_start: Position,
241 slash_position: Position,
245 current_attribute_name: String,
246 attribute_value_target: AttributeValueTarget,
247 return_state: State,
250 character_reference_start: Position,
253 character_reference_start_index: usize,
255 character_reference_code: u32,
257 current_comment_data: String,
262 current_doctype: Option<DoctypeToken>,
265 pi_temporary_buffer: String,
268 current_processing_instruction: Option<ProcessingInstructionToken>,
271 last_start_tag_name: Option<String>,
279 text_end_tag_buffer: String,
284 in_foreign_content: bool,
292 cdata_pending_brackets: Vec<Position>,
300 pending: VecDeque<Token>,
301 eof_returned: bool,
302}
303
304impl Tokenizer {
305 pub(crate) fn new(input: &str) -> Self {
306 let mut chars = Vec::new();
312 let mut positions = Vec::new();
313 let mut line = 1u32;
314 let mut column = 1u32;
315 let mut iter = input.char_indices().peekable();
316 while let Some((byte_offset, c)) = iter.next() {
317 let (emitted, skip_next) = if c == '\r' {
318 ('\n', matches!(iter.peek(), Some((_, '\n'))))
319 } else {
320 (c, false)
321 };
322 positions.push(Position {
323 line,
324 column,
325 byte_offset,
326 });
327 chars.push(emitted);
328 if emitted == '\n' {
329 line += 1;
330 column = 1;
331 } else {
332 column += 1;
333 }
334 if skip_next {
335 iter.next();
336 }
337 }
338 positions.push(Position {
339 line,
340 column,
341 byte_offset: input.len(),
342 });
343
344 let origin = Position {
345 line: 1,
346 column: 1,
347 byte_offset: 0,
348 };
349 Tokenizer {
350 chars,
351 positions,
352 index: 0,
353 saved: None,
354 state: State::Data,
355 current_tag: None,
356 current_tag_is_end: false,
357 current_tag_start: origin,
358 slash_position: origin,
359 current_attribute_name: String::new(),
360 attribute_value_target: AttributeValueTarget::Discarded,
361 return_state: State::Data,
362 character_reference_start: origin,
363 character_reference_start_index: 0,
364 character_reference_code: 0,
365 current_comment_data: String::new(),
366 current_doctype: None,
367 pi_temporary_buffer: String::new(),
368 current_processing_instruction: None,
369 last_start_tag_name: None,
370 text_end_tag_buffer: String::new(),
371 in_foreign_content: false,
372 cdata_pending_brackets: Vec::new(),
373 pending: VecDeque::new(),
374 eof_returned: false,
375 }
376 }
377
378 pub(crate) fn switch_to(&mut self, state: ExternalState) {
383 self.state = match state {
384 ExternalState::RcData => State::RcData,
385 ExternalState::RawText => State::RawText,
386 ExternalState::ScriptData => State::ScriptData,
387 ExternalState::PlainText => State::PlainText,
388 };
389 }
390
391 pub(crate) fn set_in_foreign_content(&mut self, in_foreign_content: bool) {
395 self.in_foreign_content = in_foreign_content;
396 }
397
398 fn consume(&mut self) -> (Option<char>, Position) {
399 if let Some(saved) = self.saved.take() {
400 return saved;
401 }
402 let position = self.positions[self.index];
403 let ch = self.chars.get(self.index).copied();
404 if ch.is_some() {
405 self.index += 1;
406 }
407 (ch, position)
408 }
409
410 fn is_whitespace(c: char) -> bool {
411 matches!(c, '\t' | '\n' | '\x0C' | ' ')
412 }
413
414 fn start_tag_token(&mut self, is_end: bool) {
417 self.current_tag = Some(TagToken {
418 name: String::new(),
419 self_closing: false,
420 attributes: Vec::new(),
421 });
422 self.current_tag_is_end = is_end;
423 }
424
425 fn emit_tag(&mut self, out: &mut Vec<Token>) {
426 let tag = self
427 .current_tag
428 .take()
429 .expect("emit_tag called with no tag token in progress");
430 let kind = if self.current_tag_is_end {
431 TokenKind::EndTag(tag)
432 } else {
433 self.last_start_tag_name = Some(tag.name.clone());
436 TokenKind::StartTag(tag)
437 };
438 out.push(Token {
439 kind,
440 position: self.current_tag_start,
441 });
442 }
443
444 fn close_tag(&mut self, out: &mut Vec<Token>) -> bool {
448 self.state = State::Data;
449 self.emit_tag(out);
450 false
451 }
452
453 fn emit_comment(&mut self, out: &mut Vec<Token>) {
454 let data = std::mem::take(&mut self.current_comment_data);
455 out.push(Token {
456 kind: TokenKind::Comment(data),
457 position: self.current_tag_start,
458 });
459 }
460
461 fn current_doctype_mut(&mut self) -> &mut DoctypeToken {
462 self.current_doctype
463 .as_mut()
464 .expect("doctype state reached with no doctype token in progress")
465 }
466
467 fn doctype_identifier_mut(&mut self, kind: DoctypeIdentifierKind) -> &mut String {
468 let doctype = self.current_doctype_mut();
469 let field = match kind {
470 DoctypeIdentifierKind::Public => &mut doctype.public_identifier,
471 DoctypeIdentifierKind::System => &mut doctype.system_identifier,
472 };
473 field
474 .as_mut()
475 .expect("doctype identifier appended to before being set to Some")
476 }
477
478 fn start_doctype_public_identifier(&mut self, quote_state: State) -> bool {
484 self.current_doctype_mut().public_identifier = Some(String::new());
485 self.state = quote_state;
486 false
487 }
488
489 fn start_doctype_system_identifier(&mut self, quote_state: State) -> bool {
495 self.current_doctype_mut().system_identifier = Some(String::new());
496 self.state = quote_state;
497 false
498 }
499
500 fn emit_doctype(&mut self, out: &mut Vec<Token>) {
501 let doctype = self
502 .current_doctype
503 .take()
504 .expect("emit_doctype called with no doctype token in progress");
505 out.push(Token {
506 kind: TokenKind::Doctype(doctype),
507 position: self.current_tag_start,
508 });
509 }
510
511 fn close_doctype(&mut self, out: &mut Vec<Token>) -> bool {
514 self.state = State::Data;
515 self.emit_doctype(out);
516 false
517 }
518
519 fn close_doctype_with_quirks(&mut self, out: &mut Vec<Token>) -> bool {
522 self.current_doctype_mut().force_quirks = true;
523 self.close_doctype(out)
524 }
525
526 fn eof_in_doctype(&mut self, out: &mut Vec<Token>, position: Position) -> bool {
531 self.current_doctype_mut().force_quirks = true;
532 self.emit_doctype(out);
533 push_eof(out, position);
534 false
535 }
536
537 fn bogus_doctype_with_quirks(&mut self) -> bool {
541 self.current_doctype_mut().force_quirks = true;
542 self.state = State::BogusDoctype;
543 true
544 }
545
546 fn emit_processing_instruction(&mut self, out: &mut Vec<Token>) {
547 let pi = self.current_processing_instruction.take().expect(
548 "emit_processing_instruction called with no processing instruction token in progress",
549 );
550 out.push(Token {
551 kind: TokenKind::ProcessingInstruction(pi),
552 position: self.current_tag_start,
553 });
554 }
555
556 fn convert_pi_temporary_buffer_to_comment(&mut self) {
561 let target = std::mem::take(&mut self.pi_temporary_buffer);
562 self.current_comment_data = format!("?{target}");
563 self.state = State::BogusComment;
564 }
565
566 fn peek_matches(&self, literal: &str, case_insensitive: bool) -> bool {
570 self.peek_matches_at(self.index, literal, case_insensitive)
571 }
572
573 fn peek_matches_at(&self, start: usize, literal: &str, case_insensitive: bool) -> bool {
578 literal.chars().enumerate().all(|(offset, expected)| {
579 self.chars.get(start + offset).is_some_and(|&actual| {
580 if case_insensitive {
581 actual.eq_ignore_ascii_case(&expected)
582 } else {
583 actual == expected
584 }
585 })
586 })
587 }
588
589 fn run_markup_declaration_open(&mut self) {
596 if self.peek_matches("--", false) {
597 self.index += 2;
598 self.current_comment_data.clear();
599 self.state = State::CommentStart;
600 return;
601 }
602 if self.peek_matches("DOCTYPE", true) {
603 self.index += 7;
604 self.state = State::Doctype;
605 return;
606 }
607 if self.peek_matches("[CDATA[", false) {
608 self.index += 7;
609 if self.in_foreign_content {
610 self.state = State::CdataSection;
611 } else {
612 self.current_comment_data = "[CDATA[".to_owned();
614 self.state = State::BogusComment;
615 }
616 return;
617 }
618 self.current_comment_data.clear();
620 self.state = State::BogusComment;
621 }
622
623 fn commit_attribute_name(&mut self) {
629 let name = std::mem::take(&mut self.current_attribute_name);
630 let tag = self
631 .current_tag
632 .as_mut()
633 .expect("commit_attribute_name called with no tag token in progress");
634 if tag
635 .attributes
636 .iter()
637 .any(|attribute| attribute.name == name)
638 {
639 self.attribute_value_target = AttributeValueTarget::Discarded;
642 } else {
643 tag.attributes.push(Attribute {
644 name,
645 value: String::new(),
646 });
647 self.attribute_value_target = AttributeValueTarget::Index(tag.attributes.len() - 1);
648 }
649 }
650
651 fn push_attribute_value_char(&mut self, c: char) {
652 if let AttributeValueTarget::Index(i) = self.attribute_value_target {
653 self.current_tag
654 .as_mut()
655 .expect("push_attribute_value_char called with no tag token in progress")
656 .attributes[i]
657 .value
658 .push(c);
659 }
660 }
661
662 fn run_until_token(&mut self) {
663 loop {
664 if self.state == State::NamedCharacterReference {
670 self.run_named_character_reference();
671 if !self.pending.is_empty() {
672 return;
673 }
674 continue;
675 }
676 if self.state == State::MarkupDeclarationOpen {
682 self.run_markup_declaration_open();
683 continue;
684 }
685 let (ch, position) = self.consume();
686 let mut out = Vec::new();
687 let reconsume = self.step(ch, position, &mut out);
688 if reconsume {
689 self.saved = Some((ch, position));
690 }
691 if !out.is_empty() {
692 self.pending.extend(out);
693 return;
694 }
695 }
696 }
697
698 fn step(&mut self, ch: Option<char>, position: Position, out: &mut Vec<Token>) -> bool {
702 match self.state {
703 State::Data => match ch {
704 Some('&') => {
705 self.begin_character_reference(State::Data, position);
706 false
707 }
708 Some('<') => {
709 self.current_tag_start = position;
710 self.state = State::TagOpen;
711 false
712 }
713 Some(c) => {
714 push_character(out, c, position);
718 false
719 }
720 None => {
721 push_eof(out, position);
722 false
723 }
724 },
725 State::TagOpen => match ch {
726 Some('!') => {
727 self.state = State::MarkupDeclarationOpen;
728 false
729 }
730 Some('/') => {
731 self.slash_position = position;
732 self.state = State::EndTagOpen;
733 false
734 }
735 Some(c) if c.is_ascii_alphabetic() => {
736 self.start_tag_token(false);
737 self.state = State::TagName;
738 true
739 }
740 Some('?') => {
741 self.pi_temporary_buffer.clear();
742 self.state = State::ProcessingInstructionOpen;
743 false
744 }
745 Some(_) => {
746 push_character(out, '<', self.current_tag_start);
748 self.state = State::Data;
749 true
750 }
751 None => {
752 push_character(out, '<', self.current_tag_start);
754 push_eof(out, position);
755 false
756 }
757 },
758 State::EndTagOpen => match ch {
759 Some(c) if c.is_ascii_alphabetic() => {
760 self.start_tag_token(true);
761 self.state = State::TagName;
762 true
763 }
764 Some('>') => {
765 self.state = State::Data;
767 false
768 }
769 Some(_) => {
770 self.current_comment_data.clear();
772 self.state = State::BogusComment;
773 true
774 }
775 None => {
776 push_character(out, '<', self.current_tag_start);
778 push_character(out, '/', self.slash_position);
779 push_eof(out, position);
780 false
781 }
782 },
783 State::TagName => match ch {
784 Some(c) if Self::is_whitespace(c) => {
785 self.state = State::BeforeAttributeName;
786 false
787 }
788 Some('/') => {
789 self.state = State::SelfClosingStartTag;
790 false
791 }
792 Some('>') => self.close_tag(out),
793 Some(c) if c.is_ascii_uppercase() => {
794 self.current_tag_mut().name.push(c.to_ascii_lowercase());
795 false
796 }
797 Some('\0') => {
798 self.current_tag_mut().name.push('\u{FFFD}');
799 false
800 }
801 Some(c) => {
802 self.current_tag_mut().name.push(c);
803 false
804 }
805 None => {
806 push_eof(out, position);
808 false
809 }
810 },
811 State::BeforeAttributeName => match ch {
812 Some(c) if Self::is_whitespace(c) => false,
813 Some('/') | Some('>') | None => {
814 self.state = State::AfterAttributeName;
815 true
816 }
817 Some('=') => {
818 self.current_attribute_name.clear();
822 self.current_attribute_name.push('=');
823 self.state = State::AttributeName;
824 false
825 }
826 Some(_) => {
827 self.current_attribute_name.clear();
828 self.state = State::AttributeName;
829 true
830 }
831 },
832 State::AttributeName => match ch {
833 Some(c) if Self::is_whitespace(c) || c == '/' || c == '>' => {
834 self.commit_attribute_name();
835 self.state = State::AfterAttributeName;
836 true
837 }
838 None => {
839 self.commit_attribute_name();
840 self.state = State::AfterAttributeName;
841 true
842 }
843 Some('=') => {
844 self.commit_attribute_name();
845 self.state = State::BeforeAttributeValue;
846 false
847 }
848 Some(c) if c.is_ascii_uppercase() => {
849 self.current_attribute_name.push(c.to_ascii_lowercase());
850 false
851 }
852 Some('\0') => {
853 self.current_attribute_name.push('\u{FFFD}');
854 false
855 }
856 Some(c) => {
857 self.current_attribute_name.push(c);
861 false
862 }
863 },
864 State::AfterAttributeName => match ch {
865 Some(c) if Self::is_whitespace(c) => false,
866 Some('/') => {
867 self.state = State::SelfClosingStartTag;
868 false
869 }
870 Some('=') => {
871 self.state = State::BeforeAttributeValue;
872 false
873 }
874 Some('>') => self.close_tag(out),
875 None => {
876 push_eof(out, position);
877 false
878 }
879 Some(_) => {
880 self.current_attribute_name.clear();
881 self.state = State::AttributeName;
882 true
883 }
884 },
885 State::BeforeAttributeValue => match ch {
886 Some(c) if Self::is_whitespace(c) => false,
887 Some('"') => {
888 self.state = State::AttributeValueDoubleQuoted;
889 false
890 }
891 Some('\'') => {
892 self.state = State::AttributeValueSingleQuoted;
893 false
894 }
895 Some('>') => {
896 self.close_tag(out)
898 }
899 _ => {
900 self.state = State::AttributeValueUnquoted;
901 true
902 }
903 },
904 State::AttributeValueDoubleQuoted => {
905 self.step_attribute_value_quoted(ch, position, out, '"')
906 }
907 State::AttributeValueSingleQuoted => {
908 self.step_attribute_value_quoted(ch, position, out, '\'')
909 }
910 State::AttributeValueUnquoted => match ch {
911 Some(c) if Self::is_whitespace(c) => {
912 self.state = State::BeforeAttributeName;
913 false
914 }
915 Some('&') => {
916 self.begin_character_reference(State::AttributeValueUnquoted, position);
917 false
918 }
919 Some('>') => self.close_tag(out),
920 Some('\0') => {
921 self.push_attribute_value_char('\u{FFFD}');
922 false
923 }
924 Some(c) => {
925 self.push_attribute_value_char(c);
929 false
930 }
931 None => {
932 push_eof(out, position);
933 false
934 }
935 },
936 State::AfterAttributeValueQuoted => match ch {
937 Some(c) if Self::is_whitespace(c) => {
938 self.state = State::BeforeAttributeName;
939 false
940 }
941 Some('/') => {
942 self.state = State::SelfClosingStartTag;
943 false
944 }
945 Some('>') => self.close_tag(out),
946 None => {
947 push_eof(out, position);
948 false
949 }
950 Some(_) => {
951 self.state = State::BeforeAttributeName;
953 true
954 }
955 },
956 State::SelfClosingStartTag => match ch {
957 Some('>') => {
958 self.current_tag_mut().self_closing = true;
959 self.close_tag(out)
960 }
961 None => {
962 push_eof(out, position);
963 false
964 }
965 Some(_) => {
966 self.state = State::BeforeAttributeName;
968 true
969 }
970 },
971 State::CharacterReference => match ch {
972 Some(c) if c.is_ascii_alphanumeric() => {
973 self.state = State::NamedCharacterReference;
974 true
975 }
976 Some('#') => {
977 self.state = State::NumericCharacterReference;
978 false
979 }
980 _ => {
981 let end = self.character_reference_start_index + 1;
987 self.flush_literal_character_reference_attempt(end, out);
988 self.state = self.return_state;
989 true
990 }
991 },
992 State::NamedCharacterReference => {
995 unreachable!("NamedCharacterReference is dispatched before step() is called")
996 }
997 State::AmbiguousAmpersand => match ch {
998 Some(c) if c.is_ascii_alphanumeric() => {
999 self.flush_char_as_character_reference(c, position, out);
1000 false
1001 }
1002 Some(';') => {
1003 self.state = self.return_state;
1005 true
1006 }
1007 _ => {
1008 self.state = self.return_state;
1009 true
1010 }
1011 },
1012 State::NumericCharacterReference => {
1013 self.character_reference_code = 0;
1014 match ch {
1015 Some('x') | Some('X') => {
1016 self.state = State::HexadecimalCharacterReferenceStart;
1017 false
1018 }
1019 Some(c) if c.is_ascii_digit() => {
1020 self.state = State::DecimalCharacterReference;
1021 true
1022 }
1023 _ => {
1024 let end = self.character_reference_start_index + 2;
1030 self.flush_literal_character_reference_attempt(end, out);
1031 self.state = self.return_state;
1032 true
1033 }
1034 }
1035 }
1036 State::HexadecimalCharacterReferenceStart => match ch {
1037 Some(c) if c.is_ascii_hexdigit() => {
1038 self.state = State::HexadecimalCharacterReference;
1039 true
1040 }
1041 _ => {
1042 let end = self.character_reference_start_index + 3;
1046 self.flush_literal_character_reference_attempt(end, out);
1047 self.state = self.return_state;
1048 true
1049 }
1050 },
1051 State::HexadecimalCharacterReference => match ch {
1052 Some(c) if c.is_ascii_digit() => {
1053 self.character_reference_code = self
1054 .character_reference_code
1055 .saturating_mul(16)
1056 .saturating_add(u32::from(c) - u32::from('0'));
1057 false
1058 }
1059 Some(c) if ('A'..='F').contains(&c) => {
1060 self.character_reference_code = self
1061 .character_reference_code
1062 .saturating_mul(16)
1063 .saturating_add(u32::from(c) - 0x37);
1064 false
1065 }
1066 Some(c) if ('a'..='f').contains(&c) => {
1067 self.character_reference_code = self
1068 .character_reference_code
1069 .saturating_mul(16)
1070 .saturating_add(u32::from(c) - 0x57);
1071 false
1072 }
1073 Some(';') => {
1074 self.state = State::NumericCharacterReferenceEnd;
1075 false
1076 }
1077 _ => {
1078 self.state = State::NumericCharacterReferenceEnd;
1081 true
1082 }
1083 },
1084 State::DecimalCharacterReference => match ch {
1085 Some(c) if c.is_ascii_digit() => {
1086 self.character_reference_code = self
1087 .character_reference_code
1088 .saturating_mul(10)
1089 .saturating_add(u32::from(c) - u32::from('0'));
1090 false
1091 }
1092 Some(';') => {
1093 self.state = State::NumericCharacterReferenceEnd;
1094 false
1095 }
1096 _ => {
1097 self.state = State::NumericCharacterReferenceEnd;
1100 true
1101 }
1102 },
1103 State::NumericCharacterReferenceEnd => {
1104 let resolved = self.resolve_numeric_character_reference_code();
1108 self.flush_char_as_character_reference(
1109 resolved,
1110 self.character_reference_start,
1111 out,
1112 );
1113 self.state = self.return_state;
1114 true
1115 }
1116 State::MarkupDeclarationOpen => {
1119 unreachable!("MarkupDeclarationOpen is dispatched before step() is called")
1120 }
1121 State::BogusComment => match ch {
1122 Some('>') => {
1123 self.state = State::Data;
1124 self.emit_comment(out);
1125 false
1126 }
1127 None => {
1128 self.emit_comment(out);
1129 push_eof(out, position);
1130 false
1131 }
1132 Some('\0') => {
1133 self.current_comment_data.push('\u{FFFD}');
1134 false
1135 }
1136 Some(c) => {
1137 self.current_comment_data.push(c);
1138 false
1139 }
1140 },
1141 State::CommentStart => match ch {
1142 Some('-') => {
1143 self.state = State::CommentStartDash;
1144 false
1145 }
1146 Some('>') => {
1147 self.state = State::Data;
1149 self.emit_comment(out);
1150 false
1151 }
1152 _ => {
1153 self.state = State::Comment;
1154 true
1155 }
1156 },
1157 State::CommentStartDash => match ch {
1158 Some('-') => {
1159 self.state = State::CommentEnd;
1160 false
1161 }
1162 Some('>') => {
1163 self.state = State::Data;
1165 self.emit_comment(out);
1166 false
1167 }
1168 None => {
1169 self.emit_comment(out);
1170 push_eof(out, position);
1171 false
1172 }
1173 Some(_) => {
1174 self.current_comment_data.push('-');
1175 self.state = State::Comment;
1176 true
1177 }
1178 },
1179 State::Comment => match ch {
1180 Some('<') => {
1181 self.current_comment_data.push('<');
1182 self.state = State::CommentLessThanSign;
1183 false
1184 }
1185 Some('-') => {
1186 self.state = State::CommentEndDash;
1187 false
1188 }
1189 Some('\0') => {
1190 self.current_comment_data.push('\u{FFFD}');
1191 false
1192 }
1193 None => {
1194 self.emit_comment(out);
1195 push_eof(out, position);
1196 false
1197 }
1198 Some(c) => {
1199 self.current_comment_data.push(c);
1200 false
1201 }
1202 },
1203 State::CommentLessThanSign => match ch {
1204 Some('!') => {
1205 self.current_comment_data.push('!');
1206 self.state = State::CommentLessThanSignBang;
1207 false
1208 }
1209 Some('<') => {
1210 self.current_comment_data.push('<');
1211 false
1212 }
1213 _ => {
1214 self.state = State::Comment;
1215 true
1216 }
1217 },
1218 State::CommentLessThanSignBang => match ch {
1219 Some('-') => {
1220 self.state = State::CommentLessThanSignBangDash;
1221 false
1222 }
1223 _ => {
1224 self.state = State::Comment;
1225 true
1226 }
1227 },
1228 State::CommentLessThanSignBangDash => match ch {
1229 Some('-') => {
1230 self.state = State::CommentLessThanSignBangDashDash;
1231 false
1232 }
1233 _ => {
1234 self.state = State::CommentEndDash;
1235 true
1236 }
1237 },
1238 State::CommentLessThanSignBangDashDash => {
1239 self.state = State::CommentEnd;
1244 true
1245 }
1246 State::CommentEndDash => match ch {
1247 Some('-') => {
1248 self.state = State::CommentEnd;
1249 false
1250 }
1251 None => {
1252 self.emit_comment(out);
1253 push_eof(out, position);
1254 false
1255 }
1256 Some(_) => {
1257 self.current_comment_data.push('-');
1258 self.state = State::Comment;
1259 true
1260 }
1261 },
1262 State::CommentEnd => match ch {
1263 Some('>') => {
1264 self.state = State::Data;
1265 self.emit_comment(out);
1266 false
1267 }
1268 Some('!') => {
1269 self.state = State::CommentEndBang;
1270 false
1271 }
1272 Some('-') => {
1273 self.current_comment_data.push('-');
1274 false
1275 }
1276 None => {
1277 self.emit_comment(out);
1278 push_eof(out, position);
1279 false
1280 }
1281 Some(_) => {
1282 self.current_comment_data.push_str("--");
1283 self.state = State::Comment;
1284 true
1285 }
1286 },
1287 State::CommentEndBang => match ch {
1288 Some('-') => {
1289 self.current_comment_data.push_str("--!");
1290 self.state = State::CommentEndDash;
1291 false
1292 }
1293 Some('>') => {
1294 self.state = State::Data;
1296 self.emit_comment(out);
1297 false
1298 }
1299 None => {
1300 self.emit_comment(out);
1301 push_eof(out, position);
1302 false
1303 }
1304 Some(_) => {
1305 self.current_comment_data.push_str("--!");
1306 self.state = State::Comment;
1307 true
1308 }
1309 },
1310 State::Doctype => match ch {
1311 Some(c) if Self::is_whitespace(c) => {
1312 self.state = State::BeforeDoctypeName;
1313 false
1314 }
1315 Some('>') => {
1316 self.state = State::BeforeDoctypeName;
1317 true
1318 }
1319 None => {
1320 self.current_doctype = Some(DoctypeToken {
1324 force_quirks: true,
1325 ..Default::default()
1326 });
1327 self.emit_doctype(out);
1328 push_eof(out, position);
1329 false
1330 }
1331 Some(_) => {
1332 self.state = State::BeforeDoctypeName;
1334 true
1335 }
1336 },
1337 State::BeforeDoctypeName => match ch {
1338 Some(c) if Self::is_whitespace(c) => false,
1339 Some(c) if c.is_ascii_uppercase() => {
1340 self.current_doctype = Some(DoctypeToken {
1341 name: Some(c.to_ascii_lowercase().to_string()),
1342 ..Default::default()
1343 });
1344 self.state = State::DoctypeName;
1345 false
1346 }
1347 Some('\0') => {
1348 self.current_doctype = Some(DoctypeToken {
1349 name: Some("\u{FFFD}".to_owned()),
1350 ..Default::default()
1351 });
1352 self.state = State::DoctypeName;
1353 false
1354 }
1355 Some('>') => {
1356 self.current_doctype = Some(DoctypeToken {
1357 force_quirks: true,
1358 ..Default::default()
1359 });
1360 self.close_doctype(out)
1361 }
1362 None => {
1363 self.current_doctype = Some(DoctypeToken {
1364 force_quirks: true,
1365 ..Default::default()
1366 });
1367 self.emit_doctype(out);
1368 push_eof(out, position);
1369 false
1370 }
1371 Some(c) => {
1372 self.current_doctype = Some(DoctypeToken {
1373 name: Some(c.to_string()),
1374 ..Default::default()
1375 });
1376 self.state = State::DoctypeName;
1377 false
1378 }
1379 },
1380 State::DoctypeName => match ch {
1381 Some(c) if Self::is_whitespace(c) => {
1382 self.state = State::AfterDoctypeName;
1383 false
1384 }
1385 Some('>') => self.close_doctype(out),
1386 Some(c) if c.is_ascii_uppercase() => {
1387 self.current_doctype_mut()
1388 .name
1389 .as_mut()
1390 .expect("doctype name should already be Some in DoctypeName state")
1391 .push(c.to_ascii_lowercase());
1392 false
1393 }
1394 Some('\0') => {
1395 self.current_doctype_mut()
1396 .name
1397 .as_mut()
1398 .expect("doctype name should already be Some in DoctypeName state")
1399 .push('\u{FFFD}');
1400 false
1401 }
1402 None => self.eof_in_doctype(out, position),
1403 Some(c) => {
1404 self.current_doctype_mut()
1405 .name
1406 .as_mut()
1407 .expect("doctype name should already be Some in DoctypeName state")
1408 .push(c);
1409 false
1410 }
1411 },
1412 State::AfterDoctypeName => match ch {
1413 Some(c) if Self::is_whitespace(c) => false,
1414 Some('>') => self.close_doctype(out),
1415 None => self.eof_in_doctype(out, position),
1416 Some(_) => {
1417 let start = self.index - 1;
1418 if self.peek_matches_at(start, "PUBLIC", true) {
1419 self.index = start + 6;
1420 self.state = State::AfterDoctypePublicKeyword;
1421 false
1422 } else if self.peek_matches_at(start, "SYSTEM", true) {
1423 self.index = start + 6;
1424 self.state = State::AfterDoctypeSystemKeyword;
1425 false
1426 } else {
1427 self.bogus_doctype_with_quirks()
1430 }
1431 }
1432 },
1433 State::AfterDoctypePublicKeyword => match ch {
1434 Some(c) if Self::is_whitespace(c) => {
1435 self.state = State::BeforeDoctypePublicIdentifier;
1436 false
1437 }
1438 Some('"') => {
1439 self.start_doctype_public_identifier(State::DoctypePublicIdentifierDoubleQuoted)
1440 }
1441 Some('\'') => {
1442 self.start_doctype_public_identifier(State::DoctypePublicIdentifierSingleQuoted)
1443 }
1444 Some('>') => self.close_doctype_with_quirks(out),
1445 None => self.eof_in_doctype(out, position),
1446 Some(_) => self.bogus_doctype_with_quirks(),
1447 },
1448 State::BeforeDoctypePublicIdentifier => match ch {
1449 Some(c) if Self::is_whitespace(c) => false,
1450 Some('"') => {
1451 self.start_doctype_public_identifier(State::DoctypePublicIdentifierDoubleQuoted)
1452 }
1453 Some('\'') => {
1454 self.start_doctype_public_identifier(State::DoctypePublicIdentifierSingleQuoted)
1455 }
1456 Some('>') => self.close_doctype_with_quirks(out),
1457 None => self.eof_in_doctype(out, position),
1458 Some(_) => self.bogus_doctype_with_quirks(),
1459 },
1460 State::DoctypePublicIdentifierDoubleQuoted => self.step_doctype_identifier_quoted(
1461 ch,
1462 position,
1463 out,
1464 '"',
1465 DoctypeIdentifierKind::Public,
1466 ),
1467 State::DoctypePublicIdentifierSingleQuoted => self.step_doctype_identifier_quoted(
1468 ch,
1469 position,
1470 out,
1471 '\'',
1472 DoctypeIdentifierKind::Public,
1473 ),
1474 State::AfterDoctypePublicIdentifier => match ch {
1475 Some(c) if Self::is_whitespace(c) => {
1476 self.state = State::BetweenDoctypePublicAndSystemIdentifiers;
1477 false
1478 }
1479 Some('>') => self.close_doctype(out),
1480 Some('"') => {
1483 self.start_doctype_system_identifier(State::DoctypeSystemIdentifierDoubleQuoted)
1484 }
1485 Some('\'') => {
1486 self.start_doctype_system_identifier(State::DoctypeSystemIdentifierSingleQuoted)
1487 }
1488 None => self.eof_in_doctype(out, position),
1489 Some(_) => self.bogus_doctype_with_quirks(),
1490 },
1491 State::BetweenDoctypePublicAndSystemIdentifiers => match ch {
1492 Some(c) if Self::is_whitespace(c) => false,
1493 Some('>') => self.close_doctype(out),
1494 Some('"') => {
1495 self.start_doctype_system_identifier(State::DoctypeSystemIdentifierDoubleQuoted)
1496 }
1497 Some('\'') => {
1498 self.start_doctype_system_identifier(State::DoctypeSystemIdentifierSingleQuoted)
1499 }
1500 None => self.eof_in_doctype(out, position),
1501 Some(_) => self.bogus_doctype_with_quirks(),
1502 },
1503 State::AfterDoctypeSystemKeyword => match ch {
1504 Some(c) if Self::is_whitespace(c) => {
1505 self.state = State::BeforeDoctypeSystemIdentifier;
1506 false
1507 }
1508 Some('"') => {
1509 self.start_doctype_system_identifier(State::DoctypeSystemIdentifierDoubleQuoted)
1510 }
1511 Some('\'') => {
1512 self.start_doctype_system_identifier(State::DoctypeSystemIdentifierSingleQuoted)
1513 }
1514 Some('>') => self.close_doctype_with_quirks(out),
1515 None => self.eof_in_doctype(out, position),
1516 Some(_) => self.bogus_doctype_with_quirks(),
1517 },
1518 State::BeforeDoctypeSystemIdentifier => match ch {
1519 Some(c) if Self::is_whitespace(c) => false,
1520 Some('"') => {
1521 self.start_doctype_system_identifier(State::DoctypeSystemIdentifierDoubleQuoted)
1522 }
1523 Some('\'') => {
1524 self.start_doctype_system_identifier(State::DoctypeSystemIdentifierSingleQuoted)
1525 }
1526 Some('>') => self.close_doctype_with_quirks(out),
1527 None => self.eof_in_doctype(out, position),
1528 Some(_) => self.bogus_doctype_with_quirks(),
1529 },
1530 State::DoctypeSystemIdentifierDoubleQuoted => self.step_doctype_identifier_quoted(
1531 ch,
1532 position,
1533 out,
1534 '"',
1535 DoctypeIdentifierKind::System,
1536 ),
1537 State::DoctypeSystemIdentifierSingleQuoted => self.step_doctype_identifier_quoted(
1538 ch,
1539 position,
1540 out,
1541 '\'',
1542 DoctypeIdentifierKind::System,
1543 ),
1544 State::AfterDoctypeSystemIdentifier => match ch {
1545 Some(c) if Self::is_whitespace(c) => false,
1546 Some('>') => self.close_doctype(out),
1547 None => self.eof_in_doctype(out, position),
1548 Some(_) => {
1549 self.state = State::BogusDoctype;
1553 true
1554 }
1555 },
1556 State::BogusDoctype => match ch {
1557 Some('>') => self.close_doctype(out),
1558 Some('\0') => false,
1559 None => {
1560 self.emit_doctype(out);
1561 push_eof(out, position);
1562 false
1563 }
1564 Some(_) => false,
1565 },
1566 State::ProcessingInstructionOpen => match ch {
1567 Some(c) if c.is_ascii_alphabetic() || c == '_' => {
1568 self.state = State::ProcessingInstructionTarget;
1569 true
1570 }
1571 None => {
1572 push_eof(out, position);
1577 false
1578 }
1579 Some(_) => {
1580 self.convert_pi_temporary_buffer_to_comment();
1584 true
1585 }
1586 },
1587 State::ProcessingInstructionTarget => match ch {
1588 Some(c) if Self::is_whitespace(c) || c == '?' || c == '>' => {
1589 let target = std::mem::take(&mut self.pi_temporary_buffer);
1590 if target.eq_ignore_ascii_case("xml")
1591 || target.eq_ignore_ascii_case("xml-stylesheet")
1592 {
1593 self.current_comment_data = format!("?{target}");
1596 self.state = State::BogusComment;
1597 } else {
1598 self.current_processing_instruction = Some(ProcessingInstructionToken {
1599 target,
1600 data: String::new(),
1601 });
1602 self.state = State::AfterProcessingInstructionTarget;
1603 }
1604 true
1605 }
1606 Some(c) if c.is_ascii_alphanumeric() || c == '-' || c == '_' => {
1607 self.pi_temporary_buffer.push(c);
1608 false
1609 }
1610 None => {
1611 push_eof(out, position);
1612 false
1613 }
1614 Some(_) => {
1615 self.convert_pi_temporary_buffer_to_comment();
1617 true
1618 }
1619 },
1620 State::AfterProcessingInstructionTarget => match ch {
1621 Some(c) if Self::is_whitespace(c) => false,
1622 _ => {
1623 self.state = State::ProcessingInstructionData;
1624 true
1625 }
1626 },
1627 State::ProcessingInstructionData => match ch {
1628 Some('?') => {
1629 self.state = State::ProcessingInstructionQuestionable;
1630 false
1631 }
1632 Some('>') => {
1633 self.state = State::Data;
1634 self.emit_processing_instruction(out);
1635 false
1636 }
1637 None => {
1638 self.current_processing_instruction = None;
1642 push_eof(out, position);
1643 false
1644 }
1645 Some(c) => {
1646 self.current_processing_instruction
1647 .as_mut()
1648 .expect("processing instruction token should already exist in data state")
1649 .data
1650 .push(c);
1651 false
1652 }
1653 },
1654 State::ProcessingInstructionQuestionable => match ch {
1655 Some('>') => {
1656 self.state = State::Data;
1657 self.emit_processing_instruction(out);
1658 false
1659 }
1660 None => {
1661 self.current_processing_instruction = None;
1662 push_eof(out, position);
1663 false
1664 }
1665 Some(_) => {
1666 self.current_processing_instruction
1667 .as_mut()
1668 .expect(
1669 "processing instruction token should already exist in questionable state",
1670 )
1671 .data
1672 .push('?');
1673 self.state = State::ProcessingInstructionData;
1674 true
1675 }
1676 },
1677 State::RcData => match ch {
1678 Some('&') => {
1679 self.begin_character_reference(State::RcData, position);
1680 false
1681 }
1682 Some('<') => {
1683 self.current_tag_start = position;
1684 self.state = State::RcDataLessThanSign;
1685 false
1686 }
1687 Some('\0') => {
1688 push_character(out, '\u{FFFD}', position);
1689 false
1690 }
1691 Some(c) => {
1692 push_character(out, c, position);
1693 false
1694 }
1695 None => {
1696 push_eof(out, position);
1697 false
1698 }
1699 },
1700 State::RcDataLessThanSign => self.step_text_less_than_sign(
1701 ch,
1702 position,
1703 out,
1704 State::RcDataEndTagOpen,
1705 State::RcData,
1706 ),
1707 State::RcDataEndTagOpen => self.step_text_end_tag_open(
1708 ch,
1709 position,
1710 out,
1711 State::RcDataEndTagName,
1712 State::RcData,
1713 ),
1714 State::RcDataEndTagName => {
1715 self.step_text_end_tag_name(ch, position, out, State::RcData)
1716 }
1717 State::RawText => match ch {
1718 Some('<') => {
1719 self.current_tag_start = position;
1720 self.state = State::RawTextLessThanSign;
1721 false
1722 }
1723 Some('\0') => {
1724 push_character(out, '\u{FFFD}', position);
1725 false
1726 }
1727 Some(c) => {
1728 push_character(out, c, position);
1729 false
1730 }
1731 None => {
1732 push_eof(out, position);
1733 false
1734 }
1735 },
1736 State::RawTextLessThanSign => self.step_text_less_than_sign(
1737 ch,
1738 position,
1739 out,
1740 State::RawTextEndTagOpen,
1741 State::RawText,
1742 ),
1743 State::RawTextEndTagOpen => self.step_text_end_tag_open(
1744 ch,
1745 position,
1746 out,
1747 State::RawTextEndTagName,
1748 State::RawText,
1749 ),
1750 State::RawTextEndTagName => {
1751 self.step_text_end_tag_name(ch, position, out, State::RawText)
1752 }
1753 State::PlainText => match ch {
1754 Some('\0') => {
1755 push_character(out, '\u{FFFD}', position);
1756 false
1757 }
1758 Some(c) => {
1759 push_character(out, c, position);
1760 false
1761 }
1762 None => {
1763 push_eof(out, position);
1764 false
1765 }
1766 },
1767 State::ScriptData => match ch {
1768 Some('<') => {
1769 self.current_tag_start = position;
1770 self.state = State::ScriptDataLessThanSign;
1771 false
1772 }
1773 Some('\0') => {
1774 push_character(out, '\u{FFFD}', position);
1775 false
1776 }
1777 Some(c) => {
1778 push_character(out, c, position);
1779 false
1780 }
1781 None => {
1782 push_eof(out, position);
1783 false
1784 }
1785 },
1786 State::ScriptDataLessThanSign => match ch {
1787 Some('/') => {
1788 self.text_end_tag_buffer.clear();
1789 self.slash_position = position;
1790 self.state = State::ScriptDataEndTagOpen;
1791 false
1792 }
1793 Some('!') => {
1794 self.state = State::ScriptDataEscapeStart;
1795 push_character(out, '<', self.current_tag_start);
1796 push_character(out, '!', position);
1797 false
1798 }
1799 _ => {
1800 push_character(out, '<', self.current_tag_start);
1801 self.state = State::ScriptData;
1802 true
1803 }
1804 },
1805 State::ScriptDataEndTagOpen => self.step_text_end_tag_open(
1806 ch,
1807 position,
1808 out,
1809 State::ScriptDataEndTagName,
1810 State::ScriptData,
1811 ),
1812 State::ScriptDataEndTagName => {
1813 self.step_text_end_tag_name(ch, position, out, State::ScriptData)
1814 }
1815 State::ScriptDataEscapeStart => match ch {
1816 Some('-') => {
1817 self.state = State::ScriptDataEscapeStartDash;
1818 push_character(out, '-', position);
1819 false
1820 }
1821 _ => {
1822 self.state = State::ScriptData;
1823 true
1824 }
1825 },
1826 State::ScriptDataEscapeStartDash => match ch {
1827 Some('-') => {
1828 self.state = State::ScriptDataEscapedDashDash;
1829 push_character(out, '-', position);
1830 false
1831 }
1832 _ => {
1833 self.state = State::ScriptData;
1834 true
1835 }
1836 },
1837 State::ScriptDataEscaped => match ch {
1838 Some('-') => {
1839 self.state = State::ScriptDataEscapedDash;
1840 push_character(out, '-', position);
1841 false
1842 }
1843 Some('<') => {
1844 self.current_tag_start = position;
1845 self.state = State::ScriptDataEscapedLessThanSign;
1846 false
1847 }
1848 Some('\0') => {
1849 push_character(out, '\u{FFFD}', position);
1850 false
1851 }
1852 Some(c) => {
1853 push_character(out, c, position);
1854 false
1855 }
1856 None => {
1857 push_eof(out, position);
1859 false
1860 }
1861 },
1862 State::ScriptDataEscapedDash => match ch {
1863 Some('-') => {
1864 self.state = State::ScriptDataEscapedDashDash;
1865 push_character(out, '-', position);
1866 false
1867 }
1868 Some('<') => {
1869 self.current_tag_start = position;
1870 self.state = State::ScriptDataEscapedLessThanSign;
1871 false
1872 }
1873 Some('\0') => {
1874 self.state = State::ScriptDataEscaped;
1875 push_character(out, '\u{FFFD}', position);
1876 false
1877 }
1878 None => {
1879 push_eof(out, position);
1880 false
1881 }
1882 Some(c) => {
1883 self.state = State::ScriptDataEscaped;
1884 push_character(out, c, position);
1885 false
1886 }
1887 },
1888 State::ScriptDataEscapedDashDash => match ch {
1889 Some('-') => {
1890 push_character(out, '-', position);
1891 false
1892 }
1893 Some('<') => {
1894 self.current_tag_start = position;
1895 self.state = State::ScriptDataEscapedLessThanSign;
1896 false
1897 }
1898 Some('>') => {
1899 self.state = State::ScriptData;
1900 push_character(out, '>', position);
1901 false
1902 }
1903 Some('\0') => {
1904 self.state = State::ScriptDataEscaped;
1905 push_character(out, '\u{FFFD}', position);
1906 false
1907 }
1908 None => {
1909 push_eof(out, position);
1910 false
1911 }
1912 Some(c) => {
1913 self.state = State::ScriptDataEscaped;
1914 push_character(out, c, position);
1915 false
1916 }
1917 },
1918 State::ScriptDataEscapedLessThanSign => match ch {
1919 Some('/') => {
1920 self.text_end_tag_buffer.clear();
1921 self.slash_position = position;
1922 self.state = State::ScriptDataEscapedEndTagOpen;
1923 false
1924 }
1925 Some(c) if c.is_ascii_alphabetic() => {
1926 self.text_end_tag_buffer.clear();
1927 push_character(out, '<', self.current_tag_start);
1928 self.state = State::ScriptDataDoubleEscapeStart;
1929 true
1930 }
1931 _ => {
1932 push_character(out, '<', self.current_tag_start);
1933 self.state = State::ScriptDataEscaped;
1934 true
1935 }
1936 },
1937 State::ScriptDataEscapedEndTagOpen => self.step_text_end_tag_open(
1938 ch,
1939 position,
1940 out,
1941 State::ScriptDataEscapedEndTagName,
1942 State::ScriptDataEscaped,
1943 ),
1944 State::ScriptDataEscapedEndTagName => {
1945 self.step_text_end_tag_name(ch, position, out, State::ScriptDataEscaped)
1946 }
1947 State::ScriptDataDoubleEscapeStart => match ch {
1948 Some(c) if Self::is_whitespace(c) || c == '/' || c == '>' => {
1949 self.state = if self.text_end_tag_buffer == "script" {
1950 State::ScriptDataDoubleEscaped
1951 } else {
1952 State::ScriptDataEscaped
1953 };
1954 push_character(out, c, position);
1955 false
1956 }
1957 Some(c) if c.is_ascii_uppercase() => {
1958 self.text_end_tag_buffer.push(c.to_ascii_lowercase());
1959 push_character(out, c, position);
1960 false
1961 }
1962 Some(c) if c.is_ascii_lowercase() => {
1963 self.text_end_tag_buffer.push(c);
1964 push_character(out, c, position);
1965 false
1966 }
1967 _ => {
1968 self.state = State::ScriptDataEscaped;
1969 true
1970 }
1971 },
1972 State::ScriptDataDoubleEscaped => match ch {
1973 Some('-') => {
1974 self.state = State::ScriptDataDoubleEscapedDash;
1975 push_character(out, '-', position);
1976 false
1977 }
1978 Some('<') => {
1979 self.state = State::ScriptDataDoubleEscapedLessThanSign;
1980 push_character(out, '<', position);
1981 false
1982 }
1983 Some('\0') => {
1984 push_character(out, '\u{FFFD}', position);
1985 false
1986 }
1987 Some(c) => {
1988 push_character(out, c, position);
1989 false
1990 }
1991 None => {
1992 push_eof(out, position);
1993 false
1994 }
1995 },
1996 State::ScriptDataDoubleEscapedDash => match ch {
1997 Some('-') => {
1998 self.state = State::ScriptDataDoubleEscapedDashDash;
1999 push_character(out, '-', position);
2000 false
2001 }
2002 Some('<') => {
2003 self.state = State::ScriptDataDoubleEscapedLessThanSign;
2004 push_character(out, '<', position);
2005 false
2006 }
2007 Some('\0') => {
2008 self.state = State::ScriptDataDoubleEscaped;
2009 push_character(out, '\u{FFFD}', position);
2010 false
2011 }
2012 None => {
2013 push_eof(out, position);
2014 false
2015 }
2016 Some(c) => {
2017 self.state = State::ScriptDataDoubleEscaped;
2018 push_character(out, c, position);
2019 false
2020 }
2021 },
2022 State::ScriptDataDoubleEscapedDashDash => match ch {
2023 Some('-') => {
2024 push_character(out, '-', position);
2025 false
2026 }
2027 Some('<') => {
2028 self.state = State::ScriptDataDoubleEscapedLessThanSign;
2029 push_character(out, '<', position);
2030 false
2031 }
2032 Some('>') => {
2033 self.state = State::ScriptData;
2034 push_character(out, '>', position);
2035 false
2036 }
2037 Some('\0') => {
2038 self.state = State::ScriptDataDoubleEscaped;
2039 push_character(out, '\u{FFFD}', position);
2040 false
2041 }
2042 None => {
2043 push_eof(out, position);
2044 false
2045 }
2046 Some(c) => {
2047 self.state = State::ScriptDataDoubleEscaped;
2048 push_character(out, c, position);
2049 false
2050 }
2051 },
2052 State::ScriptDataDoubleEscapedLessThanSign => match ch {
2053 Some('/') => {
2054 self.text_end_tag_buffer.clear();
2055 self.state = State::ScriptDataDoubleEscapeEnd;
2056 push_character(out, '/', position);
2057 false
2058 }
2059 _ => {
2060 self.state = State::ScriptDataDoubleEscaped;
2061 true
2062 }
2063 },
2064 State::ScriptDataDoubleEscapeEnd => match ch {
2065 Some(c) if Self::is_whitespace(c) || c == '/' || c == '>' => {
2066 self.state = if self.text_end_tag_buffer == "script" {
2067 State::ScriptDataEscaped
2068 } else {
2069 State::ScriptDataDoubleEscaped
2070 };
2071 push_character(out, c, position);
2072 false
2073 }
2074 Some(c) if c.is_ascii_uppercase() => {
2075 self.text_end_tag_buffer.push(c.to_ascii_lowercase());
2076 push_character(out, c, position);
2077 false
2078 }
2079 Some(c) if c.is_ascii_lowercase() => {
2080 self.text_end_tag_buffer.push(c);
2081 push_character(out, c, position);
2082 false
2083 }
2084 _ => {
2085 self.state = State::ScriptDataDoubleEscaped;
2086 true
2087 }
2088 },
2089 State::CdataSection => match ch {
2090 Some(']') => {
2091 self.cdata_pending_brackets.push(position);
2092 self.state = State::CdataSectionBracket;
2093 false
2094 }
2095 None => {
2096 push_eof(out, position);
2098 false
2099 }
2100 Some(c) => {
2101 push_character(out, c, position);
2106 false
2107 }
2108 },
2109 State::CdataSectionBracket => match ch {
2110 Some(']') => {
2111 self.cdata_pending_brackets.push(position);
2112 self.state = State::CdataSectionEnd;
2113 false
2114 }
2115 _ => {
2116 let bracket_position = self
2121 .cdata_pending_brackets
2122 .pop()
2123 .expect("CdataSectionBracket reached with no withheld ']'");
2124 push_character(out, ']', bracket_position);
2125 self.state = State::CdataSection;
2126 true
2127 }
2128 },
2129 State::CdataSectionEnd => match ch {
2130 Some(']') => {
2131 let oldest = self.cdata_pending_brackets.remove(0);
2138 self.cdata_pending_brackets.push(position);
2139 push_character(out, ']', oldest);
2140 false
2141 }
2142 Some('>') => {
2143 self.cdata_pending_brackets.clear();
2146 self.state = State::Data;
2147 false
2148 }
2149 _ => {
2150 for bracket_position in self.cdata_pending_brackets.drain(..) {
2153 push_character(out, ']', bracket_position);
2154 }
2155 self.state = State::CdataSection;
2156 true
2157 }
2158 },
2159 }
2160 }
2161
2162 fn step_attribute_value_quoted(
2163 &mut self,
2164 ch: Option<char>,
2165 position: Position,
2166 out: &mut Vec<Token>,
2167 quote: char,
2168 ) -> bool {
2169 match ch {
2170 Some(c) if c == quote => {
2171 self.state = State::AfterAttributeValueQuoted;
2172 false
2173 }
2174 Some('&') => {
2175 let quoted_state = self.state;
2176 self.begin_character_reference(quoted_state, position);
2177 false
2178 }
2179 Some('\0') => {
2180 self.push_attribute_value_char('\u{FFFD}');
2181 false
2182 }
2183 Some(c) => {
2184 self.push_attribute_value_char(c);
2185 false
2186 }
2187 None => {
2188 push_eof(out, position);
2189 false
2190 }
2191 }
2192 }
2193
2194 fn current_tag_mut(&mut self) -> &mut TagToken {
2195 self.current_tag
2196 .as_mut()
2197 .expect("tag-name/self-closing state reached with no tag token in progress")
2198 }
2199
2200 fn is_appropriate_end_tag(&self) -> bool {
2203 let Some(tag) = &self.current_tag else {
2204 return false;
2205 };
2206 self.last_start_tag_name.as_deref() == Some(tag.name.as_str())
2207 }
2208
2209 fn step_text_less_than_sign(
2214 &mut self,
2215 ch: Option<char>,
2216 position: Position,
2217 out: &mut Vec<Token>,
2218 end_tag_open_state: State,
2219 text_state: State,
2220 ) -> bool {
2221 match ch {
2222 Some('/') => {
2223 self.text_end_tag_buffer.clear();
2224 self.slash_position = position;
2225 self.state = end_tag_open_state;
2226 false
2227 }
2228 _ => {
2229 push_character(out, '<', self.current_tag_start);
2230 self.state = text_state;
2231 true
2232 }
2233 }
2234 }
2235
2236 fn step_text_end_tag_open(
2238 &mut self,
2239 ch: Option<char>,
2240 _position: Position,
2241 out: &mut Vec<Token>,
2242 end_tag_name_state: State,
2243 text_state: State,
2244 ) -> bool {
2245 match ch {
2246 Some(c) if c.is_ascii_alphabetic() => {
2247 self.start_tag_token(true);
2248 self.state = end_tag_name_state;
2249 true
2250 }
2251 _ => {
2252 push_character(out, '<', self.current_tag_start);
2253 push_character(out, '/', self.slash_position);
2254 self.state = text_state;
2255 true
2256 }
2257 }
2258 }
2259
2260 fn step_text_end_tag_name(
2266 &mut self,
2267 ch: Option<char>,
2268 _position: Position,
2269 out: &mut Vec<Token>,
2270 text_state: State,
2271 ) -> bool {
2272 match ch {
2273 Some(c) if Self::is_whitespace(c) => {
2274 if self.is_appropriate_end_tag() {
2275 self.state = State::BeforeAttributeName;
2276 false
2277 } else {
2278 self.abandon_text_end_tag(out, text_state)
2279 }
2280 }
2281 Some('/') => {
2282 if self.is_appropriate_end_tag() {
2283 self.state = State::SelfClosingStartTag;
2284 false
2285 } else {
2286 self.abandon_text_end_tag(out, text_state)
2287 }
2288 }
2289 Some('>') => {
2290 if self.is_appropriate_end_tag() {
2291 self.close_tag(out)
2292 } else {
2293 self.abandon_text_end_tag(out, text_state)
2294 }
2295 }
2296 Some(c) if c.is_ascii_uppercase() => {
2297 self.current_tag_mut().name.push(c.to_ascii_lowercase());
2298 self.text_end_tag_buffer.push(c);
2299 false
2300 }
2301 Some(c) if c.is_ascii_lowercase() => {
2302 self.current_tag_mut().name.push(c);
2303 self.text_end_tag_buffer.push(c);
2304 false
2305 }
2306 _ => self.abandon_text_end_tag(out, text_state),
2307 }
2308 }
2309
2310 fn abandon_text_end_tag(&mut self, out: &mut Vec<Token>, text_state: State) -> bool {
2321 push_character(out, '<', self.current_tag_start);
2322 push_character(out, '/', self.slash_position);
2323 let mut position = Position {
2324 line: self.slash_position.line,
2325 column: self.slash_position.column + 1,
2326 byte_offset: self.slash_position.byte_offset + 1,
2327 };
2328 let buffer = std::mem::take(&mut self.text_end_tag_buffer);
2329 for c in buffer.chars() {
2330 push_character(out, c, position);
2331 position.column += 1;
2332 position.byte_offset += 1;
2333 }
2334 self.current_tag = None;
2335 self.state = text_state;
2336 true
2337 }
2338
2339 fn step_doctype_identifier_quoted(
2344 &mut self,
2345 ch: Option<char>,
2346 position: Position,
2347 out: &mut Vec<Token>,
2348 quote: char,
2349 kind: DoctypeIdentifierKind,
2350 ) -> bool {
2351 match ch {
2352 Some(c) if c == quote => {
2353 self.state = match kind {
2354 DoctypeIdentifierKind::Public => State::AfterDoctypePublicIdentifier,
2355 DoctypeIdentifierKind::System => State::AfterDoctypeSystemIdentifier,
2356 };
2357 false
2358 }
2359 Some('\0') => {
2360 self.doctype_identifier_mut(kind).push('\u{FFFD}');
2361 false
2362 }
2363 Some('>') => self.close_doctype_with_quirks(out),
2364 None => self.eof_in_doctype(out, position),
2365 Some(c) => {
2366 self.doctype_identifier_mut(kind).push(c);
2367 false
2368 }
2369 }
2370 }
2371
2372 fn begin_character_reference(&mut self, return_state: State, position: Position) {
2377 self.return_state = return_state;
2378 self.character_reference_start = position;
2379 self.character_reference_start_index = self.index - 1;
2380 self.state = State::CharacterReference;
2381 }
2382
2383 fn character_reference_in_attribute(&self) -> bool {
2386 matches!(
2387 self.return_state,
2388 State::AttributeValueDoubleQuoted
2389 | State::AttributeValueSingleQuoted
2390 | State::AttributeValueUnquoted
2391 )
2392 }
2393
2394 fn flush_char_as_character_reference(
2399 &mut self,
2400 c: char,
2401 position: Position,
2402 out: &mut Vec<Token>,
2403 ) {
2404 if self.character_reference_in_attribute() {
2405 self.push_attribute_value_char(c);
2406 } else {
2407 push_character(out, c, position);
2408 }
2409 }
2410
2411 fn flush_literal_character_reference_attempt(
2421 &mut self,
2422 end_index: usize,
2423 out: &mut Vec<Token>,
2424 ) {
2425 let chars: Vec<char> = self.chars[self.character_reference_start_index..end_index].to_vec();
2426 let mut position = self.character_reference_start;
2427 for c in chars {
2428 self.flush_char_as_character_reference(c, position, out);
2429 position.column += 1;
2430 position.byte_offset += 1;
2431 }
2432 }
2433
2434 fn lookup_named_character_reference(name: &str) -> Option<&'static str> {
2437 entities::NAMED_CHARACTER_REFERENCES
2438 .binary_search_by(|&(candidate, _)| candidate.cmp(name))
2439 .ok()
2440 .map(|i| entities::NAMED_CHARACTER_REFERENCES[i].1)
2441 }
2442
2443 fn named_character_reference_has_longer_match(prefix: &str) -> bool {
2448 let start =
2449 entities::NAMED_CHARACTER_REFERENCES.partition_point(|&(name, _)| name < prefix);
2450 entities::NAMED_CHARACTER_REFERENCES[start..]
2451 .iter()
2452 .take_while(|&&(name, _)| name.starts_with(prefix))
2453 .any(|&(name, _)| name.len() > prefix.len())
2454 }
2455
2456 fn run_named_character_reference(&mut self) {
2465 let (first_char, _) = self.consume();
2466 let first_char = first_char.expect(
2467 "named character reference state entered without an alphanumeric first character",
2468 );
2469
2470 let mut candidate = String::new();
2471 candidate.push(first_char);
2472 let mut cursor = self.index;
2473 let mut best: Option<usize> = None;
2474 if Self::lookup_named_character_reference(&candidate).is_some() {
2475 best = Some(candidate.len());
2476 }
2477 loop {
2478 if !Self::named_character_reference_has_longer_match(&candidate) {
2479 break;
2480 }
2481 let Some(&c) = self.chars.get(cursor) else {
2482 break;
2483 };
2484 candidate.push(c);
2485 cursor += 1;
2486 if Self::lookup_named_character_reference(&candidate).is_some() {
2487 best = Some(candidate.len());
2488 }
2489 }
2490
2491 let mut out = Vec::new();
2492 match best {
2493 Some(matched_len) => {
2494 self.index = self.character_reference_start_index + 1 + matched_len;
2495 let matched_name = candidate[..matched_len].to_owned();
2496 let next_char = self.chars.get(self.index).copied();
2497 let last_matched_is_semicolon = matched_name.ends_with(';');
2498 let next_is_equals_or_alphanumeric = matches!(next_char, Some('='))
2499 || matches!(next_char, Some(c) if c.is_ascii_alphanumeric());
2500 let historical = self.character_reference_in_attribute()
2501 && !last_matched_is_semicolon
2502 && next_is_equals_or_alphanumeric;
2503 if historical {
2504 self.flush_literal_character_reference_attempt(self.index, &mut out);
2505 } else {
2506 let replacement = Self::lookup_named_character_reference(&matched_name)
2510 .expect("matched_name was already verified to be a table entry");
2511 let start = self.character_reference_start;
2512 for c in replacement.chars() {
2513 self.flush_char_as_character_reference(c, start, &mut out);
2514 }
2515 }
2516 self.state = self.return_state;
2517 }
2518 None => {
2519 self.index = cursor;
2524 self.flush_literal_character_reference_attempt(self.index, &mut out);
2525 self.state = State::AmbiguousAmpersand;
2526 }
2527 }
2528 self.pending.extend(out);
2529 }
2530
2531 fn resolve_numeric_character_reference_code(&self) -> char {
2538 const NUL: u32 = 0x00;
2539 const MAX_UNICODE: u32 = 0x10FFFF;
2540 let code = self.character_reference_code;
2541 let resolved = if code == NUL || code > MAX_UNICODE || is_surrogate(code) {
2542 0xFFFD
2543 } else if is_noncharacter(code) {
2544 code
2545 } else if code == 0x0D || (is_control(code) && !is_ascii_whitespace(code)) {
2546 windows_1252_override(code).unwrap_or(code)
2547 } else {
2548 code
2549 };
2550 char::from_u32(resolved).unwrap_or('\u{FFFD}')
2551 }
2552}
2553
2554fn push_eof(out: &mut Vec<Token>, position: Position) {
2560 out.push(Token {
2561 kind: TokenKind::Eof,
2562 position,
2563 });
2564}
2565
2566fn push_character(out: &mut Vec<Token>, c: char, position: Position) {
2569 out.push(Token {
2570 kind: TokenKind::Character(c),
2571 position,
2572 });
2573}
2574
2575fn is_surrogate(code: u32) -> bool {
2576 (0xD800..=0xDFFF).contains(&code)
2577}
2578
2579fn is_noncharacter(code: u32) -> bool {
2583 (0xFDD0..=0xFDEF).contains(&code) || matches!(code & 0xFFFF, 0xFFFE | 0xFFFF)
2584}
2585
2586fn is_control(code: u32) -> bool {
2589 (0x00..=0x1F).contains(&code) || (0x7F..=0x9F).contains(&code)
2590}
2591
2592fn is_ascii_whitespace(code: u32) -> bool {
2594 matches!(code, 0x09 | 0x0A | 0x0C | 0x0D | 0x20)
2595}
2596
2597fn windows_1252_override(code: u32) -> Option<u32> {
2603 const TABLE: &[(u32, u32)] = &[
2604 (0x80, 0x20AC),
2605 (0x82, 0x201A),
2606 (0x83, 0x0192),
2607 (0x84, 0x201E),
2608 (0x85, 0x2026),
2609 (0x86, 0x2020),
2610 (0x87, 0x2021),
2611 (0x88, 0x02C6),
2612 (0x89, 0x2030),
2613 (0x8A, 0x0160),
2614 (0x8B, 0x2039),
2615 (0x8C, 0x0152),
2616 (0x8E, 0x017D),
2617 (0x91, 0x2018),
2618 (0x92, 0x2019),
2619 (0x93, 0x201C),
2620 (0x94, 0x201D),
2621 (0x95, 0x2022),
2622 (0x96, 0x2013),
2623 (0x97, 0x2014),
2624 (0x98, 0x02DC),
2625 (0x99, 0x2122),
2626 (0x9A, 0x0161),
2627 (0x9B, 0x203A),
2628 (0x9C, 0x0153),
2629 (0x9E, 0x017E),
2630 (0x9F, 0x0178),
2631 ];
2632 TABLE
2633 .iter()
2634 .find(|&&(from, _)| from == code)
2635 .map(|&(_, to)| to)
2636}
2637
2638impl Iterator for Tokenizer {
2639 type Item = Token;
2640
2641 fn next(&mut self) -> Option<Token> {
2642 if self.pending.is_empty() {
2643 if self.eof_returned {
2644 return None;
2645 }
2646 self.run_until_token();
2647 }
2648 let token = self.pending.pop_front()?;
2649 if matches!(token.kind, TokenKind::Eof) {
2650 self.eof_returned = true;
2651 }
2652 Some(token)
2653 }
2654}
2655
2656#[cfg(test)]
2657mod tests {
2658 use super::*;
2663
2664 fn tokenize(input: &str) -> Vec<Token> {
2665 Tokenizer::new(input).collect()
2666 }
2667
2668 fn kinds(tokens: &[Token]) -> Vec<&TokenKind> {
2669 tokens.iter().map(|token| &token.kind).collect()
2670 }
2671
2672 fn pos(line: u32, column: u32, byte_offset: usize) -> Position {
2673 Position {
2674 line,
2675 column,
2676 byte_offset,
2677 }
2678 }
2679
2680 #[test]
2681 fn plain_text_emits_one_character_token_per_char_then_eof() {
2682 let tokens = tokenize("hi");
2683 assert_eq!(
2684 tokens,
2685 vec![
2686 Token {
2687 kind: TokenKind::Character('h'),
2688 position: pos(1, 1, 0),
2689 },
2690 Token {
2691 kind: TokenKind::Character('i'),
2692 position: pos(1, 2, 1),
2693 },
2694 Token {
2695 kind: TokenKind::Eof,
2696 position: pos(1, 3, 2),
2697 },
2698 ]
2699 );
2700 }
2701
2702 #[test]
2703 fn simple_start_tag_with_no_attributes() {
2704 let tokens = tokenize("<p>");
2705 assert_eq!(
2706 tokens,
2707 vec![
2708 Token {
2709 kind: TokenKind::StartTag(TagToken {
2710 name: "p".to_owned(),
2711 self_closing: false,
2712 attributes: vec![],
2713 }),
2714 position: pos(1, 1, 0),
2715 },
2716 Token {
2717 kind: TokenKind::Eof,
2718 position: pos(1, 4, 3),
2719 },
2720 ]
2721 );
2722 }
2723
2724 #[test]
2725 fn end_tag() {
2726 let tokens = tokenize("</p>");
2727 match &tokens[0].kind {
2728 TokenKind::EndTag(tag) => assert_eq!(tag.name, "p"),
2729 other => panic!("expected end tag, got {other:?}"),
2730 }
2731 }
2732
2733 #[test]
2734 fn self_closing_tag() {
2735 let tokens = tokenize("<br/>");
2736 match &tokens[0].kind {
2737 TokenKind::StartTag(tag) => {
2738 assert_eq!(tag.name, "br");
2739 assert!(tag.self_closing);
2740 }
2741 other => panic!("expected start tag, got {other:?}"),
2742 }
2743 }
2744
2745 #[test]
2746 fn attribute_value_quoting_forms() {
2747 for html in ["<a href=\"x\">", "<a href='x'>", "<a href=x>"] {
2748 let tokens = tokenize(html);
2749 match &tokens[0].kind {
2750 TokenKind::StartTag(tag) => assert_eq!(
2751 tag.attributes,
2752 vec![Attribute {
2753 name: "href".to_owned(),
2754 value: "x".to_owned(),
2755 }],
2756 "input: {html:?}",
2757 ),
2758 other => panic!("expected start tag for {html:?}, got {other:?}"),
2759 }
2760 }
2761 }
2762
2763 #[test]
2764 fn multiple_attributes_separated_by_whitespace() {
2765 let tokens = tokenize(r#"<a href="x" target="y">"#);
2766 match &tokens[0].kind {
2767 TokenKind::StartTag(tag) => assert_eq!(
2768 tag.attributes,
2769 vec![
2770 Attribute {
2771 name: "href".to_owned(),
2772 value: "x".to_owned(),
2773 },
2774 Attribute {
2775 name: "target".to_owned(),
2776 value: "y".to_owned(),
2777 },
2778 ]
2779 ),
2780 other => panic!("expected start tag, got {other:?}"),
2781 }
2782 }
2783
2784 #[test]
2785 fn duplicate_attribute_name_keeps_first_occurrence_only() {
2786 let tokens = tokenize(r#"<a href="x" href="y">"#);
2787 match &tokens[0].kind {
2788 TokenKind::StartTag(tag) => assert_eq!(
2789 tag.attributes,
2790 vec![Attribute {
2791 name: "href".to_owned(),
2792 value: "x".to_owned(),
2793 }]
2794 ),
2795 other => panic!("expected start tag, got {other:?}"),
2796 }
2797 }
2798
2799 #[test]
2800 fn tag_and_attribute_names_are_ascii_lowercased() {
2801 let tokens = tokenize(r#"<DIV CLASS="a">"#);
2802 match &tokens[0].kind {
2803 TokenKind::StartTag(tag) => {
2804 assert_eq!(tag.name, "div");
2805 assert_eq!(tag.attributes[0].name, "class");
2806 }
2807 other => panic!("expected start tag, got {other:?}"),
2808 }
2809 }
2810
2811 #[test]
2812 fn boolean_attribute_with_no_value() {
2813 let tokens = tokenize("<input disabled>");
2814 match &tokens[0].kind {
2815 TokenKind::StartTag(tag) => assert_eq!(
2816 tag.attributes,
2817 vec![Attribute {
2818 name: "disabled".to_owned(),
2819 value: String::new(),
2820 }]
2821 ),
2822 other => panic!("expected start tag, got {other:?}"),
2823 }
2824 }
2825
2826 #[test]
2827 fn null_character_in_tag_name_is_replaced_with_u_fffd() {
2828 let tokens = tokenize("<a\u{0}b>");
2829 match &tokens[0].kind {
2830 TokenKind::StartTag(tag) => assert_eq!(tag.name, "a\u{FFFD}b"),
2831 other => panic!("expected start tag, got {other:?}"),
2832 }
2833 }
2834
2835 #[test]
2836 fn null_character_in_data_state_is_emitted_literally_not_replaced() {
2837 let tokens = tokenize("\u{0}");
2838 assert_eq!(tokens[0].kind, TokenKind::Character('\u{0}'));
2839 }
2840
2841 #[test]
2842 fn eof_inside_a_tag_emits_only_eof_no_tag_token() {
2843 let tokens = tokenize("<div");
2844 assert_eq!(
2845 tokens,
2846 vec![Token {
2847 kind: TokenKind::Eof,
2848 position: pos(1, 5, 4),
2849 }]
2850 );
2851 }
2852
2853 #[test]
2854 fn eof_right_after_solidus_emits_synthesized_lt_and_solidus_then_eof() {
2855 let tokens = tokenize("</");
2856 assert_eq!(
2857 tokens,
2858 vec![
2859 Token {
2860 kind: TokenKind::Character('<'),
2861 position: pos(1, 1, 0),
2862 },
2863 Token {
2864 kind: TokenKind::Character('/'),
2865 position: pos(1, 2, 1),
2866 },
2867 Token {
2868 kind: TokenKind::Eof,
2869 position: pos(1, 3, 2),
2870 },
2871 ]
2872 );
2873 }
2874
2875 #[test]
2876 fn position_tracking_across_a_newline() {
2877 let tokens = tokenize("a\nb");
2878 assert_eq!(tokens[0].position, pos(1, 1, 0));
2879 assert_eq!(tokens[1].position, pos(1, 2, 1)); assert_eq!(tokens[2].position, pos(2, 1, 2));
2881 }
2882
2883 #[test]
2884 fn crlf_and_lone_cr_normalize_to_a_single_lf_character_token() {
2885 let crlf = tokenize("a\r\nb");
2886 assert_eq!(
2887 kinds(&crlf),
2888 vec![
2889 &TokenKind::Character('a'),
2890 &TokenKind::Character('\n'),
2891 &TokenKind::Character('b'),
2892 &TokenKind::Eof,
2893 ]
2894 );
2895 let lone_cr = tokenize("a\rb");
2896 assert_eq!(
2897 kinds(&lone_cr),
2898 vec![
2899 &TokenKind::Character('a'),
2900 &TokenKind::Character('\n'),
2901 &TokenKind::Character('b'),
2902 &TokenKind::Eof,
2903 ]
2904 );
2905 }
2906
2907 #[test]
2908 fn named_character_reference_with_semicolon() {
2909 assert_eq!(
2910 kinds(&tokenize("&")),
2911 vec![&TokenKind::Character('&'), &TokenKind::Eof]
2912 );
2913 }
2914
2915 #[test]
2916 fn named_character_reference_multi_codepoint() {
2917 assert_eq!(
2919 kinds(&tokenize("≂̸")),
2920 vec![
2921 &TokenKind::Character('\u{2242}'),
2922 &TokenKind::Character('\u{338}'),
2923 &TokenKind::Eof,
2924 ]
2925 );
2926 }
2927
2928 #[test]
2929 fn legacy_named_character_reference_without_semicolon_still_resolves_outside_attributes() {
2930 assert_eq!(
2934 kinds(&tokenize("& b")),
2935 vec![
2936 &TokenKind::Character('&'),
2937 &TokenKind::Character(' '),
2938 &TokenKind::Character('b'),
2939 &TokenKind::Eof,
2940 ]
2941 );
2942 }
2943
2944 #[test]
2945 fn unknown_named_character_reference_falls_back_to_ambiguous_ampersand() {
2946 assert_eq!(
2950 kinds(&tokenize("&1;")),
2951 vec![
2952 &TokenKind::Character('&'),
2953 &TokenKind::Character('1'),
2954 &TokenKind::Character(';'),
2955 &TokenKind::Eof,
2956 ]
2957 );
2958 }
2959
2960 #[test]
2961 fn named_character_reference_historical_fallback_in_unquoted_attribute_value() {
2962 let tokens = tokenize("<a href=¬it=1>");
2967 match &tokens[0].kind {
2968 TokenKind::StartTag(tag) => assert_eq!(
2969 tag.attributes,
2970 vec![Attribute {
2971 name: "href".to_owned(),
2972 value: "¬it=1".to_owned(),
2973 }]
2974 ),
2975 other => panic!("expected start tag, got {other:?}"),
2976 }
2977 }
2978
2979 #[test]
2980 fn named_character_reference_historical_fallback_applies_in_double_quoted_attributes_too() {
2981 let tokens = tokenize(r#"<a href="¬it">"#);
2987 match &tokens[0].kind {
2988 TokenKind::StartTag(tag) => assert_eq!(
2989 tag.attributes,
2990 vec![Attribute {
2991 name: "href".to_owned(),
2992 value: "¬it".to_owned(),
2993 }]
2994 ),
2995 other => panic!("expected start tag, got {other:?}"),
2996 }
2997 }
2998
2999 #[test]
3000 fn named_character_reference_ending_in_semicolon_resolves_normally_even_in_an_attribute() {
3001 let tokens = tokenize(r#"<a href="©">"#);
3005 match &tokens[0].kind {
3006 TokenKind::StartTag(tag) => assert_eq!(
3007 tag.attributes,
3008 vec![Attribute {
3009 name: "href".to_owned(),
3010 value: "\u{A9}".to_owned(),
3011 }]
3012 ),
3013 other => panic!("expected start tag, got {other:?}"),
3014 }
3015 }
3016
3017 #[test]
3018 fn decimal_and_hexadecimal_character_references() {
3019 assert_eq!(
3020 kinds(&tokenize("A")),
3021 vec![&TokenKind::Character('A'), &TokenKind::Eof]
3022 );
3023 assert_eq!(
3024 kinds(&tokenize("A")),
3025 vec![&TokenKind::Character('A'), &TokenKind::Eof]
3026 );
3027 assert_eq!(
3028 kinds(&tokenize("A")),
3029 vec![&TokenKind::Character('A'), &TokenKind::Eof]
3030 );
3031 }
3032
3033 #[test]
3034 fn numeric_character_reference_missing_semicolon_still_resolves() {
3035 assert_eq!(
3036 kinds(&tokenize("Ax")),
3037 vec![
3038 &TokenKind::Character('A'),
3039 &TokenKind::Character('x'),
3040 &TokenKind::Eof,
3041 ]
3042 );
3043 }
3044
3045 #[test]
3046 fn numeric_character_reference_null_is_replaced_with_u_fffd() {
3047 assert_eq!(
3048 kinds(&tokenize("�")),
3049 vec![&TokenKind::Character('\u{FFFD}'), &TokenKind::Eof]
3050 );
3051 }
3052
3053 #[test]
3054 fn numeric_character_reference_outside_unicode_range_is_replaced_with_u_fffd() {
3055 assert_eq!(
3056 kinds(&tokenize("�")),
3057 vec![&TokenKind::Character('\u{FFFD}'), &TokenKind::Eof]
3058 );
3059 }
3060
3061 #[test]
3062 fn numeric_character_reference_surrogate_is_replaced_with_u_fffd() {
3063 assert_eq!(
3064 kinds(&tokenize("�")),
3065 vec![&TokenKind::Character('\u{FFFD}'), &TokenKind::Eof]
3066 );
3067 }
3068
3069 #[test]
3070 fn numeric_character_reference_windows_1252_control_override() {
3071 assert_eq!(
3074 kinds(&tokenize("€")),
3075 vec![&TokenKind::Character('\u{20AC}'), &TokenKind::Eof]
3076 );
3077 }
3078
3079 #[test]
3080 fn numeric_character_reference_unmapped_c1_control_is_left_unchanged() {
3081 assert_eq!(
3085 kinds(&tokenize("")),
3086 vec![&TokenKind::Character('\u{81}'), &TokenKind::Eof]
3087 );
3088 }
3089
3090 #[test]
3091 fn absence_of_digits_in_numeric_character_reference_falls_back_to_literal_text() {
3092 assert_eq!(
3093 kinds(&tokenize("&#;")),
3094 vec![
3095 &TokenKind::Character('&'),
3096 &TokenKind::Character('#'),
3097 &TokenKind::Character(';'),
3098 &TokenKind::Eof,
3099 ]
3100 );
3101 assert_eq!(
3102 kinds(&tokenize("&#x;")),
3103 vec![
3104 &TokenKind::Character('&'),
3105 &TokenKind::Character('#'),
3106 &TokenKind::Character('x'),
3107 &TokenKind::Character(';'),
3108 &TokenKind::Eof,
3109 ]
3110 );
3111 }
3112
3113 #[test]
3114 fn lone_ampersand_at_eof_is_emitted_literally() {
3115 assert_eq!(
3116 kinds(&tokenize("&")),
3117 vec![&TokenKind::Character('&'), &TokenKind::Eof]
3118 );
3119 }
3120
3121 #[test]
3122 fn simple_comment() {
3123 assert_eq!(
3124 kinds(&tokenize("<!-- hi -->")),
3125 vec![&TokenKind::Comment(" hi ".to_owned()), &TokenKind::Eof]
3126 );
3127 }
3128
3129 #[test]
3130 fn abrupt_closing_of_empty_comment_still_emits_an_empty_comment_token() {
3131 assert_eq!(
3132 kinds(&tokenize("<!-->")),
3133 vec![&TokenKind::Comment(String::new()), &TokenKind::Eof]
3134 );
3135 }
3136
3137 #[test]
3138 fn comment_containing_a_lone_hyphen() {
3139 assert_eq!(
3140 kinds(&tokenize("<!-- a - b -->")),
3141 vec![&TokenKind::Comment(" a - b ".to_owned()), &TokenKind::Eof]
3142 );
3143 }
3144
3145 #[test]
3146 fn eof_inside_a_comment_still_emits_the_comment_token() {
3147 assert_eq!(
3148 kinds(&tokenize("<!-- unterminated")),
3149 vec![
3150 &TokenKind::Comment(" unterminated".to_owned()),
3151 &TokenKind::Eof,
3152 ]
3153 );
3154 }
3155
3156 #[test]
3157 fn nested_comment_open_sequence_is_absorbed_as_data_not_a_real_nested_comment() {
3158 assert_eq!(
3163 kinds(&tokenize("<!--<!-->")),
3164 vec![&TokenKind::Comment("<!".to_owned()), &TokenKind::Eof]
3165 );
3166 }
3167
3168 #[test]
3169 fn incorrectly_opened_comment_falls_back_to_bogus_comment() {
3170 assert_eq!(
3173 kinds(&tokenize("<!weird>")),
3174 vec![&TokenKind::Comment("weird".to_owned()), &TokenKind::Eof]
3175 );
3176 }
3177
3178 #[test]
3179 fn end_tag_with_invalid_first_character_falls_back_to_bogus_comment() {
3180 assert_eq!(
3183 kinds(&tokenize("</1>")),
3184 vec![&TokenKind::Comment("1".to_owned()), &TokenKind::Eof]
3185 );
3186 }
3187
3188 #[test]
3189 fn cdata_outside_foreign_content_becomes_a_bogus_comment() {
3190 assert_eq!(
3195 kinds(&tokenize("<![CDATA[x]]>")),
3196 vec![
3197 &TokenKind::Comment("[CDATA[x]]".to_owned()),
3198 &TokenKind::Eof,
3199 ]
3200 );
3201 }
3202
3203 fn expect_doctype(tokens: &[Token]) -> &DoctypeToken {
3204 match &tokens[0].kind {
3205 TokenKind::Doctype(doctype) => doctype,
3206 other => panic!("expected DOCTYPE token, got {other:?}"),
3207 }
3208 }
3209
3210 #[test]
3211 fn simple_doctype() {
3212 let tokens = tokenize("<!DOCTYPE html>");
3213 assert_eq!(
3214 expect_doctype(&tokens),
3215 &DoctypeToken {
3216 name: Some("html".to_owned()),
3217 ..Default::default()
3218 }
3219 );
3220 assert_eq!(kinds(&tokens)[1], &TokenKind::Eof);
3221 }
3222
3223 #[test]
3224 fn doctype_keyword_and_name_are_ascii_case_insensitive_lowercased() {
3225 let tokens = tokenize("<!doctype HTML>");
3226 assert_eq!(
3227 expect_doctype(&tokens),
3228 &DoctypeToken {
3229 name: Some("html".to_owned()),
3230 ..Default::default()
3231 }
3232 );
3233 }
3234
3235 #[test]
3236 fn doctype_with_no_name_sets_force_quirks() {
3237 let tokens = tokenize("<!DOCTYPE>");
3238 assert_eq!(
3239 expect_doctype(&tokens),
3240 &DoctypeToken {
3241 force_quirks: true,
3242 ..Default::default()
3243 }
3244 );
3245 }
3246
3247 #[test]
3248 fn doctype_with_public_and_system_identifiers() {
3249 let tokens = tokenize(
3250 r#"<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01//EN" "http://www.w3.org/TR/html4/strict.dtd">"#,
3251 );
3252 assert_eq!(
3253 expect_doctype(&tokens),
3254 &DoctypeToken {
3255 name: Some("html".to_owned()),
3256 public_identifier: Some("-//W3C//DTD HTML 4.01//EN".to_owned()),
3257 system_identifier: Some("http://www.w3.org/TR/html4/strict.dtd".to_owned()),
3258 force_quirks: false,
3259 }
3260 );
3261 }
3262
3263 #[test]
3264 fn eof_inside_doctype_name_sets_force_quirks_but_keeps_the_partial_name() {
3265 let tokens = tokenize("<!DOCTYPE html");
3266 assert_eq!(
3267 tokens,
3268 vec![
3269 Token {
3270 kind: TokenKind::Doctype(DoctypeToken {
3271 name: Some("html".to_owned()),
3272 force_quirks: true,
3273 ..Default::default()
3274 }),
3275 position: pos(1, 1, 0),
3276 },
3277 Token {
3278 kind: TokenKind::Eof,
3279 position: pos(1, 15, 14),
3280 },
3281 ]
3282 );
3283 }
3284
3285 #[test]
3286 fn unrecognized_text_after_doctype_name_falls_back_to_bogus_doctype() {
3287 let tokens = tokenize("<!DOCTYPE html GARBAGE>");
3290 assert_eq!(
3291 expect_doctype(&tokens),
3292 &DoctypeToken {
3293 name: Some("html".to_owned()),
3294 force_quirks: true,
3295 ..Default::default()
3296 }
3297 );
3298 }
3299
3300 #[test]
3301 fn simple_processing_instruction() {
3302 let tokens = tokenize("<?foo bar?>");
3303 assert_eq!(
3304 kinds(&tokens),
3305 vec![
3306 &TokenKind::ProcessingInstruction(ProcessingInstructionToken {
3307 target: "foo".to_owned(),
3308 data: "bar".to_owned(),
3309 }),
3310 &TokenKind::Eof,
3311 ]
3312 );
3313 }
3314
3315 #[test]
3316 fn xml_target_is_disallowed_and_becomes_a_bogus_comment() {
3317 let tokens = tokenize(r#"<?xml version="1.0"?>"#);
3318 assert_eq!(
3319 kinds(&tokens),
3320 vec![
3321 &TokenKind::Comment("?xml version=\"1.0\"?".to_owned()),
3322 &TokenKind::Eof,
3323 ]
3324 );
3325 }
3326
3327 #[test]
3328 fn xml_stylesheet_target_is_disallowed_and_becomes_a_bogus_comment() {
3329 let tokens = tokenize("<?xml-stylesheet foo?>");
3330 assert_eq!(
3331 kinds(&tokens),
3332 vec![
3333 &TokenKind::Comment("?xml-stylesheet foo?".to_owned()),
3334 &TokenKind::Eof,
3335 ]
3336 );
3337 }
3338
3339 #[test]
3340 fn eof_right_after_question_mark_emits_only_eof_no_token_at_all() {
3341 assert_eq!(kinds(&tokenize("<?")), vec![&TokenKind::Eof]);
3345 }
3346
3347 #[test]
3348 fn eof_inside_processing_instruction_data_discards_the_pi_token() {
3349 assert_eq!(kinds(&tokenize("<?foo bar")), vec![&TokenKind::Eof]);
3352 }
3353
3354 #[test]
3355 fn lone_question_marks_inside_processing_instruction_data_are_kept_literally() {
3356 let tokens = tokenize("<?foo a?b?>");
3357 assert_eq!(
3358 kinds(&tokens),
3359 vec![
3360 &TokenKind::ProcessingInstruction(ProcessingInstructionToken {
3361 target: "foo".to_owned(),
3362 data: "a?b".to_owned(),
3363 }),
3364 &TokenKind::Eof,
3365 ]
3366 );
3367 }
3368
3369 fn tokenize_switching_after_start_tag(input: &str, state: ExternalState) -> Vec<Token> {
3374 let mut tokenizer = Tokenizer::new(input);
3375 let mut tokens = Vec::new();
3376 while let Some(token) = tokenizer.next() {
3377 let is_start = matches!(&token.kind, TokenKind::StartTag(_));
3378 tokens.push(token);
3379 if is_start {
3380 tokenizer.switch_to(state);
3381 }
3382 }
3383 tokens
3384 }
3385
3386 fn characters_only(tokens: &[Token]) -> String {
3387 tokens
3388 .iter()
3389 .filter_map(|token| match &token.kind {
3390 TokenKind::Character(c) => Some(*c),
3391 _ => None,
3392 })
3393 .collect()
3394 }
3395
3396 #[test]
3397 fn rcdata_resolves_character_references_and_ends_on_matching_end_tag() {
3398 let tokens =
3399 tokenize_switching_after_start_tag("<title>AT&T</title>", ExternalState::RcData);
3400 assert_eq!(
3401 kinds(&tokens),
3402 vec![
3403 &TokenKind::StartTag(TagToken {
3404 name: "title".to_owned(),
3405 self_closing: false,
3406 attributes: vec![],
3407 }),
3408 &TokenKind::Character('A'),
3409 &TokenKind::Character('T'),
3410 &TokenKind::Character('&'),
3411 &TokenKind::Character('T'),
3412 &TokenKind::EndTag(TagToken {
3413 name: "title".to_owned(),
3414 self_closing: false,
3415 attributes: vec![],
3416 }),
3417 &TokenKind::Eof,
3418 ]
3419 );
3420 }
3421
3422 #[test]
3423 fn rcdata_end_tag_with_wrong_name_is_kept_as_literal_text() {
3424 let tokens =
3427 tokenize_switching_after_start_tag("<title>a</b>c</title>", ExternalState::RcData);
3428 assert_eq!(
3429 characters_only(&tokens[1..tokens.len() - 2]),
3430 "a</b>c".to_owned()
3431 );
3432 match &tokens.last().unwrap().kind {
3433 TokenKind::Eof => {}
3434 other => panic!("expected trailing Eof, got {other:?}"),
3435 }
3436 match &tokens[tokens.len() - 2].kind {
3437 TokenKind::EndTag(tag) => assert_eq!(tag.name, "title"),
3438 other => panic!("expected closing end tag, got {other:?}"),
3439 }
3440 }
3441
3442 #[test]
3443 fn rawtext_does_not_resolve_character_references() {
3444 let tokens =
3445 tokenize_switching_after_start_tag("<style>&</style>", ExternalState::RawText);
3446 assert_eq!(characters_only(&tokens), "&".to_owned());
3447 }
3448
3449 #[test]
3450 fn plaintext_never_recognizes_any_end_tag_ever_again() {
3451 let tokens = tokenize_switching_after_start_tag(
3454 "<plaintext>a</plaintext>b",
3455 ExternalState::PlainText,
3456 );
3457 assert_eq!(characters_only(&tokens), "a</plaintext>b".to_owned());
3458 match &tokens.last().unwrap().kind {
3459 TokenKind::Eof => {}
3460 other => panic!("expected trailing Eof, got {other:?}"),
3461 }
3462 }
3463
3464 #[test]
3465 fn appropriate_end_tag_requires_a_start_tag_to_have_been_emitted_first() {
3466 let mut tokenizer = Tokenizer::new("</title>");
3470 tokenizer.switch_to(ExternalState::RcData);
3471 let tokens: Vec<_> = tokenizer.collect();
3472 assert_eq!(characters_only(&tokens), "</title>".to_owned());
3473 assert_eq!(tokens.last().unwrap().kind, TokenKind::Eof);
3474 }
3475
3476 #[test]
3477 fn script_data_plain_content_no_html_comment_like_wrapper() {
3478 let tokens = tokenize_switching_after_start_tag(
3479 "<script>var x = 1;</script>",
3480 ExternalState::ScriptData,
3481 );
3482 assert_eq!(characters_only(&tokens), "var x = 1;".to_owned());
3483 match &tokens[tokens.len() - 2].kind {
3484 TokenKind::EndTag(tag) => assert_eq!(tag.name, "script"),
3485 other => panic!("expected closing end tag, got {other:?}"),
3486 }
3487 }
3488
3489 #[test]
3490 fn script_data_does_not_resolve_character_references_either() {
3491 let tokens =
3493 tokenize_switching_after_start_tag("<script>&</script>", ExternalState::ScriptData);
3494 assert_eq!(characters_only(&tokens), "&".to_owned());
3495 }
3496
3497 #[test]
3498 fn script_data_html_comment_like_wrapper_round_trips_literally() {
3499 let source = "<!--alert(1);-->";
3502 let tokens = tokenize_switching_after_start_tag(
3503 &format!("<script>{source}</script>"),
3504 ExternalState::ScriptData,
3505 );
3506 assert_eq!(characters_only(&tokens), source.to_owned());
3507 match &tokens[tokens.len() - 2].kind {
3508 TokenKind::EndTag(tag) => assert_eq!(tag.name, "script"),
3509 other => panic!("expected closing end tag, got {other:?}"),
3510 }
3511 }
3512
3513 #[test]
3514 fn nested_script_tags_inside_html_comment_like_wrapper_do_not_end_the_element_early() {
3515 let source = "<!--<script>x</script>-->";
3522 let tokens = tokenize_switching_after_start_tag(
3523 &format!("<script>{source}</script>"),
3524 ExternalState::ScriptData,
3525 );
3526 assert_eq!(characters_only(&tokens), source.to_owned());
3527 let end_tags: Vec<_> = tokens
3528 .iter()
3529 .filter(|token| matches!(&token.kind, TokenKind::EndTag(_)))
3530 .collect();
3531 assert_eq!(
3532 end_tags.len(),
3533 1,
3534 "only the real closing tag should be an end tag token, not the one nested inside the comment-like wrapper"
3535 );
3536 }
3537
3538 #[test]
3539 fn script_data_end_tag_with_wrong_name_is_kept_as_literal_text() {
3540 let tokens = tokenize_switching_after_start_tag(
3541 "<script>a</scriptx>b</script>",
3542 ExternalState::ScriptData,
3543 );
3544 assert_eq!(characters_only(&tokens), "a</scriptx>b".to_owned());
3545 let end_tags: Vec<_> = tokens
3546 .iter()
3547 .filter_map(|token| match &token.kind {
3548 TokenKind::EndTag(tag) => Some(tag.name.as_str()),
3549 _ => None,
3550 })
3551 .collect();
3552 assert_eq!(end_tags, vec!["script"]);
3553 }
3554
3555 fn tokenize_in_foreign_content(input: &str) -> Vec<Token> {
3556 let mut tokenizer = Tokenizer::new(input);
3557 tokenizer.set_in_foreign_content(true);
3558 tokenizer.collect()
3559 }
3560
3561 #[test]
3562 fn cdata_section_outside_foreign_content_still_becomes_a_bogus_comment() {
3563 assert_eq!(
3568 kinds(&tokenize("<![CDATA[x]]>")),
3569 vec![
3570 &TokenKind::Comment("[CDATA[x]]".to_owned()),
3571 &TokenKind::Eof,
3572 ]
3573 );
3574 }
3575
3576 #[test]
3577 fn cdata_section_in_foreign_content_yields_character_tokens() {
3578 assert_eq!(
3579 kinds(&tokenize_in_foreign_content("<![CDATA[hi]]>")),
3580 vec![
3581 &TokenKind::Character('h'),
3582 &TokenKind::Character('i'),
3583 &TokenKind::Eof,
3584 ]
3585 );
3586 }
3587
3588 #[test]
3589 fn cdata_section_null_character_is_kept_literal_not_replaced() {
3590 assert_eq!(
3594 kinds(&tokenize_in_foreign_content("<![CDATA[\u{0}]]>")),
3595 vec![&TokenKind::Character('\u{0}'), &TokenKind::Eof]
3596 );
3597 }
3598
3599 #[test]
3600 fn cdata_section_single_bracket_not_followed_by_another_is_literal_content() {
3601 let tokens = tokenize_in_foreign_content("<![CDATA[a]b]]>");
3602 assert_eq!(
3603 kinds(&tokens),
3604 vec![
3605 &TokenKind::Character('a'),
3606 &TokenKind::Character(']'),
3607 &TokenKind::Character('b'),
3608 &TokenKind::Eof,
3609 ]
3610 );
3611 }
3612
3613 #[test]
3614 fn cdata_section_three_consecutive_brackets_then_close_keeps_only_the_first_as_content() {
3615 let tokens = tokenize_in_foreign_content("<![CDATA[]]]>");
3620 assert_eq!(
3621 tokens,
3622 vec![
3623 Token {
3624 kind: TokenKind::Character(']'),
3625 position: pos(1, 10, 9),
3626 },
3627 Token {
3628 kind: TokenKind::Eof,
3629 position: pos(1, 14, 13),
3630 },
3631 ]
3632 );
3633 }
3634
3635 #[test]
3636 fn cdata_section_four_consecutive_brackets_then_close_keeps_first_two_as_content() {
3637 let tokens = tokenize_in_foreign_content("<![CDATA[]]]]>");
3638 assert_eq!(
3639 kinds(&tokens),
3640 vec![
3641 &TokenKind::Character(']'),
3642 &TokenKind::Character(']'),
3643 &TokenKind::Eof,
3644 ]
3645 );
3646 }
3647
3648 #[test]
3649 fn eof_inside_cdata_section_emits_eof_after_any_withheld_brackets_flush() {
3650 let tokens = tokenize_in_foreign_content("<![CDATA[a]");
3651 assert_eq!(
3652 kinds(&tokens),
3653 vec![
3654 &TokenKind::Character('a'),
3655 &TokenKind::Character(']'),
3656 &TokenKind::Eof,
3657 ]
3658 );
3659 }
3660}