browser_engine/html/
tokenizer.rs

1// Batch 1 — HTML Tokenizer
2//
3// A simplified but spec-inspired state machine, following the shape of the WHATWG
4// HTML5 tokenization states (https://html.spec.whatwg.org/multipage/parsing.html).
5// We don't implement every one of the ~80 spec states yet, but the ones here cover
6// real-world HTML: tags, attributes, comments, doctype, and text.
7//
8// SECURITY NOTE: the tokenizer must never panic on malformed input. Every state
9// transition has a defined fallback so garbage/attacker-controlled HTML degrades
10// gracefully into text/error-recovery instead of crashing the process.
11
12use super::token::Token;
13
14#[derive(Debug, Clone, Copy, PartialEq)]
15enum State {
16    Data,
17    TagOpen,
18    EndTagOpen,
19    TagName,
20    BeforeAttributeName,
21    AttributeName,
22    AfterAttributeName,
23    BeforeAttributeValue,
24    AttributeValueDoubleQuoted,
25    AttributeValueSingleQuoted,
26    AttributeValueUnquoted,
27    AfterAttributeValueQuoted,
28    SelfClosingStartTag,
29    MarkupDeclarationOpen,
30    CommentStart,
31    Comment,
32    CommentEnd,
33    Doctype,
34    DoctypeName,
35    BogusComment,
36}
37
38pub struct Tokenizer {
39    input: Vec<char>,
40    pos: usize,
41    state: State,
42    /// tokens produced so far, drained by `next_token`
43    emitted: Vec<Token>,
44
45    // scratch buffers for the token currently being built
46    current_tag_name: String,
47    current_tag_is_end: bool,
48    current_self_closing: bool,
49    current_attrs: Vec<(String, String)>,
50    current_attr_name: String,
51    current_attr_value: String,
52    current_comment: String,
53    current_doctype_name: String,
54    text_buffer: String,
55}
56
57impl Tokenizer {
58    pub fn new(input: &str) -> Self {
59        Tokenizer {
60            input: input.chars().collect(),
61            pos: 0,
62            state: State::Data,
63            emitted: Vec::new(),
64            current_tag_name: String::new(),
65            current_tag_is_end: false,
66            current_self_closing: false,
67            current_attrs: Vec::new(),
68            current_attr_name: String::new(),
69            current_attr_value: String::new(),
70            current_comment: String::new(),
71            current_doctype_name: String::new(),
72            text_buffer: String::new(),
73        }
74    }
75
76    /// Run the whole input through the state machine and return all tokens.
77    /// Malformed input never panics: unexpected states fall back to treating
78    /// characters as text (see `_ =>` arms below).
79    pub fn tokenize(mut self) -> Vec<Token> {
80        while self.pos < self.input.len() {
81            self.step();
82        }
83        self.flush_text();
84        self.emitted.push(Token::Eof);
85        self.emitted
86    }
87
88    fn peek(&self) -> Option<char> {
89        self.input.get(self.pos).copied()
90    }
91
92    fn advance(&mut self) -> Option<char> {
93        let c = self.peek();
94        if c.is_some() {
95            self.pos += 1;
96        }
97        c
98    }
99
100    fn flush_text(&mut self) {
101        if !self.text_buffer.is_empty() {
102            let text = std::mem::take(&mut self.text_buffer);
103            self.emitted.push(Token::Text(text));
104        }
105    }
106
107    fn reset_tag_scratch(&mut self) {
108        self.current_tag_name.clear();
109        self.current_tag_is_end = false;
110        self.current_self_closing = false;
111        self.current_attrs.clear();
112    }
113
114    fn emit_tag(&mut self) {
115        if self.current_tag_is_end {
116            self.emitted.push(Token::EndTag {
117                name: self.current_tag_name.clone(),
118            });
119        } else {
120            self.emitted.push(Token::StartTag {
121                name: self.current_tag_name.clone(),
122                attributes: self.current_attrs.clone(),
123                self_closing: self.current_self_closing,
124            });
125        }
126        self.reset_tag_scratch();
127    }
128
129    fn finish_current_attr(&mut self) {
130        if !self.current_attr_name.is_empty() {
131            self.current_attrs.push((
132                std::mem::take(&mut self.current_attr_name),
133                std::mem::take(&mut self.current_attr_value),
134            ));
135        } else {
136            self.current_attr_name.clear();
137            self.current_attr_value.clear();
138        }
139    }
140
141    fn step(&mut self) {
142        match self.state {
143            State::Data => self.step_data(),
144            State::TagOpen => self.step_tag_open(),
145            State::EndTagOpen => self.step_end_tag_open(),
146            State::TagName => self.step_tag_name(),
147            State::BeforeAttributeName => self.step_before_attribute_name(),
148            State::AttributeName => self.step_attribute_name(),
149            State::AfterAttributeName => self.step_after_attribute_name(),
150            State::BeforeAttributeValue => self.step_before_attribute_value(),
151            State::AttributeValueDoubleQuoted => self.step_attr_value_quoted('"'),
152            State::AttributeValueSingleQuoted => self.step_attr_value_quoted('\''),
153            State::AttributeValueUnquoted => self.step_attr_value_unquoted(),
154            State::AfterAttributeValueQuoted => self.step_after_attribute_value_quoted(),
155            State::SelfClosingStartTag => self.step_self_closing_start_tag(),
156            State::MarkupDeclarationOpen => self.step_markup_declaration_open(),
157            State::CommentStart => self.step_comment_start(),
158            State::Comment => self.step_comment(),
159            State::CommentEnd => self.step_comment_end(),
160            State::Doctype => self.step_doctype(),
161            State::DoctypeName => self.step_doctype_name(),
162            State::BogusComment => self.step_bogus_comment(),
163        }
164    }
165
166    fn step_data(&mut self) {
167        match self.advance() {
168            Some('<') => {
169                self.flush_text();
170                self.state = State::TagOpen;
171            }
172            Some(c) => self.text_buffer.push(c),
173            None => {}
174        }
175    }
176
177    fn step_tag_open(&mut self) {
178        match self.peek() {
179            Some('/') => {
180                self.pos += 1;
181                self.state = State::EndTagOpen;
182            }
183            Some('!') => {
184                self.pos += 1;
185                self.state = State::MarkupDeclarationOpen;
186            }
187            Some(c) if c.is_ascii_alphabetic() => {
188                self.reset_tag_scratch();
189                self.current_tag_is_end = false;
190                self.state = State::TagName;
191            }
192            // Malformed: '<' not followed by anything sensible -> treat as text
193            _ => {
194                self.text_buffer.push('<');
195                self.state = State::Data;
196            }
197        }
198    }
199
200    fn step_end_tag_open(&mut self) {
201        match self.peek() {
202            Some(c) if c.is_ascii_alphabetic() => {
203                self.reset_tag_scratch();
204                self.current_tag_is_end = true;
205                self.state = State::TagName;
206            }
207            Some('>') => {
208                // "</>" — bogus, just skip it per spec's error-recovery spirit
209                self.pos += 1;
210                self.state = State::Data;
211            }
212            _ => {
213                // malformed end tag; recover into bogus comment rather than crash
214                self.state = State::BogusComment;
215            }
216        }
217    }
218
219    fn step_tag_name(&mut self) {
220        match self.advance() {
221            Some(c) if c.is_whitespace() => self.state = State::BeforeAttributeName,
222            Some('/') => self.state = State::SelfClosingStartTag,
223            Some('>') => {
224                self.emit_tag();
225                self.state = State::Data;
226            }
227            Some(c) => self.current_tag_name.push(c.to_ascii_lowercase()),
228            None => {} // EOF mid-tag: just stop, don't emit malformed tag
229        }
230    }
231
232    fn step_before_attribute_name(&mut self) {
233        match self.peek() {
234            Some(c) if c.is_whitespace() => {
235                self.pos += 1;
236            }
237            Some('/') => {
238                self.pos += 1;
239                self.state = State::SelfClosingStartTag;
240            }
241            Some('>') => {
242                self.pos += 1;
243                self.emit_tag();
244                self.state = State::Data;
245            }
246            Some(_) => {
247                self.current_attr_name.clear();
248                self.current_attr_value.clear();
249                self.state = State::AttributeName;
250            }
251            None => {}
252        }
253    }
254
255    fn step_attribute_name(&mut self) {
256        match self.peek() {
257            Some(c) if c.is_whitespace() => {
258                self.pos += 1;
259                self.state = State::AfterAttributeName;
260            }
261            Some('=') => {
262                self.pos += 1;
263                self.state = State::BeforeAttributeValue;
264            }
265            Some('/') => {
266                self.finish_current_attr();
267                self.pos += 1;
268                self.state = State::SelfClosingStartTag;
269            }
270            Some('>') => {
271                self.finish_current_attr();
272                self.pos += 1;
273                self.emit_tag();
274                self.state = State::Data;
275            }
276            Some(c) => {
277                self.current_attr_name.push(c.to_ascii_lowercase());
278                self.pos += 1;
279            }
280            None => {}
281        }
282    }
283
284    fn step_after_attribute_name(&mut self) {
285        match self.peek() {
286            Some(c) if c.is_whitespace() => {
287                self.pos += 1;
288            }
289            Some('=') => {
290                self.pos += 1;
291                self.state = State::BeforeAttributeValue;
292            }
293            Some('/') => {
294                self.finish_current_attr();
295                self.pos += 1;
296                self.state = State::SelfClosingStartTag;
297            }
298            Some('>') => {
299                self.finish_current_attr();
300                self.pos += 1;
301                self.emit_tag();
302                self.state = State::Data;
303            }
304            Some(_) => {
305                self.finish_current_attr();
306                self.state = State::AttributeName;
307            }
308            None => {}
309        }
310    }
311
312    fn step_before_attribute_value(&mut self) {
313        match self.peek() {
314            Some(c) if c.is_whitespace() => {
315                self.pos += 1;
316            }
317            Some('"') => {
318                self.pos += 1;
319                self.state = State::AttributeValueDoubleQuoted;
320            }
321            Some('\'') => {
322                self.pos += 1;
323                self.state = State::AttributeValueSingleQuoted;
324            }
325            Some('>') => {
326                // "attr=>" malformed; recover by finishing attr with empty value
327                self.finish_current_attr();
328                self.pos += 1;
329                self.emit_tag();
330                self.state = State::Data;
331            }
332            Some(_) => {
333                self.state = State::AttributeValueUnquoted;
334            }
335            None => {}
336        }
337    }
338
339    fn step_attr_value_quoted(&mut self, quote: char) {
340        match self.advance() {
341            Some(c) if c == quote => {
342                self.finish_current_attr();
343                self.state = State::AfterAttributeValueQuoted;
344            }
345            Some(c) => self.current_attr_value.push(c),
346            None => {} // unterminated quoted value at EOF — just stop
347        }
348    }
349
350    fn step_attr_value_unquoted(&mut self) {
351        match self.peek() {
352            Some(c) if c.is_whitespace() => {
353                self.pos += 1;
354                self.finish_current_attr();
355                self.state = State::BeforeAttributeName;
356            }
357            Some('>') => {
358                self.finish_current_attr();
359                self.pos += 1;
360                self.emit_tag();
361                self.state = State::Data;
362            }
363            Some(c) => {
364                self.current_attr_value.push(c);
365                self.pos += 1;
366            }
367            None => {}
368        }
369    }
370
371    fn step_after_attribute_value_quoted(&mut self) {
372        match self.peek() {
373            Some(c) if c.is_whitespace() => {
374                self.pos += 1;
375                self.state = State::BeforeAttributeName;
376            }
377            Some('/') => {
378                self.pos += 1;
379                self.state = State::SelfClosingStartTag;
380            }
381            Some('>') => {
382                self.pos += 1;
383                self.emit_tag();
384                self.state = State::Data;
385            }
386            _ => {
387                // malformed spacing; recover by going back to attribute hunting
388                self.state = State::BeforeAttributeName;
389            }
390        }
391    }
392
393    fn step_self_closing_start_tag(&mut self) {
394        match self.peek() {
395            Some('>') => {
396                self.pos += 1;
397                self.current_self_closing = true;
398                self.emit_tag();
399                self.state = State::Data;
400            }
401            _ => {
402                // malformed "/" not followed by ">" — recover into attribute hunt
403                self.state = State::BeforeAttributeName;
404            }
405        }
406    }
407
408    fn step_markup_declaration_open(&mut self) {
409        let rest: String = self.input[self.pos..].iter().collect();
410        if rest.starts_with("--") {
411            self.pos += 2;
412            self.current_comment.clear();
413            self.state = State::CommentStart;
414        } else if rest.to_ascii_uppercase().starts_with("DOCTYPE") {
415            self.pos += "DOCTYPE".len();
416            self.current_doctype_name.clear();
417            self.state = State::Doctype;
418        } else {
419            // unknown declaration (e.g. CDATA outside foreign content) -> bogus comment
420            self.current_comment.clear();
421            self.state = State::BogusComment;
422        }
423    }
424
425    fn step_comment_start(&mut self) {
426        self.state = State::Comment;
427    }
428
429    fn step_comment(&mut self) {
430        let rest: String = self.input[self.pos..].iter().collect();
431        if rest.starts_with("-->") {
432            self.pos += 3;
433            self.emitted
434                .push(Token::Comment(std::mem::take(&mut self.current_comment)));
435            self.state = State::Data;
436        } else {
437            match self.advance() {
438                Some(c) => self.current_comment.push(c),
439                None => {
440                    // unterminated comment at EOF: emit what we have
441                    self.emitted
442                        .push(Token::Comment(std::mem::take(&mut self.current_comment)));
443                }
444            }
445        }
446    }
447
448    fn step_comment_end(&mut self) {
449        self.state = State::Data;
450    }
451
452    fn step_doctype(&mut self) {
453        match self.peek() {
454            Some(c) if c.is_whitespace() => self.pos += 1,
455            Some('>') => {
456                self.pos += 1;
457                self.emitted.push(Token::Doctype { name: None });
458                self.state = State::Data;
459            }
460            Some(_) => self.state = State::DoctypeName,
461            None => {}
462        }
463    }
464
465    fn step_doctype_name(&mut self) {
466        match self.peek() {
467            Some('>') => {
468                self.pos += 1;
469                let name = if self.current_doctype_name.is_empty() {
470                    None
471                } else {
472                    Some(std::mem::take(&mut self.current_doctype_name))
473                };
474                self.emitted.push(Token::Doctype { name });
475                self.state = State::Data;
476            }
477            Some(c) => {
478                self.current_doctype_name.push(c);
479                self.pos += 1;
480            }
481            None => {}
482        }
483    }
484
485    fn step_bogus_comment(&mut self) {
486        match self.advance() {
487            Some('>') => {
488                self.emitted
489                    .push(Token::Comment(std::mem::take(&mut self.current_comment)));
490                self.state = State::Data;
491            }
492            Some(c) => self.current_comment.push(c),
493            None => {
494                self.emitted
495                    .push(Token::Comment(std::mem::take(&mut self.current_comment)));
496            }
497        }
498    }
499}
browser_engine/html/tokenizer.rs

browser_engine/html/
tokenizer.rs
