rama_http/protocols/html/tokenizer/
context.rs

1//! Tree-builder *simulation*: just enough open-element context to tokenize
2//! correctly without building a DOM.
3//!
4//! Two things depend on context that a flat tokenizer otherwise can't know:
5//!
6//!   * **Text mode** — `<script>`, `<style>`, `<textarea>`, … switch their
7//!     body to a raw-text parsing mode, but only in the HTML namespace.
8//!   * **CDATA** — `<![CDATA[ … ]]>` is real character data inside foreign
9//!     content (SVG/MathML) and a bogus comment everywhere else.
10//!
11//! HTML is context-sensitive, so this is modelled with a small namespace
12//! stack plus the integration-point / breakout rules. A few non-conforming
13//! constructs (text-mode tags inside `<select>` / after `<frameset>`) make
14//! the context genuinely ambiguous for a streaming parser; following
15//! lol-html, those bail out with a [`ParsingAmbiguityError`] in strict mode
16//! rather than risk mis-tokenizing (an XSS-gadget hazard).
17//!
18//! Adapted from lol-html's `tree_builder_simulator` (BSD-3-Clause).
19
20use std::fmt;
21
22use super::name::LocalNameHash;
23use super::token::AttrRange;
24
25// --- known tag-name hashes -------------------------------------------------
26
27const SVG: LocalNameHash = LocalNameHash::from_static(b"svg");
28const MATH: LocalNameHash = LocalNameHash::from_static(b"math");
29const FONT: LocalNameHash = LocalNameHash::from_static(b"font");
30const P: LocalNameHash = LocalNameHash::from_static(b"p");
31const BR: LocalNameHash = LocalNameHash::from_static(b"br");
32
33const SCRIPT: LocalNameHash = LocalNameHash::from_static(b"script");
34const STYLE: LocalNameHash = LocalNameHash::from_static(b"style");
35const TEXTAREA: LocalNameHash = LocalNameHash::from_static(b"textarea");
36const TITLE: LocalNameHash = LocalNameHash::from_static(b"title");
37const PLAINTEXT: LocalNameHash = LocalNameHash::from_static(b"plaintext");
38const XMP: LocalNameHash = LocalNameHash::from_static(b"xmp");
39const IFRAME: LocalNameHash = LocalNameHash::from_static(b"iframe");
40const NOEMBED: LocalNameHash = LocalNameHash::from_static(b"noembed");
41const NOFRAMES: LocalNameHash = LocalNameHash::from_static(b"noframes");
42const NOSCRIPT: LocalNameHash = LocalNameHash::from_static(b"noscript");
43
44// SVG HTML-integration points.
45const DESC: LocalNameHash = LocalNameHash::from_static(b"desc");
46const FOREIGN_OBJECT: LocalNameHash = LocalNameHash::from_static(b"foreignobject");
47
48// MathML text-integration points.
49const MI: LocalNameHash = LocalNameHash::from_static(b"mi");
50const MO: LocalNameHash = LocalNameHash::from_static(b"mo");
51const MN: LocalNameHash = LocalNameHash::from_static(b"mn");
52const MS: LocalNameHash = LocalNameHash::from_static(b"ms");
53const MTEXT: LocalNameHash = LocalNameHash::from_static(b"mtext");
54
55// Ambiguity-guard insertion-mode markers.
56const SELECT: LocalNameHash = LocalNameHash::from_static(b"select");
57const FRAMESET: LocalNameHash = LocalNameHash::from_static(b"frameset");
58const TEMPLATE: LocalNameHash = LocalNameHash::from_static(b"template");
59const INPUT: LocalNameHash = LocalNameHash::from_static(b"input");
60const KEYGEN: LocalNameHash = LocalNameHash::from_static(b"keygen");
61
62/// Text-parsing-mode switching tags — the ones whose context-sensitivity is
63/// dangerous to misjudge.
64const TEXT_SWITCH_TAGS: &[LocalNameHash] = &[
65    TEXTAREA, TITLE, PLAINTEXT, SCRIPT, STYLE, IFRAME, XMP, NOEMBED, NOFRAMES, NOSCRIPT,
66];
67
68/// HTML elements that force an exit from foreign content (per the HTML
69/// "any other start tag" foreign-content breakout list).
70const BREAKOUT_TAGS: &[LocalNameHash] = &[
71    LocalNameHash::from_static(b"b"),
72    LocalNameHash::from_static(b"big"),
73    LocalNameHash::from_static(b"blockquote"),
74    LocalNameHash::from_static(b"body"),
75    LocalNameHash::from_static(b"br"),
76    LocalNameHash::from_static(b"center"),
77    LocalNameHash::from_static(b"code"),
78    LocalNameHash::from_static(b"dd"),
79    LocalNameHash::from_static(b"div"),
80    LocalNameHash::from_static(b"dl"),
81    LocalNameHash::from_static(b"dt"),
82    LocalNameHash::from_static(b"em"),
83    LocalNameHash::from_static(b"embed"),
84    LocalNameHash::from_static(b"h1"),
85    LocalNameHash::from_static(b"h2"),
86    LocalNameHash::from_static(b"h3"),
87    LocalNameHash::from_static(b"h4"),
88    LocalNameHash::from_static(b"h5"),
89    LocalNameHash::from_static(b"h6"),
90    LocalNameHash::from_static(b"head"),
91    LocalNameHash::from_static(b"hr"),
92    LocalNameHash::from_static(b"i"),
93    LocalNameHash::from_static(b"img"),
94    LocalNameHash::from_static(b"li"),
95    LocalNameHash::from_static(b"listing"),
96    LocalNameHash::from_static(b"menu"),
97    LocalNameHash::from_static(b"meta"),
98    LocalNameHash::from_static(b"nobr"),
99    LocalNameHash::from_static(b"ol"),
100    LocalNameHash::from_static(b"p"),
101    LocalNameHash::from_static(b"pre"),
102    LocalNameHash::from_static(b"ruby"),
103    LocalNameHash::from_static(b"s"),
104    LocalNameHash::from_static(b"small"),
105    LocalNameHash::from_static(b"span"),
106    LocalNameHash::from_static(b"strong"),
107    LocalNameHash::from_static(b"strike"),
108    LocalNameHash::from_static(b"sub"),
109    LocalNameHash::from_static(b"sup"),
110    LocalNameHash::from_static(b"table"),
111    LocalNameHash::from_static(b"tt"),
112    LocalNameHash::from_static(b"u"),
113    LocalNameHash::from_static(b"ul"),
114    LocalNameHash::from_static(b"var"),
115];
116
117/// How an element's body is tokenized once its start tag is seen.
118#[derive(Debug, Clone, Copy, PartialEq, Eq)]
119pub(crate) enum TextMode {
120    /// `<style>`, `<xmp>`, `<iframe>`, `<noembed>`, `<noframes>`,
121    /// `<noscript>` — raw text until the matching end tag.
122    RawText,
123    /// `<textarea>`, `<title>` — like raw text (bytes are kept raw, so the
124    /// only practical difference from `RawText` is the element name).
125    RcData,
126    /// `<script>` — raw text with the HTML script-data escape rules.
127    ScriptData,
128    /// `<plaintext>` — everything to end of input is text.
129    PlainText,
130}
131
132/// HTML element (in the HTML namespace) whose body is a special text mode.
133fn html_text_mode(name: LocalNameHash) -> Option<TextMode> {
134    if name == SCRIPT {
135        Some(TextMode::ScriptData)
136    } else if name == TEXTAREA || name == TITLE {
137        Some(TextMode::RcData)
138    } else if name == PLAINTEXT {
139        Some(TextMode::PlainText)
140    } else if name == STYLE
141        || name == XMP
142        || name == IFRAME
143        || name == NOEMBED
144        || name == NOFRAMES
145        || name == NOSCRIPT
146    {
147        Some(TextMode::RawText)
148    } else {
149        None
150    }
151}
152
153/// The namespace an element's content is parsed in.
154#[derive(Debug, Clone, Copy, PartialEq, Eq)]
155enum Namespace {
156    Html,
157    Svg,
158    MathMl,
159}
160
161/// Raised when text-mode context can't be determined in strict mode.
162#[derive(Debug, Clone, PartialEq, Eq)]
163pub struct ParsingAmbiguityError {
164    tag_name: Box<str>,
165}
166
167impl fmt::Display for ParsingAmbiguityError {
168    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
169        write!(
170            f,
171            "ambiguous parsing context at `<{}>`: cannot tell whether following \
172             content is raw text or markup (usually caused by non-conforming HTML)",
173            self.tag_name
174        )
175    }
176}
177
178impl std::error::Error for ParsingAmbiguityError {}
179
180/// Tracks the open-element context needed for correct tokenization.
181#[derive(Debug)]
182pub(crate) struct ContextTracker {
183    ns_stack: Vec<Namespace>,
184    current_ns: Namespace,
185    ambiguity: AmbiguityGuard,
186    strict: bool,
187}
188
189impl ContextTracker {
190    pub(crate) fn new(strict: bool) -> Self {
191        Self {
192            ns_stack: vec![Namespace::Html],
193            current_ns: Namespace::Html,
194            ambiguity: AmbiguityGuard::default(),
195            strict,
196        }
197    }
198
199    /// Whether `<![CDATA[ … ]]>` is currently real character data.
200    pub(crate) fn cdata_allowed(&self) -> bool {
201        self.current_ns != Namespace::Html
202    }
203
204    /// Non-mutating peek at the text mode a start tag would enter in the
205    /// current context (used by the streaming tokenizer to decide whether a
206    /// raw-text body is fully buffered before committing to it). Mirrors the
207    /// text-mode outcome of [`Self::on_start_tag`].
208    pub(crate) fn peek_text_mode(&self, name: LocalNameHash) -> Option<TextMode> {
209        if self.current_ns != Namespace::Html || name == SVG || name == MATH {
210            None
211        } else {
212            html_text_mode(name)
213        }
214    }
215
216    /// Updates context for a start tag, returning the body's text mode (only
217    /// possible in the HTML namespace).
218    pub(crate) fn on_start_tag(
219        &mut self,
220        name_hash: LocalNameHash,
221        name: &[u8],
222        attributes: &[AttrRange],
223        input: &[u8],
224        self_closing: bool,
225    ) -> Result<Option<TextMode>, ParsingAmbiguityError> {
226        if self.strict {
227            self.ambiguity.track_start_tag(name_hash, name)?;
228        }
229
230        if name_hash == SVG {
231            self.enter_ns(Namespace::Svg);
232            Ok(None)
233        } else if name_hash == MATH {
234            self.enter_ns(Namespace::MathMl);
235            Ok(None)
236        } else if self.current_ns == Namespace::Html {
237            Ok(html_text_mode(name_hash))
238        } else {
239            self.foreign_start_tag(name_hash, name, attributes, input, self_closing);
240            Ok(None)
241        }
242    }
243
244    /// Updates context for an end tag.
245    pub(crate) fn on_end_tag(&mut self, name_hash: LocalNameHash, name: &[u8]) {
246        if self.strict {
247            self.ambiguity.track_end_tag(name_hash);
248        }
249        if self.current_ns == Namespace::Html {
250            self.check_integration_point_exit(name_hash, name);
251        } else if self.should_leave_ns(name_hash) {
252            self.leave_ns();
253        }
254    }
255
256    fn enter_ns(&mut self, ns: Namespace) {
257        self.ns_stack.push(ns);
258        self.current_ns = ns;
259    }
260
261    fn leave_ns(&mut self) {
262        self.ns_stack.pop();
263        self.current_ns = self.ns_stack.last().copied().unwrap_or(Namespace::Html);
264    }
265
266    fn should_leave_ns(&self, name: LocalNameHash) -> bool {
267        match self.current_ns {
268            Namespace::Svg => name == SVG || name == P || name == BR,
269            Namespace::MathMl => name == MATH || name == P || name == BR,
270            Namespace::Html => false,
271        }
272    }
273
274    fn foreign_start_tag(
275        &mut self,
276        name_hash: LocalNameHash,
277        name: &[u8],
278        attributes: &[AttrRange],
279        input: &[u8],
280        self_closing: bool,
281    ) {
282        if BREAKOUT_TAGS.contains(&name_hash) {
283            self.leave_ns();
284        } else if self.is_integration_point_enter(name_hash) {
285            if !self_closing {
286                self.enter_ns(Namespace::Html);
287            }
288        } else if name_hash == FONT {
289            if attr_named_any(attributes, input, &[b"color", b"size", b"face"]) {
290                self.leave_ns();
291            }
292        } else if self.current_ns == Namespace::MathMl
293            && !self_closing
294            && name.eq_ignore_ascii_case(b"annotation-xml")
295            && annotation_xml_is_html(attributes, input)
296        {
297            self.enter_ns(Namespace::Html);
298        }
299    }
300
301    fn is_integration_point_enter(&self, name: LocalNameHash) -> bool {
302        match self.current_ns {
303            Namespace::Svg => is_svg_html_integration_point(name),
304            Namespace::MathMl => is_mathml_text_integration_point(name),
305            Namespace::Html => false,
306        }
307    }
308
309    fn check_integration_point_exit(&mut self, name_hash: LocalNameHash, name: &[u8]) {
310        if self.ns_stack.len() < 2 {
311            return;
312        }
313        let Some(&prev_ns) = self.ns_stack.get(self.ns_stack.len() - 2) else {
314            return;
315        };
316        let exits = match prev_ns {
317            Namespace::Svg => is_svg_html_integration_point(name_hash),
318            Namespace::MathMl => {
319                is_mathml_text_integration_point(name_hash)
320                    || name.eq_ignore_ascii_case(b"annotation-xml")
321            }
322            Namespace::Html => false,
323        };
324        if exits {
325            self.leave_ns();
326        }
327    }
328}
329
330fn is_svg_html_integration_point(name: LocalNameHash) -> bool {
331    name == FOREIGN_OBJECT || name == DESC || name == TITLE
332}
333
334fn is_mathml_text_integration_point(name: LocalNameHash) -> bool {
335    name == MI || name == MO || name == MN || name == MS || name == MTEXT
336}
337
338/// Whether any attribute's name (ASCII case-insensitive) is in `names`.
339fn attr_named_any(attributes: &[AttrRange], input: &[u8], names: &[&[u8]]) -> bool {
340    attributes.iter().any(|attr| {
341        let attr_name = input.get(attr.name.start..attr.name.end).unwrap_or(&[]);
342        names.iter().any(|n| attr_name.eq_ignore_ascii_case(n))
343    })
344}
345
346/// Whether `<annotation-xml>` carries `encoding=text/html` (or the XHTML
347/// equivalent), which makes its content an HTML integration point.
348fn annotation_xml_is_html(attributes: &[AttrRange], input: &[u8]) -> bool {
349    attributes.iter().any(|attr| {
350        let attr_name = input.get(attr.name.start..attr.name.end).unwrap_or(&[]);
351        if !attr_name.eq_ignore_ascii_case(b"encoding") {
352            return false;
353        }
354        let value = input.get(attr.value.start..attr.value.end).unwrap_or(&[]);
355        value.eq_ignore_ascii_case(b"text/html")
356            || value.eq_ignore_ascii_case(b"application/xhtml+xml")
357    })
358}
359
360/// Guards against tokenizing a text-mode element in an insertion mode where
361/// the tree builder would ignore it (`<select>`, in/after `<frameset>`).
362#[derive(Debug, Clone, Copy)]
363enum AmbiguityState {
364    Default,
365    InSelect,
366    InTemplateInSelect(u32),
367    InOrAfterFrameset,
368}
369
370#[derive(Debug)]
371struct AmbiguityGuard {
372    state: AmbiguityState,
373}
374
375impl Default for AmbiguityGuard {
376    fn default() -> Self {
377        Self {
378            state: AmbiguityState::Default,
379        }
380    }
381}
382
383impl AmbiguityGuard {
384    fn track_start_tag(
385        &mut self,
386        name_hash: LocalNameHash,
387        name: &[u8],
388    ) -> Result<(), ParsingAmbiguityError> {
389        match self.state {
390            AmbiguityState::Default => {
391                if name_hash == SELECT {
392                    self.state = AmbiguityState::InSelect;
393                } else if name_hash == FRAMESET {
394                    self.state = AmbiguityState::InOrAfterFrameset;
395                }
396            }
397            AmbiguityState::InSelect => {
398                // These prematurely exit the "in select" insertion mode.
399                if name_hash == SELECT
400                    || name_hash == TEXTAREA
401                    || name_hash == INPUT
402                    || name_hash == KEYGEN
403                {
404                    self.state = AmbiguityState::Default;
405                } else if name_hash == TEMPLATE {
406                    self.state = AmbiguityState::InTemplateInSelect(1);
407                } else if name_hash != SCRIPT {
408                    // `<script>` is allowed in "in select".
409                    assert_not_ambiguous(name_hash, name)?;
410                }
411            }
412            AmbiguityState::InTemplateInSelect(depth) => {
413                if name_hash == TEMPLATE {
414                    self.state = AmbiguityState::InTemplateInSelect(depth + 1);
415                } else {
416                    assert_not_ambiguous(name_hash, name)?;
417                }
418            }
419            AmbiguityState::InOrAfterFrameset => {
420                // `<noframes>` is allowed in and after `<frameset>`.
421                if name_hash != NOFRAMES {
422                    assert_not_ambiguous(name_hash, name)?;
423                }
424            }
425        }
426        Ok(())
427    }
428
429    fn track_end_tag(&mut self, name_hash: LocalNameHash) {
430        match self.state {
431            AmbiguityState::InSelect if name_hash == SELECT => {
432                self.state = AmbiguityState::Default;
433            }
434            AmbiguityState::InTemplateInSelect(depth) if name_hash == TEMPLATE => {
435                self.state = if depth == 1 {
436                    AmbiguityState::InSelect
437                } else {
438                    AmbiguityState::InTemplateInSelect(depth - 1)
439                };
440            }
441            _ => {}
442        }
443    }
444}
445
446fn assert_not_ambiguous(
447    name_hash: LocalNameHash,
448    name: &[u8],
449) -> Result<(), ParsingAmbiguityError> {
450    if TEXT_SWITCH_TAGS.contains(&name_hash) {
451        Err(ParsingAmbiguityError {
452            tag_name: String::from_utf8_lossy(name)
453                .to_ascii_lowercase()
454                .into_boxed_str(),
455        })
456    } else {
457        Ok(())
458    }
459}
rama_http/protocols/html/tokenizer/context.rs

rama_http/protocols/html/tokenizer/
context.rs
