rusty_xml_parser/
html.rs

1//! HTML parser matching libxml2 `HTMLparser.c` (a separate grammar, not XML recovery).
2
3use rusty_xml_tree::{NodeId, NodeKind, XmlDoc};
4
5use crate::error::XmlError;
6use crate::parse::default_parse_options;
7
8/// libxml2 `htmlParserOption` bits we honour.
9pub const HTML_PARSE_NOIMPLIED: i32 = 1 << 13;
10pub const HTML_PARSE_NONET: i32 = 1 << 11;
11
12const VOID: &[&str] = &[
13    "area", "base", "br", "col", "embed", "hr", "img", "input", "link", "meta", "param",
14    "source", "track", "wbr",
15];
16
17fn is_void(name: &str) -> bool {
18    VOID.contains(&name)
19}
20
21/// `htmlReadMemory`.
22#[doc(alias = "htmlReadMemory")]
23pub fn html_read_memory(
24    buffer: &[u8],
25    url: Option<&str>,
26    encoding: Option<&str>,
27    options: i32,
28) -> Result<XmlDoc, XmlError> {
29    let (utf8, _) = crate::encoding::xml_convert_to_utf8(buffer, encoding)?;
30    html_parse_utf8(&utf8, url, options)
31}
32
33/// `htmlReadDoc`.
34#[doc(alias = "htmlReadDoc")]
35pub fn html_read_doc(
36    cur: &str,
37    url: Option<&str>,
38    encoding: Option<&str>,
39    options: i32,
40) -> Result<XmlDoc, XmlError> {
41    html_read_memory(cur.as_bytes(), url, encoding, options)
42}
43
44/// `htmlReadFile`.
45#[doc(alias = "htmlReadFile")]
46pub fn html_read_file(filename: &str, encoding: Option<&str>, options: i32) -> Result<XmlDoc, XmlError> {
47    let b = std::fs::read(filename).map_err(|e| XmlError::new(4, e.to_string(), 0, 0))?;
48    html_read_memory(&b, Some(filename), encoding, options)
49}
50
51fn html_parse_utf8(bytes: &[u8], _url: Option<&str>, options: i32) -> Result<XmlDoc, XmlError> {
52    let text = String::from_utf8_lossy(bytes);
53    let mut p = HtmlParser {
54        src: text.as_ref(),
55        pos: 0,
56        doc: XmlDoc::xml_new_doc(Some("1.0")),
57        stack: Vec::new(),
58        noimplied: (options & HTML_PARSE_NOIMPLIED) != 0,
59        html: None,
60        head: None,
61        body: None,
62    };
63    p.doc.encoding = Some("HTML".into());
64    // Mark it as an HTML document so the serializer can tell. Without this the
65    // writer had no way to know, and emitted an XML declaration where C emits
66    // the doctype -- which, re-parsed as HTML, became a text node. An HTML
67    // round trip was not stable.
68    p.doc.node_mut(rusty_xml_tree::NodeId::DOCUMENT).kind = NodeKind::HtmlDocument;
69    p.parse()?;
70    while p.stack.len() > 1 {
71        p.stack.pop();
72    }
73    let _ = options | HTML_PARSE_NONET | default_parse_options();
74    Ok(p.doc)
75}
76
77struct HtmlParser<'a> {
78    src: &'a str,
79    pos: usize,
80    doc: XmlDoc,
81    stack: Vec<NodeId>,
82    noimplied: bool,
83    html: Option<NodeId>,
84    head: Option<NodeId>,
85    body: Option<NodeId>,
86}
87
88impl<'a> HtmlParser<'a> {
89    fn rest(&self) -> &'a str {
90        &self.src[self.pos..]
91    }
92    fn eof(&self) -> bool {
93        self.pos >= self.src.len()
94    }
95    fn bump(&mut self, n: usize) {
96        self.pos += n;
97    }
98    fn parent(&self) -> NodeId {
99        *self.stack.last().unwrap_or(&NodeId::DOCUMENT)
100    }
101    fn ensure_html(&mut self) -> NodeId {
102        if let Some(h) = self.html {
103            return h;
104        }
105        let html = self.doc.xml_new_node(None, "html");
106        self.doc.xml_doc_set_root_element(html);
107        self.html = Some(html);
108        html
109    }
110    fn ensure_head(&mut self) -> NodeId {
111        if let Some(h) = self.head {
112            return h;
113        }
114        let html = self.ensure_html();
115        let head = self.doc.xml_new_node(None, "head");
116        self.doc.xml_add_child(html, head);
117        self.head = Some(head);
118        head
119    }
120    fn ensure_body(&mut self) -> NodeId {
121        if let Some(b) = self.body {
122            return b;
123        }
124        let html = self.ensure_html();
125        let body = self.doc.xml_new_node(None, "body");
126        self.doc.xml_add_child(html, body);
127        self.body = Some(body);
128        body
129    }
130    fn ensure_html_body(&mut self) -> NodeId {
131        if self.noimplied {
132            return self.stack.last().copied().unwrap_or(NodeId::DOCUMENT);
133        }
134        self.ensure_body()
135    }
136    fn parse(&mut self) -> Result<(), XmlError> {
137        while !self.eof() {
138            if self.rest().starts_with("<!--") {
139                self.parse_comment()?;
140            } else if self.rest().starts_with("<!") {
141                self.skip_decl();
142            } else if self.rest().starts_with("</") {
143                self.parse_end_tag();
144            } else if self.rest().starts_with('<') {
145                self.parse_start_tag()?;
146            } else {
147                self.parse_text();
148            }
149        }
150        Ok(())
151    }
152    fn parse_comment(&mut self) -> Result<(), XmlError> {
153        self.bump(4);
154        if let Some(end) = self.rest().find("-->") {
155            let body = self.rest()[..end].to_string();
156            self.bump(end + 3);
157            let n = self.doc.alloc(NodeKind::Comment, "#comment");
158            self.doc.node_mut(n).content = body;
159            self.doc.xml_add_child(self.parent(), n);
160        } else {
161            self.pos = self.src.len();
162        }
163        Ok(())
164    }
165    /// Markup declaration. Only DOCTYPE carries anything we keep.
166    ///
167    /// This used to discard the lot, so an HTML document's doctype was lost and
168    /// the serializer had nothing to write. C round-trips it: `<!DOCTYPE html>`
169    /// in, `<!DOCTYPE html>` out.
170    fn skip_decl(&mut self) {
171        let decl_is_doctype = self
172            .rest()
173            .get(2..9)
174            .is_some_and(|k| k.eq_ignore_ascii_case("DOCTYPE"));
175        let end = self.rest().find('>');
176        let body = match end {
177            Some(i) => self.rest()[..i].to_string(),
178            None => self.rest().to_string(),
179        };
180        match end {
181            Some(i) => self.bump(i + 1),
182            None => self.pos = self.src.len(),
183        }
184        if decl_is_doctype && self.doc.dtd.is_none() {
185            if let Some(tail) = body.get(9..) {
186                self.doc.dtd = Some(parse_html_doctype(tail));
187            }
188        }
189    }
190    fn parse_text(&mut self) {
191        let mut i = 0;
192        let r = self.rest();
193        for (off, c) in r.char_indices() {
194            if c == '<' {
195                i = off;
196                break;
197            }
198            i = off + c.len_utf8();
199        }
200        if i == 0 {
201            return;
202        }
203        // Entities were never decoded here, so `caf&eacute;` reached the tree
204        // verbatim and came back out as `caf&amp;eacute;`.
205        let t = crate::html_entities::decode_html_text(&r[..i]).into_owned();
206        self.bump(i);
207        if t.chars().all(|c| c.is_whitespace()) && self.stack.is_empty() {
208            return;
209        }
210        let n = self.doc.alloc(NodeKind::Text, "#text");
211        self.doc.node_mut(n).content = t;
212        let parent = if self.stack.is_empty() {
213            self.ensure_html_body()
214        } else {
215            self.parent()
216        };
217        self.doc.xml_add_child(parent, n);
218    }
219    fn parse_start_tag(&mut self) -> Result<(), XmlError> {
220        self.bump(1);
221        let name = self.read_name().to_ascii_lowercase();
222        if name.is_empty() {
223            return Ok(());
224        }
225        let mut attrs: Vec<(String, String)> = Vec::new();
226        loop {
227            self.skip_ws();
228            if self.rest().starts_with('>') {
229                self.bump(1);
230                break;
231            }
232            if self.rest().starts_with("/>") {
233                self.bump(2);
234                break;
235            }
236            if self.eof() {
237                break;
238            }
239            let an = self.read_name().to_ascii_lowercase();
240            if an.is_empty() {
241                // Skip one CHARACTER, not one byte. A multi-byte character that
242                // cannot start an attribute name left self.pos mid-scalar, and
243                // the next rest() slice panicked. `<r` + U+0777 + `/>` sufficed.
244                let step = self.rest().chars().next().map_or(1, char::len_utf8);
245                self.bump(step);
246                continue;
247            }
248            self.skip_ws();
249            let av = if self.rest().starts_with('=') {
250                self.bump(1);
251                self.skip_ws();
252                self.read_attr_value()
253            } else {
254                an.clone()
255            };
256            attrs.push((an, av));
257        }
258        // autoclose p/li when another p/li starts
259        if name == "p" || name == "li" || name == "tr" || name == "td" || name == "th" {
260            while let Some(&top) = self.stack.last() {
261                if self.doc.name(top) == name {
262                    self.stack.pop();
263                } else {
264                    break;
265                }
266            }
267        }
268        // A block element also closes an open <p>: a paragraph cannot contain
269        // one. Only same-name autoclose was implemented, so <p>two<div>three
270        // nested the div INSIDE the paragraph where C makes them siblings --
271        // which puts the text at the wrong depth for anything walking the tree
272        // for structure.
273        if is_block_element(&name) {
274            while self.stack.last().is_some_and(|&t| self.doc.name(t) == "p") {
275                self.stack.pop();
276            }
277        }
278        let parent = if self.noimplied {
279            self.stack.last().copied().unwrap_or(NodeId::DOCUMENT)
280        } else if name == "html" {
281            NodeId::DOCUMENT
282        } else if name == "head" {
283            self.ensure_html()
284        } else if name == "body" || name == "frameset" {
285            self.ensure_html()
286        } else if matches!(name.as_str(), "title" | "meta" | "link" | "style" | "base") {
287            self.ensure_head()
288        } else if !self.stack.is_empty() {
289            // The open element, not the body. This branch used to return
290            // ensure_body() unconditionally, so NOTHING nested: fifty nested
291            // <div> came out as fifty empty siblings, and structure-aware
292            // consumers saw a flat document.
293            self.parent()
294        } else {
295            self.ensure_body()
296        };
297        let elem = self.doc.xml_new_node(None, &name);
298        for (k, v) in attrs {
299            self.doc.xml_set_prop(elem, &k, &v);
300        }
301        if name == "html" {
302            self.doc.xml_doc_set_root_element(elem);
303            self.html = Some(elem);
304        } else {
305            self.doc.xml_add_child(parent, elem);
306        }
307        if name == "head" {
308            self.head = Some(elem);
309        }
310        if name == "body" || name == "frameset" {
311            self.body = Some(elem);
312        }
313        if !is_void(&name) {
314            self.stack.push(elem);
315        }
316        Ok(())
317    }
318    fn parse_end_tag(&mut self) {
319        self.bump(2);
320        let name = self.read_name().to_ascii_lowercase();
321        self.skip_ws();
322        if self.rest().starts_with('>') {
323            self.bump(1);
324        }
325        if let Some(idx) = self.stack.iter().rposition(|&id| self.doc.name(id) == name) {
326            self.stack.truncate(idx);
327        }
328    }
329    fn skip_ws(&mut self) {
330        while let Some(c) = self.rest().chars().next() {
331            if c.is_whitespace() {
332                self.bump(c.len_utf8());
333            } else {
334                break;
335            }
336        }
337    }
338    fn read_name(&mut self) -> String {
339        let r = self.rest();
340        let mut n = 0;
341        for (i, c) in r.char_indices() {
342            if i == 0 {
343                if !(c.is_ascii_alphabetic() || c == '_' || c == ':') {
344                    return String::new();
345                }
346            } else if !(c.is_ascii_alphanumeric() || c == '-' || c == '_' || c == ':' || c == '.') {
347                n = i;
348                break;
349            }
350            n = i + c.len_utf8();
351        }
352        let s = r[..n].to_string();
353        self.bump(n);
354        s
355    }
356    fn read_attr_value(&mut self) -> String {
357        let r = self.rest();
358        if r.starts_with('"') || r.starts_with('\'') {
359            let q = r.as_bytes()[0] as char;
360            self.bump(1);
361            if let Some(end) = self.rest().find(q) {
362                let v = crate::html_entities::decode_html_text(&self.rest()[..end]).into_owned();
363                self.bump(end + 1);
364                return v;
365            }
366        }
367        let mut n = 0;
368        for (i, c) in self.rest().char_indices() {
369            if c.is_whitespace() || c == '>' {
370                n = i;
371                break;
372            }
373            n = i + c.len_utf8();
374        }
375        let v = crate::html_entities::decode_html_text(&self.rest()[..n]).into_owned();
376        self.bump(n);
377        v
378    }
379}
380
381/// Split the body of an HTML `<!DOCTYPE ...>` into name, public id, system id.
382///
383/// `<!DOCTYPE html>` and the HTML 4.01 form with PUBLIC/SYSTEM identifiers are
384/// the two that occur; anything else degrades to a bare name, which is what C
385/// does too.
386fn parse_html_doctype(body: &str) -> rusty_xml_tree::XmlDtd {
387    let mut dtd = rusty_xml_tree::XmlDtd::default();
388    let mut rest = body.trim_start();
389    let name_end = rest.find(char::is_whitespace).unwrap_or(rest.len());
390    if name_end > 0 {
391        dtd.name = Some(rest[..name_end].to_string());
392    }
393    rest = rest[name_end..].trim_start();
394
395    // A quoted literal, either quoting style, as HTML permits both.
396    fn literal(r: &mut &str) -> Option<String> {
397        *r = r.trim_start();
398        let q = r.chars().next().filter(|c| *c == '"' || *c == '\'')?;
399        let after = &r[1..];
400        let end = after.find(q)?;
401        let v = after[..end].to_string();
402        *r = &after[end + 1..];
403        Some(v)
404    }
405
406    // .get(..6) rather than [..6]: a byte index that lands mid-character
407    // panics, and a doctype is attacker-controlled text like anything else.
408    if rest.get(..6).is_some_and(|k| k.eq_ignore_ascii_case("PUBLIC")) {
409        rest = &rest[6..];
410        dtd.public_id = literal(&mut rest);
411        dtd.system_id = literal(&mut rest);
412    } else if rest.get(..6).is_some_and(|k| k.eq_ignore_ascii_case("SYSTEM")) {
413        rest = &rest[6..];
414        dtd.system_id = literal(&mut rest);
415    }
416    dtd
417}
418
419/// Block-level elements, which cannot appear inside a paragraph and therefore
420/// close an open one. This is libxml2's `htmlStartClose` table for `p`.
421fn is_block_element(name: &str) -> bool {
422    matches!(
423        name,
424        "address"
425            | "article"
426            | "aside"
427            | "blockquote"
428            | "center"
429            | "details"
430            | "dialog"
431            | "dir"
432            | "div"
433            | "dl"
434            | "fieldset"
435            | "figcaption"
436            | "figure"
437            | "footer"
438            | "form"
439            | "h1"
440            | "h2"
441            | "h3"
442            | "h4"
443            | "h5"
444            | "h6"
445            | "header"
446            | "hgroup"
447            | "hr"
448            | "main"
449            | "menu"
450            | "nav"
451            | "ol"
452            | "p"
453            | "pre"
454            | "section"
455            | "table"
456            | "ul"
457    )
458}
rusty_xml_parser/html.rs

rusty_xml_parser/
html.rs
