#![deny(unsafe_code)]
pub mod encoding;
pub mod events;
pub mod options;
mod sink;
mod stream;
pub use encoding::{decode_html_input, prescan_meta_charset, sniff_html_encoding};
pub use events::{HtmlAttribute, HtmlAttrs, HtmlAttrsIter, HtmlEvent};
pub use options::HtmlParseOptions;
pub use stream::{HtmlBytesReader, HtmlReader, HtmlSaxHandler, HtmlSaxParser};
use html5ever::driver::{parse_document, ParseOpts};
use html5ever::tendril::TendrilSink;
use html5ever::tokenizer::TokenizerOpts;
use html5ever::tree_builder::TreeBuilderOpts;
use crate::error::{ErrorDomain, ErrorLevel, Result, XmlError};
use sup_xml_tree::dom::Document;
pub fn parse_html_str(input: &str) -> Result<Document> {
parse_html_str_opts(input, &HtmlParseOptions::default())
}
pub fn parse_html_bytes(input: &[u8]) -> Result<Document> {
parse_html_bytes_opts(input, &HtmlParseOptions::default())
}
pub fn parse_html_str_opts(input: &str, opts: &HtmlParseOptions) -> Result<Document> {
let (doc, _recovered, fatal) = parse_html_str_inner(input, opts);
if let Some(fatal) = fatal {
return Err(fatal);
}
Ok(doc)
}
pub fn parse_html_bytes_opts(
input: &[u8],
opts: &HtmlParseOptions,
) -> Result<Document> {
let (doc, _recovered, fatal) = parse_html_bytes_inner(input, opts);
if let Some(fatal) = fatal {
return Err(fatal);
}
Ok(doc)
}
#[cfg(feature = "c-abi")]
#[allow(unsafe_code)]
pub unsafe fn parse_html_bytes_opts_with_dict(
input: &[u8],
opts: &HtmlParseOptions,
dict: *mut sup_xml_tree::dict::Dict,
) -> Result<Document> {
let (doc, _recovered, fatal) = unsafe { parse_html_bytes_inner_with_dict(input, opts, dict) };
if let Some(fatal) = fatal {
return Err(fatal);
}
Ok(doc)
}
#[cfg(feature = "c-abi")]
#[allow(unsafe_code)]
pub unsafe fn parse_html_bytes_opts_with_dict_arena(
input: &[u8],
opts: &HtmlParseOptions,
dict: *mut sup_xml_tree::dict::Dict,
arena: std::sync::Arc<bumpalo::Bump>,
) -> Result<Document> {
let (doc, _recovered, fatal) =
unsafe { parse_html_bytes_inner_with_dict_arena(input, opts, dict, arena) };
if let Some(fatal) = fatal {
return Err(fatal);
}
drop_implicit_empty_head(&doc, input);
drop_implicit_empty_body(&doc, input);
Ok(doc)
}
#[cfg(feature = "c-abi")]
#[allow(unsafe_code)]
pub unsafe fn parse_html_bytes_recovered_with_dict_arena(
input: &[u8],
opts: &HtmlParseOptions,
dict: *mut sup_xml_tree::dict::Dict,
arena: std::sync::Arc<bumpalo::Bump>,
) -> (Document, Vec<XmlError>) {
let (doc, mut recovered, fatal) =
unsafe { parse_html_bytes_inner_with_dict_arena(input, opts, dict, arena) };
if let Some(f) = fatal {
recovered.push(f);
}
drop_implicit_empty_head(&doc, input);
drop_implicit_empty_body(&doc, input);
(doc, recovered)
}
#[cfg(feature = "c-abi")]
fn source_has_explicit_head(source: &[u8]) -> bool {
let mut i = 0;
while i + 5 < source.len() {
if source[i] == b'<'
&& source[i + 1].eq_ignore_ascii_case(&b'h')
&& source[i + 2].eq_ignore_ascii_case(&b'e')
&& source[i + 3].eq_ignore_ascii_case(&b'a')
&& source[i + 4].eq_ignore_ascii_case(&b'd')
&& (source[i + 5] == b'>'
|| source[i + 5] == b'/'
|| source[i + 5].is_ascii_whitespace())
{
return true;
}
i += 1;
}
false
}
#[cfg(feature = "c-abi")]
fn drop_implicit_empty_head(doc: &Document, source: &[u8]) {
if source_has_explicit_head(source) {
return;
}
let root = doc.root();
if root.name() != "html" {
return;
}
if let Some(head) = root.children().find(|c| c.is_element() && c.name() == "head") {
if head.children().next().is_none() {
let prev = head.prev_sibling.get();
let next = head.next_sibling.get();
match prev {
Some(p) => p.next_sibling.set(next),
None => root.first_child.set(next),
}
match next {
Some(n) => n.prev_sibling.set(prev),
None => root.last_child.set(prev),
}
head.parent.set(None);
head.prev_sibling.set(None);
head.next_sibling.set(None);
}
}
}
#[cfg(feature = "c-abi")]
fn source_has_explicit_body(source: &[u8]) -> bool {
let mut i = 0;
while i + 5 < source.len() {
if source[i] == b'<'
&& source[i + 1].eq_ignore_ascii_case(&b'b')
&& source[i + 2].eq_ignore_ascii_case(&b'o')
&& source[i + 3].eq_ignore_ascii_case(&b'd')
&& source[i + 4].eq_ignore_ascii_case(&b'y')
&& (source[i + 5] == b'>'
|| source[i + 5] == b'/'
|| source[i + 5].is_ascii_whitespace())
{
return true;
}
i += 1;
}
false
}
#[cfg(feature = "c-abi")]
fn drop_implicit_empty_body(doc: &Document, source: &[u8]) {
if source_has_explicit_body(source) {
return;
}
let root = doc.root();
if root.name() != "html" {
return;
}
if let Some(body) = root.children().find(|c| c.is_element() && c.name() == "body") {
if body.children().next().is_none() && body.attributes().next().is_none() {
let prev = body.prev_sibling.get();
let next = body.next_sibling.get();
match prev {
Some(p) => p.next_sibling.set(next),
None => root.first_child.set(next),
}
match next {
Some(n) => n.prev_sibling.set(prev),
None => root.last_child.set(prev),
}
body.parent.set(None);
body.prev_sibling.set(None);
body.next_sibling.set(None);
}
}
}
pub fn parse_html_str_with_recovered(
input: &str,
opts: &HtmlParseOptions,
) -> (Result<Document>, Vec<XmlError>) {
let (doc, recovered, fatal) = parse_html_str_inner(input, opts);
let result = match fatal {
Some(e) => Err(e),
None => Ok(doc),
};
(result, recovered)
}
pub fn parse_html_bytes_with_recovered(
input: &[u8],
opts: &HtmlParseOptions,
) -> (Result<Document>, Vec<XmlError>) {
let (doc, recovered, fatal) = parse_html_bytes_inner(input, opts);
let result = match fatal {
Some(e) => Err(e),
None => Ok(doc),
};
(result, recovered)
}
fn parse_html_str_inner(
input: &str,
opts: &HtmlParseOptions,
) -> (Document, Vec<XmlError>, Option<XmlError>) {
if let Err(e) = crate::license_gate::ensure_licensed() {
let b = sup_xml_tree::dom::DocumentBuilder::new();
let synth = b.new_element(b.alloc_str("html"));
b.set_root(synth);
return (b.build(), vec![e.clone()], Some(e));
}
let sink = sink::BatchSinkArena::new(opts.clone());
let parser = parse_document(sink, make_h5_opts(opts));
let sink = match std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| parser.one(input))) {
Ok(s) => s,
Err(_) => {
let err = XmlError::new(
ErrorDomain::Html,
ErrorLevel::Fatal,
"internal HTML parser panic — input may be adversarial",
);
let b = sup_xml_tree::dom::DocumentBuilder::new();
let synth = b.new_element(b.alloc_str("html"));
b.set_root(synth);
return (b.build(), vec![err.clone()], Some(err));
}
};
sink::finalize_arena(sink)
}
fn parse_html_bytes_inner(
input: &[u8],
opts: &HtmlParseOptions,
) -> (Document, Vec<XmlError>, Option<XmlError>) {
let (utf8, _detected_encoding) = match encoding::decode_html_input(input, opts) {
Ok(out) => out,
Err(e) => {
let b = sup_xml_tree::dom::DocumentBuilder::new();
let synth = b.new_element(b.alloc_str("html"));
b.set_root(synth);
return (b.build(), vec![e.clone()], Some(e));
}
};
match std::str::from_utf8(&utf8) {
Ok(s) => parse_html_str_inner(s, opts),
Err(_) => {
let lossy = String::from_utf8_lossy(&utf8).into_owned();
parse_html_str_inner(&lossy, opts)
}
}
}
#[cfg(feature = "c-abi")]
#[allow(unsafe_code)]
unsafe fn parse_html_bytes_inner_with_dict(
input: &[u8],
opts: &HtmlParseOptions,
dict: *mut sup_xml_tree::dict::Dict,
) -> (Document, Vec<XmlError>, Option<XmlError>) {
if let Err(e) = crate::license_gate::ensure_licensed() {
let b = unsafe { sup_xml_tree::dom::DocumentBuilder::new_with_dict(dict) };
let synth = b.new_element(b.alloc_str("html"));
b.set_root(synth);
return (b.build(), vec![e.clone()], Some(e));
}
let (utf8, _detected_encoding) = match encoding::decode_html_input(input, opts) {
Ok(out) => out,
Err(e) => {
let b = unsafe { sup_xml_tree::dom::DocumentBuilder::new_with_dict(dict) };
let synth = b.new_element(b.alloc_str("html"));
b.set_root(synth);
return (b.build(), vec![e.clone()], Some(e));
}
};
let owned: String = match std::str::from_utf8(&utf8) {
Ok(s) => s.to_string(),
Err(_) => String::from_utf8_lossy(&utf8).into_owned(),
};
let sink = unsafe { sink::BatchSinkArena::new_with_dict(opts.clone(), dict) };
let parser = parse_document(sink, make_h5_opts(opts));
let sink = match std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| parser.one(owned.as_str()))) {
Ok(s) => s,
Err(_) => {
let err = XmlError::new(
ErrorDomain::Html,
ErrorLevel::Fatal,
"internal HTML parser panic — input may be adversarial",
);
let b = unsafe { sup_xml_tree::dom::DocumentBuilder::new_with_dict(dict) };
let synth = b.new_element(b.alloc_str("html"));
b.set_root(synth);
return (b.build(), vec![err.clone()], Some(err));
}
};
sink::finalize_arena(sink)
}
#[cfg(feature = "c-abi")]
#[allow(unsafe_code)]
unsafe fn parse_html_bytes_inner_with_dict_arena(
input: &[u8],
opts: &HtmlParseOptions,
dict: *mut sup_xml_tree::dict::Dict,
arena: std::sync::Arc<bumpalo::Bump>,
) -> (Document, Vec<XmlError>, Option<XmlError>) {
if let Err(e) = crate::license_gate::ensure_licensed() {
let b = unsafe {
sup_xml_tree::dom::DocumentBuilder::new_with_dict_and_arena(
dict, std::sync::Arc::clone(&arena),
)
};
let synth = b.new_element(b.alloc_str("html"));
b.set_root(synth);
return (b.build(), vec![e.clone()], Some(e));
}
let (utf8, _detected_encoding) = match encoding::decode_html_input(input, opts) {
Ok(out) => out,
Err(e) => {
let b = unsafe {
sup_xml_tree::dom::DocumentBuilder::new_with_dict_and_arena(
dict, std::sync::Arc::clone(&arena),
)
};
let synth = b.new_element(b.alloc_str("html"));
b.set_root(synth);
return (b.build(), vec![e.clone()], Some(e));
}
};
let owned: String = match std::str::from_utf8(&utf8) {
Ok(s) => s.to_string(),
Err(_) => String::from_utf8_lossy(&utf8).into_owned(),
};
let sink = unsafe {
sink::BatchSinkArena::new_with_dict_and_arena(opts.clone(), dict, std::sync::Arc::clone(&arena))
};
let parser = parse_document(sink, make_h5_opts(opts));
let sink = match std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| parser.one(owned.as_str()))) {
Ok(s) => s,
Err(_) => {
let err = XmlError::new(
ErrorDomain::Html,
ErrorLevel::Fatal,
"internal HTML parser panic — input may be adversarial",
);
let b = unsafe {
sup_xml_tree::dom::DocumentBuilder::new_with_dict_and_arena(dict, arena)
};
let synth = b.new_element(b.alloc_str("html"));
b.set_root(synth);
return (b.build(), vec![err.clone()], Some(err));
}
};
sink::finalize_arena(sink)
}
fn make_h5_opts(opts: &HtmlParseOptions) -> ParseOpts {
ParseOpts {
tokenizer: TokenizerOpts {
discard_bom: opts.discard_bom,
..Default::default()
},
tree_builder: TreeBuilderOpts {
scripting_enabled: opts.scripting_enabled,
iframe_srcdoc: opts.iframe_srcdoc,
..Default::default()
},
}
}
#[cfg(test)]
mod tests {
use super::*;
use sup_xml_tree::dom::NodeKind;
fn find_elem<'a>(
n: &'a sup_xml_tree::dom::Node<'a>,
name: &str,
) -> Option<&'a sup_xml_tree::dom::Node<'a>> {
if n.kind == NodeKind::Element && n.name() == name {
return Some(n);
}
for c in n.children() {
if let Some(hit) = find_elem(c, name) {
return Some(hit);
}
}
None
}
fn collect_text<'a>(n: &'a sup_xml_tree::dom::Node<'a>) -> String {
let mut s = String::new();
fn go<'a>(n: &'a sup_xml_tree::dom::Node<'a>, s: &mut String) {
match n.kind {
NodeKind::Text | NodeKind::CData => s.push_str(n.content()),
NodeKind::Element => {
for c in n.children() { go(c, s); }
}
_ => {}
}
}
go(n, &mut s);
s
}
#[test]
fn simple_document_parses() {
let html = "<html><body><p>hello</p></body></html>";
let doc = parse_html_str(html).unwrap();
assert!(doc.is_html());
assert_eq!(doc.root().name(),"html");
}
#[test]
fn fragment_gets_implicit_html_body() {
let doc = parse_html_str("<p>hi</p>").unwrap();
assert_eq!(doc.root().name(),"html");
assert!(find_elem(doc.root(), "body").is_some());
assert!(find_elem(doc.root(), "p").is_some());
}
#[test]
fn tag_soup_recovers() {
let doc = parse_html_str("<div><p>oops</div></p>").unwrap();
assert_eq!(doc.root().name(),"html");
}
#[test]
fn doctype_is_captured_in_metadata() {
let html = "<!DOCTYPE html><html><body></body></html>";
let doc = parse_html_str(html).unwrap();
let meta = doc.html_metadata.as_ref().unwrap();
let dt = meta.doctype.as_ref().unwrap();
assert_eq!(dt.name, "html");
assert!(dt.public_id.is_empty());
}
#[test]
fn quirks_mode_set_for_no_doctype() {
let doc = parse_html_str("<html><body></body></html>").unwrap();
let meta = doc.html_metadata.as_ref().unwrap();
assert!(matches!(meta.quirks_mode, sup_xml_tree::QuirksMode::Quirks));
}
#[test]
fn no_quirks_for_html5_doctype() {
let doc = parse_html_str("<!DOCTYPE html><html><body></body></html>").unwrap();
let meta = doc.html_metadata.as_ref().unwrap();
assert!(matches!(meta.quirks_mode, sup_xml_tree::QuirksMode::NoQuirks));
}
#[test]
fn entity_decoded() {
let doc = parse_html_str("<p>a & b © c</p>").unwrap();
let text = collect_text(doc.root());
assert!(text.contains('&'), "decoded ampersand: {text}");
assert!(text.contains('©'), "decoded named entity: {text}");
}
#[test]
fn case_folded_tag_names() {
let doc = parse_html_str("<HTML><BODY><P>X</P></BODY></HTML>").unwrap();
assert_eq!(doc.root().name(),"html");
assert!(find_elem(doc.root(), "body").is_some());
assert!(find_elem(doc.root(), "p").is_some());
}
#[test]
fn parse_bytes_works() {
let doc = parse_html_bytes(b"<p>hello</p>").unwrap();
assert!(doc.is_html());
assert_eq!(doc.root().name(),"html");
}
#[test]
fn strict_mode_returns_err_on_first_error() {
let opts = HtmlParseOptions {
recovery_mode: false,
..Default::default()
};
let result = parse_html_str_opts("<p>oops</div>", &opts);
assert!(result.is_err(), "strict mode should error on mismatched tags");
}
#[test]
fn recovered_errors_surface() {
let (doc, recovered) =
parse_html_str_with_recovered("<p>oops</div>", &HtmlParseOptions::default());
assert!(doc.is_ok());
assert!(!recovered.is_empty(), "should have recovered errors");
}
#[test]
fn depth_limit_enforced() {
let opts = HtmlParseOptions {
max_element_depth: 5,
..Default::default()
};
let html = "<div>".repeat(100) + &"</div>".repeat(100);
let result = parse_html_str_opts(&html, &opts);
assert!(result.is_err(), "must reject input exceeding depth limit");
}
#[test]
fn text_byte_limit_enforced() {
let opts = HtmlParseOptions {
max_text_bytes: 100,
..Default::default()
};
let html = format!("<p>{}</p>", "a".repeat(10_000));
let result = parse_html_str_opts(&html, &opts);
assert!(result.is_err(), "must reject input exceeding text-byte limit");
}
#[test]
fn parse_html_bytes_decodes_windows1252_via_meta() {
let mut bytes = b"<!DOCTYPE html><html><head><meta charset=\"windows-1252\"></head><body><p>".to_vec();
bytes.push(0x97);
bytes.extend_from_slice(b"</p></body></html>");
let doc = parse_html_bytes(&bytes).expect("must parse");
assert!(doc.is_html());
let p = find_elem(doc.root(), "p").expect("must find <p>");
let text = collect_text(p);
assert!(
text.contains('\u{2014}'),
"em dash (U+2014) should be decoded from byte 0x97; got {text:?}"
);
}
#[test]
fn parse_html_bytes_respects_encoding_override() {
let mut bytes = b"<html><body><p>".to_vec();
bytes.push(0x97);
bytes.extend_from_slice(b"</p></body></html>");
let opts = HtmlParseOptions {
encoding_override: Some("UTF-8".into()),
..Default::default()
};
let doc = parse_html_bytes_opts(&bytes, &opts).expect("must parse");
let p = find_elem(doc.root(), "p").expect("must find <p>");
let text = collect_text(p);
assert!(
text.contains('\u{FFFD}') || !text.contains('\u{2014}'),
"with UTF-8 override, byte 0x97 should not decode as em dash; got {text:?}"
);
}
#[test]
fn attributes_preserved() {
let doc = parse_html_str(r#"<html><body><a href="/x" class="nav">x</a></body></html>"#)
.unwrap();
let a = find_elem(doc.root(), "a").expect("<a>");
let attrs: Vec<(&str, &str)> = a.attributes().map(|x| (x.name(), x.value())).collect();
assert!(attrs.iter().any(|(n, v)| *n == "href" && *v == "/x"), "got: {:?}", attrs);
assert!(attrs.iter().any(|(n, v)| *n == "class" && *v == "nav"), "got: {:?}", attrs);
}
#[test]
fn comments_preserved() {
let doc = parse_html_str("<html><body><!-- hello --><p>x</p></body></html>")
.unwrap();
let body = find_elem(doc.root(), "body").expect("body");
let comment = body.children().find(|c| c.kind == NodeKind::Comment);
let c = comment.expect("comment");
assert_eq!(c.content(), " hello ");
}
#[test]
fn deeply_nested_recovers() {
let html = "<div>".repeat(50) + "x" + &"</div>".repeat(50);
let doc = parse_html_str(&html).unwrap();
let body = find_elem(doc.root(), "body").expect("body");
let mut depth = 0;
let mut cur = body;
loop {
let next = cur.children().find(|c| c.kind == NodeKind::Element && c.name() == "div");
match next {
Some(d) => { depth += 1; cur = d; }
None => break,
}
}
assert!(depth >= 50, "expected at least 50 nested divs; got {depth}");
}
use super::events::HtmlAttrs;
use super::stream::{HtmlBytesReader, HtmlReader, HtmlSaxHandler, HtmlSaxParser};
use super::HtmlEvent;
fn drain_pull(reader: &mut HtmlReader<'_>) -> String {
let mut out = String::new();
loop {
let event = reader.next().expect("pull errored unexpectedly");
match event {
HtmlEvent::StartElement { name, attributes } => {
out.push_str("S:");
out.push_str(name);
let mut attrs: Vec<_> = attributes
.iter()
.map(|a| format!("{}={}", a.name, a.value))
.collect();
attrs.sort();
if !attrs.is_empty() {
out.push('[');
out.push_str(&attrs.join(","));
out.push(']');
}
out.push('\n');
}
HtmlEvent::EndElement { name } => {
out.push_str("E:");
out.push_str(name);
out.push('\n');
}
HtmlEvent::Text(t) => {
out.push_str("T:");
out.push_str(t);
out.push('\n');
}
HtmlEvent::Comment(c) => {
out.push_str("C:");
out.push_str(c);
out.push('\n');
}
HtmlEvent::Doctype { name, .. } => {
out.push_str("D:");
out.push_str(name);
out.push('\n');
}
HtmlEvent::Eof => return out,
}
}
}
#[test]
fn streaming_basic_event_sequence() {
let mut r = HtmlReader::new("<html><body><p>hi</p></body></html>");
let trace = drain_pull(&mut r);
assert!(trace.contains("S:html"), "trace:\n{trace}");
assert!(trace.contains("S:body"), "trace:\n{trace}");
assert!(trace.contains("S:p"), "trace:\n{trace}");
assert!(trace.contains("T:hi"), "trace:\n{trace}");
assert!(trace.contains("E:p"), "trace:\n{trace}");
assert!(trace.contains("E:body"), "trace:\n{trace}");
assert!(trace.contains("E:html"), "trace:\n{trace}");
}
#[test]
fn streaming_text_coalescing() {
let mut r = HtmlReader::new("<p>hello world & goodbye</p>");
let trace = drain_pull(&mut r);
let text_lines: Vec<&str> = trace.lines().filter(|l| l.starts_with("T:")).collect();
assert!(
text_lines.iter().any(|l| l.contains("hello world & goodbye")),
"expected coalesced text; got lines: {:?}",
text_lines
);
}
#[test]
fn streaming_doctype_event() {
let mut r = HtmlReader::new("<!DOCTYPE html><html><body>x</body></html>");
let trace = drain_pull(&mut r);
assert!(trace.contains("D:html"));
}
#[test]
fn streaming_attributes_visible() {
let mut r = HtmlReader::new(r#"<a href="/x" class="nav">x</a>"#);
let trace = drain_pull(&mut r);
assert!(
trace.lines().any(|l| l.starts_with("S:a[") && l.contains("href=/x") && l.contains("class=nav")),
"expected start tag with attrs; got:\n{}",
trace
);
}
#[test]
fn streaming_chunk_boundary_invariance() {
let html = "<html><head><title>t</title></head><body><p>a</p><p>b & c</p></body></html>";
let mut a = HtmlReader::new(html);
let mut b = HtmlBytesReader::new(html.as_bytes()).expect("bytes reader init");
let trace_a = drain_pull(&mut a);
let trace_b = drain_pull_bytes(&mut b);
assert_eq!(trace_a, trace_b, "str and bytes readers must agree");
}
fn drain_pull_bytes(reader: &mut HtmlBytesReader<'_>) -> String {
let mut out = String::new();
loop {
let event = reader.next().expect("bytes pull errored");
match event {
HtmlEvent::StartElement { name, attributes } => {
out.push_str("S:");
out.push_str(name);
let mut attrs: Vec<_> = attributes
.iter()
.map(|a| format!("{}={}", a.name, a.value))
.collect();
attrs.sort();
if !attrs.is_empty() {
out.push('[');
out.push_str(&attrs.join(","));
out.push(']');
}
out.push('\n');
}
HtmlEvent::EndElement { name } => {
out.push_str("E:");
out.push_str(name);
out.push('\n');
}
HtmlEvent::Text(t) => {
out.push_str("T:");
out.push_str(t);
out.push('\n');
}
HtmlEvent::Comment(c) => {
out.push_str("C:");
out.push_str(c);
out.push('\n');
}
HtmlEvent::Doctype { name, .. } => {
out.push_str("D:");
out.push_str(name);
out.push('\n');
}
HtmlEvent::Eof => return out,
}
}
}
#[derive(Default)]
struct LinkCollector {
hrefs: Vec<String>,
}
impl HtmlSaxHandler for LinkCollector {
fn start_element(&mut self, name: &str, attrs: HtmlAttrs<'_>) {
if name == "a" {
if let Some(href) = attrs.get("href") {
self.hrefs.push(href.to_string());
}
}
}
}
#[test]
fn sax_parser_collects_attributes() {
let html = r#"<a href="/x">x</a><a href="/y">y</a><a>none</a>"#;
let mut p = HtmlSaxParser::new(LinkCollector::default());
p.feed(html).unwrap();
let collector = p.finish().unwrap();
assert_eq!(collector.hrefs, vec!["/x", "/y"]);
}
#[test]
fn sax_parser_chunked_feed() {
let html = r#"<a href="/x">x</a>"#;
let mut p = HtmlSaxParser::new(LinkCollector::default());
for ch in html.chars() {
let mut tmp = [0u8; 4];
let s = ch.encode_utf8(&mut tmp);
p.feed(s).unwrap();
}
let collector = p.finish().unwrap();
assert_eq!(collector.hrefs, vec!["/x"]);
}
#[derive(Default)]
struct TraceHandler {
out: String,
}
impl HtmlSaxHandler for TraceHandler {
fn start_element(&mut self, name: &str, attrs: HtmlAttrs<'_>) {
self.out.push_str("S:");
self.out.push_str(name);
let mut a: Vec<_> = attrs
.iter()
.map(|a| format!("{}={}", a.name, a.value))
.collect();
a.sort();
if !a.is_empty() {
self.out.push('[');
self.out.push_str(&a.join(","));
self.out.push(']');
}
self.out.push('\n');
}
fn end_element(&mut self, name: &str) {
self.out.push_str("E:");
self.out.push_str(name);
self.out.push('\n');
}
fn text(&mut self, content: &str) {
self.out.push_str("T:");
self.out.push_str(content);
self.out.push('\n');
}
fn comment(&mut self, content: &str) {
self.out.push_str("C:");
self.out.push_str(content);
self.out.push('\n');
}
fn doctype(&mut self, name: &str, _public_id: &str, _system_id: &str) {
self.out.push_str("D:");
self.out.push_str(name);
self.out.push('\n');
}
}
#[test]
fn pull_push_parity() {
let html = "<!DOCTYPE html><html><body><p>foo & bar</p><!-- comment --><a href=\"/x\">link</a></body></html>";
let mut pull = HtmlReader::new(html);
let pull_trace = drain_pull(&mut pull);
let mut push = HtmlSaxParser::new(TraceHandler::default());
push.feed(html).unwrap();
let push_trace = push.finish().unwrap().out;
assert_eq!(
pull_trace, push_trace,
"pull and push surfaces must produce identical event traces"
);
}
#[test]
fn streaming_recovered_errors_collected() {
let mut r = HtmlReader::new("<p>oops</div>");
let _ = drain_pull(&mut r);
assert!(
!r.recovered_errors().is_empty(),
"should have recovered at least one parse error"
);
}
#[test]
fn streaming_bytes_reader_decodes_windows1252() {
let mut bytes = b"<!DOCTYPE html><html><head><meta charset=\"windows-1252\"></head><body><p>".to_vec();
bytes.push(0x97);
bytes.extend_from_slice(b"</p></body></html>");
let mut r = HtmlBytesReader::new(&bytes).expect("init");
let mut found_em_dash = false;
loop {
match r.next().expect("pull") {
HtmlEvent::Eof => break,
HtmlEvent::Text(t) if t.contains('\u{2014}') => found_em_dash = true,
_ => {}
}
}
assert!(
found_em_dash,
"streaming bytes reader should decode windows-1252 em dash"
);
}
#[test]
fn streaming_strict_mode_errors() {
let opts = HtmlParseOptions {
recovery_mode: false,
..Default::default()
};
let mut r = HtmlReader::with_opts("<p>oops</div>", opts);
let mut hit_err = false;
loop {
match r.next() {
Ok(HtmlEvent::Eof) => break,
Ok(_) => {}
Err(_) => {
hit_err = true;
break;
}
}
}
assert!(hit_err, "strict mode should surface a parse error");
}
#[test]
fn comments_before_and_after_html_element() {
let html = "<!-- pre --><html><body>x</body></html><!-- post -->";
let doc = parse_html_str(html).unwrap();
assert_eq!(doc.root().name(), "html");
}
#[test]
fn processing_instruction_in_html_becomes_comment() {
let doc = parse_html_str("<?xml-stylesheet href='x'?><html><body>x</body></html>")
.unwrap();
assert_eq!(doc.root().name(), "html");
}
#[test]
fn max_text_bytes_exceeded_aborts_parse() {
let opts = HtmlParseOptions {
max_text_bytes: 8,
..Default::default()
};
let (result, _recovered) = parse_html_str_with_recovered(
"<p>this is more than eight bytes of text</p>", &opts,
);
assert!(result.is_err());
if let Err(e) = result {
assert!(e.message.contains("max_text_bytes"), "got {}", e.message);
}
}
#[test]
fn max_element_depth_exceeded_aborts_parse() {
let opts = HtmlParseOptions {
max_element_depth: 4,
..Default::default()
};
let html = "<div>".repeat(10) + "x" + &"</div>".repeat(10);
let (result, _) = parse_html_str_with_recovered(&html, &opts);
assert!(result.is_err());
if let Err(e) = result {
assert!(e.message.contains("max_element_depth"), "got {}", e.message);
}
}
#[test]
fn table_foster_parenting_with_text() {
let html = "<table>raw text<tr><td>cell</td></tr></table>";
let doc = parse_html_str(html).unwrap();
assert_eq!(doc.root().name(), "html");
let body = find_elem(doc.root(), "body").expect("body");
let all_text = collect_text(body);
assert!(all_text.contains("raw text"), "got: {all_text:?}");
assert!(all_text.contains("cell"), "got: {all_text:?}");
}
#[test]
fn duplicate_html_attributes_use_add_attrs_if_missing() {
let html = r#"<html lang="en"><body></body></html><html dir="ltr">"#;
let doc = parse_html_str(html).unwrap();
let root = doc.root();
let attrs: Vec<(&str, &str)> = root.attributes()
.map(|a| (a.name(), a.value())).collect();
assert!(attrs.iter().any(|(n, _)| *n == "lang"), "got {attrs:?}");
assert!(attrs.iter().any(|(n, _)| *n == "dir"), "got {attrs:?}");
}
#[test]
fn duplicate_html_attribute_does_not_overwrite() {
let html = r#"<html lang="en"><body></body></html><html lang="fr">"#;
let doc = parse_html_str(html).unwrap();
let lang = doc.root().attributes()
.find(|a| a.name() == "lang").map(|a| a.value().to_owned());
assert_eq!(lang.as_deref(), Some("en"), "first lang should win");
}
#[test]
fn limited_quirks_doctype_sets_metadata() {
let html = r#"<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
<html><body></body></html>"#;
let doc = parse_html_str(html).unwrap();
let meta = doc.html_metadata.as_ref().expect("metadata");
assert!(!matches!(meta.quirks_mode, sup_xml_tree::QuirksMode::Quirks),
"transitional DOCTYPE should not trigger full Quirks mode");
}
#[test]
fn empty_input_synthesizes_html() {
let doc = parse_html_str("").unwrap();
assert_eq!(doc.root().name(), "html");
}
#[test]
fn long_run_of_text_chunks_coalesces() {
let body = "x".repeat(10_000);
let html = format!("<html><body><p>{body}</p></body></html>");
let doc = parse_html_str(&html).unwrap();
let p = find_elem(doc.root(), "p").expect("p");
let text = collect_text(p);
assert_eq!(text.len(), 10_000);
}
#[test]
fn parse_error_is_recorded_in_strict_mode() {
let opts = HtmlParseOptions {
recovery_mode: false,
..Default::default()
};
let (result, _) = parse_html_str_with_recovered("<p>x</_invalid>", &opts);
assert!(result.is_err());
}
#[test]
fn parse_errors_in_recovery_mode_recorded_not_fatal() {
let opts = HtmlParseOptions { recovery_mode: true, ..Default::default() };
let (result, recovered) = parse_html_str_with_recovered(
"<p>x</_invalid>", &opts,
);
assert!(result.is_ok());
assert!(!recovered.is_empty() || result.unwrap().root().name() == "html");
}
#[test]
fn quirks_mode_with_old_html_doctype() {
let doc = parse_html_str("<!DOCTYPE html><html><body></body></html>").unwrap();
let meta = doc.html_metadata.as_ref().unwrap();
assert!(matches!(meta.quirks_mode, sup_xml_tree::QuirksMode::NoQuirks));
}
#[test]
fn doctype_with_public_and_system_ids() {
let html = r#"<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01//EN" "http://www.w3.org/TR/html4/strict.dtd"><html><body></body></html>"#;
let doc = parse_html_str(html).unwrap();
let meta = doc.html_metadata.as_ref().unwrap();
let dt = meta.doctype.as_ref().unwrap();
assert_eq!(dt.name, "html");
assert!(dt.public_id.contains("HTML 4.01"));
assert!(dt.system_id.contains("html4"));
}
#[test]
fn template_element_parses() {
let html = "<html><body><template><p>tmpl</p></template></body></html>";
let doc = parse_html_str(html).unwrap();
assert!(find_elem(doc.root(), "template").is_some());
}
#[test]
fn table_foster_parenting_runs_of_text() {
let html = "<table>first<tr><td>cell</td></tr>second</table>";
let doc = parse_html_str(html).unwrap();
let body = find_elem(doc.root(), "body").expect("body");
let all = collect_text(body);
assert!(all.contains("first") && all.contains("second"),
"got: {all:?}");
}
#[test]
fn nested_table_foster_parenting() {
let html = r#"<table><tr><td>a</td></tr>extra<tr><td>b</td></tr></table>"#;
let doc = parse_html_str(html).unwrap();
let body = find_elem(doc.root(), "body").expect("body");
let text = collect_text(body);
assert!(text.contains("extra"));
assert!(text.contains("a") && text.contains("b"));
}
#[test]
fn svg_with_processing_instruction() {
let html = r#"<html><body><svg xmlns="http://www.w3.org/2000/svg"><circle/></svg></body></html>"#;
let doc = parse_html_str(html).unwrap();
assert!(find_elem(doc.root(), "svg").is_some());
assert!(find_elem(doc.root(), "circle").is_some());
}
#[test]
fn text_after_html_close_appended() {
let html = "<html><body>x</body></html>extra-text";
let doc = parse_html_str(html).unwrap();
let body = find_elem(doc.root(), "body").expect("body");
let text = collect_text(body);
assert!(text.contains("x"));
}
#[test]
fn list_with_misnested_tags() {
let html = "<ul><li>one<li>two<li>three</ul>";
let doc = parse_html_str(html).unwrap();
let body = find_elem(doc.root(), "body").expect("body");
let li_count = body.children().filter_map(|c| {
find_elem(c, "li")
}).count();
assert!(li_count > 0);
}
#[test]
fn text_byte_limit_hit_during_merge() {
let opts = HtmlParseOptions {
max_text_bytes: 5,
..Default::default()
};
let html = "<html><body><script>aaa</script><script>bbbbbbbbbb</script></body></html>";
let (result, _) = parse_html_str_with_recovered(html, &opts);
assert!(result.is_err());
if let Err(e) = result {
assert!(e.message.contains("max_text_bytes"), "got {}", e.message);
}
}
#[test]
fn debug_format_on_owned_elem_name_via_strict_error() {
let html = "<html><body><div><span><a><b><i></i></b></a></span></div></body></html>";
let doc = parse_html_str(html).unwrap();
assert_eq!(doc.root().name(), "html");
}
}