pub mod arena;
#[cfg(feature = "async-tokio")]
pub mod async_parser;
pub mod builder;
pub mod node;
pub mod streaming;
pub mod traverse;
use fhp_core::tag::Tag;
use arena::{Arena, Attribute};
use builder::TreeBuilder;
use node::{NodeFlags, NodeId};
use traverse::{Ancestors, BreadthFirst, Children, DepthFirst, Siblings};
#[derive(Debug, thiserror::Error)]
pub enum HtmlError {
#[error("input too large: {size} bytes (max {max})")]
InputTooLarge {
size: usize,
max: usize,
},
#[error("encoding error: {0}")]
Encoding(#[from] fhp_core::error::EncodingError),
#[error("I/O error: {0}")]
Io(#[from] std::io::Error),
}
pub(crate) const MAX_INPUT_SIZE: usize = 256 * 1024 * 1024;
pub fn parse(input: &str) -> Result<Document, HtmlError> {
if input.len() > MAX_INPUT_SIZE {
return Err(HtmlError::InputTooLarge {
size: input.len(),
max: MAX_INPUT_SIZE,
});
}
let mut builder = TreeBuilder::with_capacity_hint(input.len());
builder.set_source(input);
fhp_tokenizer::tokenize_into(input, &mut builder);
let (arena, root) = builder.finish();
Ok(Document { arena, root })
}
pub fn parse_owned(input: String) -> Result<Document, HtmlError> {
if input.len() > MAX_INPUT_SIZE {
return Err(HtmlError::InputTooLarge {
size: input.len(),
max: MAX_INPUT_SIZE,
});
}
let mut builder = TreeBuilder::with_capacity_hint(input.len());
builder.set_source_ptr(&input);
fhp_tokenizer::tokenize_into(&input, &mut builder);
let (mut arena, root) = builder.finish();
arena.set_source_owned(input);
Ok(Document { arena, root })
}
pub fn parse_bytes(input: &[u8]) -> Result<Document, HtmlError> {
if input.len() > MAX_INPUT_SIZE {
return Err(HtmlError::InputTooLarge {
size: input.len(),
max: MAX_INPUT_SIZE,
});
}
let (text, _encoding) = fhp_encoding::decode_or_detect(input)?;
parse(&text)
}
pub struct Document {
arena: Arena,
root: NodeId,
}
impl Document {
pub fn root(&self) -> NodeRef<'_> {
NodeRef {
arena: &self.arena,
id: self.root,
}
}
pub fn get(&self, id: NodeId) -> NodeRef<'_> {
NodeRef {
arena: &self.arena,
id,
}
}
pub fn arena(&self) -> &Arena {
&self.arena
}
pub fn to_html(&self) -> String {
self.root().outer_html()
}
pub fn node_count(&self) -> usize {
self.arena.len()
}
pub fn root_id(&self) -> NodeId {
self.root
}
}
#[derive(Clone, Copy)]
pub struct NodeRef<'a> {
arena: &'a Arena,
id: NodeId,
}
impl<'a> NodeRef<'a> {
pub fn id(&self) -> NodeId {
self.id
}
pub fn tag(&self) -> Tag {
self.arena.get(self.id).tag
}
pub fn depth(&self) -> u16 {
self.arena.get(self.id).depth
}
pub fn is_text(&self) -> bool {
self.arena.get(self.id).flags.has(NodeFlags::IS_TEXT)
}
pub fn is_comment(&self) -> bool {
self.arena.get(self.id).flags.has(NodeFlags::IS_COMMENT)
}
pub fn is_doctype(&self) -> bool {
self.arena.get(self.id).flags.has(NodeFlags::IS_DOCTYPE)
}
pub fn is_void(&self) -> bool {
self.arena.get(self.id).flags.has(NodeFlags::IS_VOID)
}
pub fn has_children(&self) -> bool {
!self.arena.get(self.id).first_child.is_null()
}
pub fn text(&self) -> &'a str {
let node = self.arena.get(self.id);
if node.flags.has(NodeFlags::IS_TEXT)
|| node.flags.has(NodeFlags::IS_COMMENT)
|| node.flags.has(NodeFlags::IS_DOCTYPE)
{
self.arena.text(self.id)
} else {
""
}
}
pub fn text_content(&self) -> String {
let node = self.arena.get(self.id);
if node.flags.has(NodeFlags::IS_TEXT) {
return self.arena.text(self.id).to_string();
}
let hint = (self.arena.text_slab.len() / 4).min(4096);
let mut result = String::with_capacity(hint);
self.collect_text(&mut result);
result
}
fn collect_text(&self, out: &mut String) {
let node = self.arena.get(self.id);
if node.flags.has(NodeFlags::IS_TEXT) {
out.push_str(self.arena.text(self.id));
return;
}
let mut child = node.first_child;
while !child.is_null() {
NodeRef {
arena: self.arena,
id: child,
}
.collect_text(out);
child = self.arena.get(child).next_sibling;
}
}
pub fn inner_html(&self) -> String {
let mut result = String::new();
let node = self.arena.get(self.id);
let mut child = node.first_child;
while !child.is_null() {
NodeRef {
arena: self.arena,
id: child,
}
.write_outer_html(&mut result);
child = self.arena.get(child).next_sibling;
}
result
}
pub fn outer_html(&self) -> String {
let mut result = String::new();
self.write_outer_html(&mut result);
result
}
fn write_outer_html(&self, out: &mut String) {
let node = self.arena.get(self.id);
if node.flags.has(NodeFlags::IS_TEXT) {
let text = self.arena.text(self.id);
let parent_id = node.parent;
let is_raw_text = !parent_id.is_null() && self.arena.get(parent_id).tag.is_raw_text();
if is_raw_text {
out.push_str(text);
} else {
fhp_core::entity::escape_text(text, out);
}
return;
}
if node.flags.has(NodeFlags::IS_COMMENT) {
out.push_str("<!--");
out.push_str(self.arena.text(self.id));
out.push_str("-->");
return;
}
if node.flags.has(NodeFlags::IS_DOCTYPE) {
out.push_str("<!DOCTYPE ");
out.push_str(self.arena.text(self.id));
out.push('>');
return;
}
let tag_name = node
.tag
.as_str()
.or_else(|| self.arena.unknown_tag_name(self.id));
let is_root_wrapper = node.depth == 0 && node.parent.is_null();
if !is_root_wrapper {
if let Some(name) = tag_name {
out.push('<');
out.push_str(name);
let attrs = self.arena.attrs(self.id);
for attr in attrs {
out.push(' ');
out.push_str(self.arena.attr_name(attr));
if let Some(val) = self.arena.attr_value(attr) {
out.push_str("=\"");
fhp_core::entity::escape_attr(val, out);
out.push('"');
}
}
if node.flags.has(NodeFlags::IS_VOID) {
out.push('>');
return;
}
out.push('>');
}
}
let mut child = node.first_child;
while !child.is_null() {
NodeRef {
arena: self.arena,
id: child,
}
.write_outer_html(out);
child = self.arena.get(child).next_sibling;
}
if !is_root_wrapper {
if let Some(name) = tag_name {
out.push_str("</");
out.push_str(name);
out.push('>');
}
}
}
pub fn attr(&self, name: &str) -> Option<&'a str> {
self.arena
.attrs(self.id)
.iter()
.find(|a| self.arena.attr_name(a).eq_ignore_ascii_case(name))
.and_then(|a| self.arena.attr_value(a))
}
pub fn has_class(&self, class_name: &str) -> bool {
if let Some(classes) = self.attr("class") {
classes.split_whitespace().any(|c| c == class_name)
} else {
false
}
}
pub fn attrs(&self) -> &'a [Attribute] {
self.arena.attrs(self.id)
}
pub fn children(&self) -> Children<'a> {
Children::new(self.arena, self.id)
}
pub fn parent(&self) -> Option<NodeRef<'a>> {
let parent = self.arena.get(self.id).parent;
if parent.is_null() {
None
} else {
Some(NodeRef {
arena: self.arena,
id: parent,
})
}
}
pub fn first_child(&self) -> Option<NodeRef<'a>> {
let fc = self.arena.get(self.id).first_child;
if fc.is_null() {
None
} else {
Some(NodeRef {
arena: self.arena,
id: fc,
})
}
}
pub fn next_sibling(&self) -> Option<NodeRef<'a>> {
let ns = self.arena.get(self.id).next_sibling;
if ns.is_null() {
None
} else {
Some(NodeRef {
arena: self.arena,
id: ns,
})
}
}
pub fn prev_sibling(&self) -> Option<NodeRef<'a>> {
let ps = self.arena.get(self.id).prev_sibling;
if ps.is_null() {
None
} else {
Some(NodeRef {
arena: self.arena,
id: ps,
})
}
}
pub fn ancestors(&self) -> Ancestors<'a> {
Ancestors::new(self.arena, self.id)
}
pub fn siblings(&self) -> Siblings<'a> {
Siblings::new(self.arena, self.id)
}
pub fn descendants(&self) -> DepthFirst<'a> {
DepthFirst::new(self.arena, self.id)
}
pub fn descendants_bfs(&self) -> BreadthFirst<'a> {
BreadthFirst::new(self.arena, self.id)
}
}
impl<'a> core::fmt::Debug for NodeRef<'a> {
fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
let node = self.arena.get(self.id);
if node.flags.has(NodeFlags::IS_TEXT) {
write!(f, "Text({:?})", self.text())
} else if node.flags.has(NodeFlags::IS_COMMENT) {
write!(f, "Comment({:?})", self.text())
} else {
write!(f, "<{}>", node.tag)
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn parse_simple() {
let doc = parse("<div><p>Hello</p></div>").unwrap();
assert!(doc.node_count() > 0);
let root = doc.root();
assert!(root.has_children());
}
#[test]
fn parse_text_content() {
let doc = parse("<div><span>Hello</span> <span>World</span></div>").unwrap();
let root = doc.root();
let text = root.text_content();
assert!(text.contains("Hello"), "text: {text}");
assert!(text.contains("World"), "text: {text}");
}
#[test]
fn parse_attr() {
let doc = parse("<a href=\"https://example.com\" class=\"link primary\">text</a>").unwrap();
let root = doc.root();
let a = root.first_child().expect("should have child");
assert_eq!(a.tag(), Tag::A);
assert_eq!(a.attr("href"), Some("https://example.com"));
assert!(a.has_class("link"));
assert!(a.has_class("primary"));
assert!(!a.has_class("secondary"));
}
#[test]
fn parse_attr_case_insensitive_name_lookup() {
let doc = parse("<a HREF=\"https://example.com\" CLASS=\"link primary\">text</a>").unwrap();
let root = doc.root();
let a = root.first_child().expect("should have child");
assert_eq!(a.attr("href"), Some("https://example.com"));
assert_eq!(a.attr("HREF"), Some("https://example.com"));
assert!(a.has_class("link"));
assert!(a.has_class("primary"));
}
#[test]
fn parse_inner_html() {
let doc = parse("<div><p>Hello</p></div>").unwrap();
let root = doc.root();
let div = root.first_child().unwrap();
assert_eq!(div.tag(), Tag::Div);
let inner = div.inner_html();
assert!(inner.contains("<p>"), "inner: {inner}");
assert!(inner.contains("Hello"), "inner: {inner}");
assert!(inner.contains("</p>"), "inner: {inner}");
}
#[test]
fn parse_outer_html() {
let doc = parse("<div><p>Hello</p></div>").unwrap();
let root = doc.root();
let div = root.first_child().unwrap();
let outer = div.outer_html();
assert!(outer.starts_with("<div>"), "outer: {outer}");
assert!(outer.ends_with("</div>"), "outer: {outer}");
}
#[test]
fn parse_void_elements() {
let doc = parse("<div><br><hr></div>").unwrap();
let root = doc.root();
let div = root.first_child().unwrap();
let children: Vec<_> = div.children().collect();
assert_eq!(children.len(), 2);
let br_ref = doc.get(children[0]);
assert_eq!(br_ref.tag(), Tag::Br);
assert!(br_ref.is_void());
let hr_ref = doc.get(children[1]);
assert_eq!(hr_ref.tag(), Tag::Hr);
assert!(hr_ref.is_void());
}
#[test]
fn parse_depth_first() {
let doc = parse("<div><span>a</span><p>b</p></div>").unwrap();
let root = doc.root();
let tags: Vec<_> = root
.descendants()
.map(|id| doc.get(id))
.filter(|n| !n.is_text())
.map(|n| n.tag())
.collect();
assert!(tags.contains(&Tag::Div));
assert!(tags.contains(&Tag::Span));
assert!(tags.contains(&Tag::P));
}
#[test]
fn parse_ancestors() {
let doc = parse("<div><span><a>link</a></span></div>").unwrap();
let root = doc.root();
let div = root.first_child().unwrap();
let span = div.first_child().unwrap();
let a = span.first_child().unwrap();
let ancestor_tags: Vec<_> = a.ancestors().map(|id| doc.get(id).tag()).collect();
assert_eq!(ancestor_tags, vec![Tag::Span, Tag::Div, Tag::Unknown]);
}
#[test]
fn parse_siblings() {
let doc = parse("<ul><li>1</li><li>2</li><li>3</li></ul>").unwrap();
let root = doc.root();
let ul = root.first_child().unwrap();
let li1 = ul.first_child().unwrap();
let sibling_count = li1.siblings().count();
assert_eq!(sibling_count, 2);
}
#[test]
fn empty_input() {
let doc = parse("").unwrap();
assert!(!doc.root().has_children());
}
#[test]
fn text_only() {
let doc = parse("just text").unwrap();
assert_eq!(doc.root().text_content(), "just text");
}
#[test]
fn broken_html_unclosed() {
let doc = parse("<div><p>unclosed").unwrap();
let root = doc.root();
assert!(root.has_children());
assert_eq!(root.text_content(), "unclosed");
}
#[test]
fn broken_html_extra_close() {
let doc = parse("</div><p>ok</p>").unwrap();
let root = doc.root();
assert_eq!(root.text_content(), "ok");
}
#[test]
fn implicit_close_p_p() {
let doc = parse("<p>first<p>second").unwrap();
let root = doc.root();
let children: Vec<_> = root.children().collect();
let p_count = children
.iter()
.filter(|&c| doc.get(*c).tag() == Tag::P)
.count();
assert_eq!(p_count, 2, "both <p> should be root children");
}
#[test]
fn node_64_bytes_alignment() {
assert_eq!(std::mem::size_of::<node::Node>(), 64);
assert_eq!(std::mem::align_of::<node::Node>(), 64);
}
#[test]
fn input_too_large() {
let result = parse("");
assert!(result.is_ok());
}
#[test]
fn comment_and_doctype() {
let doc = parse("<!DOCTYPE html><!-- comment --><div>ok</div>").unwrap();
let root = doc.root();
let mut has_comment = false;
let mut has_doctype = false;
for child_id in root.children() {
let child = doc.get(child_id);
if child.is_comment() {
has_comment = true;
}
if child.is_doctype() {
has_doctype = true;
}
}
assert!(has_doctype, "should have doctype");
assert!(has_comment, "should have comment");
}
#[test]
fn void_outer_html() {
let doc = parse("<br>").unwrap();
let root = doc.root();
let br = root.first_child().unwrap();
let html = br.outer_html();
assert_eq!(html, "<br>", "outer: {html}");
}
#[test]
fn unknown_tag_outer_html_preserved() {
let doc = parse("<my-widget><x-item>ok</x-item></my-widget>").unwrap();
let root = doc.root();
let outer = root.inner_html();
assert_eq!(outer, "<my-widget><x-item>ok</x-item></my-widget>");
}
#[test]
fn parse_bytes_utf8() {
let doc = parse_bytes(b"<div><p>Hello</p></div>").unwrap();
assert_eq!(doc.root().text_content(), "Hello");
}
#[test]
fn parse_bytes_utf8_bom() {
let html = b"\xEF\xBB\xBF<div><p>BOM test</p></div>";
let doc = parse_bytes(html).unwrap();
assert!(doc.root().text_content().contains("BOM test"));
}
#[test]
fn parse_bytes_windows_1254_meta() {
let html = b"<meta charset=\"windows-1254\"><p>Merhaba d\xFCnya</p>";
let doc = parse_bytes(html).unwrap();
let text = doc.root().text_content();
assert!(text.contains("dünya"), "text: {text}");
}
#[test]
fn parse_bytes_utf16le_bom() {
let mut bytes = vec![0xFF, 0xFE]; for &ch in b"<p>UTF16</p>" {
bytes.push(ch);
bytes.push(0x00);
}
let doc = parse_bytes(&bytes).unwrap();
let text = doc.root().text_content();
assert!(text.contains("UTF16"), "text: {text}");
}
#[test]
fn parse_bytes_empty() {
let doc = parse_bytes(b"").unwrap();
assert!(!doc.root().has_children());
}
#[test]
fn text_escaping_in_inner_html() {
let doc = parse("<p>1 < 2 & 3 > 0</p>").unwrap();
let p = doc.root().first_child().unwrap();
assert_eq!(p.text_content(), "1 < 2 & 3 > 0");
let inner = p.inner_html();
assert_eq!(inner, "1 < 2 & 3 > 0");
}
#[test]
fn attr_escaping_in_outer_html() {
let doc = parse("<a href=\"x&y\">link</a>").unwrap();
let a = doc.root().first_child().unwrap();
let outer = a.outer_html();
assert!(
outer.contains("x&y"),
"attribute value should be escaped: {outer}"
);
}
#[test]
fn script_raw_text_not_escaped() {
let doc = parse("<script>if (a < b && c > d) {}</script>").unwrap();
let script = doc.root().first_child().unwrap();
let inner = script.inner_html();
assert_eq!(inner, "if (a < b && c > d) {}");
}
#[test]
fn style_raw_text_not_escaped() {
let doc = parse("<style>a > b { color: red; }</style>").unwrap();
let style = doc.root().first_child().unwrap();
let inner = style.inner_html();
assert_eq!(inner, "a > b { color: red; }");
}
#[test]
fn void_elements_no_closing_slash() {
let doc = parse("<div><br><img src=\"x.png\"><hr></div>").unwrap();
let div = doc.root().first_child().unwrap();
let inner = div.inner_html();
assert!(inner.contains("<br>"), "br: {inner}");
assert!(inner.contains("<img "), "img: {inner}");
assert!(inner.contains("<hr>"), "hr: {inner}");
assert!(!inner.contains("/>"), "should not contain />: {inner}");
}
#[test]
fn comment_not_escaped() {
let doc = parse("<!-- <b>not bold</b> & stuff -->").unwrap();
let html = doc.to_html();
assert!(
html.contains("<!-- <b>not bold</b> & stuff -->"),
"comment should be verbatim: {html}"
);
}
#[test]
fn doctype_not_escaped() {
let doc = parse("<!DOCTYPE html><p>ok</p>").unwrap();
let html = doc.to_html();
assert!(
html.contains("<!DOCTYPE html>"),
"doctype should be verbatim: {html}"
);
}
#[test]
fn document_to_html() {
let doc = parse("<!DOCTYPE html><html><body><p>Hello</p></body></html>").unwrap();
let html = doc.to_html();
assert!(html.contains("<!DOCTYPE html>"), "html: {html}");
assert!(html.contains("<p>Hello</p>"), "html: {html}");
}
#[test]
fn round_trip_structure() {
let input = "<div><p>Hello</p><span>World</span></div>";
let doc1 = parse(input).unwrap();
let html = doc1.to_html();
let doc2 = parse(&html).unwrap();
assert_eq!(doc1.root().text_content(), doc2.root().text_content());
assert_eq!(doc1.node_count(), doc2.node_count());
}
#[test]
fn round_trip_with_special_chars() {
let input = "<p>1 < 2 & 3 > 0</p>";
let doc1 = parse(input).unwrap();
assert_eq!(doc1.root().text_content(), "1 < 2 & 3 > 0");
let html = doc1.to_html();
let doc2 = parse(&html).unwrap();
assert_eq!(doc2.root().text_content(), "1 < 2 & 3 > 0");
}
#[test]
fn unknown_tag_preserved_in_to_html() {
let doc = parse("<my-widget>content</my-widget>").unwrap();
let html = doc.to_html();
assert!(html.contains("<my-widget>"), "html: {html}");
assert!(html.contains("</my-widget>"), "html: {html}");
}
}