sup_xml_core/html/stream.rs
1#![forbid(unsafe_code)]
2
3//! Streaming HTML parser surfaces.
4//!
5//! Two shapes, both built on the same underlying `StreamingSink`:
6//!
7//! - [`HtmlReader`] / [`HtmlBytesReader`] — pull-based event iterator
8//! (XmlReader twin). Caller drives via `next()`.
9//! - [`HtmlSaxParser`] — push-based callback parser
10//! (libxml2 / lxml twin). Caller feeds bytes; sink dispatches
11//! events directly to a registered handler.
12//!
13//! Both surfaces emit tree-construction events (post-tokenizer +
14//! post-tree-builder), not raw tokens. Implicit `<html>`/`<head>`
15//! /`<body>` insertion, void-element handling, named entity
16//! decoding, and tag-soup recovery have all happened by the time
17//! the consumer sees an event.
18//!
19//! # Bridging html5ever's push API to our pull API
20//!
21//! html5ever is push-based: you call `Tokenizer::feed(buffer)` and it
22//! synchronously invokes our sink's TreeSink methods for every token.
23//! To expose a pull iterator on top, we use a sink that buffers
24//! events into a `VecDeque` instead of building a tree.
25//! `HtmlReader::next()` drains that queue, feeding the next 8 KiB
26//! chunk of input when the queue empties.
27//!
28//! For the push surface, the same sink is parameterised with a
29//! callback emitter that dispatches each event directly to the
30//! caller's `HtmlSaxHandler` instead of buffering — saving the
31//! VecDeque allocation but losing the ability to pull.
32
33use std::borrow::Cow;
34use std::cell::{Cell, RefCell};
35use std::collections::VecDeque;
36
37use html5ever::driver::{parse_document, ParseOpts, Parser};
38use html5ever::interface::tree_builder::{
39 ElementFlags, NodeOrText, QuirksMode as H5QuirksMode, TreeSink,
40};
41use html5ever::tendril::{StrTendril, TendrilSink};
42use html5ever::tokenizer::TokenizerOpts;
43use html5ever::tree_builder::TreeBuilderOpts;
44use html5ever::{Attribute as H5Attribute, QualName};
45use markup5ever::interface::tree_builder::ElemName;
46use markup5ever::{LocalName, Namespace as H5Namespace};
47
48use crate::error::{ErrorDomain, ErrorLevel, Result, XmlError};
49
50use super::events::{HtmlAttrs, HtmlEvent, OwnedAttr, OwnedEvent};
51use super::options::HtmlParseOptions;
52
53// ── EventEmit trait — the shared escape hatch out of the sink ────────────────
54
55/// Strategy for "what to do with each event the sink produces."
56/// Buffered emitters queue events for later pull; callback emitters
57/// dispatch immediately.
58pub(crate) trait EventEmit {
59 fn emit(&mut self, event: OwnedEvent);
60}
61
62/// Buffered emitter — pushes events into a `VecDeque` for the pull
63/// reader to consume via `next()`.
64pub(crate) struct BufferedEmit {
65 pub queue: VecDeque<OwnedEvent>,
66}
67
68impl BufferedEmit {
69 fn new() -> Self {
70 Self {
71 queue: VecDeque::with_capacity(32),
72 }
73 }
74}
75
76impl EventEmit for BufferedEmit {
77 fn emit(&mut self, event: OwnedEvent) {
78 // Coalesce adjacent text events so consumers don't see
79 // arbitrarily-fragmented runs of character data.
80 if let OwnedEvent::Text(ref new_text) = event {
81 if let Some(OwnedEvent::Text(last)) = self.queue.back_mut() {
82 last.push_str(new_text);
83 return;
84 }
85 }
86 self.queue.push_back(event);
87 }
88}
89
90/// Callback emitter — dispatches each event directly into the
91/// caller's `HtmlSaxHandler`. Used by `HtmlSaxParser`.
92pub(crate) struct CallbackEmit<H: HtmlSaxHandler> {
93 pub handler: H,
94 /// Pending text accumulator — coalesces adjacent text events
95 /// the same way `BufferedEmit` does. Flushed on the next
96 /// non-text event or on `finish`.
97 pub pending_text: String,
98}
99
100impl<H: HtmlSaxHandler> CallbackEmit<H> {
101 fn new(handler: H) -> Self {
102 Self {
103 handler,
104 pending_text: String::new(),
105 }
106 }
107
108 fn flush_text(&mut self) {
109 if !self.pending_text.is_empty() {
110 let text = std::mem::take(&mut self.pending_text);
111 self.handler.text(&text);
112 }
113 }
114}
115
116impl<H: HtmlSaxHandler> EventEmit for CallbackEmit<H> {
117 fn emit(&mut self, event: OwnedEvent) {
118 match event {
119 OwnedEvent::Text(t) => {
120 self.pending_text.push_str(&t);
121 }
122 OwnedEvent::StartElement { name, attrs } => {
123 self.flush_text();
124 self.handler
125 .start_element(&name, HtmlAttrs { inner: &attrs });
126 }
127 OwnedEvent::EndElement { name } => {
128 self.flush_text();
129 self.handler.end_element(&name);
130 }
131 OwnedEvent::Comment(c) => {
132 self.flush_text();
133 self.handler.comment(&c);
134 }
135 OwnedEvent::Doctype {
136 name,
137 public_id,
138 system_id,
139 } => {
140 self.flush_text();
141 self.handler.doctype(&name, &public_id, &system_id);
142 }
143 OwnedEvent::ParseError(err) => {
144 self.handler.parse_error(&err);
145 }
146 OwnedEvent::Eof => {
147 self.flush_text();
148 self.handler.end_document();
149 }
150 }
151 }
152}
153
154// ── The shared TreeSink that drives both surfaces ────────────────────────────
155
156/// One slot in the streaming sink's small arena. Used to carry
157/// element identity between html5ever's `create_element` and the
158/// later `append` / `pop` / `add_attrs_if_missing` calls. Unlike
159/// `BatchSink::SinkNode` we don't track parent / children pointers
160/// here — we emit events on the fly instead of building a tree.
161enum StreamNode {
162 Document,
163 Element {
164 name: QualName,
165 attrs: Vec<H5Attribute>,
166 /// Has a `StartElement` event been emitted for this node yet?
167 /// We emit on the first `append` (to know its position).
168 /// Subsequent re-parents (adoption-agency) don't re-emit.
169 emitted: bool,
170 },
171 Comment(StrTendril),
172}
173
174/// HTML5 void elements — never have content, never get an end tag
175/// in source, never appear on the open-elements stack. We exclude
176/// them from our own stack tracking so they don't accumulate as
177/// "still open" entries to be drained at finish-time.
178fn is_void_element(name: &str) -> bool {
179 matches!(
180 name,
181 "area"
182 | "base"
183 | "br"
184 | "col"
185 | "embed"
186 | "hr"
187 | "img"
188 | "input"
189 | "link"
190 | "meta"
191 | "param"
192 | "source"
193 | "track"
194 | "wbr"
195 | "keygen"
196 | "menuitem"
197 )
198}
199
200/// Owned form of an element's qualified name, returned from
201/// `TreeSink::elem_name`. Same trick as in the batch sink — clone
202/// the cheap atoms so the returned value doesn't borrow from the
203/// sink's `RefCell`.
204pub(crate) struct OwnedElemName {
205 inner: QualName,
206}
207
208impl std::fmt::Debug for OwnedElemName {
209 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
210 std::fmt::Debug::fmt(&self.inner, f)
211 }
212}
213
214impl ElemName for OwnedElemName {
215 fn ns(&self) -> &H5Namespace {
216 &self.inner.ns
217 }
218 fn local_name(&self) -> &LocalName {
219 &self.inner.local
220 }
221}
222
223struct StreamState<E: EventEmit> {
224 arena: Vec<StreamNode>,
225 emit: E,
226 /// Our own open-elements stack, tracked from `append` calls.
227 /// We can't rely on html5ever's `pop()` callback — it isn't
228 /// invoked for every element that gets implicitly closed (e.g.
229 /// `</p>` close_p_element doesn't always go through the sink
230 /// pop path). Tracking it ourselves from append parents lets
231 /// us emit `EndElement` for every opened element correctly.
232 open_stack: Vec<usize>,
233 /// Total bytes of accumulated text content — checked against
234 /// [`HtmlParseOptions::max_text_bytes`].
235 text_bytes: u64,
236 /// First fatal error encountered, propagated from `finish()`.
237 fatal: Option<XmlError>,
238}
239
240pub(crate) struct StreamingSink<E: EventEmit> {
241 state: RefCell<StreamState<E>>,
242 quirks: Cell<H5QuirksMode>,
243 opts: HtmlParseOptions,
244 aborted: Cell<bool>,
245}
246
247impl<E: EventEmit> StreamingSink<E> {
248 fn new(opts: HtmlParseOptions, emit: E) -> Self {
249 let mut arena = Vec::with_capacity(32);
250 // Slot 0 is the Document.
251 arena.push(StreamNode::Document);
252 Self {
253 state: RefCell::new(StreamState {
254 arena,
255 emit,
256 open_stack: Vec::with_capacity(32),
257 text_bytes: 0,
258 fatal: None,
259 }),
260 quirks: Cell::new(H5QuirksMode::NoQuirks),
261 opts,
262 aborted: Cell::new(false),
263 }
264 }
265
266 /// Close all elements on the open stack down to (but not
267 /// including) `parent_id`. If `parent_id` is the Document
268 /// (id 0) or isn't on the stack at all, close everything.
269 /// Emits `EndElement` for each closed element.
270 fn close_to_parent(s: &mut StreamState<E>, parent_id: usize) {
271 // Find parent_id from the top of the stack. Pop everything above it.
272 if parent_id == 0 {
273 // Document parent — nothing to pop down to (Document is
274 // not on the stack). Don't touch the stack.
275 return;
276 }
277 // Walk from top downward.
278 while let Some(&top) = s.open_stack.last() {
279 if top == parent_id {
280 return;
281 }
282 // Top is not the parent. Pop it and emit EndElement.
283 let popped = s.open_stack.pop().unwrap();
284 if let StreamNode::Element { name, .. } = &s.arena[popped] {
285 let local = name.local.to_string();
286 s.emit.emit(OwnedEvent::EndElement { name: local });
287 }
288 // If parent_id isn't on the stack at all, this loop
289 // would drain the whole stack. That's the right
290 // behaviour for foster-parented / re-parented nodes
291 // where html5ever has lost its tree shape.
292 }
293 }
294
295 fn alloc(&self, node: StreamNode) -> usize {
296 let mut s = self.state.borrow_mut();
297 let id = s.arena.len();
298 s.arena.push(node);
299 id
300 }
301
302 fn record_error(&self, msg: impl Into<String>, level: ErrorLevel) {
303 let err = XmlError::new(ErrorDomain::Html, level, msg);
304 let mut s = self.state.borrow_mut();
305 if level == ErrorLevel::Fatal && s.fatal.is_none() {
306 s.fatal = Some(err.clone());
307 }
308 s.emit.emit(OwnedEvent::ParseError(err));
309 }
310
311 fn abort(&self, msg: impl Into<String>) {
312 if !self.aborted.get() {
313 self.aborted.set(true);
314 self.record_error(msg, ErrorLevel::Fatal);
315 }
316 }
317}
318
319impl<E: EventEmit + 'static> TreeSink for StreamingSink<E> {
320 type Handle = usize;
321 type Output = Self;
322 type ElemName<'a>
323 = OwnedElemName
324 where
325 Self: 'a;
326
327 fn finish(self) -> Self {
328 self
329 }
330
331 fn parse_error(&self, msg: Cow<'static, str>) {
332 let level = if self.opts.recovery_mode {
333 ErrorLevel::Error
334 } else if self.state.borrow().fatal.is_none() {
335 ErrorLevel::Fatal
336 } else {
337 ErrorLevel::Error
338 };
339 self.record_error(msg.into_owned(), level);
340 }
341
342 fn get_document(&self) -> usize {
343 0
344 }
345
346 fn elem_name<'a>(&'a self, target: &'a usize) -> OwnedElemName {
347 let s = self.state.borrow();
348 match &s.arena[*target] {
349 StreamNode::Element { name, .. } => OwnedElemName { inner: name.clone() },
350 _ => panic!("elem_name called on non-element node {target}"),
351 }
352 }
353
354 fn create_element(
355 &self,
356 name: QualName,
357 attrs: Vec<H5Attribute>,
358 _flags: ElementFlags,
359 ) -> usize {
360 self.alloc(StreamNode::Element {
361 name,
362 attrs,
363 emitted: false,
364 })
365 }
366
367 fn create_comment(&self, text: StrTendril) -> usize {
368 self.alloc(StreamNode::Comment(text))
369 }
370
371 fn create_pi(&self, _target: StrTendril, _data: StrTendril) -> usize {
372 // HTML5 treats source PIs as bogus comments — html5ever will
373 // not normally call this for HTML input, but the trait
374 // requires it.
375 self.alloc(StreamNode::Comment(StrTendril::new()))
376 }
377
378 fn append(&self, parent: &usize, child: NodeOrText<usize>) {
379 if self.aborted.get() {
380 return;
381 }
382 let parent = *parent;
383 match child {
384 NodeOrText::AppendText(text) => {
385 let mut s = self.state.borrow_mut();
386 let added = text.len() as u64;
387 if s.text_bytes.saturating_add(added) > self.opts.max_text_bytes {
388 drop(s);
389 self.abort(format!(
390 "max_text_bytes ({}) exceeded",
391 self.opts.max_text_bytes
392 ));
393 return;
394 }
395 s.text_bytes += added;
396 // Text doesn't move the open-elements stack. But
397 // if the text's parent isn't on top, we're either
398 // appending into an ancestor (close down to it) or
399 // into something fully outside the stack (drain).
400 Self::close_to_parent(&mut s, parent);
401 s.emit.emit(OwnedEvent::Text(text.to_string()));
402 }
403 NodeOrText::AppendNode(child_id) => {
404 let mut s = self.state.borrow_mut();
405 // Close everything above `parent` in the open-elements
406 // stack — html5ever doesn't always call sink.pop()
407 // for implicit closes, so this is how we get
408 // EndElement events out for `<p>foo<p>bar` etc.
409 Self::close_to_parent(&mut s, parent);
410 match &mut s.arena[child_id] {
411 StreamNode::Element {
412 name,
413 attrs,
414 emitted,
415 } => {
416 if *emitted {
417 // Re-parent (adoption agency) — element
418 // already emitted, don't double-emit.
419 // Push it back onto the stack so closes
420 // continue to work.
421 s.open_stack.push(child_id);
422 return;
423 }
424 *emitted = true;
425 let local_name = name.local.to_string();
426 let owned_attrs: Vec<OwnedAttr> = attrs
427 .iter()
428 .map(|a| OwnedAttr {
429 name: a.name.local.to_string(),
430 value: a.value.to_string(),
431 })
432 .collect();
433 // Void elements never close — don't push.
434 // Non-void elements get pushed so we can
435 // emit EndElement when they're closed.
436 if !is_void_element(&local_name) {
437 s.open_stack.push(child_id);
438 }
439 s.emit.emit(OwnedEvent::StartElement {
440 name: local_name,
441 attrs: owned_attrs,
442 });
443 }
444 StreamNode::Comment(text) => {
445 let content = text.to_string();
446 s.emit.emit(OwnedEvent::Comment(content));
447 }
448 StreamNode::Document => {
449 // Document never gets appended to anything.
450 }
451 }
452 }
453 }
454 }
455
456 fn append_based_on_parent_node(
457 &self,
458 element: &usize,
459 prev_element: &usize,
460 child: NodeOrText<usize>,
461 ) {
462 // Same logic as BatchSink: if the element has a parent we
463 // append before-sibling, otherwise append to prev_element.
464 // For streaming, both paths go through `append` — the event
465 // semantics don't depend on physical insertion position.
466 let _ = prev_element;
467 self.append(element, child);
468 }
469
470 fn append_doctype_to_document(
471 &self,
472 name: StrTendril,
473 public_id: StrTendril,
474 system_id: StrTendril,
475 ) {
476 if self.aborted.get() {
477 return;
478 }
479 let mut s = self.state.borrow_mut();
480 s.emit.emit(OwnedEvent::Doctype {
481 name: name.to_string(),
482 public_id: public_id.to_string(),
483 system_id: system_id.to_string(),
484 });
485 }
486
487 fn pop(&self, _node: &usize) {
488 // Intentionally no-op: html5ever doesn't call pop() for
489 // every implicitly-closed element (close_p_element and
490 // similar paths bypass it), so we can't rely on it as the
491 // sole "end of element" signal. Instead we track the
492 // open-elements stack ourselves from `append` calls and
493 // emit `EndElement` via `close_to_parent` when needed
494 // (and from `finish`-time draining of remaining opens).
495 }
496
497 fn get_template_contents(&self, target: &usize) -> usize {
498 // We don't model template-contents documents in v1 — see
499 // batch sink note. Returning the element itself works for
500 // html5ever's bookkeeping purposes; consumers see the
501 // template element's children in the regular event stream.
502 *target
503 }
504
505 fn same_node(&self, x: &usize, y: &usize) -> bool {
506 x == y
507 }
508
509 fn set_quirks_mode(&self, mode: H5QuirksMode) {
510 self.quirks.set(mode);
511 }
512
513 fn append_before_sibling(&self, sibling: &usize, new_node: NodeOrText<usize>) {
514 // Streaming doesn't model insertion position — just emit
515 // the event the same way `append` does. Consumers
516 // operating on event streams can't observe the structural
517 // distinction anyway.
518 self.append(sibling, new_node);
519 }
520
521 fn add_attrs_if_missing(&self, target: &usize, attrs: Vec<H5Attribute>) {
522 // The HTML5 spec calls this in two cases:
523 // - Duplicate `<html>` / `<body>` start tags (rare)
524 // - Attribute deduplication
525 // For streaming, we've already emitted StartElement by now;
526 // we'd have to emit a synthetic "attr-update" event to
527 // surface this, which doesn't fit the standard event model.
528 // We update the arena entry so subsequent elem_name and
529 // pop work consistently, but don't emit anything.
530 let mut s = self.state.borrow_mut();
531 if let StreamNode::Element { attrs: existing, .. } = &mut s.arena[*target] {
532 for new_attr in attrs {
533 if !existing.iter().any(|a| a.name == new_attr.name) {
534 existing.push(new_attr);
535 }
536 }
537 }
538 }
539
540 fn remove_from_parent(&self, _target: &usize) {
541 // Adoption agency uses this before re-inserting elsewhere.
542 // We don't emit any synthetic "removed" event — the
543 // subsequent re-insert is suppressed by the `emitted` flag,
544 // so the consumer sees the element only once (in its
545 // original position). Acceptable approximation; full
546 // adoption-agency event modelling is out of scope for v1.
547 }
548
549 fn reparent_children(&self, _node: &usize, _new_parent: &usize) {
550 // Same rationale as remove_from_parent — children stay
551 // emitted in their original position; we do not emit
552 // synthetic events for the move.
553 }
554}
555
556// ── Push API: HtmlSaxHandler + HtmlSaxParser ─────────────────────────────────
557
558/// Callback trait for the push-based [`HtmlSaxParser`]. All methods
559/// have default no-op implementations — implementers override only
560/// the events they care about (e.g. an `<a href>` extractor only
561/// needs `start_element`).
562///
563/// libxml2 / lxml / Nokogiri analogue. Argument shape mirrors
564/// libxml2's xmlSAXHandler — flat parameters per callback rather
565/// than packing them into payload structs.
566pub trait HtmlSaxHandler {
567 fn start_element(&mut self, _name: &str, _attrs: HtmlAttrs<'_>) {}
568 fn end_element(&mut self, _name: &str) {}
569 fn text(&mut self, _content: &str) {}
570 fn comment(&mut self, _content: &str) {}
571 fn doctype(&mut self, _name: &str, _public_id: &str, _system_id: &str) {}
572 fn parse_error(&mut self, _err: &XmlError) {}
573 /// Called once after the last token has been processed.
574 fn end_document(&mut self) {}
575}
576
577/// Push-based HTML parser. Caller drives input chunks via `feed`;
578/// the parser dispatches events synchronously into the registered
579/// handler as they're produced.
580///
581/// libxml2 `htmlCreatePushParserCtxt` + `htmlParseChunk` analogue,
582/// or lxml's `HTMLParser(target=…)` + `parser.feed(bytes)`.
583///
584/// # Example
585/// ```
586/// use sup_xml_core::html::{HtmlAttrs, HtmlSaxHandler, HtmlSaxParser};
587///
588/// #[derive(Default)]
589/// struct LinkCollector {
590/// hrefs: Vec<String>,
591/// }
592///
593/// impl HtmlSaxHandler for LinkCollector {
594/// fn start_element(&mut self, name: &str, attrs: HtmlAttrs<'_>) {
595/// if name == "a" {
596/// if let Some(href) = attrs.get("href") {
597/// self.hrefs.push(href.to_string());
598/// }
599/// }
600/// }
601/// }
602///
603/// let html = r#"<a href="/x">x</a><a href="/y">y</a>"#;
604/// let mut p = HtmlSaxParser::new(LinkCollector::default());
605/// p.feed(html).unwrap();
606/// let collector = p.finish().unwrap();
607/// assert_eq!(collector.hrefs, vec!["/x", "/y"]);
608/// ```
609pub struct HtmlSaxParser<H: HtmlSaxHandler + 'static> {
610 parser: Parser<StreamingSink<CallbackEmit<H>>>,
611}
612
613impl<H: HtmlSaxHandler + 'static> HtmlSaxParser<H> {
614 /// Create a push parser with default options.
615 pub fn new(handler: H) -> Self {
616 Self::with_opts(handler, HtmlParseOptions::default())
617 }
618
619 /// Create a push parser with explicit options.
620 pub fn with_opts(handler: H, opts: HtmlParseOptions) -> Self {
621 let sink = StreamingSink::new(opts.clone(), CallbackEmit::new(handler));
622 let parser = parse_document(sink, make_h5_opts(&opts));
623 Self { parser }
624 }
625
626 /// Feed a chunk of UTF-8 text. Drives the tokenizer and
627 /// dispatches any emitted events synchronously into the handler
628 /// before returning.
629 ///
630 /// Errors only when an internal limit (e.g. `max_text_bytes`)
631 /// is exceeded.
632 pub fn feed(&mut self, chunk: &str) -> Result<()> {
633 let tendril = StrTendril::from_slice(chunk);
634 self.parser.process(tendril);
635 // After process(), check for fatal aborts.
636 let s = self.parser.tokenizer.sink.sink.state.borrow();
637 if let Some(err) = &s.fatal {
638 if self.parser.tokenizer.sink.sink.aborted.get() {
639 return Err(err.clone());
640 }
641 }
642 Ok(())
643 }
644
645 /// Signal end-of-input. Flushes any pending text, dispatches
646 /// EndElement events for any still-open elements, then calls
647 /// `end_document`. Returns the handler.
648 pub fn finish(self) -> Result<H> {
649 // Capture fatal-error state before consuming the parser.
650 let fatal = self.parser.tokenizer.sink.sink.state.borrow().fatal.clone();
651 let aborted = self.parser.tokenizer.sink.sink.aborted.get();
652 let sink = self.parser.finish();
653 // sink.finish() returned `Self` from our TreeSink impl.
654 let mut state = sink.state.into_inner();
655 // Drain any still-open elements as EndElement events first
656 // — same reason as `HtmlReader::finish_parser`: html5ever
657 // doesn't reliably call pop() for everything at end-of-input.
658 while let Some(open_id) = state.open_stack.pop() {
659 if let StreamNode::Element { name, .. } = &state.arena[open_id] {
660 let local = name.local.to_string();
661 state.emit.emit(OwnedEvent::EndElement { name: local });
662 }
663 }
664 // Final end_document dispatch (also flushes any pending text).
665 state.emit.emit(OwnedEvent::Eof);
666 if aborted {
667 return Err(fatal.unwrap_or_else(|| {
668 XmlError::new(ErrorDomain::Html, ErrorLevel::Fatal, "aborted")
669 }));
670 }
671 Ok(state.emit.handler)
672 }
673
674 // No `handler()` borrow accessor in v1: the natural path
675 // through `RefCell::borrow().emit.handler` produces a `Ref<'_,
676 // _>` that doesn't compose cleanly with returning `&H`. Users
677 // who want to inspect progress should look at the returned
678 // handler after `finish()`.
679}
680
681// ── Pull API: HtmlReader / HtmlBytesReader ───────────────────────────────────
682
683/// Pull-based HTML event reader (XmlReader twin). Iterate events
684/// with `next()`; the reader feeds the next 8 KiB chunk into
685/// html5ever whenever its internal queue runs dry.
686///
687/// # Example
688/// ```
689/// use sup_xml_core::html::{HtmlReader, HtmlEvent};
690///
691/// let mut r = HtmlReader::new("<p>hello <b>world</b></p>");
692/// loop {
693/// match r.next().unwrap() {
694/// HtmlEvent::Eof => break,
695/// HtmlEvent::StartElement { name, .. } => println!("start: {name}"),
696/// _ => {}
697/// }
698/// }
699/// ```
700pub struct HtmlReader<'src> {
701 inner: ReaderInner<'src>,
702}
703
704/// Bytes-flavoured pull reader. Same as [`HtmlReader`] but takes
705/// raw bytes — html5ever does lossy UTF-8 decoding internally.
706pub struct HtmlBytesReader<'src> {
707 inner: ReaderInner<'src>,
708}
709
710struct ReaderInner<'src> {
711 parser: Parser<StreamingSink<BufferedEmit>>,
712 /// Input bytes the reader feeds to html5ever. Borrowed for the
713 /// str path (already UTF-8), owned for the bytes path after
714 /// encoding sniffing + transcoding.
715 input: Cow<'src, [u8]>,
716 fed_pos: usize,
717 chunk_size: usize,
718 finished: bool,
719 /// Owned data backing the most recent event. Held so the
720 /// borrowed `HtmlEvent<'_>` returned from `next()` stays valid
721 /// until the next call.
722 current: Option<OwnedEvent>,
723 /// Errors recovered while parsing — same contract as
724 /// `parse_html_str_with_recovered`.
725 recovered: Vec<XmlError>,
726 /// Whether to return Err on the next strict-mode parse error.
727 strict: bool,
728}
729
730impl<'src> ReaderInner<'src> {
731 /// Create from already-UTF-8 input. Used by `HtmlReader::new`.
732 fn from_utf8_str(input: &'src str, opts: HtmlParseOptions) -> Self {
733 Self::with_cow(Cow::Borrowed(input.as_bytes()), opts)
734 }
735
736 /// Create from raw bytes. Runs WHATWG byte-stream sniffing and
737 /// transcodes to UTF-8 before feeding to html5ever.
738 fn from_bytes(input: &'src [u8], opts: HtmlParseOptions) -> Result<Self> {
739 let (decoded, _enc) = super::encoding::decode_html_input(input, &opts)?;
740 let cow: Cow<'src, [u8]> = match decoded {
741 Cow::Borrowed(b) => {
742 // We can borrow as long as transcoding was a no-op
743 // (input was already UTF-8 / ASCII). But the
744 // borrow's lifetime is tied to the *input* slice,
745 // and `decode_html_input` may have stripped a BOM
746 // — which still produces a borrow into `input`,
747 // so this is sound.
748 if std::ptr::eq(b.as_ptr(), input.as_ptr()) || input.starts_with(b) {
749 // SAFETY-equivalent (no unsafe used): we're just
750 // re-asserting that `b` is a subslice of `input`,
751 // which has lifetime 'src. The Cow we got from
752 // `decode_html_input` had a shorter lifetime
753 // because of the function signature, but the
754 // bytes themselves live as long as `input`.
755 // Reconstruct the borrow with the wider lifetime
756 // by slicing `input` directly.
757 let offset = b.as_ptr() as usize - input.as_ptr() as usize;
758 Cow::Borrowed(&input[offset..offset + b.len()])
759 } else {
760 Cow::Owned(b.to_vec())
761 }
762 }
763 Cow::Owned(v) => Cow::Owned(v),
764 };
765 Ok(Self::with_cow(cow, opts))
766 }
767
768 fn with_cow(input: Cow<'src, [u8]>, opts: HtmlParseOptions) -> Self {
769 let strict = !opts.recovery_mode;
770 let sink = StreamingSink::new(opts.clone(), BufferedEmit::new());
771 let parser = parse_document(sink, make_h5_opts(&opts));
772 Self {
773 parser,
774 input,
775 fed_pos: 0,
776 chunk_size: 8 * 1024,
777 finished: false,
778 current: None,
779 recovered: Vec::new(),
780 strict,
781 }
782 }
783
784 /// Borrow the queue mutably to pop the next event.
785 fn pop_event(&mut self) -> Option<OwnedEvent> {
786 let mut s = self.parser.tokenizer.sink.sink.state.borrow_mut();
787 s.emit.queue.pop_front()
788 }
789
790 /// True if the sink has aborted (depth/text-byte limit, or
791 /// strict-mode parse error).
792 fn aborted(&self) -> bool {
793 self.parser.tokenizer.sink.sink.aborted.get()
794 }
795
796 fn fatal(&self) -> Option<XmlError> {
797 self.parser.tokenizer.sink.sink.state.borrow().fatal.clone()
798 }
799
800 /// Feed the next chunk into the tokenizer. Returns true if any
801 /// input was fed.
802 fn feed_chunk(&mut self) -> bool {
803 if self.fed_pos >= self.input.len() {
804 return false;
805 }
806 let end = (self.fed_pos + self.chunk_size).min(self.input.len());
807 // Walk back to a UTF-8 char boundary so we don't split
808 // mid-codepoint between chunks. At most 3 bytes back.
809 let chunk_bytes = trim_to_utf8_boundary(&self.input[self.fed_pos..end]);
810 let actual_end = self.fed_pos + chunk_bytes.len();
811 if chunk_bytes.is_empty() {
812 // Pathological — single multi-byte char split across
813 // 8 KiB? Feed the original chunk lossily.
814 let lossy = String::from_utf8_lossy(&self.input[self.fed_pos..end]).into_owned();
815 self.parser.process(StrTendril::from_slice(&lossy));
816 self.fed_pos = end;
817 } else {
818 let lossy = String::from_utf8_lossy(chunk_bytes);
819 self.parser.process(StrTendril::from_slice(&lossy));
820 self.fed_pos = actual_end;
821 }
822 true
823 }
824
825 fn finish_parser(&mut self) {
826 if !self.finished {
827 // `Parser::finish` consumes; we replace with a sentinel
828 // by swapping a fresh parser. But the parser owns the
829 // sink which holds our queue — we can't just discard it.
830 // Instead, drive the tokenizer to end-of-input directly.
831 self.parser.tokenizer.end();
832 self.finished = true;
833 // Drain any still-open elements as EndElement events.
834 // (html5ever doesn't reliably call pop() at end-of-input
835 // for everything; we close them ourselves.)
836 let mut s = self.parser.tokenizer.sink.sink.state.borrow_mut();
837 while let Some(open_id) = s.open_stack.pop() {
838 if let StreamNode::Element { name, .. } = &s.arena[open_id] {
839 let local = name.local.to_string();
840 s.emit.emit(OwnedEvent::EndElement { name: local });
841 }
842 }
843 }
844 }
845
846 /// Pull events repeatedly: drain queue, feed more, until we
847 /// have an event to return or hit Eof.
848 fn pull(&mut self) -> Result<OwnedEvent> {
849 loop {
850 if let Some(event) = self.pop_event() {
851 // Check for parse errors / fatals that should
852 // surface as Err in strict mode or as recovered
853 // entries in lenient mode.
854 if let OwnedEvent::ParseError(err) = &event {
855 let err = err.clone();
856 if self.strict {
857 return Err(err);
858 } else {
859 self.recovered.push(err);
860 continue;
861 }
862 }
863 if self.aborted() {
864 if let Some(fatal) = self.fatal() {
865 return Err(fatal);
866 }
867 }
868 return Ok(event);
869 }
870 // Queue empty — feed more or finalise.
871 if !self.finished && self.feed_chunk() {
872 continue;
873 }
874 if !self.finished {
875 self.finish_parser();
876 // After end(), the sink may have queued a final
877 // burst of events. Loop once more to drain them.
878 continue;
879 }
880 // No more input, no more events.
881 return Ok(OwnedEvent::Eof);
882 }
883 }
884
885 /// Recovered errors so far. Same semantics as
886 /// `XmlReader::recovered_errors`.
887 fn recovered_errors(&self) -> &[XmlError] {
888 &self.recovered
889 }
890}
891
892/// Walk back from the end of `bytes` to the nearest UTF-8 char
893/// boundary. Returns the truncated slice. Used to avoid splitting
894/// a multi-byte codepoint across two chunks fed to html5ever.
895fn trim_to_utf8_boundary(bytes: &[u8]) -> &[u8] {
896 if bytes.is_empty() {
897 return bytes;
898 }
899 let mut end = bytes.len();
900 while end > 0 {
901 // ASCII byte: definitely a boundary.
902 // UTF-8 continuation byte (10xxxxxx): not a boundary.
903 // UTF-8 start byte (0xxxxxxx, 11xxxxxx): boundary.
904 let b = bytes[end - 1];
905 if b < 0x80 || b >= 0xC0 {
906 // Either ASCII (already a boundary at `end`) or a
907 // start byte — meaning the byte at `end - 1` starts a
908 // multi-byte sequence. Truncate to *before* the start
909 // byte so we don't split it.
910 if b >= 0xC0 {
911 end -= 1;
912 }
913 break;
914 }
915 end -= 1;
916 // Limit the walk-back to 4 bytes (max UTF-8 sequence length).
917 if bytes.len() - end > 4 {
918 break;
919 }
920 }
921 &bytes[..end]
922}
923
924impl<'src> HtmlReader<'src> {
925 pub fn new(input: &'src str) -> Self {
926 Self::with_opts(input, HtmlParseOptions::default())
927 }
928
929 pub fn with_opts(input: &'src str, opts: HtmlParseOptions) -> Self {
930 Self {
931 inner: ReaderInner::from_utf8_str(input, opts),
932 }
933 }
934
935 /// Pull the next event. Returns `Eof` when input is exhausted.
936 pub fn next(&mut self) -> Result<HtmlEvent<'_>> {
937 let event = self.inner.pull()?;
938 self.inner.current = Some(event);
939 Ok(event_borrow(self.inner.current.as_ref().unwrap()))
940 }
941
942 /// Errors recovered while parsing (only populated in lenient
943 /// mode). Strict mode returns an `Err` from `next()` instead.
944 pub fn recovered_errors(&self) -> &[XmlError] {
945 self.inner.recovered_errors()
946 }
947}
948
949impl<'src> HtmlBytesReader<'src> {
950 /// Construct from raw bytes using default options. Runs WHATWG
951 /// byte-stream encoding sniffing (BOM → meta charset prescan →
952 /// Windows-1252 fallback) and transcodes to UTF-8 before
953 /// streaming events. See [`HtmlParseOptions::encoding_override`]
954 /// to bypass sniffing with a known encoding.
955 pub fn new(input: &'src [u8]) -> Result<Self> {
956 Self::with_opts(input, HtmlParseOptions::default())
957 }
958
959 pub fn with_opts(input: &'src [u8], opts: HtmlParseOptions) -> Result<Self> {
960 Ok(Self {
961 inner: ReaderInner::from_bytes(input, opts)?,
962 })
963 }
964
965 pub fn next(&mut self) -> Result<HtmlEvent<'_>> {
966 let event = self.inner.pull()?;
967 self.inner.current = Some(event);
968 Ok(event_borrow(self.inner.current.as_ref().unwrap()))
969 }
970
971 pub fn recovered_errors(&self) -> &[XmlError] {
972 self.inner.recovered_errors()
973 }
974}
975
976/// Convert an owned event into the borrowing public form.
977fn event_borrow(owned: &OwnedEvent) -> HtmlEvent<'_> {
978 match owned {
979 OwnedEvent::StartElement { name, attrs } => HtmlEvent::StartElement {
980 name: name.as_str(),
981 attributes: HtmlAttrs { inner: attrs.as_slice() },
982 },
983 OwnedEvent::EndElement { name } => HtmlEvent::EndElement {
984 name: name.as_str(),
985 },
986 OwnedEvent::Text(content) => HtmlEvent::Text(content.as_str()),
987 OwnedEvent::Comment(content) => HtmlEvent::Comment(content.as_str()),
988 OwnedEvent::Doctype {
989 name,
990 public_id,
991 system_id,
992 } => HtmlEvent::Doctype {
993 name: name.as_str(),
994 public_id: public_id.as_str(),
995 system_id: system_id.as_str(),
996 },
997 OwnedEvent::Eof => HtmlEvent::Eof,
998 OwnedEvent::ParseError(_) => {
999 // ParseErrors are filtered out in `pull()` (returned as
1000 // Err in strict mode or pushed into `recovered` in
1001 // lenient mode). Should never reach here.
1002 HtmlEvent::Eof
1003 }
1004 }
1005}
1006
1007// ── shared helper: html5ever ParseOpts construction ──────────────────────────
1008
1009fn make_h5_opts(opts: &HtmlParseOptions) -> ParseOpts {
1010 ParseOpts {
1011 tokenizer: TokenizerOpts {
1012 discard_bom: opts.discard_bom,
1013 ..Default::default()
1014 },
1015 tree_builder: TreeBuilderOpts {
1016 scripting_enabled: opts.scripting_enabled,
1017 iframe_srcdoc: opts.iframe_srcdoc,
1018 ..Default::default()
1019 },
1020 }
1021}