text_mirror/convert/
html.rs

1//! Strip-to-text conversion for html.
2//!
3//! A streaming pass over the markup collects the visible text and
4//! nothing else. Script, style, template, and noscript content is
5//! suppressed, head content other than the title is dropped, and the
6//! title text becomes the first line. Block boundaries become line
7//! breaks: the ruled set, `p`, `div`, `li`, `tr`, `br`, the headings,
8//! `blockquote`, and `pre`, plus the HTML5 sectioning and grouping
9//! containers, and table cells within a row separate with a tab.
10//! Character references decode, whitespace runs collapse to one
11//! space, and blank-line runs collapse to one blank line. Reading
12//! order and table reconstruction are out of scope.
13//!
14//! The stream never builds a document tree, so memory stays bounded
15//! under hostile nesting, and a run over the memory ceiling or the
16//! working budget fails closed. Captured text, block breaks, and cell
17//! tabs all charge one working budget, so a dense page of short
18//! blocks cannot grow converter-owned state past the order of the
19//! output ceiling. Suppression of script, style, and
20//! noscript rides the lexer's own text typing rather than end-tag
21//! bookkeeping, so a missing closing tag cannot leak script text into
22//! the capture.
23
24use std::cell::RefCell;
25use std::rc::Rc;
26
27use lol_html::html_content::{TextChunk, TextType};
28use lol_html::{HtmlRewriter, MemorySettings, Settings, doc_text, element, end_tag};
29
30use super::{ConvertError, Converter, MAX_OUTPUT_BYTES, MAX_PART_BYTES, Outcome, normalize_text};
31use crate::segments::Segment;
32
33/// Registry id of the html strip converter.
34pub const HTML_STRIP_ID: &str = "html-strip";
35/// Version of the html strip converter.
36pub const HTML_STRIP_VERSION: &str = "1.0.0";
37
38/// Strip-to-text converter for html.
39pub struct HtmlStrip;
40
41/// Elements whose start tag does not imply the head has ended.
42///
43/// The stream has no tree builder, so an unclosed `<head>` would
44/// otherwise swallow the whole document. The first start tag outside
45/// this set ends head suppression, which mirrors the implied close a
46/// tree builder performs.
47const HEAD_CONTENT: &[&str] = &[
48    "base", "basefont", "bgsound", "head", "html", "link", "meta", "noscript", "script", "style",
49    "template", "title",
50];
51
52/// The block set: the ruled elements plus the HTML5 sectioning and
53/// grouping containers, so adjacent blocks never concatenate words.
54/// Start and end both mark a line break.
55const BLOCK_ELEMENTS: &[&str] = &[
56    "address",
57    "article",
58    "aside",
59    "blockquote",
60    "dd",
61    "div",
62    "dl",
63    "dt",
64    "figure",
65    "footer",
66    "h1",
67    "h2",
68    "h3",
69    "h4",
70    "h5",
71    "h6",
72    "header",
73    "li",
74    "main",
75    "nav",
76    "ol",
77    "p",
78    "pre",
79    "section",
80    "table",
81    "tr",
82    "ul",
83];
84
85#[derive(Default)]
86struct StripState {
87    /// Captured content: collapsed text, `\n` for block boundaries,
88    /// `\t` for cell separators. One flat buffer, so the working
89    /// memory stays on the order of the budget.
90    raw: String,
91    title: String,
92    in_title: bool,
93    title_done: bool,
94    in_head: bool,
95    template_depth: u32,
96    row_has_cell: bool,
97    node_buf: String,
98    node_type: Option<TextType>,
99    charged_bytes: usize,
100    budget: usize,
101}
102
103impl StripState {
104    /// Charges bytes against the working budget. Text, block breaks,
105    /// and cell tabs all pay, so no shape of markup grows the
106    /// converter's own memory past the order of the budget.
107    fn charge(&mut self, bytes: usize) -> std::result::Result<(), String> {
108        self.charged_bytes += bytes;
109        if self.charged_bytes > self.budget {
110            Err(format!(
111                "captured content exceeds the {} byte ceiling",
112                self.budget
113            ))
114        } else {
115            Ok(())
116        }
117    }
118
119    fn push_break(&mut self) -> std::result::Result<(), String> {
120        self.charge(1)?;
121        self.raw.push('\n');
122        Ok(())
123    }
124
125    fn push_tab(&mut self) -> std::result::Result<(), String> {
126        self.charge(1)?;
127        self.raw.push('\t');
128        Ok(())
129    }
130
131    fn flush_node(&mut self, fallback_type: TextType) {
132        let text = std::mem::take(&mut self.node_buf);
133        let text_type = self.node_type.take().unwrap_or(fallback_type);
134        if text.is_empty() || self.template_depth > 0 {
135            return;
136        }
137        let keep = matches!(
138            text_type,
139            TextType::Data | TextType::RCData | TextType::PlainText
140        );
141        if !keep {
142            return;
143        }
144        let decoded = if text_type.allows_html_entities() {
145            decode_entities(&text)
146        } else {
147            text
148        };
149        if self.in_title {
150            self.title.push_str(&decoded);
151        } else if !self.in_head {
152            // Collapsed at capture, so the buffer holds no literal
153            // newline or tab except the structural ones above.
154            self.raw.push_str(&collapse_whitespace(&decoded));
155        }
156    }
157}
158
159/// Strips markup from html bytes and returns the visible text beside
160/// conversion warnings. Shared with the eml converter for html-only
161/// message bodies.
162pub(crate) fn strip_html(source: &[u8]) -> Result<(String, Vec<String>), ConvertError> {
163    strip_html_with_budget(source, MAX_OUTPUT_BYTES)
164}
165
166/// [`strip_html`] with an explicit working budget, so tests can prove
167/// the charging without gigabyte fixtures.
168fn strip_html_with_budget(
169    source: &[u8],
170    budget: usize,
171) -> Result<(String, Vec<String>), ConvertError> {
172    let text_source = std::str::from_utf8(source).map_err(|e| ConvertError {
173        code: "invalid_utf8",
174        message: format!("invalid UTF-8 at byte {}", e.valid_up_to()),
175    })?;
176    let mut warnings = Vec::new();
177    let text_source = match text_source.strip_prefix('\u{feff}') {
178        Some(stripped) => {
179            warnings.push("stripped leading byte order mark".to_string());
180            stripped
181        }
182        None => text_source,
183    };
184
185    let state = Rc::new(RefCell::new(StripState {
186        budget,
187        ..StripState::default()
188    }));
189    let element_state = state.clone();
190    let text_state = state.clone();
191    let settings = Settings::new()
192        .with_memory_settings(
193            MemorySettings::new().with_max_allowed_memory_usage(MAX_PART_BYTES as usize),
194        )
195        .append_element_content_handler(element!("*", move |el| {
196            let tag = el.tag_name();
197            let mut st = element_state.borrow_mut();
198            if st.in_head && !HEAD_CONTENT.contains(&tag.as_str()) {
199                st.in_head = false;
200            }
201            match tag.as_str() {
202                "head" => {
203                    st.in_head = true;
204                    let state = element_state.clone();
205                    drop(st);
206                    el.on_end_tag(end_tag!(move |_| {
207                        state.borrow_mut().in_head = false;
208                        Ok(())
209                    }))?;
210                }
211                "title" if !st.title_done => {
212                    st.in_title = true;
213                    let state = element_state.clone();
214                    drop(st);
215                    el.on_end_tag(end_tag!(move |_| {
216                        let mut st = state.borrow_mut();
217                        st.in_title = false;
218                        st.title_done = true;
219                        Ok(())
220                    }))?;
221                }
222                "template" => {
223                    st.template_depth += 1;
224                    let state = element_state.clone();
225                    drop(st);
226                    el.on_end_tag(end_tag!(move |_| {
227                        let mut st = state.borrow_mut();
228                        st.template_depth = st.template_depth.saturating_sub(1);
229                        Ok(())
230                    }))?;
231                }
232                "br" => st.push_break()?,
233                "td" | "th" => {
234                    if st.row_has_cell {
235                        st.push_tab()?;
236                    } else {
237                        st.row_has_cell = true;
238                    }
239                }
240                block if BLOCK_ELEMENTS.contains(&block) => {
241                    st.push_break()?;
242                    if block == "tr" {
243                        st.row_has_cell = false;
244                    }
245                    let state = element_state.clone();
246                    drop(st);
247                    el.on_end_tag(end_tag!(move |_| {
248                        state.borrow_mut().push_break()?;
249                        Ok(())
250                    }))?;
251                }
252                _ => {}
253            }
254            Ok(())
255        }))
256        .append_document_content_handler(doc_text!(move |chunk: &mut TextChunk| {
257            let mut st = text_state.borrow_mut();
258            let piece = chunk.as_str();
259            st.charge(piece.len())?;
260            if st.node_type.is_none() && !piece.is_empty() {
261                st.node_type = Some(chunk.text_type());
262            }
263            st.node_buf.push_str(piece);
264            if chunk.last_in_text_node() {
265                let text_type = chunk.text_type();
266                st.flush_node(text_type);
267            }
268            Ok(())
269        }));
270
271    let mut rewriter = HtmlRewriter::new(settings, |_: &[u8]| {});
272    rewriter
273        .write(text_source.as_bytes())
274        .and_then(|()| rewriter.end())
275        .map_err(|e| match e {
276            lol_html::errors::RewritingError::MemoryLimitExceeded(_) => ConvertError {
277                code: "resource_limit",
278                message: format!("markup working memory exceeds the {MAX_PART_BYTES} byte ceiling"),
279            },
280            lol_html::errors::RewritingError::ContentHandlerError(detail) => ConvertError {
281                code: "resource_limit",
282                message: detail.to_string(),
283            },
284            other => ConvertError {
285                code: "html_parse_error",
286                message: other.to_string(),
287            },
288        })?;
289
290    let mut state = state.borrow_mut();
291    state.flush_node(TextType::Data);
292    Ok((assemble(&state.title, &state.raw), warnings))
293}
294
295impl Converter for HtmlStrip {
296    fn id(&self) -> &'static str {
297        HTML_STRIP_ID
298    }
299
300    fn version(&self) -> &'static str {
301        HTML_STRIP_VERSION
302    }
303
304    fn convert(
305        &self,
306        source: &[u8],
307        detected_format: &str,
308    ) -> std::result::Result<Outcome, ConvertError> {
309        let (stripped, warnings) = strip_html(source)?;
310        let text = normalize_text(&stripped);
311        if !source.is_empty() && text.is_empty() {
312            return Err(ConvertError {
313                code: "empty_output",
314                message: format!("source is {} bytes but holds no visible text", source.len()),
315            });
316        }
317        let segments = vec![Segment::span(0, text.len(), "document")];
318        Ok(Outcome {
319            artifact_kind: crate::manifest::ArtifactKind::Text,
320            converter_id: HTML_STRIP_ID.to_string(),
321            converter_version: HTML_STRIP_VERSION.to_string(),
322            detected_format: detected_format.to_string(),
323            text,
324            warnings,
325            segments,
326        })
327    }
328}
329
330/// Collapses ASCII whitespace runs to one space. Non-breaking spaces
331/// stay, matching how a browser lays the text out.
332fn collapse_whitespace(text: &str) -> String {
333    let mut out = String::with_capacity(text.len());
334    let mut in_run = false;
335    for c in text.chars() {
336        if matches!(c, ' ' | '\t' | '\n' | '\r' | '\x0c') {
337            if !in_run {
338                out.push(' ');
339                in_run = true;
340            }
341        } else {
342            out.push(c);
343            in_run = false;
344        }
345    }
346    out
347}
348
349/// Renders the captured buffer to the final text: the title line
350/// first, one line per block run, cells joined with tabs, whitespace
351/// collapsed, and blank-line runs reduced to one blank line.
352fn assemble(title: &str, raw: &str) -> String {
353    let mut lines: Vec<String> = Vec::new();
354    for raw_line in raw.split('\n') {
355        let mut cells: Vec<String> = raw_line
356            .split('\t')
357            .map(|cell| collapse_whitespace(cell).trim().to_string())
358            .collect();
359        while cells.last().is_some_and(String::is_empty) && cells.len() > 1 {
360            cells.pop();
361        }
362        lines.push(cells.join("\t"));
363    }
364
365    let mut out = String::new();
366    let mut body_started = false;
367    let mut blank_pending = false;
368    for line in &lines {
369        if line.is_empty() {
370            // A blank line renders only between body lines, so
371            // neither the title nor leading block breaks open the
372            // text with one.
373            blank_pending = body_started;
374            continue;
375        }
376        if blank_pending {
377            out.push('\n');
378            blank_pending = false;
379        }
380        out.push_str(line);
381        out.push('\n');
382        body_started = true;
383    }
384    let title_line = collapse_whitespace(title);
385    let title_line = title_line.trim();
386    if title_line.is_empty() {
387        out
388    } else {
389        format!("{title_line}\n{out}")
390    }
391}
392
393/// Decodes numeric character references and the named references in
394/// [`NAMED_ENTITIES`]. Anything else stays literal.
395pub(crate) fn decode_entities(text: &str) -> String {
396    if !text.contains('&') {
397        return text.to_string();
398    }
399    let mut out = String::with_capacity(text.len());
400    let mut rest = text;
401    while let Some(position) = rest.find('&') {
402        out.push_str(&rest[..position]);
403        rest = &rest[position..];
404        match decode_reference(rest) {
405            Some((decoded, consumed)) => {
406                out.push(decoded);
407                rest = &rest[consumed..];
408            }
409            None => {
410                out.push('&');
411                rest = &rest[1..];
412            }
413        }
414    }
415    out.push_str(rest);
416    out
417}
418
419/// Decodes one reference at the start of `text`, which begins with an
420/// ampersand. Returns the character and the bytes consumed.
421fn decode_reference(text: &str) -> Option<(char, usize)> {
422    let semicolon = text.as_bytes().iter().take(40).position(|&b| b == b';')?;
423    let body = &text[1..semicolon];
424    if let Some(number) = body.strip_prefix('#') {
425        let value = if let Some(hex) = number.strip_prefix(['x', 'X']) {
426            u32::from_str_radix(hex, 16).ok()?
427        } else if number.is_empty() {
428            return None;
429        } else {
430            number.parse::<u32>().ok()?
431        };
432        let decoded = char::from_u32(value)?;
433        if decoded == '\0' {
434            return None;
435        }
436        return Some((decoded, semicolon + 1));
437    }
438    NAMED_ENTITIES
439        .binary_search_by(|(name, _)| (*name).cmp(body))
440        .ok()
441        .map(|index| (NAMED_ENTITIES[index].1, semicolon + 1))
442}
443
444/// Named character references this converter decodes: the XML five,
445/// the Latin-1 set, and the common HTML punctuation and symbol names.
446/// Sorted by name for binary search. An unrecognized reference stays
447/// literal, which is the text as authored.
448///
449/// Known subset, tracked for a later quality pass: extend toward the
450/// full HTML named set, the Greek block first, and accept a
451/// semicolon-less numeric reference.
452const NAMED_ENTITIES: &[(&str, char)] = &[
453    ("AElig", '\u{c6}'),
454    ("Aacute", '\u{c1}'),
455    ("Acirc", '\u{c2}'),
456    ("Agrave", '\u{c0}'),
457    ("Aring", '\u{c5}'),
458    ("Atilde", '\u{c3}'),
459    ("Auml", '\u{c4}'),
460    ("Ccedil", '\u{c7}'),
461    ("Dagger", '\u{2021}'),
462    ("ETH", '\u{d0}'),
463    ("Eacute", '\u{c9}'),
464    ("Ecirc", '\u{ca}'),
465    ("Egrave", '\u{c8}'),
466    ("Euml", '\u{cb}'),
467    ("Iacute", '\u{cd}'),
468    ("Icirc", '\u{ce}'),
469    ("Igrave", '\u{cc}'),
470    ("Iuml", '\u{cf}'),
471    ("Ntilde", '\u{d1}'),
472    ("OElig", '\u{152}'),
473    ("Oacute", '\u{d3}'),
474    ("Ocirc", '\u{d4}'),
475    ("Ograve", '\u{d2}'),
476    ("Oslash", '\u{d8}'),
477    ("Otilde", '\u{d5}'),
478    ("Ouml", '\u{d6}'),
479    ("Prime", '\u{2033}'),
480    ("Scaron", '\u{160}'),
481    ("THORN", '\u{de}'),
482    ("Uacute", '\u{da}'),
483    ("Ucirc", '\u{db}'),
484    ("Ugrave", '\u{d9}'),
485    ("Uuml", '\u{dc}'),
486    ("Yacute", '\u{dd}'),
487    ("Yuml", '\u{178}'),
488    ("aacute", '\u{e1}'),
489    ("acirc", '\u{e2}'),
490    ("acute", '\u{b4}'),
491    ("aelig", '\u{e6}'),
492    ("agrave", '\u{e0}'),
493    ("amp", '\u{26}'),
494    ("apos", '\u{27}'),
495    ("aring", '\u{e5}'),
496    ("atilde", '\u{e3}'),
497    ("auml", '\u{e4}'),
498    ("bdquo", '\u{201e}'),
499    ("brvbar", '\u{a6}'),
500    ("bull", '\u{2022}'),
501    ("ccedil", '\u{e7}'),
502    ("cedil", '\u{b8}'),
503    ("cent", '\u{a2}'),
504    ("circ", '\u{2c6}'),
505    ("copy", '\u{a9}'),
506    ("curren", '\u{a4}'),
507    ("dagger", '\u{2020}'),
508    ("deg", '\u{b0}'),
509    ("divide", '\u{f7}'),
510    ("eacute", '\u{e9}'),
511    ("ecirc", '\u{ea}'),
512    ("egrave", '\u{e8}'),
513    ("emsp", '\u{2003}'),
514    ("ensp", '\u{2002}'),
515    ("eth", '\u{f0}'),
516    ("euml", '\u{eb}'),
517    ("euro", '\u{20ac}'),
518    ("fnof", '\u{192}'),
519    ("frac12", '\u{bd}'),
520    ("frac14", '\u{bc}'),
521    ("frac34", '\u{be}'),
522    ("frasl", '\u{2044}'),
523    ("gt", '\u{3e}'),
524    ("hellip", '\u{2026}'),
525    ("iacute", '\u{ed}'),
526    ("icirc", '\u{ee}'),
527    ("iexcl", '\u{a1}'),
528    ("igrave", '\u{ec}'),
529    ("iquest", '\u{bf}'),
530    ("iuml", '\u{ef}'),
531    ("laquo", '\u{ab}'),
532    ("ldquo", '\u{201c}'),
533    ("lrm", '\u{200e}'),
534    ("lsaquo", '\u{2039}'),
535    ("lsquo", '\u{2018}'),
536    ("lt", '\u{3c}'),
537    ("macr", '\u{af}'),
538    ("mdash", '\u{2014}'),
539    ("micro", '\u{b5}'),
540    ("middot", '\u{b7}'),
541    ("minus", '\u{2212}'),
542    ("nbsp", '\u{a0}'),
543    ("ndash", '\u{2013}'),
544    ("not", '\u{ac}'),
545    ("ntilde", '\u{f1}'),
546    ("oacute", '\u{f3}'),
547    ("ocirc", '\u{f4}'),
548    ("oelig", '\u{153}'),
549    ("ograve", '\u{f2}'),
550    ("oline", '\u{203e}'),
551    ("ordf", '\u{aa}'),
552    ("ordm", '\u{ba}'),
553    ("oslash", '\u{f8}'),
554    ("otilde", '\u{f5}'),
555    ("ouml", '\u{f6}'),
556    ("para", '\u{b6}'),
557    ("permil", '\u{2030}'),
558    ("plusmn", '\u{b1}'),
559    ("pound", '\u{a3}'),
560    ("prime", '\u{2032}'),
561    ("quot", '\u{22}'),
562    ("raquo", '\u{bb}'),
563    ("rdquo", '\u{201d}'),
564    ("reg", '\u{ae}'),
565    ("rlm", '\u{200f}'),
566    ("rsaquo", '\u{203a}'),
567    ("rsquo", '\u{2019}'),
568    ("sbquo", '\u{201a}'),
569    ("scaron", '\u{161}'),
570    ("sect", '\u{a7}'),
571    ("shy", '\u{ad}'),
572    ("sup1", '\u{b9}'),
573    ("sup2", '\u{b2}'),
574    ("sup3", '\u{b3}'),
575    ("szlig", '\u{df}'),
576    ("thinsp", '\u{2009}'),
577    ("thorn", '\u{fe}'),
578    ("tilde", '\u{2dc}'),
579    ("times", '\u{d7}'),
580    ("trade", '\u{2122}'),
581    ("uacute", '\u{fa}'),
582    ("ucirc", '\u{fb}'),
583    ("ugrave", '\u{f9}'),
584    ("uml", '\u{a8}'),
585    ("uuml", '\u{fc}'),
586    ("yacute", '\u{fd}'),
587    ("yen", '\u{a5}'),
588    ("yuml", '\u{ff}'),
589    ("zwj", '\u{200d}'),
590    ("zwnj", '\u{200c}'),
591];
592
593#[cfg(test)]
594mod tests {
595    use super::*;
596
597    // Prototype proof 1: character references decode, and the raw
598    // reference never ships.
599    #[test]
600    fn entities_decode_and_the_raw_reference_never_ships() {
601        let (text, _) =
602            strip_html(b"<p>a &amp; b &#169; &lt;tag&gt; &euro;9 &bogus; x</p>").unwrap();
603        assert_eq!(text, "a & b \u{a9} <tag> \u{20ac}9 &bogus; x\n");
604        assert!(!text.contains("&amp;"));
605        // Sensitivity control: a doubly escaped reference decodes to
606        // the literal reference string, so the leak assertion above
607        // can fire.
608        let (control, _) = strip_html(b"<p>&amp;amp;</p>").unwrap();
609        assert!(control.contains("&amp;"));
610    }
611
612    // Prototype proof 2: suppressed element text never reaches the
613    // capture, a missing closing tag included.
614    #[test]
615    fn suppressed_element_text_never_reaches_the_capture() {
616        let html = b"<p>keep</p><script>var leak = 'scriptleak';</script>\
617            <style>.styleleak{}</style><noscript>noscriptleak</noscript>\
618            <template>templateleak</template><script>runs to eof scriptleaktwo";
619        let (text, _) = strip_html(html).unwrap();
620        assert!(text.contains("keep"));
621        for leak in [
622            "scriptleak",
623            "styleleak",
624            "noscriptleak",
625            "templateleak",
626            "scriptleaktwo",
627        ] {
628            assert!(!text.contains(leak), "{leak} leaked into {text:?}");
629        }
630        // Sensitivity control: the same marker in a paragraph is
631        // captured, so the leak assertions above can fire.
632        let (control, _) = strip_html(b"<p>var leak = 'scriptleak';</p>").unwrap();
633        assert!(control.contains("scriptleak"));
634    }
635
636    // Prototype proof 3: implied closures still yield deterministic
637    // line breaks with every item on its own line.
638    #[test]
639    fn implied_closures_break_lines_deterministically() {
640        let source = b"<ul><li>one<li>two<li>three</ul><p>para";
641        let (first, _) = strip_html(source).unwrap();
642        let (second, _) = strip_html(source).unwrap();
643        assert_eq!(first, second);
644        assert!(first.contains("one\ntwo\nthree"), "{first:?}");
645        assert!(first.contains("para"));
646        // Sensitivity control: without the list markup the same words
647        // stay on one line, so the own-line assertion can fire.
648        let (control, _) = strip_html(b"<p>one two three</p>").unwrap();
649        assert_eq!(control, "one two three\n");
650    }
651
652    // Sectioning and grouping containers are boundaries, so adjacent
653    // blocks never concatenate words.
654    #[test]
655    fn sectioning_containers_break_instead_of_concatenating() {
656        let (text, _) = strip_html(b"<article>alpha</article><section>beta</section>").unwrap();
657        assert_eq!(
658            text,
659            "alpha
660
661beta
662"
663        );
664        assert!(!text.contains("alphabeta"));
665        let (more, _) = strip_html(
666            b"<header>top</header><nav>menu</nav><main>body</main><footer>legal</footer>",
667        )
668        .unwrap();
669        assert_eq!(
670            more,
671            "top
672
673menu
674
675body
676
677legal
678"
679        );
680    }
681
682    // Block breaks and cell tabs charge the same working budget as
683    // captured text, so a dense page of short blocks fails closed
684    // instead of growing converter-owned state without bound.
685    #[test]
686    fn dense_short_blocks_fail_closed_on_the_working_budget() {
687        let page = "<p>x</p>".repeat(64);
688        // 64 blocks charge 64 text bytes plus 128 break bytes.
689        let err = strip_html_with_budget(page.as_bytes(), 100).unwrap_err();
690        assert_eq!(err.code, "resource_limit");
691        assert!(err.message.contains("byte ceiling"), "{}", err.message);
692        // Sensitivity control: the same page inside the budget
693        // converts, so the ceiling is what fired above.
694        let (text, _) = strip_html_with_budget(page.as_bytes(), 4096).unwrap();
695        assert_eq!(text, vec!["x"; 64].join("\n\n") + "\n");
696    }
697
698    #[test]
699    fn title_is_the_first_line_and_head_content_drops() {
700        let html = b"<html><head><meta charset=\"utf-8\"><title>Q3 Plan</title>\
701            <style>.x{}</style></head><body><p>body text</p></body></html>";
702        let (text, _) = strip_html(html).unwrap();
703        assert_eq!(text, "Q3 Plan\nbody text\n");
704    }
705
706    #[test]
707    fn an_unclosed_head_does_not_swallow_the_body() {
708        let (text, _) =
709            strip_html(b"<html><head><title>T</title><p>after an unclosed head").unwrap();
710        assert_eq!(text, "T\nafter an unclosed head\n");
711    }
712
713    #[test]
714    fn paragraphs_separate_with_one_blank_line() {
715        let (text, _) =
716            strip_html(b"<p>first</p>\n\n\n<p>second</p><div><div><p>third</p></div></div>")
717                .unwrap();
718        assert_eq!(text, "first\n\nsecond\n\nthird\n");
719    }
720
721    #[test]
722    fn table_cells_join_with_tabs_within_a_row() {
723        let (text, _) = strip_html(
724            b"<table><tr><td>a</td><td>b</td></tr><tr><td>c</td><td>d</td></tr></table>",
725        )
726        .unwrap();
727        assert!(text.contains("a\tb"), "{text:?}");
728        assert!(text.contains("c\td"), "{text:?}");
729    }
730
731    #[test]
732    fn whitespace_collapses_and_inline_markup_flows() {
733        let (text, _) = strip_html(b"<p>a\n   lot\t of <b>bold</b>\nspace</p>").unwrap();
734        assert_eq!(text, "a lot of bold space\n");
735    }
736
737    #[test]
738    fn invalid_utf8_fails_closed() {
739        let err = strip_html(b"<p>ok \xff</p>").unwrap_err();
740        assert_eq!(err.code, "invalid_utf8");
741    }
742
743    #[test]
744    fn a_page_with_no_visible_text_is_suspiciously_empty() {
745        let err = HtmlStrip
746            .convert(b"<script>var x = 1;</script>", "html")
747            .unwrap_err();
748        assert_eq!(err.code, "empty_output");
749    }
750}
text_mirror/convert/html.rs

text_mirror/convert/
html.rs
