text_mirror/convert/
html.rs

1//! Strip-to-text conversion for html.
2//!
3//! A streaming pass over the markup collects the visible text and
4//! nothing else. Script, style, template, and noscript content is
5//! suppressed, head content other than the title is dropped, and the
6//! title text becomes the first line. Block boundaries become line
7//! breaks: the ruled set, `p`, `div`, `li`, `tr`, `br`, the headings,
8//! `blockquote`, and `pre`, plus the HTML5 sectioning and grouping
9//! containers, and table cells within a row separate with a tab.
10//! Character references decode, whitespace runs collapse to one
11//! space, and blank-line runs collapse to one blank line. Reading
12//! order and table reconstruction are out of scope.
13//!
14//! The stream never builds a document tree, so memory stays bounded
15//! under hostile nesting, and a run over the memory ceiling or the
16//! working budget fails closed. Captured text, block breaks, and cell
17//! tabs all charge one working budget, so a dense page of short
18//! blocks cannot grow converter-owned state past the order of the
19//! output ceiling. Suppression of script, style, and
20//! noscript rides the lexer's own text typing rather than end-tag
21//! bookkeeping, so a missing closing tag cannot leak script text into
22//! the capture.
23
24use std::cell::RefCell;
25use std::rc::Rc;
26
27use lol_html::html_content::{TextChunk, TextType};
28use lol_html::{HtmlRewriter, MemorySettings, Settings, doc_text, element, end_tag};
29
30use super::{ConvertError, Converter, MAX_OUTPUT_BYTES, MAX_PART_BYTES, Outcome, normalize_text};
31use crate::segments::Segment;
32
33/// Registry id of the html strip converter.
34pub const HTML_STRIP_ID: &str = "html-strip";
35/// Version of the html strip converter.
36pub const HTML_STRIP_VERSION: &str = "1.0.0";
37
38/// Strip-to-text converter for html.
39pub struct HtmlStrip;
40
41/// Elements whose start tag does not imply the head has ended.
42///
43/// The stream has no tree builder, so an unclosed `<head>` would
44/// otherwise swallow the whole document. The first start tag outside
45/// this set ends head suppression, which mirrors the implied close a
46/// tree builder performs.
47const HEAD_CONTENT: &[&str] = &[
48    "base", "basefont", "bgsound", "head", "html", "link", "meta", "noscript", "script", "style",
49    "template", "title",
50];
51
52/// The block set: the ruled elements plus the HTML5 sectioning and
53/// grouping containers, so adjacent blocks never concatenate words.
54/// Start and end both mark a line break.
55const BLOCK_ELEMENTS: &[&str] = &[
56    "address",
57    "article",
58    "aside",
59    "blockquote",
60    "dd",
61    "div",
62    "dl",
63    "dt",
64    "figure",
65    "footer",
66    "h1",
67    "h2",
68    "h3",
69    "h4",
70    "h5",
71    "h6",
72    "header",
73    "li",
74    "main",
75    "nav",
76    "ol",
77    "p",
78    "pre",
79    "section",
80    "table",
81    "tr",
82    "ul",
83];
84
85#[derive(Default)]
86struct StripState {
87    /// Captured content: collapsed text, `\n` for block boundaries,
88    /// `\t` for cell separators. One flat buffer, so the working
89    /// memory stays on the order of the budget.
90    raw: String,
91    title: String,
92    in_title: bool,
93    title_done: bool,
94    in_head: bool,
95    template_depth: u32,
96    row_has_cell: bool,
97    node_buf: String,
98    node_type: Option<TextType>,
99    charged_bytes: usize,
100    budget: usize,
101}
102
103impl StripState {
104    /// Charges bytes against the working budget. Text, block breaks,
105    /// and cell tabs all pay, so no shape of markup grows the
106    /// converter's own memory past the order of the budget.
107    fn charge(&mut self, bytes: usize) -> std::result::Result<(), String> {
108        self.charged_bytes += bytes;
109        if self.charged_bytes > self.budget {
110            Err(format!(
111                "captured content exceeds the {} byte ceiling",
112                self.budget
113            ))
114        } else {
115            Ok(())
116        }
117    }
118
119    fn push_break(&mut self) -> std::result::Result<(), String> {
120        self.charge(1)?;
121        self.raw.push('\n');
122        Ok(())
123    }
124
125    fn push_tab(&mut self) -> std::result::Result<(), String> {
126        self.charge(1)?;
127        self.raw.push('\t');
128        Ok(())
129    }
130
131    fn flush_node(&mut self, fallback_type: TextType) {
132        let text = std::mem::take(&mut self.node_buf);
133        let text_type = self.node_type.take().unwrap_or(fallback_type);
134        if text.is_empty() || self.template_depth > 0 {
135            return;
136        }
137        let keep = matches!(
138            text_type,
139            TextType::Data | TextType::RCData | TextType::PlainText
140        );
141        if !keep {
142            return;
143        }
144        let decoded = if text_type.allows_html_entities() {
145            decode_entities(&text)
146        } else {
147            text
148        };
149        if self.in_title {
150            self.title.push_str(&decoded);
151        } else if !self.in_head {
152            // Collapsed at capture, so the buffer holds no literal
153            // newline or tab except the structural ones above.
154            self.raw.push_str(&collapse_whitespace(&decoded));
155        }
156    }
157}
158
159/// Strips markup from html bytes and returns the visible text beside
160/// conversion warnings. Shared with the eml converter for html-only
161/// message bodies.
162pub(crate) fn strip_html(source: &[u8]) -> Result<(String, Vec<String>), ConvertError> {
163    strip_html_with_budget(source, MAX_OUTPUT_BYTES)
164}
165
166/// [`strip_html`] with an explicit working budget, so tests can prove
167/// the charging without gigabyte fixtures.
168fn strip_html_with_budget(
169    source: &[u8],
170    budget: usize,
171) -> Result<(String, Vec<String>), ConvertError> {
172    let text_source = std::str::from_utf8(source).map_err(|e| ConvertError {
173        code: "invalid_utf8",
174        message: format!("invalid UTF-8 at byte {}", e.valid_up_to()),
175    })?;
176    let mut warnings = Vec::new();
177    let text_source = match text_source.strip_prefix('\u{feff}') {
178        Some(stripped) => {
179            warnings.push("stripped leading byte order mark".to_string());
180            stripped
181        }
182        None => text_source,
183    };
184
185    let state = Rc::new(RefCell::new(StripState {
186        budget,
187        ..StripState::default()
188    }));
189    let element_state = state.clone();
190    let text_state = state.clone();
191    let settings = Settings::new()
192        .with_memory_settings(
193            MemorySettings::new().with_max_allowed_memory_usage(MAX_PART_BYTES as usize),
194        )
195        .append_element_content_handler(element!("*", move |el| {
196            let tag = el.tag_name();
197            let mut st = element_state.borrow_mut();
198            if st.in_head && !HEAD_CONTENT.contains(&tag.as_str()) {
199                st.in_head = false;
200            }
201            match tag.as_str() {
202                "head" => {
203                    st.in_head = true;
204                    let state = element_state.clone();
205                    drop(st);
206                    el.on_end_tag(end_tag!(move |_| {
207                        state.borrow_mut().in_head = false;
208                        Ok(())
209                    }))?;
210                }
211                "title" if !st.title_done => {
212                    st.in_title = true;
213                    let state = element_state.clone();
214                    drop(st);
215                    el.on_end_tag(end_tag!(move |_| {
216                        let mut st = state.borrow_mut();
217                        st.in_title = false;
218                        st.title_done = true;
219                        Ok(())
220                    }))?;
221                }
222                "template" => {
223                    st.template_depth += 1;
224                    let state = element_state.clone();
225                    drop(st);
226                    el.on_end_tag(end_tag!(move |_| {
227                        let mut st = state.borrow_mut();
228                        st.template_depth = st.template_depth.saturating_sub(1);
229                        Ok(())
230                    }))?;
231                }
232                "br" => st.push_break()?,
233                "td" | "th" => {
234                    if st.row_has_cell {
235                        st.push_tab()?;
236                    } else {
237                        st.row_has_cell = true;
238                    }
239                }
240                block if BLOCK_ELEMENTS.contains(&block) => {
241                    st.push_break()?;
242                    if block == "tr" {
243                        st.row_has_cell = false;
244                    }
245                    let state = element_state.clone();
246                    drop(st);
247                    el.on_end_tag(end_tag!(move |_| {
248                        state.borrow_mut().push_break()?;
249                        Ok(())
250                    }))?;
251                }
252                _ => {}
253            }
254            Ok(())
255        }))
256        .append_document_content_handler(doc_text!(move |chunk: &mut TextChunk| {
257            let mut st = text_state.borrow_mut();
258            let piece = chunk.as_str();
259            st.charge(piece.len())?;
260            if st.node_type.is_none() && !piece.is_empty() {
261                st.node_type = Some(chunk.text_type());
262            }
263            st.node_buf.push_str(piece);
264            if chunk.last_in_text_node() {
265                let text_type = chunk.text_type();
266                st.flush_node(text_type);
267            }
268            Ok(())
269        }));
270
271    let mut rewriter = HtmlRewriter::new(settings, |_: &[u8]| {});
272    rewriter
273        .write(text_source.as_bytes())
274        .and_then(|()| rewriter.end())
275        .map_err(|e| match e {
276            lol_html::errors::RewritingError::MemoryLimitExceeded(_) => ConvertError {
277                code: "resource_limit",
278                message: format!("markup working memory exceeds the {MAX_PART_BYTES} byte ceiling"),
279            },
280            lol_html::errors::RewritingError::ContentHandlerError(detail) => ConvertError {
281                code: "resource_limit",
282                message: detail.to_string(),
283            },
284            other => ConvertError {
285                code: "html_parse_error",
286                message: other.to_string(),
287            },
288        })?;
289
290    let mut state = state.borrow_mut();
291    state.flush_node(TextType::Data);
292    Ok((assemble(&state.title, &state.raw), warnings))
293}
294
295impl Converter for HtmlStrip {
296    fn id(&self) -> &'static str {
297        HTML_STRIP_ID
298    }
299
300    fn version(&self) -> &'static str {
301        HTML_STRIP_VERSION
302    }
303
304    fn convert(
305        &self,
306        source: &[u8],
307        detected_format: &str,
308    ) -> std::result::Result<Outcome, ConvertError> {
309        let (stripped, warnings) = strip_html(source)?;
310        let text = normalize_text(&stripped);
311        if !source.is_empty() && text.is_empty() {
312            return Err(ConvertError {
313                code: "empty_output",
314                message: format!("source is {} bytes but holds no visible text", source.len()),
315            });
316        }
317        let segments = vec![Segment::span(0, text.len(), "document")];
318        Ok(Outcome {
319            converter_id: HTML_STRIP_ID.to_string(),
320            converter_version: HTML_STRIP_VERSION.to_string(),
321            detected_format: detected_format.to_string(),
322            text,
323            warnings,
324            segments,
325        })
326    }
327}
328
329/// Collapses ASCII whitespace runs to one space. Non-breaking spaces
330/// stay, matching how a browser lays the text out.
331fn collapse_whitespace(text: &str) -> String {
332    let mut out = String::with_capacity(text.len());
333    let mut in_run = false;
334    for c in text.chars() {
335        if matches!(c, ' ' | '\t' | '\n' | '\r' | '\x0c') {
336            if !in_run {
337                out.push(' ');
338                in_run = true;
339            }
340        } else {
341            out.push(c);
342            in_run = false;
343        }
344    }
345    out
346}
347
348/// Renders the captured buffer to the final text: the title line
349/// first, one line per block run, cells joined with tabs, whitespace
350/// collapsed, and blank-line runs reduced to one blank line.
351fn assemble(title: &str, raw: &str) -> String {
352    let mut lines: Vec<String> = Vec::new();
353    for raw_line in raw.split('\n') {
354        let mut cells: Vec<String> = raw_line
355            .split('\t')
356            .map(|cell| collapse_whitespace(cell).trim().to_string())
357            .collect();
358        while cells.last().is_some_and(String::is_empty) && cells.len() > 1 {
359            cells.pop();
360        }
361        lines.push(cells.join("\t"));
362    }
363
364    let mut out = String::new();
365    let mut body_started = false;
366    let mut blank_pending = false;
367    for line in &lines {
368        if line.is_empty() {
369            // A blank line renders only between body lines, so
370            // neither the title nor leading block breaks open the
371            // text with one.
372            blank_pending = body_started;
373            continue;
374        }
375        if blank_pending {
376            out.push('\n');
377            blank_pending = false;
378        }
379        out.push_str(line);
380        out.push('\n');
381        body_started = true;
382    }
383    let title_line = collapse_whitespace(title);
384    let title_line = title_line.trim();
385    if title_line.is_empty() {
386        out
387    } else {
388        format!("{title_line}\n{out}")
389    }
390}
391
392/// Decodes numeric character references and the named references in
393/// [`NAMED_ENTITIES`]. Anything else stays literal.
394pub(crate) fn decode_entities(text: &str) -> String {
395    if !text.contains('&') {
396        return text.to_string();
397    }
398    let mut out = String::with_capacity(text.len());
399    let mut rest = text;
400    while let Some(position) = rest.find('&') {
401        out.push_str(&rest[..position]);
402        rest = &rest[position..];
403        match decode_reference(rest) {
404            Some((decoded, consumed)) => {
405                out.push(decoded);
406                rest = &rest[consumed..];
407            }
408            None => {
409                out.push('&');
410                rest = &rest[1..];
411            }
412        }
413    }
414    out.push_str(rest);
415    out
416}
417
418/// Decodes one reference at the start of `text`, which begins with an
419/// ampersand. Returns the character and the bytes consumed.
420fn decode_reference(text: &str) -> Option<(char, usize)> {
421    let semicolon = text.as_bytes().iter().take(40).position(|&b| b == b';')?;
422    let body = &text[1..semicolon];
423    if let Some(number) = body.strip_prefix('#') {
424        let value = if let Some(hex) = number.strip_prefix(['x', 'X']) {
425            u32::from_str_radix(hex, 16).ok()?
426        } else if number.is_empty() {
427            return None;
428        } else {
429            number.parse::<u32>().ok()?
430        };
431        let decoded = char::from_u32(value)?;
432        if decoded == '\0' {
433            return None;
434        }
435        return Some((decoded, semicolon + 1));
436    }
437    NAMED_ENTITIES
438        .binary_search_by(|(name, _)| (*name).cmp(body))
439        .ok()
440        .map(|index| (NAMED_ENTITIES[index].1, semicolon + 1))
441}
442
443/// Named character references this converter decodes: the XML five,
444/// the Latin-1 set, and the common HTML punctuation and symbol names.
445/// Sorted by name for binary search. An unrecognized reference stays
446/// literal, which is the text as authored.
447///
448/// Known subset, tracked for a later quality pass: extend toward the
449/// full HTML named set, the Greek block first, and accept a
450/// semicolon-less numeric reference.
451const NAMED_ENTITIES: &[(&str, char)] = &[
452    ("AElig", '\u{c6}'),
453    ("Aacute", '\u{c1}'),
454    ("Acirc", '\u{c2}'),
455    ("Agrave", '\u{c0}'),
456    ("Aring", '\u{c5}'),
457    ("Atilde", '\u{c3}'),
458    ("Auml", '\u{c4}'),
459    ("Ccedil", '\u{c7}'),
460    ("Dagger", '\u{2021}'),
461    ("ETH", '\u{d0}'),
462    ("Eacute", '\u{c9}'),
463    ("Ecirc", '\u{ca}'),
464    ("Egrave", '\u{c8}'),
465    ("Euml", '\u{cb}'),
466    ("Iacute", '\u{cd}'),
467    ("Icirc", '\u{ce}'),
468    ("Igrave", '\u{cc}'),
469    ("Iuml", '\u{cf}'),
470    ("Ntilde", '\u{d1}'),
471    ("OElig", '\u{152}'),
472    ("Oacute", '\u{d3}'),
473    ("Ocirc", '\u{d4}'),
474    ("Ograve", '\u{d2}'),
475    ("Oslash", '\u{d8}'),
476    ("Otilde", '\u{d5}'),
477    ("Ouml", '\u{d6}'),
478    ("Prime", '\u{2033}'),
479    ("Scaron", '\u{160}'),
480    ("THORN", '\u{de}'),
481    ("Uacute", '\u{da}'),
482    ("Ucirc", '\u{db}'),
483    ("Ugrave", '\u{d9}'),
484    ("Uuml", '\u{dc}'),
485    ("Yacute", '\u{dd}'),
486    ("Yuml", '\u{178}'),
487    ("aacute", '\u{e1}'),
488    ("acirc", '\u{e2}'),
489    ("acute", '\u{b4}'),
490    ("aelig", '\u{e6}'),
491    ("agrave", '\u{e0}'),
492    ("amp", '\u{26}'),
493    ("apos", '\u{27}'),
494    ("aring", '\u{e5}'),
495    ("atilde", '\u{e3}'),
496    ("auml", '\u{e4}'),
497    ("bdquo", '\u{201e}'),
498    ("brvbar", '\u{a6}'),
499    ("bull", '\u{2022}'),
500    ("ccedil", '\u{e7}'),
501    ("cedil", '\u{b8}'),
502    ("cent", '\u{a2}'),
503    ("circ", '\u{2c6}'),
504    ("copy", '\u{a9}'),
505    ("curren", '\u{a4}'),
506    ("dagger", '\u{2020}'),
507    ("deg", '\u{b0}'),
508    ("divide", '\u{f7}'),
509    ("eacute", '\u{e9}'),
510    ("ecirc", '\u{ea}'),
511    ("egrave", '\u{e8}'),
512    ("emsp", '\u{2003}'),
513    ("ensp", '\u{2002}'),
514    ("eth", '\u{f0}'),
515    ("euml", '\u{eb}'),
516    ("euro", '\u{20ac}'),
517    ("fnof", '\u{192}'),
518    ("frac12", '\u{bd}'),
519    ("frac14", '\u{bc}'),
520    ("frac34", '\u{be}'),
521    ("frasl", '\u{2044}'),
522    ("gt", '\u{3e}'),
523    ("hellip", '\u{2026}'),
524    ("iacute", '\u{ed}'),
525    ("icirc", '\u{ee}'),
526    ("iexcl", '\u{a1}'),
527    ("igrave", '\u{ec}'),
528    ("iquest", '\u{bf}'),
529    ("iuml", '\u{ef}'),
530    ("laquo", '\u{ab}'),
531    ("ldquo", '\u{201c}'),
532    ("lrm", '\u{200e}'),
533    ("lsaquo", '\u{2039}'),
534    ("lsquo", '\u{2018}'),
535    ("lt", '\u{3c}'),
536    ("macr", '\u{af}'),
537    ("mdash", '\u{2014}'),
538    ("micro", '\u{b5}'),
539    ("middot", '\u{b7}'),
540    ("minus", '\u{2212}'),
541    ("nbsp", '\u{a0}'),
542    ("ndash", '\u{2013}'),
543    ("not", '\u{ac}'),
544    ("ntilde", '\u{f1}'),
545    ("oacute", '\u{f3}'),
546    ("ocirc", '\u{f4}'),
547    ("oelig", '\u{153}'),
548    ("ograve", '\u{f2}'),
549    ("oline", '\u{203e}'),
550    ("ordf", '\u{aa}'),
551    ("ordm", '\u{ba}'),
552    ("oslash", '\u{f8}'),
553    ("otilde", '\u{f5}'),
554    ("ouml", '\u{f6}'),
555    ("para", '\u{b6}'),
556    ("permil", '\u{2030}'),
557    ("plusmn", '\u{b1}'),
558    ("pound", '\u{a3}'),
559    ("prime", '\u{2032}'),
560    ("quot", '\u{22}'),
561    ("raquo", '\u{bb}'),
562    ("rdquo", '\u{201d}'),
563    ("reg", '\u{ae}'),
564    ("rlm", '\u{200f}'),
565    ("rsaquo", '\u{203a}'),
566    ("rsquo", '\u{2019}'),
567    ("sbquo", '\u{201a}'),
568    ("scaron", '\u{161}'),
569    ("sect", '\u{a7}'),
570    ("shy", '\u{ad}'),
571    ("sup1", '\u{b9}'),
572    ("sup2", '\u{b2}'),
573    ("sup3", '\u{b3}'),
574    ("szlig", '\u{df}'),
575    ("thinsp", '\u{2009}'),
576    ("thorn", '\u{fe}'),
577    ("tilde", '\u{2dc}'),
578    ("times", '\u{d7}'),
579    ("trade", '\u{2122}'),
580    ("uacute", '\u{fa}'),
581    ("ucirc", '\u{fb}'),
582    ("ugrave", '\u{f9}'),
583    ("uml", '\u{a8}'),
584    ("uuml", '\u{fc}'),
585    ("yacute", '\u{fd}'),
586    ("yen", '\u{a5}'),
587    ("yuml", '\u{ff}'),
588    ("zwj", '\u{200d}'),
589    ("zwnj", '\u{200c}'),
590];
591
592#[cfg(test)]
593mod tests {
594    use super::*;
595
596    // Prototype proof 1: character references decode, and the raw
597    // reference never ships.
598    #[test]
599    fn entities_decode_and_the_raw_reference_never_ships() {
600        let (text, _) =
601            strip_html(b"<p>a &amp; b &#169; &lt;tag&gt; &euro;9 &bogus; x</p>").unwrap();
602        assert_eq!(text, "a & b \u{a9} <tag> \u{20ac}9 &bogus; x\n");
603        assert!(!text.contains("&amp;"));
604        // Sensitivity control: a doubly escaped reference decodes to
605        // the literal reference string, so the leak assertion above
606        // can fire.
607        let (control, _) = strip_html(b"<p>&amp;amp;</p>").unwrap();
608        assert!(control.contains("&amp;"));
609    }
610
611    // Prototype proof 2: suppressed element text never reaches the
612    // capture, a missing closing tag included.
613    #[test]
614    fn suppressed_element_text_never_reaches_the_capture() {
615        let html = b"<p>keep</p><script>var leak = 'scriptleak';</script>\
616            <style>.styleleak{}</style><noscript>noscriptleak</noscript>\
617            <template>templateleak</template><script>runs to eof scriptleaktwo";
618        let (text, _) = strip_html(html).unwrap();
619        assert!(text.contains("keep"));
620        for leak in [
621            "scriptleak",
622            "styleleak",
623            "noscriptleak",
624            "templateleak",
625            "scriptleaktwo",
626        ] {
627            assert!(!text.contains(leak), "{leak} leaked into {text:?}");
628        }
629        // Sensitivity control: the same marker in a paragraph is
630        // captured, so the leak assertions above can fire.
631        let (control, _) = strip_html(b"<p>var leak = 'scriptleak';</p>").unwrap();
632        assert!(control.contains("scriptleak"));
633    }
634
635    // Prototype proof 3: implied closures still yield deterministic
636    // line breaks with every item on its own line.
637    #[test]
638    fn implied_closures_break_lines_deterministically() {
639        let source = b"<ul><li>one<li>two<li>three</ul><p>para";
640        let (first, _) = strip_html(source).unwrap();
641        let (second, _) = strip_html(source).unwrap();
642        assert_eq!(first, second);
643        assert!(first.contains("one\ntwo\nthree"), "{first:?}");
644        assert!(first.contains("para"));
645        // Sensitivity control: without the list markup the same words
646        // stay on one line, so the own-line assertion can fire.
647        let (control, _) = strip_html(b"<p>one two three</p>").unwrap();
648        assert_eq!(control, "one two three\n");
649    }
650
651    // Sectioning and grouping containers are boundaries, so adjacent
652    // blocks never concatenate words.
653    #[test]
654    fn sectioning_containers_break_instead_of_concatenating() {
655        let (text, _) = strip_html(b"<article>alpha</article><section>beta</section>").unwrap();
656        assert_eq!(
657            text,
658            "alpha
659
660beta
661"
662        );
663        assert!(!text.contains("alphabeta"));
664        let (more, _) = strip_html(
665            b"<header>top</header><nav>menu</nav><main>body</main><footer>legal</footer>",
666        )
667        .unwrap();
668        assert_eq!(
669            more,
670            "top
671
672menu
673
674body
675
676legal
677"
678        );
679    }
680
681    // Block breaks and cell tabs charge the same working budget as
682    // captured text, so a dense page of short blocks fails closed
683    // instead of growing converter-owned state without bound.
684    #[test]
685    fn dense_short_blocks_fail_closed_on_the_working_budget() {
686        let page = "<p>x</p>".repeat(64);
687        // 64 blocks charge 64 text bytes plus 128 break bytes.
688        let err = strip_html_with_budget(page.as_bytes(), 100).unwrap_err();
689        assert_eq!(err.code, "resource_limit");
690        assert!(err.message.contains("byte ceiling"), "{}", err.message);
691        // Sensitivity control: the same page inside the budget
692        // converts, so the ceiling is what fired above.
693        let (text, _) = strip_html_with_budget(page.as_bytes(), 4096).unwrap();
694        assert_eq!(text, vec!["x"; 64].join("\n\n") + "\n");
695    }
696
697    #[test]
698    fn title_is_the_first_line_and_head_content_drops() {
699        let html = b"<html><head><meta charset=\"utf-8\"><title>Q3 Plan</title>\
700            <style>.x{}</style></head><body><p>body text</p></body></html>";
701        let (text, _) = strip_html(html).unwrap();
702        assert_eq!(text, "Q3 Plan\nbody text\n");
703    }
704
705    #[test]
706    fn an_unclosed_head_does_not_swallow_the_body() {
707        let (text, _) =
708            strip_html(b"<html><head><title>T</title><p>after an unclosed head").unwrap();
709        assert_eq!(text, "T\nafter an unclosed head\n");
710    }
711
712    #[test]
713    fn paragraphs_separate_with_one_blank_line() {
714        let (text, _) =
715            strip_html(b"<p>first</p>\n\n\n<p>second</p><div><div><p>third</p></div></div>")
716                .unwrap();
717        assert_eq!(text, "first\n\nsecond\n\nthird\n");
718    }
719
720    #[test]
721    fn table_cells_join_with_tabs_within_a_row() {
722        let (text, _) = strip_html(
723            b"<table><tr><td>a</td><td>b</td></tr><tr><td>c</td><td>d</td></tr></table>",
724        )
725        .unwrap();
726        assert!(text.contains("a\tb"), "{text:?}");
727        assert!(text.contains("c\td"), "{text:?}");
728    }
729
730    #[test]
731    fn whitespace_collapses_and_inline_markup_flows() {
732        let (text, _) = strip_html(b"<p>a\n   lot\t of <b>bold</b>\nspace</p>").unwrap();
733        assert_eq!(text, "a lot of bold space\n");
734    }
735
736    #[test]
737    fn invalid_utf8_fails_closed() {
738        let err = strip_html(b"<p>ok \xff</p>").unwrap_err();
739        assert_eq!(err.code, "invalid_utf8");
740    }
741
742    #[test]
743    fn a_page_with_no_visible_text_is_suspiciously_empty() {
744        let err = HtmlStrip
745            .convert(b"<script>var x = 1;</script>", "html")
746            .unwrap_err();
747        assert_eq!(err.code, "empty_output");
748    }
749}
text_mirror/convert/html.rs

text_mirror/convert/
html.rs
