text_mirror/convert/
html.rs

1//! Strip-to-text conversion for html.
2//!
3//! A streaming pass over the markup collects the visible text and
4//! nothing else. Script, style, template, and noscript content is
5//! suppressed, head content other than the title is dropped, and the
6//! title text becomes the first line. Block boundaries become line
7//! breaks: the ruled set, `p`, `div`, `li`, `tr`, `br`, the headings,
8//! `blockquote`, and `pre`, plus the HTML5 sectioning and grouping
9//! containers, and table cells within a row separate with a tab.
10//! Character references decode, whitespace runs collapse to one
11//! space, and blank-line runs collapse to one blank line. Reading
12//! order and table reconstruction are out of scope.
13//!
14//! The stream never builds a document tree, so memory stays bounded
15//! under hostile nesting, and a run over the memory ceiling or the
16//! working budget fails closed. Captured text, block breaks, and cell
17//! tabs all charge one working budget, so a dense page of short
18//! blocks cannot grow converter-owned state past the order of the
19//! output ceiling. Suppression of script, style, and
20//! noscript rides the lexer's own text typing rather than end-tag
21//! bookkeeping, so a missing closing tag cannot leak script text into
22//! the capture.
23
24use std::cell::RefCell;
25use std::rc::Rc;
26
27use lol_html::html_content::{TextChunk, TextType};
28use lol_html::{HtmlRewriter, MemorySettings, Settings, doc_text, element, end_tag};
29
30use super::{ConvertError, Converter, MAX_OUTPUT_BYTES, MAX_PART_BYTES, Outcome, normalize_text};
31use crate::segments::Segment;
32
33/// Registry id of the html strip converter.
34pub const HTML_STRIP_ID: &str = "html-strip";
35/// Version of the html strip converter.
36pub const HTML_STRIP_VERSION: &str = "1.0.0";
37
38/// Strip-to-text converter for html.
39pub struct HtmlStrip;
40
41/// Elements whose start tag does not imply the head has ended.
42///
43/// The stream has no tree builder, so an unclosed `<head>` would
44/// otherwise swallow the whole document. The first start tag outside
45/// this set ends head suppression, which mirrors the implied close a
46/// tree builder performs.
47const HEAD_CONTENT: &[&str] = &[
48    "base", "basefont", "bgsound", "head", "html", "link", "meta", "noscript", "script", "style",
49    "template", "title",
50];
51
52/// The block set: the ruled elements plus the HTML5 sectioning and
53/// grouping containers, so adjacent blocks never concatenate words.
54/// Start and end both mark a line break.
55const BLOCK_ELEMENTS: &[&str] = &[
56    "address",
57    "article",
58    "aside",
59    "blockquote",
60    "dd",
61    "div",
62    "dl",
63    "dt",
64    "figure",
65    "footer",
66    "h1",
67    "h2",
68    "h3",
69    "h4",
70    "h5",
71    "h6",
72    "header",
73    "li",
74    "main",
75    "nav",
76    "ol",
77    "p",
78    "pre",
79    "section",
80    "table",
81    "tr",
82    "ul",
83];
84
85#[derive(Default)]
86struct StripState {
87    /// Captured content: collapsed text, `\n` for block boundaries,
88    /// `\t` for cell separators. One flat buffer, so the working
89    /// memory stays on the order of the budget.
90    raw: String,
91    title: String,
92    in_title: bool,
93    title_done: bool,
94    in_head: bool,
95    template_depth: u32,
96    row_has_cell: bool,
97    node_buf: String,
98    node_type: Option<TextType>,
99    charged_bytes: usize,
100    budget: usize,
101}
102
103impl StripState {
104    /// Charges bytes against the working budget. Text, block breaks,
105    /// and cell tabs all pay, so no shape of markup grows the
106    /// converter's own memory past the order of the budget.
107    fn charge(&mut self, bytes: usize) -> std::result::Result<(), String> {
108        self.charged_bytes += bytes;
109        if self.charged_bytes > self.budget {
110            Err(format!(
111                "captured content exceeds the {} byte ceiling",
112                self.budget
113            ))
114        } else {
115            Ok(())
116        }
117    }
118
119    fn push_break(&mut self) -> std::result::Result<(), String> {
120        self.charge(1)?;
121        self.raw.push('\n');
122        Ok(())
123    }
124
125    fn push_tab(&mut self) -> std::result::Result<(), String> {
126        self.charge(1)?;
127        self.raw.push('\t');
128        Ok(())
129    }
130
131    fn flush_node(&mut self, fallback_type: TextType) {
132        let text = std::mem::take(&mut self.node_buf);
133        let text_type = self.node_type.take().unwrap_or(fallback_type);
134        if text.is_empty() || self.template_depth > 0 {
135            return;
136        }
137        let keep = matches!(
138            text_type,
139            TextType::Data | TextType::RCData | TextType::PlainText
140        );
141        if !keep {
142            return;
143        }
144        let decoded = if text_type.allows_html_entities() {
145            decode_entities(&text)
146        } else {
147            text
148        };
149        if self.in_title {
150            self.title.push_str(&decoded);
151        } else if !self.in_head {
152            // Collapsed at capture, so the buffer holds no literal
153            // newline or tab except the structural ones above.
154            self.raw.push_str(&collapse_whitespace(&decoded));
155        }
156    }
157}
158
159/// Strips markup from html bytes and returns the visible text beside
160/// conversion warnings. Shared with the eml converter for html-only
161/// message bodies.
162pub(crate) fn strip_html(source: &[u8]) -> Result<(String, Vec<String>), ConvertError> {
163    strip_html_with_budget(source, MAX_OUTPUT_BYTES)
164}
165
166/// [`strip_html`] with an explicit working budget, so tests can prove
167/// the charging without gigabyte fixtures.
168fn strip_html_with_budget(
169    source: &[u8],
170    budget: usize,
171) -> Result<(String, Vec<String>), ConvertError> {
172    let text_source = std::str::from_utf8(source).map_err(|e| ConvertError {
173        code: "invalid_utf8",
174        message: format!("invalid UTF-8 at byte {}", e.valid_up_to()),
175    })?;
176    let mut warnings = Vec::new();
177    let text_source = match text_source.strip_prefix('\u{feff}') {
178        Some(stripped) => {
179            warnings.push("stripped leading byte order mark".to_string());
180            stripped
181        }
182        None => text_source,
183    };
184
185    let state = Rc::new(RefCell::new(StripState {
186        budget,
187        ..StripState::default()
188    }));
189    let element_state = state.clone();
190    let text_state = state.clone();
191    let settings = Settings::new()
192        .with_memory_settings(
193            MemorySettings::new().with_max_allowed_memory_usage(MAX_PART_BYTES as usize),
194        )
195        .append_element_content_handler(element!("*", move |el| {
196            let tag = el.tag_name();
197            let mut st = element_state.borrow_mut();
198            if st.in_head && !HEAD_CONTENT.contains(&tag.as_str()) {
199                st.in_head = false;
200            }
201            match tag.as_str() {
202                "head" => {
203                    st.in_head = true;
204                    let state = element_state.clone();
205                    drop(st);
206                    el.on_end_tag(end_tag!(move |_| {
207                        state.borrow_mut().in_head = false;
208                        Ok(())
209                    }))?;
210                }
211                "title" if !st.title_done => {
212                    st.in_title = true;
213                    let state = element_state.clone();
214                    drop(st);
215                    el.on_end_tag(end_tag!(move |_| {
216                        let mut st = state.borrow_mut();
217                        st.in_title = false;
218                        st.title_done = true;
219                        Ok(())
220                    }))?;
221                }
222                "template" => {
223                    st.template_depth += 1;
224                    let state = element_state.clone();
225                    drop(st);
226                    el.on_end_tag(end_tag!(move |_| {
227                        let mut st = state.borrow_mut();
228                        st.template_depth = st.template_depth.saturating_sub(1);
229                        Ok(())
230                    }))?;
231                }
232                "br" => st.push_break()?,
233                "td" | "th" => {
234                    if st.row_has_cell {
235                        st.push_tab()?;
236                    } else {
237                        st.row_has_cell = true;
238                    }
239                }
240                block if BLOCK_ELEMENTS.contains(&block) => {
241                    st.push_break()?;
242                    if block == "tr" {
243                        st.row_has_cell = false;
244                    }
245                    let state = element_state.clone();
246                    drop(st);
247                    el.on_end_tag(end_tag!(move |_| {
248                        state.borrow_mut().push_break()?;
249                        Ok(())
250                    }))?;
251                }
252                _ => {}
253            }
254            Ok(())
255        }))
256        .append_document_content_handler(doc_text!(move |chunk: &mut TextChunk| {
257            let mut st = text_state.borrow_mut();
258            let piece = chunk.as_str();
259            st.charge(piece.len())?;
260            if st.node_type.is_none() && !piece.is_empty() {
261                st.node_type = Some(chunk.text_type());
262            }
263            st.node_buf.push_str(piece);
264            if chunk.last_in_text_node() {
265                let text_type = chunk.text_type();
266                st.flush_node(text_type);
267            }
268            Ok(())
269        }));
270
271    let mut rewriter = HtmlRewriter::new(settings, |_: &[u8]| {});
272    rewriter
273        .write(text_source.as_bytes())
274        .and_then(|()| rewriter.end())
275        .map_err(|e| match e {
276            lol_html::errors::RewritingError::MemoryLimitExceeded(_) => ConvertError {
277                code: "resource_limit",
278                message: format!("markup working memory exceeds the {MAX_PART_BYTES} byte ceiling"),
279            },
280            lol_html::errors::RewritingError::ContentHandlerError(detail) => ConvertError {
281                code: "resource_limit",
282                message: detail.to_string(),
283            },
284            other => ConvertError {
285                code: "html_parse_error",
286                message: other.to_string(),
287            },
288        })?;
289
290    let mut state = state.borrow_mut();
291    state.flush_node(TextType::Data);
292    Ok((assemble(&state.title, &state.raw), warnings))
293}
294
295impl Converter for HtmlStrip {
296    fn id(&self) -> &'static str {
297        HTML_STRIP_ID
298    }
299
300    fn version(&self) -> &'static str {
301        HTML_STRIP_VERSION
302    }
303
304    fn convert(
305        &self,
306        source: &[u8],
307        detected_format: &str,
308    ) -> std::result::Result<Outcome, ConvertError> {
309        let (stripped, warnings) = strip_html(source)?;
310        let text = normalize_text(&stripped);
311        if !source.is_empty() && text.is_empty() {
312            return Err(ConvertError {
313                code: "empty_output",
314                message: format!("source is {} bytes but holds no visible text", source.len()),
315            });
316        }
317        let segments = vec![Segment::span(0, text.len(), "document")];
318        Ok(Outcome {
319            artifact_kind: crate::manifest::ArtifactKind::Text,
320            converter_id: HTML_STRIP_ID.to_string(),
321            converter_version: HTML_STRIP_VERSION.to_string(),
322            detected_format: detected_format.to_string(),
323            text,
324            warnings,
325            segments,
326            media: None,
327        })
328    }
329}
330
331/// Collapses ASCII whitespace runs to one space. Non-breaking spaces
332/// stay, matching how a browser lays the text out.
333fn collapse_whitespace(text: &str) -> String {
334    let mut out = String::with_capacity(text.len());
335    let mut in_run = false;
336    for c in text.chars() {
337        if matches!(c, ' ' | '\t' | '\n' | '\r' | '\x0c') {
338            if !in_run {
339                out.push(' ');
340                in_run = true;
341            }
342        } else {
343            out.push(c);
344            in_run = false;
345        }
346    }
347    out
348}
349
350/// Renders the captured buffer to the final text: the title line
351/// first, one line per block run, cells joined with tabs, whitespace
352/// collapsed, and blank-line runs reduced to one blank line.
353fn assemble(title: &str, raw: &str) -> String {
354    let mut lines: Vec<String> = Vec::new();
355    for raw_line in raw.split('\n') {
356        let mut cells: Vec<String> = raw_line
357            .split('\t')
358            .map(|cell| collapse_whitespace(cell).trim().to_string())
359            .collect();
360        while cells.last().is_some_and(String::is_empty) && cells.len() > 1 {
361            cells.pop();
362        }
363        lines.push(cells.join("\t"));
364    }
365
366    let mut out = String::new();
367    let mut body_started = false;
368    let mut blank_pending = false;
369    for line in &lines {
370        if line.is_empty() {
371            // A blank line renders only between body lines, so
372            // neither the title nor leading block breaks open the
373            // text with one.
374            blank_pending = body_started;
375            continue;
376        }
377        if blank_pending {
378            out.push('\n');
379            blank_pending = false;
380        }
381        out.push_str(line);
382        out.push('\n');
383        body_started = true;
384    }
385    let title_line = collapse_whitespace(title);
386    let title_line = title_line.trim();
387    if title_line.is_empty() {
388        out
389    } else {
390        format!("{title_line}\n{out}")
391    }
392}
393
394/// Decodes numeric character references and the named references in
395/// [`NAMED_ENTITIES`]. Anything else stays literal.
396pub(crate) fn decode_entities(text: &str) -> String {
397    if !text.contains('&') {
398        return text.to_string();
399    }
400    let mut out = String::with_capacity(text.len());
401    let mut rest = text;
402    while let Some(position) = rest.find('&') {
403        out.push_str(&rest[..position]);
404        rest = &rest[position..];
405        match decode_reference(rest) {
406            Some((decoded, consumed)) => {
407                out.push(decoded);
408                rest = &rest[consumed..];
409            }
410            None => {
411                out.push('&');
412                rest = &rest[1..];
413            }
414        }
415    }
416    out.push_str(rest);
417    out
418}
419
420/// Decodes one reference at the start of `text`, which begins with an
421/// ampersand. Returns the character and the bytes consumed.
422fn decode_reference(text: &str) -> Option<(char, usize)> {
423    let semicolon = text.as_bytes().iter().take(40).position(|&b| b == b';')?;
424    let body = &text[1..semicolon];
425    if let Some(number) = body.strip_prefix('#') {
426        let value = if let Some(hex) = number.strip_prefix(['x', 'X']) {
427            u32::from_str_radix(hex, 16).ok()?
428        } else if number.is_empty() {
429            return None;
430        } else {
431            number.parse::<u32>().ok()?
432        };
433        let decoded = char::from_u32(value)?;
434        if decoded == '\0' {
435            return None;
436        }
437        return Some((decoded, semicolon + 1));
438    }
439    NAMED_ENTITIES
440        .binary_search_by(|(name, _)| (*name).cmp(body))
441        .ok()
442        .map(|index| (NAMED_ENTITIES[index].1, semicolon + 1))
443}
444
445/// Named character references this converter decodes: the XML five,
446/// the Latin-1 set, and the common HTML punctuation and symbol names.
447/// Sorted by name for binary search. An unrecognized reference stays
448/// literal, which is the text as authored.
449///
450/// Known subset, tracked for a later quality pass: extend toward the
451/// full HTML named set, the Greek block first, and accept a
452/// semicolon-less numeric reference.
453const NAMED_ENTITIES: &[(&str, char)] = &[
454    ("AElig", '\u{c6}'),
455    ("Aacute", '\u{c1}'),
456    ("Acirc", '\u{c2}'),
457    ("Agrave", '\u{c0}'),
458    ("Aring", '\u{c5}'),
459    ("Atilde", '\u{c3}'),
460    ("Auml", '\u{c4}'),
461    ("Ccedil", '\u{c7}'),
462    ("Dagger", '\u{2021}'),
463    ("ETH", '\u{d0}'),
464    ("Eacute", '\u{c9}'),
465    ("Ecirc", '\u{ca}'),
466    ("Egrave", '\u{c8}'),
467    ("Euml", '\u{cb}'),
468    ("Iacute", '\u{cd}'),
469    ("Icirc", '\u{ce}'),
470    ("Igrave", '\u{cc}'),
471    ("Iuml", '\u{cf}'),
472    ("Ntilde", '\u{d1}'),
473    ("OElig", '\u{152}'),
474    ("Oacute", '\u{d3}'),
475    ("Ocirc", '\u{d4}'),
476    ("Ograve", '\u{d2}'),
477    ("Oslash", '\u{d8}'),
478    ("Otilde", '\u{d5}'),
479    ("Ouml", '\u{d6}'),
480    ("Prime", '\u{2033}'),
481    ("Scaron", '\u{160}'),
482    ("THORN", '\u{de}'),
483    ("Uacute", '\u{da}'),
484    ("Ucirc", '\u{db}'),
485    ("Ugrave", '\u{d9}'),
486    ("Uuml", '\u{dc}'),
487    ("Yacute", '\u{dd}'),
488    ("Yuml", '\u{178}'),
489    ("aacute", '\u{e1}'),
490    ("acirc", '\u{e2}'),
491    ("acute", '\u{b4}'),
492    ("aelig", '\u{e6}'),
493    ("agrave", '\u{e0}'),
494    ("amp", '\u{26}'),
495    ("apos", '\u{27}'),
496    ("aring", '\u{e5}'),
497    ("atilde", '\u{e3}'),
498    ("auml", '\u{e4}'),
499    ("bdquo", '\u{201e}'),
500    ("brvbar", '\u{a6}'),
501    ("bull", '\u{2022}'),
502    ("ccedil", '\u{e7}'),
503    ("cedil", '\u{b8}'),
504    ("cent", '\u{a2}'),
505    ("circ", '\u{2c6}'),
506    ("copy", '\u{a9}'),
507    ("curren", '\u{a4}'),
508    ("dagger", '\u{2020}'),
509    ("deg", '\u{b0}'),
510    ("divide", '\u{f7}'),
511    ("eacute", '\u{e9}'),
512    ("ecirc", '\u{ea}'),
513    ("egrave", '\u{e8}'),
514    ("emsp", '\u{2003}'),
515    ("ensp", '\u{2002}'),
516    ("eth", '\u{f0}'),
517    ("euml", '\u{eb}'),
518    ("euro", '\u{20ac}'),
519    ("fnof", '\u{192}'),
520    ("frac12", '\u{bd}'),
521    ("frac14", '\u{bc}'),
522    ("frac34", '\u{be}'),
523    ("frasl", '\u{2044}'),
524    ("gt", '\u{3e}'),
525    ("hellip", '\u{2026}'),
526    ("iacute", '\u{ed}'),
527    ("icirc", '\u{ee}'),
528    ("iexcl", '\u{a1}'),
529    ("igrave", '\u{ec}'),
530    ("iquest", '\u{bf}'),
531    ("iuml", '\u{ef}'),
532    ("laquo", '\u{ab}'),
533    ("ldquo", '\u{201c}'),
534    ("lrm", '\u{200e}'),
535    ("lsaquo", '\u{2039}'),
536    ("lsquo", '\u{2018}'),
537    ("lt", '\u{3c}'),
538    ("macr", '\u{af}'),
539    ("mdash", '\u{2014}'),
540    ("micro", '\u{b5}'),
541    ("middot", '\u{b7}'),
542    ("minus", '\u{2212}'),
543    ("nbsp", '\u{a0}'),
544    ("ndash", '\u{2013}'),
545    ("not", '\u{ac}'),
546    ("ntilde", '\u{f1}'),
547    ("oacute", '\u{f3}'),
548    ("ocirc", '\u{f4}'),
549    ("oelig", '\u{153}'),
550    ("ograve", '\u{f2}'),
551    ("oline", '\u{203e}'),
552    ("ordf", '\u{aa}'),
553    ("ordm", '\u{ba}'),
554    ("oslash", '\u{f8}'),
555    ("otilde", '\u{f5}'),
556    ("ouml", '\u{f6}'),
557    ("para", '\u{b6}'),
558    ("permil", '\u{2030}'),
559    ("plusmn", '\u{b1}'),
560    ("pound", '\u{a3}'),
561    ("prime", '\u{2032}'),
562    ("quot", '\u{22}'),
563    ("raquo", '\u{bb}'),
564    ("rdquo", '\u{201d}'),
565    ("reg", '\u{ae}'),
566    ("rlm", '\u{200f}'),
567    ("rsaquo", '\u{203a}'),
568    ("rsquo", '\u{2019}'),
569    ("sbquo", '\u{201a}'),
570    ("scaron", '\u{161}'),
571    ("sect", '\u{a7}'),
572    ("shy", '\u{ad}'),
573    ("sup1", '\u{b9}'),
574    ("sup2", '\u{b2}'),
575    ("sup3", '\u{b3}'),
576    ("szlig", '\u{df}'),
577    ("thinsp", '\u{2009}'),
578    ("thorn", '\u{fe}'),
579    ("tilde", '\u{2dc}'),
580    ("times", '\u{d7}'),
581    ("trade", '\u{2122}'),
582    ("uacute", '\u{fa}'),
583    ("ucirc", '\u{fb}'),
584    ("ugrave", '\u{f9}'),
585    ("uml", '\u{a8}'),
586    ("uuml", '\u{fc}'),
587    ("yacute", '\u{fd}'),
588    ("yen", '\u{a5}'),
589    ("yuml", '\u{ff}'),
590    ("zwj", '\u{200d}'),
591    ("zwnj", '\u{200c}'),
592];
593
594#[cfg(test)]
595mod tests {
596    use super::*;
597
598    // Prototype proof 1: character references decode, and the raw
599    // reference never ships.
600    #[test]
601    fn entities_decode_and_the_raw_reference_never_ships() {
602        let (text, _) =
603            strip_html(b"<p>a &amp; b &#169; &lt;tag&gt; &euro;9 &bogus; x</p>").unwrap();
604        assert_eq!(text, "a & b \u{a9} <tag> \u{20ac}9 &bogus; x\n");
605        assert!(!text.contains("&amp;"));
606        // Sensitivity control: a doubly escaped reference decodes to
607        // the literal reference string, so the leak assertion above
608        // can fire.
609        let (control, _) = strip_html(b"<p>&amp;amp;</p>").unwrap();
610        assert!(control.contains("&amp;"));
611    }
612
613    // Prototype proof 2: suppressed element text never reaches the
614    // capture, a missing closing tag included.
615    #[test]
616    fn suppressed_element_text_never_reaches_the_capture() {
617        let html = b"<p>keep</p><script>var leak = 'scriptleak';</script>\
618            <style>.styleleak{}</style><noscript>noscriptleak</noscript>\
619            <template>templateleak</template><script>runs to eof scriptleaktwo";
620        let (text, _) = strip_html(html).unwrap();
621        assert!(text.contains("keep"));
622        for leak in [
623            "scriptleak",
624            "styleleak",
625            "noscriptleak",
626            "templateleak",
627            "scriptleaktwo",
628        ] {
629            assert!(!text.contains(leak), "{leak} leaked into {text:?}");
630        }
631        // Sensitivity control: the same marker in a paragraph is
632        // captured, so the leak assertions above can fire.
633        let (control, _) = strip_html(b"<p>var leak = 'scriptleak';</p>").unwrap();
634        assert!(control.contains("scriptleak"));
635    }
636
637    // Prototype proof 3: implied closures still yield deterministic
638    // line breaks with every item on its own line.
639    #[test]
640    fn implied_closures_break_lines_deterministically() {
641        let source = b"<ul><li>one<li>two<li>three</ul><p>para";
642        let (first, _) = strip_html(source).unwrap();
643        let (second, _) = strip_html(source).unwrap();
644        assert_eq!(first, second);
645        assert!(first.contains("one\ntwo\nthree"), "{first:?}");
646        assert!(first.contains("para"));
647        // Sensitivity control: without the list markup the same words
648        // stay on one line, so the own-line assertion can fire.
649        let (control, _) = strip_html(b"<p>one two three</p>").unwrap();
650        assert_eq!(control, "one two three\n");
651    }
652
653    // Sectioning and grouping containers are boundaries, so adjacent
654    // blocks never concatenate words.
655    #[test]
656    fn sectioning_containers_break_instead_of_concatenating() {
657        let (text, _) = strip_html(b"<article>alpha</article><section>beta</section>").unwrap();
658        assert_eq!(
659            text,
660            "alpha
661
662beta
663"
664        );
665        assert!(!text.contains("alphabeta"));
666        let (more, _) = strip_html(
667            b"<header>top</header><nav>menu</nav><main>body</main><footer>legal</footer>",
668        )
669        .unwrap();
670        assert_eq!(
671            more,
672            "top
673
674menu
675
676body
677
678legal
679"
680        );
681    }
682
683    // Block breaks and cell tabs charge the same working budget as
684    // captured text, so a dense page of short blocks fails closed
685    // instead of growing converter-owned state without bound.
686    #[test]
687    fn dense_short_blocks_fail_closed_on_the_working_budget() {
688        let page = "<p>x</p>".repeat(64);
689        // 64 blocks charge 64 text bytes plus 128 break bytes.
690        let err = strip_html_with_budget(page.as_bytes(), 100).unwrap_err();
691        assert_eq!(err.code, "resource_limit");
692        assert!(err.message.contains("byte ceiling"), "{}", err.message);
693        // Sensitivity control: the same page inside the budget
694        // converts, so the ceiling is what fired above.
695        let (text, _) = strip_html_with_budget(page.as_bytes(), 4096).unwrap();
696        assert_eq!(text, vec!["x"; 64].join("\n\n") + "\n");
697    }
698
699    #[test]
700    fn title_is_the_first_line_and_head_content_drops() {
701        let html = b"<html><head><meta charset=\"utf-8\"><title>Q3 Plan</title>\
702            <style>.x{}</style></head><body><p>body text</p></body></html>";
703        let (text, _) = strip_html(html).unwrap();
704        assert_eq!(text, "Q3 Plan\nbody text\n");
705    }
706
707    #[test]
708    fn an_unclosed_head_does_not_swallow_the_body() {
709        let (text, _) =
710            strip_html(b"<html><head><title>T</title><p>after an unclosed head").unwrap();
711        assert_eq!(text, "T\nafter an unclosed head\n");
712    }
713
714    #[test]
715    fn paragraphs_separate_with_one_blank_line() {
716        let (text, _) =
717            strip_html(b"<p>first</p>\n\n\n<p>second</p><div><div><p>third</p></div></div>")
718                .unwrap();
719        assert_eq!(text, "first\n\nsecond\n\nthird\n");
720    }
721
722    #[test]
723    fn table_cells_join_with_tabs_within_a_row() {
724        let (text, _) = strip_html(
725            b"<table><tr><td>a</td><td>b</td></tr><tr><td>c</td><td>d</td></tr></table>",
726        )
727        .unwrap();
728        assert!(text.contains("a\tb"), "{text:?}");
729        assert!(text.contains("c\td"), "{text:?}");
730    }
731
732    #[test]
733    fn whitespace_collapses_and_inline_markup_flows() {
734        let (text, _) = strip_html(b"<p>a\n   lot\t of <b>bold</b>\nspace</p>").unwrap();
735        assert_eq!(text, "a lot of bold space\n");
736    }
737
738    #[test]
739    fn invalid_utf8_fails_closed() {
740        let err = strip_html(b"<p>ok \xff</p>").unwrap_err();
741        assert_eq!(err.code, "invalid_utf8");
742    }
743
744    #[test]
745    fn a_page_with_no_visible_text_is_suspiciously_empty() {
746        let err = HtmlStrip
747            .convert(b"<script>var x = 1;</script>", "html")
748            .unwrap_err();
749        assert_eq!(err.code, "empty_output");
750    }
751}
text_mirror/convert/html.rs

text_mirror/convert/
html.rs
