rmut_core/
html.rs

1//! text/html into readable text, rmut's own few hundred lines: mutt
2//! shows html-only mail as raw source unless a mailcap filter is
3//! configured, and machines without lynx/w3m deserve better. This is
4//! the fallback the pager uses by default (`[pager] html = "raw"`
5//! restores mutt's literal behavior); a `[filters]`/mailcap
6//! auto_view entry still wins, since it is consulted first.
7//!
8//! The shape of the output, not fidelity, is the goal: block
9//! elements break lines, `<blockquote>` becomes `> ` prefixes so the
10//! quote machinery works on html mail, links keep their targets as
11//! `text <url>` so the window can click them, and everything inside
12//! `<style>`/`<script>`/`<head>` disappears. Unknown tags are
13//! ignored rather than feared; broken html must never panic.
14
15/// The rendered text, one trailing newline, never more than one
16/// blank line in a row.
17pub fn to_text(html: &str) -> String {
18    let mut out = Render::default();
19    let mut rest = html;
20    while !rest.is_empty() {
21        match rest.find('<') {
22            None => {
23                out.text(rest);
24                break;
25            }
26            Some(i) => {
27                out.text(&rest[..i]);
28                rest = &rest[i..];
29                match tag_end(rest) {
30                    Some(end) => {
31                        out.tag(&rest[..end]);
32                        rest = &rest[end..];
33                    }
34                    None => {
35                        // A stray '<' with no closing '>': literal.
36                        out.text("<");
37                        rest = &rest[1..];
38                    }
39                }
40            }
41        }
42    }
43    out.finish()
44}
45
46/// Where the tag starting at `s` (which begins with '<') ends, past
47/// its '>': quotes protect '>', comments end at `-->`.
48fn tag_end(s: &str) -> Option<usize> {
49    if let Some(rest) = s.strip_prefix("<!--") {
50        return rest.find("-->").map(|i| 4 + i + 3);
51    }
52    let bytes = s.as_bytes();
53    let mut quote: Option<u8> = None;
54    for (i, &b) in bytes.iter().enumerate().skip(1) {
55        match quote {
56            Some(q) => {
57                if b == q {
58                    quote = None;
59                }
60            }
61            None => match b {
62                b'"' | b'\'' => quote = Some(b),
63                b'>' => return Some(i + 1),
64                _ => {}
65            },
66        }
67    }
68    None
69}
70
71#[derive(Default)]
72struct Render {
73    out: String,
74    /// `<blockquote>` depth: each line starts with this many `> `.
75    quote: usize,
76    /// Open lists: an `<ol>` counts its items, a `<ul>` does not.
77    lists: Vec<Option<usize>>,
78    /// `<pre>` depth: whitespace kept verbatim inside.
79    pre: usize,
80    /// Inside `<style>`/`<script>`/`<head>`...: text is dropped
81    /// until this tag closes (the innermost such tag).
82    skip: Vec<String>,
83    /// Open `<a href>` targets, said after their anchor text.
84    links: Vec<(String, usize)>,
85    /// A collapsed space is owed before the next word.
86    space: bool,
87    /// A blank line is owed before the next text (paragraph break).
88    blank: bool,
89}
90
91/// Elements whose content is machinery, not prose.
92const SKIPPED: &[&str] = &[
93    "style", "script", "head", "title", "template", "svg", "noscript",
94];
95
96/// Elements that break the line without a blank (a row, a cell run).
97const LINE_BREAKERS: &[&str] = &[
98    "div", "tr", "section", "article", "aside", "main", "figure", "nav", "form", "header", "footer",
99];
100
101/// Elements that earn a blank line around them.
102const PARA_BREAKERS: &[&str] = &[
103    "p",
104    "h1",
105    "h2",
106    "h3",
107    "h4",
108    "h5",
109    "h6",
110    "table",
111    "ul",
112    "ol",
113    "pre",
114    "blockquote",
115];
116
117impl Render {
118    fn text(&mut self, s: &str) {
119        if !self.skip.is_empty() {
120            return;
121        }
122        if self.pre > 0 {
123            let decoded = decode_entities(s);
124            for (i, line) in decoded.split('\n').enumerate() {
125                if i > 0 {
126                    self.newline();
127                }
128                self.raw(line);
129            }
130            return;
131        }
132        if s.starts_with(char::is_whitespace) {
133            self.space = true;
134        }
135        for piece in s.split_whitespace() {
136            if self.space && !self.at_line_start() {
137                self.raw(" ");
138            }
139            let word = decode_entities(piece);
140            self.raw(&word);
141            self.space = true;
142        }
143        // Trailing whitespace in the run keeps the pending space for
144        // the next run ("a <b>b</b>" has a space, "a<b>b</b>" not).
145        if !s.is_empty() {
146            self.space = s.ends_with(char::is_whitespace) || self.space;
147            if !s.trim().is_empty() && !s.ends_with(char::is_whitespace) {
148                self.space = false;
149            }
150        }
151    }
152
153    fn tag(&mut self, tag: &str) {
154        if tag.starts_with("<!") || tag.starts_with("<?") {
155            return;
156        }
157        let inner = tag.trim_start_matches('<').trim_end_matches('>');
158        let closing = inner.starts_with('/');
159        let inner = inner.trim_start_matches('/');
160        let name: String = inner
161            .chars()
162            .take_while(|c| c.is_ascii_alphanumeric())
163            .collect::<String>()
164            .to_lowercase();
165        if name.is_empty() {
166            return;
167        }
168        // Inside a skipped element, only its own closing tag matters.
169        if let Some(top) = self.skip.last() {
170            if closing && *top == name {
171                self.skip.pop();
172            }
173            return;
174        }
175        if SKIPPED.contains(&name.as_str()) {
176            if !closing && !inner.ends_with('/') {
177                self.skip.push(name);
178            }
179            return;
180        }
181        match (name.as_str(), closing) {
182            ("br", _) => self.newline(),
183            ("hr", _) => {
184                self.want_blank();
185                self.flush_break();
186                self.raw("--");
187                self.want_blank();
188            }
189            ("blockquote", false) => {
190                self.want_blank();
191                self.quote += 1;
192            }
193            ("blockquote", true) => {
194                self.quote = self.quote.saturating_sub(1);
195                self.want_blank();
196            }
197            ("ul", false) => {
198                self.want_blank();
199                self.lists.push(None);
200            }
201            ("ol", false) => {
202                self.want_blank();
203                self.lists.push(Some(0));
204            }
205            ("ul", true) | ("ol", true) => {
206                self.lists.pop();
207                if self.lists.is_empty() {
208                    self.want_blank();
209                }
210            }
211            ("li", false) => {
212                self.fresh_line();
213                let depth = self.lists.len().saturating_sub(1);
214                let marker = match self.lists.last_mut() {
215                    Some(Some(n)) => {
216                        *n += 1;
217                        format!("{n}. ")
218                    }
219                    _ => "- ".to_string(),
220                };
221                self.raw(&"  ".repeat(depth));
222                self.raw(&marker);
223                self.space = false;
224            }
225            ("pre", false) => {
226                self.want_blank();
227                self.flush_break();
228                self.pre += 1;
229            }
230            ("pre", true) => {
231                self.pre = self.pre.saturating_sub(1);
232                self.want_blank();
233            }
234            ("a", false) => {
235                let href = attr(inner, "href").unwrap_or_default();
236                self.links.push((href, self.out.len()));
237            }
238            ("a", true) => {
239                if let Some((href, start)) = self.links.pop() {
240                    let text = &self.out[start.min(self.out.len())..];
241                    if worth_showing(&href, text) {
242                        let href = href.clone();
243                        self.text(&format!(" <{href}>"));
244                    }
245                }
246            }
247            ("img", _) => {
248                if let Some(alt) = attr(inner, "alt")
249                    && !alt.trim().is_empty()
250                {
251                    let alt = alt.clone();
252                    self.text(&format!("[{alt}]"));
253                }
254            }
255            ("td", false) | ("th", false) => {
256                if !self.at_line_start() {
257                    self.raw("  ");
258                    self.space = false;
259                }
260            }
261            (n, _) if PARA_BREAKERS.contains(&n) => self.want_blank(),
262            (n, _) if LINE_BREAKERS.contains(&n) => self.fresh_line(),
263            _ => {}
264        }
265    }
266
267    /// Text goes through here so line starts get their quote prefix
268    /// and an owed paragraph break lands first.
269    fn raw(&mut self, s: &str) {
270        if s.is_empty() {
271            return;
272        }
273        self.flush_break();
274        if self.at_line_start() && self.quote > 0 {
275            let prefix = "> ".repeat(self.quote);
276            self.out.push_str(&prefix);
277        }
278        self.out.push_str(s);
279    }
280
281    fn at_line_start(&self) -> bool {
282        self.out.is_empty() || self.out.ends_with('\n')
283    }
284
285    /// A hard line break (`<br>`): breaks even at a line start, so
286    /// two of them make a blank line.
287    fn newline(&mut self) {
288        self.flush_break();
289        self.out.push('\n');
290        self.space = false;
291    }
292
293    /// Break to a fresh line without stacking: a `</tr><tr>` pair or
294    /// a list item wants one break, however many tags said so.
295    fn fresh_line(&mut self) {
296        self.flush_break();
297        if !self.at_line_start() {
298            self.out.push('\n');
299        }
300        self.space = false;
301    }
302
303    fn want_blank(&mut self) {
304        if !self.out.is_empty() {
305            self.blank = true;
306        }
307        self.space = false;
308    }
309
310    /// An owed paragraph break becomes at most one blank line.
311    fn flush_break(&mut self) {
312        if std::mem::take(&mut self.blank) {
313            while !self.out.is_empty() && !self.out.ends_with("\n\n") {
314                self.out.push('\n');
315            }
316        }
317    }
318
319    fn finish(mut self) -> String {
320        let mut text: String = self
321            .out
322            .split('\n')
323            .map(str::trim_end)
324            .collect::<Vec<_>>()
325            .join("\n");
326        while text.ends_with('\n') {
327            text.pop();
328        }
329        let trimmed = text.trim_start_matches('\n').to_string();
330        self.out = trimmed;
331        if self.out.is_empty() {
332            return String::new();
333        }
334        self.out.push('\n');
335        self.out
336    }
337}
338
339/// A link target worth saying after its anchor text: not empty, not
340/// a fragment or script, and not already the text itself.
341fn worth_showing(href: &str, text: &str) -> bool {
342    let href = href.trim();
343    if href.is_empty() || href.starts_with('#') || href.to_lowercase().starts_with("javascript:") {
344        return false;
345    }
346    let text = text.trim();
347    !text.contains(href.trim_end_matches('/'))
348}
349
350/// The value of `name=` in a tag's innards, quotes stripped.
351fn attr(inner: &str, name: &str) -> Option<String> {
352    let lower = inner.to_lowercase();
353    let mut from = 0;
354    loop {
355        let i = lower[from..].find(name)? + from;
356        // A whole attribute name, not the tail of another.
357        let clean_start = i == 0
358            || !lower.as_bytes()[i - 1].is_ascii_alphanumeric() && lower.as_bytes()[i - 1] != b'-';
359        let after = &inner[i + name.len()..];
360        let after_trim = after.trim_start();
361        if clean_start && after_trim.starts_with('=') {
362            let value = after_trim[1..].trim_start();
363            let value = match value.as_bytes().first() {
364                Some(b'"') => value[1..].split('"').next().unwrap_or(""),
365                Some(b'\'') => value[1..].split('\'').next().unwrap_or(""),
366                _ => value.split_whitespace().next().unwrap_or(""),
367            };
368            return Some(decode_entities(value));
369        }
370        from = i + name.len();
371        if from >= lower.len() {
372            return None;
373        }
374    }
375}
376
377/// `&amp;` and friends, plus numeric `&#123;` / `&#x1f;`. An entity
378/// this table does not know stays literal.
379fn decode_entities(s: &str) -> String {
380    if !s.contains('&') {
381        return s.to_string();
382    }
383    let mut out = String::with_capacity(s.len());
384    let mut rest = s;
385    while let Some(i) = rest.find('&') {
386        out.push_str(&rest[..i]);
387        rest = &rest[i..];
388        let semi = rest[..rest.len().min(32)].find(';');
389        let Some(semi) = semi else {
390            out.push('&');
391            rest = &rest[1..];
392            continue;
393        };
394        let name = &rest[1..semi];
395        let decoded = match name {
396            "amp" => Some('&'),
397            "lt" => Some('<'),
398            "gt" => Some('>'),
399            "quot" => Some('"'),
400            "apos" | "#39" => Some('\''),
401            "nbsp" => Some(' '),
402            "copy" => Some('\u{a9}'),
403            "reg" => Some('\u{ae}'),
404            "trade" => Some('\u{2122}'),
405            "mdash" => Some('\u{2014}'),
406            "ndash" => Some('\u{2013}'),
407            "hellip" => Some('\u{2026}'),
408            "lsquo" => Some('\u{2018}'),
409            "rsquo" => Some('\u{2019}'),
410            "ldquo" => Some('\u{201c}'),
411            "rdquo" => Some('\u{201d}'),
412            "bull" => Some('\u{2022}'),
413            "middot" => Some('\u{b7}'),
414            "laquo" => Some('\u{ab}'),
415            "raquo" => Some('\u{bb}'),
416            "deg" => Some('\u{b0}'),
417            "times" => Some('\u{d7}'),
418            "euro" => Some('\u{20ac}'),
419            "pound" => Some('\u{a3}'),
420            "shy" | "zwnj" | "zwj" | "lrm" | "rlm" => Some('\u{0}'),
421            _ => name
422                .strip_prefix("#x")
423                .or_else(|| name.strip_prefix("#X"))
424                .and_then(|h| u32::from_str_radix(h, 16).ok())
425                .or_else(|| name.strip_prefix('#').and_then(|d| d.parse().ok()))
426                .and_then(char::from_u32),
427        };
428        match decoded {
429            Some('\u{0}') => rest = &rest[semi + 1..],
430            Some(c) => {
431                out.push(c);
432                rest = &rest[semi + 1..];
433            }
434            None => {
435                out.push('&');
436                rest = &rest[1..];
437            }
438        }
439    }
440    out.push_str(rest);
441    out
442}
443
444#[cfg(test)]
445mod tests {
446    use super::*;
447
448    #[test]
449    fn paragraphs_and_headings_break_lines() {
450        let text = to_text("<html><body><h1>Title</h1><p>one two</p>\n<p>three</p></body></html>");
451        assert_eq!(text, "Title\n\none two\n\nthree\n");
452    }
453
454    #[test]
455    fn style_script_and_head_disappear() {
456        let text = to_text(
457            "<head><title>t</title><style>p { color: red }</style></head>\
458             <body><script>alert(1)</script><p>kept</p></body>",
459        );
460        assert_eq!(text, "kept\n");
461    }
462
463    #[test]
464    fn entities_decode_and_unknown_stay() {
465        let text = to_text("<p>Hello &amp; goodbye &lt;3 &#65; &#x42; &unknown; &rsquo;</p>");
466        assert_eq!(text, "Hello & goodbye <3 A B &unknown; \u{2019}\n");
467    }
468
469    #[test]
470    fn links_keep_their_targets() {
471        let text = to_text("<p>see <a href=\"https://example.com/x\">the docs</a> now</p>");
472        assert_eq!(text, "see the docs <https://example.com/x> now\n");
473        // The text already saying the target says it once.
474        let text = to_text("<a href=\"https://example.com\">https://example.com</a>");
475        assert_eq!(text, "https://example.com\n");
476        // Fragments and scripts are not links worth words.
477        let text = to_text("<a href=\"#top\">up</a> <a href=\"javascript:x()\">no</a>");
478        assert_eq!(text, "up no\n");
479    }
480
481    #[test]
482    fn blockquotes_become_quote_prefixes() {
483        let text = to_text(
484            "<p>said:</p><blockquote>first<br>second\
485             <blockquote>deeper</blockquote></blockquote><p>after</p>",
486        );
487        assert_eq!(text, "said:\n\n> first\n> second\n\n> > deeper\n\nafter\n");
488    }
489
490    #[test]
491    fn lists_get_markers_and_numbers() {
492        let text = to_text("<ul><li>one</li><li>two<ol><li>a</li><li>b</li></ol></li></ul>");
493        assert_eq!(text, "- one\n- two\n\n  1. a\n  2. b\n");
494    }
495
496    #[test]
497    fn pre_keeps_its_whitespace() {
498        let text = to_text("<p>code:</p><pre>  indented\n    more</pre>");
499        assert_eq!(text, "code:\n\n  indented\n    more\n");
500    }
501
502    #[test]
503    fn tables_space_their_cells() {
504        let text =
505            to_text("<table><tr><th>a</th><th>b</th></tr><tr><td>1</td><td>2</td></tr></table>");
506        assert_eq!(text, "a  b\n1  2\n");
507    }
508
509    #[test]
510    fn images_say_their_alt_text() {
511        let text = to_text("<p><img src=\"cid:x\" alt=\"a chart\"> and <img src=\"y\"></p>");
512        assert_eq!(text, "[a chart] and\n");
513    }
514
515    #[test]
516    fn broken_html_never_panics() {
517        for bad in [
518            "<",
519            "<p",
520            "<a href=\"unclosed>text",
521            "</closes><nothing",
522            "<p>&#xffffffff; &#; &",
523            "<blockquote></blockquote></blockquote>",
524            "<!-- unterminated",
525            "text < 3 and > 2",
526        ] {
527            let _ = to_text(bad);
528        }
529        assert_eq!(to_text("a < b"), "a < b\n");
530    }
531}
rmut_core/html.rs

rmut_core/
html.rs
