mail_parser/decoders/html/
mod.rs

1/*
2 * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <hello@stalw.art>
3 *
4 * SPDX-License-Identifier: Apache-2.0 OR MIT
5 */
6
7//! HTML to text, text to HTML, and HTML charset declarations.
8
9mod entities;
10mod meta;
11mod text;
12
13use std::char::REPLACEMENT_CHARACTER;
14
15use memchr::{memchr_iter, memchr2, memchr3};
16
17pub use meta::strip_charset_meta;
18pub(crate) use text::html_to_text_prefix;
19
20const HTML_PREFIX: &str = "<html><body>";
21const HTML_SUFFIX: &str = "</body></html>";
22const LINE_BREAK: &str = "<br/>";
23
24/// Converts HTML to plain text: tags are removed, entities decoded and
25/// white space collapsed; `<br>` and `</p>` become line breaks; the content
26/// of `<head>`, `<style>`, `<script>` and `<template>` is skipped.
27pub fn html_to_text(input: &str) -> String {
28    text::html_to_text(input)
29}
30
31/// Appends one text token of an HTML document to `result`, decoded when it
32/// is an entity (`&amp;`, `&#233;`), after a space when `add_space` is set.
33/// Invalid UTF-8 becomes U+FFFD. For callers with their own HTML tokenizer.
34pub fn add_html_token(result: &mut String, token: &[u8], add_space: bool) {
35    let token = String::from_utf8_lossy(token);
36    if add_space {
37        result.push(' ');
38    }
39    match decode_entity(&token) {
40        Some(ch) => result.push(ch),
41        None => result.push_str(&token),
42    }
43}
44
45/// Converts plain text to HTML: `<`, `>` and `&` are escaped, line breaks
46/// become `<br/>`, carriage returns are dropped, and the result is wrapped
47/// in `<html><body>`.
48pub fn text_to_html(input: &str) -> String {
49    let bytes = input.as_bytes();
50    let line_breaks = memchr_iter(b'\n', bytes).count();
51    let mut result = String::with_capacity(
52        bytes.len() + line_breaks * (LINE_BREAK.len() - 1) + HTML_PREFIX.len() + HTML_SUFFIX.len(),
53    );
54    result.push_str(HTML_PREFIX);
55    let find_common = |from: usize| find(bytes, from, |rest| memchr3(b'\n', b'\r', b'<', rest));
56    let find_rare = |from: usize| find(bytes, from, |rest| memchr2(b'>', b'&', rest));
57    let mut next_common = find_common(0);
58    let mut next_rare = find_rare(0);
59    let mut literal_start = 0;
60    while let Some(special) = earliest(next_common, next_rare) {
61        result.push_str(input.get(literal_start..special).unwrap_or_default());
62        result.push_str(match bytes.get(special) {
63            Some(b'\n') => LINE_BREAK,
64            Some(b'<') => "&lt;",
65            Some(b'>') => "&gt;",
66            Some(b'&') => "&amp;",
67            _ => "",
68        });
69        literal_start = special + 1;
70        if next_common == Some(special) {
71            next_common = find_common(literal_start);
72        }
73        if next_rare == Some(special) {
74            next_rare = find_rare(literal_start);
75        }
76    }
77    result.push_str(input.get(literal_start..).unwrap_or_default());
78    result.push_str(HTML_SUFFIX);
79    result
80}
81
82fn find(bytes: &[u8], from: usize, search: impl Fn(&[u8]) -> Option<usize>) -> Option<usize> {
83    bytes
84        .get(from..)
85        .and_then(search)
86        .map(|offset| from + offset)
87}
88
89fn earliest(first: Option<usize>, second: Option<usize>) -> Option<usize> {
90    match (first, second) {
91        (Some(first), Some(second)) => Some(first.min(second)),
92        (first, second) => first.or(second),
93    }
94}
95
96fn decode_entity(token: &str) -> Option<char> {
97    let entity = token.strip_prefix('&')?.strip_suffix(';')?;
98    let code = match entity.strip_prefix('#') {
99        Some(number) => match number.strip_prefix(['x', 'X']) {
100            Some(hex) => u32::from_str_radix(hex, 16).ok()?,
101            None => number.parse::<u32>().ok()?,
102        },
103        None => entities::lookup(entity.as_bytes()).copied()?,
104    };
105    Some(char::from_u32(code).unwrap_or(REPLACEMENT_CHARACTER))
106}
107
108#[cfg(test)]
109mod tests {
110    use super::{add_html_token, html_to_text, strip_charset_meta, text_to_html};
111    use crate::fields::tests::load_text_pairs;
112    use std::borrow::Cow;
113
114    #[test]
115    fn text_to_html_fixtures() {
116        let tests = load_text_pairs("decoders/text_to_html.json");
117        assert_eq!(tests.len(), 3);
118        for (input, expected) in tests {
119            assert_eq!(text_to_html(&input), expected, "{input:?}");
120        }
121    }
122
123    #[test]
124    fn html_to_text_fixtures() {
125        let tests = load_text_pairs("decoders/html_to_text.json");
126        assert_eq!(tests.len(), 24);
127        for (input, expected) in tests {
128            assert_eq!(html_to_text(&input), expected, "{input:?}");
129        }
130    }
131
132    #[test]
133    fn html_entity_fixtures() {
134        let tests = load_text_pairs("decoders/html_entities.json");
135        assert_eq!(tests.len(), 9);
136        for (input, expected) in tests {
137            let mut result = String::with_capacity(input.len());
138            add_html_token(&mut result, input.as_bytes(), false);
139            assert_eq!(result, expected, "{input:?}");
140        }
141    }
142
143    #[test]
144    fn charset_meta_fixtures() {
145        let tests = load_text_pairs("decoders/charset_meta.json");
146        assert_eq!(tests.len(), 7);
147        for (input, expected) in tests {
148            let stripped = strip_charset_meta(&input);
149            assert_eq!(
150                matches!(stripped, Cow::Borrowed(_)),
151                input == expected,
152                "{input:?}"
153            );
154            assert_eq!(stripped, expected, "{input:?}");
155        }
156    }
157}
mail_parser/decoders/html/mod.rs

mail_parser/decoders/html/
mod.rs
