moss_core/
html_entities.rs

1//! Turn the HTML entities that real text arrives wrapped in back into characters.
2//!
3//! Three callers in moss need the same small decoder and none of them is an
4//! HTML parser:
5//!
6//! - the article scraper reads JSON-LD out of a `<script>` block, where `<` and
7//!   `&` are HTML-special, so titles and bylines arrive as `China&#8217;s` —
8//!   `serde_json` parses the JSON faithfully and leaves the entities alone;
9//! - the build's HTML post-pass rewrites attribute values that the synthesizer
10//!   already escaped, so it has to decode before it can split a URL on `?`/`#`
11//!   (a `#` inside `&#39;` is not a fragment);
12//! - the orphan sweep compares an author's media reference against the files on
13//!   disk, and `&amp;` is not a filename.
14//!
15//! Deliberately not a full entity table. It covers what those three actually
16//! meet: numeric references in both spellings (`&#8217;`, `&#x2019;`) and the
17//! handful of named entities an escaper emits (`&amp;`, `&lt;`, `&gt;`,
18//! `&quot;`, `&apos;`, `&nbsp;`). Anything else is left exactly as written,
19//! which is the safe answer for text that was never entity-encoded to begin
20//! with — `Cats & dogs` comes back unchanged.
21
22/// Decode the entity subset above; return everything else byte-for-byte.
23///
24/// Total: an unrecognized token, a bare `&`, or an unterminated entity all
25/// pass through literally rather than being dropped or erroring.
26pub fn decode(s: &str) -> String {
27    if !s.contains('&') {
28        return s.to_string();
29    }
30    let mut out = String::with_capacity(s.len());
31    let mut rest = s;
32    while let Some((before, after_amp)) = rest.split_once('&') {
33        out.push_str(before);
34        match after_amp.split_once(';') {
35            Some((token, after_semi)) => match decode_token(token) {
36                Some(ch) => {
37                    out.push(ch);
38                    rest = after_semi;
39                }
40                // Not an entity we know: emit the `&` literally and rescan
41                // from just past it (the `;` may end a later entity).
42                None => {
43                    out.push('&');
44                    rest = after_amp;
45                }
46            },
47            None => {
48                out.push('&');
49                out.push_str(after_amp);
50                rest = "";
51            }
52        }
53    }
54    out.push_str(rest);
55    out
56}
57
58/// The text between `&` and `;`, or `None` when it names nothing we decode.
59fn decode_token(token: &str) -> Option<char> {
60    if let Some(rest) = token.strip_prefix('#') {
61        let n: u32 = if let Some(hex) = rest.strip_prefix(['x', 'X']) {
62            u32::from_str_radix(hex, 16).ok()?
63        } else {
64            rest.parse().ok()?
65        };
66        return char::from_u32(n);
67    }
68    match token {
69        "amp" => Some('&'),
70        "lt" => Some('<'),
71        "gt" => Some('>'),
72        "quot" => Some('"'),
73        "apos" => Some('\''),
74        "nbsp" => Some('\u{a0}'),
75        _ => None,
76    }
77}
78
79#[cfg(test)]
80mod tests {
81    use super::*;
82
83    #[test]
84    fn decodes_numeric_and_named_entities() {
85        assert_eq!(decode("Finding China&#8217;s Voice"), "Finding China’s Voice");
86        assert_eq!(decode("AT&amp;T"), "AT&T");
87        assert_eq!(decode("a&#x2014;b"), "a—b");
88        assert_eq!(decode("no entities"), "no entities");
89    }
90
91    /// The three callers all run this over text that was never encoded, so a
92    /// bare ampersand has to survive intact — mangling it would corrupt a
93    /// filename in the orphan sweep and a URL in the HTML post-pass.
94    #[test]
95    fn text_that_was_never_encoded_survives_unchanged() {
96        assert_eq!(decode("Cats & dogs"), "Cats & dogs");
97        assert_eq!(decode("a&notanentity;b"), "a&notanentity;b");
98        assert_eq!(decode("50% & rising"), "50% & rising");
99    }
100
101    /// `&#39;` contains a `#`, which the HTML post-pass would otherwise read as
102    /// the start of a URL fragment — decoding first is what makes that split
103    /// correct, so the decode itself must not stop at the `#`.
104    #[test]
105    fn a_numeric_entity_is_decoded_whole() {
106        assert_eq!(decode("Jo&#39;s photos/a.jpg"), "Jo's photos/a.jpg");
107    }
108}
moss_core/html_entities.rs

moss_core/
html_entities.rs
