webfetch_core/
charset.rs

1//! Decoding response bodies that are not UTF-8.
2//!
3//! Most of the web is UTF-8, and `String::from_utf8_lossy` handles it. The
4//! common exception by a wide margin is the single-byte Western European
5//! family — `windows-1252`, and `ISO-8859-1`/`latin1`, which browsers treat as
6//! windows-1252 anyway. Those are a 128-entry table, decoded here directly.
7//!
8//! Multi-byte legacy encodings (Shift_JIS, GBK, Big5, EUC-KR) need real tables,
9//! and pulling in a full encoding library would add roughly a megabyte to a
10//! binary whose whole pitch is being small. Those are decoded lossily and the
11//! declared charset is reported on the result instead, so a caller can see why
12//! the text looks wrong rather than guessing.
13
14/// Upper half of windows-1252 (0x80-0x9F); 0xA0-0xFF matches Latin-1, which
15/// matches the Unicode code points of the same value.
16const CP1252_HIGH: [char; 32] = [
17    '\u{20AC}', '\u{FFFD}', '\u{201A}', '\u{0192}', '\u{201E}', '\u{2026}', '\u{2020}', '\u{2021}',
18    '\u{02C6}', '\u{2030}', '\u{0160}', '\u{2039}', '\u{0152}', '\u{FFFD}', '\u{017D}', '\u{FFFD}',
19    '\u{FFFD}', '\u{2018}', '\u{2019}', '\u{201C}', '\u{201D}', '\u{2022}', '\u{2013}', '\u{2014}',
20    '\u{02DC}', '\u{2122}', '\u{0161}', '\u{203A}', '\u{0153}', '\u{FFFD}', '\u{017E}', '\u{0178}',
21];
22
23/// What decoding a body needs, given its declared charset.
24#[derive(Debug, Clone, PartialEq, Eq)]
25pub enum Charset {
26    /// UTF-8 (or nothing declared): decode lossily as UTF-8.
27    Utf8,
28    /// windows-1252 / ISO-8859-1: decoded here, exactly.
29    Cp1252,
30    /// Something else. Decoded as UTF-8, which will mangle it; the label is
31    /// carried on the result so the caller knows.
32    Unsupported(String),
33}
34
35/// Classify a charset label.
36pub fn classify(label: &str) -> Charset {
37    let normalized = label
38        .trim()
39        .trim_matches('"')
40        .to_ascii_lowercase()
41        .replace('_', "-");
42    match normalized.as_str() {
43        "" | "utf-8" | "utf8" | "us-ascii" | "ascii" => Charset::Utf8,
44        // Browsers decode every one of these as windows-1252, which is a strict
45        // superset of ISO-8859-1 over the bytes that matter.
46        "windows-1252" | "cp1252" | "iso-8859-1" | "latin1" | "latin-1" | "iso8859-1"
47        | "iso-latin-1" | "ansi-x3.4-1968" => Charset::Cp1252,
48        _ => Charset::Unsupported(label.trim().trim_matches('"').to_string()),
49    }
50}
51
52/// Decode a body according to its declared charset.
53///
54/// Returns the text, plus the charset label to report when the result may be
55/// garbled (`None` when the decoding is exact).
56pub fn decode(bytes: &[u8], label: Option<&str>) -> (String, Option<String>) {
57    match label.map(classify).unwrap_or(Charset::Utf8) {
58        Charset::Utf8 => (String::from_utf8_lossy(bytes).into_owned(), None),
59        Charset::Cp1252 => (decode_cp1252(bytes), None),
60        Charset::Unsupported(name) => (String::from_utf8_lossy(bytes).into_owned(), Some(name)),
61    }
62}
63
64/// Decode windows-1252. Every byte maps to exactly one character, so this
65/// cannot fail.
66pub fn decode_cp1252(bytes: &[u8]) -> String {
67    let mut out = String::with_capacity(bytes.len());
68    for &b in bytes {
69        match b {
70            0x00..=0x7F => out.push(b as char),
71            0x80..=0x9F => out.push(CP1252_HIGH[(b - 0x80) as usize]),
72            _ => out.push(b as char), // 0xA0-0xFF: Latin-1 == Unicode
73        }
74    }
75    out
76}
77
78/// Pull a `charset=` value out of a `Content-Type` header.
79pub fn from_content_type(header: &str) -> Option<String> {
80    header.split(';').find_map(|part| {
81        let part = part.trim();
82        let rest = part.strip_prefix("charset=").or_else(|| {
83            part.to_ascii_lowercase()
84                .starts_with("charset=")
85                .then(|| &part["charset=".len()..])
86        })?;
87        let value = rest.trim().trim_matches('"');
88        (!value.is_empty()).then(|| value.to_string())
89    })
90}
91
92#[cfg(test)]
93mod tests {
94    use super::*;
95
96    #[test]
97    fn utf8_labels_need_no_special_handling() {
98        assert_eq!(classify("utf-8"), Charset::Utf8);
99        assert_eq!(classify("UTF-8"), Charset::Utf8);
100        assert_eq!(classify(" \"utf8\" "), Charset::Utf8);
101        assert_eq!(classify("us-ascii"), Charset::Utf8);
102    }
103
104    #[test]
105    fn latin_labels_all_decode_as_cp1252() {
106        for label in [
107            "ISO-8859-1",
108            "iso_8859-1",
109            "latin1",
110            "windows-1252",
111            "CP1252",
112        ] {
113            assert_eq!(classify(label), Charset::Cp1252, "{label}");
114        }
115    }
116
117    #[test]
118    fn a_latin1_body_decodes_correctly() {
119        // "Café — naïve" in windows-1252: é=0xE9, em dash=0x97, ï=0xEF.
120        let bytes = b"Caf\xe9 \x97 na\xefve";
121        let (text, reported) = decode(bytes, Some("ISO-8859-1"));
122        assert_eq!(text, "Café — naïve");
123        assert_eq!(reported, None, "an exact decode reports no problem");
124    }
125
126    #[test]
127    fn cp1252_smart_quotes_survive() {
128        let bytes = b"\x93quoted\x94 \x85";
129        assert_eq!(decode_cp1252(bytes), "“quoted” …");
130    }
131
132    #[test]
133    fn an_unsupported_charset_is_reported_rather_than_hidden() {
134        let (_, reported) = decode(b"\x82\xa0", Some("Shift_JIS"));
135        assert_eq!(reported.as_deref(), Some("Shift_JIS"));
136    }
137
138    #[test]
139    fn utf8_bodies_round_trip() {
140        let (text, reported) = decode("Café — naïve".as_bytes(), Some("utf-8"));
141        assert_eq!(text, "Café — naïve");
142        assert!(reported.is_none());
143    }
144
145    #[test]
146    fn charset_is_read_out_of_a_content_type() {
147        assert_eq!(
148            from_content_type("text/html; charset=ISO-8859-1").as_deref(),
149            Some("ISO-8859-1")
150        );
151        assert_eq!(
152            from_content_type("text/html;charset=\"utf-8\"").as_deref(),
153            Some("utf-8")
154        );
155        assert_eq!(from_content_type("text/html").as_deref(), None);
156    }
157}
webfetch_core/charset.rs

webfetch_core/
charset.rs
