webfetch_core/
charset.rs

1//! Decoding response bodies that are not UTF-8.
2//!
3//! Most of the web is UTF-8 and takes the fast path here. The rest is decoded
4//! through `encoding_rs`, the same implementation Firefox uses, which covers
5//! the whole WHATWG Encoding Standard: the single-byte Western family
6//! (`windows-1252`, `ISO-8859-*`), the CJK multi-byte encodings (Shift_JIS,
7//! GBK, GB18030, Big5, EUC-KR, EUC-JP, ISO-2022-JP), KOI8, and UTF-16.
8//!
9//! An earlier version hand-rolled a windows-1252 table and reported everything
10//! else as undecodable, on the grounds that conversion tables would bloat a
11//! binary that advertises being small. Measured, the tables cost about 0.2 MB —
12//! against handing back mojibake for every Japanese, Chinese and Korean page,
13//! which is most of the non-Latin web.
14//!
15//! Label lookup follows the WHATWG rules, so the aliases real pages use
16//! (`latin1`, `sjis`, `x-gbk`, `ms949`, …) all resolve.
17
18/// What decoding a body needs, given its declared charset.
19///
20/// Non-exhaustive: the set of recognized encodings is `encoding_rs`', not ours,
21/// and this should be able to grow without breaking callers again.
22#[derive(Debug, Clone, PartialEq, Eq)]
23#[non_exhaustive]
24pub enum Charset {
25    /// UTF-8 (or nothing declared): decoded as UTF-8, lossily.
26    Utf8,
27    /// A label we can decode exactly. Carries the encoding's canonical name.
28    Supported(String),
29    /// A label no known encoding matches. Decoded as UTF-8, which will mangle
30    /// it, and reported so the caller can see why.
31    Unknown(String),
32}
33
34/// Is this label UTF-8 (or absent), and therefore the fast path?
35fn is_utf8_label(normalized: &str) -> bool {
36    matches!(
37        normalized,
38        "" | "utf-8" | "utf8" | "unicode-1-1-utf-8" | "us-ascii" | "ascii"
39    )
40}
41
42/// Classify a charset label.
43pub fn classify(label: &str) -> Charset {
44    let trimmed = label.trim().trim_matches('"');
45    if is_utf8_label(&trimmed.to_ascii_lowercase()) {
46        return Charset::Utf8;
47    }
48    match encoding_rs::Encoding::for_label(trimmed.as_bytes()) {
49        // `for_label` maps the UTF-8 aliases too; keep them on the fast path.
50        Some(encoding) if encoding == encoding_rs::UTF_8 => Charset::Utf8,
51        Some(encoding) => Charset::Supported(encoding.name().to_string()),
52        None => Charset::Unknown(trimmed.to_string()),
53    }
54}
55
56/// Decode a body according to its declared charset.
57///
58/// Returns the text, plus the charset label to report when the result may be
59/// garbled — `None` whenever the decoding was exact.
60pub fn decode(bytes: &[u8], label: Option<&str>) -> (String, Option<String>) {
61    let Some(label) = label else {
62        return (String::from_utf8_lossy(bytes).into_owned(), None);
63    };
64    let trimmed = label.trim().trim_matches('"');
65    if is_utf8_label(&trimmed.to_ascii_lowercase()) {
66        return (String::from_utf8_lossy(bytes).into_owned(), None);
67    }
68
69    match encoding_rs::Encoding::for_label(trimmed.as_bytes()) {
70        Some(encoding) => {
71            // `decode` strips a BOM when present and substitutes replacement
72            // characters for malformed sequences rather than failing.
73            let (text, _, _) = encoding.decode(bytes);
74            (text.into_owned(), None)
75        }
76        None => (
77            String::from_utf8_lossy(bytes).into_owned(),
78            Some(trimmed.to_string()),
79        ),
80    }
81}
82
83/// Find `<meta charset=…>` in the head of a body whose header declared nothing.
84///
85/// Only the first 2 KiB are searched: the declaration is required to appear
86/// early, and scanning a whole 5 MiB body for it would be wasted work.
87pub fn sniff_meta(raw: &[u8]) -> Option<String> {
88    const WINDOW: usize = 2048;
89    let head = &raw[..raw.len().min(WINDOW)];
90    let text = String::from_utf8_lossy(head).to_ascii_lowercase();
91    let at = text.find("charset")? + "charset".len();
92    let rest = text[at..].trim_start().strip_prefix('=')?.trim_start();
93    let value: String = rest
94        .trim_start_matches(['"', '\''])
95        .chars()
96        .take_while(|c| c.is_ascii_alphanumeric() || *c == '-' || *c == '_')
97        .collect();
98    (!value.is_empty()).then_some(value)
99}
100
101/// Pull a `charset=` value out of a `Content-Type` header.
102pub fn from_content_type(header: &str) -> Option<String> {
103    header.split(';').find_map(|part| {
104        let part = part.trim();
105        let rest = part.strip_prefix("charset=").or_else(|| {
106            part.to_ascii_lowercase()
107                .starts_with("charset=")
108                .then(|| &part["charset=".len()..])
109        })?;
110        let value = rest.trim().trim_matches('"');
111        (!value.is_empty()).then(|| value.to_string())
112    })
113}
114
115#[cfg(test)]
116mod tests {
117    use super::*;
118
119    #[test]
120    fn utf8_labels_take_the_fast_path() {
121        for label in ["utf-8", "UTF-8", " \"utf8\" ", "us-ascii", ""] {
122            assert_eq!(classify(label), Charset::Utf8, "{label}");
123        }
124    }
125
126    #[test]
127    fn a_latin1_body_decodes_correctly() {
128        // "Café — naïve" in windows-1252: é=0xE9, em dash=0x97, ï=0xEF.
129        let bytes = b"Caf\xe9 \x97 na\xefve";
130        let (text, reported) = decode(bytes, Some("ISO-8859-1"));
131        assert_eq!(text, "Café — naïve");
132        assert_eq!(reported, None, "an exact decode reports no problem");
133    }
134
135    #[test]
136    fn cp1252_smart_quotes_survive() {
137        let (text, _) = decode(b"\x93quoted\x94 \x85", Some("windows-1252"));
138        assert_eq!(text, "“quoted” …");
139    }
140
141    /// The case the hand-rolled table could not handle: CJK pages came back as
142    /// replacement characters.
143    #[test]
144    fn shift_jis_decodes() {
145        // "こんにちは" in Shift_JIS.
146        let bytes = b"\x82\xb1\x82\xf1\x82\xc9\x82\xbf\x82\xcd";
147        let (text, reported) = decode(bytes, Some("Shift_JIS"));
148        assert_eq!(text, "こんにちは");
149        assert!(reported.is_none());
150    }
151
152    #[test]
153    fn gbk_decodes() {
154        // "中文" in GBK.
155        let (text, reported) = decode(b"\xd6\xd0\xce\xc4", Some("GBK"));
156        assert_eq!(text, "中文");
157        assert!(reported.is_none());
158    }
159
160    #[test]
161    fn big5_decodes() {
162        // "中文" in Big5.
163        let (text, reported) = decode(b"\xa4\xa4\xa4\xe5", Some("Big5"));
164        assert_eq!(text, "中文");
165        assert!(reported.is_none());
166    }
167
168    #[test]
169    fn euc_kr_decodes() {
170        // "한국" in EUC-KR.
171        let (text, reported) = decode(b"\xc7\xd1\xb1\xb9", Some("EUC-KR"));
172        assert_eq!(text, "한국");
173        assert!(reported.is_none());
174    }
175
176    /// Real pages declare aliases, not canonical names.
177    #[test]
178    fn whatwg_aliases_resolve() {
179        for (label, canonical) in [
180            ("latin1", "windows-1252"),
181            ("sjis", "Shift_JIS"),
182            ("x-gbk", "GBK"),
183            ("windows-949", "EUC-KR"),
184            ("korean", "EUC-KR"),
185            ("iso-2022-jp", "ISO-2022-JP"),
186        ] {
187            assert_eq!(
188                classify(label),
189                Charset::Supported(canonical.to_string()),
190                "{label}"
191            );
192        }
193    }
194
195    #[test]
196    fn an_unrecognized_label_is_reported_rather_than_hidden() {
197        assert_eq!(
198            classify("x-not-a-real-encoding"),
199            Charset::Unknown("x-not-a-real-encoding".into())
200        );
201        let (_, reported) = decode(b"bytes", Some("x-not-a-real-encoding"));
202        assert_eq!(reported.as_deref(), Some("x-not-a-real-encoding"));
203    }
204
205    #[test]
206    fn utf8_bodies_round_trip() {
207        let (text, reported) = decode("Café — naïve".as_bytes(), Some("utf-8"));
208        assert_eq!(text, "Café — naïve");
209        assert!(reported.is_none());
210    }
211
212    #[test]
213    fn a_utf16_body_decodes_including_its_bom() {
214        // "hi" in UTF-16LE with a BOM.
215        let (text, reported) = decode(b"\xff\xfeh\x00i\x00", Some("utf-16"));
216        assert_eq!(text, "hi");
217        assert!(reported.is_none());
218    }
219
220    #[test]
221    fn meta_charset_is_sniffed_from_the_head() {
222        assert_eq!(
223            sniff_meta(br#"<html><head><meta charset="Shift_JIS"></head>"#).as_deref(),
224            Some("shift_jis")
225        );
226        assert_eq!(
227            sniff_meta(
228                b"<html><head><meta http-equiv=content-type content='text/html; charset=gbk'>"
229            )
230            .as_deref(),
231            Some("gbk")
232        );
233        assert_eq!(sniff_meta(b"<html><head></head>").as_deref(), None);
234    }
235
236    #[test]
237    fn charset_is_read_out_of_a_content_type() {
238        assert_eq!(
239            from_content_type("text/html; charset=ISO-8859-1").as_deref(),
240            Some("ISO-8859-1")
241        );
242        assert_eq!(
243            from_content_type("text/html;charset=\"utf-8\"").as_deref(),
244            Some("utf-8")
245        );
246        assert_eq!(from_content_type("text/html").as_deref(), None);
247    }
248}
webfetch_core/charset.rs

webfetch_core/
charset.rs
