sup_xml_core/html/
encoding.rs

1#![forbid(unsafe_code)]
2
3//! WHATWG byte-stream encoding sniffing for HTML input.
4//!
5//! Sniffing precedence per [WHATWG § 12.2.3]:
6//!
7//! 1. Caller-supplied encoding (HTTP `Content-Type`, manual override
8//!    via [`HtmlParseOptions::encoding_override`]).
9//! 2. Byte-order mark (UTF-8 `EF BB BF`, UTF-16BE `FE FF`, UTF-16LE
10//!    `FF FE`).
11//! 3. Pre-scan up to [`HtmlParseOptions::encoding_sniff_window`]
12//!    bytes (default 1024) for a `<meta charset>` or `<meta
13//!    http-equiv="Content-Type">` declaration.
14//! 4. Fall back to **Windows-1252** — explicitly *not* Latin-1.
15//!    The HTML5 spec mandates this for backward compatibility with
16//!    legacy web content.
17//!
18//! [WHATWG § 12.2.3]: https://html.spec.whatwg.org/multipage/parsing.html#determining-the-character-encoding
19//!
20//! # What's *not* implemented (deferred to v2 if a user asks)
21//!
22//! - The full WHATWG prescan algorithm for `<meta>` is approximated
23//!   here.  We handle the two common forms (charset attribute and
24//!   http-equiv content-type) with reasonable attribute parsing,
25//!   but not every WHATWG edge case (deeply nested comments inside
26//!   the head, weird whitespace, etc.).
27//! - Encoding switches *during* parsing (the spec allows the parser
28//!   to detect a different encoding in a later `<meta>` and restart).
29//!   Our sniffer reads up to the configured window once and then
30//!   commits to that encoding for the whole document.
31
32use std::borrow::Cow;
33
34use crate::encoding::{transcode_to_utf8_as, Encoding};
35use crate::error::Result;
36
37use super::options::HtmlParseOptions;
38
39/// Sniff the encoding of an HTML byte stream per the WHATWG
40/// algorithm.  Always returns *some* encoding — falls back to
41/// Windows-1252 when no signal is found.
42pub fn sniff_html_encoding(bytes: &[u8], opts: &HtmlParseOptions) -> Encoding {
43    // 1. Caller-supplied encoding wins.
44    if let Some(label) = opts.encoding_override.as_deref() {
45        return label_to_encoding(label);
46    }
47
48    // 2. BOM.
49    if let Some(enc) = sniff_bom(bytes) {
50        return enc;
51    }
52
53    // 3. Pre-scan for meta charset.
54    let window = opts.encoding_sniff_window.max(64).min(bytes.len());
55    if let Some(label) = prescan_meta_charset(&bytes[..window]) {
56        return label_to_encoding(&label);
57    }
58
59    // 4. Fall back to Windows-1252 (NOT Latin-1; spec is explicit).
60    Encoding::Windows1252
61}
62
63/// Sniff + transcode in one step.  Returns UTF-8 bytes plus the
64/// encoding that was used so the caller can record it on the
65/// resulting `Document`.
66pub fn decode_html_input<'a>(
67    bytes: &'a [u8],
68    opts: &HtmlParseOptions,
69) -> Result<(Cow<'a, [u8]>, Encoding)> {
70    let enc = sniff_html_encoding(bytes, opts);
71    let decoded = transcode_to_utf8_as(bytes, enc.clone())?;
72    Ok((decoded, enc))
73}
74
75/// Detect a byte-order mark.  Returns the implied encoding or
76/// `None` if no BOM is present.
77fn sniff_bom(bytes: &[u8]) -> Option<Encoding> {
78    if bytes.starts_with(&[0xEF, 0xBB, 0xBF]) {
79        return Some(Encoding::Utf8);
80    }
81    if bytes.starts_with(&[0xFE, 0xFF]) {
82        return Some(Encoding::Utf16Be);
83    }
84    if bytes.starts_with(&[0xFF, 0xFE]) {
85        return Some(Encoding::Utf16Le);
86    }
87    None
88}
89
90/// Pre-scan the first chunk of input for a `<meta>` tag with a
91/// charset declaration.  Returns the label string as written, or
92/// `None` if no usable declaration is found.
93///
94/// Pragmatic implementation — handles the two common WHATWG forms:
95///   `<meta charset="UTF-8">`
96///   `<meta http-equiv="Content-Type" content="text/html; charset=UTF-8">`
97/// plus the various quote/whitespace/case variations.  Skips
98/// HTML comments so a charset inside `<!-- -->` doesn't trigger.
99pub fn prescan_meta_charset(bytes: &[u8]) -> Option<String> {
100    let mut pos = 0;
101    while pos < bytes.len() {
102        let b = bytes[pos];
103
104        // Skip HTML comments.
105        if starts_with_ascii(&bytes[pos..], b"<!--") {
106            pos += 4;
107            // Skip until "-->" or end.
108            while pos + 2 < bytes.len()
109                && !(bytes[pos] == b'-' && bytes[pos + 1] == b'-' && bytes[pos + 2] == b'>')
110            {
111                pos += 1;
112            }
113            pos = (pos + 3).min(bytes.len());
114            continue;
115        }
116
117        // `<meta` followed by a tag-name terminator.
118        if starts_with_ascii_ignore_case(&bytes[pos..], b"<meta")
119            && bytes
120                .get(pos + 5)
121                .is_some_and(|&c| matches!(c, b' ' | b'\t' | b'\n' | b'\r' | 0x0C | b'/' | b'>'))
122        {
123            pos += 5;
124            if let Some(label) = parse_meta_attrs(bytes, &mut pos) {
125                return Some(label);
126            }
127            // parse_meta_attrs already advanced past the tag.
128            continue;
129        }
130
131        // Other tag — skip past the closing `>`.
132        if b == b'<' {
133            // `</`, `<!`, `<?`, or `<letter` — all "skip past `>`".
134            if let Some(end) = find_byte(&bytes[pos..], b'>') {
135                pos += end + 1;
136                continue;
137            }
138            // No `>` in remaining bytes — bail out.
139            return None;
140        }
141
142        pos += 1;
143    }
144    None
145}
146
147/// Parse attributes of a `<meta>` tag starting at `*pos` (just past
148/// `<meta`).  Returns the discovered charset label if found.
149/// Advances `*pos` past the tag's `>` regardless.
150fn parse_meta_attrs(bytes: &[u8], pos: &mut usize) -> Option<String> {
151    let mut http_equiv: Option<Vec<u8>> = None;
152    let mut content: Option<Vec<u8>> = None;
153    let mut charset: Option<String> = None;
154
155    loop {
156        // Skip whitespace and stray slashes.
157        while *pos < bytes.len()
158            && matches!(bytes[*pos], b' ' | b'\t' | b'\n' | b'\r' | 0x0C | b'/')
159        {
160            *pos += 1;
161        }
162        if *pos >= bytes.len() {
163            break;
164        }
165        if bytes[*pos] == b'>' {
166            *pos += 1;
167            break;
168        }
169
170        // Read attribute name (lower-cased) until '=', whitespace, '/', or '>'.
171        let mut name = Vec::new();
172        while *pos < bytes.len() {
173            let c = bytes[*pos];
174            if matches!(c, b'=' | b' ' | b'\t' | b'\n' | b'\r' | 0x0C | b'/' | b'>') {
175                break;
176            }
177            name.push(c.to_ascii_lowercase());
178            *pos += 1;
179        }
180        if name.is_empty() {
181            // Stray `/` or `>` already advanced; loop again.
182            if *pos < bytes.len() && bytes[*pos] == b'>' {
183                *pos += 1;
184                break;
185            }
186            *pos += 1;
187            continue;
188        }
189
190        // Skip whitespace before potential `=`.
191        while *pos < bytes.len() && matches!(bytes[*pos], b' ' | b'\t' | b'\n' | b'\r' | 0x0C) {
192            *pos += 1;
193        }
194
195        let mut value: Vec<u8> = Vec::new();
196        if *pos < bytes.len() && bytes[*pos] == b'=' {
197            *pos += 1;
198            // Skip whitespace after `=`.
199            while *pos < bytes.len()
200                && matches!(bytes[*pos], b' ' | b'\t' | b'\n' | b'\r' | 0x0C)
201            {
202                *pos += 1;
203            }
204            if *pos < bytes.len() && (bytes[*pos] == b'"' || bytes[*pos] == b'\'') {
205                let quote = bytes[*pos];
206                *pos += 1;
207                while *pos < bytes.len() && bytes[*pos] != quote {
208                    value.push(bytes[*pos]);
209                    *pos += 1;
210                }
211                if *pos < bytes.len() {
212                    *pos += 1; // consume closing quote
213                }
214            } else {
215                while *pos < bytes.len() {
216                    let c = bytes[*pos];
217                    if matches!(c, b' ' | b'\t' | b'\n' | b'\r' | 0x0C | b'>') {
218                        break;
219                    }
220                    value.push(c);
221                    *pos += 1;
222                }
223            }
224        }
225
226        // Dispatch on attribute name.
227        match name.as_slice() {
228            b"charset" => {
229                if charset.is_none() {
230                    charset = Some(String::from_utf8_lossy(&value).into_owned());
231                }
232            }
233            b"http-equiv" => {
234                if http_equiv.is_none() {
235                    http_equiv = Some(value);
236                }
237            }
238            b"content" => {
239                if content.is_none() {
240                    content = Some(value);
241                }
242            }
243            _ => {}
244        }
245    }
246
247    // Direct charset attribute wins.
248    if let Some(c) = charset {
249        return Some(c);
250    }
251    // Otherwise check http-equiv content-type.
252    if let (Some(equiv), Some(content_val)) = (http_equiv, content) {
253        if ascii_equal_ignore_case(&equiv, b"content-type") {
254            return extract_charset_from_content(&content_val);
255        }
256    }
257    None
258}
259
260/// Pull a `charset=NAME` value out of an `http-equiv` `content`
261/// attribute string.  Looks for the substring `charset=` (case
262/// insensitive) and reads the value (quoted or unquoted, terminated
263/// by `;` or whitespace).
264fn extract_charset_from_content(content: &[u8]) -> Option<String> {
265    let lower: Vec<u8> = content.iter().map(|b| b.to_ascii_lowercase()).collect();
266    let needle = b"charset=";
267    let pos = lower.windows(needle.len()).position(|w| w == needle)?;
268    let mut start = pos + needle.len();
269    if start < content.len() && (content[start] == b'"' || content[start] == b'\'') {
270        let quote = content[start];
271        start += 1;
272        let end = content[start..].iter().position(|&c| c == quote)?;
273        Some(String::from_utf8_lossy(&content[start..start + end]).into_owned())
274    } else {
275        let end = content[start..]
276            .iter()
277            .position(|&c| matches!(c, b';' | b' ' | b'\t' | b'\n' | b'\r' | 0x0C))
278            .unwrap_or(content.len() - start);
279        Some(String::from_utf8_lossy(&content[start..start + end]).into_owned())
280    }
281}
282
283/// Map a WHATWG encoding label to our [`Encoding`] enum.  Unknown
284/// labels fall through to [`Encoding::Other`] which routes to
285/// `encoding_rs` (when the `full-encodings` feature is on) or
286/// errors out.
287///
288/// Common labels covered inline so we don't always go through
289/// `encoding_rs`.  Note: WHATWG explicitly maps `iso-8859-1` to
290/// Windows-1252 because legacy web content labels Latin-1 documents
291/// that actually use Win1252 characters.
292pub fn label_to_encoding(label: &str) -> Encoding {
293    let trimmed = label.trim().to_ascii_lowercase();
294    match trimmed.as_str() {
295        "utf-8" | "utf8" | "unicode-1-1-utf-8" | "unicode11utf8" | "unicode20utf8"
296        | "x-unicode20utf8" => Encoding::Utf8,
297        "us-ascii" | "ascii" | "ansi_x3.4-1968" | "iso646-us" | "iso-ir-6"
298        | "iso_646.irv:1991" | "csascii" => Encoding::Ascii,
299        // WHATWG: iso-8859-1 / latin1 / cp1252 all map to Windows-1252.
300        "iso-8859-1" | "latin1" | "iso8859-1" | "iso_8859-1" | "iso_8859-1:1987"
301        | "windows-1252" | "cp1252" | "cp819" | "csisolatin1" | "ibm819" | "l1"
302        | "x-cp1252" => Encoding::Windows1252,
303        "utf-16" | "utf-16le" | "csunicode" | "ucs-2" | "unicode" | "unicodefeff" => {
304            Encoding::Utf16Le
305        }
306        "utf-16be" | "unicodefffe" => Encoding::Utf16Be,
307        // Py_UCS4 / wide-unicode source buffers: lxml hands these in as
308        // UTF-32 (Python's internal representation for strings whose max
309        // codepoint exceeds U+FFFF).  UCS-4 is the historical synonym.
310        "utf-32" | "utf-32le" | "ucs-4" | "ucs-4le" | "ucs4" => Encoding::Utf32Le,
311        "utf-32be" | "ucs-4be" => Encoding::Utf32Be,
312        "ibm037" | "cp037" | "csibm037" => Encoding::Ebcdic037,
313        // Anything else: hand to the Other path so encoding_rs can take a swing.
314        _ => Encoding::Other(trimmed),
315    }
316}
317
318// ── small byte-slice helpers ─────────────────────────────────────────────────
319
320fn starts_with_ascii(haystack: &[u8], needle: &[u8]) -> bool {
321    haystack.starts_with(needle)
322}
323
324fn starts_with_ascii_ignore_case(haystack: &[u8], needle: &[u8]) -> bool {
325    if haystack.len() < needle.len() {
326        return false;
327    }
328    haystack[..needle.len()]
329        .iter()
330        .zip(needle)
331        .all(|(a, b)| a.to_ascii_lowercase() == b.to_ascii_lowercase())
332}
333
334fn ascii_equal_ignore_case(a: &[u8], b: &[u8]) -> bool {
335    a.len() == b.len()
336        && a.iter()
337            .zip(b)
338            .all(|(x, y)| x.to_ascii_lowercase() == y.to_ascii_lowercase())
339}
340
341fn find_byte(haystack: &[u8], needle: u8) -> Option<usize> {
342    memchr::memchr(needle, haystack)
343}
344
345#[cfg(test)]
346mod tests {
347    use super::*;
348
349    fn opts() -> HtmlParseOptions {
350        HtmlParseOptions::default()
351    }
352
353    #[test]
354    fn bom_utf8() {
355        let mut bytes = vec![0xEF, 0xBB, 0xBF];
356        bytes.extend_from_slice(b"<html></html>");
357        assert_eq!(sniff_html_encoding(&bytes, &opts()), Encoding::Utf8);
358    }
359
360    #[test]
361    fn bom_utf16le() {
362        let bytes = [0xFF, 0xFE, b'<', 0, b'a', 0];
363        assert_eq!(sniff_html_encoding(&bytes, &opts()), Encoding::Utf16Le);
364    }
365
366    #[test]
367    fn bom_utf16be() {
368        let bytes = [0xFE, 0xFF, 0, b'<', 0, b'a'];
369        assert_eq!(sniff_html_encoding(&bytes, &opts()), Encoding::Utf16Be);
370    }
371
372    #[test]
373    fn meta_charset_simple() {
374        let html = br#"<!DOCTYPE html><html><head><meta charset="UTF-8"></head></html>"#;
375        assert_eq!(prescan_meta_charset(html), Some("UTF-8".into()));
376    }
377
378    #[test]
379    fn meta_charset_unquoted() {
380        let html = br"<meta charset=UTF-8>";
381        assert_eq!(prescan_meta_charset(html), Some("UTF-8".into()));
382    }
383
384    #[test]
385    fn meta_charset_single_quotes() {
386        let html = br"<meta charset='windows-1252'>";
387        assert_eq!(prescan_meta_charset(html), Some("windows-1252".into()));
388    }
389
390    #[test]
391    fn meta_charset_case_insensitive() {
392        let html = br#"<META CHARSET="ISO-8859-1">"#;
393        assert_eq!(prescan_meta_charset(html), Some("ISO-8859-1".into()));
394    }
395
396    #[test]
397    fn meta_http_equiv_content_type() {
398        let html = br#"<meta http-equiv="Content-Type" content="text/html; charset=Shift_JIS">"#;
399        assert_eq!(prescan_meta_charset(html), Some("Shift_JIS".into()));
400    }
401
402    #[test]
403    fn meta_http_equiv_quoted_charset() {
404        let html =
405            br#"<meta http-equiv="content-type" content='text/html;charset="EUC-KR"'>"#;
406        assert_eq!(prescan_meta_charset(html), Some("EUC-KR".into()));
407    }
408
409    #[test]
410    fn meta_inside_comment_ignored() {
411        let html =
412            br#"<!-- <meta charset="UTF-8"> --><meta charset="ISO-8859-1">"#;
413        assert_eq!(prescan_meta_charset(html), Some("ISO-8859-1".into()));
414    }
415
416    #[test]
417    fn no_meta_falls_back_to_windows1252() {
418        let html = br"<html><head><title>x</title></head><body>x</body></html>";
419        assert_eq!(sniff_html_encoding(html, &opts()), Encoding::Windows1252);
420    }
421
422    #[test]
423    fn override_wins_over_bom() {
424        let mut bytes = vec![0xEF, 0xBB, 0xBF];
425        bytes.extend_from_slice(b"<html></html>");
426        let mut o = opts();
427        o.encoding_override = Some("ISO-8859-1".into());
428        assert_eq!(sniff_html_encoding(&bytes, &o), Encoding::Windows1252);
429    }
430
431    #[test]
432    fn label_iso88591_is_windows1252() {
433        assert_eq!(label_to_encoding("iso-8859-1"), Encoding::Windows1252);
434        assert_eq!(label_to_encoding("ISO-8859-1"), Encoding::Windows1252);
435        assert_eq!(label_to_encoding(" Latin1 "), Encoding::Windows1252);
436    }
437
438    #[test]
439    fn label_unknown_routes_to_other() {
440        match label_to_encoding("shift_jis") {
441            Encoding::Other(name) => assert_eq!(name, "shift_jis"),
442            _ => panic!("expected Other"),
443        }
444    }
445
446    #[test]
447    fn decode_with_meta_windows1252_content() {
448        // Windows-1252 byte 0x85 = ellipsis (U+2026), which is illegal
449        // in Latin-1.  Verify we treat it as Windows-1252 per the meta
450        // tag and decode correctly.
451        let mut bytes = b"<meta charset=\"windows-1252\"><body>".to_vec();
452        bytes.push(0x85);
453        bytes.extend_from_slice(b"</body>");
454        let (decoded, enc) = decode_html_input(&bytes, &opts()).unwrap();
455        assert_eq!(enc, Encoding::Windows1252);
456        let s = std::str::from_utf8(&decoded).expect("decoded must be UTF-8");
457        assert!(s.contains('\u{2026}'), "ellipsis should be decoded: {s:?}");
458    }
459
460    #[test]
461    fn decode_no_signal_uses_windows1252() {
462        // No BOM, no meta charset — fall back to Windows-1252.
463        let bytes = b"<html><body>plain</body></html>";
464        let (_, enc) = decode_html_input(bytes, &opts()).unwrap();
465        assert_eq!(enc, Encoding::Windows1252);
466    }
467}
sup_xml_core/html/encoding.rs

sup_xml_core/html/
encoding.rs
