fhp_encoding/
detect.rs

1//! Encoding detection from raw HTML bytes.
2//!
3//! Implements a simplified version of the HTML spec's encoding sniffing
4//! algorithm: BOM → meta prescan → UTF-8 fallback.
5
6use encoding_rs::Encoding;
7
8/// Maximum number of bytes to prescan for `<meta>` tags.
9const PRESCAN_LIMIT: usize = 1024;
10
11/// Detect the character encoding of raw HTML bytes.
12///
13/// The detection order is:
14/// 1. **BOM** — UTF-8 (`EF BB BF`), UTF-16 LE (`FF FE`), UTF-16 BE (`FE FF`)
15/// 2. **`<meta charset="...">`** — first occurrence in the first 1 KB
16/// 3. **`<meta http-equiv="Content-Type" content="...charset=...">`**
17/// 4. **Fallback** — UTF-8
18///
19/// # Example
20///
21/// ```
22/// use fhp_encoding::detect;
23///
24/// let html = b"\xEF\xBB\xBF<html>UTF-8 with BOM</html>";
25/// assert_eq!(detect(html).name(), "UTF-8");
26/// ```
27pub fn detect(input: &[u8]) -> &'static Encoding {
28    // 1. BOM detection.
29    if let Some(enc) = detect_bom(input) {
30        return enc;
31    }
32
33    // 2–3. Meta prescan.
34    if let Some(enc) = prescan_meta(input) {
35        return enc;
36    }
37
38    // 4. Fallback.
39    encoding_rs::UTF_8
40}
41
42/// Check for a Byte Order Mark at the start of the input.
43fn detect_bom(input: &[u8]) -> Option<&'static Encoding> {
44    if input.len() >= 3 && input[0] == 0xEF && input[1] == 0xBB && input[2] == 0xBF {
45        return Some(encoding_rs::UTF_8);
46    }
47    if input.len() >= 2 {
48        if input[0] == 0xFF && input[1] == 0xFE {
49            return Some(encoding_rs::UTF_16LE);
50        }
51        if input[0] == 0xFE && input[1] == 0xFF {
52            return Some(encoding_rs::UTF_16BE);
53        }
54    }
55    None
56}
57
58/// Prescan the first [`PRESCAN_LIMIT`] bytes for `<meta>` encoding declarations.
59///
60/// Looks for two patterns:
61/// - `<meta charset="ENCODING">`
62/// - `<meta http-equiv="Content-Type" content="...charset=ENCODING...">`
63///
64/// This is a byte-level (ASCII-oriented) scan, so it does not detect a `<meta>`
65/// declaration inside a BOM-less UTF-16 document (where bytes are interleaved
66/// with NULs). Such documents are expected to carry a BOM or an HTTP charset;
67/// matching browser behaviour, a BOM-less UTF-16 body without one falls back to
68/// UTF-8 detection.
69fn prescan_meta(input: &[u8]) -> Option<&'static Encoding> {
70    let limit = input.len().min(PRESCAN_LIMIT);
71    let haystack = &input[..limit];
72
73    // Find each '<' and check if it starts a <meta tag.
74    let mut pos = 0;
75    while pos < haystack.len() {
76        // Find next '<'.
77        let Some(lt) = memchr_byte(b'<', &haystack[pos..]) else {
78            break;
79        };
80        let lt = pos + lt;
81        pos = lt + 1;
82
83        // Check for "<meta" (case-insensitive).
84        if !starts_with_ci(&haystack[lt..], b"<meta") {
85            continue;
86        }
87
88        // Extract everything up to the closing '>'.
89        let tag_start = lt;
90        let Some(gt_offset) = memchr_byte(b'>', &haystack[tag_start..]) else {
91            break;
92        };
93        let tag_bytes = &haystack[tag_start..tag_start + gt_offset + 1];
94        pos = tag_start + gt_offset + 1;
95
96        // Try <meta charset="...">
97        if let Some(enc) = extract_charset_attr(tag_bytes) {
98            return Some(remap_meta_encoding(enc));
99        }
100
101        // Try <meta http-equiv="Content-Type" content="...charset=...">
102        if let Some(enc) = extract_http_equiv_charset(tag_bytes) {
103            return Some(remap_meta_encoding(enc));
104        }
105    }
106    None
107}
108
109/// Extract encoding from `charset="VALUE"` or `charset=VALUE` in a `<meta>` tag.
110fn extract_charset_attr(tag: &[u8]) -> Option<&'static Encoding> {
111    let charset_needle = b"charset";
112
113    let idx = find_subsequence_ci(tag, charset_needle)?;
114    let rest = &tag[idx + charset_needle.len()..];
115
116    // Skip whitespace and '='.
117    let rest = skip_ws(rest);
118    if rest.first() != Some(&b'=') {
119        return None;
120    }
121    let rest = skip_ws(&rest[1..]);
122
123    // Read the value (quoted or unquoted).
124    let value = read_attr_value(rest)?;
125    Encoding::for_label(value.as_bytes())
126}
127
128/// Extract encoding from `http-equiv="Content-Type" content="...charset=..."`.
129fn extract_http_equiv_charset(tag: &[u8]) -> Option<&'static Encoding> {
130    // Must have http-equiv="content-type".
131    if !contains_subsequence_ci(tag, b"http-equiv") {
132        return None;
133    }
134    if !contains_subsequence_ci(tag, b"content-type") {
135        return None;
136    }
137
138    // Find the `content` attribute — skip occurrences inside "content-type".
139    // We look for "content" followed by optional whitespace then "=".
140    let content_needle = b"content";
141    let mut search_start = 0;
142    let content_value = loop {
143        let idx = find_subsequence_ci(&tag[search_start..], content_needle)?;
144        let abs_idx = search_start + idx;
145        let after = &tag[abs_idx + content_needle.len()..];
146        let after = skip_ws(after);
147        if after.first() == Some(&b'=') {
148            let rest = skip_ws(&after[1..]);
149            break read_attr_value(rest)?;
150        }
151        // Not followed by '=', skip past and try again.
152        search_start = abs_idx + content_needle.len();
153    };
154
155    // Find charset= inside the content value.
156    let cv_lower: String = content_value.to_ascii_lowercase();
157    let charset_pos = cv_lower.find("charset=")?;
158    let enc_str = &cv_lower[charset_pos + 8..];
159    // Trim trailing ';' or whitespace.
160    let enc_str = enc_str.split(';').next().unwrap_or("").trim();
161
162    Encoding::for_label(enc_str.as_bytes())
163}
164
165/// Apply the HTML spec's meta-prescan encoding remap.
166///
167/// A `<meta>`-declared `utf-16` / `utf-16le` / `utf-16be` label is changed to
168/// UTF-8: a document whose `<meta>` tag was found by an ASCII byte scan cannot
169/// actually be UTF-16 (UTF-16 interleaves NUL bytes that would have prevented
170/// the literal ASCII match), so decoding it as UTF-16 would produce mojibake.
171fn remap_meta_encoding(enc: &'static Encoding) -> &'static Encoding {
172    if enc == encoding_rs::UTF_16LE || enc == encoding_rs::UTF_16BE {
173        encoding_rs::UTF_8
174    } else {
175        enc
176    }
177}
178
179// ---------------------------------------------------------------------------
180// Helper utilities
181// ---------------------------------------------------------------------------
182
183/// Simple `memchr`-like byte search (no external dep, just for 1KB prescan).
184#[inline]
185fn memchr_byte(needle: u8, haystack: &[u8]) -> Option<usize> {
186    haystack.iter().position(|&b| b == needle)
187}
188
189/// Case-insensitive prefix check.
190fn starts_with_ci(haystack: &[u8], needle: &[u8]) -> bool {
191    if haystack.len() < needle.len() {
192        return false;
193    }
194    haystack[..needle.len()]
195        .iter()
196        .zip(needle)
197        .all(|(&a, &b)| a.eq_ignore_ascii_case(&b))
198}
199
200/// Find the first occurrence of `needle` in `haystack` (case-insensitive).
201fn find_subsequence_ci(haystack: &[u8], needle: &[u8]) -> Option<usize> {
202    haystack
203        .windows(needle.len())
204        .position(|w| w.eq_ignore_ascii_case(needle))
205}
206
207/// Check if `haystack` contains `needle` (case-insensitive).
208fn contains_subsequence_ci(haystack: &[u8], needle: &[u8]) -> bool {
209    find_subsequence_ci(haystack, needle).is_some()
210}
211
212/// Skip leading ASCII whitespace.
213fn skip_ws(input: &[u8]) -> &[u8] {
214    let start = input
215        .iter()
216        .position(|b| !b.is_ascii_whitespace())
217        .unwrap_or(input.len());
218    &input[start..]
219}
220
221/// Read an attribute value — either quoted (`"..."` / `'...'`) or bare token.
222fn read_attr_value(input: &[u8]) -> Option<String> {
223    if input.is_empty() {
224        return None;
225    }
226    let quote = input[0];
227    if quote == b'"' || quote == b'\'' {
228        let end = memchr_byte(quote, &input[1..])?;
229        let value = &input[1..1 + end];
230        Some(String::from_utf8_lossy(value).into_owned())
231    } else {
232        // Bare token: ends at whitespace, '>', '/', or ';'.
233        let end = input
234            .iter()
235            .position(|&b| b.is_ascii_whitespace() || b == b'>' || b == b'/' || b == b';')
236            .unwrap_or(input.len());
237        if end == 0 {
238            return None;
239        }
240        Some(String::from_utf8_lossy(&input[..end]).into_owned())
241    }
242}
243
244#[cfg(test)]
245mod tests {
246    use super::*;
247
248    #[test]
249    fn bom_utf8() {
250        let input = b"\xEF\xBB\xBF<html></html>";
251        assert_eq!(detect(input).name(), "UTF-8");
252    }
253
254    #[test]
255    fn bom_utf16le() {
256        let input = b"\xFF\xFE<\x00h\x00t\x00m\x00l\x00";
257        assert_eq!(detect(input).name(), "UTF-16LE");
258    }
259
260    #[test]
261    fn bom_utf16be() {
262        let input = b"\xFE\xFF\x00<\x00h\x00t\x00m\x00l";
263        assert_eq!(detect(input).name(), "UTF-16BE");
264    }
265
266    #[test]
267    fn meta_charset_double_quote() {
268        let input = b"<html><head><meta charset=\"windows-1252\"></head></html>";
269        assert_eq!(detect(input).name(), "windows-1252");
270    }
271
272    #[test]
273    fn meta_charset_single_quote() {
274        let input = b"<html><head><meta charset='iso-8859-1'></head></html>";
275        assert_eq!(detect(input).name(), "windows-1252"); // encoding_rs maps iso-8859-1 → windows-1252
276    }
277
278    #[test]
279    fn meta_charset_case_insensitive() {
280        let input = b"<HTML><HEAD><META CHARSET=\"UTF-8\"></HEAD></HTML>";
281        assert_eq!(detect(input).name(), "UTF-8");
282    }
283
284    #[test]
285    fn meta_http_equiv() {
286        let input = b"<html><head><meta http-equiv=\"Content-Type\" content=\"text/html; charset=windows-1254\"></head></html>";
287        assert_eq!(detect(input).name(), "windows-1254");
288    }
289
290    #[test]
291    fn fallback_utf8() {
292        let input = b"<html><head></head><body>Hello</body></html>";
293        assert_eq!(detect(input).name(), "UTF-8");
294    }
295
296    #[test]
297    fn empty_input() {
298        assert_eq!(detect(b"").name(), "UTF-8");
299    }
300
301    #[test]
302    fn no_meta_in_first_1kb() {
303        // Put meta after 1KB — should not be detected.
304        let mut input = vec![b' '; 1100];
305        let meta = b"<meta charset=\"iso-8859-1\">";
306        input.extend_from_slice(meta);
307        assert_eq!(detect(&input).name(), "UTF-8"); // fallback
308    }
309
310    #[test]
311    fn meta_charset_bare_value() {
312        let input = b"<meta charset=utf-8>";
313        assert_eq!(detect(input).name(), "UTF-8");
314    }
315
316    #[test]
317    fn bom_takes_priority_over_meta() {
318        // UTF-8 BOM but meta says windows-1252. BOM wins.
319        let input = b"\xEF\xBB\xBF<html><head><meta charset=\"windows-1252\"></head></html>";
320        assert_eq!(detect(input).name(), "UTF-8");
321    }
322
323    #[test]
324    fn meta_charset_utf16_remaps_to_utf8() {
325        // A document that was ASCII-prescannable cannot actually be UTF-16;
326        // the HTML spec mandates remapping a meta-declared utf-16 label to UTF-8.
327        let input = b"<html><head><meta charset=\"utf-16\"></head><body>Hello</body></html>";
328        assert_eq!(detect(input).name(), "UTF-8");
329    }
330
331    #[test]
332    fn meta_charset_utf16le_remaps_to_utf8() {
333        let input = b"<meta charset=\"utf-16le\">Hello";
334        assert_eq!(detect(input).name(), "UTF-8");
335    }
336
337    #[test]
338    fn meta_charset_utf16be_remaps_to_utf8() {
339        let input = b"<meta charset=\"utf-16be\">Hello";
340        assert_eq!(detect(input).name(), "UTF-8");
341    }
342
343    #[test]
344    fn meta_http_equiv_utf16_remaps_to_utf8() {
345        let input =
346            b"<meta http-equiv=\"Content-Type\" content=\"text/html; charset=utf-16\">Hello";
347        assert_eq!(detect(input).name(), "UTF-8");
348    }
349}
fhp_encoding/detect.rs

fhp_encoding/
detect.rs
