fhp_encoding/
detect.rs

1//! Encoding detection from raw HTML bytes.
2//!
3//! Implements a simplified version of the HTML spec's encoding sniffing
4//! algorithm: BOM → meta prescan → UTF-8 fallback.
5
6use encoding_rs::Encoding;
7
8/// Maximum number of bytes to prescan for `<meta>` tags.
9const PRESCAN_LIMIT: usize = 1024;
10
11/// Detect the character encoding of raw HTML bytes.
12///
13/// The detection order is:
14/// 1. **BOM** — UTF-8 (`EF BB BF`), UTF-16 LE (`FF FE`), UTF-16 BE (`FE FF`)
15/// 2. **`<meta charset="...">`** — first occurrence in the first 1 KB
16/// 3. **`<meta http-equiv="Content-Type" content="...charset=...">`**
17/// 4. **Fallback** — UTF-8
18///
19/// # Example
20///
21/// ```
22/// use fhp_encoding::detect;
23///
24/// let html = b"\xEF\xBB\xBF<html>UTF-8 with BOM</html>";
25/// assert_eq!(detect(html).name(), "UTF-8");
26/// ```
27pub fn detect(input: &[u8]) -> &'static Encoding {
28    // 1. BOM detection.
29    if let Some(enc) = detect_bom(input) {
30        return enc;
31    }
32
33    // 2–3. Meta prescan.
34    if let Some(enc) = prescan_meta(input) {
35        return enc;
36    }
37
38    // 4. Fallback.
39    encoding_rs::UTF_8
40}
41
42/// Check for a Byte Order Mark at the start of the input.
43fn detect_bom(input: &[u8]) -> Option<&'static Encoding> {
44    if input.len() >= 3 && input[0] == 0xEF && input[1] == 0xBB && input[2] == 0xBF {
45        return Some(encoding_rs::UTF_8);
46    }
47    if input.len() >= 2 {
48        if input[0] == 0xFF && input[1] == 0xFE {
49            return Some(encoding_rs::UTF_16LE);
50        }
51        if input[0] == 0xFE && input[1] == 0xFF {
52            return Some(encoding_rs::UTF_16BE);
53        }
54    }
55    None
56}
57
58/// Prescan the first [`PRESCAN_LIMIT`] bytes for `<meta>` encoding declarations.
59///
60/// Looks for two patterns:
61/// - `<meta charset="ENCODING">`
62/// - `<meta http-equiv="Content-Type" content="...charset=ENCODING...">`
63///
64/// This is a byte-level (ASCII-oriented) scan, so it does not detect a `<meta>`
65/// declaration inside a BOM-less UTF-16 document (where bytes are interleaved
66/// with NULs). Such documents are expected to carry a BOM or an HTTP charset;
67/// matching browser behaviour, a BOM-less UTF-16 body without one falls back to
68/// UTF-8 detection.
69fn prescan_meta(input: &[u8]) -> Option<&'static Encoding> {
70    let limit = input.len().min(PRESCAN_LIMIT);
71    let haystack = &input[..limit];
72
73    // Find each '<' and check if it starts a <meta tag.
74    let mut pos = 0;
75    while pos < haystack.len() {
76        // Find next '<'.
77        let Some(lt) = memchr_byte(b'<', &haystack[pos..]) else {
78            break;
79        };
80        let lt = pos + lt;
81        pos = lt + 1;
82
83        // Check for "<meta" (case-insensitive).
84        if !starts_with_ci(&haystack[lt..], b"<meta") {
85            continue;
86        }
87
88        // Extract everything up to the closing '>'.
89        let tag_start = lt;
90        let Some(gt_offset) = memchr_byte(b'>', &haystack[tag_start..]) else {
91            break;
92        };
93        let tag_bytes = &haystack[tag_start..tag_start + gt_offset + 1];
94        pos = tag_start + gt_offset + 1;
95
96        // Try <meta charset="...">
97        if let Some(enc) = extract_charset_attr(tag_bytes) {
98            return Some(enc);
99        }
100
101        // Try <meta http-equiv="Content-Type" content="...charset=...">
102        if let Some(enc) = extract_http_equiv_charset(tag_bytes) {
103            return Some(enc);
104        }
105    }
106    None
107}
108
109/// Extract encoding from `charset="VALUE"` or `charset=VALUE` in a `<meta>` tag.
110fn extract_charset_attr(tag: &[u8]) -> Option<&'static Encoding> {
111    let charset_needle = b"charset";
112
113    let idx = find_subsequence_ci(tag, charset_needle)?;
114    let rest = &tag[idx + charset_needle.len()..];
115
116    // Skip whitespace and '='.
117    let rest = skip_ws(rest);
118    if rest.first() != Some(&b'=') {
119        return None;
120    }
121    let rest = skip_ws(&rest[1..]);
122
123    // Read the value (quoted or unquoted).
124    let value = read_attr_value(rest)?;
125    Encoding::for_label(value.as_bytes())
126}
127
128/// Extract encoding from `http-equiv="Content-Type" content="...charset=..."`.
129fn extract_http_equiv_charset(tag: &[u8]) -> Option<&'static Encoding> {
130    // Must have http-equiv="content-type".
131    if !contains_subsequence_ci(tag, b"http-equiv") {
132        return None;
133    }
134    if !contains_subsequence_ci(tag, b"content-type") {
135        return None;
136    }
137
138    // Find the `content` attribute — skip occurrences inside "content-type".
139    // We look for "content" followed by optional whitespace then "=".
140    let content_needle = b"content";
141    let mut search_start = 0;
142    let content_value = loop {
143        let idx = find_subsequence_ci(&tag[search_start..], content_needle)?;
144        let abs_idx = search_start + idx;
145        let after = &tag[abs_idx + content_needle.len()..];
146        let after = skip_ws(after);
147        if after.first() == Some(&b'=') {
148            let rest = skip_ws(&after[1..]);
149            break read_attr_value(rest)?;
150        }
151        // Not followed by '=', skip past and try again.
152        search_start = abs_idx + content_needle.len();
153    };
154
155    // Find charset= inside the content value.
156    let cv_lower: String = content_value.to_ascii_lowercase();
157    let charset_pos = cv_lower.find("charset=")?;
158    let enc_str = &cv_lower[charset_pos + 8..];
159    // Trim trailing ';' or whitespace.
160    let enc_str = enc_str.split(';').next().unwrap_or("").trim();
161
162    Encoding::for_label(enc_str.as_bytes())
163}
164
165// ---------------------------------------------------------------------------
166// Helper utilities
167// ---------------------------------------------------------------------------
168
169/// Simple `memchr`-like byte search (no external dep, just for 1KB prescan).
170#[inline]
171fn memchr_byte(needle: u8, haystack: &[u8]) -> Option<usize> {
172    haystack.iter().position(|&b| b == needle)
173}
174
175/// Case-insensitive prefix check.
176fn starts_with_ci(haystack: &[u8], needle: &[u8]) -> bool {
177    if haystack.len() < needle.len() {
178        return false;
179    }
180    haystack[..needle.len()]
181        .iter()
182        .zip(needle)
183        .all(|(&a, &b)| a.eq_ignore_ascii_case(&b))
184}
185
186/// Find the first occurrence of `needle` in `haystack` (case-insensitive).
187fn find_subsequence_ci(haystack: &[u8], needle: &[u8]) -> Option<usize> {
188    haystack
189        .windows(needle.len())
190        .position(|w| w.eq_ignore_ascii_case(needle))
191}
192
193/// Check if `haystack` contains `needle` (case-insensitive).
194fn contains_subsequence_ci(haystack: &[u8], needle: &[u8]) -> bool {
195    find_subsequence_ci(haystack, needle).is_some()
196}
197
198/// Skip leading ASCII whitespace.
199fn skip_ws(input: &[u8]) -> &[u8] {
200    let start = input
201        .iter()
202        .position(|b| !b.is_ascii_whitespace())
203        .unwrap_or(input.len());
204    &input[start..]
205}
206
207/// Read an attribute value — either quoted (`"..."` / `'...'`) or bare token.
208fn read_attr_value(input: &[u8]) -> Option<String> {
209    if input.is_empty() {
210        return None;
211    }
212    let quote = input[0];
213    if quote == b'"' || quote == b'\'' {
214        let end = memchr_byte(quote, &input[1..])?;
215        let value = &input[1..1 + end];
216        Some(String::from_utf8_lossy(value).into_owned())
217    } else {
218        // Bare token: ends at whitespace, '>', '/', or ';'.
219        let end = input
220            .iter()
221            .position(|&b| b.is_ascii_whitespace() || b == b'>' || b == b'/' || b == b';')
222            .unwrap_or(input.len());
223        if end == 0 {
224            return None;
225        }
226        Some(String::from_utf8_lossy(&input[..end]).into_owned())
227    }
228}
229
230#[cfg(test)]
231mod tests {
232    use super::*;
233
234    #[test]
235    fn bom_utf8() {
236        let input = b"\xEF\xBB\xBF<html></html>";
237        assert_eq!(detect(input).name(), "UTF-8");
238    }
239
240    #[test]
241    fn bom_utf16le() {
242        let input = b"\xFF\xFE<\x00h\x00t\x00m\x00l\x00";
243        assert_eq!(detect(input).name(), "UTF-16LE");
244    }
245
246    #[test]
247    fn bom_utf16be() {
248        let input = b"\xFE\xFF\x00<\x00h\x00t\x00m\x00l";
249        assert_eq!(detect(input).name(), "UTF-16BE");
250    }
251
252    #[test]
253    fn meta_charset_double_quote() {
254        let input = b"<html><head><meta charset=\"windows-1252\"></head></html>";
255        assert_eq!(detect(input).name(), "windows-1252");
256    }
257
258    #[test]
259    fn meta_charset_single_quote() {
260        let input = b"<html><head><meta charset='iso-8859-1'></head></html>";
261        assert_eq!(detect(input).name(), "windows-1252"); // encoding_rs maps iso-8859-1 → windows-1252
262    }
263
264    #[test]
265    fn meta_charset_case_insensitive() {
266        let input = b"<HTML><HEAD><META CHARSET=\"UTF-8\"></HEAD></HTML>";
267        assert_eq!(detect(input).name(), "UTF-8");
268    }
269
270    #[test]
271    fn meta_http_equiv() {
272        let input = b"<html><head><meta http-equiv=\"Content-Type\" content=\"text/html; charset=windows-1254\"></head></html>";
273        assert_eq!(detect(input).name(), "windows-1254");
274    }
275
276    #[test]
277    fn fallback_utf8() {
278        let input = b"<html><head></head><body>Hello</body></html>";
279        assert_eq!(detect(input).name(), "UTF-8");
280    }
281
282    #[test]
283    fn empty_input() {
284        assert_eq!(detect(b"").name(), "UTF-8");
285    }
286
287    #[test]
288    fn no_meta_in_first_1kb() {
289        // Put meta after 1KB — should not be detected.
290        let mut input = vec![b' '; 1100];
291        let meta = b"<meta charset=\"iso-8859-1\">";
292        input.extend_from_slice(meta);
293        assert_eq!(detect(&input).name(), "UTF-8"); // fallback
294    }
295
296    #[test]
297    fn meta_charset_bare_value() {
298        let input = b"<meta charset=utf-8>";
299        assert_eq!(detect(input).name(), "UTF-8");
300    }
301
302    #[test]
303    fn bom_takes_priority_over_meta() {
304        // UTF-8 BOM but meta says windows-1252. BOM wins.
305        let input = b"\xEF\xBB\xBF<html><head><meta charset=\"windows-1252\"></head></html>";
306        assert_eq!(detect(input).name(), "UTF-8");
307    }
308}
fhp_encoding/detect.rs

fhp_encoding/
detect.rs
