Skip to main content

parsec_encoding/
prescan.rs

1// SPDX-License-Identifier: Apache-2.0
2// Copyright 2026 James Conway
3
4//! The HTML Standard's "prescan a byte stream to determine its encoding",
5//! "get an attribute", "get an XML encoding" and "extracting a character
6//! encoding from a meta element", step for step.
7
8use encoding_rs::{Encoding, UTF_8, UTF_16BE, UTF_16LE, WINDOWS_1252, X_USER_DEFINED};
9
10/// The encoding a `<meta>` (or an XML declaration) in `bytes` declares, as
11/// the HTML Standard's prescan finds it, reading `bytes` and nothing
12/// past them. [`crate::sniff`] gives it the first 1,024 bytes, as the
13/// spec advises; pass more or fewer to scan further or less.
14///
15/// ```
16/// let found = parsec_encoding::prescan(b"<meta charset=shift_jis>");
17/// assert_eq!(found, Some(encoding_rs::SHIFT_JIS));
18/// ```
19pub fn prescan(bytes: &[u8]) -> Option<&'static Encoding> {
20    // Running out of bytes anywhere in the scan ends it, and then only an
21    // XML declaration can still give an answer.
22    scan(bytes).or_else(|| xml_encoding(bytes))
23}
24
25/// The prescan's steps. `None` is running out of bytes; the loop has no
26/// other way to end. A `Some` comes from the first six bytes or a
27/// `<meta>` read to its `>`, so more bytes can't change it.
28pub(crate) fn scan(b: &[u8]) -> Option<&'static Encoding> {
29    if b.starts_with(&[0x3C, 0, 0x3F, 0, 0x78, 0]) {
30        return Some(UTF_16LE);
31    }
32    if b.starts_with(&[0, 0x3C, 0, 0x3F, 0, 0x78]) {
33        return Some(UTF_16BE);
34    }
35    let mut p = 0;
36    loop {
37        let rest = b.get(p..).filter(|r| !r.is_empty())?;
38        if rest.starts_with(b"<!--") {
39            // The first `>` after two `-`, which may be the comment
40            // opener's own: `<!-->` ends here.
41            p += 2 + find(&rest[2..], b"-->")? + 2;
42        } else if rest.len() >= 6
43            && rest[0] == b'<'
44            && rest[1..5].eq_ignore_ascii_case(b"meta")
45            && (is_space(rest[5]) || rest[5] == b'/')
46        {
47            p += 5;
48            if let Some(found) = meta(b, &mut p)? {
49                return Some(found);
50            }
51        } else if rest[0] == b'<' && starts_tag(&rest[1..]) {
52            p += rest
53                .iter()
54                .position(|&c| matches!(c, b'\t' | b'\n' | b'\x0C' | b'\r' | b' ' | b'>'))?;
55            while attribute(b, &mut p)?.is_some() {}
56        } else if rest.starts_with(b"<!") || rest.starts_with(b"</") || rest.starts_with(b"<?") {
57            p += 1 + rest[1..].iter().position(|&c| c == b'>')?;
58        }
59        p += 1;
60    }
61}
62
63/// `<` is followed by an optional `/` and an ASCII letter.
64fn starts_tag(after: &[u8]) -> bool {
65    let after = after.strip_prefix(b"/").unwrap_or(after);
66    after.first().is_some_and(u8::is_ascii_alphabetic)
67}
68
69/// A `<meta>`'s attributes, from `p` just after `<meta`: `Some(None)` when
70/// it declares nothing usable, `None` when the bytes ran out.
71fn meta(b: &[u8], p: &mut usize) -> Option<Option<&'static Encoding>> {
72    let mut seen: Vec<Vec<u8>> = Vec::new();
73    let mut got_pragma = false;
74    let mut need_pragma: Option<bool> = None;
75    // The spec's null (`None`) is distinct from failure (`Some(None)`): a
76    // `charset` that names no encoding still stops `content` from counting.
77    let mut charset: Option<Option<&'static Encoding>> = None;
78    while let Some((name, value)) = attribute(b, p)? {
79        if seen.contains(&name) {
80            continue;
81        }
82        match name.as_slice() {
83            b"http-equiv" if value == b"content-type" => got_pragma = true,
84            b"content" if charset.is_none() => {
85                if let Some(found) = charset_in_content(&value) {
86                    charset = Some(Some(found));
87                    need_pragma = Some(true);
88                }
89            }
90            b"charset" => {
91                charset = Some(Encoding::for_label(&value));
92                need_pragma = Some(false);
93            }
94            _ => {}
95        }
96        seen.push(name);
97    }
98    Some(match (need_pragma, charset) {
99        (Some(need), Some(Some(found))) if !need || got_pragma => Some(as_declared(found)),
100        _ => None,
101    })
102}
103
104/// A declared encoding as HTML uses it: a page that says UTF-16 in ASCII
105/// can't be UTF-16, and x-user-defined means windows-1252.
106pub(crate) fn as_declared(e: &'static Encoding) -> &'static Encoding {
107    if e == UTF_16BE || e == UTF_16LE {
108        UTF_8
109    } else if e == X_USER_DEFINED {
110        WINDOWS_1252
111    } else {
112        e
113    }
114}
115
116fn is_space(c: u8) -> bool {
117    matches!(c, b'\t' | b'\n' | b'\x0C' | b'\r' | b' ')
118}
119
120/// "Get an attribute": `Some(None)` when there isn't one (a `>`), `None`
121/// when the bytes ran out. Names and values come back lowercased.
122#[allow(clippy::type_complexity)]
123fn attribute(b: &[u8], p: &mut usize) -> Option<Option<(Vec<u8>, Vec<u8>)>> {
124    while is_space(*b.get(*p)?) || b[*p] == b'/' {
125        *p += 1;
126    }
127    if b[*p] == b'>' {
128        return Some(None);
129    }
130    let mut name = Vec::new();
131    let mut value = Vec::new();
132    loop {
133        match *b.get(*p)? {
134            b'=' if !name.is_empty() => {
135                *p += 1;
136                break;
137            }
138            c if is_space(c) => {
139                while is_space(*b.get(*p)?) {
140                    *p += 1;
141                }
142                if b[*p] != b'=' {
143                    return Some(Some((name, value)));
144                }
145                *p += 1;
146                break;
147            }
148            b'/' | b'>' => return Some(Some((name, value))),
149            c => name.push(c.to_ascii_lowercase()),
150        }
151        *p += 1;
152    }
153    while is_space(*b.get(*p)?) {
154        *p += 1;
155    }
156    match b[*p] {
157        quote @ (b'"' | b'\'') => loop {
158            *p += 1;
159            let c = *b.get(*p)?;
160            if c == quote {
161                *p += 1;
162                return Some(Some((name, value)));
163            }
164            value.push(c.to_ascii_lowercase());
165        },
166        b'>' => return Some(Some((name, value))),
167        c => {
168            value.push(c.to_ascii_lowercase());
169            *p += 1;
170        }
171    }
172    loop {
173        match *b.get(*p)? {
174            c if is_space(c) || c == b'>' => return Some(Some((name, value))),
175            c => value.push(c.to_ascii_lowercase()),
176        }
177        *p += 1;
178    }
179}
180
181/// "Get an XML encoding": the encoding in a leading `<?xml ... ?>`.
182fn xml_encoding(b: &[u8]) -> Option<&'static Encoding> {
183    if !b.starts_with(b"<?xml") {
184        return None;
185    }
186    let end = b.iter().position(|&c| c == b'>')?;
187    let mut p = find(&b[..end], b"encoding")? + b"encoding".len();
188    while *b.get(p)? <= 0x20 {
189        p += 1;
190    }
191    if b[p] != b'=' {
192        return None;
193    }
194    p += 1;
195    while *b.get(p)? <= 0x20 {
196        p += 1;
197    }
198    let quote = b[p];
199    if quote != b'"' && quote != b'\'' {
200        return None;
201    }
202    p += 1;
203    let len = b[p..].iter().position(|&c| c == quote)?;
204    let name = &b[p..p + len];
205    if name.iter().any(|&c| c <= 0x20) {
206        return None;
207    }
208    let found = Encoding::for_label(name)?;
209    Some(if found == UTF_16BE || found == UTF_16LE {
210        UTF_8
211    } else {
212        found
213    })
214}
215
216/// "The algorithm for extracting a character encoding from a meta
217/// element", given a `content` attribute's value.
218pub(crate) fn charset_in_content(s: &[u8]) -> Option<&'static Encoding> {
219    let mut p = 0;
220    loop {
221        p += s[p..]
222            .windows(7)
223            .position(|w| w.eq_ignore_ascii_case(b"charset"))?
224            + 7;
225        while s.get(p).is_some_and(|c| c.is_ascii_whitespace()) {
226            p += 1;
227        }
228        if s.get(p) != Some(&b'=') {
229            continue;
230        }
231        p += 1;
232        while s.get(p).is_some_and(|c| c.is_ascii_whitespace()) {
233            p += 1;
234        }
235        let rest = &s[p..];
236        return match *rest.first()? {
237            quote @ (b'"' | b'\'') => {
238                let len = rest[1..].iter().position(|&c| c == quote)?;
239                Encoding::for_label(&rest[1..1 + len])
240            }
241            _ => {
242                let len = rest
243                    .iter()
244                    .position(|&c| c.is_ascii_whitespace() || c == b';')
245                    .unwrap_or(rest.len());
246                Encoding::for_label(&rest[..len])
247            }
248        };
249    }
250}
251
252fn find(haystack: &[u8], needle: &[u8]) -> Option<usize> {
253    haystack.windows(needle.len()).position(|w| w == needle)
254}
255
256#[cfg(test)]
257mod tests {
258    use super::*;
259    use encoding_rs::{EUC_JP, ISO_8859_2, SHIFT_JIS, WINDOWS_1251};
260
261    #[test]
262    fn meta_charset_and_pragma() {
263        assert_eq!(prescan(b"<meta charset='euc-jp'>"), Some(EUC_JP));
264        assert_eq!(
265            prescan(b"<meta http-equiv=Content-Type content='text/html; charset=windows-1251'>"),
266            Some(WINDOWS_1251)
267        );
268        // A content charset without the pragma counts for nothing.
269        assert_eq!(prescan(b"<meta content='charset=windows-1251'>"), None);
270        // The pragma may come after the content.
271        assert_eq!(
272            prescan(b"<meta content='charset=iso-8859-2' http-equiv=content-type>"),
273            Some(ISO_8859_2)
274        );
275    }
276
277    #[test]
278    fn charset_failure_blocks_content() {
279        let b = b"<meta charset=nonsense http-equiv=content-type content='charset=utf-8'>";
280        assert_eq!(prescan(b), None);
281    }
282
283    #[test]
284    fn duplicates_and_comments() {
285        assert_eq!(
286            prescan(b"<meta charset=sjis charset=euc-jp>"),
287            Some(SHIFT_JIS)
288        );
289        assert_eq!(prescan(b"<!-- <meta charset=euc-jp> --><p>"), None);
290        // `<!-->` is a whole comment.
291        assert_eq!(prescan(b"<!--><meta charset=euc-jp>"), Some(EUC_JP));
292        // Inside another tag's attribute value, a meta isn't seen.
293        assert_eq!(prescan(b"<a title='<meta charset=euc-jp>'>x"), None);
294    }
295
296    #[test]
297    fn utf16_and_user_defined_are_mapped() {
298        assert_eq!(prescan(b"<meta charset=utf-16le>"), Some(UTF_8));
299        assert_eq!(
300            prescan(b"<meta charset=x-user-defined>"),
301            Some(WINDOWS_1252)
302        );
303        assert_eq!(prescan(b"<\0?\0x\0m\0l\0"), Some(UTF_16LE));
304        assert_eq!(prescan(b"\0<\0?\0x\0m\0l"), Some(UTF_16BE));
305    }
306
307    #[test]
308    fn running_out_falls_back_to_xml() {
309        assert_eq!(prescan(b"<meta charset=euc-jp"), None);
310        assert_eq!(
311            prescan(b"<?xml version='1.0' encoding='Shift_JIS'?><p"),
312            Some(SHIFT_JIS)
313        );
314        assert_eq!(
315            prescan(b"<?xml version='1.0' encoding='utf-16'?><p"),
316            Some(UTF_8)
317        );
318        assert_eq!(prescan(b"<?xml encoding = \"euc-jp\"?>"), Some(EUC_JP));
319        // A scan that finds no `<meta>` always ends by running out.
320        assert_eq!(prescan(b"<?xml encoding='euc-jp'?><p>"), Some(EUC_JP));
321    }
322
323    #[test]
324    fn content_extraction() {
325        assert_eq!(
326            charset_in_content(b"text/html; charset=sjis"),
327            Some(SHIFT_JIS)
328        );
329        assert_eq!(
330            charset_in_content(b"charsetcharset = \"euc-jp\""),
331            Some(EUC_JP)
332        );
333        assert_eq!(charset_in_content(b"charset='euc-jp"), None);
334        assert_eq!(charset_in_content(b"charset="), None);
335        assert_eq!(charset_in_content(b"CHARSET=sjis;x"), Some(SHIFT_JIS));
336    }
337
338    #[test]
339    fn every_prefix_runs_out_cleanly() {
340        let page: &[u8] = b"<?xml encoding = 'x'?><!-- c --><a b='c' d=e f><meta \
341            http-equiv=content-type content=\"charset = 'sjis'\" charset>";
342        for end in 0..=page.len() {
343            prescan(&page[..end]);
344        }
345        // The bare `charset` at the end names nothing, and it overrides
346        // the `content`.
347        assert_eq!(prescan(page), None);
348        let without = b"<meta http-equiv=content-type content=\"charset = 'sjis'\">";
349        assert_eq!(prescan(without), Some(SHIFT_JIS));
350    }
351}