Skip to main content

parsec_encoding/
lib.rs

1// SPDX-License-Identifier: Apache-2.0
2// Copyright 2026 James Conway
3
4//! Bytes into Parsec. [`parse_bytes`] finds a page's encoding the way the
5//! HTML Standard says a browser does, decodes it and parses it.
6//! [`ByteParser`] does the same with the bytes as they arrive.
7//!
8//! The decoding is [`encoding_rs`], Firefox's implementation of the WHATWG
9//! Encoding Standard, re-exported here. It has every encoding the web
10//! uses, decoding and encoding, whole or streamed:
11//!
12//! ```
13//! use parsec_encoding::Encoding;
14//!
15//! let sjis = Encoding::for_label(b"sjis").unwrap();
16//! assert_eq!(sjis.name(), "Shift_JIS");
17//! let (text, _, had_errors) = sjis.decode(b"\x93\xfa\x96\x7b");
18//! assert_eq!((&*text, had_errors), ("日本", false));
19//! let (bytes, _, _) = sjis.encode("日本");
20//! assert_eq!(&*bytes, b"\x93\xfa\x96\x7b");
21//! ```
22
23use std::time::Instant;
24
25pub use encoding_rs::{self, Encoding};
26use encoding_rs::{UTF_8, UTF_16BE, UTF_16LE};
27use parsec::{AttrNs, Document, Kind, NodeId, Ns, ParseError};
28
29mod prescan;
30pub use prescan::prescan;
31mod stream;
32pub use stream::ByteParser;
33mod offsets;
34pub use offsets::Offsets;
35
36/// How sure the parser is of an encoding, in the HTML Standard's terms.
37#[derive(Debug, Clone, Copy, PartialEq, Eq)]
38pub enum Confidence {
39    /// A guess: from the prescan, the UTF-8 autodetection or the
40    /// fallback. A `<meta>` the parser
41    /// meets can still change it.
42    Tentative,
43    /// From a byte order mark, the transport layer, or a `<meta>` that
44    /// agreed with the guess or replaced it.
45    Certain,
46}
47
48/// How to turn bytes into a document.
49#[derive(Debug, Clone, Copy)]
50pub struct Options {
51    /// The parse itself.
52    pub html: parsec::Options,
53    /// The encoding the transport layer gave, such as the `charset` in an
54    /// HTTP `Content-Type`: look it up with [`Encoding::for_label`]. A byte
55    /// order mark still wins over it, as in browsers.
56    pub transport: Option<&'static Encoding>,
57    /// The encoding when nothing in or around the page names one and its
58    /// bytes aren't valid UTF-8: windows-1252 by default, as in browsers set to English.
59    /// [`locale_default`] gives the one a browser in another language
60    /// would use.
61    pub fallback: &'static Encoding,
62}
63
64impl Default for Options {
65    fn default() -> Self {
66        Options {
67            html: parsec::Options::default(),
68            transport: None,
69            fallback: encoding_rs::WINDOWS_1252,
70        }
71    }
72}
73
74/// A document parsed from bytes, and how its encoding was decided.
75pub struct Parsed {
76    pub document: Document,
77    /// The encoding the document was decoded with: what JavaScript's
78    /// `document.characterSet` would say.
79    pub encoding: &'static Encoding,
80    pub confidence: Confidence,
81}
82
83/// The HTML Standard's encoding sniffing algorithm, for a parser that has
84/// every byte: the byte order mark, then `transport`, then the prescan of
85/// the first 1,024 bytes, then autodetection, then `fallback`.
86///
87/// The spec lets a browser guess from the bytes before falling back, and
88/// browsers do: Firefox's chardetng and Chrome's CED both recognise
89/// UTF-8. Parsec's guess is that one only, and deterministic: bytes with
90/// something beyond ASCII that are all valid UTF-8 are UTF-8, tentative,
91/// so a later `<meta>` can still change it. Anything else falls back.
92///
93/// ```
94/// use parsec_encoding::{Confidence, sniff};
95/// let page = b"<meta charset=windows-1251><p>\xcf\xf0\xe8\xe2\xe5\xf2";
96/// let found = sniff(page, None, encoding_rs::WINDOWS_1252);
97/// assert_eq!(found, (encoding_rs::WINDOWS_1251, Confidence::Tentative));
98/// ```
99pub fn sniff(
100    bytes: &[u8],
101    transport: Option<&'static Encoding>,
102    fallback: &'static Encoding,
103) -> (&'static Encoding, Confidence) {
104    guess(bytes, transport, fallback, true)
105}
106
107/// [`sniff`] for `bytes` that are the whole input when `complete`, or
108/// only its start, where a UTF-8 sequence cut off at the end still counts.
109pub(crate) fn guess(
110    bytes: &[u8],
111    transport: Option<&'static Encoding>,
112    fallback: &'static Encoding,
113    complete: bool,
114) -> (&'static Encoding, Confidence) {
115    if let Some((bom, _)) = Encoding::for_bom(bytes) {
116        return (bom, Confidence::Certain);
117    }
118    if let Some(given) = transport {
119        return (given, Confidence::Certain);
120    }
121    match prescan(&bytes[..bytes.len().min(1024)]) {
122        Some(declared) => (declared, Confidence::Tentative),
123        None if looks_like_utf8(bytes, complete) => (UTF_8, Confidence::Tentative),
124        None => (fallback, Confidence::Tentative),
125    }
126}
127
128/// The autodetection step: something beyond ASCII, and every byte valid
129/// UTF-8. Pure ASCII says nothing, so it's left to the fallback.
130fn looks_like_utf8(bytes: &[u8], complete: bool) -> bool {
131    if bytes.is_ascii() {
132        return false;
133    }
134    match std::str::from_utf8(bytes) {
135        Ok(_) => true,
136        // `error_len` is None for a sequence the end cut short.
137        Err(e) => !complete && e.error_len().is_none(),
138    }
139}
140
141/// Parses a page given as bytes, as a browser would: sniffs its encoding
142/// ([`sniff`]), decodes it and parses it.
143///
144/// When the encoding was only a guess and the parser meets a `<meta>`
145/// declaring another one (past the first 1,024 bytes, say), the spec has
146/// the browser start again in the declared encoding. Here that's a second
147/// decode and parse of the same bytes, once at most, since the encoding
148/// is certain after it. The time budget covers both parses together.
149///
150/// ```
151/// use parsec_encoding::{Options, parse_bytes};
152/// let page = b"<meta charset=shift_jis><title>\x93\xfa\x96\x7b</title>";
153/// let parsed = parse_bytes(page, Options::default()).unwrap();
154/// assert_eq!(parsed.encoding, encoding_rs::SHIFT_JIS);
155/// assert_eq!(parsed.document.text(parsed.document.root()), "日本");
156/// ```
157pub fn parse_bytes(bytes: &[u8], options: Options) -> Result<Parsed, ParseError> {
158    // The clock is read only for a budget, so that WebAssembly, which has
159    // none, still works.
160    let start = options.html.budget.map(|_| Instant::now());
161    let (mut encoding, mut confidence) = sniff(bytes, options.transport, options.fallback);
162    let mut document = decode_and_parse(bytes, encoding, options.html)?;
163    if confidence == Confidence::Tentative
164        && let Some(declared) = declared(&document)
165    {
166        confidence = Confidence::Certain;
167        if let Some(new) = changed(encoding, declared) {
168            encoding = new;
169            let mut html = options.html;
170            let spent = start.map_or(Default::default(), |t| t.elapsed());
171            html.budget = html.budget.map(|b| b.saturating_sub(spent));
172            document = decode_and_parse(bytes, encoding, html)?;
173        }
174    }
175    Ok(Parsed {
176        document,
177        encoding,
178        confidence,
179    })
180}
181
182fn decode_and_parse(
183    bytes: &[u8],
184    encoding: &'static Encoding,
185    options: parsec::Options,
186) -> Result<Document, ParseError> {
187    // Any byte order mark already chose `encoding`, so removing one for
188    // `encoding` is the Encoding Standard's "decode".
189    let (text, _) = encoding.decode_with_bom_removal(bytes);
190    parsec::parse_with(&text, options)
191}
192
193/// The first encoding a `<meta>` declares as the tree builder sees it: by
194/// `charset`, or by `http-equiv="Content-Type"` with a `content` naming
195/// one. The parser makes elements in the order it meets their tags, so
196/// node order is that order, `<template>` contents included.
197fn declared(doc: &Document) -> Option<&'static Encoding> {
198    (0..doc.len() as NodeId).find_map(|id| {
199        if !matches!(doc.kind(id), Kind::Element { ns: Ns::Html, .. })
200            || doc.lower_name(id) != Some("meta")
201        {
202            return None;
203        }
204        let attr = |want: &str| {
205            doc.attrs(id)
206                .find(|&(ns, name, _)| ns == AttrNs::None && name == want)
207                .map(|(_, _, value)| value)
208        };
209        if let Some(found) = attr("charset").and_then(|v| Encoding::for_label(v.as_bytes())) {
210            return Some(found);
211        }
212        attr("http-equiv")
213            .filter(|v| v.eq_ignore_ascii_case("content-type"))
214            .and(attr("content"))
215            .and_then(|v| prescan::charset_in_content(v.as_bytes()))
216    })
217}
218
219/// "Change the encoding": the encoding to parse again in, or `None` when
220/// the one in use stands.
221fn changed(current: &'static Encoding, declared: &'static Encoding) -> Option<&'static Encoding> {
222    if current == UTF_16BE || current == UTF_16LE {
223        return None;
224    }
225    let new = prescan::as_declared(declared);
226    (new != current).then_some(new)
227}
228
229/// The fallback encoding a browser uses for a user whose language is
230/// `locale` (a BCP 47 tag such as `ja`, `ru-RU` or `zh-Hant`), from the
231/// HTML Standard's table of suggested defaults. Pass it as
232/// [`Options::fallback`] when you know your users' language, or a site's.
233///
234/// ```
235/// use parsec_encoding::locale_default;
236/// assert_eq!(locale_default("ja-JP"), encoding_rs::SHIFT_JIS);
237/// assert_eq!(locale_default("zh-TW"), encoding_rs::BIG5);
238/// assert_eq!(locale_default("en-GB"), encoding_rs::WINDOWS_1252);
239/// ```
240pub fn locale_default(locale: &str) -> &'static Encoding {
241    use encoding_rs::*;
242    let locale = locale.to_ascii_lowercase();
243    let mut tags = locale.split(['-', '_']);
244    match tags.next().unwrap_or("") {
245        "ar" | "fa" => WINDOWS_1256,
246        "az" | "ku" | "tr" => WINDOWS_1254,
247        "ba" | "be" | "bg" | "kk" | "ky" | "mk" | "ru" | "sah" | "sr" | "tg" | "tt" | "uk" => {
248            WINDOWS_1251
249        }
250        "cs" | "hr" | "sk" => WINDOWS_1250,
251        "el" => ISO_8859_7,
252        "et" | "lt" | "lv" => WINDOWS_1257,
253        "he" => WINDOWS_1255,
254        "hu" | "pl" | "sl" => ISO_8859_2,
255        "ja" => SHIFT_JIS,
256        "ko" => EUC_KR,
257        "th" => WINDOWS_874,
258        "vi" => WINDOWS_1258,
259        // The script subtag comes before the region, so it decides first.
260        "zh" => tags
261            .find_map(|t| match t {
262                "hans" | "cn" | "sg" => Some(GBK),
263                "hant" | "hk" | "mo" | "tw" => Some(BIG5),
264                _ => None,
265            })
266            .unwrap_or(WINDOWS_1252),
267        _ => WINDOWS_1252,
268    }
269}
270
271#[cfg(test)]
272mod tests {
273    use super::*;
274    use encoding_rs::{EUC_JP, SHIFT_JIS, UTF_8, WINDOWS_1251, WINDOWS_1252};
275
276    fn parse(bytes: &[u8]) -> Parsed {
277        parse_bytes(bytes, Options::default()).unwrap()
278    }
279
280    #[test]
281    fn bom_beats_transport_and_meta() {
282        let page = b"\xef\xbb\xbf<meta charset=euc-jp>\xc3\xa9";
283        let options = Options {
284            transport: Some(SHIFT_JIS),
285            ..Options::default()
286        };
287        let p = parse_bytes(page, options).unwrap();
288        assert_eq!((p.encoding, p.confidence), (UTF_8, Confidence::Certain));
289        assert_eq!(p.document.text(p.document.root()), "é");
290    }
291
292    #[test]
293    fn utf16_bom_is_removed() {
294        let p = parse(b"\xff\xfe<\0p\0>\0h\0i\0");
295        assert_eq!(p.encoding, UTF_16LE);
296        assert_eq!(p.document.text(p.document.root()), "hi");
297    }
298
299    #[test]
300    fn transport_is_certain() {
301        let options = Options {
302            transport: Some(WINDOWS_1251),
303            ..Options::default()
304        };
305        let p = parse_bytes(b"<meta charset=euc-jp>\xcf", options).unwrap();
306        assert_eq!(
307            (p.encoding, p.confidence),
308            (WINDOWS_1251, Confidence::Certain)
309        );
310        assert_eq!(p.document.text(p.document.root()), "П");
311    }
312
313    #[test]
314    fn late_meta_reparses() {
315        let mut page = b"<!doctype html><title>".to_vec();
316        page.extend(std::iter::repeat_n(b'x', 1100));
317        page.extend(b"</title><meta charset=windows-1251><p>\xcf");
318        let p = parse(&page);
319        assert_eq!(
320            (p.encoding, p.confidence),
321            (WINDOWS_1251, Confidence::Certain)
322        );
323        assert!(p.document.text(p.document.root()).ends_with('П'));
324    }
325
326    #[test]
327    fn agreeing_meta_makes_it_certain() {
328        let p = parse(b"<meta charset=euc-jp>");
329        assert_eq!((p.encoding, p.confidence), (EUC_JP, Confidence::Certain));
330        let p = parse(b"<p>no declaration");
331        assert_eq!(
332            (p.encoding, p.confidence),
333            (WINDOWS_1252, Confidence::Tentative)
334        );
335    }
336
337    #[test]
338    fn late_http_equiv_and_svg_meta() {
339        let mut page = b"<!-- ".to_vec();
340        page.extend(std::iter::repeat_n(b'x', 1100));
341        page.extend(b" --><svg><meta charset=euc-jp></svg>");
342        // An SVG <meta> breaks out of foreign content, so it's HTML's.
343        assert_eq!(parse(&page).encoding, EUC_JP);
344        let mut page = b"<!-- ".to_vec();
345        page.extend(std::iter::repeat_n(b'x', 1100));
346        page.extend(b" --><meta http-equiv=content-type content='text/html;charset=sjis'>");
347        assert_eq!(parse(&page).encoding, SHIFT_JIS);
348    }
349
350    #[test]
351    fn later_metas_and_utf16_declarations() {
352        let mut page = b"<!-- ".to_vec();
353        page.extend(std::iter::repeat_n(b'x', 1100));
354        page.extend(b" --><meta charset=bogus><meta charset=utf-16><meta charset=sjis>");
355        // The first that names an encoding decides; UTF-16 means UTF-8.
356        assert_eq!(parse(&page).encoding, UTF_8);
357    }
358
359    #[test]
360    fn budget_covers_both_parses() {
361        let mut page = b"<title>".to_vec();
362        page.extend(std::iter::repeat_n(b'x', 1100));
363        page.extend(b"</title><meta charset=windows-1251>");
364        let options = Options {
365            html: parsec::Options {
366                budget: Some(std::time::Duration::from_secs(60)),
367                ..parsec::Options::default()
368            },
369            ..Options::default()
370        };
371        assert_eq!(parse_bytes(&page, options).unwrap().encoding, WINDOWS_1251);
372    }
373
374    #[test]
375    fn valid_utf8_is_detected() {
376        let p = parse("<p>caf\u{e9}".as_bytes());
377        assert_eq!((p.encoding, p.confidence), (UTF_8, Confidence::Tentative));
378        assert_eq!(p.document.text(p.document.root()), "caf\u{e9}");
379        // Not valid UTF-8, or nothing but ASCII: the fallback.
380        assert_eq!(parse(b"<p>caf\xe9").encoding, WINDOWS_1252);
381        assert_eq!(parse(b"<p>cafe").encoding, WINDOWS_1252);
382        // Cut off at the end of the whole input isn't valid.
383        assert_eq!(parse(b"<p>\xe6\x97").encoding, WINDOWS_1252);
384        // A transport encoding or a <meta> comes first.
385        let page = "<meta charset=windows-1251><p>\u{e9}".as_bytes();
386        assert_eq!(parse(page).encoding, WINDOWS_1251);
387        // And a late <meta> can still change the guess.
388        let mut page = "<title>\u{e9}".as_bytes().to_vec();
389        page.extend(std::iter::repeat_n(b'x', 1100));
390        page.extend(b"</title><meta charset=windows-1251>");
391        let p = parse(&page);
392        assert_eq!(
393            (p.encoding, p.confidence),
394            (WINDOWS_1251, Confidence::Certain)
395        );
396    }
397
398    #[test]
399    fn locales() {
400        assert_eq!(locale_default("RU"), WINDOWS_1251);
401        assert_eq!(locale_default("zh-Hans-HK"), encoding_rs::GBK);
402        assert_eq!(locale_default("zh"), WINDOWS_1252);
403        assert_eq!(locale_default(""), WINDOWS_1252);
404    }
405}