Skip to main content

parsec_encoding/
stream.rs

1// SPDX-License-Identifier: Apache-2.0
2// Copyright 2026 James Conway
3
4//! Bytes as they arrive: [`ByteParser`] sniffs, decodes and parses a page
5//! a piece at a time into what [`parse_bytes`](crate::parse_bytes) makes
6//! of the whole.
7
8use std::time::{Duration, Instant};
9
10use encoding_rs::{Decoder, Encoding, UTF_16BE, UTF_16LE};
11use parsec::ParseError;
12
13use crate::{Confidence, Options, Parsed, changed, declared, prescan};
14
15/// Parses a page given as bytes a piece at a time, as a browser does while
16/// it downloads one. You get what [`parse_bytes`](crate::parse_bytes)
17/// gives for all the pieces joined: the same document, encoding and
18/// confidence, however the bytes were cut.
19///
20/// ```
21/// use parsec_encoding::{ByteParser, Confidence, Options};
22///
23/// let mut parser = ByteParser::new(Options::default());
24/// // Shift_JIS for 日本, cut inside the first character.
25/// for piece in [&b"<meta charset=sjis><p>\x93"[..], b"\xfa\x96\x7b"] {
26///     parser.feed(piece);
27/// }
28/// let parsed = parser.finish().unwrap();
29/// assert_eq!(parsed.encoding, encoding_rs::SHIFT_JIS);
30/// assert_eq!(parsed.confidence, Confidence::Certain);
31/// assert_eq!(parsed.document.text(parsed.document.root()), "日本");
32/// ```
33///
34/// How it goes, in the HTML Standard's terms:
35///
36/// - Until it knows the encoding, it holds the bytes back. A byte order
37///   mark decides at once, even cut over two pieces. Then the transport
38///   layer's encoding, if you gave one. Otherwise the prescan does, which
39///   the spec lets a browser wait up to 1,024 bytes for: this waits until
40///   the prescan finds a whole `<meta>` naming an encoding, or there are
41///   1,024 bytes, or the input ends. With no `<meta>`, the bytes held back
42///   go through the autodetection ([`sniff`](crate::sniff)): non-ASCII
43///   and valid UTF-8, with a character cut off at the end allowed, is
44///   UTF-8; otherwise it's the fallback.
45/// - From then on each piece goes through encoding_rs's [`Decoder`],
46///   which keeps a character cut between pieces for the next one, and on
47///   into [`parsec::Parser`].
48/// - When the encoding was only a guess, a `<meta>` the parser meets later
49///   can name another. The spec has the browser start again in that one.
50///   Here that's at [`ByteParser::finish`]: one more decode and parse,
51///   of every byte, once at most, as `parse_bytes` does. So is a guess
52///   the whole input wouldn't make (UTF-8 that starts after 1,024 bytes
53///   of ASCII, or bytes after the first 1,024 that aren't UTF-8): with
54///   every byte kept, `finish` checks and, if so, parses as
55///   `parse_bytes` would.
56///
57/// Memory: while the encoding is a guess (no byte order mark, no
58/// transport encoding), every byte is kept for that second parse, so the
59/// input sits in memory once as bytes beside the document, which holds
60/// the decoded text anyway. Give the transport layer's encoding when you
61/// have it and only the decoder's few bytes of state are kept. Before the
62/// encoding is known, the pieces fed so far wait, which stops once there
63/// are 1,024 bytes.
64///
65/// A time budget in [`Options::html`] counts the time spent in
66/// [`ByteParser::feed`] and [`ByteParser::finish`], both parses together,
67/// not the time waiting for input.
68pub struct ByteParser {
69    options: Options,
70    /// The bytes held back while sniffing, then, while the encoding is a
71    /// guess, every byte for a second parse.
72    bytes: Vec<u8>,
73    state: Option<Running>,
74    /// Time spent in `feed` and `finish`, counted only with a budget, so
75    /// that WebAssembly, which has no clock, still works.
76    spent: Duration,
77}
78
79struct Running {
80    encoding: &'static Encoding,
81    confidence: Confidence,
82    /// Whether a later `<meta>` could still change the encoding, and so
83    /// whether every byte has to be kept.
84    keep: bool,
85    decoder: Decoder,
86    parser: parsec::Parser,
87    /// Where decoded text goes on its way into the parser, reused.
88    text: String,
89}
90
91impl ByteParser {
92    pub fn new(options: Options) -> ByteParser {
93        ByteParser {
94            options,
95            bytes: Vec::new(),
96            state: None,
97            spent: Duration::ZERO,
98        }
99    }
100
101    /// The next piece of the input.
102    pub fn feed(&mut self, chunk: &[u8]) {
103        let start = self.options.html.budget.map(|_| Instant::now());
104        match &mut self.state {
105            Some(running) => {
106                if running.keep {
107                    self.bytes.extend_from_slice(chunk);
108                }
109                running.decode(chunk, false);
110            }
111            None => {
112                self.bytes.extend_from_slice(chunk);
113                if let Some((encoding, confidence)) = self.decided() {
114                    self.run(encoding, confidence);
115                }
116            }
117        }
118        self.spent += start.map_or(Duration::ZERO, |t| t.elapsed());
119    }
120
121    /// The end of the input: the document and its encoding, or why there's
122    /// none, as [`parse_bytes`](crate::parse_bytes) gives them.
123    pub fn finish(mut self) -> Result<Parsed, ParseError> {
124        let start = self.options.html.budget.map(|_| Instant::now());
125        if self.state.is_none() {
126            let (encoding, confidence) =
127                crate::sniff(&self.bytes, self.options.transport, self.options.fallback);
128            self.run(encoding, confidence);
129        }
130        let mut running = self.state.take().expect("set just above");
131        // The guess was made from the first bytes; `parse_bytes` makes it
132        // from all of them. Where the UTF-8 autodetection gives another
133        // answer for the whole (UTF-8 past the first 1,024 bytes of ASCII,
134        // or bytes later on that aren't UTF-8), parse again as it would,
135        // as rarely as a late `<meta>` does. Guesses keep every byte.
136        if running.keep {
137            let (whole, _) =
138                crate::sniff(&self.bytes, self.options.transport, self.options.fallback);
139            if whole != running.encoding {
140                let mut options = self.options;
141                let spent = self.spent + start.map_or(Duration::ZERO, |t| t.elapsed());
142                options.html.budget = options.html.budget.map(|b| b.saturating_sub(spent));
143                return crate::parse_bytes(&self.bytes, options);
144            }
145        }
146        running.decode(&[], true);
147        let mut document = running.parser.finish()?;
148        let (mut encoding, mut confidence) = (running.encoding, running.confidence);
149        if confidence == Confidence::Tentative
150            && let Some(declared) = declared(&document)
151        {
152            confidence = Confidence::Certain;
153            if let Some(new) = changed(encoding, declared) {
154                encoding = new;
155                let mut html = self.options.html;
156                let spent = self.spent + start.map_or(Duration::ZERO, |t| t.elapsed());
157                html.budget = html.budget.map(|b| b.saturating_sub(spent));
158                document = crate::decode_and_parse(&self.bytes, encoding, html)?;
159            }
160        }
161        Ok(Parsed {
162            document,
163            encoding,
164            confidence,
165        })
166    }
167
168    /// The encoding sniffing's answer, once the bytes so far are enough to
169    /// give the one it would give for the whole input.
170    fn decided(&self) -> Option<(&'static Encoding, Confidence)> {
171        let b = &self.bytes;
172        if let Some((bom, _)) = Encoding::for_bom(b) {
173            return Some((bom, Confidence::Certain));
174        }
175        // `EF BB` or `FE` alone could still be the start of one.
176        let boms: [&[u8]; 3] = [b"\xEF\xBB\xBF", b"\xFE\xFF", b"\xFF\xFE"];
177        if boms.iter().any(|bom| bom.starts_with(b)) {
178            return None;
179        }
180        if let Some(given) = self.options.transport {
181            return Some((given, Confidence::Certain));
182        }
183        if b.len() >= 1024 {
184            // More may come, so UTF-8 cut off at the end still counts.
185            return Some(crate::guess(b, None, self.options.fallback, false));
186        }
187        // A `<meta>` the scan read to its `>` says the same whatever comes
188        // after it, so there's no need to wait for the rest.
189        prescan::scan(b).map(|found| (found, Confidence::Tentative))
190    }
191
192    /// Starts decoding and parsing in `encoding`, with the bytes held back.
193    fn run(&mut self, encoding: &'static Encoding, confidence: Confidence) {
194        // A guess of UTF-16 (from the prescan's `<\0?\0x\0`) is never
195        // changed, so the bytes aren't needed again.
196        let keep =
197            confidence == Confidence::Tentative && encoding != UTF_16BE && encoding != UTF_16LE;
198        let mut running = Running {
199            encoding,
200            confidence,
201            keep,
202            // As `parse_bytes` decodes: only this encoding's own byte order
203            // mark is removed.
204            decoder: encoding.new_decoder_with_bom_removal(),
205            parser: parsec::Parser::new(self.options.html),
206            text: String::new(),
207        };
208        running.decode(&self.bytes, false);
209        if !keep {
210            self.bytes = Vec::new();
211        }
212        self.state = Some(running);
213    }
214}
215
216impl Running {
217    fn decode(&mut self, mut bytes: &[u8], last: bool) {
218        loop {
219            self.text.clear();
220            let room = self
221                .decoder
222                .max_utf8_buffer_length(bytes.len())
223                .unwrap_or(usize::MAX);
224            // A cap on the piece of text made at once, for huge chunks.
225            self.text.reserve(room.min(1 << 20));
226            let (result, read, _) = self.decoder.decode_to_string(bytes, &mut self.text, last);
227            self.parser.feed(&self.text);
228            bytes = &bytes[read..];
229            if result == encoding_rs::CoderResult::InputEmpty {
230                return;
231            }
232        }
233    }
234}
235
236#[cfg(test)]
237mod tests {
238    use super::*;
239    use encoding_rs::{ISO_8859_15, SHIFT_JIS, UTF_8, UTF_16LE, WINDOWS_1252};
240
241    fn decided(p: &ByteParser) -> Option<(&'static Encoding, Confidence, bool)> {
242        p.state.as_ref().map(|r| (r.encoding, r.confidence, r.keep))
243    }
244
245    #[test]
246    fn a_whole_meta_decides_before_1024_bytes() {
247        let mut p = ByteParser::new(Options::default());
248        // `iso-8859-1` is windows-1252's label: the scan mustn't stop on it.
249        p.feed(b"<meta charset=iso-8859-1");
250        assert_eq!(decided(&p), None);
251        p.feed(b"5><p>\xa4");
252        assert_eq!(
253            decided(&p),
254            Some((ISO_8859_15, Confidence::Tentative, true))
255        );
256        let parsed = p.finish().unwrap();
257        assert_eq!(parsed.confidence, Confidence::Certain);
258        assert_eq!(parsed.document.text(parsed.document.root()), "€");
259    }
260
261    #[test]
262    fn no_meta_waits_for_1024_bytes() {
263        let mut p = ByteParser::new(Options::default());
264        p.feed(&[b'x'; 1023]);
265        assert_eq!(decided(&p), None);
266        p.feed(b"x");
267        assert_eq!(
268            decided(&p),
269            Some((WINDOWS_1252, Confidence::Tentative, true))
270        );
271    }
272
273    #[test]
274    fn a_bom_cut_in_three_decides_on_its_last_byte() {
275        let mut p = ByteParser::new(Options {
276            transport: Some(SHIFT_JIS),
277            ..Options::default()
278        });
279        p.feed(b"\xef");
280        p.feed(b"\xbb");
281        assert_eq!(decided(&p), None);
282        p.feed(b"\xbf\xc3");
283        assert_eq!(decided(&p), Some((UTF_8, Confidence::Certain, false)));
284        assert!(p.bytes.is_empty());
285        p.feed(b"\xa9");
286        assert_eq!(p.finish().unwrap().document.text(0), "é");
287    }
288
289    #[test]
290    fn transport_decides_once_no_bom_can_come() {
291        let options = Options {
292            transport: Some(SHIFT_JIS),
293            ..Options::default()
294        };
295        let mut p = ByteParser::new(options);
296        p.feed(b"\xfe");
297        assert_eq!(decided(&p), None);
298        p.feed(b"<");
299        assert_eq!(decided(&p), Some((SHIFT_JIS, Confidence::Certain, false)));
300        let mut p = ByteParser::new(options);
301        p.feed(b"\xff");
302        p.feed(b"\xfe");
303        assert_eq!(decided(&p), Some((UTF_16LE, Confidence::Certain, false)));
304    }
305
306    #[test]
307    fn a_late_meta_reparses_within_the_budget() {
308        let mut p = ByteParser::new(Options {
309            html: parsec::Options {
310                budget: Some(Duration::from_secs(60)),
311                ..parsec::Options::default()
312            },
313            ..Options::default()
314        });
315        p.feed(b"<title>");
316        for _ in 0..20 {
317            p.feed(&[b'x'; 100]);
318        }
319        p.feed(b"</title><meta charset=shift_jis><p>\x93\xfa");
320        p.feed(b"\x96\x7b");
321        let parsed = p.finish().unwrap();
322        assert_eq!(
323            (parsed.encoding, parsed.confidence),
324            (SHIFT_JIS, Confidence::Certain)
325        );
326        assert!(parsed.document.text(0).ends_with("日本"));
327    }
328
329    #[test]
330    fn utf8_is_detected_from_the_bytes_held_back() {
331        let mut p = ByteParser::new(Options::default());
332        let mut page = b"<p>".to_vec();
333        page.extend(std::iter::repeat_n(b'x', 1018));
334        // 1,021 bytes, then the first two of a three-byte character.
335        page.extend("\u{65e5}".as_bytes());
336        p.feed(&page[..1023]);
337        assert_eq!(decided(&p), None);
338        p.feed(&page[1023..]);
339        assert_eq!(decided(&p), Some((UTF_8, Confidence::Tentative, true)));
340        let parsed = p.finish().unwrap();
341        assert_eq!(parsed.encoding, UTF_8);
342        assert!(parsed.document.text(0).ends_with('\u{65e5}'));
343    }
344
345    #[test]
346    fn a_guess_the_whole_wouldnt_make_is_parsed_again() {
347        // ASCII for the first 1,024 bytes guesses windows-1252; the UTF-8
348        // after them makes it UTF-8, as parse_bytes says.
349        let mut page = std::iter::repeat_n(b'x', 1100).collect::<Vec<u8>>();
350        page.extend("caf\u{e9}".as_bytes());
351        let mut p = ByteParser::new(Options::default());
352        p.feed(&page[..1024]);
353        assert_eq!(decided(&p).map(|d| d.0), Some(WINDOWS_1252));
354        p.feed(&page[1024..]);
355        let parsed = p.finish().unwrap();
356        assert_eq!(parsed.encoding, UTF_8);
357        assert!(parsed.document.text(0).ends_with("caf\u{e9}"));
358        // UTF-8 at first, then a byte that isn't: windows-1252 after all.
359        let mut page = "\u{e9}".as_bytes().to_vec();
360        page.extend(std::iter::repeat_n(b'x', 1100));
361        page.push(0xe9);
362        let mut p = ByteParser::new(Options::default());
363        p.feed(&page[..1024]);
364        assert_eq!(decided(&p).map(|d| d.0), Some(UTF_8));
365        p.feed(&page[1024..]);
366        let parsed = p.finish().unwrap();
367        assert_eq!(parsed.encoding, WINDOWS_1252);
368        assert!(parsed.document.text(0).starts_with("\u{c3}\u{a9}"));
369    }
370
371    #[test]
372    fn nothing_at_all() {
373        let p = ByteParser::new(Options::default());
374        let parsed = p.finish().unwrap();
375        assert_eq!(
376            (parsed.encoding, parsed.confidence),
377            (WINDOWS_1252, Confidence::Tentative)
378        );
379    }
380}