1use std::time::{Duration, Instant};
9
10use encoding_rs::{Decoder, Encoding, UTF_16BE, UTF_16LE};
11use parsec::ParseError;
12
13use crate::{Confidence, Options, Parsed, changed, declared, prescan};
14
15pub struct ByteParser {
69 options: Options,
70 bytes: Vec<u8>,
73 state: Option<Running>,
74 spent: Duration,
77}
78
79struct Running {
80 encoding: &'static Encoding,
81 confidence: Confidence,
82 keep: bool,
85 decoder: Decoder,
86 parser: parsec::Parser,
87 text: String,
89}
90
91impl ByteParser {
92 pub fn new(options: Options) -> ByteParser {
93 ByteParser {
94 options,
95 bytes: Vec::new(),
96 state: None,
97 spent: Duration::ZERO,
98 }
99 }
100
101 pub fn feed(&mut self, chunk: &[u8]) {
103 let start = self.options.html.budget.map(|_| Instant::now());
104 match &mut self.state {
105 Some(running) => {
106 if running.keep {
107 self.bytes.extend_from_slice(chunk);
108 }
109 running.decode(chunk, false);
110 }
111 None => {
112 self.bytes.extend_from_slice(chunk);
113 if let Some((encoding, confidence)) = self.decided() {
114 self.run(encoding, confidence);
115 }
116 }
117 }
118 self.spent += start.map_or(Duration::ZERO, |t| t.elapsed());
119 }
120
121 pub fn finish(mut self) -> Result<Parsed, ParseError> {
124 let start = self.options.html.budget.map(|_| Instant::now());
125 if self.state.is_none() {
126 let (encoding, confidence) =
127 crate::sniff(&self.bytes, self.options.transport, self.options.fallback);
128 self.run(encoding, confidence);
129 }
130 let mut running = self.state.take().expect("set just above");
131 if running.keep {
137 let (whole, _) =
138 crate::sniff(&self.bytes, self.options.transport, self.options.fallback);
139 if whole != running.encoding {
140 let mut options = self.options;
141 let spent = self.spent + start.map_or(Duration::ZERO, |t| t.elapsed());
142 options.html.budget = options.html.budget.map(|b| b.saturating_sub(spent));
143 return crate::parse_bytes(&self.bytes, options);
144 }
145 }
146 running.decode(&[], true);
147 let mut document = running.parser.finish()?;
148 let (mut encoding, mut confidence) = (running.encoding, running.confidence);
149 if confidence == Confidence::Tentative
150 && let Some(declared) = declared(&document)
151 {
152 confidence = Confidence::Certain;
153 if let Some(new) = changed(encoding, declared) {
154 encoding = new;
155 let mut html = self.options.html;
156 let spent = self.spent + start.map_or(Duration::ZERO, |t| t.elapsed());
157 html.budget = html.budget.map(|b| b.saturating_sub(spent));
158 document = crate::decode_and_parse(&self.bytes, encoding, html)?;
159 }
160 }
161 Ok(Parsed {
162 document,
163 encoding,
164 confidence,
165 })
166 }
167
168 fn decided(&self) -> Option<(&'static Encoding, Confidence)> {
171 let b = &self.bytes;
172 if let Some((bom, _)) = Encoding::for_bom(b) {
173 return Some((bom, Confidence::Certain));
174 }
175 let boms: [&[u8]; 3] = [b"\xEF\xBB\xBF", b"\xFE\xFF", b"\xFF\xFE"];
177 if boms.iter().any(|bom| bom.starts_with(b)) {
178 return None;
179 }
180 if let Some(given) = self.options.transport {
181 return Some((given, Confidence::Certain));
182 }
183 if b.len() >= 1024 {
184 return Some(crate::guess(b, None, self.options.fallback, false));
186 }
187 prescan::scan(b).map(|found| (found, Confidence::Tentative))
190 }
191
192 fn run(&mut self, encoding: &'static Encoding, confidence: Confidence) {
194 let keep =
197 confidence == Confidence::Tentative && encoding != UTF_16BE && encoding != UTF_16LE;
198 let mut running = Running {
199 encoding,
200 confidence,
201 keep,
202 decoder: encoding.new_decoder_with_bom_removal(),
205 parser: parsec::Parser::new(self.options.html),
206 text: String::new(),
207 };
208 running.decode(&self.bytes, false);
209 if !keep {
210 self.bytes = Vec::new();
211 }
212 self.state = Some(running);
213 }
214}
215
216impl Running {
217 fn decode(&mut self, mut bytes: &[u8], last: bool) {
218 loop {
219 self.text.clear();
220 let room = self
221 .decoder
222 .max_utf8_buffer_length(bytes.len())
223 .unwrap_or(usize::MAX);
224 self.text.reserve(room.min(1 << 20));
226 let (result, read, _) = self.decoder.decode_to_string(bytes, &mut self.text, last);
227 self.parser.feed(&self.text);
228 bytes = &bytes[read..];
229 if result == encoding_rs::CoderResult::InputEmpty {
230 return;
231 }
232 }
233 }
234}
235
236#[cfg(test)]
237mod tests {
238 use super::*;
239 use encoding_rs::{ISO_8859_15, SHIFT_JIS, UTF_8, UTF_16LE, WINDOWS_1252};
240
241 fn decided(p: &ByteParser) -> Option<(&'static Encoding, Confidence, bool)> {
242 p.state.as_ref().map(|r| (r.encoding, r.confidence, r.keep))
243 }
244
245 #[test]
246 fn a_whole_meta_decides_before_1024_bytes() {
247 let mut p = ByteParser::new(Options::default());
248 p.feed(b"<meta charset=iso-8859-1");
250 assert_eq!(decided(&p), None);
251 p.feed(b"5><p>\xa4");
252 assert_eq!(
253 decided(&p),
254 Some((ISO_8859_15, Confidence::Tentative, true))
255 );
256 let parsed = p.finish().unwrap();
257 assert_eq!(parsed.confidence, Confidence::Certain);
258 assert_eq!(parsed.document.text(parsed.document.root()), "€");
259 }
260
261 #[test]
262 fn no_meta_waits_for_1024_bytes() {
263 let mut p = ByteParser::new(Options::default());
264 p.feed(&[b'x'; 1023]);
265 assert_eq!(decided(&p), None);
266 p.feed(b"x");
267 assert_eq!(
268 decided(&p),
269 Some((WINDOWS_1252, Confidence::Tentative, true))
270 );
271 }
272
273 #[test]
274 fn a_bom_cut_in_three_decides_on_its_last_byte() {
275 let mut p = ByteParser::new(Options {
276 transport: Some(SHIFT_JIS),
277 ..Options::default()
278 });
279 p.feed(b"\xef");
280 p.feed(b"\xbb");
281 assert_eq!(decided(&p), None);
282 p.feed(b"\xbf\xc3");
283 assert_eq!(decided(&p), Some((UTF_8, Confidence::Certain, false)));
284 assert!(p.bytes.is_empty());
285 p.feed(b"\xa9");
286 assert_eq!(p.finish().unwrap().document.text(0), "é");
287 }
288
289 #[test]
290 fn transport_decides_once_no_bom_can_come() {
291 let options = Options {
292 transport: Some(SHIFT_JIS),
293 ..Options::default()
294 };
295 let mut p = ByteParser::new(options);
296 p.feed(b"\xfe");
297 assert_eq!(decided(&p), None);
298 p.feed(b"<");
299 assert_eq!(decided(&p), Some((SHIFT_JIS, Confidence::Certain, false)));
300 let mut p = ByteParser::new(options);
301 p.feed(b"\xff");
302 p.feed(b"\xfe");
303 assert_eq!(decided(&p), Some((UTF_16LE, Confidence::Certain, false)));
304 }
305
306 #[test]
307 fn a_late_meta_reparses_within_the_budget() {
308 let mut p = ByteParser::new(Options {
309 html: parsec::Options {
310 budget: Some(Duration::from_secs(60)),
311 ..parsec::Options::default()
312 },
313 ..Options::default()
314 });
315 p.feed(b"<title>");
316 for _ in 0..20 {
317 p.feed(&[b'x'; 100]);
318 }
319 p.feed(b"</title><meta charset=shift_jis><p>\x93\xfa");
320 p.feed(b"\x96\x7b");
321 let parsed = p.finish().unwrap();
322 assert_eq!(
323 (parsed.encoding, parsed.confidence),
324 (SHIFT_JIS, Confidence::Certain)
325 );
326 assert!(parsed.document.text(0).ends_with("日本"));
327 }
328
329 #[test]
330 fn utf8_is_detected_from_the_bytes_held_back() {
331 let mut p = ByteParser::new(Options::default());
332 let mut page = b"<p>".to_vec();
333 page.extend(std::iter::repeat_n(b'x', 1018));
334 page.extend("\u{65e5}".as_bytes());
336 p.feed(&page[..1023]);
337 assert_eq!(decided(&p), None);
338 p.feed(&page[1023..]);
339 assert_eq!(decided(&p), Some((UTF_8, Confidence::Tentative, true)));
340 let parsed = p.finish().unwrap();
341 assert_eq!(parsed.encoding, UTF_8);
342 assert!(parsed.document.text(0).ends_with('\u{65e5}'));
343 }
344
345 #[test]
346 fn a_guess_the_whole_wouldnt_make_is_parsed_again() {
347 let mut page = std::iter::repeat_n(b'x', 1100).collect::<Vec<u8>>();
350 page.extend("caf\u{e9}".as_bytes());
351 let mut p = ByteParser::new(Options::default());
352 p.feed(&page[..1024]);
353 assert_eq!(decided(&p).map(|d| d.0), Some(WINDOWS_1252));
354 p.feed(&page[1024..]);
355 let parsed = p.finish().unwrap();
356 assert_eq!(parsed.encoding, UTF_8);
357 assert!(parsed.document.text(0).ends_with("caf\u{e9}"));
358 let mut page = "\u{e9}".as_bytes().to_vec();
360 page.extend(std::iter::repeat_n(b'x', 1100));
361 page.push(0xe9);
362 let mut p = ByteParser::new(Options::default());
363 p.feed(&page[..1024]);
364 assert_eq!(decided(&p).map(|d| d.0), Some(UTF_8));
365 p.feed(&page[1024..]);
366 let parsed = p.finish().unwrap();
367 assert_eq!(parsed.encoding, WINDOWS_1252);
368 assert!(parsed.document.text(0).starts_with("\u{c3}\u{a9}"));
369 }
370
371 #[test]
372 fn nothing_at_all() {
373 let p = ByteParser::new(Options::default());
374 let parsed = p.finish().unwrap();
375 assert_eq!(
376 (parsed.encoding, parsed.confidence),
377 (WINDOWS_1252, Confidence::Tentative)
378 );
379 }
380}