use std::time::{Duration, Instant};
use encoding_rs::{Decoder, Encoding, UTF_16BE, UTF_16LE};
use parsec::ParseError;
use crate::{Confidence, Options, Parsed, changed, declared, prescan};
pub struct ByteParser {
options: Options,
bytes: Vec<u8>,
state: Option<Running>,
spent: Duration,
}
struct Running {
encoding: &'static Encoding,
confidence: Confidence,
keep: bool,
decoder: Decoder,
parser: parsec::Parser,
text: String,
}
impl ByteParser {
pub fn new(options: Options) -> ByteParser {
ByteParser {
options,
bytes: Vec::new(),
state: None,
spent: Duration::ZERO,
}
}
pub fn feed(&mut self, chunk: &[u8]) {
let start = self.options.html.budget.map(|_| Instant::now());
match &mut self.state {
Some(running) => {
if running.keep {
self.bytes.extend_from_slice(chunk);
}
running.decode(chunk, false);
}
None => {
self.bytes.extend_from_slice(chunk);
if let Some((encoding, confidence)) = self.decided() {
self.run(encoding, confidence);
}
}
}
self.spent += start.map_or(Duration::ZERO, |t| t.elapsed());
}
pub fn finish(mut self) -> Result<Parsed, ParseError> {
let start = self.options.html.budget.map(|_| Instant::now());
if self.state.is_none() {
let (encoding, confidence) =
crate::sniff(&self.bytes, self.options.transport, self.options.fallback);
self.run(encoding, confidence);
}
let mut running = self.state.take().expect("set just above");
if running.keep {
let (whole, _) =
crate::sniff(&self.bytes, self.options.transport, self.options.fallback);
if whole != running.encoding {
let mut options = self.options;
let spent = self.spent + start.map_or(Duration::ZERO, |t| t.elapsed());
options.html.budget = options.html.budget.map(|b| b.saturating_sub(spent));
return crate::parse_bytes(&self.bytes, options);
}
}
running.decode(&[], true);
let mut document = running.parser.finish()?;
let (mut encoding, mut confidence) = (running.encoding, running.confidence);
if confidence == Confidence::Tentative
&& let Some(declared) = declared(&document)
{
confidence = Confidence::Certain;
if let Some(new) = changed(encoding, declared) {
encoding = new;
let mut html = self.options.html;
let spent = self.spent + start.map_or(Duration::ZERO, |t| t.elapsed());
html.budget = html.budget.map(|b| b.saturating_sub(spent));
document = crate::decode_and_parse(&self.bytes, encoding, html)?;
}
}
Ok(Parsed {
document,
encoding,
confidence,
})
}
fn decided(&self) -> Option<(&'static Encoding, Confidence)> {
let b = &self.bytes;
if let Some((bom, _)) = Encoding::for_bom(b) {
return Some((bom, Confidence::Certain));
}
let boms: [&[u8]; 3] = [b"\xEF\xBB\xBF", b"\xFE\xFF", b"\xFF\xFE"];
if boms.iter().any(|bom| bom.starts_with(b)) {
return None;
}
if let Some(given) = self.options.transport {
return Some((given, Confidence::Certain));
}
if b.len() >= 1024 {
return Some(crate::guess(b, None, self.options.fallback, false));
}
prescan::scan(b).map(|found| (found, Confidence::Tentative))
}
fn run(&mut self, encoding: &'static Encoding, confidence: Confidence) {
let keep =
confidence == Confidence::Tentative && encoding != UTF_16BE && encoding != UTF_16LE;
let mut running = Running {
encoding,
confidence,
keep,
decoder: encoding.new_decoder_with_bom_removal(),
parser: parsec::Parser::new(self.options.html),
text: String::new(),
};
running.decode(&self.bytes, false);
if !keep {
self.bytes = Vec::new();
}
self.state = Some(running);
}
}
impl Running {
fn decode(&mut self, mut bytes: &[u8], last: bool) {
loop {
self.text.clear();
let room = self
.decoder
.max_utf8_buffer_length(bytes.len())
.unwrap_or(usize::MAX);
self.text.reserve(room.min(1 << 20));
let (result, read, _) = self.decoder.decode_to_string(bytes, &mut self.text, last);
self.parser.feed(&self.text);
bytes = &bytes[read..];
if result == encoding_rs::CoderResult::InputEmpty {
return;
}
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use encoding_rs::{ISO_8859_15, SHIFT_JIS, UTF_8, UTF_16LE, WINDOWS_1252};
fn decided(p: &ByteParser) -> Option<(&'static Encoding, Confidence, bool)> {
p.state.as_ref().map(|r| (r.encoding, r.confidence, r.keep))
}
#[test]
fn a_whole_meta_decides_before_1024_bytes() {
let mut p = ByteParser::new(Options::default());
p.feed(b"<meta charset=iso-8859-1");
assert_eq!(decided(&p), None);
p.feed(b"5><p>\xa4");
assert_eq!(
decided(&p),
Some((ISO_8859_15, Confidence::Tentative, true))
);
let parsed = p.finish().unwrap();
assert_eq!(parsed.confidence, Confidence::Certain);
assert_eq!(parsed.document.text(parsed.document.root()), "€");
}
#[test]
fn no_meta_waits_for_1024_bytes() {
let mut p = ByteParser::new(Options::default());
p.feed(&[b'x'; 1023]);
assert_eq!(decided(&p), None);
p.feed(b"x");
assert_eq!(
decided(&p),
Some((WINDOWS_1252, Confidence::Tentative, true))
);
}
#[test]
fn a_bom_cut_in_three_decides_on_its_last_byte() {
let mut p = ByteParser::new(Options {
transport: Some(SHIFT_JIS),
..Options::default()
});
p.feed(b"\xef");
p.feed(b"\xbb");
assert_eq!(decided(&p), None);
p.feed(b"\xbf\xc3");
assert_eq!(decided(&p), Some((UTF_8, Confidence::Certain, false)));
assert!(p.bytes.is_empty());
p.feed(b"\xa9");
assert_eq!(p.finish().unwrap().document.text(0), "é");
}
#[test]
fn transport_decides_once_no_bom_can_come() {
let options = Options {
transport: Some(SHIFT_JIS),
..Options::default()
};
let mut p = ByteParser::new(options);
p.feed(b"\xfe");
assert_eq!(decided(&p), None);
p.feed(b"<");
assert_eq!(decided(&p), Some((SHIFT_JIS, Confidence::Certain, false)));
let mut p = ByteParser::new(options);
p.feed(b"\xff");
p.feed(b"\xfe");
assert_eq!(decided(&p), Some((UTF_16LE, Confidence::Certain, false)));
}
#[test]
fn a_late_meta_reparses_within_the_budget() {
let mut p = ByteParser::new(Options {
html: parsec::Options {
budget: Some(Duration::from_secs(60)),
..parsec::Options::default()
},
..Options::default()
});
p.feed(b"<title>");
for _ in 0..20 {
p.feed(&[b'x'; 100]);
}
p.feed(b"</title><meta charset=shift_jis><p>\x93\xfa");
p.feed(b"\x96\x7b");
let parsed = p.finish().unwrap();
assert_eq!(
(parsed.encoding, parsed.confidence),
(SHIFT_JIS, Confidence::Certain)
);
assert!(parsed.document.text(0).ends_with("日本"));
}
#[test]
fn utf8_is_detected_from_the_bytes_held_back() {
let mut p = ByteParser::new(Options::default());
let mut page = b"<p>".to_vec();
page.extend(std::iter::repeat_n(b'x', 1018));
page.extend("\u{65e5}".as_bytes());
p.feed(&page[..1023]);
assert_eq!(decided(&p), None);
p.feed(&page[1023..]);
assert_eq!(decided(&p), Some((UTF_8, Confidence::Tentative, true)));
let parsed = p.finish().unwrap();
assert_eq!(parsed.encoding, UTF_8);
assert!(parsed.document.text(0).ends_with('\u{65e5}'));
}
#[test]
fn a_guess_the_whole_wouldnt_make_is_parsed_again() {
let mut page = std::iter::repeat_n(b'x', 1100).collect::<Vec<u8>>();
page.extend("caf\u{e9}".as_bytes());
let mut p = ByteParser::new(Options::default());
p.feed(&page[..1024]);
assert_eq!(decided(&p).map(|d| d.0), Some(WINDOWS_1252));
p.feed(&page[1024..]);
let parsed = p.finish().unwrap();
assert_eq!(parsed.encoding, UTF_8);
assert!(parsed.document.text(0).ends_with("caf\u{e9}"));
let mut page = "\u{e9}".as_bytes().to_vec();
page.extend(std::iter::repeat_n(b'x', 1100));
page.push(0xe9);
let mut p = ByteParser::new(Options::default());
p.feed(&page[..1024]);
assert_eq!(decided(&p).map(|d| d.0), Some(UTF_8));
p.feed(&page[1024..]);
let parsed = p.finish().unwrap();
assert_eq!(parsed.encoding, WINDOWS_1252);
assert!(parsed.document.text(0).starts_with("\u{c3}\u{a9}"));
}
#[test]
fn nothing_at_all() {
let p = ByteParser::new(Options::default());
let parsed = p.finish().unwrap();
assert_eq!(
(parsed.encoding, parsed.confidence),
(WINDOWS_1252, Confidence::Tentative)
);
}
}