use std::time::Instant;
pub use encoding_rs::{self, Encoding};
use encoding_rs::{UTF_8, UTF_16BE, UTF_16LE};
use parsec::{AttrNs, Document, Kind, NodeId, Ns, ParseError};
mod prescan;
pub use prescan::prescan;
mod stream;
pub use stream::ByteParser;
mod offsets;
pub use offsets::Offsets;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Confidence {
Tentative,
Certain,
}
#[derive(Debug, Clone, Copy)]
pub struct Options {
pub html: parsec::Options,
pub transport: Option<&'static Encoding>,
pub fallback: &'static Encoding,
}
impl Default for Options {
fn default() -> Self {
Options {
html: parsec::Options::default(),
transport: None,
fallback: encoding_rs::WINDOWS_1252,
}
}
}
pub struct Parsed {
pub document: Document,
pub encoding: &'static Encoding,
pub confidence: Confidence,
}
pub fn sniff(
bytes: &[u8],
transport: Option<&'static Encoding>,
fallback: &'static Encoding,
) -> (&'static Encoding, Confidence) {
guess(bytes, transport, fallback, true)
}
pub(crate) fn guess(
bytes: &[u8],
transport: Option<&'static Encoding>,
fallback: &'static Encoding,
complete: bool,
) -> (&'static Encoding, Confidence) {
if let Some((bom, _)) = Encoding::for_bom(bytes) {
return (bom, Confidence::Certain);
}
if let Some(given) = transport {
return (given, Confidence::Certain);
}
match prescan(&bytes[..bytes.len().min(1024)]) {
Some(declared) => (declared, Confidence::Tentative),
None if looks_like_utf8(bytes, complete) => (UTF_8, Confidence::Tentative),
None => (fallback, Confidence::Tentative),
}
}
fn looks_like_utf8(bytes: &[u8], complete: bool) -> bool {
if bytes.is_ascii() {
return false;
}
match std::str::from_utf8(bytes) {
Ok(_) => true,
Err(e) => !complete && e.error_len().is_none(),
}
}
pub fn parse_bytes(bytes: &[u8], options: Options) -> Result<Parsed, ParseError> {
let start = options.html.budget.map(|_| Instant::now());
let (mut encoding, mut confidence) = sniff(bytes, options.transport, options.fallback);
let mut document = decode_and_parse(bytes, encoding, options.html)?;
if confidence == Confidence::Tentative
&& let Some(declared) = declared(&document)
{
confidence = Confidence::Certain;
if let Some(new) = changed(encoding, declared) {
encoding = new;
let mut html = options.html;
let spent = start.map_or(Default::default(), |t| t.elapsed());
html.budget = html.budget.map(|b| b.saturating_sub(spent));
document = decode_and_parse(bytes, encoding, html)?;
}
}
Ok(Parsed {
document,
encoding,
confidence,
})
}
fn decode_and_parse(
bytes: &[u8],
encoding: &'static Encoding,
options: parsec::Options,
) -> Result<Document, ParseError> {
let (text, _) = encoding.decode_with_bom_removal(bytes);
parsec::parse_with(&text, options)
}
fn declared(doc: &Document) -> Option<&'static Encoding> {
(0..doc.len() as NodeId).find_map(|id| {
if !matches!(doc.kind(id), Kind::Element { ns: Ns::Html, .. })
|| doc.lower_name(id) != Some("meta")
{
return None;
}
let attr = |want: &str| {
doc.attrs(id)
.find(|&(ns, name, _)| ns == AttrNs::None && name == want)
.map(|(_, _, value)| value)
};
if let Some(found) = attr("charset").and_then(|v| Encoding::for_label(v.as_bytes())) {
return Some(found);
}
attr("http-equiv")
.filter(|v| v.eq_ignore_ascii_case("content-type"))
.and(attr("content"))
.and_then(|v| prescan::charset_in_content(v.as_bytes()))
})
}
fn changed(current: &'static Encoding, declared: &'static Encoding) -> Option<&'static Encoding> {
if current == UTF_16BE || current == UTF_16LE {
return None;
}
let new = prescan::as_declared(declared);
(new != current).then_some(new)
}
pub fn locale_default(locale: &str) -> &'static Encoding {
use encoding_rs::*;
let locale = locale.to_ascii_lowercase();
let mut tags = locale.split(['-', '_']);
match tags.next().unwrap_or("") {
"ar" | "fa" => WINDOWS_1256,
"az" | "ku" | "tr" => WINDOWS_1254,
"ba" | "be" | "bg" | "kk" | "ky" | "mk" | "ru" | "sah" | "sr" | "tg" | "tt" | "uk" => {
WINDOWS_1251
}
"cs" | "hr" | "sk" => WINDOWS_1250,
"el" => ISO_8859_7,
"et" | "lt" | "lv" => WINDOWS_1257,
"he" => WINDOWS_1255,
"hu" | "pl" | "sl" => ISO_8859_2,
"ja" => SHIFT_JIS,
"ko" => EUC_KR,
"th" => WINDOWS_874,
"vi" => WINDOWS_1258,
"zh" => tags
.find_map(|t| match t {
"hans" | "cn" | "sg" => Some(GBK),
"hant" | "hk" | "mo" | "tw" => Some(BIG5),
_ => None,
})
.unwrap_or(WINDOWS_1252),
_ => WINDOWS_1252,
}
}
#[cfg(test)]
mod tests {
use super::*;
use encoding_rs::{EUC_JP, SHIFT_JIS, UTF_8, WINDOWS_1251, WINDOWS_1252};
fn parse(bytes: &[u8]) -> Parsed {
parse_bytes(bytes, Options::default()).unwrap()
}
#[test]
fn bom_beats_transport_and_meta() {
let page = b"\xef\xbb\xbf<meta charset=euc-jp>\xc3\xa9";
let options = Options {
transport: Some(SHIFT_JIS),
..Options::default()
};
let p = parse_bytes(page, options).unwrap();
assert_eq!((p.encoding, p.confidence), (UTF_8, Confidence::Certain));
assert_eq!(p.document.text(p.document.root()), "é");
}
#[test]
fn utf16_bom_is_removed() {
let p = parse(b"\xff\xfe<\0p\0>\0h\0i\0");
assert_eq!(p.encoding, UTF_16LE);
assert_eq!(p.document.text(p.document.root()), "hi");
}
#[test]
fn transport_is_certain() {
let options = Options {
transport: Some(WINDOWS_1251),
..Options::default()
};
let p = parse_bytes(b"<meta charset=euc-jp>\xcf", options).unwrap();
assert_eq!(
(p.encoding, p.confidence),
(WINDOWS_1251, Confidence::Certain)
);
assert_eq!(p.document.text(p.document.root()), "П");
}
#[test]
fn late_meta_reparses() {
let mut page = b"<!doctype html><title>".to_vec();
page.extend(std::iter::repeat_n(b'x', 1100));
page.extend(b"</title><meta charset=windows-1251><p>\xcf");
let p = parse(&page);
assert_eq!(
(p.encoding, p.confidence),
(WINDOWS_1251, Confidence::Certain)
);
assert!(p.document.text(p.document.root()).ends_with('П'));
}
#[test]
fn agreeing_meta_makes_it_certain() {
let p = parse(b"<meta charset=euc-jp>");
assert_eq!((p.encoding, p.confidence), (EUC_JP, Confidence::Certain));
let p = parse(b"<p>no declaration");
assert_eq!(
(p.encoding, p.confidence),
(WINDOWS_1252, Confidence::Tentative)
);
}
#[test]
fn late_http_equiv_and_svg_meta() {
let mut page = b"<!-- ".to_vec();
page.extend(std::iter::repeat_n(b'x', 1100));
page.extend(b" --><svg><meta charset=euc-jp></svg>");
assert_eq!(parse(&page).encoding, EUC_JP);
let mut page = b"<!-- ".to_vec();
page.extend(std::iter::repeat_n(b'x', 1100));
page.extend(b" --><meta http-equiv=content-type content='text/html;charset=sjis'>");
assert_eq!(parse(&page).encoding, SHIFT_JIS);
}
#[test]
fn later_metas_and_utf16_declarations() {
let mut page = b"<!-- ".to_vec();
page.extend(std::iter::repeat_n(b'x', 1100));
page.extend(b" --><meta charset=bogus><meta charset=utf-16><meta charset=sjis>");
assert_eq!(parse(&page).encoding, UTF_8);
}
#[test]
fn budget_covers_both_parses() {
let mut page = b"<title>".to_vec();
page.extend(std::iter::repeat_n(b'x', 1100));
page.extend(b"</title><meta charset=windows-1251>");
let options = Options {
html: parsec::Options {
budget: Some(std::time::Duration::from_secs(60)),
..parsec::Options::default()
},
..Options::default()
};
assert_eq!(parse_bytes(&page, options).unwrap().encoding, WINDOWS_1251);
}
#[test]
fn valid_utf8_is_detected() {
let p = parse("<p>caf\u{e9}".as_bytes());
assert_eq!((p.encoding, p.confidence), (UTF_8, Confidence::Tentative));
assert_eq!(p.document.text(p.document.root()), "caf\u{e9}");
assert_eq!(parse(b"<p>caf\xe9").encoding, WINDOWS_1252);
assert_eq!(parse(b"<p>cafe").encoding, WINDOWS_1252);
assert_eq!(parse(b"<p>\xe6\x97").encoding, WINDOWS_1252);
let page = "<meta charset=windows-1251><p>\u{e9}".as_bytes();
assert_eq!(parse(page).encoding, WINDOWS_1251);
let mut page = "<title>\u{e9}".as_bytes().to_vec();
page.extend(std::iter::repeat_n(b'x', 1100));
page.extend(b"</title><meta charset=windows-1251>");
let p = parse(&page);
assert_eq!(
(p.encoding, p.confidence),
(WINDOWS_1251, Confidence::Certain)
);
}
#[test]
fn locales() {
assert_eq!(locale_default("RU"), WINDOWS_1251);
assert_eq!(locale_default("zh-Hans-HK"), encoding_rs::GBK);
assert_eq!(locale_default("zh"), WINDOWS_1252);
assert_eq!(locale_default(""), WINDOWS_1252);
}
}