1use std::time::Instant;
24
25pub use encoding_rs::{self, Encoding};
26use encoding_rs::{UTF_8, UTF_16BE, UTF_16LE};
27use parsec::{AttrNs, Document, Kind, NodeId, Ns, ParseError};
28
29mod prescan;
30pub use prescan::prescan;
31mod stream;
32pub use stream::ByteParser;
33mod offsets;
34pub use offsets::Offsets;
35
36#[derive(Debug, Clone, Copy, PartialEq, Eq)]
38pub enum Confidence {
39 Tentative,
43 Certain,
46}
47
48#[derive(Debug, Clone, Copy)]
50pub struct Options {
51 pub html: parsec::Options,
53 pub transport: Option<&'static Encoding>,
57 pub fallback: &'static Encoding,
62}
63
64impl Default for Options {
65 fn default() -> Self {
66 Options {
67 html: parsec::Options::default(),
68 transport: None,
69 fallback: encoding_rs::WINDOWS_1252,
70 }
71 }
72}
73
74pub struct Parsed {
76 pub document: Document,
77 pub encoding: &'static Encoding,
80 pub confidence: Confidence,
81}
82
83pub fn sniff(
100 bytes: &[u8],
101 transport: Option<&'static Encoding>,
102 fallback: &'static Encoding,
103) -> (&'static Encoding, Confidence) {
104 guess(bytes, transport, fallback, true)
105}
106
107pub(crate) fn guess(
110 bytes: &[u8],
111 transport: Option<&'static Encoding>,
112 fallback: &'static Encoding,
113 complete: bool,
114) -> (&'static Encoding, Confidence) {
115 if let Some((bom, _)) = Encoding::for_bom(bytes) {
116 return (bom, Confidence::Certain);
117 }
118 if let Some(given) = transport {
119 return (given, Confidence::Certain);
120 }
121 match prescan(&bytes[..bytes.len().min(1024)]) {
122 Some(declared) => (declared, Confidence::Tentative),
123 None if looks_like_utf8(bytes, complete) => (UTF_8, Confidence::Tentative),
124 None => (fallback, Confidence::Tentative),
125 }
126}
127
128fn looks_like_utf8(bytes: &[u8], complete: bool) -> bool {
131 if bytes.is_ascii() {
132 return false;
133 }
134 match std::str::from_utf8(bytes) {
135 Ok(_) => true,
136 Err(e) => !complete && e.error_len().is_none(),
138 }
139}
140
141pub fn parse_bytes(bytes: &[u8], options: Options) -> Result<Parsed, ParseError> {
158 let start = options.html.budget.map(|_| Instant::now());
161 let (mut encoding, mut confidence) = sniff(bytes, options.transport, options.fallback);
162 let mut document = decode_and_parse(bytes, encoding, options.html)?;
163 if confidence == Confidence::Tentative
164 && let Some(declared) = declared(&document)
165 {
166 confidence = Confidence::Certain;
167 if let Some(new) = changed(encoding, declared) {
168 encoding = new;
169 let mut html = options.html;
170 let spent = start.map_or(Default::default(), |t| t.elapsed());
171 html.budget = html.budget.map(|b| b.saturating_sub(spent));
172 document = decode_and_parse(bytes, encoding, html)?;
173 }
174 }
175 Ok(Parsed {
176 document,
177 encoding,
178 confidence,
179 })
180}
181
182fn decode_and_parse(
183 bytes: &[u8],
184 encoding: &'static Encoding,
185 options: parsec::Options,
186) -> Result<Document, ParseError> {
187 let (text, _) = encoding.decode_with_bom_removal(bytes);
190 parsec::parse_with(&text, options)
191}
192
193fn declared(doc: &Document) -> Option<&'static Encoding> {
198 (0..doc.len() as NodeId).find_map(|id| {
199 if !matches!(doc.kind(id), Kind::Element { ns: Ns::Html, .. })
200 || doc.lower_name(id) != Some("meta")
201 {
202 return None;
203 }
204 let attr = |want: &str| {
205 doc.attrs(id)
206 .find(|&(ns, name, _)| ns == AttrNs::None && name == want)
207 .map(|(_, _, value)| value)
208 };
209 if let Some(found) = attr("charset").and_then(|v| Encoding::for_label(v.as_bytes())) {
210 return Some(found);
211 }
212 attr("http-equiv")
213 .filter(|v| v.eq_ignore_ascii_case("content-type"))
214 .and(attr("content"))
215 .and_then(|v| prescan::charset_in_content(v.as_bytes()))
216 })
217}
218
219fn changed(current: &'static Encoding, declared: &'static Encoding) -> Option<&'static Encoding> {
222 if current == UTF_16BE || current == UTF_16LE {
223 return None;
224 }
225 let new = prescan::as_declared(declared);
226 (new != current).then_some(new)
227}
228
229pub fn locale_default(locale: &str) -> &'static Encoding {
241 use encoding_rs::*;
242 let locale = locale.to_ascii_lowercase();
243 let mut tags = locale.split(['-', '_']);
244 match tags.next().unwrap_or("") {
245 "ar" | "fa" => WINDOWS_1256,
246 "az" | "ku" | "tr" => WINDOWS_1254,
247 "ba" | "be" | "bg" | "kk" | "ky" | "mk" | "ru" | "sah" | "sr" | "tg" | "tt" | "uk" => {
248 WINDOWS_1251
249 }
250 "cs" | "hr" | "sk" => WINDOWS_1250,
251 "el" => ISO_8859_7,
252 "et" | "lt" | "lv" => WINDOWS_1257,
253 "he" => WINDOWS_1255,
254 "hu" | "pl" | "sl" => ISO_8859_2,
255 "ja" => SHIFT_JIS,
256 "ko" => EUC_KR,
257 "th" => WINDOWS_874,
258 "vi" => WINDOWS_1258,
259 "zh" => tags
261 .find_map(|t| match t {
262 "hans" | "cn" | "sg" => Some(GBK),
263 "hant" | "hk" | "mo" | "tw" => Some(BIG5),
264 _ => None,
265 })
266 .unwrap_or(WINDOWS_1252),
267 _ => WINDOWS_1252,
268 }
269}
270
271#[cfg(test)]
272mod tests {
273 use super::*;
274 use encoding_rs::{EUC_JP, SHIFT_JIS, UTF_8, WINDOWS_1251, WINDOWS_1252};
275
276 fn parse(bytes: &[u8]) -> Parsed {
277 parse_bytes(bytes, Options::default()).unwrap()
278 }
279
280 #[test]
281 fn bom_beats_transport_and_meta() {
282 let page = b"\xef\xbb\xbf<meta charset=euc-jp>\xc3\xa9";
283 let options = Options {
284 transport: Some(SHIFT_JIS),
285 ..Options::default()
286 };
287 let p = parse_bytes(page, options).unwrap();
288 assert_eq!((p.encoding, p.confidence), (UTF_8, Confidence::Certain));
289 assert_eq!(p.document.text(p.document.root()), "é");
290 }
291
292 #[test]
293 fn utf16_bom_is_removed() {
294 let p = parse(b"\xff\xfe<\0p\0>\0h\0i\0");
295 assert_eq!(p.encoding, UTF_16LE);
296 assert_eq!(p.document.text(p.document.root()), "hi");
297 }
298
299 #[test]
300 fn transport_is_certain() {
301 let options = Options {
302 transport: Some(WINDOWS_1251),
303 ..Options::default()
304 };
305 let p = parse_bytes(b"<meta charset=euc-jp>\xcf", options).unwrap();
306 assert_eq!(
307 (p.encoding, p.confidence),
308 (WINDOWS_1251, Confidence::Certain)
309 );
310 assert_eq!(p.document.text(p.document.root()), "П");
311 }
312
313 #[test]
314 fn late_meta_reparses() {
315 let mut page = b"<!doctype html><title>".to_vec();
316 page.extend(std::iter::repeat_n(b'x', 1100));
317 page.extend(b"</title><meta charset=windows-1251><p>\xcf");
318 let p = parse(&page);
319 assert_eq!(
320 (p.encoding, p.confidence),
321 (WINDOWS_1251, Confidence::Certain)
322 );
323 assert!(p.document.text(p.document.root()).ends_with('П'));
324 }
325
326 #[test]
327 fn agreeing_meta_makes_it_certain() {
328 let p = parse(b"<meta charset=euc-jp>");
329 assert_eq!((p.encoding, p.confidence), (EUC_JP, Confidence::Certain));
330 let p = parse(b"<p>no declaration");
331 assert_eq!(
332 (p.encoding, p.confidence),
333 (WINDOWS_1252, Confidence::Tentative)
334 );
335 }
336
337 #[test]
338 fn late_http_equiv_and_svg_meta() {
339 let mut page = b"<!-- ".to_vec();
340 page.extend(std::iter::repeat_n(b'x', 1100));
341 page.extend(b" --><svg><meta charset=euc-jp></svg>");
342 assert_eq!(parse(&page).encoding, EUC_JP);
344 let mut page = b"<!-- ".to_vec();
345 page.extend(std::iter::repeat_n(b'x', 1100));
346 page.extend(b" --><meta http-equiv=content-type content='text/html;charset=sjis'>");
347 assert_eq!(parse(&page).encoding, SHIFT_JIS);
348 }
349
350 #[test]
351 fn later_metas_and_utf16_declarations() {
352 let mut page = b"<!-- ".to_vec();
353 page.extend(std::iter::repeat_n(b'x', 1100));
354 page.extend(b" --><meta charset=bogus><meta charset=utf-16><meta charset=sjis>");
355 assert_eq!(parse(&page).encoding, UTF_8);
357 }
358
359 #[test]
360 fn budget_covers_both_parses() {
361 let mut page = b"<title>".to_vec();
362 page.extend(std::iter::repeat_n(b'x', 1100));
363 page.extend(b"</title><meta charset=windows-1251>");
364 let options = Options {
365 html: parsec::Options {
366 budget: Some(std::time::Duration::from_secs(60)),
367 ..parsec::Options::default()
368 },
369 ..Options::default()
370 };
371 assert_eq!(parse_bytes(&page, options).unwrap().encoding, WINDOWS_1251);
372 }
373
374 #[test]
375 fn valid_utf8_is_detected() {
376 let p = parse("<p>caf\u{e9}".as_bytes());
377 assert_eq!((p.encoding, p.confidence), (UTF_8, Confidence::Tentative));
378 assert_eq!(p.document.text(p.document.root()), "caf\u{e9}");
379 assert_eq!(parse(b"<p>caf\xe9").encoding, WINDOWS_1252);
381 assert_eq!(parse(b"<p>cafe").encoding, WINDOWS_1252);
382 assert_eq!(parse(b"<p>\xe6\x97").encoding, WINDOWS_1252);
384 let page = "<meta charset=windows-1251><p>\u{e9}".as_bytes();
386 assert_eq!(parse(page).encoding, WINDOWS_1251);
387 let mut page = "<title>\u{e9}".as_bytes().to_vec();
389 page.extend(std::iter::repeat_n(b'x', 1100));
390 page.extend(b"</title><meta charset=windows-1251>");
391 let p = parse(&page);
392 assert_eq!(
393 (p.encoding, p.confidence),
394 (WINDOWS_1251, Confidence::Certain)
395 );
396 }
397
398 #[test]
399 fn locales() {
400 assert_eq!(locale_default("RU"), WINDOWS_1251);
401 assert_eq!(locale_default("zh-Hans-HK"), encoding_rs::GBK);
402 assert_eq!(locale_default("zh"), WINDOWS_1252);
403 assert_eq!(locale_default(""), WINDOWS_1252);
404 }
405}