#[derive(Copy, Clone, Debug, PartialEq, Eq)]
pub(crate) enum Encoding {
Utf16Be,
Utf16Le,
Utf8Bom,
Utf8Sniffed,
PdfDoc,
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub(crate) struct Decoded {
pub text: String,
pub encoding: Encoding,
pub undefined: usize,
}
pub(crate) fn decode(bytes: &[u8]) -> Decoded {
if let Some(rest) = bytes.strip_prefix(&[0xFE, 0xFF]) {
return Decoded {
text: from_utf16(rest, u16::from_be_bytes),
encoding: Encoding::Utf16Be,
undefined: 0,
};
}
if let Some(rest) = bytes.strip_prefix(&[0xFF, 0xFE]) {
return Decoded {
text: from_utf16(rest, u16::from_le_bytes),
encoding: Encoding::Utf16Le,
undefined: 0,
};
}
if let Some(rest) = bytes.strip_prefix(&[0xEF, 0xBB, 0xBF]) {
return Decoded {
text: String::from_utf8_lossy(rest).into_owned(),
encoding: Encoding::Utf8Bom,
undefined: 0,
};
}
if bytes.iter().any(|&b| b >= 0x80)
&& let Ok(s) = std::str::from_utf8(bytes)
{
return Decoded {
text: s.to_string(),
encoding: Encoding::Utf8Sniffed,
undefined: 0,
};
}
let mut text = String::with_capacity(bytes.len());
let mut undefined = 0;
for &b in bytes {
let (c, defined) = pdfdoc(b);
if !defined {
undefined += 1;
}
text.push(c);
}
Decoded {
text,
encoding: Encoding::PdfDoc,
undefined,
}
}
fn pdfdoc(b: u8) -> (char, bool) {
match b {
0x18..=0x1F => (ACCENTS[(b - 0x18) as usize], true),
0x7F | 0x9F | 0xAD => (b as char, false),
0x80..=0xA0 => (HIGH[(b - 0x80) as usize], true),
_ => (b as char, true),
}
}
static ACCENTS: [char; 8] = [
'\u{02D8}', '\u{02C7}', '\u{02C6}', '\u{02D9}', '\u{02DD}', '\u{02DB}', '\u{02DA}', '\u{02DC}', ];
static HIGH: [char; 33] = [
'\u{2022}', '\u{2020}', '\u{2021}', '\u{2026}', '\u{2014}', '\u{2013}', '\u{0192}', '\u{2044}', '\u{2039}', '\u{203A}', '\u{2212}', '\u{2030}', '\u{201E}', '\u{201C}', '\u{201D}', '\u{2018}', '\u{2019}', '\u{201A}', '\u{2122}', '\u{FB01}', '\u{FB02}', '\u{0141}', '\u{0152}', '\u{0160}', '\u{0178}', '\u{017D}', '\u{0131}', '\u{0142}', '\u{0153}', '\u{0161}', '\u{017E}', '\u{FFFD}', '\u{20AC}', ];
fn from_utf16(bytes: &[u8], unit: fn([u8; 2]) -> u16) -> String {
let units: Vec<u16> = bytes.as_chunks::<2>().0.iter().copied().map(unit).collect();
String::from_utf16_lossy(&units)
}
#[cfg(test)]
mod tests {
use super::*;
fn text(bytes: &[u8]) -> String {
decode(bytes).text
}
#[test]
fn utf16be_mark_wins() {
let d = decode(b"\xFE\xFF\x00C\x00a\x00f\x00\xE9");
assert_eq!(d.text, "Café");
assert_eq!(d.encoding, Encoding::Utf16Be);
}
#[test]
fn utf16le_mark_is_accepted_though_off_spec() {
let d = decode(b"\xFF\xFEC\x00a\x00f\x00\xE9\x00");
assert_eq!(d.text, "Café");
assert_eq!(d.encoding, Encoding::Utf16Le);
}
#[test]
fn utf8_mark_is_accepted_though_off_spec() {
let d = decode(b"\xEF\xBB\xBFCaf\xC3\xA9");
assert_eq!(d.text, "Café");
assert_eq!(d.encoding, Encoding::Utf8Bom);
}
#[test]
fn a_bare_mark_decodes_to_nothing() {
assert_eq!(text(b"\xFE\xFF"), "");
assert_eq!(text(b""), "");
}
#[test]
fn an_odd_trailing_byte_loses_a_character_not_the_string() {
assert_eq!(text(b"\xFE\xFF\x00T\x00"), "T");
}
#[test]
fn decodes_a_surrogate_pair() {
assert_eq!(text(b"\xFE\xFF\xD8=\xDE\x00"), "\u{1F600}");
}
#[test]
fn ascii_is_unchanged_and_reported_as_pdfdoc() {
let d = decode(b"plain comment");
assert_eq!(d.text, "plain comment");
assert_eq!(d.encoding, Encoding::PdfDoc);
assert_eq!(d.undefined, 0);
}
#[test]
fn decodes_the_punctuation_block() {
assert_eq!(text(b"don\x90t"), "don\u{2019}t");
assert_eq!(text(b"\x8Dquoted\x8E"), "\u{201C}quoted\u{201D}");
assert_eq!(text(b"a \x84 dash"), "a — dash");
assert_eq!(text(b"an \x85 dash"), "an – dash");
assert_eq!(text(b"\x80 item"), "• item");
assert_eq!(text(b"and so on\x83"), "and so on…");
assert_eq!(text(b"\x8Flow\x91"), "\u{2018}low\u{201A}");
}
#[test]
fn decodes_the_ligature_and_symbol_slots() {
assert_eq!(text(b"\x93nal"), "\u{FB01}nal");
assert_eq!(text(b"\x94ow"), "\u{FB02}ow");
assert_eq!(text(b"Widget\x92"), "Widget™");
assert_eq!(text(b"\xA0100"), "€100");
assert_eq!(text(b"\x8A5"), "\u{2212}5");
assert_eq!(text(b"\x8B"), "\u{2030}");
assert_eq!(text(b"\x9Ccuvre"), "\u{0153}cuvre");
}
#[test]
fn decodes_the_accent_block_where_ascii_has_controls() {
assert_eq!(text(b"\x18\x19\x1A"), "\u{02D8}\u{02C7}\u{02C6}");
assert_eq!(text(b"\x1F"), "\u{02DC}");
}
#[test]
fn tab_newline_and_return_survive() {
assert_eq!(text(b"a\tb\nc\rd"), "a\tb\nc\rd");
}
#[test]
fn agrees_with_latin1_from_a1_upwards() {
assert_eq!(text(b"caf\xE9"), "café");
assert_eq!(text(b"\xDCber"), "Über");
assert_eq!(text(b"\xA1"), "¡");
}
#[test]
fn undefined_slots_are_counted_not_dropped() {
let d = decode(b"soft\xADhyphen");
assert_eq!(d.text, "soft\u{00AD}hyphen");
assert_eq!(d.undefined, 1);
assert_eq!(decode(b"\x7F\x9F\xAD").undefined, 3);
}
#[test]
fn bare_utf8_is_sniffed_when_multi_byte() {
let d = decode(b"caf\xC3\xA9");
assert_eq!(d.text, "café");
assert_eq!(d.encoding, Encoding::Utf8Sniffed);
}
#[test]
fn high_bytes_that_are_not_utf8_fall_to_the_table() {
let d = decode(b"don\x90t");
assert_eq!(d.text, "don\u{2019}t");
assert_eq!(d.encoding, Encoding::PdfDoc);
}
#[test]
fn the_sniff_does_not_claim_pure_ascii() {
assert_eq!(decode(b"plain").encoding, Encoding::PdfDoc);
}
}