use encoding_rs::{CoderResult, Encoding, UTF_8, UTF_16BE, UTF_16LE};
pub struct Offsets {
points: Vec<(u32, u32)>,
}
impl Offsets {
pub fn new(bytes: &[u8], encoding: &'static Encoding) -> Offsets {
let mut map = Offsets { points: Vec::new() };
let bom = match Encoding::for_bom(bytes) {
Some((found, len)) if found == encoding => len,
_ => 0,
};
map.at(0, bom);
if encoding == UTF_8 {
let (mut decoded, mut source) = (0, bom);
for chunk in bytes[bom..].utf8_chunks() {
decoded += chunk.valid().len();
source += chunk.valid().len();
if !chunk.invalid().is_empty() {
decoded += '\u{FFFD}'.len_utf8();
source += chunk.invalid().len();
map.at(decoded, source);
}
}
return map;
}
if encoding.is_single_byte() {
let (text, _) = encoding.decode_without_bom_handling(bytes);
let mut decoded = 0;
for (i, c) in text.chars().enumerate() {
map.at(decoded, i);
decoded += c.len_utf8();
}
return map;
}
let narrow = encoding != UTF_16BE && encoding != UTF_16LE;
let mut decoder = encoding.new_decoder_without_bom_handling();
let mut out = [0u8; 64];
let mut decoded = 0;
let mut from = bom;
for i in bom..=bytes.len() {
let last = i == bytes.len();
let mut input = if last { &[][..] } else { &bytes[i..i + 1] };
let mut first = true;
loop {
let (result, read, written, _) = decoder.decode_to_utf8(input, &mut out, last);
input = &input[read..];
for (k, &b) in out[..written].iter().enumerate() {
if (b as i8) >= -0x40 {
let start = if first && !(narrow && b < 0x80) {
from
} else {
i
};
map.at(decoded + k, start);
first = false;
}
}
decoded += written;
if result == CoderResult::InputEmpty {
break;
}
}
if !first {
from = i + 1;
}
}
map.at(decoded, bytes.len());
map
}
fn at(&mut self, decoded: usize, source: usize) {
let (d, s) = self.points.last().copied().unwrap_or((0, 0));
if source as u64 != u64::from(s) + (decoded as u64 - u64::from(d)) {
self.points.push((decoded as u32, source as u32));
}
}
pub fn is_identity(&self) -> bool {
self.points.is_empty()
}
pub fn to_source(&self, decoded: usize) -> usize {
let i = self.points.partition_point(|&(d, _)| d as usize <= decoded);
match i.checked_sub(1).map(|i| self.points[i]) {
Some((d, s)) => s as usize + (decoded - d as usize),
None => decoded,
}
}
pub fn to_decoded(&self, source: usize) -> usize {
let i = self.points.partition_point(|&(_, s)| s as usize <= source);
match i.checked_sub(1).map(|i| self.points[i]) {
Some((d, s)) => {
let at = d as usize + (source - s as usize);
self.points
.get(i)
.map_or(at, |&(next, _)| at.min(next as usize))
}
None => source,
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use encoding_rs::*;
fn check(bytes: &[u8], encoding: &'static Encoding) -> Offsets {
let (text, _) = encoding.decode_with_bom_removal(bytes);
let map = Offsets::new(bytes, encoding);
for (d, c) in text.char_indices() {
if c.is_ascii() && encoding != UTF_16BE {
let s = map.to_source(d);
assert_eq!(bytes[s], c as u8, "{} at {d}", encoding.name());
assert_eq!(map.to_decoded(s), d);
}
}
assert_eq!(map.to_source(text.len()), bytes.len());
map
}
#[test]
fn ascii_and_clean_utf8_are_the_identity() {
assert!(check(b"<p>plain", UTF_8).is_identity());
assert!(check("<p>caf\u{e9}".as_bytes(), UTF_8).is_identity());
assert!(check(b"<p>plain", WINDOWS_1252).is_identity());
}
#[test]
fn utf8_boms_and_bad_bytes() {
let map = check(b"\xef\xbb\xbf<p>\xff\xfe</p><b>x", UTF_8);
assert_eq!(map.to_source(3 + 6 + 2), 3 + 3 + 2 + 2);
}
#[test]
fn legacy_multibyte() {
check(b"<p>\x93\xfa\x96\x7b</p><b>x</b>", SHIFT_JIS);
check(b"<p>\xc6\xfc\xcb\xdc</p><b>x</b>", EUC_JP);
check(b"<p>\x1b$BF|K\\\x1b(B</p><b>x</b>", ISO_2022_JP);
check(b"<p>\xc8\xd5\xb1\xbe</p><b>x</b>", GBK);
check(b"<p>\x81\x30\x81\x30\xc8\xd5</p><b>x</b>", GB18030);
check(b"<p>\xa4\xe9\xa5\xbb</p><b>x</b>", BIG5);
check(b"<p>\xc0\xcf\xba\xbb</p><b>x</b>", EUC_KR);
check(b"<p>\xcf\xf0\xe8</p><b>x</b>", WINDOWS_1251);
}
#[test]
fn a_bad_lead_byte_then_ascii() {
let map = check(b"<p>\x93<b>x</b>", SHIFT_JIS);
assert_eq!(map.to_source(6), 4);
check(b"<p>x\x93", SHIFT_JIS);
}
#[test]
fn utf16() {
let mut bytes = vec![0xff, 0xfe];
for u in "<p>\u{65e5}\u{1F600}x".encode_utf16() {
bytes.extend(u.to_le_bytes());
}
let map = check(&bytes, UTF_16LE);
assert_eq!(map.to_source(0), 2);
assert_eq!(map.to_source(3), 8);
assert_eq!(map.to_source(6), 10);
assert_eq!(map.to_source(10), 14);
assert_eq!(map.to_decoded(9), 4);
}
}