use std::borrow::Cow;
pub const BROKEN_UTF8: &str = "UTF-8";
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum RStr {
Text(String),
Bytes(Vec<u8>, &'static str),
}
impl RStr {
pub fn encoding(&self) -> Option<&'static str> {
match self {
RStr::Text(_) => None,
RStr::Bytes(_, enc) => Some(enc),
}
}
pub fn bytes(&self) -> &[u8] {
match self {
RStr::Text(s) => s.as_bytes(),
RStr::Bytes(b, _) => b,
}
}
pub fn view(&self) -> Cow<'_, str> {
match self {
RStr::Text(s) => Cow::Borrowed(s),
RStr::Bytes(b, _) => Cow::Owned(bytes_to_latin1(b)),
}
}
pub fn set_view(&mut self, s: String) {
match self {
RStr::Text(t) => *t = s,
RStr::Bytes(b, _) => *b = latin1_to_bytes(&s),
}
}
pub fn set_encoding(&mut self, enc: &'static str) {
let bytes = match std::mem::replace(self, RStr::Text(String::new())) {
RStr::Text(s) => latin1_to_bytes(&s),
RStr::Bytes(b, _) => b,
};
*self = if enc == BROKEN_UTF8 {
RStr::from_utf8(bytes)
} else {
RStr::Bytes(bytes, enc)
};
}
pub fn clear_encoding(&mut self) {
if let RStr::Bytes(b, _) = self {
*self = RStr::Text(bytes_to_latin1(b));
}
}
pub fn from_utf8(bytes: Vec<u8>) -> RStr {
match String::from_utf8(bytes) {
Ok(s) => RStr::Text(s),
Err(e) => RStr::Bytes(e.into_bytes(), BROKEN_UTF8),
}
}
pub fn is_broken_utf8(&self) -> bool {
matches!(self, RStr::Bytes(_, BROKEN_UTF8))
}
}
pub fn bytes_to_latin1(bytes: &[u8]) -> String {
bytes.iter().map(|&b| char::from(b)).collect()
}
pub fn latin1_to_bytes(s: &str) -> Vec<u8> {
let mut out = Vec::with_capacity(s.len());
for c in s.chars() {
match u8::try_from(u32::from(c)) {
Ok(b) => out.push(b),
Err(_) => out.extend_from_slice(c.encode_utf8(&mut [0; 4]).as_bytes()),
}
}
out
}
pub fn utf8_units(bytes: &[u8]) -> impl Iterator<Item = &[u8]> + '_ {
bytes.utf8_chunks().flat_map(|chunk| {
let valid = chunk.valid();
let chars = valid
.char_indices()
.map(move |(i, c)| &valid.as_bytes()[i..i + c.len_utf8()]);
let invalid = chunk.invalid().chunks(1);
chars.chain(invalid)
})
}
pub mod utf8 {
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Mbc {
Found(usize),
NeedMore,
Invalid,
}
pub fn precise(b: &[u8]) -> Mbc {
let Some(&first) = b.first() else {
return Mbc::NeedMore;
};
let (second, len) = match first {
0x00..=0x7F => return Mbc::Found(1),
0xC2..=0xDF => (0x80..=0xBF, 2),
0xE0 => (0xA0..=0xBF, 3),
0xE1..=0xEC | 0xEE..=0xEF => (0x80..=0xBF, 3),
0xED => (0x80..=0x9F, 3),
0xF0 => (0x90..=0xBF, 4),
0xF1..=0xF3 => (0x80..=0xBF, 4),
0xF4 => (0x80..=0x8F, 4),
_ => return Mbc::Invalid,
};
for i in 1..len {
let Some(&c) = b.get(i) else {
return Mbc::NeedMore;
};
let ok = if i == 1 {
second.contains(&c)
} else {
(0x80..=0xBF).contains(&c)
};
if !ok {
return Mbc::Invalid;
}
}
Mbc::Found(len)
}
pub fn mbclen(b: &[u8], p: usize) -> usize {
match precise(&b[p..]) {
Mbc::Found(n) => n,
_ => 1,
}
}
pub fn fast_mbclen(b: &[u8], p: usize) -> usize {
match precise(&b[p..]) {
Mbc::Found(n) => n,
Mbc::NeedMore => b.len() - p,
Mbc::Invalid => 1,
}
}
pub fn left_head(b: &[u8], start: usize, s: usize) -> usize {
let mut p = s;
while p > start && p < b.len() && b[p] & 0xC0 == 0x80 {
p -= 1;
}
p
}
pub fn right_head(b: &[u8], start: usize, s: usize) -> usize {
let p = left_head(b, start, s);
if p < s {
p + fast_mbclen(b, p)
} else {
p
}
}
pub fn prev_char(b: &[u8], start: usize, s: usize) -> Option<usize> {
(s > start).then(|| left_head(b, start, s - 1))
}
pub fn at_char_boundary(b: &[u8], start: usize, s: usize) -> bool {
left_head(b, start, s) == s
}
pub fn at_char_right_boundary(b: &[u8], start: usize, s: usize) -> bool {
right_head(b, start, s) == s
}
pub fn char_starts(b: &[u8]) -> Vec<usize> {
let mut at = Vec::with_capacity(b.len() + 1);
let mut p = 0;
while p < b.len() {
at.push(p);
p += mbclen(b, p);
}
at.push(b.len());
at
}
pub fn strlen(b: &[u8]) -> usize {
let mut n = 0;
let mut p = 0;
while p < b.len() {
p += mbclen(b, p);
n += 1;
}
n
}
pub fn offset(b: &[u8], nth: usize) -> usize {
let mut p = 0;
for _ in 0..nth {
if p >= b.len() {
break;
}
p += mbclen(b, p);
}
p.min(b.len())
}
pub fn search(b: &[u8], sub: &[u8], from: usize) -> Option<usize> {
let mut start = from;
loop {
let hay = b.get(start..)?;
let pos = start + find(hay, sub)?;
let t = right_head(b, start, pos);
if t == pos {
return Some(pos);
}
if t >= b.len() {
return None;
}
start = t;
}
}
pub fn rsearch(b: &[u8], sub: &[u8], s: usize) -> Option<usize> {
if sub.is_empty() {
return Some(s);
}
if b[s..].starts_with(sub) {
return Some(s);
}
let mut searchlen = s + 1;
while searchlen > 0 {
let hit = b[..searchlen.min(b.len())]
.iter()
.rposition(|&c| c == sub[0])?;
let adjusted = left_head(b, 0, hit);
if hit == adjusted && b[hit..].starts_with(sub) {
return Some(hit);
}
searchlen = adjusted;
}
None
}
fn find(hay: &[u8], needle: &[u8]) -> Option<usize> {
if needle.is_empty() {
return Some(0);
}
hay.windows(needle.len()).position(|w| w == needle)
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn walking_forward_and_stepping_back_disagree_on_a_cut_sequence() {
let b = b"a\xE3\x81";
assert_eq!(utf8::strlen(b), 3);
assert_eq!(utf8::prev_char(b, 0, b.len()), Some(1));
assert_eq!(utf8::char_starts(b), vec![0, 1, 2, 3]);
}
#[test]
fn a_search_hit_inside_a_sequence_is_skipped() {
assert_eq!(utf8::search(b"\xE3\x81\x82x", b"\x81\x82x", 0), None);
assert_eq!(utf8::search(b"a\xFFb", b"b", 0), Some(2));
}
#[test]
fn units_split_invalid_sequences_byte_by_byte() {
let units: Vec<&[u8]> = utf8_units(b"\xE3\x81a\xE3\x81\x82\xFF").collect();
assert_eq!(
units,
vec![&b"\xE3"[..], b"\x81", b"a", "\u{3042}".as_bytes(), b"\xFF"]
);
}
#[test]
fn latin1_view_round_trips_every_byte() {
let all: Vec<u8> = (0..=255).collect();
assert_eq!(latin1_to_bytes(&bytes_to_latin1(&all)), all);
}
#[test]
fn tagging_utf8_normalizes_valid_bytes_to_text() {
let mut s = RStr::Text("\u{e9}".into()); s.set_encoding(BROKEN_UTF8);
assert_eq!(s, RStr::Bytes(vec![0xE9], BROKEN_UTF8));
let mut t = RStr::Text("\u{c3}\u{a9}".into()); t.set_encoding(BROKEN_UTF8);
assert_eq!(t, RStr::Text("\u{e9}".into()));
}
}