retex-engine 0.9.0

//! UTF-8 encoding for TeX characters.
//!
//! TeX characters are code points 0–0x10FFFF, including surrogates, which
//! LuaTeX encodes like any other value (a superset of standard UTF-8, as in
//! WTF-8). Strings of TeX characters are therefore kept as byte vectors.

/// Appends the encoding of `c` to `out`.
pub fn push(out: &mut Vec<u8>, c: u32) {
    if c < 0x80 {
        out.push(c as u8);
    } else if c < 0x800 {
        out.push(0xC0 | (c >> 6) as u8);
        out.push(0x80 | (c & 0x3F) as u8);
    } else if c < 0x10000 {
        out.push(0xE0 | (c >> 12) as u8);
        out.push(0x80 | ((c >> 6) & 0x3F) as u8);
        out.push(0x80 | (c & 0x3F) as u8);
    } else {
        out.push(0xF0 | (c >> 18) as u8);
        out.push(0x80 | ((c >> 12) & 0x3F) as u8);
        out.push(0x80 | ((c >> 6) & 0x3F) as u8);
        out.push(0x80 | (c & 0x3F) as u8);
    }
}

/// The encoding of a single character.
pub fn encode(c: u32) -> Vec<u8> {
    let mut v = Vec::with_capacity(4);
    push(&mut v, c);
    v
}

/// Number of bytes in the encoding of `c`.
pub fn len(c: u32) -> usize {
    match c {
        0..0x80 => 1,
        0x80..0x800 => 2,
        0x800..0x10000 => 3,
        _ => 4,
    }
}

/// Decodes the character starting at `bytes[i]`, returning it and its length.
/// Malformed sequences decode as their first byte (length 1), which is how
/// LuaTeX treats invalid input bytes.
pub fn decode(bytes: &[u8], i: usize) -> (u32, usize) {
    let b0 = bytes[i];
    let need = match b0 {
        0x00..=0x7F => return (u32::from(b0), 1),
        0xC0..=0xDF => 2,
        0xE0..=0xEF => 3,
        0xF0..=0xF7 => 4,
        _ => return (u32::from(b0), 1),
    };
    if i + need > bytes.len() {
        return (u32::from(b0), 1);
    }
    let mut c = u32::from(b0) & (0x7F >> need);
    for k in 1..need {
        let b = bytes[i + k];
        if b & 0xC0 != 0x80 {
            return (u32::from(b0), 1);
        }
        c = (c << 6) | u32::from(b & 0x3F);
    }
    (c, need)
}

/// Decodes a whole byte string into characters.
pub fn chars(bytes: &[u8]) -> Vec<u32> {
    let mut out = Vec::with_capacity(bytes.len());
    let mut i = 0;
    while i < bytes.len() {
        let (c, n) = decode(bytes, i);
        out.push(c);
        i += n;
    }
    out
}

/// If `bytes` encodes exactly one character, returns it.
pub fn single_char(bytes: &[u8]) -> Option<u32> {
    if bytes.is_empty() {
        return None;
    }
    let (c, n) = decode(bytes, 0);
    (n == bytes.len()).then_some(c)
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn round_trip() {
        for c in [
            0, 0x41, 0x7F, 0x80, 0xE9, 0x7FF, 0x800, 0xD800, 0xFFFF, 0x10000, 0x10FFFF,
        ] {
            let e = encode(c);
            assert_eq!(e.len(), len(c));
            assert_eq!(decode(&e, 0), (c, e.len()));
            assert_eq!(single_char(&e), Some(c));
        }
        assert_eq!(chars("aé😀".as_bytes()), vec![0x61, 0xE9, 0x1F600]);
    }
}