retex_engine/
utf8x.rs

1//! UTF-8 encoding for TeX characters.
2//!
3//! TeX characters are code points 0–0x10FFFF, including surrogates, which
4//! LuaTeX encodes like any other value (a superset of standard UTF-8, as in
5//! WTF-8). Strings of TeX characters are therefore kept as byte vectors.
6
7/// Appends the encoding of `c` to `out`.
8pub fn push(out: &mut Vec<u8>, c: u32) {
9    if c < 0x80 {
10        out.push(c as u8);
11    } else if c < 0x800 {
12        out.push(0xC0 | (c >> 6) as u8);
13        out.push(0x80 | (c & 0x3F) as u8);
14    } else if c < 0x10000 {
15        out.push(0xE0 | (c >> 12) as u8);
16        out.push(0x80 | ((c >> 6) & 0x3F) as u8);
17        out.push(0x80 | (c & 0x3F) as u8);
18    } else {
19        out.push(0xF0 | (c >> 18) as u8);
20        out.push(0x80 | ((c >> 12) & 0x3F) as u8);
21        out.push(0x80 | ((c >> 6) & 0x3F) as u8);
22        out.push(0x80 | (c & 0x3F) as u8);
23    }
24}
25
26/// The encoding of a single character.
27pub fn encode(c: u32) -> Vec<u8> {
28    let mut v = Vec::with_capacity(4);
29    push(&mut v, c);
30    v
31}
32
33/// Number of bytes in the encoding of `c`.
34pub fn len(c: u32) -> usize {
35    match c {
36        0..0x80 => 1,
37        0x80..0x800 => 2,
38        0x800..0x10000 => 3,
39        _ => 4,
40    }
41}
42
43/// Decodes the character starting at `bytes[i]`, returning it and its length.
44/// Malformed sequences decode as their first byte (length 1), which is how
45/// LuaTeX treats invalid input bytes.
46pub fn decode(bytes: &[u8], i: usize) -> (u32, usize) {
47    let b0 = bytes[i];
48    let need = match b0 {
49        0x00..=0x7F => return (u32::from(b0), 1),
50        0xC0..=0xDF => 2,
51        0xE0..=0xEF => 3,
52        0xF0..=0xF7 => 4,
53        _ => return (u32::from(b0), 1),
54    };
55    if i + need > bytes.len() {
56        return (u32::from(b0), 1);
57    }
58    let mut c = u32::from(b0) & (0x7F >> need);
59    for k in 1..need {
60        let b = bytes[i + k];
61        if b & 0xC0 != 0x80 {
62            return (u32::from(b0), 1);
63        }
64        c = (c << 6) | u32::from(b & 0x3F);
65    }
66    (c, need)
67}
68
69/// Decodes a whole byte string into characters.
70pub fn chars(bytes: &[u8]) -> Vec<u32> {
71    let mut out = Vec::with_capacity(bytes.len());
72    let mut i = 0;
73    while i < bytes.len() {
74        let (c, n) = decode(bytes, i);
75        out.push(c);
76        i += n;
77    }
78    out
79}
80
81/// Marks, in a decoded input line, a byte that does not start a valid
82/// UTF-8 sequence: `RAW_BYTE | b`. The tokenizer reports it (LuaTeX's
83/// "String contains an invalid utf-8 sequence") and error contexts show
84/// the byte itself.
85pub const RAW_BYTE: u32 = 0x12_0000;
86
87/// Whether `c` is a [`RAW_BYTE`] marker.
88#[inline]
89pub fn is_raw(c: u32) -> bool {
90    c & !0xFF == RAW_BYTE
91}
92
93/// Decodes an input line like [`chars`], but marks the bytes of invalid
94/// sequences with [`RAW_BYTE`].
95pub fn chars_marking_invalid(bytes: &[u8]) -> Vec<u32> {
96    let mut out = Vec::with_capacity(bytes.len());
97    let mut i = 0;
98    while i < bytes.len() {
99        let (c, n) = decode(bytes, i);
100        if n == 1 && c >= 0x80 {
101            out.push(RAW_BYTE | c);
102        } else {
103            out.push(c);
104        }
105        i += n;
106    }
107    out
108}
109
110/// Appends the bytes of input-line entry `c` to `out`: a [`RAW_BYTE`]
111/// marker stands for its byte, any other value for its encoding.
112pub fn push_entry(out: &mut Vec<u8>, c: u32) {
113    if is_raw(c) {
114        out.push((c & 0xFF) as u8);
115    } else {
116        push(out, c);
117    }
118}
119
120/// If `bytes` encodes exactly one character, returns it.
121pub fn single_char(bytes: &[u8]) -> Option<u32> {
122    if bytes.is_empty() {
123        return None;
124    }
125    let (c, n) = decode(bytes, 0);
126    (n == bytes.len()).then_some(c)
127}
128
129#[cfg(test)]
130mod tests {
131    use super::*;
132
133    #[test]
134    fn round_trip() {
135        for c in [
136            0, 0x41, 0x7F, 0x80, 0xE9, 0x7FF, 0x800, 0xD800, 0xFFFF, 0x10000, 0x10FFFF,
137        ] {
138            let e = encode(c);
139            assert_eq!(e.len(), len(c));
140            assert_eq!(decode(&e, 0), (c, e.len()));
141            assert_eq!(single_char(&e), Some(c));
142        }
143        assert_eq!(chars("aé😀".as_bytes()), vec![0x61, 0xE9, 0x1F600]);
144    }
145}
retex_engine/utf8x.rs

retex_engine/
utf8x.rs
