ferroni/encodings/
utf8.rs

1//! Port of `utf8.c`: the UTF-8 encoding, in the RFC 3629 range U+0000 to
2//! U+10FFFF. [`ONIG_ENCODING_UTF8`] is the static that exposes it.
3
4use crate::oniguruma::*;
5use crate::regenc::*;
6
7// === UTF-8 Helpers ===
8
9#[inline]
10fn utf8_islead(c: u8) -> bool {
11    (c & 0xc0) != 0x80
12}
13
14#[inline]
15fn utf8_istail(c: u8) -> bool {
16    (c & 0xc0) == 0x80
17}
18
19// === EncLen_UTF8 Table ===
20// Maps first byte to character length (RFC 3629: max 4 bytes).
21
22static ENC_LEN_UTF8: [u8; 256] = [
23    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
24    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
25    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
26    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
27    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
28    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
29    1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
30    3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
31];
32
33// === UTF-8 Encoding Struct ===
34
35/// The UTF-8 encoding type behind [`ONIG_ENCODING_UTF8`].
36pub struct Utf8Encoding;
37
38/// UTF-8, the encoding to pass to `onig_new` for UTF-8 patterns and text.
39pub static ONIG_ENCODING_UTF8: Utf8Encoding = Utf8Encoding;
40
41impl Encoding for Utf8Encoding {
42    fn mbc_enc_len(&self, p: &[u8]) -> usize {
43        ENC_LEN_UTF8[p[0] as usize] as usize
44    }
45
46    #[cfg_attr(coverage_nightly, coverage(off))]
47    fn name(&self) -> &str {
48        "UTF-8"
49    }
50
51    fn max_enc_len(&self) -> usize {
52        4
53    }
54
55    fn min_enc_len(&self) -> usize {
56        1
57    }
58
59    fn is_mbc_newline(&self, p: &[u8], end: usize) -> bool {
60        onigenc_is_mbc_newline_0x0a(p, end)
61    }
62
63    fn mbc_to_code(&self, p: &[u8], end: usize) -> OnigCodePoint {
64        let mut len = ENC_LEN_UTF8[p[0] as usize] as usize;
65        let available = end.min(p.len());
66        if len > available {
67            len = available;
68        }
69
70        let c = p[0] as u32;
71        if len > 1 {
72            let remaining = len - 1;
73            let mut n = c & ((1u32 << (6 - remaining)) - 1);
74            for &byte in p[1..len].iter() {
75                n = (n << 6) | ((byte as u32) & 0x3f);
76            }
77            n
78        } else {
79            c
80        }
81    }
82
83    fn code_to_mbclen(&self, code: OnigCodePoint) -> i32 {
84        if (code & 0xffffff80) == 0 {
85            1
86        } else if (code & 0xfffff800) == 0 {
87            2
88        } else if (code & 0xffff0000) == 0 {
89            3
90        } else if (code & 0xffe00000) == 0 {
91            4
92        } else {
93            ONIGERR_INVALID_CODE_POINT_VALUE
94        }
95    }
96
97    fn code_to_mbc(&self, code: OnigCodePoint, buf: &mut [u8]) -> i32 {
98        if (code & 0xffffff80) == 0 {
99            buf[0] = code as u8;
100            1
101        } else {
102            let mut i = 0;
103            if (code & 0xfffff800) == 0 {
104                buf[i] = ((code >> 6) & 0x1f) as u8 | 0xc0;
105                i += 1;
106            } else if (code & 0xffff0000) == 0 {
107                buf[i] = ((code >> 12) & 0x0f) as u8 | 0xe0;
108                i += 1;
109                buf[i] = ((code >> 6) & 0x3f) as u8 | 0x80;
110                i += 1;
111            } else if (code & 0xffe00000) == 0 {
112                buf[i] = ((code >> 18) & 0x07) as u8 | 0xf0;
113                i += 1;
114                buf[i] = ((code >> 12) & 0x3f) as u8 | 0x80;
115                i += 1;
116                buf[i] = ((code >> 6) & 0x3f) as u8 | 0x80;
117                i += 1;
118            } else {
119                return ONIGERR_TOO_BIG_WIDE_CHAR_VALUE;
120            }
121            buf[i] = (code & 0x3f) as u8 | 0x80;
122            i += 1;
123            i as i32
124        }
125    }
126
127    fn mbc_case_fold(
128        &self,
129        flag: OnigCaseFoldType,
130        pp: &mut usize,
131        end: usize,
132        source: &[u8],
133        fold_buf: &mut [u8],
134    ) -> i32 {
135        if source[*pp] < 128 {
136            // ASCII range: direct lookup
137            fold_buf[0] = onigenc_ascii_code_to_lower_case(source[*pp]);
138            *pp += 1;
139            1
140        } else {
141            // Non-ASCII: delegate to Unicode case fold
142            crate::unicode::onigenc_unicode_mbc_case_fold(self, flag, pp, end, source, fold_buf)
143        }
144    }
145
146    fn apply_all_case_fold(
147        &self,
148        flag: OnigCaseFoldType,
149        f: &mut dyn FnMut(OnigCodePoint, &[OnigCodePoint]) -> i32,
150    ) -> i32 {
151        crate::unicode::onigenc_unicode_apply_all_case_fold(flag, f)
152    }
153
154    fn get_case_fold_codes_by_str(
155        &self,
156        flag: OnigCaseFoldType,
157        p: &[u8],
158        end: usize,
159        items: &mut [OnigCaseFoldCodeItem],
160    ) -> i32 {
161        crate::unicode::onigenc_unicode_get_case_fold_codes_by_str(self, flag, p, end, items)
162    }
163
164    fn property_name_to_ctype(&self, p: &[u8]) -> i32 {
165        crate::unicode::onigenc_unicode_property_name_to_ctype(p)
166    }
167
168    fn is_code_ctype(&self, code: OnigCodePoint, ctype: u32) -> bool {
169        crate::unicode::onigenc_unicode_is_code_ctype(code, ctype)
170    }
171
172    fn get_ctype_code_range(
173        &self,
174        ctype: u32,
175        sb_out: &mut OnigCodePoint,
176    ) -> Option<&'static [OnigCodePoint]> {
177        *sb_out = 0x80;
178        crate::unicode::onigenc_unicode_ctype_code_range(ctype)
179    }
180
181    fn left_adjust_char_head(&self, start: usize, s: usize, data: &[u8]) -> usize {
182        if s <= start {
183            return s;
184        }
185        let mut p = s;
186        while !utf8_islead(data[p]) && p > start {
187            p -= 1;
188        }
189        p
190    }
191
192    fn is_allowed_reverse_match(&self, _p: &[u8]) -> bool {
193        true
194    }
195
196    fn is_valid_mbc_string(&self, s: &[u8]) -> bool {
197        let mut p = 0;
198        while p < s.len() {
199            if s[p] > 0xf4 || (s[p] > 0x7f && s[p] < 0xc2) {
200                return false;
201            }
202            let len = ENC_LEN_UTF8[s[p] as usize] as usize;
203            p += 1;
204            if len > 1 {
205                for _ in 1..len {
206                    if p >= s.len() {
207                        return false;
208                    }
209                    if !utf8_istail(s[p]) {
210                        return false;
211                    }
212                    p += 1;
213                }
214            }
215        }
216        true
217    }
218
219    fn flag(&self) -> u32 {
220        ENC_FLAG_ASCII_COMPATIBLE | ENC_FLAG_UNICODE | ENC_FLAG_SKIP_OFFSET_1_OR_0
221    }
222
223    #[cfg_attr(coverage_nightly, coverage(off))]
224    fn sb_range(&self) -> OnigCodePoint {
225        0
226    }
227
228    #[cfg_attr(coverage_nightly, coverage(off))]
229    fn index(&self) -> i32 {
230        0
231    }
232}
233
234#[cfg(test)]
235mod tests {
236    use super::*;
237
238    #[test]
239    fn mbc_to_code_respects_the_logical_end() {
240        let euro = [0xE2, 0x82, 0xAC];
241        assert_eq!(ONIG_ENCODING_UTF8.mbc_to_code(&euro, 2), 0x82);
242        assert_eq!(ONIG_ENCODING_UTF8.mbc_to_code(&euro, euro.len()), 0x20AC);
243    }
244}
ferroni/encodings/utf8.rs

ferroni/encodings/
utf8.rs
