luau_vm/libs/
utf8.rs

1use crate::VmResult;
2use crate::native::{NativeCallContext, NativeCallResult, NativeFunction};
3use crate::thread::{LuaStringBuilder, LuaStringBuilderStorage, Thread};
4
5const MAX_UNICODE: u32 = 0x10ffff;
6const UTF8_BUFF_SIZE: usize = 8;
7const UTF8_PATTERN: &[u8] = b"[\0-\x7F\xC2-\xF4][\x80-\xBF]*";
8
9static UTF8_FUNCS: [NativeFunction; 5] = [
10    NativeFunction {
11        name: "offset",
12        function: byte_offset,
13    },
14    NativeFunction {
15        name: "codepoint",
16        function: codepoint,
17    },
18    NativeFunction {
19        name: "char",
20        function: utfchar,
21    },
22    NativeFunction {
23        name: "len",
24        function: utf_len,
25    },
26    NativeFunction {
27        name: "codes",
28        function: iter_codes,
29    },
30];
31
32/// `iscont`
33fn is_cont(byte: u8) -> bool {
34    (byte & 0xc0) == 0x80
35}
36
37/// `u_posrelat`
38fn u_posrelat(pos: i32, len: usize) -> i32 {
39    if pos >= 0 {
40        pos
41    } else if (0usize).wrapping_sub(pos as usize) > len {
42        0
43    } else {
44        len as i32 + pos + 1
45    }
46}
47
48/// `utf8_decode`
49fn utf8_decode(bytes: &[u8], start: usize) -> Option<(usize, i32)> {
50    const LIMITS: [u32; 4] = [0xff, 0x7f, 0x7ff, 0xffff];
51
52    let mut c = *bytes.get(start)? as u32;
53    let mut result = 0u32;
54    if c < 0x80 {
55        return Some((start + 1, c as i32));
56    }
57
58    let mut count = 0usize;
59    while (c & 0x40) != 0 {
60        count += 1;
61        let cc = *bytes.get(start + count)? as u32;
62        if (cc & 0xc0) != 0x80 {
63            return None;
64        }
65        result = (result << 6) | (cc & 0x3f);
66        c <<= 1;
67    }
68
69    result |= (c & 0x7f) << (count * 5);
70    if count > 3 || result > MAX_UNICODE || result <= LIMITS[count] {
71        return None;
72    }
73    if (0xd800..=0xdfff).contains(&result) {
74        return None;
75    }
76
77    Some((start + count + 1, result as i32))
78}
79
80/// `utflen`
81fn utf_len(ctx: NativeCallContext) -> NativeCallResult {
82    let thread = ctx.raw_thread();
83    unsafe {
84        let bytes = thread.check_string(1)?;
85        let mut pos_i = u_posrelat(thread.opt_integer(2, 1)?, bytes.len());
86        let mut pos_j = u_posrelat(thread.opt_integer(3, -1)?, bytes.len());
87
88        if !(1 <= pos_i && {
89            pos_i -= 1;
90            pos_i as usize <= bytes.len()
91        }) {
92            return thread
93                .lua_arg_error(2, "initial position out of string")
94                .map_err(Into::into);
95        }
96
97        pos_j -= 1;
98        if !bytes.is_empty() && pos_j as usize >= bytes.len() {
99            return thread
100                .lua_arg_error(3, "final position out of string")
101                .map_err(Into::into);
102        }
103
104        let mut count = 0;
105        while pos_i <= pos_j {
106            let Some((next, _)) = utf8_decode(bytes, pos_i as usize) else {
107                thread.push_nil()?;
108                thread.push_integer(pos_i + 1)?;
109                return Ok(2);
110            };
111            pos_i = next as i32;
112            count += 1;
113        }
114
115        thread.push_integer(count)?;
116        Ok(1)
117    }
118}
119
120/// `codepoint`
121fn codepoint(ctx: NativeCallContext) -> NativeCallResult {
122    let thread = ctx.raw_thread();
123    unsafe {
124        let bytes = thread.check_string(1)?;
125        let pos_i = u_posrelat(thread.opt_integer(2, 1)?, bytes.len());
126        let pos_e = u_posrelat(thread.opt_integer(3, pos_i)?, bytes.len());
127
128        if pos_i < 1 {
129            return thread.lua_arg_error(2, "out of range").map_err(Into::into);
130        }
131        if pos_e as usize > bytes.len() {
132            return thread.lua_arg_error(3, "out of range").map_err(Into::into);
133        }
134        if pos_i > pos_e {
135            return Ok(0);
136        }
137        thread.lua_check_stack(pos_e - pos_i + 1, Some("string slice too long"))?;
138
139        let mut count = 0;
140        let mut pos = (pos_i - 1) as usize;
141        let end = pos_e as usize;
142        while pos < end {
143            let Some((next, code)) = utf8_decode(bytes, pos) else {
144                return crate::error!(thread, "invalid UTF-8 code").map_err(Into::into);
145            };
146            thread.push_integer(code)?;
147            count += 1;
148            pos = next;
149        }
150        Ok(count)
151    }
152}
153
154/// `luaO_utf8esc`
155fn lua_o_utf8esc(buffer: &mut [u8; UTF8_BUFF_SIZE], mut x: u32) -> usize {
156    let mut n = 1usize;
157    debug_assert!(x <= MAX_UNICODE);
158    if x < 0x80 {
159        buffer[UTF8_BUFF_SIZE - 1] = x as u8;
160    } else {
161        let mut mfb = 0x3f;
162        while x > mfb {
163            buffer[UTF8_BUFF_SIZE - n] = (0x80 | (x & 0x3f)) as u8;
164            n += 1;
165            x >>= 6;
166            mfb >>= 1;
167        }
168        buffer[UTF8_BUFF_SIZE - n] = (((!mfb) << 1) | x) as u8;
169    }
170    n
171}
172
173/// `buffutfchar`
174fn buff_utfchar<'a>(
175    thread: &Thread,
176    argument: i32,
177    buffer: &'a mut [u8; UTF8_BUFF_SIZE],
178) -> VmResult<&'a [u8]> {
179    let code = unsafe { thread.check_integer(argument)? };
180    if !(0..=MAX_UNICODE as i32).contains(&code) {
181        return unsafe { thread.lua_arg_error(argument, "value out of range") }.map_err(Into::into);
182    }
183    let len = lua_o_utf8esc(buffer, code as u32);
184    Ok(&buffer[UTF8_BUFF_SIZE - len..])
185}
186
187/// `utfchar`
188fn utfchar(ctx: NativeCallContext) -> NativeCallResult {
189    let thread = ctx.raw_thread();
190    unsafe {
191        let count = thread.get_top();
192        let mut buffer = [0u8; UTF8_BUFF_SIZE];
193        if count == 1 {
194            let bytes = buff_utfchar(thread, 1, &mut buffer)?;
195            thread.push_string(bytes)?;
196        } else {
197            let mut out_storage = LuaStringBuilderStorage::uninit();
198            let mut out = LuaStringBuilder::new(thread, &mut out_storage);
199            for index in 1..=count {
200                let bytes = buff_utfchar(thread, index, &mut buffer)?;
201                out.push_bytes(bytes)?;
202            }
203            out.finish()?;
204        }
205    }
206    Ok(1)
207}
208
209/// `byteoffset`
210fn byte_offset(ctx: NativeCallContext) -> NativeCallResult {
211    let thread = ctx.raw_thread();
212    unsafe {
213        let bytes = thread.check_string(1)?;
214        let n = thread.check_integer(2)?;
215        let mut pos_i = if n >= 0 { 1 } else { bytes.len() as i32 + 1 };
216        pos_i = u_posrelat(thread.opt_integer(3, pos_i)?, bytes.len());
217
218        if !(1 <= pos_i && {
219            pos_i -= 1;
220            pos_i as usize <= bytes.len()
221        }) {
222            return thread
223                .lua_arg_error(3, "position out of range")
224                .map_err(Into::into);
225        }
226
227        if n == 0 {
228            while pos_i > 0 && is_cont(bytes[pos_i as usize]) {
229                pos_i -= 1;
230            }
231        } else {
232            if pos_i < bytes.len() as i32 && is_cont(bytes[pos_i as usize]) {
233                return crate::error!(thread, "initial position is a continuation byte")
234                    .map_err(Into::into);
235            }
236
237            let mut remaining = n;
238            if remaining < 0 {
239                while remaining < 0 && pos_i > 0 {
240                    loop {
241                        pos_i -= 1;
242                        if pos_i == 0 || !is_cont(bytes[pos_i as usize]) {
243                            break;
244                        }
245                    }
246                    remaining += 1;
247                }
248            } else {
249                remaining -= 1;
250                while remaining > 0 && pos_i < bytes.len() as i32 {
251                    loop {
252                        pos_i += 1;
253                        if pos_i as usize >= bytes.len() || !is_cont(bytes[pos_i as usize]) {
254                            break;
255                        }
256                    }
257                    remaining -= 1;
258                }
259            }
260
261            if remaining != 0 {
262                thread.push_nil()?;
263                return Ok(1);
264            }
265        }
266
267        thread.push_integer(pos_i + 1)?;
268        Ok(1)
269    }
270}
271
272/// `iter_aux`
273fn iter_aux(ctx: NativeCallContext) -> NativeCallResult {
274    let thread = ctx.raw_thread();
275    unsafe {
276        let bytes = thread.check_string(1)?;
277        let mut n = thread.to_integer(2).unwrap_or(0) - 1;
278        if n < 0 {
279            n = 0;
280        } else if (n as usize) < bytes.len() {
281            n += 1;
282            while (n as usize) < bytes.len() && is_cont(bytes[n as usize]) {
283                n += 1;
284            }
285        }
286
287        if (n as usize) >= bytes.len() {
288            return Ok(0);
289        }
290
291        let Some((next, code)) = utf8_decode(bytes, n as usize) else {
292            return crate::error!(thread, "invalid UTF-8 code").map_err(Into::into);
293        };
294        if next < bytes.len() && is_cont(bytes[next]) {
295            return crate::error!(thread, "invalid UTF-8 code").map_err(Into::into);
296        }
297
298        thread.push_integer(n + 1)?;
299        thread.push_integer(code)?;
300        Ok(2)
301    }
302}
303
304/// `iter_codes`
305fn iter_codes(ctx: NativeCallContext) -> NativeCallResult {
306    let thread = ctx.raw_thread();
307    let _ = unsafe { thread.check_string(1)? };
308    unsafe {
309        thread.push_native_closure_k(iter_aux, None, 0, None)?;
310        thread.push_value(1)?;
311        thread.push_integer(0)?;
312    }
313    Ok(3)
314}
315
316impl Thread {
317    /// `luaopen_utf8`
318    pub unsafe fn open_utf8(&self) -> NativeCallResult {
319        unsafe {
320            self.register(Some(super::LUA_UTF8LIB_NAME), &UTF8_FUNCS[..])?;
321            self.push_string(UTF8_PATTERN)?;
322            self.raw_set_field(-2, "charpattern")?;
323        }
324        Ok(1)
325    }
326}
luau_vm/libs/utf8.rs

luau_vm/libs/
utf8.rs
