---
source: src/main.rs
expression: compiled
input_file: test-data/lua5.3-tests/utf8.lua
---
print("testing UTF-8 library");
local utf8 = require('utf8')
local fn checkerror(msg, f, ...) {
local s, err = pcall(f, ...)
assert(!s && string.find(err, msg));
}
local fn len(s) {
return #string.gsub(s, "[\x80-\xBF]", "")
}
local justone = "^" .. utf8.charpattern .. "$"
local fn checksyntax(s, t) {
local ts = {
"return '"
}
for i = 1, #t {
ts[(i + 1)] = string.format("\\u{%x}", t[(i)])
}
ts[(#t + 2)] = "'"
ts = table.concat(ts)
assert(assert(load(ts))() == s);
}
assert(utf8.offset("alo", 5) == nil);
assert(utf8.offset("alo", -4) == nil);
local fn check(s, t) {
local l = utf8.len(s)
assert(#t == l && len(s) == l);
assert(utf8.char(table.unpack(t)) == s);
assert(utf8.offset(s, 0) == 1);
checksyntax(s, t);
local t1 = {
utf8.codepoint(s, 1, -1)
}
assert(#t == #t1);
for i = 1, #t {
assert(t[(i)] == t1[(i)]);
}
for i = 1, l {
local pi = utf8.offset(s, i)
local pi1 = utf8.offset(s, 2, pi)
assert(string.find(string.sub(s, pi, pi1 - 1), justone));
assert(utf8.offset(s, -1, pi1) == pi);
assert(utf8.offset(s, i - l - 1) == pi);
assert(pi1 - pi == #utf8.char(utf8.codepoint(s, pi)));
for j = pi, pi1 - 1 {
assert(utf8.offset(s, 0, j) == pi);
}
for j = pi + 1, pi1 - 1 {
assert(!utf8.len(s, j));
}
assert(utf8.len(s, pi, pi) == 1);
assert(utf8.len(s, pi, pi1 - 1) == 1);
assert(utf8.len(s, pi) == l - i + 1);
assert(utf8.len(s, pi1) == l - i);
assert(utf8.len(s, 1, pi) == i);
}
local i = 0
for p, c with utf8.codes(s) {
i = i + 1
assert(c == t[(i)] && p == utf8.offset(s, i));
assert(utf8.codepoint(s, p) == c);
}
assert(i == #t);
i = 0
for p, c with utf8.codes(s) {
i = i + 1
assert(c == t[(i)] && p == utf8.offset(s, i));
}
assert(i == #t);
i = 0
for c with string.gmatch(s, utf8.charpattern) {
i = i + 1
assert(c == utf8.char(t[(i)]));
}
assert(i == #t);
for i = 1, l {
assert(utf8.offset(s, i) == utf8.offset(s, i - l - 1, #s + 1));
}
}
{
local fn check(s, p) {
local a, b = utf8.len(s)
assert(!a && b == p);
}
check("abc\xE3def", 4);
check("汉字\x80", #("汉字") + 1);
check("\xF4\x9F\xBF", 1);
check("\xF4\x9F\xBF\xBF", 1);
}
checkerror("invalid UTF%-8 code", fn () {
local s = "ab\xff"
for c with utf8.codes(s) {
assert(c);
}
});
checkerror("position out of range", utf8.offset, "abc", 1, 5);
checkerror("position out of range", utf8.offset, "abc", 1, -4);
checkerror("position out of range", utf8.offset, "", 1, 2);
checkerror("position out of range", utf8.offset, "", 1, -1);
checkerror("continuation byte", utf8.offset, "𦧺", 1, 2);
checkerror("continuation byte", utf8.offset, "𦧺", 1, 2);
checkerror("continuation byte", utf8.offset, "\x80", 1);
local s = "hello World"
local t = {
string.byte(s, 1, -1)
}
for i = 1, utf8.len(s) {
assert(t[(i)] == string.byte(s, i));
}
check(s, t);
check("汉字/漢字", {
27721,
23383,
47,
28450,
23383
});
{
local s = "áéí\128"
local t = {
utf8.codepoint(s, 1, #s - 1)
}
assert(#t == 3 && t[(1)] == 225 && t[(2)] == 233 && t[(3)] == 237);
checkerror("invalid UTF%-8 code", utf8.codepoint, s, 1, #s);
checkerror("out of range", utf8.codepoint, s, #s + 1);
t = {
utf8.codepoint(s, 4, 3)
}
assert(#t == 0);
checkerror("out of range", utf8.codepoint, s, -(#s + 1), 1);
checkerror("out of range", utf8.codepoint, s, 1, #s + 1);
}
assert(utf8.char() == "");
assert(utf8.char(97, 98, 99) == "abc");
assert(utf8.codepoint(utf8.char(0x10FFFF)) == 0x10FFFF);
checkerror("value out of range", utf8.char, 0x10FFFF + 1);
local fn invalid(s) {
checkerror("invalid UTF%-8 code", utf8.codepoint, s);
assert(!utf8.len(s));
}
invalid("\xF4\x9F\xBF\xBF");
invalid("\xC0\x80");
invalid("\xC1\xBF");
invalid("\xE0\x9F\xBF");
invalid("\xF0\x8F\xBF\xBF");
invalid("\x80");
invalid("\xBF");
invalid("\xFE");
invalid("\xFF");
check("", {});
s = "\0 \x7F\z
\xC2\x80 \xDF\xBF\z
\xE0\xA0\x80 \xEF\xBF\xBF\z
\xF0\x90\x80\x80 \xF4\x8F\xBF\xBF"
s = string.gsub(s, " ", "")
check(s, {
0,
0x7F,
0x80,
0x7FF,
0x800,
0xFFFF,
0x10000,
0x10FFFF
});
x = "日本語a-4\0éó"
check(x, {
26085,
26412,
35486,
97,
45,
52,
0,
233,
243
});
check("𣲷𠜎𠱓𡁻𠵼ab𠺢", {
0x23CB7,
0x2070E,
0x20C53,
0x2107B,
0x20D7C,
0x61,
0x62,
0x20EA2
});
check("𨳊𩶘𦧺𨳒𥄫𤓓\xF4\x8F\xBF\xBF", {
0x28CCA,
0x29D98,
0x269FA,
0x28CD2,
0x2512B,
0x244D3,
0x10ffff
});
local i = 0
for p, c with string.gmatch(x, "()(" .. utf8.charpattern .. ")") {
i = i + 1
assert(utf8.offset(x, i) == p);
assert(utf8.len(x, p) == utf8.len(x) - i + 1);
assert(utf8.len(c) == 1);
for j = 1, #c - 1 {
assert(utf8.offset(x, 0, p + j - 1) == p);
}
}
print('ok');