import fileinput
import re
import os
import sys
preamble = '''// Copyright 2012-2018 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.
// NOTE: The following code was generated by "scripts/unicode.py", do not edit directly
#![allow(missing_docs, non_upper_case_globals, non_snake_case)]
'''
UNICODE_VERSION = (17, 0, 0)
UNICODE_VERSION_NUMBER = "%s.%s.%s" % UNICODE_VERSION
def escape_char(c):
return "'\\u{%x}'" % c
def fetch(f):
if not os.path.exists(os.path.basename(f)):
if "emoji" in f:
os.system("curl -O https://www.unicode.org/Public/emoji/%s.%s/%s"
% (UNICODE_VERSION[0], UNICODE_VERSION[1], f))
else:
os.system("curl -O http://www.unicode.org/Public/%s/ucd/%s"
% (UNICODE_VERSION_NUMBER, f))
if not os.path.exists(os.path.basename(f)):
sys.stderr.write("cannot load %s" % f)
exit(1)
def load_names(f, interestingprops):
fetch(f)
normal_names = {}
special_names = []
re1 = re.compile(r"^([0-9A-F]+);([^;]+);.*$")
for line in fileinput.input(os.path.basename(f)):
d_ch = 0
d_name = ''
m = re1.match(line)
if not m:
continue
d_ch = m.group(1)
d_name = m.group(2).strip()
if d_name.startswith('<'):
special_names.append((d_ch, d_name))
else:
normal_names[d_ch] = d_name
return (normal_names, special_names)
SPACE_SYMBOL = ' '
CODEPOINT_SYMBOL = '@'
SPECIAL_SYMBOLS = ['-', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9']
def tokenize(str, codepoint):
name_sep = str
name_sep = name_sep.replace(codepoint, CODEPOINT_SYMBOL)
for symbol in SPECIAL_SYMBOLS:
name_sep = name_sep.replace(
symbol, SPACE_SYMBOL + symbol + SPACE_SYMBOL)
return name_sep
def make_wordset(names):
word_set = {}
word_set[SPACE_SYMBOL] = SPACE_SYMBOL
for codepoint, name in names.items():
name_sep = tokenize(name, codepoint)
word_list = name_sep.split(SPACE_SYMBOL)
for word in word_list:
word_set[word] = word
return word_set
class WordIndex:
def __init__(self, normal_names):
word_set = make_wordset(normal_names)
word_list = []
for word, _ in word_set.items():
word_list.append(word)
word_list.sort()
word_map = {}
for idx, word in enumerate(word_list):
word_map[word] = idx
special_map = {}
special_list = []
for symbol in SPECIAL_SYMBOLS:
if symbol not in word_map:
continue
symbol_idx = word_map[symbol]
special_map[symbol_idx] = True
special_list.append(symbol_idx)
codepoint_symbol_idx = word_map[CODEPOINT_SYMBOL]
special_map[codepoint_symbol_idx] = True
special_list.append(codepoint_symbol_idx)
space_symbol_idx = word_map[SPACE_SYMBOL]
special_map[space_symbol_idx] = True
special_list.append(space_symbol_idx)
special_list.sort()
self.word_list = word_list
self.word_map = word_map
self.special_map = special_map
self.special_list = special_list
def encode(self, name, codepoint):
name_sep = tokenize(name, codepoint)
word_list = name_sep.split(SPACE_SYMBOL)
last_is_special = True
name_build = ''
encoded_sequence = []
for word in word_list:
if word == '':
continue
word_idx = self.word_map[word]
word_is_special = word_idx in self.special_map
auto_add_space = not last_is_special and not word_is_special
if word == CODEPOINT_SYMBOL:
word = codepoint
if name[len(name_build):].startswith(SPACE_SYMBOL):
if not auto_add_space:
encoded_sequence.append(self.word_map[SPACE_SYMBOL])
name_build += SPACE_SYMBOL
if not name[len(name_build):].startswith(word):
raise Exception(
"Divergence! [%s] - [%s] vs [%s]" % (name, name[len(name_build):], word))
name_build += word
encoded_sequence.append(word_idx)
last_is_special = word_is_special
if name_build != name:
raise Exception("Different! [%s] vs [%s]" % (name, name_build))
return encoded_sequence
def create_intervals(list):
list.sort()
in_group = False
group_start = 0
result = []
for (idx, val) in enumerate(list):
if not in_group:
group_start = val
in_group = True
if idx + 1 >= len(list) or list[idx + 1] != val + 1:
result.append((group_start, val))
in_group = False
return result
def create_normal_groups(normal_names):
normal_intervals = create_intervals(
[int(key, 16) for key in normal_names.keys()])
encoded_groups = []
for first, last in normal_intervals:
group_buffer = []
pos_buffer = []
for ch in range(first, last + 1):
ch_buffer_pos = len(group_buffer)
ch_str = "%04X" % ch
ch_namebuf = word_index.encode(normal_names[ch_str], ch_str)
group_buffer = group_buffer + ch_namebuf
pos_buffer.append(ch_buffer_pos)
final_buffer_pos = len(group_buffer)
pos_buffer.append(final_buffer_pos)
encoded_groups.append((first, last, group_buffer, pos_buffer))
return encoded_groups
def create_special_groups(special_names):
item_idx = 0
item_count = len(special_names)
result = []
re1 = re.compile(r"^<(.*), First>$")
re2 = re.compile(r"^<(.*), Last>$")
re3 = re.compile(r"^<(.*)>$")
while item_idx < item_count:
item_text = special_names[item_idx][1]
m1 = re1.match(item_text)
if m1 and item_idx + 1 < item_count:
label = m1.group(1)
m2 = re2.match(special_names[item_idx + 1][1])
if not m2 or m2.group(1) != label:
raise Exception("Pair mismatch! [%s] vs [%s]" % (
special_names[item_idx], special_names[item_idx + 1]))
result.append((int(special_names[item_idx][0], 16), int(
special_names[item_idx + 1][0], 16), label))
item_idx += 2
continue
m3 = re3.match(item_text)
if m3:
label = m3.group(1)
last_repeat_item_idx = item_idx
try_item_idx = item_idx + 1
while try_item_idx < item_count and special_names[try_item_idx][1] == item_text:
last_repeat_item_idx = try_item_idx
try_item_idx += 1
result.append((int(special_names[item_idx][0], 16), int(
special_names[last_repeat_item_idx][0], 16), label))
item_idx = last_repeat_item_idx + 1
continue
raise Exception("Unexpected item: %s" % item_text)
return result
def write_enumeration_char_names(rf, encoded_groups):
rf.write("""
pub const ENUMERATION_CHAR_NAMES: &'static [(u32, u32, &'static [u16], &'static [u32])] = &[
""")
for (first, last, group_buffer, pos_buffer) in encoded_groups:
rf.write("\t(%d, %d, &%s, &%s),\n" %
(first, last, group_buffer, pos_buffer))
rf.write("""];
""")
def write_special_groups(rf, special_groups):
rf.write("""
#[allow(non_camel_case_types)]
#[derive(Copy, Clone, PartialEq, Eq)]
pub enum SpecialGroup {
""")
for ((_, _, groupname)) in special_groups:
group_variant = groupname.replace(' ', '')
rf.write("\t%s,\n" % group_variant)
rf.write("""}
""")
rf.write("""
pub const SPECIAL_GROUPS: &'static [(u32, u32, SpecialGroup)] = &[
""")
for (idx, (first, last, groupname)) in enumerate(special_groups):
if idx % 2 == 0:
rf.write("\t")
group_variant = groupname.replace(' ', '')
group_variant_full = "SpecialGroup::" + group_variant
rf.write("(%d, %d, %s), " % (first, last, group_variant_full))
if (idx + 1) % 2 == 0:
rf.write('\n')
rf.write("""];
pub fn find_in_special_groups(ch: u32) -> Option<SpecialGroup> {
let record_idx = SPECIAL_GROUPS
.binary_search_by(|record| {
use core::cmp::Ordering;
if record.1 < ch {
Ordering::Less
} else if record.0 > ch {
Ordering::Greater
} else {
Ordering::Equal
}
})
.ok()?;
let group = SPECIAL_GROUPS[record_idx].2;
Some(group)
}
""")
def write_word_table(rf, word_table):
rf.write("""
pub const ENUMERATION_WORD_TABLE: &'static [&'static str] = &[
""")
for (idx, word) in enumerate(word_table):
if idx % 8 == 0:
rf.write("\t")
rf.write("\"%s\", " % word)
if (idx + 1) % 8 == 0:
rf.write('\n')
rf.write("""];
pub fn find_in_enumerate_names(ch: u32) -> Option<&'static [u16]> {
let record_idx = ENUMERATION_CHAR_NAMES
.binary_search_by(|record| {
use core::cmp::Ordering;
if record.1 < ch {
Ordering::Less
} else if record.0 > ch {
Ordering::Greater
} else {
Ordering::Equal
}
})
.ok()?;
let offset = (ch - ENUMERATION_CHAR_NAMES[record_idx].0) as usize;
let index_slice = ENUMERATION_CHAR_NAMES[record_idx].2;
let offset_slice = ENUMERATION_CHAR_NAMES[record_idx].3;
let range = (offset_slice[offset] as usize)..(offset_slice[offset + 1] as usize);
Some(&index_slice[range])
}
""")
def write_special_symbols(rf, word_index):
rf.write("""
pub const WORD_TABLE_INDEX_SPACE: u16 = %d;
""" % word_index.word_map[SPACE_SYMBOL])
rf.write("""
pub const WORD_TABLE_INDEX_CODEPOINT: u16 = %d;
""" % word_index.word_map[CODEPOINT_SYMBOL])
special_intervals = create_intervals(word_index.special_list)
rf.write("""
pub fn is_special_word_index(v: u16) -> bool {
match v {
""")
for (first, last) in special_intervals:
rf.write("\t\t%d..=%d => true,\n" % (first, last))
rf.write("""\t\t_ => false,
}
}
""")
if __name__ == "__main__":
r = "tables.rs"
if os.path.exists(r):
os.remove(r)
with open(r, "w") as rf:
rf.write(preamble)
rf.write("""
/// The version of [Unicode](http://www.unicode.org/)
/// that this version of unicode-charname is based on.
pub const UNICODE_VERSION: (u64, u64, u64) = (%s, %s, %s);
""" % UNICODE_VERSION)
normal_names, special_names = load_names("UnicodeData.txt", [])
word_index = WordIndex(normal_names)
normal_encoded_groups = create_normal_groups(normal_names)
special_groups = create_special_groups(special_names)
write_enumeration_char_names(rf, normal_encoded_groups)
write_special_groups(rf, special_groups)
write_word_table(rf, word_index.word_list)
write_special_symbols(rf, word_index)