unicode-normalization 0.1.8

#!/usr/bin/env python
#
# Copyright 2011-2018 The Rust Project Developers. See the COPYRIGHT
# file at the top-level directory of this distribution and at
# http://rust-lang.org/COPYRIGHT.
#
# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
# http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
# <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
# option. This file may not be copied, modified, or distributed
# except according to those terms.

# This script uses the following Unicode tables:
# - DerivedNormalizationProps.txt
# - NormalizationTest.txt
# - UnicodeData.txt
#
# Since this should not require frequent updates, we just store this
# out-of-line and check the unicode.rs file into git.
import collections
import requests

UNICODE_VERSION = "9.0.0"
UCD_URL = "https://www.unicode.org/Public/%s/ucd/" % UNICODE_VERSION

PREAMBLE = """// Copyright 2012-2018 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

// NOTE: The following code was generated by "scripts/unicode.py", do not edit directly

#![allow(missing_docs)]
"""

NormalizationTest = collections.namedtuple(
    "NormalizationTest",
    ["source", "nfc", "nfd", "nfkc", "nfkd"],
)

# Mapping taken from Table 12 from:
# http://www.unicode.org/reports/tr44/#General_Category_Values
expanded_categories = {
    'Lu': ['LC', 'L'], 'Ll': ['LC', 'L'], 'Lt': ['LC', 'L'],
    'Lm': ['L'], 'Lo': ['L'],
    'Mn': ['M'], 'Mc': ['M'], 'Me': ['M'],
    'Nd': ['N'], 'Nl': ['N'], 'No': ['No'],
    'Pc': ['P'], 'Pd': ['P'], 'Ps': ['P'], 'Pe': ['P'],
    'Pi': ['P'], 'Pf': ['P'], 'Po': ['P'],
    'Sm': ['S'], 'Sc': ['S'], 'Sk': ['S'], 'So': ['S'],
    'Zs': ['Z'], 'Zl': ['Z'], 'Zp': ['Z'],
    'Cc': ['C'], 'Cf': ['C'], 'Cs': ['C'], 'Co': ['C'], 'Cn': ['C'],
}

class UnicodeData(object):
    def __init__(self):
        self._load_unicode_data()
        self.norm_props = self._load_norm_props()
        self.norm_tests = self._load_norm_tests()

        self.canon_comp = self._compute_canonical_comp()
        self.canon_fully_decomp, self.compat_fully_decomp = self._compute_fully_decomposed()

        def stats(name, table):
            count = sum(len(v) for v in table.values())
            print "%s: %d chars => %d decomposed chars" % (name, len(table), count)

        print "Decomposition table stats:"
        stats("Canonical decomp", self.canon_decomp)
        stats("Compatible decomp", self.compat_decomp)
        stats("Canonical fully decomp", self.canon_fully_decomp)
        stats("Compatible fully decomp", self.compat_fully_decomp)

        self.ss_leading, self.ss_trailing = self._compute_stream_safe_tables()

    def _fetch(self, filename):
        resp = requests.get(UCD_URL + filename)
        return resp.text

    def _load_unicode_data(self):
        self.combining_classes = {}
        self.compat_decomp = {}
        self.canon_decomp = {}
        self.general_category_mark = []

        for line in self._fetch("UnicodeData.txt").splitlines():
            # See ftp://ftp.unicode.org/Public/3.0-Update/UnicodeData-3.0.0.html
            pieces = line.split(';')
            assert len(pieces) == 15
            char, category, cc, decomp = pieces[0], pieces[2], pieces[3], pieces[5]
            char_int = int(char, 16)

            if cc != '0':
                self.combining_classes[char_int] = cc

            if decomp.startswith('<'):
                self.compat_decomp[char_int] = [int(c, 16) for c in decomp.split()[1:]]
            elif decomp != '':
                self.canon_decomp[char_int] = [int(c, 16) for c in decomp.split()]

            if category == 'M' or 'M' in expanded_categories.get(category, []):
                self.general_category_mark.append(char_int)

    def _load_norm_props(self):
        props = collections.defaultdict(list)

        for line in self._fetch("DerivedNormalizationProps.txt").splitlines():
            (prop_data, _, _) = line.partition("#")
            prop_pieces = prop_data.split(";")

            if len(prop_pieces) < 2:
                continue

            assert len(prop_pieces) <= 3
            (low, _, high) = prop_pieces[0].strip().partition("..")

            prop = prop_pieces[1].strip()

            data = None
            if len(prop_pieces) == 3:
                data = prop_pieces[2].strip()

            props[prop].append((low, high, data))

        return props

    def _load_norm_tests(self):
        tests = []
        for line in self._fetch("NormalizationTest.txt").splitlines():
            (test_data, _, _) = line.partition("#")
            test_pieces = test_data.split(";")

            if len(test_pieces) < 5:
                continue

            source, nfc, nfd, nfkc, nfkd = [[c.strip() for c in p.split()] for p in test_pieces[:5]]
            tests.append(NormalizationTest(source, nfc, nfd, nfkc, nfkd))

        return tests

    def _compute_canonical_comp(self):
        canon_comp = {}
        comp_exclusions = [
            (int(low, 16), int(high or low, 16))
            for low, high, _ in self.norm_props["Full_Composition_Exclusion"]
        ]
        for char_int, decomp in self.canon_decomp.items():
            if any(lo <= char_int <= hi for lo, hi in comp_exclusions):
                continue

            assert len(decomp) == 2
            assert (decomp[0], decomp[1]) not in canon_comp
            canon_comp[(decomp[0], decomp[1])] = char_int

        return canon_comp

    def _compute_fully_decomposed(self):
        """
        Even though the decomposition algorithm is recursive, it is possible
        to precompute the recursion at table generation time with modest
        increase to the table size.  Then, for these precomputed tables, we
        note that 1) compatible decomposition is a subset of canonical
        decomposition and 2) they mostly agree on their intersection.
        Therefore, we don't store entries in the compatible table for
        characters that decompose the same way under canonical decomposition.

            Decomposition table stats:
            Canonical decomp: 2060 chars => 3085 decomposed chars
            Compatible decomp: 3662 chars => 5440 decomposed chars
            Canonical fully decomp: 2060 chars => 3404 decomposed chars
            Compatible fully decomp: 3678 chars => 5599 decomposed chars

        The upshot is that decomposition code is very simple and easy to inline
        at mild code size cost.
        """
        # Constants from Unicode 9.0.0 Section 3.12 Conjoining Jamo Behavior
        # http://www.unicode.org/versions/Unicode9.0.0/ch03.pdf#M9.32468.Heading.310.Combining.Jamo.Behavior
        S_BASE, L_COUNT, V_COUNT, T_COUNT = 0xAC00, 19, 21, 28
        S_COUNT = L_COUNT * V_COUNT * T_COUNT

        def _decompose(char_int, compatible):
            # 7-bit ASCII never decomposes
            if char_int <= 0x7f:
                yield char_int
                return

            # Assert that we're handling Hangul separately.
            assert not (S_BASE <= char_int < S_BASE + S_COUNT)

            decomp = self.canon_decomp.get(char_int)
            if decomp is not None:
                for decomposed_ch in decomp:
                    for fully_decomposed_ch in _decompose(decomposed_ch, compatible):
                        yield fully_decomposed_ch
                return

            if compatible and char_int in self.compat_decomp:
                for decomposed_ch in self.compat_decomp[char_int]:
                    for fully_decomposed_ch in _decompose(decomposed_ch, compatible):
                        yield fully_decomposed_ch
                return

            yield char_int
            return

        end_codepoint = max(
            max(self.canon_decomp.keys()),
            max(self.compat_decomp.keys()),
        )

        canon_fully_decomp = {}
        compat_fully_decomp = {}

        for char_int in range(0, end_codepoint + 1):
            # Always skip Hangul, since it's more efficient to represent its
            # decomposition programmatically.
            if S_BASE <= char_int < S_BASE + S_COUNT:
                continue

            canon = list(_decompose(char_int, False))
            if not (len(canon) == 1 and canon[0] == char_int):
                canon_fully_decomp[char_int] = canon

            compat = list(_decompose(char_int, True))
            if not (len(compat) == 1 and compat[0] == char_int):
                compat_fully_decomp[char_int] = compat

        # Since canon_fully_decomp is a subset of compat_fully_decomp, we don't
        # need to store their overlap when they agree.  When they don't agree,
        # store the decomposition in the compatibility table since we'll check
        # that first when normalizing to NFKD.
        assert canon_fully_decomp <= compat_fully_decomp

        for ch in set(canon_fully_decomp) & set(compat_fully_decomp):
            if canon_fully_decomp[ch] == compat_fully_decomp[ch]:
                del compat_fully_decomp[ch]

        return canon_fully_decomp, compat_fully_decomp

    def _compute_stream_safe_tables(self):
        """
        To make a text stream-safe with the Stream-Safe Text Process (UAX15-D4),
        we need to be able to know the number of contiguous non-starters *after*
        applying compatibility decomposition to each character.

        We can do this incrementally by computing the number of leading and
        trailing non-starters for each character's compatibility decomposition
        with the following rules:

        1) If a character is not affected by compatibility decomposition, look
           up its canonical combining class to find out if it's a non-starter.
        2) All Hangul characters are starters, even under decomposition.
        3) Otherwise, very few decomposing characters have a nonzero count
           of leading or trailing non-starters, so store these characters
           with their associated counts in a separate table.
        """
        leading_nonstarters = {}
        trailing_nonstarters = {}

        for c in set(self.canon_fully_decomp) | set(self.compat_fully_decomp):
            decomposed = self.compat_fully_decomp.get(c) or self.canon_fully_decomp[c]

            num_leading = 0
            for d in decomposed:
                if d not in self.combining_classes:
                    break
                num_leading += 1

            num_trailing = 0
            for d in reversed(decomposed):
                if d not in self.combining_classes:
                    break
                num_trailing += 1

            if num_leading > 0:
                leading_nonstarters[c] = num_leading
            if num_trailing > 0:
                trailing_nonstarters[c] = num_trailing

        return leading_nonstarters, trailing_nonstarters

hexify = lambda c: hex(c)[2:].upper().rjust(4, '0')

def gen_combining_class(combining_classes, out):
    out.write("#[inline]\n")
    out.write("pub fn canonical_combining_class(c: char) -> u8 {\n")
    out.write("    match c {\n")

    for char, combining_class in sorted(combining_classes.items()):
        out.write("        '\u{%s}' => %s,\n" % (hexify(char), combining_class))

    out.write("        _ => 0,\n")
    out.write("    }\n")
    out.write("}\n")

def gen_composition_table(canon_comp, out):
    out.write("#[inline]\n")
    out.write("pub fn composition_table(c1: char, c2: char) -> Option<char> {\n")
    out.write("    match (c1, c2) {\n")

    for (c1, c2), c3 in sorted(canon_comp.items()):
        out.write("        ('\u{%s}', '\u{%s}') => Some('\u{%s}'),\n" % (hexify(c1), hexify(c2), hexify(c3)))

    out.write("        _ => None,\n")
    out.write("    }\n")
    out.write("}\n")

def gen_decomposition_tables(canon_decomp, compat_decomp, out):
    tables = [(canon_decomp, 'canonical'), (compat_decomp, 'compatibility')]
    for table, name in tables:
        out.write("#[inline]\n")
        out.write("pub fn %s_fully_decomposed(c: char) -> Option<&'static [char]> {\n" % name)
        # The "Some" constructor is around the match statement here, because
        # putting it into the individual arms would make the item_bodies
        # checking of rustc takes almost twice as long, and it's already pretty
        # slow because of the huge number of match arms and the fact that there
        # is a borrow inside each arm
        out.write("    Some(match c {\n")

        for char, chars in sorted(table.items()):
            d = ", ".join("'\u{%s}'" % hexify(c) for c in chars)
            out.write("        '\u{%s}' => &[%s],\n" % (hexify(char), d))

        out.write("        _ => return None,\n")
        out.write("    })\n")
        out.write("}\n")
        out.write("\n")

def gen_qc_match(prop_table, out):
    out.write("    match c {\n")

    for low, high, data in prop_table:
        assert data in ('N', 'M')
        result = "No" if data == 'N' else "Maybe"
        if high:
            out.write(r"        '\u{%s}'...'\u{%s}' => %s," % (low, high, result))
        else:
            out.write(r"        '\u{%s}' => %s," % (low, result))
        out.write("\n")

    out.write("        _ => Yes,\n")
    out.write("    }\n")

def gen_nfc_qc(prop_tables, out):
    out.write("#[inline]\n")
    out.write("pub fn qc_nfc(c: char) -> IsNormalized {\n")
    gen_qc_match(prop_tables['NFC_QC'], out)
    out.write("}\n")

def gen_nfkc_qc(prop_tables, out):
    out.write("#[inline]\n")
    out.write("pub fn qc_nfkc(c: char) -> IsNormalized {\n")
    gen_qc_match(prop_tables['NFKC_QC'], out)
    out.write("}\n")

def gen_nfd_qc(prop_tables, out):
    out.write("#[inline]\n")
    out.write("pub fn qc_nfd(c: char) -> IsNormalized {\n")
    gen_qc_match(prop_tables['NFD_QC'], out)
    out.write("}\n")

def gen_nfkd_qc(prop_tables, out):
    out.write("#[inline]\n")
    out.write("pub fn qc_nfkd(c: char) -> IsNormalized {\n")
    gen_qc_match(prop_tables['NFKD_QC'], out)
    out.write("}\n")

def gen_combining_mark(general_category_mark, out):
    out.write("#[inline]\n")
    out.write("pub fn is_combining_mark(c: char) -> bool {\n")
    out.write("    match c {\n")

    for char in general_category_mark:
        out.write("        '\u{%s}' => true,\n" % hexify(char))

    out.write("        _ => false,\n")
    out.write("    }\n")
    out.write("}\n")

def gen_stream_safe(leading, trailing, out):
    out.write("#[inline]\n")
    out.write("pub fn stream_safe_leading_nonstarters(c: char) -> usize {\n")
    out.write("    match c {\n")

    for char, num_leading in leading.items():
        out.write("        '\u{%s}' => %d,\n" % (hexify(char), num_leading))

    out.write("        _ => 0,\n")
    out.write("    }\n")
    out.write("}\n")
    out.write("\n")

    out.write("#[inline]\n")
    out.write("pub fn stream_safe_trailing_nonstarters(c: char) -> usize {\n")
    out.write("    match c {\n")

    for char, num_trailing in trailing.items():
        out.write("        '\u{%s}' => %d,\n" % (hexify(char), num_trailing))

    out.write("        _ => 0,\n")
    out.write("    }\n")
    out.write("}\n")

def gen_tests(tests, out):
    out.write("""#[derive(Debug)]
pub struct NormalizationTest {
    pub source: &'static str,
    pub nfc: &'static str,
    pub nfd: &'static str,
    pub nfkc: &'static str,
    pub nfkd: &'static str,
}

""")

    out.write("pub const NORMALIZATION_TESTS: &[NormalizationTest] = &[\n")
    str_literal = lambda s: '"%s"' % "".join("\u{%s}" % c for c in s)

    for test in tests:
        out.write("    NormalizationTest {\n")
        out.write("        source: %s,\n" % str_literal(test.source))
        out.write("        nfc: %s,\n" % str_literal(test.nfc))
        out.write("        nfd: %s,\n" % str_literal(test.nfd))
        out.write("        nfkc: %s,\n" % str_literal(test.nfkc))
        out.write("        nfkd: %s,\n" % str_literal(test.nfkd))
        out.write("    },\n")

    out.write("];\n")

if __name__ == '__main__':
    data = UnicodeData()
    with open("tables.rs", "w") as out:
        out.write(PREAMBLE)
        out.write("use quick_check::IsNormalized;\n")
        out.write("use quick_check::IsNormalized::*;\n")
        out.write("\n")

        version = "(%s, %s, %s)" % tuple(UNICODE_VERSION.split("."))
        out.write("#[allow(unused)]\n")
        out.write("pub const UNICODE_VERSION: (u64, u64, u64) = %s;\n\n" % version)

        gen_combining_class(data.combining_classes, out)
        out.write("\n")

        gen_composition_table(data.canon_comp, out)
        out.write("\n")

        gen_decomposition_tables(data.canon_fully_decomp, data.compat_fully_decomp, out)

        gen_combining_mark(data.general_category_mark, out)
        out.write("\n")

        gen_nfc_qc(data.norm_props, out)
        out.write("\n")

        gen_nfkc_qc(data.norm_props, out)
        out.write("\n")

        gen_nfd_qc(data.norm_props, out)
        out.write("\n")

        gen_nfkd_qc(data.norm_props, out)
        out.write("\n")

        gen_stream_safe(data.ss_leading, data.ss_trailing, out)
        out.write("\n")

    with open("normalization_tests.rs", "w") as out:
        out.write(PREAMBLE)
        gen_tests(data.norm_tests, out)