kataan 0.0.8

#!/usr/bin/env python3
"""Generate `src/regex/props_data.rs` — code-point range tables for the Unicode
binary properties that ECMAScript `\\p{…}` escapes expose but the `intl` crate
does not publish a predicate for.

Usage:

    tools/gen_unicode_props.py <ucd-dir> > src/regex/props_data.rs

`<ucd-dir>` must contain these files, downloaded from
https://www.unicode.org/Public/<version>/ucd/ :

    PropList.txt  DerivedCoreProperties.txt  DerivedNormalizationProps.txt
    emoji/emoji-data.txt

Each UCD file lists `START..END ; Property_Name` (or a single code point) per
line, `#`-comments to end of line. We collect the code points per property,
coalesce them into sorted, non-adjacent ranges, and emit one `&[(u32, u32)]`
static per property. `src/regex/props.rs` binary-searches these.
"""

import sys
import os
import re

UNICODE_VERSION = "17.0.0"

# property name -> (file, ucd property name). Order here is the emit order.
PROPS = [
    # DerivedCoreProperties.txt
    ("CASE_IGNORABLE", "DerivedCoreProperties.txt", "Case_Ignorable"),
    ("GRAPHEME_BASE", "DerivedCoreProperties.txt", "Grapheme_Base"),
    ("GRAPHEME_EXTEND", "DerivedCoreProperties.txt", "Grapheme_Extend"),
    ("ID_CONTINUE", "DerivedCoreProperties.txt", "ID_Continue"),
    ("ID_START", "DerivedCoreProperties.txt", "ID_Start"),
    # DerivedNormalizationProps.txt
    (
        "CHANGES_WHEN_NFKC_CASEFOLDED",
        "DerivedNormalizationProps.txt",
        "Changes_When_NFKC_Casefolded",
    ),
    # emoji/emoji-data.txt
    ("EMOJI", "emoji-data.txt", "Emoji"),
    ("EMOJI_COMPONENT", "emoji-data.txt", "Emoji_Component"),
    ("EMOJI_MODIFIER", "emoji-data.txt", "Emoji_Modifier"),
    ("EMOJI_MODIFIER_BASE", "emoji-data.txt", "Emoji_Modifier_Base"),
    ("EMOJI_PRESENTATION", "emoji-data.txt", "Emoji_Presentation"),
    ("EXTENDED_PICTOGRAPHIC", "emoji-data.txt", "Extended_Pictographic"),
    # PropList.txt
    ("DEPRECATED", "PropList.txt", "Deprecated"),
    ("EXTENDER", "PropList.txt", "Extender"),
    ("IDS_BINARY_OPERATOR", "PropList.txt", "IDS_Binary_Operator"),
    ("IDS_TRINARY_OPERATOR", "PropList.txt", "IDS_Trinary_Operator"),
    ("IDEOGRAPHIC", "PropList.txt", "Ideographic"),
    ("LOGICAL_ORDER_EXCEPTION", "PropList.txt", "Logical_Order_Exception"),
    ("PATTERN_SYNTAX", "PropList.txt", "Pattern_Syntax"),
    ("PATTERN_WHITE_SPACE", "PropList.txt", "Pattern_White_Space"),
    ("RADICAL", "PropList.txt", "Radical"),
    ("SENTENCE_TERMINAL", "PropList.txt", "Sentence_Terminal"),
    ("SOFT_DOTTED", "PropList.txt", "Soft_Dotted"),
    ("TERMINAL_PUNCTUATION", "PropList.txt", "Terminal_Punctuation"),
    ("UNIFIED_IDEOGRAPH", "PropList.txt", "Unified_Ideograph"),
]

LINE = re.compile(
    r"^\s*([0-9A-Fa-f]{4,6})(?:\.\.([0-9A-Fa-f]{4,6}))?\s*;\s*([A-Za-z_0-9]+)"
)


def parse(path):
    """file -> {property name: sorted list of coalesced (lo, hi) ranges}."""
    raw = {}
    with open(path, encoding="utf-8") as fh:
        for line in fh:
            line = line.split("#", 1)[0]
            m = LINE.match(line)
            if not m:
                continue
            lo = int(m.group(1), 16)
            hi = int(m.group(2), 16) if m.group(2) else lo
            raw.setdefault(m.group(3), []).append((lo, hi))
    return {name: coalesce(rs) for name, rs in raw.items()}


def coalesce(ranges):
    out = []
    for lo, hi in sorted(ranges):
        if out and lo <= out[-1][1] + 1:
            out[-1] = (out[-1][0], max(out[-1][1], hi))
        else:
            out.append((lo, hi))
    return out


def main():
    if len(sys.argv) != 2:
        sys.exit(__doc__)
    ucd = sys.argv[1]
    files = {}
    for _, fname, _ in PROPS:
        if fname not in files:
            path = os.path.join(ucd, fname)
            if not os.path.exists(path):
                path = os.path.join(ucd, "emoji", fname)
            files[fname] = parse(path)

    major, minor, patch = UNICODE_VERSION.split(".")
    out = sys.stdout.write
    out(
        f"""//! Unicode binary-property range tables for regex `\\p{{…}}` escapes.
//!
//! @generated by `tools/gen_unicode_props.py` from the Unicode Character
//! Database v{UNICODE_VERSION} (`PropList.txt`, `DerivedCoreProperties.txt`,
//! `DerivedNormalizationProps.txt`, `emoji/emoji-data.txt`) — DO NOT EDIT.
//! Regenerate with:
//!
//! ```text
//! tools/gen_unicode_props.py <ucd-dir> > src/regex/props_data.rs
//! ```
//!
//! Only the properties the `intl` crate does not expose a public predicate for
//! live here; everything else in [`super::props`] delegates to `intl` (whose
//! tables are the same UCD version) or to a closed form. Each table is a sorted,
//! coalesced list of inclusive `(lo, hi)` code-point ranges, searched by
//! [`in_ranges`].

/// The UCD version these tables were generated from. Kept in sync with the
/// `intl` crate's `intl::unicode::UNICODE_VERSION` by the regex property tests.
#[cfg_attr(not(test), allow(dead_code))]
pub(crate) const UNICODE_VERSION: (u8, u8, u8) = ({major}, {minor}, {patch});

/// Whether `cp` falls inside one of `ranges` (sorted, non-overlapping, inclusive).
pub(crate) fn in_ranges(ranges: &[(u32, u32)], cp: u32) -> bool {{
    ranges
        .binary_search_by(|&(lo, hi)| {{
            if cp < lo {{
                core::cmp::Ordering::Greater
            }} else if cp > hi {{
                core::cmp::Ordering::Less
            }} else {{
                core::cmp::Ordering::Equal
            }}
        }})
        .is_ok()
}}
"""
    )

    for const, fname, prop in PROPS:
        ranges = files[fname].get(prop)
        if not ranges:
            sys.exit(f"error: {prop} not found in {fname}")
        out(f"\n/// `\\p{{{prop}}}` — {len(ranges)} ranges.\n")
        out(f"pub(crate) static {const}: &[(u32, u32)] = &[\n")
        for lo, hi in ranges:
            out(f"    (0x{lo:04X}, 0x{hi:04X}),\n")
        out("];\n")


if __name__ == "__main__":
    main()