from __future__ import annotations
import re
import sys
import unicodedata
import urllib.error
import urllib.request
from collections.abc import Iterable, Iterator
from pathlib import Path
UCD = "https://www.unicode.org/Public/UCD/latest/ucd/UnicodeData.txt"
README = "https://www.unicode.org/Public/UCD/latest/ReadMe.txt"
OUTPUT = Path(__file__).resolve().parent.parent / "src" / "unicode_data.rs"
MAX_CODE_POINT = 0x10FFFF
TABLES: tuple[tuple[str, str, frozenset[str]], ...] = (
("LETTER", "L", frozenset({"Lu", "Ll", "Lt", "Lm", "Lo"})),
("NUMBER", "N", frozenset({"Nd", "Nl", "No"})),
("DECIMAL_DIGIT", "Nd", frozenset({"Nd"})),
("MARK", "M", frozenset({"Mn", "Mc", "Me"})),
)
def fetch(url: str) -> str:
with urllib.request.urlopen(url, timeout=60) as response: return response.read().decode("utf-8")
def categories_from_ucd(data: str) -> Iterator[tuple[int, int, str]]:
pending_first: tuple[int, str] | None = None
for line in data.splitlines():
if not line.strip():
continue
fields = line.split(";")
code, name, category = int(fields[0], 16), fields[1], fields[2]
if name.endswith(", First>"):
pending_first = (code, category)
continue
if name.endswith(", Last>"):
if pending_first is None or pending_first[1] != category:
raise ValueError(f"unmatched range end at U+{code:04X}: {line}")
yield pending_first[0], code, category
pending_first = None
continue
yield code, code, category
if pending_first is not None:
raise ValueError("UnicodeData.txt ends inside a First/Last range")
def categories_from_unicodedata() -> Iterator[tuple[int, int, str]]:
for code in range(MAX_CODE_POINT + 1):
yield code, code, unicodedata.category(chr(code))
def collect(
triples: Iterable[tuple[int, int, str]], wanted: frozenset[str]
) -> list[tuple[int, int]]:
ranges: list[tuple[int, int]] = []
for first, last, category in sorted(triples):
if category not in wanted:
continue
if ranges and ranges[-1][1] + 1 >= first:
ranges[-1] = (ranges[-1][0], max(ranges[-1][1], last))
else:
ranges.append((first, last))
return ranges
def version_from_readme(readme: str) -> str:
match = re.search(r"Version\s+(\d+\.\d+\.\d+)", readme)
if match is None:
raise ValueError("could not find the Unicode version in ReadMe.txt")
return match.group(1)
def render(version: str, source: str, tables: dict[str, list[tuple[int, int]]]) -> str:
out: list[str] = []
out.append("// @generated by tools/gen_unicode_data.py -- do not edit by hand.")
out.append(f"// Unicode {version} general categories, from {source}.")
out.append("")
out.append("//! Unicode general-category range tables (`L`, `N`, `Nd`, `M`).")
out.append("//!")
out.append("//! Rust's standard library exposes no general-category lookup, and the crate")
out.append("//! has no runtime dependencies, so the categories the SPEC §3 scanner needs are")
out.append("//! baked in as sorted, inclusive `(first, last)` code-point ranges and looked")
out.append("//! up by binary search. Regenerate with `python3 tools/gen_unicode_data.py`.")
out.append("")
out.append(f'/// The Unicode version the tables were generated from ("{version}").')
out.append(f'pub const UNICODE_VERSION: &str = "{version}";')
for name, category, _ in TABLES:
ranges = tables[name]
out.append("")
out.append(f"/// Inclusive code-point ranges of general category `{category}` ({len(ranges)} ranges).")
out.append("#[rustfmt::skip]")
out.append(f"const {name}: &[(u32, u32)] = &[")
for first, last in ranges:
out.append(f" (0x{first:04X}, 0x{last:04X}),")
out.append("];")
out.append("")
out.append("/// Whether `code` lies in one of the sorted, inclusive ranges of `table`.")
out.append("pub(crate) fn contains(table: &[(u32, u32)], code: u32) -> bool {")
out.append(" table")
out.append(" .binary_search_by(|&(first, last)| {")
out.append(" if last < code {")
out.append(" std::cmp::Ordering::Less")
out.append(" } else if first > code {")
out.append(" std::cmp::Ordering::Greater")
out.append(" } else {")
out.append(" std::cmp::Ordering::Equal")
out.append(" }")
out.append(" })")
out.append(" .is_ok()")
out.append("}")
lookups = (
("is_letter", "LETTER", "`\\p{L}`: any letter (`Lu`, `Ll`, `Lt`, `Lm`, `Lo`)."),
("is_number", "NUMBER", "`\\p{N}`: any number (`Nd`, `Nl`, `No`)."),
("is_decimal_digit", "DECIMAL_DIGIT", "`\\p{Nd}`: a decimal digit of any script."),
("is_mark", "MARK", "`\\p{M}`: a combining mark (`Mn`, `Mc`, `Me`)."),
)
for function, table, doc in lookups:
out.append("")
out.append(f"/// {doc}")
out.append("#[must_use]")
out.append(f"pub fn {function}(c: char) -> bool {{")
out.append(f" contains({table}, u32::from(c))")
out.append("}")
out.append("")
return "\n".join(out)
def main(argv: list[str]) -> int:
offline = "--offline" in argv
triples: list[tuple[int, int, str]]
if offline:
triples, version, source = list(categories_from_unicodedata()), unicodedata.unidata_version, "Python unicodedata"
else:
try:
data = fetch(UCD)
version = version_from_readme(fetch(README))
triples, source = list(categories_from_ucd(data)), "UnicodeData.txt"
except (urllib.error.URLError, OSError, ValueError) as error:
print(f"download failed ({error}); falling back to unicodedata", file=sys.stderr)
triples = list(categories_from_unicodedata())
version, source = unicodedata.unidata_version, "Python unicodedata"
tables = {name: collect(triples, wanted) for name, _, wanted in TABLES}
OUTPUT.write_text(render(version, source, tables), encoding="utf-8")
total = sum(len(ranges) for ranges in tables.values())
print(f"wrote {OUTPUT} (Unicode {version}, {total} ranges)")
return 0
if __name__ == "__main__":
raise SystemExit(main(sys.argv[1:]))