smooth-reading 0.3.0

Emphasise the leading letters of every word to create artificial fixation points.
Documentation
#!/usr/bin/env python3
"""Generate ``src/unicode_data.rs`` from the Unicode Character Database.

Rust's standard library has no Unicode general-category lookup, and the crate
has zero runtime dependencies, so the categories the algorithm needs are baked
in as sorted, inclusive code-point range tables:

* ``L``  -- every letter (``Lu Ll Lt Lm Lo``),
* ``N``  -- every number (``Nd Nl No``),
* ``Nd`` -- decimal digits only,
* ``M``  -- every mark (``Mn Mc Me``).

The script downloads ``UnicodeData.txt`` (and the version from ``ReadMe.txt``)
from unicode.org with only the standard library. If the download fails it
falls back to Python's own ``unicodedata`` module and records that module's
``unidata_version`` instead.

Usage (from ``rust/``)::

    python3 tools/gen_unicode_data.py            # writes src/unicode_data.rs
    python3 tools/gen_unicode_data.py --offline  # force the unicodedata fallback
"""

from __future__ import annotations

import re
import sys
import unicodedata
import urllib.error
import urllib.request
from collections.abc import Iterable, Iterator
from pathlib import Path

UCD = "https://www.unicode.org/Public/UCD/latest/ucd/UnicodeData.txt"
README = "https://www.unicode.org/Public/UCD/latest/ReadMe.txt"
OUTPUT = Path(__file__).resolve().parent.parent / "src" / "unicode_data.rs"
MAX_CODE_POINT = 0x10FFFF

TABLES: tuple[tuple[str, str, frozenset[str]], ...] = (
    ("LETTER", "L", frozenset({"Lu", "Ll", "Lt", "Lm", "Lo"})),
    ("NUMBER", "N", frozenset({"Nd", "Nl", "No"})),
    ("DECIMAL_DIGIT", "Nd", frozenset({"Nd"})),
    ("MARK", "M", frozenset({"Mn", "Mc", "Me"})),
)


def fetch(url: str) -> str:
    with urllib.request.urlopen(url, timeout=60) as response:  # noqa: S310
        return response.read().decode("utf-8")


def categories_from_ucd(data: str) -> Iterator[tuple[int, int, str]]:
    """Yield ``(first, last, category)`` triples from ``UnicodeData.txt``.

    Large blocks are listed as a ``<..., First>`` / ``<..., Last>`` pair of
    lines; everything between the two code points shares the category.
    """
    pending_first: tuple[int, str] | None = None
    for line in data.splitlines():
        if not line.strip():
            continue
        fields = line.split(";")
        code, name, category = int(fields[0], 16), fields[1], fields[2]
        if name.endswith(", First>"):
            pending_first = (code, category)
            continue
        if name.endswith(", Last>"):
            if pending_first is None or pending_first[1] != category:
                raise ValueError(f"unmatched range end at U+{code:04X}: {line}")
            yield pending_first[0], code, category
            pending_first = None
            continue
        yield code, code, category
    if pending_first is not None:
        raise ValueError("UnicodeData.txt ends inside a First/Last range")


def categories_from_unicodedata() -> Iterator[tuple[int, int, str]]:
    for code in range(MAX_CODE_POINT + 1):
        yield code, code, unicodedata.category(chr(code))


def collect(
    triples: Iterable[tuple[int, int, str]], wanted: frozenset[str]
) -> list[tuple[int, int]]:
    """Sorted, merged inclusive ranges of the code points in ``wanted``."""
    ranges: list[tuple[int, int]] = []
    for first, last, category in sorted(triples):
        if category not in wanted:
            continue
        if ranges and ranges[-1][1] + 1 >= first:
            ranges[-1] = (ranges[-1][0], max(ranges[-1][1], last))
        else:
            ranges.append((first, last))
    return ranges


def version_from_readme(readme: str) -> str:
    match = re.search(r"Version\s+(\d+\.\d+\.\d+)", readme)
    if match is None:
        raise ValueError("could not find the Unicode version in ReadMe.txt")
    return match.group(1)


def render(version: str, source: str, tables: dict[str, list[tuple[int, int]]]) -> str:
    out: list[str] = []
    out.append("// @generated by tools/gen_unicode_data.py -- do not edit by hand.")
    out.append(f"// Unicode {version} general categories, from {source}.")
    out.append("")
    out.append("//! Unicode general-category range tables (`L`, `N`, `Nd`, `M`).")
    out.append("//!")
    out.append("//! Rust's standard library exposes no general-category lookup, and the crate")
    out.append("//! has no runtime dependencies, so the categories the SPEC §3 scanner needs are")
    out.append("//! baked in as sorted, inclusive `(first, last)` code-point ranges and looked")
    out.append("//! up by binary search. Regenerate with `python3 tools/gen_unicode_data.py`.")
    out.append("")
    out.append(f'/// The Unicode version the tables were generated from ("{version}").')
    out.append(f'pub const UNICODE_VERSION: &str = "{version}";')
    for name, category, _ in TABLES:
        ranges = tables[name]
        out.append("")
        out.append(f"/// Inclusive code-point ranges of general category `{category}` ({len(ranges)} ranges).")
        out.append("#[rustfmt::skip]")
        out.append(f"const {name}: &[(u32, u32)] = &[")
        for first, last in ranges:
            out.append(f"    (0x{first:04X}, 0x{last:04X}),")
        out.append("];")
    out.append("")
    out.append("/// Whether `code` lies in one of the sorted, inclusive ranges of `table`.")
    out.append("pub(crate) fn contains(table: &[(u32, u32)], code: u32) -> bool {")
    out.append("    table")
    out.append("        .binary_search_by(|&(first, last)| {")
    out.append("            if last < code {")
    out.append("                std::cmp::Ordering::Less")
    out.append("            } else if first > code {")
    out.append("                std::cmp::Ordering::Greater")
    out.append("            } else {")
    out.append("                std::cmp::Ordering::Equal")
    out.append("            }")
    out.append("        })")
    out.append("        .is_ok()")
    out.append("}")
    lookups = (
        ("is_letter", "LETTER", "`\\p{L}`: any letter (`Lu`, `Ll`, `Lt`, `Lm`, `Lo`)."),
        ("is_number", "NUMBER", "`\\p{N}`: any number (`Nd`, `Nl`, `No`)."),
        ("is_decimal_digit", "DECIMAL_DIGIT", "`\\p{Nd}`: a decimal digit of any script."),
        ("is_mark", "MARK", "`\\p{M}`: a combining mark (`Mn`, `Mc`, `Me`)."),
    )
    for function, table, doc in lookups:
        out.append("")
        out.append(f"/// {doc}")
        out.append("#[must_use]")
        out.append(f"pub fn {function}(c: char) -> bool {{")
        out.append(f"    contains({table}, u32::from(c))")
        out.append("}")
    out.append("")
    return "\n".join(out)


def main(argv: list[str]) -> int:
    offline = "--offline" in argv
    triples: list[tuple[int, int, str]]
    if offline:
        triples, version, source = list(categories_from_unicodedata()), unicodedata.unidata_version, "Python unicodedata"
    else:
        try:
            data = fetch(UCD)
            version = version_from_readme(fetch(README))
            triples, source = list(categories_from_ucd(data)), "UnicodeData.txt"
        except (urllib.error.URLError, OSError, ValueError) as error:
            print(f"download failed ({error}); falling back to unicodedata", file=sys.stderr)
            triples = list(categories_from_unicodedata())
            version, source = unicodedata.unidata_version, "Python unicodedata"
    tables = {name: collect(triples, wanted) for name, _, wanted in TABLES}
    OUTPUT.write_text(render(version, source, tables), encoding="utf-8")
    total = sum(len(ranges) for ranges in tables.values())
    print(f"wrote {OUTPUT} (Unicode {version}, {total} ranges)")
    return 0


if __name__ == "__main__":
    raise SystemExit(main(sys.argv[1:]))