html5-parser 0.2.0

#!/usr/bin/env python3
"""Generates src/entities.rs from the WHATWG named-character-reference table.

Source: https://html.spec.whatwg.org/entities.json
Referenced by: WHATWG HTML spec, §13.2.5.73 "Named character reference
state" (https://html.spec.whatwg.org/multipage/parsing.html#tokenization).

The upstream JSON keys already encode whether a trailing `;` is part of a
given name (legacy, pre-HTML5 entities are present both with and without
`;`; all other entries require the `;`) — this script does not need to
special-case that, it just emits one table row per JSON key.

Usage:
    python3 xtask/gen-entities.py > src/entities.rs

Network access required (fetches entities.json directly); pass a local file
path as the first argument to regenerate offline from a saved copy instead:
    python3 xtask/gen-entities.py /path/to/entities.json > src/entities.rs
"""

import json
import sys
import urllib.request

SOURCE_URL = "https://html.spec.whatwg.org/entities.json"


def load_entities():
    if len(sys.argv) > 1:
        with open(sys.argv[1], encoding="utf-8") as f:
            return json.load(f)
    with urllib.request.urlopen(SOURCE_URL) as response:
        return json.load(response)


def rust_str_literal(characters):
    """Renders `characters` as a Rust string literal using \\u{...} escapes
    per codepoint, so combining marks and non-BMP characters stay legible
    (and diffable) in the generated source instead of appearing as raw,
    often invisible Unicode in the file."""
    escaped = "".join(f"\\u{{{ord(ch):x}}}" for ch in characters)
    return f'"{escaped}"'


def main():
    entities = load_entities()
    # Sort explicitly rather than relying on upstream JSON key order, so
    # regeneration is deterministic regardless of upstream ordering changes.
    rows = sorted(entities.items())

    print("// Generated by `xtask/gen-entities.py` from")
    print(f"// <{SOURCE_URL}> — do not edit by hand, regenerate instead.")
    print("//")
    print("// Named character reference table for the tokenizer's")
    print("// \"Named character reference state\" (WHATWG HTML spec §13.2.5).")
    print("// Each entry's name already includes a trailing `;` where the")
    print("// spec requires one; legacy (pre-HTML5) names appear twice, once")
    print("// with and once without `;`, both mapping to the same")
    print("// replacement text — matching upstream `entities.json` exactly.")
    print("#![allow(dead_code)] // not yet consumed; see plan/02-tokenizer.md")
    print()
    print(f"pub(crate) static NAMED_CHARACTER_REFERENCES: [(&str, &str); {len(rows)}] = [")
    for name, entry in rows:
        # Upstream keys include the leading `&`; the tokenizer matches the
        # name *after* the `&` has already been consumed, so it is dropped
        # here rather than re-stripped on every lookup at runtime.
        assert name.startswith("&")
        bare_name = name[1:]
        replacement = rust_str_literal(entry["characters"])
        print(f'    ("{bare_name}", {replacement}),')
    print("];")


if __name__ == "__main__":
    main()