domq 0.5.0

Tiny, zero-dependency, forgiving HTML parser: turn messy real-world HTML into a clean DOM tree (and JSON). WASM-first, no_std-friendly.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76

"""Generate src/entities_full.rs (the `entities-full` feature) from the WHATWG
named character reference table.

Usage (from the repository root; `-I` keeps Python from importing anything
next to the downloaded file):

    curl -sSfLo /tmp/entities/entities.json https://html.spec.whatwg.org/entities.json
    python3 -I scripts/gen_entities.py /tmp/entities/entities.json > src/entities_full.rs

Only the `;`-terminated names are emitted: like the curated default table, the
decoder requires the semicolon, so the legacy no-semicolon spellings
(`&amp`, `&copy`, ...) are left verbatim. The table is sorted by name in byte
order, which is what `str::cmp` and the binary search in src/entities.rs use.
"""

import json
import sys

SOURCE = "https://html.spec.whatwg.org/entities.json"


def rust_str(s):
    """A Rust string literal for `s`: printable ASCII as-is, the rest escaped."""
    out = []
    for ch in s:
        if ch in '"\\':
            out.append("\\" + ch)
        elif " " <= ch <= "~":
            out.append(ch)
        else:
            out.append("\\u{%x}" % ord(ch))
    return '"' + "".join(out) + '"'


def main():
    if len(sys.argv) != 2:
        sys.exit("usage: python3 -I scripts/gen_entities.py <entities.json>")
    with open(sys.argv[1], encoding="utf-8") as f:
        table = json.load(f)

    entries = {}
    for key, value in table.items():
        if not (key.startswith("&") and key.endswith(";")):
            continue  # legacy no-semicolon spelling
        name = key[1:-1]
        if not (name.isascii() and name.isalnum()):
            sys.exit("unexpected entity name: %r" % key)
        chars = "".join(chr(c) for c in value["codepoints"])
        if chars != value["characters"]:
            sys.exit("codepoints and characters disagree for %r" % key)
        entries[name] = chars

    names = sorted(entries, key=lambda n: n.encode("ascii"))
    longest = max(len(n) for n in names)
    if longest + 1 > 32:
        sys.exit("a name no longer fits entities::MAX_ENTITY_LEN (32 bytes)")

    w = sys.stdout.write
    w("//! The complete WHATWG named character reference table, for the\n")
    w("//! `entities-full` feature.\n")
    w("//!\n")
    w("//! GENERATED by `scripts/gen_entities.py` from\n")
    w("//! <%s> -- do not edit by hand.\n" % SOURCE)
    w("//! Only `;`-terminated names: the legacy no-semicolon spellings are\n")
    w("//! left verbatim, as with the curated table.\n")
    w("\n")
    w("/// `(name, replacement)` pairs, sorted by name in byte order.\n")
    w("#[rustfmt::skip]\n")
    w("pub(crate) static ENTITIES: [(&str, &str); %d] = [\n" % len(names))
    for n in names:
        w("    (%s, %s),\n" % (rust_str(n), rust_str(entries[n])))
    w("];\n")


if __name__ == "__main__":
    main()