hermes-parser 0.1.1

A Rust port of the Hermes JavaScript/Flow/TypeScript parser (front-end) by Tzvetan Mikov, the architect of Hermes. Not an official Meta project.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113

#!/usr/bin/env python3
# Copyright (c) Meta Platforms, Inc. and affiliates.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.
#
# Generate rust/crates/parser/src/html_entities.rs from
# include/hermes/Parser/HTMLEntities.def. The table is the XHTML named-entity
# map the JSX lexer needs (`&name;` -> code point). Re-run when the .def is
# updated:
#   python3 rust/crates/parser/gen_html_entities.py
import re
import sys
from pathlib import Path

ROOT = Path(__file__).resolve().parents[3]
DEF = ROOT / "include/hermes/Parser/HTMLEntities.def"
OUT = Path(__file__).resolve().parent / "src/html_entities.rs"

# Match `HTML_ENTITY(name, 0xHEX)` data rows (not the `#define` stub).
ROW = re.compile(r"^HTML_ENTITY\((\w+),\s*(0x[0-9a-fA-F]+)\)", re.M)

# The number of data rows in HTMLEntities.def (anti-drift guard). The C++ source
# of truth currently has 253 `HTML_ENTITY(name, 0xHEX)` rows.
EXPECTED = 253


def main():
    if not DEF.exists():
        sys.exit(f"error: {DEF} not found — run from a full hermes checkout")
    src = DEF.read_text()
    rows = [(name, int(value, 0)) for name, value in ROW.findall(src)]

    if len(rows) != EXPECTED:
        sys.exit(
            f"error: HTMLEntities.def has {len(rows)} rows, expected {EXPECTED}"
        )

    # Sort by name (so a binary search over the table works).
    rows.sort(key=lambda nv: nv[0])

    # Sanity: names are unique and strictly ascending after sorting.
    for i in range(1, len(rows)):
        if rows[i][0] <= rows[i - 1][0]:
            sys.exit(
                f"error: duplicate/unsorted entity name at index {i}: "
                f"{rows[i][0]!r}"
            )

    out = []
    out.append("// Copyright (c) Meta Platforms, Inc. and affiliates.")
    out.append("//")
    out.append("// This source code is licensed under the MIT license found in the")
    out.append("// LICENSE file in the root directory of this source tree.")
    out.append("//")
    out.append(
        "// GENERATED by gen_html_entities.py from "
        "include/hermes/Parser/HTMLEntities.def."
    )
    out.append("// DO NOT EDIT BY HAND.")
    out.append("//")
    out.append("//! XHTML named-entity table for the JSX lexer. Port of the")
    out.append("//! `initializeHTMLEntities` map in `lib/Parser/JSLexer.cpp`, but")
    out.append("//! emitted sorted by name so `&name;` lookup is a binary search.")
    out.append("")
    out.append(
        f"/// The XHTML named entities, sorted by name. Each entry maps an "
        f"entity\n/// name (the bytes between `&` and `;`) to its Unicode code "
        f"point."
    )
    out.append(f"pub static HTML_ENTITIES: [(&[u8], u32); {len(rows)}] = [")
    for name, value in rows:
        out.append(f'    (b"{name}", 0x{value:04x}),')
    out.append("];")
    out.append("")
    out.append("/// Look up an XHTML named entity by name (the bytes between `&`")
    out.append("/// and `;`). Returns its code point, or `None` if unknown.")
    out.append("pub fn lookup(name: &[u8]) -> Option<u32> {")
    out.append(
        "    HTML_ENTITIES"
    )
    out.append("        .binary_search_by(|(n, _)| (*n).cmp(name))")
    out.append("        .ok()")
    out.append("        .map(|i| HTML_ENTITIES[i].1)")
    out.append("}")
    out.append("")
    out.append("#[cfg(test)]")
    out.append("mod tests {")
    out.append("    use super::*;")
    out.append("")
    out.append("    #[test]")
    out.append("    fn known_entities() {")
    out.append("        assert_eq!(lookup(b\"amp\"), Some(0x26));")
    out.append("        assert_eq!(lookup(b\"lt\"), Some(0x3c));")
    out.append("        assert_eq!(lookup(b\"nope\"), None);")
    out.append("    }")
    out.append("")
    out.append("    #[test]")
    out.append("    fn table_is_sorted_and_full() {")
    out.append(f"        assert_eq!(HTML_ENTITIES.len(), {len(rows)});")
    out.append("        for w in HTML_ENTITIES.windows(2) {")
    out.append("            assert!(w[0].0 < w[1].0, \"table not sorted by name\");")
    out.append("        }")
    out.append("    }")
    out.append("}")
    out.append("")

    OUT.write_text("\n".join(out))
    print(f"wrote {OUT} ({len(rows)} entities)")


if __name__ == "__main__":
    main()