rsconstruct 0.9.132

Rust based fast build system
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126

#!/usr/bin/env python
"""Generate src/engines/checkstyle/tables.rs from checkstyle's Java grammar.

icheckstyle (processor.checker.icheckstyle) is a port of checkstyle 8.36.1,
whose Java parser is generated by ANTLR 2.7.7 from `grammar/java.g`. The
port follows the generated recognizer and lexer rule by rule; the lookahead
sets those decide with, the token types and names and the keyword table are
data, so they are taken from ANTLR's output rather than retyped.

Usage: gen-icheckstyle-tables.py <checkstyle-8.36.1 checkout> [antlr.jar] \
           > src/engines/checkstyle/tables.rs
       rustfmt --edition 2024 src/engines/checkstyle/tables.rs
(antlr.jar defaults to /usr/share/java/antlr.jar, Debian's antlr 2.7.7.)
"""

import re
import subprocess
import sys
import tempfile
from pathlib import Path

GRAMMAR = "src/main/resources/com/puppycrawl/tools/checkstyle/grammar/java.g"


def generate(checkout: Path, jar: str, out: Path) -> None:
    cmd = ["java", "-cp", jar, "antlr.Tool", "-o", str(out), str(checkout / GRAMMAR)]
    subprocess.run(cmd, check=True, capture_output=True)


def token_types(out: Path) -> list[tuple[str, int]]:
    """NAME=number pairs of GeneratedJavaTokenTypes.txt (literals as LITERAL_x)."""
    pairs = []
    for line in (out / "GeneratedJavaTokenTypes.txt").read_text().splitlines():
        m = re.fullmatch(r'(\w+)(?:="[^"]*")?=(\d+)', line.strip())
        if m:
            pairs.append((m.group(1), int(m.group(2))))
    return pairs


def token_names(java: str) -> list[str]:
    body = java[java.index("_tokenNames = {") :]
    body = body[: body.index("};")]
    return [bytes(s, "ascii").decode("unicode_escape") for s in re.findall(r'"((?:[^"\\]|\\.)*)"', body)]


def long_arrays(java: str) -> dict[int, list[int]]:
    """The long[] data of each mk_tokenSet_N(), as unsigned words."""
    sets = {}
    for m in re.finditer(r"mk_tokenSet_(\d+)\(\) \{(.*?)return data;", java, re.S):
        body = m.group(2)
        size_m = re.search(r"new long\[(\d+)\]", body)
        if size_m:
            data = [0] * int(size_m.group(1))
            for i, v in re.findall(r"data\[(\d+)\]=(-?\d+)L;", body):
                data[int(i)] = int(v) & (2**64 - 1)
            for lo, hi, v in re.findall(r"for \(int i = (\d+); i<=(\d+); i\+\+\) \{ data\[i\]=(-?\d+)L; \}", body):
                for i in range(int(lo), int(hi) + 1):
                    data[i] = int(v) & (2**64 - 1)
        else:
            init = re.search(r"long\[\] data = \{([^}]*)\}", body).group(1)
            data = [int(v.strip().rstrip("L")) & (2**64 - 1) for v in init.split(",")]
        sets[int(m.group(1))] = data
    return sets


def char_ranges(words: list[int]) -> list[tuple[int, int]]:
    ranges, start = [], None
    for c in range(0x10000):
        member = c // 64 < len(words) and (words[c // 64] >> (c % 64)) & 1
        if member and start is None:
            start = c
        if not member and start is not None:
            ranges.append((start, c - 1))
            start = None
    if start is not None:
        ranges.append((start, 0xFFFF))
    return ranges


def main() -> None:
    checkout = Path(sys.argv[1])
    jar = sys.argv[2] if len(sys.argv) > 2 else "/usr/share/java/antlr.jar"
    with tempfile.TemporaryDirectory() as tmp:
        out = Path(tmp)
        generate(checkout, jar, out)
        parser = (out / "GeneratedJavaRecognizer.java").read_text()
        lexer = (out / "GeneratedJavaLexer.java").read_text()
        types = token_types(out)
    names = token_names(parser)
    literals = sorted(
        (word, int(t)) for word, t in re.findall(r'literals\.put\(new ANTLRHashString\("(\w+)", this\), new Integer\((\d+)\)\);', lexer)
    )
    parser_sets = long_arrays(parser)
    lexer_sets = long_arrays(lexer)

    w = sys.stdout.write
    w("//! Generated by scripts/gen-icheckstyle-tables.py from checkstyle 8.36.1's\n")
    w("//! `java.g` (ANTLR 2.7.7 output): token types and names, the keyword\n")
    w("//! table, and the lookahead sets of the recognizer and the lexer. Do not edit.\n\n")
    w("use super::lexer::CharSet;\nuse super::parser::TokenSet;\n\n")
    for name, value in types:
        w(f"pub const {name}: u16 = {value};\n")
    w(f"\n/// The recognizer's `_tokenNames`, as its error messages print them.\npub const TOKEN_NAMES: [&str; {len(names)}] = [\n")
    for n in names:
        escaped = n.replace("\\", "\\\\").replace('"', '\\"')
        w(f'    "{escaped}",\n')
    w("];\n\n/// The lexer's literals table: identifiers that are keywords.\n")
    w(f"pub const LITERALS: [(&str, u16); {len(literals)}] = [\n")
    for word, t in literals:
        w(f'    ("{word}", {t}),\n')
    w("];\n\n")
    for i in sorted(parser_sets):
        words = (parser_sets[i] + [0] * 4)[:4]
        if any(parser_sets[i][4:]):
            raise ValueError(f"parser token set {i} has types above 255")
        hex = [f"0x{x >> 48:04x}_{(x >> 32) & 0xFFFF:04x}_{(x >> 16) & 0xFFFF:04x}_{x & 0xFFFF:04x}" for x in words]
        w(f"pub const SET_{i}: TokenSet = TokenSet([{', '.join(hex)}]);\n")
    w("\n")
    for i in sorted(lexer_sets):
        ranges = char_ranges(lexer_sets[i])
        body = ", ".join(f"(0x{lo:04x}, 0x{hi:04x})" for lo, hi in ranges)
        w(f"pub const CHARS_{i}: CharSet = CharSet(&[{body}]);\n")


if __name__ == "__main__":
    main()