import argparse
import re
import sys
PROP = re.compile(r"\.\s*([A-Za-z_$][A-Za-z0-9_$]*)")
WORD = re.compile(r"[A-Za-z_$][A-Za-z0-9_$]*")
LIT = re.compile(r"""(?<!\\)(['"])((?:\\.|(?!\1).){0,24})\1""")
ESCAPE = {"\\n": "\n", "\\t": "\t", "\\r": "\r", "\\\\": "\\", "\\0": "\0",
"\\'": "'", '\\"': '"'}
def unescape(s: str) -> str:
for k, v in ESCAPE.items():
s = s.replace(k, v)
return s
def add(seen: set, out: list, s: str) -> None:
if not s or len(s) > 32 or s in seen:
return
if any(ord(c) < 0x20 for c in s):
return
seen.add(s)
out.append(s)
def main() -> None:
ap = argparse.ArgumentParser()
ap.add_argument("--out", required=True)
ap.add_argument("--no-punct", action="store_true",
help="不补标点/数字等短串(只要语料里出现的)")
ap.add_argument("dirs", nargs="+")
args = ap.parse_args()
seen: set = set()
out: list = []
if not args.no_punct:
for c in range(0x20, 0x7F):
add(seen, out, chr(c))
punct = "!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~"
for a in punct:
for b in punct:
add(seen, out, a + b)
for a in punct:
for b in punct:
for c in punct:
if a == b == c: continue
add(seen, out, a + b + c)
for s in ("...", "===", "!==", "=>", "<!--", "-->", "//", "/*", "*/",
"&&", "||", "??", "?.", "**", "++", "--", "0", "00"):
add(seen, out, s)
for n in range(0, 1000):
add(seen, out, str(n))
for n in ("-1", "0x", "1e", "Infinity", "NaN", "true", "false", "null",
"undefined", "none", "TRUE", "FALSE", "NULL"):
add(seen, out, n)
import os
files = 0
for d in args.dirs:
if os.path.isfile(d):
paths = [d]
else:
paths = []
for root, _dirs, names in os.walk(d):
for n in names:
if n.endswith(".js"):
paths.append(os.path.join(root, n))
for p in paths:
try:
text = open(p, encoding="utf-8", errors="ignore").read()
except OSError:
continue
files += 1
for m in PROP.finditer(text):
add(seen, out, m.group(1))
for m in LIT.finditer(text):
add(seen, out, unescape(m.group(2)))
for m in WORD.finditer(text):
add(seen, out, m.group(0))
with open(args.out, "w", encoding="utf-8") as f:
for s in out:
f.write(s + "\n")
print(f"{len(out)} 条候选(扫了 {files} 个文件)→ {args.out}", file=sys.stderr)
if __name__ == "__main__":
main()