from __future__ import annotations
import re
import subprocess
import sys
from pathlib import Path
from typing import Iterator, NamedTuple
SAFE_INLINE = re.compile(r"\$(`+)(.+?)\1\$")
FENCE = re.compile(r"^\s*```(\w*)")
HTML_TAG = re.compile(r"</?[A-Za-z!][^<>]*>")
HTML_ATTR_VALUE = re.compile(r"=\s*(\"[^\"]*\"|'[^']*')")
MID = re.compile(r"\\mid(?![A-Za-z])")
INERT_MATH = re.compile(r"^[OoΘΩθω𝒪]\s*\(.+\)$|^[ΘΩΣΠΛ]$")
MATH_UNICODE_ALLOWLIST: frozenset = frozenset()
GRANDFATHERED: frozenset[str] = frozenset(
{
}
)
BODY_RULES = frozenset({"mid-delimiter", "unicode-in-math", "inert-code-math"})
def is_archival(path: str) -> bool:
return (
"/design/history/" in path
or path.endswith("-ledger.md")
or "ledger" in path.rsplit("/", 1)[-1]
or "handoff" in path
)
def check_math_body(path: str, line: int, body: str) -> Iterator["Violation"]:
if MID.search(body):
yield Violation(path, line, "mid-delimiter", body.strip()[:90])
bad = sorted({ch for ch in body if ord(ch) > 0x7F and ch not in MATH_UNICODE_ALLOWLIST})
if bad:
glyphs = ", ".join(f"{ch!r} (U+{ord(ch):04X})" for ch in bad)
yield Violation(path, line, "unicode-in-math", f"{glyphs} in {body.strip()[:70]}")
class Token(NamedTuple):
kind: str text: str body: str
class Violation(NamedTuple):
path: str
line: int
kind: str
snippet: str
FIXES = {
"inline": "use $`…`$",
"display": "use a ```math fence",
"letter-before": "insert a space before the opening `$` (GitHub renders no math otherwise)",
"literal-dollar": "wrap each literal dollar in inline code, e.g. `$₁`",
"mid-delimiter": r"\mid is a relation, not a delimiter — use \lvert…\rvert for length, ':' for set-builder",
"unicode-in-math": r"replace the glyph with its LaTeX command (− → -, ≤ → \le, Σ → \Sigma, s̄ → \bar{s})",
"inert-code-math": "typeset as inline MathJax — $`O(1)`$, not a `O(1)` code span",
}
def tokenize(line: str, open_run: int = 0) -> tuple[list[Token], int]:
out: list[Token] = []
i, n = 0, len(line)
if open_run:
k = 0
while k < n:
if line[k] == "`":
m = k
while m < n and line[m] == "`":
m += 1
if m - k == open_run:
out.append(Token("code", line[: m], ""))
i = m
open_run = 0
break
k = m
else:
k += 1
else:
return [Token("code", line, "")], open_run
while i < n:
c = line[i]
prev_i = i
if c == "\\":
out.append(Token("esc", line[i : i + 2], ""))
i += 2
elif c == "`":
j = i
while j < n and line[j] == "`":
j += 1
run = j - i
close = -1
k = j
while k < n:
if line[k] == "`":
m = k
while m < n and line[m] == "`":
m += 1
if m - k == run:
close = k
break
k = m
else:
k += 1
if close < 0:
out.append(Token("code", line[i:], ""))
return out, run
out.append(Token("code", line[i : close + run], ""))
i = close + run
elif c == "$":
if line.startswith("$$", i):
end = line.find("$$", i + 2)
if end != -1:
out.append(Token("display", line[i : end + 2], line[i + 2 : end]))
i = end + 2
else:
out.append(Token("text", "$$", ""))
i += 2
else:
m = SAFE_INLINE.match(line, i)
if m:
out.append(Token("safe", m.group(0), m.group(2)))
i = m.end()
continue
j, close = i + 1, -1
while j < n:
if line[j] == "\\":
j += 2
continue
if line[j] == "$":
close = j
break
j += 1
if close > i + 1:
out.append(Token("inline", line[i : close + 1], line[i + 1 : close]))
i = close + 1
else:
out.append(Token("text", "$", ""))
i += 1
else:
j = i
while j < n and line[j] not in "\\`$":
j += 1
out.append(Token("text", line[i:j], ""))
i = j
if i <= prev_i: raise AssertionError(
f"tokenize made no progress at column {i} of {line!r} — "
"this would loop forever and exhaust memory"
)
return out, 0
def blank_attribute_values(line: str) -> str:
out = list(line)
for tag in HTML_TAG.finditer(line):
base = tag.start()
for attr in HTML_ATTR_VALUE.finditer(tag.group(0)):
start, end = attr.span(1)
for k in range(base + start + 1, base + end - 1):
out[k] = " "
return "".join(out)
def scan(path: Path) -> Iterator[Violation]:
lines = path.read_text(errors="replace").split("\n")
archival = is_archival(str(path))
in_fence = False
fence_lang = "" open_run = 0 i = 0
while i < len(lines):
line = lines[i]
stripped = line.strip()
m = FENCE.match(line)
if m:
fence_lang = "" if in_fence else (m.group(1) or "").lower()
in_fence = not in_fence
open_run = 0
i += 1
continue
if in_fence:
if fence_lang == "math" and not archival:
yield from check_math_body(str(path), i + 1, line)
i += 1
continue
if not stripped: open_run = 0
i += 1
continue
if stripped == "$$":
end = i + 1
while end < len(lines) and lines[end].strip() != "$$":
end += 1
yield Violation(str(path), i + 1, "display", "$$ … $$ block")
i = end + 1
continue
scanned = blank_attribute_values(line)
toks, open_run = tokenize(scanned, open_run)
literal_dollars = 0
col = 0
for tok in toks:
if tok.kind in ("inline", "display"):
yield Violation(str(path), i + 1, tok.kind, tok.text)
elif tok.kind == "esc" and tok.text == "\\$":
literal_dollars += 1
elif tok.kind == "safe":
if col > 0 and scanned[col - 1].isascii() and scanned[col - 1].isalpha():
yield Violation(str(path), i + 1, "letter-before", scanned[max(0, col - 12) : col + len(tok.text)])
if not archival:
yield from check_math_body(str(path), i + 1, tok.body)
elif tok.kind == "code" and not archival:
content = tok.text.strip("`").strip()
if INERT_MATH.match(content):
yield Violation(str(path), i + 1, "inert-code-math", tok.text)
col += len(tok.text)
if literal_dollars >= 2:
yield Violation(str(path), i + 1, "literal-dollar", line.strip()[:90])
i += 1
def selftest() -> int:
import itertools
import tempfile
calls = 0
for length in range(5):
for tup in itertools.product("\\`$x", repeat=length):
line = "".join(tup)
for run in (0, 1, 2):
toks, _ = tokenize(line, run)
calls += 1
if run == 0 and not line.endswith("\\"):
rebuilt = "".join(t.text for t in toks)
assert rebuilt == line, f"reconstruction: {line!r} -> {rebuilt!r}"
assert tokenize("trailing backslash \\")[0], "trailing backslash must terminate"
expected = {
"bare inline $x\\_y$": {"inline"},
"$$\n\\max\\{\\,L\\,\\}\n$$": {"display"},
"letter before InMem$`\\Rightarrow`$descend": {"letter-before"},
"| markers | \\$₁, \\$₂ |": {"literal-dollar"},
"safe $`x\\_y`$ and\n\n```math\n\\max\\{\\,L\\,\\}\n```": set(),
'<img src="d.svg" alt="a $x\\_y$ b" width="100%"/>': set(),
"<td>bare $x\\_y$ in a cell</td>": {"inline"},
"<summary>bare $x\\_y$ in a summary</summary>": {"inline"},
'<img src="d.svg" alt="fine"/> then bare $x\\_y$': {"inline"},
"`Option<V>` and `Vec<u8>` with bare $x\\_y$": {"inline"},
'<img alt="cost \\$1 and \\$2" src="d.svg"/>': set(),
'<img alt="InMem$`\\Rightarrow`$descend" src="d.svg"/>': set(),
"<td>safe $`x\\_y`$ in a cell</td>": set(),
"bad $`O(\\mid q\\mid )`$": {"mid-delimiter"},
"setbuilder $`\\{\\, t : s = h\\cdot t \\in V \\,\\}`$": set(),
"minus $`\\le 2\\cdot \\mid T\\mid − 1`$": {"mid-delimiter", "unicode-in-math"},
"macron $`O(\\mid key\\mid / s̄)`$": {"mid-delimiter", "unicode-in-math"},
"arrow-only span $`→`$ here": {"unicode-in-math"},
"inert `O(1)` here": {"inert-code-math"},
"inert `O(N log N)` here": {"inert-code-math"},
"bare greek `Σ` here": {"inert-code-math"},
"safe $`O(\\lvert q\\rvert)`$ row": set(),
"idents `O_DIRECT` and `Vec<u8>` and `to(x)`": set(),
"```rust\nlet a = 1; // O(1) amortised\n```": set(),
"```math\nO(\\mid q\\mid )\n```": {"mid-delimiter"},
"```math\nO(\\lvert q\\rvert)\n```": set(),
}
with tempfile.TemporaryDirectory() as td:
for i, (src, want) in enumerate(expected.items()):
p = Path(td) / f"t{i}.md"
p.write_text(src + "\n")
got = {v.kind for v in scan(p)}
assert got == want, f"{src!r}: expected {want or '{}'} , got {got or '{}'}"
hist = Path(td) / "docs" / "design" / "history" / "campaign.md"
hist.parent.mkdir(parents=True)
hist.write_text("inert `O(1)` and bad $`O(\\mid q\\mid )`$ but a real $x\\_y$ delimiter\n")
got = {v.kind for v in scan(hist)}
assert got == {"inline"}, f"archival: expected {{'inline'}}, got {got or '{}'}"
assert is_archival("docs/design/history/slice3/x.md")
assert is_archival("docs/experiments/loading-optimization-ledger.md")
assert is_archival("docs/benchmarks/c1-dawg-core-handoff.md")
assert not is_archival("docs/notation.md")
assert not is_archival("docs/theory/scdawg/04-scdawg.md")
V = Violation
grand = frozenset({"a.md"})
vs = [V("a.md", 1, "inert-code-math", "`O(1)`"), V("b.md", 2, "inert-code-math", "`O(n)`")]
fatal, warned, stale = partition(vs, grand, full_run=True)
assert [v.path for v in fatal] == ["b.md"], "off-list body rule must be fatal"
assert [v.path for v in warned] == ["a.md"], "on-list body rule must only warn"
assert stale == [], "a.md still offends, so it is not stale"
fatal2, _, _ = partition([V("a.md", 1, "inline", "$x$")], grand, full_run=True)
assert [v.path for v in fatal2] == ["a.md"], "delimiter rule is fatal regardless of grandfather"
_, _, stale2 = partition([], grand, full_run=True)
assert stale2 == ["a.md"], "clean-but-grandfathered file must be reported stale"
_, _, stale3 = partition([], grand, full_run=False)
assert stale3 == [], "staleness is only meaningful over the whole tree"
print(
f"doc-math selftest: OK — {calls} tokenize() calls terminated; "
f"7 rules fire over {len(expected)} scan cases + archival + ratchet, 0 false positives"
)
return 0
def partition(
violations: list[Violation], grandfathered: frozenset[str], full_run: bool
) -> tuple[list[Violation], list[Violation], list[str]]:
fatal = [v for v in violations if not (v.kind in BODY_RULES and v.path in grandfathered)]
warned = [v for v in violations if v.kind in BODY_RULES and v.path in grandfathered]
stale: list[str] = []
if full_run:
still_offending = {v.path for v in violations if v.kind in BODY_RULES}
stale = sorted(grandfathered - still_offending)
return fatal, warned, stale
def tracked_markdown() -> list[Path]:
out = subprocess.run(
["git", "ls-files", "-z", "*.md"], capture_output=True, text=True, check=True
).stdout
return [Path(p) for p in out.split("\0") if p]
def main(argv: list[str]) -> int:
if "--selftest" in argv[1:]:
return selftest()
explicit = [a for a in argv[1:] if not a.startswith("-")]
paths = [Path(a) for a in explicit] or tracked_markdown()
full_run = not explicit
violations = [v for p in paths for v in scan(p)]
fatal, warned, stale = partition(violations, GRANDFATHERED, full_run)
for v in fatal:
print(f"{v.path}:{v.line}: {v.kind} is GitHub-unsafe — {FIXES[v.kind]}\n {v.snippet}")
if warned:
wfiles = len({v.path for v in warned})
if not full_run:
for v in warned:
print(f"{v.path}:{v.line}: {v.kind} (grandfathered, report-only) — {FIXES[v.kind]}\n {v.snippet}")
print(
f"\nnote: {len(warned)} grandfathered body-rule site(s) in {wfiles} file(s) "
"(report-only until that file is cleaned; see GRANDFATHERED in this script).",
file=sys.stderr,
)
if stale:
print(
"\nFAIL: these files are grandfathered but now clean — remove them from GRANDFATHERED:\n "
+ "\n ".join(stale),
file=sys.stderr,
)
if not fatal and not stale:
msg = f"doc-math: OK — {len(paths)} file(s), no GitHub-unsafe math"
if warned:
msg += f" ({len(warned)} grandfathered body-rule site(s) pending, report-only)"
print(msg)
return 0
if fatal:
files = len({v.path for v in fatal})
print(
f"\nFAIL: {len(fatal)} GitHub-unsafe math site(s) in {files} file(s).\n"
"GitHub strips backslash-escapes inside $…$ and $$…$$ before MathJax sees them,\n"
"corrupting \\_ \\{ \\} \\; \\, \\# — loudly (parse error) or silently (wrong output).\n"
"Delimiters: inline $`…`$ · display ```math fence · literal dollars in inline code.\n"
"Bodies: \\lvert…\\rvert not \\mid · ASCII LaTeX not unicode · $`O(1)`$ not `O(1)`.\n"
"See docs/notation.md.",
file=sys.stderr,
)
return 1
if __name__ == "__main__":
sys.exit(main(sys.argv))