from pathlib import Path
import json
import re
import sys
from urllib.parse import unquote, urlsplit
ROOT = Path(__file__).resolve().parents[1]
LINK = re.compile(r"\[([^\]]*)\]\(([^)]+)\)")
IDENTIFIER = re.compile(r"`([A-Za-z_][A-Za-z_0-9]*)`")
def markdown_prose(text: str) -> str:
lines = []
fence = None
for line in text.splitlines(keepends=True):
marker = re.match(r"^ {0,3}(`{3,}|~{3,})(.*)$", line)
if fence is None and marker:
fence = marker[1]
elif fence and marker and marker[1][0] == fence[0]:
if len(marker[1]) >= len(fence) and not marker[2].strip():
fence = None
elif fence is None:
lines.append(line)
continue
lines.append("\n" if line.endswith("\n") else "")
return re.sub(
r"<!--.*?(?:-->|$)",
lambda match: re.sub(r"[^\n]", " ", match[0]),
"".join(lines),
flags=re.DOTALL,
)
def markdown_anchors(text: str) -> set[str]:
prose = markdown_prose(text)
anchors = set(re.findall(r'<a\s+id=[\'"]([^\'"]+)[\'"]', prose))
used = set()
for heading in re.findall(r"^ {0,3}#{1,6}\s+(.+?)\s*$", prose, re.MULTILINE):
heading = re.sub(r"\s+#+$", "", heading)
heading = LINK.sub(lambda match: match[1], heading)
heading = re.sub(r"<[^>]*>", "", heading)
slug = re.sub(r"[^\w\- ]", "", heading.lower()).replace(" ", "-")
anchor = slug
suffix = 0
while anchor in used:
suffix += 1
anchor = f"{slug}-{suffix}"
used.add(anchor)
anchors.add(anchor)
return anchors
RUST_LITERAL = re.compile(
r'//[^\n]*|/\*|\b(?:br|cr|r)(#*)"|(?:b|c)?"(?:\\.|[^"\\])*"'
r"|b?'(?:\\.|[^'\\])'",
re.DOTALL,
)
def rust_code(text: str) -> str:
chunks = []
start = 0
while match := RUST_LITERAL.search(text, start):
chunks.append(text[start:match.start()])
end = match.end()
if match[0] == "/*":
depth = 1
while depth and (comment := re.search(r"/\*|\*/", text[end:])):
depth += 1 if comment[0] == "/*" else -1
end += comment.end()
if depth:
end = len(text)
elif match[1] is not None:
terminator = '"' + match[1]
closing = text.find(terminator, end)
end = len(text) if closing == -1 else closing + len(terminator)
chunks.append(re.sub(r"[^\n]", " ", text[match.start():end]))
start = end
chunks.append(text[start:])
return "".join(chunks)
def rust_references(text: str) -> tuple[set[str], set[str]]:
code = rust_code(text)
visibility = r"(?:pub(?:\([^)]*\))?\s+)?"
names = set(re.findall(
r"^\s*" + visibility + r"(?:(?:async|unsafe|const)\s+)*"
r"(?:fn|struct|enum|type|trait|const|static|mod)\s+([A-Za-z_]\w*)\b",
code, re.MULTILINE,
))
tests = set(re.findall(
r"#\[\s*test\s*\]\s*(?:#\[[^]\n]*\]\s*)*" + visibility
+ r"(?:async\s+)?fn\s+([A-Za-z_]\w*)\s*\(", code,
))
return names, tests
def check_links(root: Path, docs: list[Path]) -> tuple[list[str], dict[str, int]]:
errors = []
counts = dict(links=0, anchors=0, symbols=0, tests=0)
source_cache = {}
for path in sorted(docs):
prose = markdown_prose(path.read_text(encoding="utf-8"))
for match in LINK.finditer(prose):
label, target = match.groups()
target = target.strip().split(' "', 1)[0].strip("<>")
url = urlsplit(target)
if url.scheme or url.netloc:
continue
counts["links"] += 1
location = f"{path.relative_to(root)}:{prose.count(chr(10), 0, match.start()) + 1}"
destination = (path.parent / unquote(url.path)).resolve() if url.path else path
if not destination.exists():
errors.append(f"{location}: missing target {target}")
continue
if destination.suffix == ".md" and url.fragment:
counts["anchors"] += 1
if unquote(url.fragment) not in markdown_anchors(destination.read_text(encoding="utf-8")):
errors.append(f"{location}: missing Markdown anchor {target}")
identifier = IDENTIFIER.fullmatch(label)
if destination.suffix != ".rs" or identifier is None:
continue
name = identifier[1]
if destination not in source_cache:
source_cache[destination] = rust_references(destination.read_text(encoding="utf-8"))
symbols, tests = source_cache[destination]
is_test = (root / "tests") in destination.parents
counts["tests" if is_test else "symbols"] += 1
if name not in (tests if is_test else symbols):
kind = "#[test] function" if is_test else "Rust declaration"
errors.append(f"{location}: missing {kind} {name} in {target}")
return errors, counts
LEAN_MODULE_NAME = re.compile(r"[^\W\d][\w']*(?:\.[^\W\d][\w']*)*")
def lean_imports(text: str) -> list[str]:
position = 0
def token() -> str | None:
nonlocal position
while position < len(text):
if text[position].isspace():
position += 1
elif text.startswith("--", position):
end = text.find("\n", position)
position = len(text) if end == -1 else end + 1
elif text.startswith(("/--", "/-!"), position):
return None elif text.startswith("/-", position):
depth = 1
position += 2
while position < len(text) and depth:
if text.startswith("/-", position):
depth += 1
position += 2
elif text.startswith("-/", position):
depth -= 1
position += 2
else:
position += 1
else:
match = LEAN_MODULE_NAME.match(text, position)
if match is None:
return None
end = match.end()
if end < len(text) and not (
text[end].isspace() or text.startswith(("--", "/-"), end)
):
return None position = end
return match[0]
return None
current = token()
if current == "module":
current = token()
if current == "prelude":
current = token()
imports = []
while current is not None:
if current == "public":
current = token()
if current == "meta":
current = token()
if current != "import":
break
dependency = token()
if dependency == "all":
dependency = token()
if dependency is None:
break
imports.append(dependency)
current = token()
return imports
def check_lean(root: Path) -> tuple[list[str], int]:
errors = []
lean = root / "lean"
sources = {"Qleisli": lean / "Qleisli.lean"}
for path in (lean / "Qleisli").rglob("*.lean"):
sources[".".join(path.relative_to(lean).with_suffix("").parts)] = path
visited = set()
def visit(module: str) -> None:
if module in visited:
return
visited.add(module)
if module not in sources or not sources[module].is_file():
errors.append(f"missing Lean module: {module}")
return
for dependency in lean_imports(sources[module].read_text(encoding="utf-8")):
if dependency == "Qleisli" or dependency.startswith("Qleisli."):
visit(dependency)
visit("Qleisli")
for module in sorted(sources.keys() - visited):
errors.append(f"Lean module absent from root import/audit: {module}")
return errors, len(sources)
def render_status(root: Path, *, inventory_only: bool = False) -> str:
data = json.loads((root / "docs/project-status.json").read_text(encoding="utf-8"))
if set(data) != {"format", "release_state", "current", "milestones", "inventory"} or data["format"] != 3:
raise ValueError("unsupported project-status format or fields")
versions = []
for manifest in ["Cargo.toml", "lean/lakefile.toml", "lean-kernel/lakefile.toml", "python/pyproject.toml"]:
match = re.search(r'^version = "([^"]+)"$', (root / manifest).read_text(), re.MULTILINE)
if not match:
raise ValueError(f"missing project version in {manifest}")
versions.append(match[1])
if len(set(versions)) != 1:
raise ValueError("Rust, Lean and Python project versions differ")
python_version = re.search(r'^__version__ = "([^"]+)"$', (root / "python/qleisli/__init__.py").read_text(), re.MULTILINE)
if python_version is None or python_version[1] != versions[0]:
raise ValueError("Python runtime version differs from project manifests")
for notice in ["LICENSE", "NOTICE"]:
if (root / "python" / notice).read_bytes() != (root / notice).read_bytes():
raise ValueError(f"Python package {notice} differs from repository {notice}")
def cell(value):
if not isinstance(value, str) or not value.strip() or "\n" in value or "\r" in value:
raise ValueError("status cells must be nonempty single-line strings")
return value.replace("|", "\\|")
def rows(items, fields):
if not isinstance(items, list) or not items:
raise ValueError("status tables must be nonempty lists")
result = []
for item in items:
if not isinstance(item, dict) or set(item) != set(fields):
raise ValueError("status row has missing or unknown fields")
result.append("| " + " | ".join(cell(item[key]) for key in fields) + " |")
return result
milestone_rows = rows(data["milestones"], ["id", "state", "evidence", "open"])
if [item["id"] for item in data["milestones"]] != [f"M{i}" for i in range(6)]:
raise ValueError("milestones must contain M0 through M5 once, in order")
inventory_rows = rows(data["inventory"], ["rule", "implementation", "tests", "proof"])
current_rows = rows(data["current"], ["topic", "state", "detail"])
if inventory_only:
return "\n".join([
"# Finite rule and implementation inventory", "",
"<!-- Generated by scripts/check_docs.py --write-status; edit docs/project-status.json. -->", "",
"For the current overview, read [current status](current-status.md).", "",
"This scoped inventory identifies implementation boundaries, existing regressions",
"and proved/open obligations. It is not a fresh execution report or complete soundness audit.", "",
"| Rule | Implementation boundary | Existing regression evidence | Proof status and gap |",
"| --- | --- | --- | --- |", *inventory_rows, "",
])
return "\n".join([
"# Current project status and finite-rule inventory", "",
"<!-- Generated by scripts/check_docs.py --write-status; edit docs/project-status.json. -->", "",
f"Selected product version: **{versions[0]}**. {cell(data['release_state'])}",
f"See the [validation/publication record](releases/v{versions[0]}.md).", "",
"## Where we are now", "",
"| Area | Current state | Details |",
"| --- | --- | --- |", *current_rows, "",
"Historical narrative pages are retired; lookup uses Git history and published tags.", "",
"This view is generated from [project-status.json](project-status.json) and the project manifests.",
"It centralizes current scheduling states and an initial rule/implementation/test/proof inventory.",
"The inventory names inspected boundaries and existing regressions; it is not a complete",
"conformance audit, a test execution report or a proof of Rust acceptance soundness.",
"Execution evidence remains beside the source and [validation fixtures](../tests/fixtures/authoring_sessions/README.md).", "",
"## Active milestone states", "",
"| Milestone | State | Evidence / direction | Remaining work |",
"| --- | --- | --- | --- |", *milestone_rows, "",
"## Finite rule inventory", "",
"Read the [separate implementation/test/proof inventory](rule-inventory.md).", "",
"[M0–M5 and legacy IDs](v0x-roadmap.md#legacy-id-mapping) separate active scheduling",
"from requirements, theorem names, acceptance criteria and historical release steps.",
"Regenerate with `python3 scripts/check_docs.py --write-status`; ordinary checks reject drift.", "",
])
def check_status(root: Path, write: bool = False) -> list[str]:
try:
views = {"docs/current-status.md": render_status(root),
"docs/rule-inventory.md": render_status(root, inventory_only=True)}
for name, expected in views.items():
path = root / name
if write:
path.write_text(expected, encoding="utf-8")
elif not path.exists() or path.read_text(encoding="utf-8") != expected:
return [f"{name} is stale; run python3 scripts/check_docs.py --write-status"]
except (OSError, ValueError, TypeError, KeyError) as failure:
return [f"project status: {failure}"]
return []
def markdown_paths(root: Path) -> list[Path]:
docs = [*root.glob("*.md"), *root.joinpath("docs").rglob("*.md")]
docs += list(root.joinpath("lean").glob("*.md"))
docs += list(root.joinpath("lean-kernel").glob("*.md"))
docs += list(root.joinpath("research").glob("*/README.md"))
for directory in ["examples", "corpus", "tests/fixtures"]:
docs += list(root.joinpath(directory).rglob("*.md"))
return docs
def main() -> int:
if sys.argv[1:] not in ([], ["--write-status"]):
print("usage: check_docs.py [--write-status]", file=sys.stderr)
return 2
status_errors = check_status(ROOT, write=sys.argv[1:] == ["--write-status"])
errors, counts = check_links(ROOT, markdown_paths(ROOT))
errors.extend(status_errors)
lean_errors, modules = check_lean(ROOT)
errors.extend(lean_errors)
from check_lean_kernel import check_kernel
kernel_errors, kernel_modules = check_kernel(ROOT)
errors.extend(kernel_errors)
modules += kernel_modules
if errors:
print("\n".join(errors), file=sys.stderr)
return 1
print(
f"Checked {counts['links']} local links, {counts['anchors']} Markdown anchors, "
f"{counts['symbols']} Rust declarations, {counts['tests']} Rust tests, "
f"and {modules} Lean root modules."
)
return 0
if __name__ == "__main__":
sys.exit(main())