import ast
import json
import pathlib
import re
import sys
import rustai
PROBES = {
"Client": lambda kw: rustai.Client(**{k: _sample(k) for k in kw}),
"research": lambda kw: rustai.research(
"q", **{**{k: _sample(k) for k in kw}, "providers": []}
),
"slim": lambda kw: rustai.slim("q", [], **{k: _sample(k) for k in kw}),
"extract": lambda kw: rustai.extract("<p>x</p>", **{k: _sample(k) for k in kw}),
"extract_many": lambda kw: rustai.extract_many([], **{k: _sample(k) for k in kw}),
"chunk_many": lambda kw: rustai.chunk_many([], **{k: _sample(k) for k in kw}),
"count_tokens": None,
"tokenize": None,
"density": None,
"canonical_url": None,
"parse_feed": None,
"parse_sitemap": None,
}
SAMPLES = {
"providers": ["duckduckgo"], "concurrency": 4, "timeout": 5.0,
"impersonate": "chrome", "respect_robots": True, "respect_crawl_delay": True,
"per_host_delay": 0.0, "retries": 0, "max_body_bytes": 1024,
"accept_language": "en", "browser_fallback": False, "contact_email": None,
"proxies": None, "max_retry_after": 1.0, "cookie_file": None,
"max_tokens": 128, "max_tokens_per_source": 64, "diversity": 0.3,
"include_links": True, "include_images": False, "include_tables": True,
"index_mode": "auto", "limit": 3, "max_results": 3, "max_sources": 1,
"include_breadcrumbs": True, "query_vector": None, "unit_vectors": None,
"semantic_weight": 0.5, "target_tokens": 512, "overlap_tokens": 64,
"min_tokens": 24, "urls": None, "url": None, "strict": False,
"raise_on_error": False,
}
def _sample(name: str):
if name not in SAMPLES:
raise KeyError(name)
return SAMPLES[name]
def guarded(tree: ast.AST) -> set[int]:
lines = set()
for node in ast.walk(tree):
if not isinstance(node, ast.Try):
continue
catches = any(
isinstance(h.type, ast.Name) and h.type.id == "TypeError"
or isinstance(h.type, ast.Tuple)
and any(isinstance(e, ast.Name) and e.id == "TypeError" for e in h.type.elts)
for h in node.handlers
)
if catches:
for child in ast.walk(node):
if hasattr(child, "lineno"):
lines.add(child.lineno)
return lines
def calls(source: str):
tree = ast.parse(source)
skip = guarded(tree)
for node in ast.walk(tree):
if not isinstance(node, ast.Call):
continue
f = node.func
if not (isinstance(f, ast.Attribute) and isinstance(f.value, ast.Name)
and f.value.id == "rustai"):
continue
if node.lineno in skip:
continue
yield f.attr, [k.arg for k in node.keywords if k.arg], node.lineno
def blocks(path: pathlib.Path):
if path.suffix == ".ipynb":
cells = [c for c in json.loads(path.read_text(encoding="utf-8"))["cells"]
if c["cell_type"] == "code"]
for number, cell in enumerate(cells, start=1):
yield f"cell {number}", "".join(cell["source"])
return
text = path.read_text(encoding="utf-8")
for number, match in enumerate(
re.finditer(r"^```(?:python|py)\n(.*?)^```", text, re.M | re.S), start=1
):
line = text[: match.start()].count("\n") + 1
yield f"block {number} at line {line}", match.group(1)
def main() -> None:
which = sys.argv[1] if len(sys.argv) > 1 else "all"
if which not in ("all", "notebooks", "docs"):
sys.exit(f"usage: {sys.argv[0]} [all|notebooks|docs]")
paths = []
if which in ("all", "notebooks"):
paths += sorted(pathlib.Path("notebooks").glob("*.ipynb"))
if which in ("all", "docs"):
paths += [p for p in (pathlib.Path("README.md"), pathlib.Path("CONTRIBUTING.md"))
if p.exists()]
if not paths:
sys.exit("nothing to check")
print(f"checking against rustai {rustai.__version__}\n")
problems = 0
for path in paths:
print(f"{path.name}")
found = []
for label, raw in blocks(path):
source = "\n".join(
(l[: len(l) - len(l.lstrip())] + "pass")
if l.lstrip().startswith(("%", "!")) else l
for l in raw.splitlines()
)
try:
found.extend(calls(source))
except SyntaxError:
if path.suffix == ".ipynb":
print(f" {label}: could not be parsed")
problems += 1
for name, keywords, line in found:
if not hasattr(rustai, name):
print(f" line {line}: rustai.{name} does not exist")
problems += 1
continue
probe = PROBES.get(name)
if probe is None:
continue
try:
probe(keywords)
except TypeError as error:
print(f" line {line}: rustai.{name}({', '.join(keywords)}) -> {error}")
problems += 1
except KeyError as error:
print(f" line {line}: rustai.{name}(... {error} ...) - no such argument, "
f"or a new one that needs a value in SAMPLES")
problems += 1
except Exception:
pass
print()
if problems:
sys.exit(f"{problems} problem(s). Examples must run on the rustai they target.")
print("every rustai call in the docs and notebooks works on this version.")
if __name__ == "__main__":
main()