import json
import os
import re
import subprocess
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parent.parent
DOCTOR_KINDS = [
(r"_is_pad|source_fps|^next\.", "auxiliary_column"),
(r"^\.\.\.and \d+ more|episode\(s\) flagged$", None),
(r"consecutive identical|frozen", "action_frozen"),
(r"sudden large action jumps|action jump\(s\)", "action_jump"),
(r"distribution shift", "distribution_shift"),
(r"stuck/static", "stuck_in_some_episodes"),
(r"ALL \d+ dimensions constant", "all_dimensions_constant"),
(r"zero variance", "constant_dimension"),
(r"% of values at (minimum|maximum)", "clipped_dimension"),
(r"extreme outlier", "outlier_values"),
(r"length mismatch", "episode_length_mismatch"),
(r"abnormally long episode", "long_episode"),
(r"shorter than chunk_size", "shorter_than_chunk_size"),
(r"total_frames=\d+ but actual", "total_frames_mismatch"),
(r"Global index column is not sequential", "index_not_sequential"),
(r"Missing episode indices", "missing_episode_indices"),
(r"No stats\.json -- normalization", "no_stats_for_training"),
(r"stats\.json not found", "stats_json_missing"),
(r"has \d+ frames", "video_frame_count"),
]
def episodes_in(message):
m = re.search(r"[Ee]pisode (\d+)", message)
if m:
return [int(m.group(1))]
m = re.search(r"abnormally long episode\(s\).*: \[([\d, ]+)\]", message)
if m:
return [int(x) for x in m.group(1).split(",")]
m = re.search(r"Missing episode indices: \[([\d, ]+)", message)
if m:
return [None]
m = re.search(r"episode_(\d+)\.mp4", message)
if m:
return [int(m.group(1))]
m = re.search(r"\[\((\d+), 'data=", message)
if m:
return [int(x) for x in re.findall(r"\((\d+), 'data=", message)]
return [None]
def run(cmd):
out = subprocess.run(cmd, capture_output=True, text=True)
return json.loads(out.stdout) if out.stdout.strip() else None
def doctor(tool, path):
d = run([tool, "check", "--json", str(path)])
if d is None:
return None
version = subprocess.run([tool, "--version"], capture_output=True, text=True).stdout.split()[-1]
findings = set()
for check in d["checks"]:
for m in check["messages"]:
if m["severity"] not in ("WARN", "FAIL"):
continue
kind = next((k for pat, k in DOCTOR_KINDS if re.search(pat, m["message"])), "unclassified")
if kind is None:
continue
if kind == "stats_json_missing" and d["codebase_version"].startswith("v2.1"):
kind = "stats_json_missing_v2_1"
for ep in episodes_in(m["message"]):
findings.add((kind, ep, m["severity"].lower()))
return {"version": version, "findings": sorted(findings, key=str)}
def physlint(tool, path):
d = run([tool, "check", "-f", "json", str(path)])
if d is None:
return None
findings = set()
for r in d["results"]:
for f in r["findings"]:
ep = f["location"].get("episode")
ep = int(re.sub(r"\D", "", ep)) if ep else None
findings.add((r["rule_id"], ep, f["severity"]))
return {"version": d["physlint_version"], "findings": sorted(findings, key=str)}
def main():
tools = {"lerobot-doctor": (os.environ.get("LERO_DOCTOR"), doctor), "physlint": (os.environ.get("PHYSLINT"), physlint)}
out_dir = ROOT / "tests" / "goldens"
out_dir.mkdir(exist_ok=True)
for line in (ROOT / "scripts" / "datasets.tsv").read_text().splitlines():
if line.startswith("#") or not line.strip():
continue
name = line.split("\t")[0]
path = ROOT / "datasets" / name
if not path.exists():
print(f"skipping {name}: not downloaded", file=sys.stderr)
continue
golden = {"dataset": name, "tools": {}}
for tool_name, (exe, fn) in tools.items():
if not exe:
print(f"{tool_name}: set its path in the environment", file=sys.stderr)
continue
r = fn(exe, path)
if r is None:
print(f"{name}: {tool_name} gave no output", file=sys.stderr)
continue
golden["tools"][tool_name] = {
"version": r["version"],
"findings": [{"kind": k, "episode": e, "severity": s} for k, e, s in r["findings"]],
}
(out_dir / f"{name}.json").write_text(json.dumps(golden, indent=1) + "\n")
print(name, {t: len(v["findings"]) for t, v in golden["tools"].items()})
if __name__ == "__main__":
main()