use clap::{Parser, Subcommand, ValueEnum};
use indicatif::{HumanBytes, HumanCount, ProgressBar, ProgressDrawTarget, ProgressStyle};
use olean_export::{
blean::{self, Blean},
ndjson::{self, Ndjson},
resolve, search_path, statement, Env, Exporter, NameId, Sink, Summary,
};
use std::{
cell::Cell,
cmp::Reverse,
error::Error,
fs::File,
io::{self, BufWriter, Write},
mem,
path::PathBuf,
process::ExitCode,
rc::Rc,
thread,
time::{Duration, Instant},
};
#[cfg(feature = "mimalloc")]
#[global_allocator]
static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc;
#[derive(Debug, Parser)]
#[command(
version,
args_conflicts_with_subcommands = true,
subcommand_negates_reqs = true,
after_help = "Run under `lake env` so LEAN_PATH covers your build and the toolchain:\n lake env olean-export Mathlib -o mathlib.ndjson"
)]
struct Cli {
#[command(subcommand)]
command: Option<Command>,
#[arg(required = true, value_name = "MODULE")]
modules: Vec<String>,
#[arg(short, long = "const", value_name = "NAME")]
consts: Vec<String>,
#[arg(short, long, value_name = "FILE")]
output: Option<PathBuf>,
#[arg(short = 'L', long = "search", value_name = "DIR", global = true)]
search: Vec<PathBuf>,
#[arg(short, long, value_name = "N", global = true)]
jobs: Option<usize>,
#[arg(short, long, value_enum, default_value_t = Format::Ndjson)]
format: Format,
#[arg(short, long)]
quiet: bool,
#[arg(long)]
rss: bool,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, ValueEnum)]
enum Format {
Ndjson,
Blean,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, ValueEnum)]
enum Shape {
Text,
Ndjson,
Blean,
}
#[derive(Debug, Subcommand)]
enum Command {
Inspect {
#[arg(value_name = "MODULE|FILE")]
target: String,
#[arg(short, long)]
consts: bool,
},
Convert {
input: PathBuf,
#[arg(short, long, value_name = "FILE")]
output: Option<PathBuf>,
},
Statement {
#[arg(required = true, value_name = "MODULE")]
modules: Vec<String>,
#[arg(short, long = "const", value_name = "NAME", required = true)]
consts: Vec<String>,
#[arg(short, long, value_delimiter = ',', value_name = "MODULES")]
trust: Vec<String>,
#[arg(short, long, value_enum, default_value_t = Shape::Text)]
format: Shape,
#[arg(short, long, value_name = "N", default_value_t = 2000)]
max: usize,
#[arg(short, long, value_name = "FILE")]
output: Option<PathBuf>,
},
Compare {
a: PathBuf,
b: PathBuf,
#[arg(short = 'n', long, value_name = "N", default_value_t = 20)]
limit: usize,
},
}
struct Counting<W> {
inner: W,
bytes: Rc<Cell<u64>>,
}
impl<W: Write> Write for Counting<W> {
fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
let n = self.inner.write(buf)?;
self.bytes.set(self.bytes.get() + n as u64);
Ok(n)
}
fn flush(&mut self) -> io::Result<()> {
self.inner.flush()
}
}
fn main() -> ExitCode {
let cli = Cli::parse();
let result = match &cli.command {
Some(Command::Inspect { target, consts }) => inspect(&cli, target, *consts),
Some(Command::Convert { input, output }) => convert(input, output.as_ref()),
Some(Command::Compare { a, b, limit }) => compare(a, b, *limit),
Some(Command::Statement {
modules,
consts,
trust,
format,
max,
output,
}) => statement(&cli, modules, consts, trust, *format, *max, output.as_ref()),
None => run(&cli),
};
match result {
Ok(()) => ExitCode::SUCCESS,
Err(e) if broken_pipe(&*e) => ExitCode::SUCCESS,
Err(e) => {
eprintln!("error: {e}");
ExitCode::FAILURE
}
}
}
fn broken_pipe(e: &(dyn Error + 'static)) -> bool {
let io = e
.downcast_ref::<io::Error>()
.or_else(|| match e.downcast_ref() {
Some(olean_export::Error::Io(io)) => Some(io),
_ => None,
});
io.is_some_and(|io| io.kind() == io::ErrorKind::BrokenPipe)
}
#[cfg(unix)]
#[allow(unsafe_code)]
fn peak_rss() -> Option<u64> {
let mut u: libc::rusage = unsafe { mem::zeroed() };
if unsafe { libc::getrusage(libc::RUSAGE_SELF, &raw mut u) } != 0 {
return None;
}
let n = u64::try_from(u.ru_maxrss).ok()?;
Some(if cfg!(target_os = "macos") {
n
} else {
n << 10
})
}
#[cfg(not(unix))]
const fn peak_rss() -> Option<u64> {
None
}
fn rss(on: bool) -> String {
match peak_rss() {
Some(n) if on => format!(", peak {}", HumanBytes(n)),
_ => String::new(),
}
}
fn bar(quiet: bool, len: Option<u64>, template: &str) -> ProgressBar {
let pb = len.map_or_else(ProgressBar::new_spinner, ProgressBar::new);
if quiet {
pb.set_draw_target(ProgressDrawTarget::hidden());
}
pb.set_style(
ProgressStyle::with_template(template)
.expect("valid template")
.progress_chars("=> "),
);
pb.enable_steady_tick(Duration::from_millis(100));
pb
}
type Res = Result<(), Box<dyn Error + Send + Sync>>;
fn search(cli: &Cli) -> Result<Vec<PathBuf>, Box<dyn Error + Send + Sync>> {
let dirs: Vec<PathBuf> = cli.search.iter().cloned().chain(search_path()).collect();
if dirs.is_empty() {
return Err("no search path: run under `lake env` or pass -L DIR".into());
}
Ok(dirs)
}
fn inspect(cli: &Cli, target: &str, list: bool) -> Res {
let path = PathBuf::from(target);
let path = if path.is_file() {
path
} else {
resolve(&search(cli)?, target)?
};
let s = Summary::open(&path)?;
let flag = |on: bool, s: &'static str| if on { s } else { "" };
let mut out = io::stdout().lock();
writeln!(
out,
"Lean {} ({}){}{}",
s.header.version,
s.header.githash,
flag(s.header.gmp, ", gmp"),
flag(s.is_module, ", module")
)?;
for (p, n) in &s.parts {
writeln!(out, " {:>11} {}", HumanBytes(*n).to_string(), p.display())?;
}
writeln!(out, "\nimports ({})", s.imports.len())?;
for i in &s.imports {
writeln!(
out,
" {}{}{}{}",
i.module,
flag(i.all, " all"),
flag(!i.exported, " private"),
flag(i.meta, " meta")
)?;
}
let mut kinds: Vec<(&str, usize)> = Vec::new();
for d in &s.consts {
match kinds.iter_mut().find(|k| k.0 == d.kind) {
Some(k) => k.1 += 1,
None => kinds.push((d.kind, 1)),
}
}
kinds.sort_by_key(|k| Reverse(k.1));
let kinds: Vec<String> = kinds.iter().map(|(k, n)| format!("{n} {k}")).collect();
writeln!(
out,
"\nconstants ({}): {}",
s.consts.len(),
kinds.join(", ")
)?;
if list {
let mut consts: Vec<_> = s.consts.iter().collect();
consts.sort_by(|a, b| a.name.cmp(&b.name));
for d in consts {
let line = format!(" {:<9} {} {}", d.kind, d.name, d.safety);
writeln!(out, "{}", line.trim_end())?;
}
}
let mut sections: Vec<_> = s.sections.iter().filter(|x| x.items > 0).collect();
sections.sort_by_key(|x| Reverse(x.bytes));
let total = s.bytes().max(1);
let row = |out: &mut io::StdoutLock<'_>, bytes: u64, items: String, name: &str| {
#[allow(clippy::cast_precision_loss)]
let share = 100.0 * bytes as f64 / total as f64;
let size = HumanBytes(bytes).to_string();
writeln!(out, "{size:>11} {share:>5.1}% {items:>6} {name}")
};
writeln!(
out,
"\n{:>11} {:>6} {:>6} section",
"bytes", "share", "items"
)?;
for x in §ions {
row(&mut out, x.bytes, x.items.to_string(), &x.name)?;
}
let reached: u64 = s.sections.iter().map(|x| x.bytes).sum();
row(
&mut out,
total.saturating_sub(reached),
String::new(),
"(headers and unreached)",
)?;
Ok(())
}
fn run(cli: &Cli) -> Res {
let start = Instant::now();
let search = search(cli)?;
let roots: Vec<&str> = cli.modules.iter().map(String::as_str).collect();
let pb = bar(
cli.quiet,
None,
"{spinner:.cyan} decoding [{elapsed}] {msg}",
);
let jobs = jobs(cli);
pb.set_message(format!("finding modules, {jobs} threads"));
let tick = |modules: usize, consts: usize| {
pb.set_message(format!(
"{} modules, {} constants{}",
HumanCount(modules as u64),
HumanCount(consts as u64),
rss(cli.rss)
));
};
let env = if cli.consts.is_empty() {
Env::load_with(&search, &roots, jobs, &tick)?
} else {
let consts: Vec<&str> = cli.consts.iter().map(String::as_str).collect();
Env::load_cone(&search, &roots, &consts, jobs, &tick)?
};
pb.finish_and_clear();
let loaded = start.elapsed();
let targets = if cli.consts.is_empty() {
env.roots().collect()
} else {
cli.consts
.iter()
.map(|n| {
env.find_name(n)
.ok_or_else(|| format!("unknown constant {n}"))
})
.collect::<Result<Vec<_>, _>>()?
};
let sink: Box<dyn Write> = match &cli.output {
Some(path) => Box::new(File::create(path)?),
None => Box::new(io::stdout().lock()),
};
let bytes = Rc::new(Cell::new(0));
let out = BufWriter::with_capacity(
1 << 20,
Counting {
inner: sink,
bytes: Rc::clone(&bytes),
},
);
let pb = bar(
cli.quiet,
Some(targets.len() as u64),
"{spinner:.cyan} emitting [{elapsed}] [{bar:30.cyan/blue}] {human_pos}/{human_len} {msg}",
);
let tick = |i: usize| {
pb.set_position(i as u64);
pb.set_message(format!("{}{}", HumanBytes(bytes.get()), rss(cli.rss)));
};
let counts = match cli.format {
Format::Ndjson => export(Exporter::new(&env, out), &targets, tick)?,
Format::Blean => export(Exporter::with_sink(&env, Blean::new(out)), &targets, tick)?,
};
pb.finish_and_clear();
if !cli.quiet {
let ms = start.elapsed().saturating_sub(loaded).as_millis().max(1);
let rate = u64::try_from(u128::from(bytes.get()) * 1000 / ms).unwrap_or(u64::MAX);
eprintln!(
"{} modules, {} constants decoded in {:.2}s\n{} of {} ({} names, {} levels, {} exprs) in {:.2}s total, {}/s{}",
HumanCount(env.modules.len() as u64),
HumanCount(env.consts.len() as u64),
loaded.as_secs_f64(),
HumanBytes(bytes.get()),
match cli.format {
Format::Ndjson => "NDJSON",
Format::Blean => "blean",
},
HumanCount(counts.names.into()),
HumanCount(counts.levels.into()),
HumanCount(counts.exprs.into()),
start.elapsed().as_secs_f64(),
HumanBytes(rate),
rss(cli.rss),
);
}
Ok(())
}
fn export<S: Sink>(
mut ex: Exporter<'_, S>,
targets: &[NameId],
tick: impl Fn(usize),
) -> Result<olean_export::Counts, Box<dyn Error + Send + Sync>> {
ex.meta()?;
for (i, &c) in targets.iter().enumerate() {
ex.constant(c)?;
if i % 256 == 0 {
tick(i);
}
}
let counts = ex.counts();
ex.finish()?;
Ok(counts)
}
fn jobs(cli: &Cli) -> usize {
cli.jobs
.unwrap_or_else(|| thread::available_parallelism().map_or(1, usize::from))
}
fn statement(
cli: &Cli,
modules: &[String],
consts: &[String],
trust: &[String],
format: Shape,
max: usize,
output: Option<&PathBuf>,
) -> Res {
let search = search(cli)?;
let roots: Vec<&str> = modules.iter().map(String::as_str).collect();
let env = Env::load_statements(&search, &roots, jobs(cli), &|_, _| {})?;
let targets = consts
.iter()
.map(|n| {
env.find_name(n)
.ok_or_else(|| format!("unknown constant {n}"))
})
.collect::<Result<Vec<_>, _>>()?;
let sink: Box<dyn Write + Send> = match output {
Some(path) => Box::new(File::create(path)?),
None => Box::new(io::stdout()),
};
let mut out = BufWriter::with_capacity(1 << 20, sink);
match format {
Shape::Text => statement::report(&env, &search, &targets, trust, max, &mut out)?,
Shape::Ndjson | Shape::Blean => {
let cone: Vec<NameId> = env
.statement(&targets)?
.into_iter()
.map(|(_, c)| c)
.collect();
if format == Shape::Ndjson {
export(
Exporter::new(&env, out.by_ref()).statements(),
&cone,
|_| {},
)?;
} else {
let sink = Blean::new(out.by_ref());
export(Exporter::with_sink(&env, sink).statements(), &cone, |_| {})?;
}
}
}
out.flush()?;
Ok(())
}
fn mmap(path: &PathBuf) -> io::Result<memmap2::Mmap> {
let file = File::open(path)?;
#[allow(unsafe_code)]
unsafe {
memmap2::Mmap::map(&file)
}
}
fn compare(a: &PathBuf, b: &PathBuf, limit: usize) -> Res {
let d = olean_export::compare(&mmap(a)?, &mmap(b)?)?;
let mut out = io::stdout().lock();
let mut list = |what: &str, names: &[String]| -> io::Result<()> {
for n in names.iter().take(limit) {
writeln!(out, "{what}: {n}")?;
}
if names.len() > limit {
writeln!(out, "{what}: ... {} more", names.len() - limit)?;
}
Ok(())
};
list(&format!("only in {}", a.display()), &d.only_a)?;
list(&format!("only in {}", b.display()), &d.only_b)?;
let differ: Vec<String> = d.differ.iter().map(|(n, w)| format!("{n} ({w})")).collect();
list("differs", &differ)?;
list(&format!("duplicate in {}", a.display()), &d.dups.0)?;
list(&format!("duplicate in {}", b.display()), &d.dups.1)?;
writeln!(
out,
"{} / {} declarations: {} only in the first, {} only in the second, {} differ, {} duplicate",
d.decls.0,
d.decls.1,
d.only_a.len(),
d.only_b.len(),
d.differ.len(),
d.dups.0.len() + d.dups.1.len()
)?;
if d.is_empty() {
Ok(())
} else {
Err("the exports differ".into())
}
}
fn convert(input: &PathBuf, output: Option<&PathBuf>) -> Res {
let map = mmap(input)?;
let sink: Box<dyn Write> = match output {
Some(path) => Box::new(File::create(path)?),
None => Box::new(io::stdout().lock()),
};
let out = BufWriter::with_capacity(1 << 20, sink);
if blean::sniff(&map) {
blean::read(&map, Ndjson::new(out))?;
} else {
ndjson::read(&map[..], Blean::new(out))?;
}
Ok(())
}