use cpp_parser::{CppParser, CppSyntaxKind, ParserConfig};
use std::path::{Path, PathBuf};
use std::time::Instant;
fn project() -> Option<PathBuf> {
let named = std::env::var_os("CPPLS_PROJECT_ROOT").map(PathBuf::from)?;
named.is_dir().then_some(named)
}
fn sources(root: &Path, limit: usize) -> Vec<PathBuf> {
let mut found = Vec::new();
let mut stack = vec![root.to_path_buf()];
while let Some(dir) = stack.pop() {
let Ok(entries) = std::fs::read_dir(&dir) else {
continue;
};
for entry in entries.flatten() {
let path = entry.path();
if path.is_dir() {
let name = path.file_name().unwrap_or_default().to_string_lossy();
if !matches!(name.as_ref(), "build" | "3rd" | ".git" | "target" | "out") {
stack.push(path);
}
continue;
}
let text = path.to_string_lossy();
let name = path.file_name().unwrap_or_default().to_string_lossy();
let extensionless = !name.contains('.');
if text.ends_with(".cpp")
|| text.ends_with(".h")
|| text.ends_with(".hpp")
|| text.ends_with(".hxx")
|| text.ends_with(".tcc")
|| text.ends_with(".inl")
|| extensionless
{
found.push(path);
if found.len() >= limit * 4 {
return found;
}
}
}
}
found.sort();
found
}
fn mbps(bytes: usize, took: std::time::Duration) -> f64 {
let seconds = took.as_secs_f64();
if seconds == 0.0 {
return 0.0;
}
bytes as f64 / (1024.0 * 1024.0) / seconds
}
fn weigh(root: &cpp_parser::CppSyntaxTree) -> (usize, usize, usize) {
let mut nodes = 0usize;
let mut tokens = 0usize;
let mut token_bytes = 0usize;
let mut stack = vec![root.get_green_root().clone()];
while let Some(node) = stack.pop() {
nodes += 1;
for child in node.children() {
match child {
rowan::NodeOrToken::Node(node) => stack.push(node.to_owned()),
rowan::NodeOrToken::Token(token) => {
tokens += 1;
token_bytes += token.text().len();
}
}
}
}
(nodes, tokens, token_bytes)
}
#[test]
#[ignore = "reads a real project; run with --ignored and CPPLS_PROJECT_ROOT"]
fn what_parsing_costs() {
let Some(root) = project() else {
eprintln!("skipped: set CPPLS_PROJECT_ROOT");
return;
};
let files = sources(&root, 60);
let mut total_bytes = 0usize;
let mut total_parse = std::time::Duration::ZERO;
let mut worst: Vec<(u128, String, usize)> = Vec::new();
for path in files.iter().take(60) {
let Ok(text) = std::fs::read_to_string(path) else {
continue;
};
if text.len() < 1024 {
continue;
}
total_bytes += text.len();
let started = Instant::now();
let tree = CppParser::parse(&text, ParserConfig::default());
let took = started.elapsed();
total_parse += took;
let errors = tree.get_errors().len();
worst.push((
took.as_micros(),
format!("{} ({errors} error(s))", path.display()),
text.len(),
));
}
worst.sort_by_key(|(micros, _, _)| std::cmp::Reverse(*micros));
let mut dense: Vec<(u128, String)> = worst
.iter()
.map(|(micros, which, bytes)| {
let per_kb = *micros as f64 * 1024.0 / (*bytes as f64).max(1.0);
(per_kb as u128, which.clone())
})
.collect();
dense.sort_by_key(|(per_kb, _)| std::cmp::Reverse(*per_kb));
println!(" slowest per KB:");
for (per_kb, which) in dense.iter().take(10) {
println!(" {per_kb:>6} µs/KB {which}");
}
println!(
"parsed {} file(s), {:.1} MB in {:?} — {:.2} MB/s",
worst.len(),
total_bytes as f64 / (1024.0 * 1024.0),
total_parse,
mbps(total_bytes, total_parse)
);
for (micros, which, bytes) in worst.iter().take(5) {
println!(
" {:>7.1} ms {:>6.1} MB/s {}",
*micros as f64 / 1000.0,
mbps(*bytes, std::time::Duration::from_micros(*micros as u64)),
which
);
}
let mut checked = 0usize;
let mut with_errors = 0usize;
let mut total_errors = 0usize;
let mut not_lossless: Vec<String> = Vec::new();
for path in files.iter().take(120) {
let Ok(text) = std::fs::read_to_string(path) else {
continue;
};
if text.len() < 256 {
continue;
}
checked += 1;
let tree = CppParser::parse(&text, ParserConfig::default());
let errors = tree.get_errors().len();
if errors > 0 {
with_errors += 1;
total_errors += errors;
}
if tree.to_source_text() != text {
not_lossless.push(path.display().to_string());
}
}
println!(
"\nreal code: {checked} file(s), {with_errors} with syntax error(s) ({total_errors} in total), \
{} not lossless",
not_lossless.len()
);
for path in not_lossless.iter().take(5) {
println!(" lost bytes: {path}");
if let Ok(text) = std::fs::read_to_string(path) {
let tree = CppParser::parse(&text, ParserConfig::default());
let rebuilt = tree.to_source_text();
if rebuilt != text {
let file_lines: Vec<&str> = text.lines().collect();
let tree_lines: Vec<&str> = rebuilt.lines().collect();
let at = file_lines
.iter()
.zip(tree_lines.iter())
.position(|(left, right)| left != right)
.unwrap_or(file_lines.len().min(tree_lines.len()));
println!(
" the file is {} bytes in {} line(s), the tree gives {} bytes in {} line(s) — first \
difference at line {}",
text.len(),
file_lines.len(),
rebuilt.len(),
tree_lines.len(),
at + 1
);
for (label, lines) in [("file", &file_lines), ("tree", &tree_lines)] {
for offset in 0..4 {
let line = at.saturating_sub(1) + offset;
if let Some(text) = lines.get(line) {
println!(" {label} {}: {}", line + 1, text.escape_debug());
}
}
}
}
}
}
let biggest = files
.iter()
.filter_map(|path| std::fs::read_to_string(path).ok().map(|text| (path, text)))
.max_by_key(|(_, text)| text.len());
if let Some((path, text)) = biggest {
let tree = CppParser::parse(&text, ParserConfig::default());
let (nodes, tokens, token_bytes) = weigh(&tree);
let token_allocations = tokens * (24 + 16 + ((token_bytes / tokens.max(1)).max(8)));
let node_allocations = nodes * (24 + 16);
println!(
"\n{} — {:.2} MB, {nodes} node(s), {tokens} token(s), {} bytes of token text",
path.display(),
text.len() as f64 / (1024.0 * 1024.0),
token_bytes
);
println!(
" green tree ≈ {:.1} MB ({:.1}× the file): token allocations ≈ {:.1} MB, node allocations ≈ {:.1} MB",
(token_allocations + node_allocations) as f64 / (1024.0 * 1024.0),
(token_allocations + node_allocations) as f64 / text.len().max(1) as f64,
token_allocations as f64 / (1024.0 * 1024.0),
node_allocations as f64 / (1024.0 * 1024.0),
);
let mut big = String::new();
for path in files.iter() {
if big.len() > 4 * 1024 * 1024 {
break;
}
if let Ok(text) = std::fs::read_to_string(path) {
big.push_str(&text);
big.push('\n');
}
}
let started = Instant::now();
let lexed = cpp_parser::lex(&big, &cpp_parser::LexerConfig::default());
let lexing_took = started.elapsed();
println!(
"lexing alone: {} token(s) in {:?} — {:.2} MB/s",
lexed.0.len(),
lexing_took,
mbps(big.len(), lexing_took)
);
let started = Instant::now();
let big_tree = CppParser::parse(&big, ParserConfig::default());
let parse_took = started.elapsed();
let (big_nodes, big_tokens, _) = weigh(&big_tree);
println!(
"large unit: {:.2} MB, {big_nodes} node(s), {big_tokens} token(s), parsed in {:?} — {:.2} MB/s",
big.len() as f64 / (1024.0 * 1024.0),
parse_took,
mbps(big.len(), parse_took)
);
{
let mut cache = rowan::NodeCache::default();
let config = ParserConfig::new(cpp_parser::CppLanguageLevel::Cpp20, Some(&mut cache));
let started = Instant::now();
let cached = CppParser::parse(&big, config);
let took = started.elapsed();
let (cached_nodes, cached_tokens, _) = weigh(&cached);
println!(
" with a NodeCache: {:?} — {:.2} MB/s, {cached_nodes} node(s), {cached_tokens} token(s)",
took,
mbps(big.len(), took)
);
}
let (_, events) = CppParser::parse_with_events(&big, ParserConfig::default());
let started = Instant::now();
let mut builder = cpp_parser::CppTreeBuilder::new(&big, events, None);
builder.build();
let green = builder.finish();
let building_took = started.elapsed();
println!(
" tree building alone: {:?} — {:.1}% of the parse, {:.2} MB/s",
building_took,
building_took.as_secs_f64() / parse_took.as_secs_f64() * 100.0,
mbps(big.len(), building_took)
);
let _ = green;
println!("\nscaling, one construct repeated:");
for count in [2000usize, 4000, 8000, 16000] {
let mut text = String::with_capacity(count * 22);
for at in 0..count {
text.push_str(&format!("int f{at}(int a);\n"));
}
let started = Instant::now();
let _ = CppParser::parse(&text, ParserConfig::default());
let took = started.elapsed();
println!(
" {count:>6} declaration(s) {:>7.2} MB: {:>8.2?} — {:>7.2} MB/s",
text.len() as f64 / (1024.0 * 1024.0),
took,
mbps(text.len(), took)
);
}
println!("scaling, one template declaration repeated:");
for count in [2000usize, 4000, 8000, 16000] {
let mut text = String::with_capacity(count * 40);
for at in 0..count {
text.push_str(&format!(
"template <class T> void g{at}(Box<T> &b) {{ b.get(); }}\n"
));
}
let started = Instant::now();
let _ = CppParser::parse(&text, ParserConfig::default());
let took = started.elapsed();
println!(
" {count:>6} declaration(s) {:>7.2} MB: {:>8.2?} — {:>7.2} MB/s",
text.len() as f64 / (1024.0 * 1024.0),
took,
mbps(text.len(), took)
);
}
let started = Instant::now();
let red = big_tree.get_red_root();
let mut count = 0usize;
for _node in red.descendants() {
count += 1;
}
let walk_took = started.elapsed();
println!(
"red walk over {count} node(s): {:?} — {:.1} ns/node",
walk_took,
walk_took.as_nanos() as f64 / count.max(1) as f64
);
let started = Instant::now();
let red = tree.get_red_root();
let mut seen = 0usize;
for node in red.descendants() {
if cpp_parser::CppSyntaxKind::from(node.kind()) == CppSyntaxKind::ErrorNode {
seen += 1;
}
}
println!(
" red walk (descendants): {:?} for {nodes} node(s), {seen} error node(s)",
started.elapsed()
);
}
}