#![allow(dead_code)]
use anyhow::Result;
use crate::bench::api::Bencher;
use crate::bench::BenchResult;
use crate::warehouse::dep_graph::WorkspaceGraph;
use crate::workspace::WorkspaceDescriptor;
#[path = "../tests/support/genworkspace.rs"]
mod genworkspace;
use genworkspace::{gen_workspace, GenSpec, Shape};
struct DepGraphBuild {
id: &'static str,
repos: usize,
ext_crates: usize,
}
impl Bencher for DepGraphBuild {
fn id(&self) -> &'static str {
self.id
}
fn run(&self) -> Result<BenchResult> {
let mut spec = GenSpec::new(Shape::LayeredDag, self.repos).seed(42);
if self.ext_crates > 0 {
spec = spec.ext_crates(self.ext_crates);
}
let ws = gen_workspace(&spec);
let desc = WorkspaceDescriptor::load(&ws.descriptor_path)?;
let t0 = std::time::Instant::now();
let g = WorkspaceGraph::build(&desc)?;
let build = t0.elapsed();
let t1 = std::time::Instant::now();
let mut answered = 0u64;
for name in &ws.repo_names {
let _ = g.dependents_transitive(name);
let _ = g.deps_transitive(name);
answered += 2;
}
let queries = t1.elapsed();
let crates = ws.repo_names.len() as u64;
let queries_per_sec = if queries.as_secs_f64() > 0.0 {
answered as f64 / queries.as_secs_f64()
} else {
0.0
};
let mut metrics = serde_json::Map::new();
metrics.insert("build_ms".into(), serde_json::json!(build.as_secs_f64() * 1000.0));
metrics.insert("crates".into(), serde_json::json!(crates));
metrics.insert("queries".into(), serde_json::json!(answered));
metrics.insert("queries_per_sec".into(), serde_json::json!(queries_per_sec));
metrics.insert("ext_crates".into(), serde_json::json!(self.ext_crates));
metrics.insert(
"elapsed_ms".into(),
serde_json::json!((build + queries).as_secs_f64() * 1000.0),
);
Ok(BenchResult { name: self.id.into(), metrics , ..Default::default() })
}
}
crate::register_bench!(DepGraphBuild {
id: "nornir.dep_graph_build",
repos: 50,
ext_crates: 0,
});
struct DeltaRescan {
id: &'static str,
files: usize,
}
impl Bencher for DeltaRescan {
fn id(&self) -> &'static str {
self.id
}
fn run(&self) -> Result<BenchResult> {
use crate::knowledge::delta::SymbolDelta;
let dir = tempfile::tempdir()?;
std::fs::write(
dir.path().join("Cargo.toml"),
"[package]\nname = \"deltabench\"\nversion = \"0.0.0\"\nedition = \"2021\"\n",
)?;
let src = dir.path().join("src");
std::fs::create_dir_all(&src)?;
let mut lib = String::new();
for i in 0..self.files {
std::fs::write(
src.join(format!("m{i}.rs")),
format!("pub fn f{i}() {{ let _ = {i}; }}\npub struct S{i};\n"),
)?;
lib.push_str(&format!("pub mod m{i};\n"));
}
std::fs::write(src.join("lib.rs"), lib)?;
let now = chrono::Utc::now();
let t0 = std::time::Instant::now();
let mut wh = SymbolDelta::full_build(dir.path(), "deltabench", uuid::Uuid::new_v4(), now)?;
let full_build = t0.elapsed();
let baseline_syms = wh.scan.symbols.len();
let target = src.join(format!("m{}.rs", self.files / 2));
std::thread::sleep(std::time::Duration::from_millis(12));
std::fs::write(&target, "pub fn edited() {}\npub struct Edited;\npub struct Extra;\n")?;
let t1 = std::time::Instant::now();
let plan = wh.rescan(dir.path(), uuid::Uuid::new_v4(), now)?;
let delta = t1.elapsed();
let t2 = std::time::Instant::now();
let _fresh =
SymbolDelta::full_build(dir.path(), "deltabench", uuid::Uuid::new_v4(), now)?;
let full_rebuild = t2.elapsed();
let speedup = if delta.as_secs_f64() > 0.0 {
full_rebuild.as_secs_f64() / delta.as_secs_f64()
} else {
0.0
};
let mut metrics = serde_json::Map::new();
metrics.insert("files".into(), serde_json::json!(self.files));
metrics.insert("full_build_ms".into(), serde_json::json!(full_build.as_secs_f64() * 1000.0));
metrics.insert("delta_rescan_ms".into(), serde_json::json!(delta.as_secs_f64() * 1000.0));
metrics.insert("full_rebuild_ms".into(), serde_json::json!(full_rebuild.as_secs_f64() * 1000.0));
metrics.insert("speedup_vs_full".into(), serde_json::json!(speedup));
metrics.insert("files_touched".into(), serde_json::json!(plan.touched()));
metrics.insert("baseline_symbols".into(), serde_json::json!(baseline_syms));
metrics.insert("elapsed_ms".into(), serde_json::json!((full_build + delta + full_rebuild).as_secs_f64() * 1000.0));
Ok(BenchResult { name: self.id.into(), metrics, ..Default::default() })
}
}
crate::register_bench!(DeltaRescan {
id: "nornir.delta_rescan",
files: 400,
});
crate::register_bench!(DepGraphBuild {
id: "nornir.dep_graph_build_ext",
repos: 50,
ext_crates: 300,
});
struct KnowledgeScan {
id: &'static str,
files: usize,
}
impl Bencher for KnowledgeScan {
fn id(&self) -> &'static str {
self.id
}
fn run(&self) -> Result<BenchResult> {
use crate::knowledge::symbols::scan_repo;
let dir = tempfile::tempdir()?;
std::fs::write(
dir.path().join("Cargo.toml"),
"[package]\nname = \"scanbench\"\nversion = \"0.0.0\"\nedition = \"2021\"\n",
)?;
let src = dir.path().join("src");
std::fs::create_dir_all(&src)?;
let mut lib = String::new();
for i in 0..self.files {
let next = (i + 1) % self.files;
std::fs::write(
src.join(format!("m{i}.rs")),
format!(
"pub fn f{i}() {{ let _ = crate::m{next}::g{next}(); }}\n\
pub fn g{i}() -> u32 {{ {i} }}\n\
pub struct S{i} {{ pub v: u32 }}\n"
),
)?;
lib.push_str(&format!("pub mod m{i};\n"));
}
std::fs::write(src.join("lib.rs"), lib)?;
let now = chrono::Utc::now();
let t0 = std::time::Instant::now();
let scan = scan_repo(dir.path(), "scanbench", uuid::Uuid::new_v4(), now)?;
let elapsed = t0.elapsed();
let symbols = scan.symbols.len() as u64;
let calls = scan.calls.len() as u64;
let symbols_per_sec = if elapsed.as_secs_f64() > 0.0 {
symbols as f64 / elapsed.as_secs_f64()
} else {
0.0
};
let mut metrics = serde_json::Map::new();
metrics.insert("files".into(), serde_json::json!(self.files));
metrics.insert("symbols".into(), serde_json::json!(symbols));
metrics.insert("call_edges".into(), serde_json::json!(calls));
metrics.insert("symbols_per_sec".into(), serde_json::json!(symbols_per_sec));
metrics.insert(
"elapsed_ms".into(),
serde_json::json!(elapsed.as_secs_f64() * 1000.0),
);
Ok(BenchResult { name: self.id.into(), metrics, ..Default::default() })
}
}
crate::register_bench!(KnowledgeScan {
id: "nornir.knowledge_scan",
files: 300,
});
#[cfg(feature = "vector")]
struct VectorSearch {
id: &'static str,
rows: usize,
dim: usize,
queries: usize,
k: usize,
}
#[cfg(feature = "vector")]
impl Bencher for VectorSearch {
fn id(&self) -> &'static str {
self.id
}
fn run(&self) -> Result<BenchResult> {
use crate::vector::VectorIndex;
let mk = |seed: u64, dim: usize| -> Vec<f32> {
(0..dim)
.map(|d| {
let x = (seed.wrapping_mul(2_654_435_761).wrapping_add(d as u64)) as f32;
(x * 0.000_1).sin() + 0.001 * (d as f32 + 1.0)
})
.collect()
};
let mut idx = VectorIndex::new(self.dim)?;
let mut flat = Vec::with_capacity(self.rows * self.dim);
let mut ids = Vec::with_capacity(self.rows);
for r in 0..self.rows as u64 {
flat.extend(mk(r, self.dim));
ids.push(r * 7 + 3);
}
idx.add(&flat, &ids)?;
let t0 = std::time::Instant::now();
let mut hits = 0u64;
for q in 0..self.queries as u64 {
let query = mk(q.wrapping_mul(31).wrapping_add(1_000), self.dim);
hits += idx.search_auto(&query, self.k).len() as u64;
}
let elapsed = t0.elapsed();
let queries_per_sec = if elapsed.as_secs_f64() > 0.0 {
self.queries as f64 / elapsed.as_secs_f64()
} else {
0.0
};
let mut metrics = serde_json::Map::new();
metrics.insert("rows".into(), serde_json::json!(self.rows));
metrics.insert("dim".into(), serde_json::json!(self.dim));
metrics.insert("queries".into(), serde_json::json!(self.queries));
metrics.insert("k".into(), serde_json::json!(self.k));
metrics.insert("hits".into(), serde_json::json!(hits));
metrics.insert("queries_per_sec".into(), serde_json::json!(queries_per_sec));
metrics.insert(
"elapsed_ms".into(),
serde_json::json!(elapsed.as_secs_f64() * 1000.0),
);
Ok(BenchResult { name: self.id.into(), metrics, ..Default::default() })
}
}
#[cfg(feature = "vector")]
crate::register_bench!(VectorSearch {
id: "nornir.vector_search",
rows: 8192,
dim: 128,
queries: 256,
k: 10,
});
struct WarehouseQuery {
id: &'static str,
files: usize,
}
impl Bencher for WarehouseQuery {
fn id(&self) -> &'static str {
self.id
}
fn run(&self) -> Result<BenchResult> {
use crate::coverage::{summarise, Boundary, FileCoverage, FnCoverage};
use crate::warehouse::coverage::{append_report, query_coverage, rows_for};
use crate::warehouse::iceberg::IcebergWarehouse;
let file_cov: Vec<FileCoverage> = (0..self.files)
.map(|i| {
let file = format!("src/gen/m{i}.rs");
FileCoverage {
file: file.clone(),
krate: "nornir".into(),
lines: 20,
lines_covered: (i as u64 % 21),
regions: 12,
regions_covered: (i as u64 % 13),
functions: vec![FnCoverage {
name: format!("nornir::gen::m{i}::f{i}"),
file,
lines: 20,
lines_covered: (i as u64 % 21),
regions: 12,
regions_covered: (i as u64 % 13),
boundary: Boundary::Core,
}],
}
})
.collect();
let report = summarise("nornir", file_cov)?;
let (rows, fn_rows) = rows_for(&report, "bench-run", "nornir", 1_000);
let dir = tempfile::tempdir()?;
let wh = IcebergWarehouse::open(dir.path())?;
wh.block_on(append_report(&wh, &rows, &fn_rows))?;
let t0 = std::time::Instant::now();
let back = wh.block_on(query_coverage(&wh, "nornir"))?;
let elapsed = t0.elapsed();
let read = back.len() as u64;
let rows_per_sec = if elapsed.as_secs_f64() > 0.0 {
read as f64 / elapsed.as_secs_f64()
} else {
0.0
};
let mut metrics = serde_json::Map::new();
metrics.insert("files".into(), serde_json::json!(self.files));
metrics.insert("rows_written".into(), serde_json::json!(rows.len()));
metrics.insert("rows_read".into(), serde_json::json!(read));
metrics.insert("rows_per_sec".into(), serde_json::json!(rows_per_sec));
metrics.insert(
"elapsed_ms".into(),
serde_json::json!(elapsed.as_secs_f64() * 1000.0),
);
Ok(BenchResult { name: self.id.into(), metrics, ..Default::default() })
}
}
crate::register_bench!(WarehouseQuery {
id: "nornir.warehouse_query",
files: 500,
});
struct PlanDagAssemble {
id: &'static str,
nodes: usize,
}
impl Bencher for PlanDagAssemble {
fn id(&self) -> &'static str {
self.id
}
fn run(&self) -> Result<BenchResult> {
use funnel_core::event::{Event, ItemKind, PlanStatus};
use funnel_core::ids::{IdeaId, NodeId, PlanId};
use funnel_core::plan_dag::PlanDag;
use funnel_core::state::Funnel;
let mut f = Funnel::default();
let now = chrono::Utc::now();
let idea = IdeaId::seq(1);
let plan = PlanId::seq(1);
f.apply(&Event::IdeaSubmitted {
id: idea.clone(),
source: "bench".into(),
text: "assemble".into(),
refs: vec![],
item_kind: ItemKind::Idea,
ts: now,
})?;
f.apply(&Event::PlanCreated {
id: plan.clone(),
idea_id: idea,
summary: "chain".into(),
planner: "bench".into(),
ts: now,
})?;
f.apply(&Event::PlanStatusChanged { plan_id: plan.clone(), status: PlanStatus::Active, why: None, ts: now })?;
for i in 1..=self.nodes as u64 {
f.apply(&Event::NodeAdded {
plan_id: plan.clone(),
node_id: NodeId::seq(i),
kind: "code:write".into(),
params: Default::default(),
targets: vec![],
prompt_excerpt: Some(format!("task {i}")),
ts: now,
})?;
if i > 1 {
f.apply(&Event::EdgeAdded {
plan_id: plan.clone(),
from_node: NodeId::seq(i - 1),
to_node: NodeId::seq(i),
ts: now,
})?;
}
}
let iters = 200u64;
let t0 = std::time::Instant::now();
let mut last_nodes = 0usize;
for _ in 0..iters {
let dag = PlanDag::from_funnel(&f, None);
last_nodes = dag.node_count();
}
let elapsed = t0.elapsed();
let per = elapsed.as_secs_f64() * 1000.0 / iters as f64;
let mut metrics = serde_json::Map::new();
metrics.insert("nodes".into(), serde_json::json!(last_nodes));
metrics.insert("assemble_ms".into(), serde_json::json!(per));
metrics.insert("elapsed_ms".into(), serde_json::json!(elapsed.as_secs_f64() * 1000.0));
metrics.insert(
"assemblies_per_sec".into(),
serde_json::json!(if elapsed.as_secs_f64() > 0.0 { iters as f64 / elapsed.as_secs_f64() } else { 0.0 }),
);
Ok(BenchResult { name: self.id.into(), metrics, ..Default::default() })
}
}
crate::register_bench!(PlanDagAssemble {
id: "nornir.plan_dag_assemble",
nodes: 200,
});
#[cfg(test)]
mod tests {
#[test]
fn dep_graph_benchers_are_linked_and_discoverable() {
let ids = crate::bench::api::registered_bench_ids();
assert!(!ids.is_empty(), "linked build must register at least one bencher: {ids:?}");
assert!(
ids.iter().any(|id| id == "nornir.dep_graph_build"),
"nornir.dep_graph_build must be linked + discoverable: {ids:?}"
);
assert!(
ids.iter().any(|id| id == "nornir.dep_graph_build_ext"),
"nornir.dep_graph_build_ext must be linked + discoverable: {ids:?}"
);
}
#[test]
fn dep_graph_hot_path_is_covered_by_registry() {
let registered = crate::bench::api::registered_bench_ids();
let rows = crate::autonom::bench_rows(&["nornir.dep_graph"], ®istered);
let dg = rows.iter().find(|r| r.module == "nornir.dep_graph").unwrap();
assert!(dg.has_bencher, "dep_graph must be covered by the linked registry: {registered:?}");
}
#[test]
fn wired_hot_paths_are_covered_by_registry() {
let registered = crate::bench::api::registered_bench_ids();
let mut keys = vec!["nornir.knowledge_scan", "nornir.warehouse_query"];
#[cfg(feature = "vector")]
keys.push("nornir.vector_search");
let rows = crate::autonom::bench_rows(&keys, ®istered);
for key in &keys {
let row = rows.iter().find(|r| &r.module == key).unwrap();
assert!(
row.has_bencher,
"{key} must be covered by the linked registry: {registered:?}"
);
}
}
}