use assert_cmd::cargo::cargo_bin;
use chrono::Utc;
use criterion::{criterion_group, criterion_main, BatchSize, Criterion};
use moonlight_core::{
Adapter, BodyCapture, Classification, ComparisonRun, ComparisonSummary, RunInput,
TargetObservation,
};
use std::{
collections::BTreeMap,
fs,
path::{Path, PathBuf},
process::Command,
};
use tempfile::{tempdir, TempDir};
use uuid::Uuid;
fn binary() -> PathBuf {
cargo_bin("moonlight-cli")
}
fn command_json(value: &str) -> String {
format!("printf '%s\\n' '{}'", value)
}
fn argv_json(args: &[&str]) -> String {
serde_json::to_string(args).expect("serialize argv")
}
fn fresh_storage() -> (TempDir, PathBuf) {
let dir = tempdir().expect("temp dir");
let path = dir.path().join("runs.jsonl");
(dir, path)
}
fn run_cli(args: &[&str]) {
let output = Command::new(binary())
.args(args)
.output()
.expect("run moonlight-cli");
assert!(
output.status.success(),
"moonlight-cli failed: {}",
String::from_utf8_lossy(&output.stderr)
);
}
fn run_scenario(storage_path: &Path, args: &[&str]) {
let storage = storage_path.to_string_lossy();
let mut full_args = vec!["run", "--storage-path", storage.as_ref(), "--compact"];
full_args.extend_from_slice(args);
run_cli(&full_args);
}
fn run_batch_scenario(input_path: &Path, storage_path: &Path, jobs: usize) {
let input = input_path.to_string_lossy();
let storage = storage_path.to_string_lossy();
let jobs = jobs.to_string();
run_cli(&[
"batch",
"--input",
input.as_ref(),
"--storage-path",
storage.as_ref(),
"--quiet",
"--jobs",
&jobs,
]);
}
fn body() -> BodyCapture {
BodyCapture {
size_bytes: 0,
sha256: "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855".to_string(),
preview: String::new(),
truncated: false,
}
}
fn target(latency_ms: u128) -> TargetObservation {
TargetObservation {
status: Some(0),
headers: BTreeMap::new(),
body: body(),
stderr: Some(body()),
latency_ms,
error: None,
}
}
fn fixture_run(index: usize) -> ComparisonRun {
ComparisonRun {
id: Uuid::new_v4(),
timestamp: Utc::now(),
adapter: Adapter::Cli,
input: RunInput::Cli {
primary_command: format!("fixture-{index}-primary"),
candidate_command: format!("fixture-{index}-candidate"),
secondary_command: Some(format!("fixture-{index}-secondary")),
},
request_headers: BTreeMap::new(),
request_body: body(),
primary: target(10),
candidate: target(20),
secondary: Some(target(30)),
comparison: ComparisonSummary {
classification: if index.is_multiple_of(2) {
Classification::Match
} else {
Classification::SuspiciousDifference
},
..Default::default()
},
}
}
fn write_fixture(count: usize) -> (TempDir, PathBuf, Uuid) {
let (dir, path) = fresh_storage();
let mut show_id = Uuid::nil();
let lines = (0..count)
.map(|index| {
let run = fixture_run(index);
if index == count / 2 {
show_id = run.id;
}
serde_json::to_string(&run).expect("serialize fixture run")
})
.collect::<Vec<_>>()
.join("\n");
fs::write(&path, format!("{lines}\n")).expect("write fixture");
(dir, path, show_id)
}
fn write_sibling_polluted_fixture(count: usize) -> (TempDir, PathBuf, Uuid) {
let dir = tempdir().expect("temp dir");
let requested_path = dir.path().join("requested-runs.jsonl");
let sibling_a = dir.path().join("sibling-a.jsonl");
let sibling_b = dir.path().join("sibling-b.jsonl");
let mut show_id = Uuid::nil();
for (path, id_offset) in [
(&requested_path, 0),
(&sibling_a, count),
(&sibling_b, count * 2),
] {
let lines = (0..count)
.map(|index| {
let run = fixture_run(index + id_offset);
if path == &requested_path && index == count / 2 {
show_id = run.id;
}
serde_json::to_string(&run).expect("serialize fixture run")
})
.collect::<Vec<_>>()
.join("\n");
fs::write(path, format!("{lines}\n")).expect("write fixture");
}
(dir, requested_path, show_id)
}
fn write_batch_fixture(count: usize) -> (TempDir, PathBuf, PathBuf) {
let dir = tempdir().expect("temp dir");
let input = dir.path().join("cases.jsonl");
let storage = dir.path().join("runs.jsonl");
let case = serde_json::json!({
"primary_argv": ["printf", "%s\n", "{\"value\":42}"],
"candidate_argv": ["printf", "%s\n", "{\"value\":42}"]
});
let lines = (0..count)
.map(|_| case.to_string())
.collect::<Vec<_>>()
.join("\n");
fs::write(&input, format!("{lines}\n")).expect("write batch fixture");
(dir, input, storage)
}
fn bench_run_commands(c: &mut Criterion) {
c.bench_function("run_match_small_json", |b| {
let primary = command_json(r#"{"value":42}"#);
let candidate = command_json(r#"{"value":42}"#);
let secondary = command_json(r#"{"value":42}"#);
b.iter_batched(
fresh_storage,
|(_dir, storage)| {
run_scenario(
&storage,
&[
"--primary",
&primary,
"--candidate",
&candidate,
"--secondary",
&secondary,
],
);
},
BatchSize::SmallInput,
);
});
c.bench_function("run_match_small_json_argv", |b| {
let primary = argv_json(&["printf", "%s\n", r#"{"value":42}"#]);
let candidate = argv_json(&["printf", "%s\n", r#"{"value":42}"#]);
let secondary = argv_json(&["printf", "%s\n", r#"{"value":42}"#]);
b.iter_batched(
fresh_storage,
|(_dir, storage)| {
run_scenario(
&storage,
&[
"--primary-argv",
&primary,
"--candidate-argv",
&candidate,
"--secondary-argv",
&secondary,
],
);
},
BatchSize::SmallInput,
);
});
c.bench_function("run_suspicious_difference_small_json", |b| {
let primary = command_json(r#"{"value":42}"#);
let candidate = command_json(r#"{"value":43}"#);
b.iter_batched(
fresh_storage,
|(_dir, storage)| {
run_scenario(
&storage,
&["--primary", &primary, "--candidate", &candidate],
);
},
BatchSize::SmallInput,
);
});
c.bench_function("run_reference_noise_small_json", |b| {
let primary = command_json(r#"{"region":"a","value":1}"#);
let candidate = command_json(r#"{"region":"a","value":1}"#);
let secondary = command_json(r#"{"region":"b","value":1}"#);
b.iter_batched(
fresh_storage,
|(_dir, storage)| {
run_scenario(
&storage,
&[
"--primary",
&primary,
"--candidate",
&candidate,
"--secondary",
&secondary,
],
);
},
BatchSize::SmallInput,
);
});
c.bench_function("run_large_stdout_64kb", |b| {
let primary = "python3 -c 'print(\"a\" * 65536, end=\"\")'";
let candidate = "python3 -c 'print(\"a\" * 65536, end=\"\")'";
b.iter_batched(
fresh_storage,
|(_dir, storage)| {
run_scenario(&storage, &["--primary", primary, "--candidate", candidate]);
},
BatchSize::SmallInput,
);
});
}
fn bench_batch_commands(c: &mut Criterion) {
for jobs in [1, 4, 8, 16] {
c.bench_function(&format!("batch_many_small_json_argv_jobs_{jobs}"), |b| {
b.iter_batched(
|| write_batch_fixture(64),
|(_dir, input, storage)| run_batch_scenario(&input, &storage, jobs),
BatchSize::SmallInput,
);
});
}
}
fn bench_read_commands(c: &mut Criterion) {
let (_dir, storage, show_id) = write_fixture(1000);
let storage = storage.to_string_lossy().into_owned();
let show_id = show_id.to_string();
c.bench_function("stats_1000_runs", |b| {
b.iter(|| run_cli(&["stats", "--storage-path", &storage]));
});
c.bench_function("list_1000_runs", |b| {
b.iter(|| run_cli(&["list", "--storage-path", &storage]));
});
c.bench_function("list_latest_20_of_1000_runs", |b| {
b.iter(|| run_cli(&["list", "--storage-path", &storage, "--limit", "20"]));
});
c.bench_function("show_middle_run_1000_runs", |b| {
b.iter(|| run_cli(&["show", &show_id, "--storage-path", &storage]));
});
let (_polluted_dir, polluted_storage, polluted_show_id) = write_sibling_polluted_fixture(1000);
let polluted_storage = polluted_storage.to_string_lossy().into_owned();
let polluted_show_id = polluted_show_id.to_string();
c.bench_function("stats_1000_runs_with_sibling_jsonl", |b| {
b.iter(|| run_cli(&["stats", "--storage-path", &polluted_storage]));
});
c.bench_function("list_latest_20_of_1000_runs_with_sibling_jsonl", |b| {
b.iter(|| run_cli(&["list", "--storage-path", &polluted_storage, "--limit", "20"]));
});
c.bench_function("show_middle_run_1000_runs_with_sibling_jsonl", |b| {
b.iter(|| {
run_cli(&[
"show",
&polluted_show_id,
"--storage-path",
&polluted_storage,
])
});
});
}
criterion_group!(
benches,
bench_run_commands,
bench_batch_commands,
bench_read_commands
);
criterion_main!(benches);