#![cfg(any(windows, target_os = "linux"))]
use std::alloc::{GlobalAlloc, Layout, System};
use std::collections::VecDeque;
use std::hint::black_box;
use std::ptr::NonNull;
use std::sync::{Arc, Barrier, mpsc};
use std::thread;
use std::thread::JoinHandle;
use std::time::{Duration, Instant};
use bomalloc::{BoAllocator, Config, TrimMode};
const SAMPLE_BATCH: usize = 100;
const DEFAULT_THREAD_COUNTS: [usize; 5] = [1, 2, 4, 8, 16];
const LOCAL_SIZES: [usize; 5] = [16, 96, 1024, 16 * 1024, 32 * 1024];
const MIXED_SIZES: [usize; 8] = [16, 24, 96, 192, 1024, 4096, 16 * 1024, 32 * 1024];
#[derive(Clone, Copy, Debug)]
enum AffinityMode {
Compact,
Spread,
}
fn benchmark_affinity_mode() -> Option<AffinityMode> {
match std::env::var("BOMALLOC_BENCH_AFFINITY").ok().as_deref() {
None | Some("") | Some("none") => None,
Some("compact") => Some(AffinityMode::Compact),
Some("spread") => Some(AffinityMode::Spread),
Some(value) => {
panic!("invalid BOMALLOC_BENCH_AFFINITY={value:?}; expected none, compact, or spread")
}
}
}
#[cfg(windows)]
fn available_cpu_ids() -> Vec<usize> {
use windows_sys::Win32::System::Threading::{GetCurrentProcess, GetProcessAffinityMask};
let mut process_mask = 0usize;
let mut system_mask = 0usize;
let succeeded =
unsafe { GetProcessAffinityMask(GetCurrentProcess(), &mut process_mask, &mut system_mask) };
assert_ne!(succeeded, 0, "GetProcessAffinityMask failed");
(0..usize::BITS as usize)
.filter(|cpu| process_mask & (1usize << cpu) != 0)
.collect()
}
#[cfg(target_os = "linux")]
fn available_cpu_ids() -> Vec<usize> {
let mut cpu_set = unsafe { std::mem::zeroed::<libc::cpu_set_t>() };
let result =
unsafe { libc::sched_getaffinity(0, std::mem::size_of::<libc::cpu_set_t>(), &mut cpu_set) };
assert_eq!(result, 0, "sched_getaffinity failed");
(0..libc::CPU_SETSIZE as usize)
.filter(|cpu| unsafe { libc::CPU_ISSET(*cpu, &cpu_set) })
.collect()
}
#[cfg(not(any(windows, target_os = "linux")))]
fn available_cpu_ids() -> Vec<usize> {
(0..thread::available_parallelism().map_or(1, usize::from)).collect()
}
fn affinity_cpu(worker_index: usize, worker_count: usize) -> Option<usize> {
let mode = benchmark_affinity_mode()?;
let available = available_cpu_ids();
assert!(
!available.is_empty(),
"no CPUs are available to the benchmark process"
);
let slot = match mode {
AffinityMode::Compact => worker_index % available.len(),
AffinityMode::Spread if worker_count <= available.len() => {
worker_index * available.len() / worker_count
}
AffinityMode::Spread => worker_index % available.len(),
};
Some(available[slot])
}
#[cfg(windows)]
fn pin_current_thread(cpu: usize) {
use windows_sys::Win32::System::Threading::{GetCurrentThread, SetThreadAffinityMask};
assert!(
cpu < usize::BITS as usize,
"CPU {cpu} requires processor-group support"
);
let previous = unsafe { SetThreadAffinityMask(GetCurrentThread(), 1usize << cpu) };
assert_ne!(previous, 0, "SetThreadAffinityMask failed for CPU {cpu}");
}
#[cfg(target_os = "linux")]
fn pin_current_thread(cpu: usize) {
let mut cpu_set = unsafe { std::mem::zeroed::<libc::cpu_set_t>() };
unsafe { libc::CPU_SET(cpu, &mut cpu_set) };
let result = unsafe {
libc::pthread_setaffinity_np(
libc::pthread_self(),
std::mem::size_of::<libc::cpu_set_t>(),
&cpu_set,
)
};
assert_eq!(result, 0, "pthread_setaffinity_np failed for CPU {cpu}");
}
#[cfg(not(any(windows, target_os = "linux")))]
fn pin_current_thread(_cpu: usize) {}
fn spawn_bench_worker<F, T>(worker_index: usize, worker_count: usize, worker: F) -> JoinHandle<T>
where
F: FnOnce() -> T + Send + 'static,
T: Send + 'static,
{
let cpu = affinity_cpu(worker_index, worker_count);
thread::spawn(move || {
if let Some(cpu) = cpu {
pin_current_thread(cpu);
}
worker()
})
}
#[derive(Clone, Copy, Debug)]
enum Backend {
Bomalloc,
System,
}
impl Backend {
fn label(self) -> &'static str {
match self {
Self::Bomalloc => "bomalloc",
Self::System => "system",
}
}
fn output_label(self, variant: &str) -> String {
if !matches!(self, Self::System) {
return variant.to_owned();
}
let label =
std::env::var("BOMALLOC_BENCH_SYSTEM_LABEL").unwrap_or_else(|_| variant.to_owned());
assert!(
!label.is_empty()
&& label
.bytes()
.all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_' | b'.')),
"BOMALLOC_BENCH_SYSTEM_LABEL must be a non-empty token"
);
label
}
}
struct Metrics {
rate: f64,
p50_ns: u128,
p99_ns: u128,
locality_hits: u64,
global_searches: u64,
global_candidate_probes: u64,
random_jumps: u64,
jump_successes: u64,
post_jump_probes: u64,
cas_failures: u64,
steals: u64,
reserved_activations: u64,
territory_extensions: u64,
territory_extension_spans: u64,
reserved_run_activations: u64,
run_territory_extensions: u64,
run_territory_extension_extents: u64,
run_local_window_refills: u64,
run_local_window_hits: u64,
run_local_window_releases: u64,
snapshot_publishes: u64,
reserved_bytes: usize,
committed_bytes: usize,
local_free_spans: usize,
tree_leaf_splits: u64,
tree_parent_inserts: u64,
tree_internal_splits: u64,
tree_logical_removes: u64,
tree_leaf_upserts: u64,
tree_redirects: u64,
tree_parent_cleanups: u64,
tree_batch_replacements: u64,
tree_help_calls: u64,
tree_operation_aborts: u64,
tree_candidate_discards: u64,
}
struct TailMetrics {
rate: f64,
p50_ticks: u64,
p99_ticks: u64,
p9999_ticks: u64,
max_ticks: u64,
samples: usize,
}
struct RemoteMetrics {
rate: f64,
batches: u64,
blocks: u64,
cas_failures: u64,
locality_hits: u64,
global_searches: u64,
reserved_activations: u64,
}
struct ServiceMetrics {
rate: f64,
p50_ns: u128,
p99_ns: u128,
committed_bytes: usize,
reserved_bytes: usize,
remote_batches: u64,
remote_blocks: u64,
cas_failures: u64,
steals: u64,
global_searches: u64,
}
const SERVICE_BATCH: usize = 32;
const SERVICE_SURVIVOR_STRIDE: usize = 16;
const SERVICE_SURVIVOR_WAVES: usize = 8;
const SERVICE_SIZES: [usize; 7] = [48, 96, 192, 512, 1024, 4096, 16 * 1024];
#[derive(Clone, Copy)]
struct ServiceAllocation {
address: usize,
layout: Layout,
}
#[derive(Clone, Copy)]
struct ServiceBatch {
entries: [ServiceAllocation; SERVICE_BATCH],
}
#[derive(Clone, Copy)]
struct ServiceSurvivorCohort {
entries: [ServiceAllocation; SERVICE_BATCH / SERVICE_SURVIVOR_STRIDE],
len: usize,
}
struct MixedMetrics {
rate: f64,
p50_ns: u128,
p99_ns: u128,
committed_bytes: usize,
reserved_bytes: usize,
locality_hits: u64,
global_searches: u64,
}
struct FragmentationMetrics {
locality_reserve_spans: usize,
allocation_rate: f64,
baseline_rss: usize,
peak_rss: usize,
fragmented_rss: usize,
fragmented_trim_rss: usize,
empty_rss: usize,
final_rss: usize,
peak_committed: usize,
fragmented_committed: usize,
final_committed: usize,
peak_reserved: usize,
final_reserved: usize,
}
struct ChurnMetrics {
rate: f64,
p50_ns: u128,
p99_ns: u128,
logical_removes: u64,
redirects: u64,
cross_parent_redirects: u64,
internal_redirects: u64,
internal_redistributions: u64,
root_collapses: u64,
help_calls: u64,
aborts: u64,
node_slots: usize,
live_nodes: usize,
retired_nodes: u64,
reused_nodes: u64,
node_object_slots: usize,
live_node_objects: usize,
reused_node_objects: u64,
span_slots: usize,
live_spans: usize,
reused_spans: u64,
bitmap_slots: usize,
live_bitmaps: usize,
reused_bitmaps: u64,
payload_vector_slots: usize,
live_payload_vectors: usize,
reused_payload_vectors: u64,
metadata_blocks: usize,
metadata_reserved_bytes: usize,
decommits: u64,
chunk_releases: u64,
}
struct GenerationMetrics {
rate: f64,
p50_ns: u128,
p99_ns: u128,
steady_requested_bytes: usize,
steady_usable_bytes: usize,
peak_requested_bytes: usize,
peak_committed_bytes: usize,
reserved_bytes: usize,
global_candidate_probes: u64,
locality_probes: u64,
locality_hits: u64,
global_searches: u64,
reserved_span_activations: u64,
random_jumps: u64,
young_spans: usize,
sealed_spans: usize,
sealed_hole_bytes: usize,
cohort_promotions: u64,
cohort_recycles: u64,
phases: Option<GenerationPhaseMetrics>,
}
struct GenerationPhaseMetrics {
birth_rate: f64,
birth_p50_ns: u128,
birth_p99_ns: u128,
reclaim_rate: f64,
reclaim_p50_ns: u128,
reclaim_p99_ns: u128,
expire_rate: f64,
expire_p50_ns: u128,
expire_p99_ns: u128,
}
#[derive(Default)]
struct GenerationWorkerPhases {
birth_samples: Vec<u128>,
reclaim_samples: Vec<u128>,
expire_samples: Vec<u128>,
birth_elapsed: Duration,
reclaim_elapsed: Duration,
expire_elapsed: Duration,
birth_ops: usize,
reclaim_ops: usize,
expire_ops: usize,
}
fn percentile(samples: &mut [u128], percentile: usize) -> u128 {
samples.sort_unstable();
samples[(samples.len() - 1) * percentile / 100]
}
fn percentile_basis_points(samples: &mut [u64], basis_points: usize) -> u64 {
assert!(!samples.is_empty());
assert!(basis_points <= 10_000);
samples.sort_unstable();
samples[(samples.len() - 1) * basis_points / 10_000]
}
#[cfg(target_arch = "x86_64")]
#[inline]
fn tail_clock() -> u64 {
unsafe {
core::arch::x86_64::_mm_lfence();
let cycles = core::arch::x86_64::_rdtsc();
core::arch::x86_64::_mm_lfence();
cycles
}
}
#[cfg(not(target_arch = "x86_64"))]
#[inline]
fn tail_clock() -> u64 {
use std::sync::OnceLock;
static ORIGIN: OnceLock<Instant> = OnceLock::new();
ORIGIN
.get_or_init(Instant::now)
.elapsed()
.as_nanos()
.min(u64::MAX as u128) as u64
}
fn tail_clock_unit() -> &'static str {
if cfg!(target_arch = "x86_64") {
"cycles"
} else {
"ns"
}
}
fn layout_for(size: usize) -> Layout {
Layout::from_size_align(size, size.next_power_of_two().min(4096)).unwrap()
}
#[cfg(target_os = "linux")]
fn resident_bytes() -> usize {
let statm =
std::fs::read_to_string("/proc/self/statm").expect("failed to read /proc/self/statm");
let resident_pages = statm
.split_ascii_whitespace()
.nth(1)
.expect("statm has no resident field")
.parse::<usize>()
.expect("statm resident field is not an integer");
let page_size = unsafe { libc::sysconf(libc::_SC_PAGESIZE) };
assert!(page_size > 0, "sysconf(_SC_PAGESIZE) failed");
resident_pages * page_size as usize
}
#[cfg(windows)]
fn resident_bytes() -> usize {
0
}
unsafe fn touch_requested_pages(pointer: NonNull<u8>, layout: Layout) {
const PAGE: usize = 4096;
let mut offset = 0;
while offset < layout.size() {
unsafe {
pointer
.as_ptr()
.add(offset)
.write_volatile((offset / PAGE) as u8)
};
offset += PAGE;
}
unsafe { pointer.as_ptr().add(layout.size() - 1).write_volatile(0xa5) };
}
fn selected_size(size: usize) -> bool {
std::env::var("BOMALLOC_BENCH_SIZE")
.ok()
.and_then(|value| value.parse::<usize>().ok())
.is_none_or(|selected| selected == size)
}
fn selected_thread_count(threads: usize) -> bool {
std::env::var("BOMALLOC_BENCH_THREADS")
.ok()
.and_then(|value| value.parse::<usize>().ok())
.is_none_or(|selected| threads == 1 || threads == selected)
}
fn benchmark_thread_counts() -> Vec<usize> {
match std::env::var("BOMALLOC_BENCH_THREADS") {
Ok(value) => {
let selected = value
.parse::<usize>()
.expect("BOMALLOC_BENCH_THREADS must be a positive integer");
assert!(selected != 0, "BOMALLOC_BENCH_THREADS must be non-zero");
if selected == 1 {
vec![1]
} else {
vec![1, selected]
}
}
Err(_) => DEFAULT_THREAD_COUNTS.to_vec(),
}
}
fn benchmark_service_thread_counts() -> Vec<usize> {
match std::env::var("BOMALLOC_BENCH_THREADS") {
Ok(value) => {
let selected = value
.parse::<usize>()
.expect("BOMALLOC_BENCH_THREADS must be an even integer");
assert!(selected >= 2 && selected % 2 == 0);
if selected == 2 {
vec![2]
} else {
vec![2, selected]
}
}
Err(_) => vec![2, 4, 8, 16],
}
}
fn selected_remote_thread_count(threads: usize) -> bool {
std::env::var("BOMALLOC_BENCH_THREADS")
.ok()
.and_then(|value| value.parse::<usize>().ok())
.is_none_or(|selected| threads == 2 || threads == selected)
}
fn selected_allocator(label: &str) -> bool {
std::env::var("BOMALLOC_BENCH_ALLOCATOR")
.ok()
.is_none_or(|selected| {
selected == label
|| (label == "bomalloc"
&& matches!(selected.as_str(), "bomalloc-pageq0" | "bomalloc-pageq32"))
})
}
fn benchmark_bomalloc_config() -> Config {
let mut config = Config::default();
if let Ok(value) = std::env::var("BOMALLOC_BENCH_TERRITORY_EXTENSION") {
config.adaptive_territory_extension_spans = value
.parse()
.expect("BOMALLOC_BENCH_TERRITORY_EXTENSION must be an integer");
}
if let Ok(value) = std::env::var("BOMALLOC_BENCH_RUN_TERRITORY_EXTENSION") {
config.adaptive_run_territory_extension_extents = value
.parse()
.expect("BOMALLOC_BENCH_RUN_TERRITORY_EXTENSION must be an integer");
}
if let Ok(value) = std::env::var("BOMALLOC_BENCH_RUN_LOCAL_WINDOW") {
config.run_local_window_capacity = value
.parse()
.expect("BOMALLOC_BENCH_RUN_LOCAL_WINDOW must be an integer");
}
if let Ok(value) = std::env::var("BOMALLOC_BENCH_LOCALITY_RESERVE") {
config.locality_reserve_spans = value
.parse()
.expect("BOMALLOC_BENCH_LOCALITY_RESERVE must be an integer");
}
if let Ok(value) = std::env::var("BOMALLOC_BENCH_JUMPS") {
config.max_random_jumps = value
.parse()
.expect("BOMALLOC_BENCH_JUMPS must be an integer");
}
if let Ok(value) = std::env::var("BOMALLOC_BENCH_RANDOM_SEED") {
config.random_seed = Some(
value
.parse()
.expect("BOMALLOC_BENCH_RANDOM_SEED must be an integer"),
);
}
if let Ok(value) = std::env::var("BOMALLOC_BENCH_YOUNG_RING") {
config.young_ring_spans = value
.parse()
.expect("BOMALLOC_BENCH_YOUNG_RING must be an integer");
}
if let Ok(value) = std::env::var("BOMALLOC_BENCH_REMOTE_CACHE") {
config.remote_cache_capacity = value
.parse()
.expect("BOMALLOC_BENCH_REMOTE_CACHE must be an integer");
}
if let Ok(value) = std::env::var("BOMALLOC_BENCH_MAGAZINE") {
config.local_magazine_capacity = value
.parse()
.expect("BOMALLOC_BENCH_MAGAZINE must be an integer");
}
if let Ok(value) = std::env::var("BOMALLOC_BENCH_BLACK_RING") {
config.black_span_ring_capacity = value
.parse()
.expect("BOMALLOC_BENCH_BLACK_RING must be an integer");
}
match std::env::var("BOMALLOC_BENCH_ALLOCATOR").ok().as_deref() {
Some("bomalloc-pageq0") => config.low_capacity_page_queue_spans = 0,
Some("bomalloc-pageq32") => config.low_capacity_page_queue_spans = 32,
_ => {}
}
config
}
fn bomalloc_pair(allocator: &BoAllocator, layout: Layout) {
let allocation = allocator.allocate(layout).unwrap();
let pointer = NonNull::new(allocation.as_ptr() as *mut u8).unwrap();
black_box(pointer);
unsafe { allocator.deallocate(pointer, layout) };
}
fn system_pair(layout: Layout) {
unsafe {
let pointer = System.alloc(layout);
assert!(!pointer.is_null());
black_box(pointer);
System.dealloc(pointer, layout);
}
}
fn backend_allocate(backend: Backend, allocator: &BoAllocator, layout: Layout) -> NonNull<u8> {
match backend {
Backend::Bomalloc => {
let allocation = allocator.allocate(layout).unwrap();
NonNull::new(allocation.as_ptr() as *mut u8).unwrap()
}
Backend::System => {
unsafe { NonNull::new(System.alloc(layout)).expect("system allocation failed") }
}
}
}
unsafe fn backend_deallocate(
backend: Backend,
allocator: &BoAllocator,
pointer: NonNull<u8>,
layout: Layout,
) {
match backend {
Backend::Bomalloc => {
unsafe { allocator.deallocate(pointer, layout) }
}
Backend::System => {
unsafe { System.dealloc(pointer.as_ptr(), layout) }
}
}
}
#[inline]
unsafe fn initialize_service_object(pointer: NonNull<u8>, layout: Layout, tag: u64) {
debug_assert!(layout.size() >= std::mem::size_of::<u64>());
unsafe {
(pointer.as_ptr() as *mut u64).write_volatile(tag);
pointer
.as_ptr()
.add(layout.size() - 1)
.write_volatile(tag as u8);
}
}
#[inline]
unsafe fn visit_service_object(allocation: ServiceAllocation) {
let observed = unsafe { (allocation.address as *const u64).read_volatile() };
black_box(observed);
}
fn service_roundtrip(backend: Backend, threads: usize, iterations: usize) -> ServiceMetrics {
assert!(threads >= 2 && threads % 2 == 0);
let pairs = threads / 2;
let rounds = (iterations / SERVICE_BATCH).max(256);
let layouts = SERVICE_SIZES.map(layout_for);
let allocator = BoAllocator::new(benchmark_bomalloc_config()).unwrap();
let barrier = Arc::new(Barrier::new(threads + 1));
let mut workers = Vec::with_capacity(threads);
for pair in 0..pairs {
let (request_sender, request_receiver) = mpsc::sync_channel::<ServiceBatch>(1);
let (response_sender, response_receiver) = mpsc::sync_channel::<ServiceBatch>(1);
let server_allocator = allocator.clone();
let server_barrier = Arc::clone(&barrier);
workers.push(spawn_bench_worker(pair * 2 + 1, threads, move || {
let empty = ServiceAllocation {
address: 0,
layout: Layout::new::<u8>(),
};
let mut generations = VecDeque::with_capacity(SERVICE_SURVIVOR_WAVES + 1);
server_barrier.wait();
for round in 0..rounds {
if generations.len() == SERVICE_SURVIVOR_WAVES {
let expired: ServiceSurvivorCohort = generations.pop_front().unwrap();
for allocation in &expired.entries[..expired.len] {
unsafe {
backend_deallocate(
backend,
&server_allocator,
NonNull::new(allocation.address as *mut u8).unwrap(),
allocation.layout,
)
};
}
}
let request = request_receiver.recv().unwrap();
let mut survivors = ServiceSurvivorCohort {
entries: [empty; SERVICE_BATCH / SERVICE_SURVIVOR_STRIDE],
len: 0,
};
for (index, allocation) in request.entries.into_iter().enumerate() {
unsafe { visit_service_object(allocation) };
if index % SERVICE_SURVIVOR_STRIDE == 0 {
survivors.entries[survivors.len] = allocation;
survivors.len += 1;
} else {
unsafe {
backend_deallocate(
backend,
&server_allocator,
NonNull::new(allocation.address as *mut u8).unwrap(),
allocation.layout,
)
};
}
}
generations.push_back(survivors);
let mut response = ServiceBatch {
entries: [empty; SERVICE_BATCH],
};
for (index, entry) in response.entries.iter_mut().enumerate() {
let layout = layouts[(round * 3 + index * 5 + pair) % layouts.len()];
let pointer = backend_allocate(backend, &server_allocator, layout);
unsafe {
initialize_service_object(
pointer,
layout,
((pair as u64) << 48) | ((round as u64) << 8) | index as u64,
)
};
*entry = ServiceAllocation {
address: pointer.as_ptr() as usize,
layout,
};
}
response_sender.send(response).unwrap();
}
for cohort in generations {
for allocation in &cohort.entries[..cohort.len] {
unsafe {
backend_deallocate(
backend,
&server_allocator,
NonNull::new(allocation.address as *mut u8).unwrap(),
allocation.layout,
)
};
}
}
None
}));
let client_allocator = allocator.clone();
let client_barrier = Arc::clone(&barrier);
workers.push(spawn_bench_worker(pair * 2, threads, move || {
let empty = ServiceAllocation {
address: 0,
layout: Layout::new::<u8>(),
};
let mut samples = Vec::with_capacity(rounds);
client_barrier.wait();
let started = Instant::now();
for round in 0..rounds {
let sample_start = Instant::now();
let mut request = ServiceBatch {
entries: [empty; SERVICE_BATCH],
};
for (index, entry) in request.entries.iter_mut().enumerate() {
let layout = layouts[(round + index * 3 + pair) % layouts.len()];
let pointer = backend_allocate(backend, &client_allocator, layout);
unsafe {
initialize_service_object(
pointer,
layout,
((pair as u64) << 48) | ((round as u64) << 8) | index as u64,
)
};
*entry = ServiceAllocation {
address: pointer.as_ptr() as usize,
layout,
};
}
request_sender.send(request).unwrap();
let response = response_receiver.recv().unwrap();
for allocation in response.entries {
unsafe { visit_service_object(allocation) };
unsafe {
backend_deallocate(
backend,
&client_allocator,
NonNull::new(allocation.address as *mut u8).unwrap(),
allocation.layout,
)
};
}
samples.push(sample_start.elapsed().as_nanos() / SERVICE_BATCH as u128);
}
Some((samples, started.elapsed()))
}));
}
barrier.wait();
let mut samples = Vec::with_capacity(pairs * rounds);
let mut elapsed = Duration::ZERO;
for worker in workers {
if let Some((worker_samples, worker_elapsed)) = worker.join().unwrap() {
samples.extend(worker_samples);
elapsed = elapsed.max(worker_elapsed);
}
}
let stats = allocator.stats();
if matches!(backend, Backend::Bomalloc) {
assert_eq!(
stats.requested_live_bytes, 0,
"service workload leaked requested bytes"
);
allocator
.validate()
.expect("service workload violated allocator invariants");
}
ServiceMetrics {
rate: (pairs * rounds * SERVICE_BATCH) as f64 / elapsed.as_secs_f64(),
p50_ns: percentile(&mut samples, 50),
p99_ns: percentile(&mut samples, 99),
committed_bytes: stats.committed_bytes,
reserved_bytes: stats.reserved_bytes,
remote_batches: stats.remote_batches_drained,
remote_blocks: stats.remote_blocks_drained,
cas_failures: stats.cas_failures,
steals: stats.steals,
global_searches: stats.global_searches,
}
}
fn local_hot(backend: Backend, size: usize, threads: usize, iterations: usize) -> Metrics {
local_hot_with_config(
backend,
size,
threads,
iterations,
benchmark_bomalloc_config(),
)
}
fn local_hot_with_config(
backend: Backend,
size: usize,
threads: usize,
iterations: usize,
config: Config,
) -> Metrics {
let layout = layout_for(size);
let allocator = BoAllocator::new(config).unwrap();
let barrier = Arc::new(Barrier::new(threads + 1));
let mut workers = Vec::with_capacity(threads);
for worker_index in 0..threads {
let allocator = allocator.clone();
let barrier = Arc::clone(&barrier);
workers.push(spawn_bench_worker(worker_index, threads, move || {
let warmup = match backend {
Backend::Bomalloc => 1_000,
Backend::System => 20_000,
};
for _ in 0..warmup {
match backend {
Backend::Bomalloc => bomalloc_pair(&allocator, layout),
Backend::System => system_pair(layout),
}
}
barrier.wait();
let worker_start = Instant::now();
let mut samples = Vec::with_capacity(iterations / SAMPLE_BATCH);
for _ in 0..(iterations / SAMPLE_BATCH) {
let sample_start = Instant::now();
for _ in 0..SAMPLE_BATCH {
match backend {
Backend::Bomalloc => bomalloc_pair(&allocator, layout),
Backend::System => system_pair(layout),
}
}
samples.push(sample_start.elapsed().as_nanos() / SAMPLE_BATCH as u128);
}
(samples, worker_start.elapsed())
}));
}
barrier.wait();
let mut samples = Vec::new();
let mut elapsed = Duration::ZERO;
for worker in workers {
let (worker_samples, worker_elapsed) = worker.join().unwrap();
samples.extend(worker_samples);
elapsed = elapsed.max(worker_elapsed);
}
let stats = allocator.stats();
Metrics {
rate: (threads * iterations) as f64 / elapsed.as_secs_f64(),
p50_ns: percentile(&mut samples, 50),
p99_ns: percentile(&mut samples, 99),
locality_hits: stats.locality_hits,
global_searches: stats.global_searches,
global_candidate_probes: stats.global_candidate_probes,
random_jumps: stats.random_jumps,
jump_successes: stats.jump_successes,
post_jump_probes: stats.post_jump_probes,
cas_failures: stats.cas_failures,
steals: stats.steals,
reserved_activations: stats.reserved_span_activations,
territory_extensions: stats.territory_extensions,
territory_extension_spans: stats.territory_extension_spans,
reserved_run_activations: stats.reserved_run_activations,
run_territory_extensions: stats.run_territory_extensions,
run_territory_extension_extents: stats.run_territory_extension_extents,
run_local_window_refills: stats.run_local_window_refills,
run_local_window_hits: stats.run_local_window_hits,
run_local_window_releases: stats.run_local_window_releases,
snapshot_publishes: stats.topology_snapshot_publishes,
reserved_bytes: stats.reserved_bytes,
committed_bytes: stats.committed_bytes,
local_free_spans: stats.local_free_spans,
tree_leaf_splits: stats.tree_leaf_splits,
tree_parent_inserts: stats.tree_parent_inserts,
tree_internal_splits: stats.tree_internal_splits,
tree_logical_removes: stats.tree_logical_removes,
tree_leaf_upserts: stats.tree_leaf_upserts,
tree_redirects: stats.tree_redirects,
tree_parent_cleanups: stats.tree_parent_cleanups,
tree_batch_replacements: stats.tree_batch_replacements,
tree_help_calls: stats.tree_help_calls,
tree_operation_aborts: stats.tree_operation_aborts,
tree_candidate_discards: stats.tree_candidate_discards,
}
}
fn local_tail(
backend: Backend,
size: usize,
threads: usize,
samples_per_thread: usize,
) -> TailMetrics {
let layout = layout_for(size);
let allocator = BoAllocator::new(benchmark_bomalloc_config()).unwrap();
let barrier = Arc::new(Barrier::new(threads + 1));
let mut workers = Vec::with_capacity(threads);
for worker_index in 0..threads {
let allocator = allocator.clone();
let barrier = Arc::clone(&barrier);
workers.push(spawn_bench_worker(worker_index, threads, move || {
for _ in 0..10_000 {
match backend {
Backend::Bomalloc => bomalloc_pair(&allocator, layout),
Backend::System => system_pair(layout),
}
}
let mut samples = Vec::with_capacity(samples_per_thread);
barrier.wait();
let worker_start = Instant::now();
for _ in 0..samples_per_thread {
let sample_start = tail_clock();
match backend {
Backend::Bomalloc => bomalloc_pair(&allocator, layout),
Backend::System => system_pair(layout),
}
samples.push(tail_clock().wrapping_sub(sample_start));
}
(samples, worker_start.elapsed())
}));
}
barrier.wait();
let mut samples = Vec::with_capacity(threads * samples_per_thread);
let mut elapsed = Duration::ZERO;
for worker in workers {
let (worker_samples, worker_elapsed) = worker.join().unwrap();
samples.extend(worker_samples);
elapsed = elapsed.max(worker_elapsed);
}
let sample_count = samples.len();
let p50_ticks = percentile_basis_points(&mut samples, 5_000);
let p99_ticks = percentile_basis_points(&mut samples, 9_900);
let p9999_ticks = percentile_basis_points(&mut samples, 9_999);
let max_ticks = *samples.last().unwrap();
TailMetrics {
rate: sample_count as f64 / elapsed.as_secs_f64(),
p50_ticks,
p99_ticks,
p9999_ticks,
max_ticks,
samples: sample_count,
}
}
fn magazine_burst_with_config(
backend: Backend,
size: usize,
threads: usize,
iterations: usize,
config: Config,
) -> Metrics {
burst_with_config(backend, size, threads, iterations, config, 64)
}
fn print_magazine_matrix(
workload: &str,
size: usize,
iterations: usize,
runner: fn(Backend, usize, usize, usize, Config) -> Metrics,
) {
for (backend, label, capacity) in [
(Backend::System, "system", 0),
(Backend::Bomalloc, "bomalloc-shared", 0),
(Backend::Bomalloc, "bomalloc-adaptive8", 8),
] {
if !selected_allocator(label) {
continue;
}
let output_label = backend.output_label(label);
let mut single_rate = 0.0;
for threads in benchmark_thread_counts() {
if !selected_thread_count(threads) {
continue;
}
let metrics = runner(
backend,
size,
threads,
iterations,
Config {
local_magazine_capacity: capacity,
..Config::default()
},
);
if threads == 1 {
single_rate = metrics.rate;
}
let speedup = metrics.rate / single_rate;
let efficiency = speedup / threads as f64 * 100.0;
println!(
"workload={workload:<13} allocator={output_label:<18} size={size:>6} threads={threads:>2} \
pairs/s={:>12.0} speedup={speedup:>5.2} efficiency={efficiency:>6.1}% \
p50={:>5}ns p99={:>5}ns",
metrics.rate, metrics.p50_ns, metrics.p99_ns,
);
}
}
}
fn recent_free_holes(threads: usize, iterations: usize, hint_depth: usize) -> Metrics {
const SIZE: usize = 16;
const SPAN_CAPACITY: usize = (64 * 1024 - 4 * 1024) / SIZE;
const HOLE_INDICES: [usize; 4] = [32, 1024, 2048, 3072];
let layout = layout_for(SIZE);
let allocator = BoAllocator::new(Config {
local_magazine_capacity: 0,
recent_free_word_hints: hint_depth,
..Config::default()
})
.unwrap();
let barrier = Arc::new(Barrier::new(threads + 1));
let samples_per_worker = (iterations / SAMPLE_BATCH).max(1);
let rounds_per_sample = SAMPLE_BATCH / HOLE_INDICES.len();
let mut workers = Vec::with_capacity(threads);
for worker_index in 0..threads {
let allocator = allocator.clone();
let barrier = Arc::clone(&barrier);
workers.push(spawn_bench_worker(worker_index, threads, move || {
let mut retained = Vec::with_capacity(SPAN_CAPACITY - HOLE_INDICES.len());
let mut holes = Vec::with_capacity(HOLE_INDICES.len());
for index in 0..SPAN_CAPACITY {
let allocation = allocator.allocate(layout).unwrap();
let pointer = NonNull::new(allocation.as_ptr() as *mut u8).unwrap();
if HOLE_INDICES.contains(&index) {
holes.push(pointer);
} else {
retained.push(pointer);
}
}
for pointer in holes {
unsafe { allocator.deallocate(pointer, layout) };
}
barrier.wait();
let worker_start = Instant::now();
let mut samples = Vec::with_capacity(samples_per_worker);
for _ in 0..samples_per_worker {
let start = Instant::now();
for _ in 0..rounds_per_sample {
let mut recycled = [NonNull::<u8>::dangling(); HOLE_INDICES.len()];
for pointer in &mut recycled {
let allocation = allocator.allocate(layout).unwrap();
*pointer = NonNull::new(allocation.as_ptr() as *mut u8).unwrap();
}
for pointer in recycled {
unsafe { allocator.deallocate(pointer, layout) };
}
}
samples.push(start.elapsed().as_nanos() / SAMPLE_BATCH as u128);
}
let elapsed = worker_start.elapsed();
for pointer in retained {
unsafe { allocator.deallocate(pointer, layout) };
}
(samples, elapsed)
}));
}
barrier.wait();
let mut samples = Vec::with_capacity(threads * samples_per_worker);
let mut elapsed = Duration::ZERO;
for worker in workers {
let (worker_samples, worker_elapsed) = worker.join().unwrap();
samples.extend(worker_samples);
elapsed = elapsed.max(worker_elapsed);
}
let stats = allocator.stats();
Metrics {
rate: (threads * samples_per_worker * SAMPLE_BATCH) as f64 / elapsed.as_secs_f64(),
p50_ns: percentile(&mut samples, 50),
p99_ns: percentile(&mut samples, 99),
locality_hits: stats.locality_hits,
global_searches: stats.global_searches,
global_candidate_probes: stats.global_candidate_probes,
random_jumps: stats.random_jumps,
jump_successes: stats.jump_successes,
post_jump_probes: stats.post_jump_probes,
cas_failures: stats.cas_failures,
steals: stats.steals,
reserved_activations: stats.reserved_span_activations,
territory_extensions: stats.territory_extensions,
territory_extension_spans: stats.territory_extension_spans,
reserved_run_activations: stats.reserved_run_activations,
run_territory_extensions: stats.run_territory_extensions,
run_territory_extension_extents: stats.run_territory_extension_extents,
run_local_window_refills: stats.run_local_window_refills,
run_local_window_hits: stats.run_local_window_hits,
run_local_window_releases: stats.run_local_window_releases,
snapshot_publishes: stats.topology_snapshot_publishes,
reserved_bytes: stats.reserved_bytes,
committed_bytes: stats.committed_bytes,
local_free_spans: stats.local_free_spans,
tree_leaf_splits: stats.tree_leaf_splits,
tree_parent_inserts: stats.tree_parent_inserts,
tree_internal_splits: stats.tree_internal_splits,
tree_logical_removes: stats.tree_logical_removes,
tree_leaf_upserts: stats.tree_leaf_upserts,
tree_redirects: stats.tree_redirects,
tree_parent_cleanups: stats.tree_parent_cleanups,
tree_batch_replacements: stats.tree_batch_replacements,
tree_help_calls: stats.tree_help_calls,
tree_operation_aborts: stats.tree_operation_aborts,
tree_candidate_discards: stats.tree_candidate_discards,
}
}
fn print_recent_free_matrix(iterations: usize) {
for hint_depth in [1, 2, 4] {
let label = match hint_depth {
1 => "bomalloc-hint1",
2 => "bomalloc-hint2",
_ => "bomalloc-hint4",
};
if !selected_allocator(label) {
continue;
}
let mut single_rate = 0.0;
for threads in benchmark_thread_counts() {
if !selected_thread_count(threads) {
continue;
}
let metrics = recent_free_holes(threads, iterations, hint_depth);
if threads == 1 {
single_rate = metrics.rate;
}
let speedup = metrics.rate / single_rate;
let efficiency = speedup / threads as f64 * 100.0;
println!(
"workload=recent-free allocator={label:<18} size={:>6} threads={threads:>2} \
pairs/s={:>12.0} speedup={speedup:>5.2} efficiency={efficiency:>6.1}% \
p50={:>5}ns p99={:>5}ns",
16, metrics.rate, metrics.p50_ns, metrics.p99_ns,
);
}
}
}
fn burst(backend: Backend, size: usize, threads: usize, iterations: usize) -> Metrics {
burst_with_config(
backend,
size,
threads,
iterations,
benchmark_bomalloc_config(),
64,
)
}
fn run_burst(backend: Backend, size: usize, threads: usize, iterations: usize) -> Metrics {
burst_with_config(backend, size, threads, iterations, Config::default(), 8)
}
fn run_growth(backend: Backend, size: usize, threads: usize, allocations: usize) -> Metrics {
let layout = layout_for(size);
let allocator = BoAllocator::new(benchmark_bomalloc_config()).unwrap();
let barrier = Arc::new(Barrier::new(threads + 1));
let release_barrier = Arc::new(Barrier::new(threads + 1));
let stats_barrier = Arc::new(Barrier::new(threads + 1));
let mut workers = Vec::with_capacity(threads);
for worker_index in 0..threads {
let allocator = allocator.clone();
let barrier = Arc::clone(&barrier);
let release_barrier = Arc::clone(&release_barrier);
let stats_barrier = Arc::clone(&stats_barrier);
workers.push(spawn_bench_worker(worker_index, threads, move || {
barrier.wait();
let worker_start = Instant::now();
let mut samples = Vec::with_capacity(allocations);
match backend {
Backend::Bomalloc => {
let mut pointers = Vec::with_capacity(allocations);
for _ in 0..allocations {
let start = Instant::now();
let allocation = allocator.allocate(layout).unwrap();
samples.push(start.elapsed().as_nanos());
pointers.push(NonNull::new(allocation.as_ptr() as *mut u8).unwrap());
}
let elapsed = worker_start.elapsed();
release_barrier.wait();
stats_barrier.wait();
for pointer in pointers.into_iter().rev() {
unsafe { allocator.deallocate(pointer, layout) };
}
(samples, elapsed)
}
Backend::System => {
let mut pointers = Vec::with_capacity(allocations);
for _ in 0..allocations {
let start = Instant::now();
let pointer = unsafe { System.alloc(layout) };
assert!(!pointer.is_null());
samples.push(start.elapsed().as_nanos());
pointers.push(pointer);
}
let elapsed = worker_start.elapsed();
release_barrier.wait();
stats_barrier.wait();
for pointer in pointers.into_iter().rev() {
unsafe { System.dealloc(pointer, layout) };
}
(samples, elapsed)
}
}
}));
}
barrier.wait();
release_barrier.wait();
let live_stats = allocator.stats();
stats_barrier.wait();
let mut samples = Vec::with_capacity(threads * allocations);
let mut elapsed = Duration::ZERO;
for worker in workers {
let (worker_samples, worker_elapsed) = worker.join().unwrap();
samples.extend(worker_samples);
elapsed = elapsed.max(worker_elapsed);
}
Metrics {
rate: (threads * allocations) as f64 / elapsed.as_secs_f64(),
p50_ns: percentile(&mut samples, 50),
p99_ns: percentile(&mut samples, 99),
locality_hits: live_stats.locality_hits,
global_searches: live_stats.global_searches,
global_candidate_probes: live_stats.global_candidate_probes,
random_jumps: live_stats.random_jumps,
jump_successes: live_stats.jump_successes,
post_jump_probes: live_stats.post_jump_probes,
cas_failures: live_stats.cas_failures,
steals: live_stats.steals,
reserved_activations: live_stats.reserved_span_activations,
territory_extensions: live_stats.territory_extensions,
territory_extension_spans: live_stats.territory_extension_spans,
reserved_run_activations: live_stats.reserved_run_activations,
run_territory_extensions: live_stats.run_territory_extensions,
run_territory_extension_extents: live_stats.run_territory_extension_extents,
run_local_window_refills: live_stats.run_local_window_refills,
run_local_window_hits: live_stats.run_local_window_hits,
run_local_window_releases: live_stats.run_local_window_releases,
snapshot_publishes: live_stats.topology_snapshot_publishes,
reserved_bytes: live_stats.reserved_bytes,
committed_bytes: live_stats.committed_bytes,
local_free_spans: live_stats.local_free_spans,
tree_leaf_splits: live_stats.tree_leaf_splits,
tree_parent_inserts: live_stats.tree_parent_inserts,
tree_internal_splits: live_stats.tree_internal_splits,
tree_logical_removes: live_stats.tree_logical_removes,
tree_leaf_upserts: live_stats.tree_leaf_upserts,
tree_redirects: live_stats.tree_redirects,
tree_parent_cleanups: live_stats.tree_parent_cleanups,
tree_batch_replacements: live_stats.tree_batch_replacements,
tree_help_calls: live_stats.tree_help_calls,
tree_operation_aborts: live_stats.tree_operation_aborts,
tree_candidate_discards: live_stats.tree_candidate_discards,
}
}
fn locality_burst(backend: Backend, size: usize, threads: usize, iterations: usize) -> Metrics {
burst_with_config(
backend,
size,
threads,
iterations,
Config {
recent_spans_per_class: 1,
locality_window_spans: 32,
locality_reserve_spans: 5,
red_cold_slow_paths: u64::MAX,
..Config::default()
},
64,
)
}
fn burst_with_config(
backend: Backend,
size: usize,
threads: usize,
iterations: usize,
config: Config,
burst_size: usize,
) -> Metrics {
let layout = layout_for(size);
let rounds = (iterations / burst_size).max(1);
let allocator = BoAllocator::new(config).unwrap();
let barrier = Arc::new(Barrier::new(threads + 1));
let mut workers = Vec::with_capacity(threads);
for worker_index in 0..threads {
let allocator = allocator.clone();
let barrier = Arc::clone(&barrier);
workers.push(spawn_bench_worker(worker_index, threads, move || {
for _ in 0..100 {
match backend {
Backend::Bomalloc => bomalloc_pair(&allocator, layout),
Backend::System => system_pair(layout),
}
}
barrier.wait();
let worker_start = Instant::now();
let mut samples = Vec::with_capacity(rounds);
for _ in 0..rounds {
let start = Instant::now();
match backend {
Backend::Bomalloc => {
let mut pointers = Vec::with_capacity(burst_size);
for _ in 0..burst_size {
let allocation = allocator.allocate(layout).unwrap();
pointers.push(NonNull::new(allocation.as_ptr() as *mut u8).unwrap());
}
for pointer in pointers.into_iter().rev() {
unsafe { allocator.deallocate(pointer, layout) };
}
}
Backend::System => {
let mut pointers = Vec::with_capacity(burst_size);
for _ in 0..burst_size {
unsafe {
let pointer = System.alloc(layout);
assert!(!pointer.is_null());
pointers.push(pointer);
}
}
for pointer in pointers.into_iter().rev() {
unsafe { System.dealloc(pointer, layout) };
}
}
}
samples.push(start.elapsed().as_nanos() / burst_size as u128);
}
(samples, worker_start.elapsed())
}));
}
barrier.wait();
let mut samples = Vec::new();
let mut elapsed = Duration::ZERO;
for worker in workers {
let (worker_samples, worker_elapsed) = worker.join().unwrap();
samples.extend(worker_samples);
elapsed = elapsed.max(worker_elapsed);
}
let total = threads * rounds * burst_size;
let stats = allocator.stats();
Metrics {
rate: total as f64 / elapsed.as_secs_f64(),
p50_ns: percentile(&mut samples, 50),
p99_ns: percentile(&mut samples, 99),
locality_hits: stats.locality_hits,
global_searches: stats.global_searches,
global_candidate_probes: stats.global_candidate_probes,
random_jumps: stats.random_jumps,
jump_successes: stats.jump_successes,
post_jump_probes: stats.post_jump_probes,
cas_failures: stats.cas_failures,
steals: stats.steals,
reserved_activations: stats.reserved_span_activations,
territory_extensions: stats.territory_extensions,
territory_extension_spans: stats.territory_extension_spans,
reserved_run_activations: stats.reserved_run_activations,
run_territory_extensions: stats.run_territory_extensions,
run_territory_extension_extents: stats.run_territory_extension_extents,
run_local_window_refills: stats.run_local_window_refills,
run_local_window_hits: stats.run_local_window_hits,
run_local_window_releases: stats.run_local_window_releases,
snapshot_publishes: stats.topology_snapshot_publishes,
reserved_bytes: stats.reserved_bytes,
committed_bytes: stats.committed_bytes,
local_free_spans: stats.local_free_spans,
tree_leaf_splits: stats.tree_leaf_splits,
tree_parent_inserts: stats.tree_parent_inserts,
tree_internal_splits: stats.tree_internal_splits,
tree_logical_removes: stats.tree_logical_removes,
tree_leaf_upserts: stats.tree_leaf_upserts,
tree_redirects: stats.tree_redirects,
tree_parent_cleanups: stats.tree_parent_cleanups,
tree_batch_replacements: stats.tree_batch_replacements,
tree_help_calls: stats.tree_help_calls,
tree_operation_aborts: stats.tree_operation_aborts,
tree_candidate_discards: stats.tree_candidate_discards,
}
}
fn mixed_burst(backend: Backend, threads: usize, iterations: usize) -> MixedMetrics {
const BURST_SIZE: usize = 64;
let layouts = MIXED_SIZES.map(layout_for);
let rounds = (iterations / BURST_SIZE).max(1);
let allocator = BoAllocator::new(benchmark_bomalloc_config()).unwrap();
let barrier = Arc::new(Barrier::new(threads + 1));
let mut workers = Vec::with_capacity(threads);
for worker_index in 0..threads {
let allocator = allocator.clone();
let barrier = Arc::clone(&barrier);
workers.push(spawn_bench_worker(worker_index, threads, move || {
for index in 0..MIXED_SIZES.len() * 16 {
let layout = layouts[index % layouts.len()];
match backend {
Backend::Bomalloc => bomalloc_pair(&allocator, layout),
Backend::System => system_pair(layout),
}
}
barrier.wait();
let worker_start = Instant::now();
let mut samples = Vec::with_capacity(rounds);
for _ in 0..rounds {
let start = Instant::now();
let mut pointers = Vec::with_capacity(BURST_SIZE);
for index in 0..BURST_SIZE {
let layout = layouts[index % layouts.len()];
let pointer = match backend {
Backend::Bomalloc => {
let allocation = allocator.allocate(layout).unwrap();
NonNull::new(allocation.as_ptr() as *mut u8).unwrap()
}
Backend::System => unsafe {
let pointer = System.alloc(layout);
NonNull::new(pointer).expect("system mixed allocation failed")
},
};
pointers.push((pointer, layout));
}
for (pointer, layout) in pointers.into_iter().rev() {
match backend {
Backend::Bomalloc => unsafe { allocator.deallocate(pointer, layout) },
Backend::System => unsafe { System.dealloc(pointer.as_ptr(), layout) },
}
}
samples.push(start.elapsed().as_nanos() / BURST_SIZE as u128);
}
(samples, worker_start.elapsed())
}));
}
barrier.wait();
let mut samples = Vec::new();
let mut elapsed = Duration::ZERO;
for worker in workers {
let (worker_samples, worker_elapsed) = worker.join().unwrap();
samples.extend(worker_samples);
elapsed = elapsed.max(worker_elapsed);
}
let stats = allocator.stats();
MixedMetrics {
rate: (threads * rounds * BURST_SIZE) as f64 / elapsed.as_secs_f64(),
p50_ns: percentile(&mut samples, 50),
p99_ns: percentile(&mut samples, 99),
committed_bytes: stats.committed_bytes,
reserved_bytes: stats.reserved_bytes,
locality_hits: stats.locality_hits,
global_searches: stats.global_searches,
}
}
#[derive(Clone, Copy)]
struct GenerationProfile {
label: &'static str,
survivor_stride: usize,
survivor_waves: usize,
}
#[derive(Clone, Copy)]
enum GenerationCollection {
Immediate,
NurseryReverse,
NurseryForward,
RuntimeReverse,
RuntimeForward,
}
impl GenerationCollection {
fn workload_prefix(self) -> &'static str {
match self {
Self::Immediate => "gen",
Self::NurseryReverse | Self::NurseryForward => "nursery",
Self::RuntimeReverse => "runtime-gc",
Self::RuntimeForward => "runtime-gc",
}
}
}
fn generational_wave(
backend: Backend,
threads: usize,
iterations: usize,
profile: GenerationProfile,
collection: GenerationCollection,
) -> GenerationMetrics {
const SIZES: [usize; 4] = [32, 96, 256, 1024];
let layouts = std::env::var("BOMALLOC_BENCH_NURSERY_SIZE")
.ok()
.and_then(|value| value.parse::<usize>().ok())
.map_or_else(
|| SIZES.map(layout_for),
|size| [layout_for(size); SIZES.len()],
);
let waves = (iterations / 2048).clamp(8, 64);
let allocations_per_wave = (iterations / waves).max(256);
let report_phases = std::env::var("BOMALLOC_BENCH_NURSERY_PHASES")
.ok()
.as_deref()
== Some("1")
&& !matches!(collection, GenerationCollection::Immediate);
let touch_objects = matches!(
collection,
GenerationCollection::RuntimeReverse | GenerationCollection::RuntimeForward
);
let allocator = BoAllocator::new(benchmark_bomalloc_config()).unwrap();
let barrier = Arc::new(Barrier::new(threads + 1));
let mut workers = Vec::with_capacity(threads);
for worker_index in 0..threads {
let allocator = allocator.clone();
let barrier = Arc::clone(&barrier);
workers.push(spawn_bench_worker(worker_index, threads, move || {
let mut generations: VecDeque<Vec<(NonNull<u8>, Layout)>> =
VecDeque::with_capacity(profile.survivor_waves + 1);
let mut newborns = Vec::with_capacity(allocations_per_wave);
let survivor_capacity = allocations_per_wave.div_ceil(profile.survivor_stride);
let mut spare_generations: Vec<Vec<(NonNull<u8>, Layout)>> = (0..=profile
.survivor_waves)
.map(|_| Vec::with_capacity(survivor_capacity))
.collect();
let mut samples = Vec::with_capacity(waves);
let phase_capacity = if report_phases { waves } else { 0 };
let mut phases = GenerationWorkerPhases {
birth_samples: Vec::with_capacity(phase_capacity),
reclaim_samples: Vec::with_capacity(phase_capacity),
expire_samples: Vec::with_capacity(phase_capacity),
..GenerationWorkerPhases::default()
};
let mut requested_live = 0_usize;
let mut peak_requested = 0_usize;
barrier.wait();
let started = Instant::now();
for wave in 0..waves {
let sample_start = Instant::now();
if touch_objects {
for generation in &generations {
for &(pointer, layout) in generation {
unsafe {
visit_service_object(ServiceAllocation {
address: pointer.as_ptr() as usize,
layout,
})
};
}
}
}
let expire_start = report_phases.then(Instant::now);
let mut expired_ops = 0;
let mut survivors = if generations.len() == profile.survivor_waves {
let mut expired = generations.pop_front().unwrap();
expired_ops = expired.len();
for (pointer, layout) in expired.drain(..) {
requested_live -= layout.size();
unsafe { backend_deallocate(backend, &allocator, pointer, layout) };
}
expired
} else {
spare_generations
.pop()
.expect("one generation buffer exists for each live cohort")
};
if let Some(expire_start) = expire_start {
if expired_ops != 0 {
let phase_elapsed = expire_start.elapsed();
phases.expire_elapsed += phase_elapsed;
phases.expire_ops += expired_ops;
phases
.expire_samples
.push(phase_elapsed.as_nanos() / expired_ops as u128);
}
}
match collection {
GenerationCollection::Immediate => {
for index in 0..allocations_per_wave {
let layout = layouts[(index + wave + worker_index) % layouts.len()];
let pointer = backend_allocate(backend, &allocator, layout);
requested_live += layout.size();
peak_requested = peak_requested.max(requested_live);
if touch_objects {
unsafe {
initialize_service_object(
pointer,
layout,
((worker_index as u64) << 48)
| ((wave as u64) << 24)
| index as u64,
)
};
}
if index % profile.survivor_stride == 0 {
survivors.push((pointer, layout));
} else {
requested_live -= layout.size();
unsafe { backend_deallocate(backend, &allocator, pointer, layout) };
}
}
}
GenerationCollection::NurseryReverse
| GenerationCollection::NurseryForward
| GenerationCollection::RuntimeReverse
| GenerationCollection::RuntimeForward => {
let birth_start = report_phases.then(Instant::now);
newborns.clear();
for index in 0..allocations_per_wave {
let layout = layouts[(index + wave + worker_index) % layouts.len()];
let pointer = backend_allocate(backend, &allocator, layout);
requested_live += layout.size();
peak_requested = peak_requested.max(requested_live);
if touch_objects {
unsafe {
initialize_service_object(
pointer,
layout,
((worker_index as u64) << 48)
| ((wave as u64) << 24)
| index as u64,
)
};
}
newborns.push((pointer, layout));
}
if let Some(birth_start) = birth_start {
let phase_elapsed = birth_start.elapsed();
phases.birth_elapsed += phase_elapsed;
phases.birth_ops += allocations_per_wave;
phases
.birth_samples
.push(phase_elapsed.as_nanos() / allocations_per_wave as u128);
}
let reclaim_start = report_phases.then(Instant::now);
let mut reclaim =
|index: usize, survivors: &mut Vec<(NonNull<u8>, Layout)>| {
let (pointer, layout) = newborns[index];
if index % profile.survivor_stride == 0 {
survivors.push((pointer, layout));
} else {
requested_live -= layout.size();
unsafe {
backend_deallocate(backend, &allocator, pointer, layout)
};
}
};
match collection {
GenerationCollection::NurseryReverse
| GenerationCollection::RuntimeReverse => {
for index in (0..newborns.len()).rev() {
reclaim(index, &mut survivors);
}
}
GenerationCollection::NurseryForward => {
for index in 0..newborns.len() {
reclaim(index, &mut survivors);
}
}
GenerationCollection::RuntimeForward => {
for index in 0..newborns.len() {
reclaim(index, &mut survivors);
}
}
GenerationCollection::Immediate => unreachable!(),
}
newborns.clear();
if let Some(reclaim_start) = reclaim_start {
let phase_elapsed = reclaim_start.elapsed();
phases.reclaim_elapsed += phase_elapsed;
phases.reclaim_ops += allocations_per_wave;
phases
.reclaim_samples
.push(phase_elapsed.as_nanos() / allocations_per_wave as u128);
}
}
}
generations.push_back(survivors);
samples.push(sample_start.elapsed().as_nanos() / allocations_per_wave as u128);
}
let elapsed = started.elapsed();
barrier.wait();
if matches!(backend, Backend::Bomalloc) {
let _ = allocator.stats();
}
barrier.wait();
let stats = matches!(backend, Backend::Bomalloc).then(|| allocator.stats());
let steady_requested = requested_live;
let observed_requested = stats
.as_ref()
.map_or(0, |snapshot| snapshot.requested_live_bytes);
let steady_usable = stats
.as_ref()
.map_or(0, |snapshot| snapshot.usable_live_bytes);
let peak_committed = stats
.as_ref()
.map_or(0, |snapshot| snapshot.committed_bytes);
let peak_young_spans = stats.as_ref().map_or(0, |snapshot| snapshot.young_spans);
let peak_sealed_spans = stats
.as_ref()
.map_or(0, |snapshot| snapshot.sealed_black_spans);
let peak_sealed_hole_bytes = stats
.as_ref()
.map_or(0, |snapshot| snapshot.sealed_hole_bytes);
for generation in generations {
for (pointer, layout) in generation {
requested_live -= layout.size();
unsafe { backend_deallocate(backend, &allocator, pointer, layout) };
}
}
debug_assert_eq!(requested_live, 0);
(
samples,
elapsed,
peak_requested,
steady_requested,
observed_requested,
steady_usable,
peak_committed,
peak_young_spans,
peak_sealed_spans,
peak_sealed_hole_bytes,
phases,
)
}));
}
barrier.wait();
barrier.wait();
barrier.wait();
let mut samples = Vec::with_capacity(threads * waves);
let mut elapsed = Duration::ZERO;
let mut peak_requested = 0;
let mut steady_requested = 0;
let mut observed_requested = 0;
let mut steady_usable = 0;
let mut peak_committed = 0;
let mut peak_young_spans = 0;
let mut peak_sealed_spans = 0;
let mut peak_sealed_hole_bytes = 0;
let mut phase_birth_samples = Vec::with_capacity(threads * waves);
let mut phase_reclaim_samples = Vec::with_capacity(threads * waves);
let mut phase_expire_samples = Vec::with_capacity(threads * waves);
let mut phase_birth_elapsed = Duration::ZERO;
let mut phase_reclaim_elapsed = Duration::ZERO;
let mut phase_expire_elapsed = Duration::ZERO;
let mut phase_birth_ops = 0;
let mut phase_reclaim_ops = 0;
let mut phase_expire_ops = 0;
for worker in workers {
let (
worker_samples,
worker_elapsed,
worker_peak_requested,
requested,
observed,
usable,
committed,
young_spans,
sealed_spans,
sealed_hole_bytes,
worker_phases,
) = worker.join().unwrap();
samples.extend(worker_samples);
elapsed = elapsed.max(worker_elapsed);
peak_requested += worker_peak_requested;
steady_requested += requested;
observed_requested = observed_requested.max(observed);
steady_usable = steady_usable.max(usable);
peak_committed = peak_committed.max(committed);
peak_young_spans = peak_young_spans.max(young_spans);
peak_sealed_spans = peak_sealed_spans.max(sealed_spans);
peak_sealed_hole_bytes = peak_sealed_hole_bytes.max(sealed_hole_bytes);
phase_birth_samples.extend(worker_phases.birth_samples);
phase_reclaim_samples.extend(worker_phases.reclaim_samples);
phase_expire_samples.extend(worker_phases.expire_samples);
phase_birth_elapsed = phase_birth_elapsed.max(worker_phases.birth_elapsed);
phase_reclaim_elapsed = phase_reclaim_elapsed.max(worker_phases.reclaim_elapsed);
phase_expire_elapsed = phase_expire_elapsed.max(worker_phases.expire_elapsed);
phase_birth_ops += worker_phases.birth_ops;
phase_reclaim_ops += worker_phases.reclaim_ops;
phase_expire_ops += worker_phases.expire_ops;
}
let stats = allocator.stats();
if matches!(backend, Backend::Bomalloc) {
assert_eq!(
observed_requested, steady_requested,
"modeled and allocator steady requested bytes diverged"
);
assert_eq!(
stats.requested_live_bytes, 0,
"generation workload leaked requested bytes"
);
allocator
.validate()
.expect("generation workload violated allocator invariants");
}
let phase_rate = |ops: usize, elapsed: Duration| {
if elapsed.is_zero() {
0.0
} else {
ops as f64 / elapsed.as_secs_f64()
}
};
let phase_percentile = |samples: &mut [u128], value: usize| {
if samples.is_empty() {
0
} else {
percentile(samples, value)
}
};
let phases = report_phases.then(|| GenerationPhaseMetrics {
birth_rate: phase_rate(phase_birth_ops, phase_birth_elapsed),
birth_p50_ns: phase_percentile(&mut phase_birth_samples, 50),
birth_p99_ns: phase_percentile(&mut phase_birth_samples, 99),
reclaim_rate: phase_rate(phase_reclaim_ops, phase_reclaim_elapsed),
reclaim_p50_ns: phase_percentile(&mut phase_reclaim_samples, 50),
reclaim_p99_ns: phase_percentile(&mut phase_reclaim_samples, 99),
expire_rate: phase_rate(phase_expire_ops, phase_expire_elapsed),
expire_p50_ns: phase_percentile(&mut phase_expire_samples, 50),
expire_p99_ns: phase_percentile(&mut phase_expire_samples, 99),
});
GenerationMetrics {
rate: (threads * waves * allocations_per_wave) as f64 / elapsed.as_secs_f64(),
p50_ns: percentile(&mut samples, 50),
p99_ns: percentile(&mut samples, 99),
steady_requested_bytes: steady_requested,
steady_usable_bytes: steady_usable,
peak_requested_bytes: peak_requested,
peak_committed_bytes: peak_committed,
reserved_bytes: stats.reserved_bytes,
global_candidate_probes: stats.global_candidate_probes,
locality_probes: stats.locality_probes,
locality_hits: stats.locality_hits,
global_searches: stats.global_searches,
reserved_span_activations: stats.reserved_span_activations,
random_jumps: stats.random_jumps,
young_spans: peak_young_spans,
sealed_spans: peak_sealed_spans,
sealed_hole_bytes: peak_sealed_hole_bytes,
cohort_promotions: stats.cohort_promotions,
cohort_recycles: stats.cohort_recycles,
phases,
}
}
fn fragmentation_wave(
backend: Backend,
threads: usize,
objects_per_thread: usize,
) -> FragmentationMetrics {
let layouts = MIXED_SIZES.map(layout_for);
let config = benchmark_bomalloc_config();
let locality_reserve_spans = config.locality_reserve_spans;
let allocator = BoAllocator::new(config).unwrap();
let baseline_rss = resident_bytes();
let barrier = Arc::new(Barrier::new(threads + 1));
let mut workers = Vec::with_capacity(threads);
for worker_index in 0..threads {
let allocator = allocator.clone();
let barrier = Arc::clone(&barrier);
workers.push(spawn_bench_worker(worker_index, threads, move || {
let start = Instant::now();
let mut allocations = Vec::with_capacity(objects_per_thread);
for index in 0..objects_per_thread {
let layout = layouts[index % layouts.len()];
let pointer = match backend {
Backend::Bomalloc => {
let allocation = allocator.allocate(layout).unwrap();
NonNull::new(allocation.as_ptr() as *mut u8).unwrap()
}
Backend::System => unsafe {
let pointer = System.alloc(layout);
NonNull::new(pointer).expect("system fragmentation allocation failed")
},
};
unsafe { touch_requested_pages(pointer, layout) };
allocations.push(Some((pointer, layout)));
}
let allocation_elapsed = start.elapsed();
barrier.wait();
barrier.wait();
for (index, allocation) in allocations.iter_mut().enumerate() {
if (index + worker_index) & 1 != 0 {
continue;
}
let (pointer, layout) = allocation.take().unwrap();
match backend {
Backend::Bomalloc => unsafe { allocator.deallocate(pointer, layout) },
Backend::System => unsafe { System.dealloc(pointer.as_ptr(), layout) },
}
}
barrier.wait();
barrier.wait();
for allocation in allocations.into_iter().flatten() {
let (pointer, layout) = allocation;
match backend {
Backend::Bomalloc => unsafe { allocator.deallocate(pointer, layout) },
Backend::System => unsafe { System.dealloc(pointer.as_ptr(), layout) },
}
}
allocation_elapsed
}));
}
barrier.wait();
let peak_rss = resident_bytes();
let peak_stats = allocator.stats();
barrier.wait();
barrier.wait();
let fragmented_rss = resident_bytes();
let fragmented_stats = allocator.stats();
if matches!(backend, Backend::Bomalloc) {
allocator.trim(TrimMode::Excess);
}
let fragmented_trim_rss = resident_bytes();
barrier.wait();
let mut allocation_elapsed = Duration::ZERO;
for worker in workers {
allocation_elapsed = allocation_elapsed.max(worker.join().unwrap());
}
let empty_rss = resident_bytes();
if matches!(backend, Backend::Bomalloc) {
allocator.trim(TrimMode::Aggressive);
}
let final_rss = resident_bytes();
let final_stats = allocator.stats();
let allocations = threads * objects_per_thread;
FragmentationMetrics {
locality_reserve_spans,
allocation_rate: allocations as f64 / allocation_elapsed.as_secs_f64(),
baseline_rss,
peak_rss,
fragmented_rss,
fragmented_trim_rss,
empty_rss,
final_rss,
peak_committed: peak_stats.committed_bytes,
fragmented_committed: fragmented_stats.committed_bytes,
final_committed: final_stats.committed_bytes,
peak_reserved: peak_stats.reserved_bytes,
final_reserved: final_stats.reserved_bytes,
}
}
fn remote_free(
backend: Backend,
size: usize,
pairs: usize,
iterations: usize,
remote_free_batching: bool,
) -> RemoteMetrics {
let layout = layout_for(size);
let allocator = BoAllocator::new(Config {
remote_free_batching,
..benchmark_bomalloc_config()
})
.unwrap();
let barrier = Arc::new(Barrier::new(pairs * 2 + 1));
let mut workers = Vec::with_capacity(pairs * 2);
for pair_index in 0..pairs {
let (sender, receiver) = mpsc::sync_channel::<usize>(1024);
let producer_allocator = allocator.clone();
let producer_barrier = Arc::clone(&barrier);
workers.push(spawn_bench_worker(pair_index * 2, pairs * 2, move || {
producer_barrier.wait();
let worker_start = Instant::now();
for _ in 0..iterations {
let address = match backend {
Backend::Bomalloc => {
let allocation = producer_allocator.allocate(layout).unwrap();
allocation.as_ptr() as *mut u8 as usize
}
Backend::System => unsafe {
let pointer = System.alloc(layout);
assert!(!pointer.is_null());
pointer as usize
},
};
sender.send(address).unwrap();
}
worker_start.elapsed()
}));
let consumer_allocator = allocator.clone();
let consumer_barrier = Arc::clone(&barrier);
workers.push(spawn_bench_worker(
pair_index * 2 + 1,
pairs * 2,
move || {
consumer_barrier.wait();
let worker_start = Instant::now();
for _ in 0..iterations {
let address = receiver.recv().unwrap();
match backend {
Backend::Bomalloc => unsafe {
consumer_allocator
.deallocate(NonNull::new(address as *mut u8).unwrap(), layout);
},
Backend::System => unsafe {
System.dealloc(address as *mut u8, layout);
},
}
}
worker_start.elapsed()
},
));
}
barrier.wait();
let mut elapsed = Duration::ZERO;
for worker in workers {
elapsed = elapsed.max(worker.join().unwrap());
}
let stats = allocator.stats();
RemoteMetrics {
rate: pairs as f64 * iterations as f64 / elapsed.as_secs_f64(),
batches: stats.remote_batches_drained,
blocks: stats.remote_blocks_drained,
cas_failures: stats.cas_failures,
locality_hits: stats.locality_hits,
global_searches: stats.global_searches,
reserved_activations: stats.reserved_span_activations,
}
}
fn remote_free_fanout(
backend: Backend,
consumers: usize,
iterations_per_consumer: usize,
remote_free_batching: bool,
) -> RemoteMetrics {
let layout = layout_for(96);
let allocator = BoAllocator::new(Config {
remote_free_batching,
..benchmark_bomalloc_config()
})
.unwrap();
let mut buckets: Vec<Vec<usize>> = (0..consumers)
.map(|_| Vec::with_capacity(iterations_per_consumer))
.collect();
for iteration in 0..(consumers * iterations_per_consumer) {
let address = match backend {
Backend::Bomalloc => {
let allocation = allocator.allocate(layout).unwrap();
allocation.as_ptr() as *mut u8 as usize
}
Backend::System => unsafe {
let pointer = System.alloc(layout);
assert!(!pointer.is_null());
pointer as usize
},
};
buckets[iteration % consumers].push(address);
}
let start = Arc::new(Barrier::new(consumers + 1));
let mut workers = Vec::with_capacity(consumers);
for (worker_index, addresses) in buckets.into_iter().enumerate() {
let allocator = allocator.clone();
let start = Arc::clone(&start);
workers.push(spawn_bench_worker(worker_index, consumers, move || {
start.wait();
let worker_start = Instant::now();
for address in addresses {
match backend {
Backend::Bomalloc => unsafe {
allocator.deallocate(NonNull::new(address as *mut u8).unwrap(), layout);
},
Backend::System => unsafe { System.dealloc(address as *mut u8, layout) },
}
}
worker_start.elapsed()
}));
}
start.wait();
let mut elapsed = Duration::ZERO;
for worker in workers {
elapsed = elapsed.max(worker.join().unwrap());
}
let stats = allocator.stats();
RemoteMetrics {
rate: (consumers * iterations_per_consumer) as f64 / elapsed.as_secs_f64(),
batches: stats.remote_batches_drained,
blocks: stats.remote_blocks_drained,
cas_failures: stats.cas_failures,
locality_hits: stats.locality_hits,
global_searches: stats.global_searches,
reserved_activations: stats.reserved_span_activations,
}
}
fn channel_only(pairs: usize, iterations: usize) -> f64 {
let barrier = Arc::new(Barrier::new(pairs * 2 + 1));
let mut workers = Vec::with_capacity(pairs * 2);
for pair_index in 0..pairs {
let (sender, receiver) = mpsc::sync_channel::<usize>(1024);
let producer_barrier = Arc::clone(&barrier);
workers.push(spawn_bench_worker(pair_index * 2, pairs * 2, move || {
producer_barrier.wait();
let start = Instant::now();
for value in 0..iterations {
sender.send(value).unwrap();
}
start.elapsed()
}));
let consumer_barrier = Arc::clone(&barrier);
workers.push(spawn_bench_worker(
pair_index * 2 + 1,
pairs * 2,
move || {
consumer_barrier.wait();
let start = Instant::now();
for _ in 0..iterations {
black_box(receiver.recv().unwrap());
}
start.elapsed()
},
));
}
barrier.wait();
let mut elapsed = Duration::ZERO;
for worker in workers {
elapsed = elapsed.max(worker.join().unwrap());
}
pairs as f64 * iterations as f64 / elapsed.as_secs_f64()
}
fn structural_churn(
backend: Backend,
threads: usize,
waves: usize,
allocations_per_wave: usize,
) -> ChurnMetrics {
let layout = layout_for(64 * 1024);
let allocator = BoAllocator::new(Config::default()).unwrap();
let phase = Arc::new(Barrier::new(threads + 1));
let mut workers = Vec::with_capacity(threads);
for worker_index in 0..threads {
let allocator = allocator.clone();
let phase = Arc::clone(&phase);
workers.push(spawn_bench_worker(worker_index, threads, move || {
let mut samples = Vec::with_capacity(waves);
for _ in 0..waves {
phase.wait();
let wave_start = Instant::now();
match backend {
Backend::Bomalloc => {
let mut pointers = Vec::with_capacity(allocations_per_wave);
for _ in 0..allocations_per_wave {
let allocation = allocator.allocate(layout).unwrap();
pointers.push(NonNull::new(allocation.as_ptr() as *mut u8).unwrap());
}
phase.wait();
for pointer in pointers.into_iter().rev() {
unsafe { allocator.deallocate(pointer, layout) };
}
}
Backend::System => {
let mut pointers = Vec::with_capacity(allocations_per_wave);
for _ in 0..allocations_per_wave {
let pointer = unsafe { System.alloc(layout) };
assert!(!pointer.is_null());
pointers.push(pointer);
}
phase.wait();
for pointer in pointers.into_iter().rev() {
unsafe { System.dealloc(pointer, layout) };
}
}
}
phase.wait();
phase.wait();
samples.push(wave_start.elapsed().as_nanos() / allocations_per_wave as u128);
}
samples
}));
}
let start = Instant::now();
let trace = std::env::var_os("BOMALLOC_BENCH_TRACE").is_some();
for wave in 0..waves {
if trace {
eprintln!("tree-churn threads={threads} wave={wave} phase=start");
}
phase.wait();
phase.wait();
if trace {
eprintln!("tree-churn threads={threads} wave={wave} phase=allocated");
}
phase.wait();
if trace {
eprintln!("tree-churn threads={threads} wave={wave} phase=freed");
}
if matches!(backend, Backend::Bomalloc) {
black_box(allocator.trim(TrimMode::Aggressive));
}
if trace {
eprintln!("tree-churn threads={threads} wave={wave} phase=trimmed");
}
phase.wait();
}
let elapsed = start.elapsed();
let mut samples = Vec::with_capacity(threads * waves);
for worker in workers {
samples.extend(worker.join().unwrap());
}
let stats = allocator.stats();
ChurnMetrics {
rate: (threads * waves * allocations_per_wave) as f64 / elapsed.as_secs_f64(),
p50_ns: percentile(&mut samples, 50),
p99_ns: percentile(&mut samples, 99),
logical_removes: stats.tree_logical_removes,
redirects: stats.tree_redirects,
cross_parent_redirects: stats.tree_cross_parent_redirects,
internal_redirects: stats.tree_internal_redirects,
internal_redistributions: stats.tree_internal_redistributions,
root_collapses: stats.tree_root_collapses,
help_calls: stats.tree_help_calls,
aborts: stats.tree_operation_aborts,
node_slots: stats.tree_node_slots,
live_nodes: stats.tree_live_nodes,
retired_nodes: stats.tree_retired_nodes,
reused_nodes: stats.tree_reused_nodes,
node_object_slots: stats.tree_node_object_slots,
live_node_objects: stats.tree_live_node_objects,
reused_node_objects: stats.tree_reused_node_objects,
span_slots: stats.span_descriptor_slots,
live_spans: stats.live_span_descriptors,
reused_spans: stats.reused_span_descriptors,
bitmap_slots: stats.bitmap_slots,
live_bitmaps: stats.live_bitmaps,
reused_bitmaps: stats.reused_bitmaps,
payload_vector_slots: stats.payload_vector_slots,
live_payload_vectors: stats.live_payload_vectors,
reused_payload_vectors: stats.reused_payload_vectors,
metadata_blocks: stats.tree_metadata_blocks,
metadata_reserved_bytes: stats.tree_metadata_reserved_bytes,
decommits: stats.decommits,
chunk_releases: stats.chunk_releases,
}
}
fn print_churn_matrix(backend: Backend, waves: usize, allocations_per_wave: usize) {
let mut single_rate = 0.0;
for threads in benchmark_thread_counts() {
if let Ok(selected) = std::env::var("BOMALLOC_BENCH_THREADS") {
if selected.parse::<usize>().ok() != Some(threads) {
continue;
}
}
let metrics = structural_churn(backend, threads, waves, allocations_per_wave);
if threads == 1 {
single_rate = metrics.rate;
}
let (speedup, efficiency) = if single_rate > 0.0 {
let speedup = metrics.rate / single_rate;
(speedup, speedup / threads as f64 * 100.0)
} else {
(f64::NAN, f64::NAN)
};
println!(
"workload=tree_churn allocator={:<8} threads={threads:>2} pairs/s={:>11.0} \
speedup={speedup:>5.2} efficiency={efficiency:>6.1}% p50={:>8}ns \
p99={:>8}ns remove={:>5} redirect={:>4}/{:>4}/{:>4}/{:>4} \
collapse={:>3} helps={:>6} aborts={:>5} nodes={:>4}/{:>4}/{:>4}/{:>4} \
objects={:>4}/{:>4}/{:>4} spans={:>4}/{:>4}/{:>4} \
bitmaps={:>4}/{:>4}/{:>4} \
payloads={:>4}/{:>4}/{:>4} \
vm={:>2}/{:>4}KiB decommit={:>4} release={:>3}",
backend.output_label(backend.label()),
metrics.rate,
metrics.p50_ns,
metrics.p99_ns,
metrics.logical_removes,
metrics.redirects,
metrics.cross_parent_redirects,
metrics.internal_redirects,
metrics.internal_redistributions,
metrics.root_collapses,
metrics.help_calls,
metrics.aborts,
metrics.node_slots,
metrics.live_nodes,
metrics.retired_nodes,
metrics.reused_nodes,
metrics.node_object_slots,
metrics.live_node_objects,
metrics.reused_node_objects,
metrics.span_slots,
metrics.live_spans,
metrics.reused_spans,
metrics.bitmap_slots,
metrics.live_bitmaps,
metrics.reused_bitmaps,
metrics.payload_vector_slots,
metrics.live_payload_vectors,
metrics.reused_payload_vectors,
metrics.metadata_blocks,
metrics.metadata_reserved_bytes / 1024,
metrics.decommits,
metrics.chunk_releases,
);
}
}
fn print_matrix(
workload: &str,
backend: Backend,
size: usize,
iterations: usize,
runner: fn(Backend, usize, usize, usize) -> Metrics,
) {
black_box(runner(backend, size, 2, (iterations / 10).max(10_000)));
let mut single_rate = 0.0;
for threads in benchmark_thread_counts() {
if !selected_thread_count(threads) {
continue;
}
let metrics = runner(backend, size, threads, iterations);
if threads == 1 {
single_rate = metrics.rate;
}
let speedup = metrics.rate / single_rate;
let efficiency = speedup / threads as f64 * 100.0;
println!(
"workload={workload:<9} allocator={:<8} size={size:>6} threads={threads:>2} \
pairs/s={:>12.0} speedup={speedup:>5.2} efficiency={efficiency:>6.1}% \
p50={:>5}ns p99={:>5}ns locality_hits={:>8} global={:>6} \
probes={:>7} jumps={:>5}/{:>5} post_jump={:>6} cas={:>7} \
activations={:>5} run_activations={:>5} publishes={:>4} steals={:>6}",
backend.output_label(backend.label()),
metrics.rate,
metrics.p50_ns,
metrics.p99_ns,
metrics.locality_hits,
metrics.global_searches,
metrics.global_candidate_probes,
metrics.jump_successes,
metrics.random_jumps,
metrics.post_jump_probes,
metrics.cas_failures,
metrics.reserved_activations,
metrics.reserved_run_activations,
metrics.snapshot_publishes,
metrics.steals,
);
}
}
fn print_tail_matrix(backend: Backend, size: usize, samples_per_thread: usize) {
let clock_unit = tail_clock_unit();
for threads in benchmark_thread_counts() {
let metrics = local_tail(backend, size, threads, samples_per_thread);
println!(
"workload=tail-local allocator={:<8} size={size:>6} threads={threads:>3} \
measured_pairs/s={:>11.0} samples={:>8} sample_unit=pair tail_unit={clock_unit} \
p50={:>6} p99={:>7} p99.99={:>9} max={:>10}",
backend.output_label(backend.label()),
metrics.rate,
metrics.samples,
metrics.p50_ticks,
metrics.p99_ticks,
metrics.p9999_ticks,
metrics.max_ticks,
);
}
}
fn print_headroom_matrix(backend: Backend, size: usize, iterations: usize) {
let burst_size = std::env::var("BOMALLOC_BENCH_HEADROOM_BURST")
.ok()
.and_then(|value| value.parse::<usize>().ok())
.unwrap_or(2048);
assert!(
burst_size != 0,
"BOMALLOC_BENCH_HEADROOM_BURST must be non-zero"
);
let extension_spans = std::env::var("BOMALLOC_BENCH_HEADROOM_EXTENSION")
.ok()
.and_then(|value| value.parse::<usize>().ok())
.unwrap_or(Config::default().adaptive_territory_extension_spans);
assert!(
extension_spans <= bomalloc::MAX_ADAPTIVE_TERRITORY_EXTENSION_SPANS,
"BOMALLOC_BENCH_HEADROOM_EXTENSION exceeds the supported maximum"
);
let territory_caps = match backend {
Backend::System => vec![0],
Backend::Bomalloc => match std::env::var("BOMALLOC_BENCH_HEADROOM_CAP").ok().as_deref() {
None | Some("") | Some("all") => vec![1, 5, 16, 32, 64],
Some(value) => {
let cap = value
.parse::<usize>()
.expect("BOMALLOC_BENCH_HEADROOM_CAP must be all, 1, 5, 16, 32, or 64");
assert!(
matches!(cap, 1 | 5 | 16 | 32 | 64),
"BOMALLOC_BENCH_HEADROOM_CAP must be all, 1, 5, 16, 32, or 64"
);
vec![cap]
}
},
};
for territory_cap in territory_caps {
let effective_extension = if matches!(backend, Backend::Bomalloc) {
extension_spans
} else {
0
};
let mut single_rate = 0.0;
for threads in benchmark_thread_counts() {
let mut config = benchmark_bomalloc_config();
if matches!(backend, Backend::Bomalloc) {
config.locality_reserve_spans = territory_cap;
config.adaptive_territory_extension_spans = extension_spans;
}
let metrics = burst_with_config(backend, size, threads, iterations, config, burst_size);
if threads == 1 {
single_rate = metrics.rate;
}
let speedup = metrics.rate / single_rate;
let efficiency = speedup / threads as f64 * 100.0;
println!(
"workload=headroom allocator={:<8} size={size:>6} threads={threads:>3} \
burst={burst_size:>5} territory_cap={territory_cap:>2} extension={effective_extension:>1} \
soft_cap_max/thread={:>4}KiB pairs/s={:>11.0} speedup={speedup:>5.2} \
efficiency={efficiency:>6.1}% p50={:>5}ns p99={:>6}ns \
committed={:>7}KiB reserved={:>5}MiB activations={:>6} \
global={:>6} probes={:>8} extensions={:>4}/{:>5}",
backend.output_label(backend.label()),
(territory_cap + effective_extension).min(bomalloc::MAX_LOCALITY_WINDOW_SPANS) * 64,
metrics.rate,
metrics.p50_ns,
metrics.p99_ns,
metrics.committed_bytes / 1024,
metrics.reserved_bytes / (1024 * 1024),
metrics.reserved_activations,
metrics.global_searches,
metrics.global_candidate_probes,
metrics.territory_extensions,
metrics.territory_extension_spans,
);
}
}
}
fn print_generation_matrix(
backend: Backend,
iterations: usize,
profile: GenerationProfile,
collection: GenerationCollection,
) {
for threads in benchmark_thread_counts() {
if !selected_thread_count(threads) {
continue;
}
let metrics = generational_wave(backend, threads, iterations, profile, collection);
let committed_demand = if metrics.peak_requested_bytes == 0 {
0.0
} else {
metrics.peak_committed_bytes as f64 / metrics.peak_requested_bytes as f64
};
let usable_steady = if metrics.steady_requested_bytes == 0 {
0.0
} else {
metrics.steady_usable_bytes as f64 / metrics.steady_requested_bytes as f64
};
println!(
"workload={}-{:<11} allocator={:<8} threads={threads:>2} allocs/s={:>11.0} \
p50={:>6}ns p99={:>6}ns steady_live={:>7}KiB peak_demand={:>7}KiB \
committed={:>7}KiB committed/demand={committed_demand:>5.2} \
usable/steady={usable_steady:>5.2} \
reserved={:>4}MiB probes={:>8} local={:>8}/{:>8} searches={:>6} \
activates={:>6} jumps={:>6} cohort={:>4}/{:>4} \
promote/recycle={:>5}/{:>5} sealed_holes={:>7}KiB",
collection.workload_prefix(),
profile.label,
backend.output_label(backend.label()),
metrics.rate,
metrics.p50_ns,
metrics.p99_ns,
metrics.steady_requested_bytes / 1024,
metrics.peak_requested_bytes / 1024,
metrics.peak_committed_bytes / 1024,
metrics.reserved_bytes / (1024 * 1024),
metrics.global_candidate_probes,
metrics.locality_hits,
metrics.locality_probes,
metrics.global_searches,
metrics.reserved_span_activations,
metrics.random_jumps,
metrics.young_spans,
metrics.sealed_spans,
metrics.cohort_promotions,
metrics.cohort_recycles,
metrics.sealed_hole_bytes / 1024,
);
if let Some(phases) = metrics.phases {
for (phase, rate, p50, p99) in [
(
"birth",
phases.birth_rate,
phases.birth_p50_ns,
phases.birth_p99_ns,
),
(
"reclaim",
phases.reclaim_rate,
phases.reclaim_p50_ns,
phases.reclaim_p99_ns,
),
(
"expire",
phases.expire_rate,
phases.expire_p50_ns,
phases.expire_p99_ns,
),
] {
println!(
"workload={}-phase-{phase:<7} allocator={:<8} threads={threads:>2} \
ops/s={rate:>11.0} p50={p50:>6}ns p99={p99:>6}ns",
collection.workload_prefix(),
backend.output_label(backend.label()),
);
}
}
}
}
fn print_mixed_matrix(backend: Backend, iterations: usize) {
let mut single_rate = 0.0;
for threads in benchmark_thread_counts() {
if !selected_thread_count(threads) {
continue;
}
let metrics = mixed_burst(backend, threads, iterations);
if threads == 1 {
single_rate = metrics.rate;
}
let speedup = metrics.rate / single_rate;
let efficiency = speedup / threads as f64 * 100.0;
println!(
"workload=mixed64 allocator={:<8} sizes=8 threads={threads:>2} \
pairs/s={:>12.0} speedup={speedup:>5.2} efficiency={efficiency:>6.1}% \
p50={:>5}ns p99={:>5}ns committed={:>6}KiB reserved={:>4}MiB \
locality={:>6} global={:>4}",
backend.output_label(backend.label()),
metrics.rate,
metrics.p50_ns,
metrics.p99_ns,
metrics.committed_bytes / 1024,
metrics.reserved_bytes / (1024 * 1024),
metrics.locality_hits,
metrics.global_searches,
);
}
}
fn print_service_matrix(backend: Backend, iterations: usize) {
let mut pair_rate = 0.0;
for threads in benchmark_service_thread_counts() {
let metrics = service_roundtrip(backend, threads, iterations);
if threads == 2 {
pair_rate = metrics.rate;
}
let pairs = threads / 2;
let speedup = metrics.rate / pair_rate;
let efficiency = speedup / pairs as f64 * 100.0;
println!(
"workload=service allocator={:<8} pairs={pairs:>2} threads={threads:>2} \
requests/s={:>11.0} speedup={speedup:>5.2} efficiency={efficiency:>6.1}% \
p50={:>6}ns p99={:>7}ns committed={:>7}KiB reserved={:>5}MiB \
remote={:>6}/{:>8} cas={:>7} steals={:>6} global={:>6}",
backend.output_label(backend.label()),
metrics.rate,
metrics.p50_ns,
metrics.p99_ns,
metrics.committed_bytes / 1024,
metrics.reserved_bytes / (1024 * 1024),
metrics.remote_batches,
metrics.remote_blocks,
metrics.cas_failures,
metrics.steals,
metrics.global_searches,
);
}
}
fn print_fragmentation_matrix(backend: Backend, objects_per_thread: usize) {
for threads in benchmark_thread_counts() {
if !selected_thread_count(threads) {
continue;
}
let metrics = fragmentation_wave(backend, threads, objects_per_thread);
let rss_kib = |value: usize| value.saturating_sub(metrics.baseline_rss) / 1024;
println!(
"workload=fragmentation allocator={:<8} threads={threads:>2} objects/thread={objects_per_thread:>5} territory_max={:>2} \
allocs/s={:>11.0} rss_delta_kib={:>7}/{:>7}/{:>7}/{:>7}/{:>7} \
committed_kib={:>7}/{:>7}/{:>7} reserved_mib={:>4}/{:>4}",
backend.output_label(backend.label()),
metrics.locality_reserve_spans,
metrics.allocation_rate,
rss_kib(metrics.peak_rss),
rss_kib(metrics.fragmented_rss),
rss_kib(metrics.fragmented_trim_rss),
rss_kib(metrics.empty_rss),
rss_kib(metrics.final_rss),
metrics.peak_committed / 1024,
metrics.fragmented_committed / 1024,
metrics.final_committed / 1024,
metrics.peak_reserved / (1024 * 1024),
metrics.final_reserved / (1024 * 1024),
);
}
}
fn print_growth_matrix(workload: &str, backend: Backend, size: usize, allocations: usize) {
let mut single_rate = 0.0;
for threads in benchmark_thread_counts() {
if !selected_thread_count(threads) {
continue;
}
let metrics = run_growth(backend, size, threads, allocations);
if threads == 1 {
single_rate = metrics.rate;
}
let speedup = metrics.rate / single_rate;
let efficiency = speedup / threads as f64 * 100.0;
println!(
"workload={workload:<10} allocator={:<8} size={size:>6} threads={threads:>2} \
allocs/s={:>11.0} speedup={speedup:>5.2} efficiency={efficiency:>6.1}% \
p50={:>7}ns p99={:>7}ns committed={:>6}KiB reserved={:>4}MiB \
local_free={:>4} run_activations={:>5} run_extensions={:>3}/{:>4} publishes={:>4} steals={:>6} \
run_window={:>4}/{:>5}/{:>4} \
probes={:>7} jumps={:>4}/{:>4} post_jump={:>6} cas={:>7} \
tree={:>3}/{:>3}/{:>3} update={:>3}/{:>3}/{:>3}/{:>3}/{:>3} \
helps={:>5} aborts={:>4} discarded={:>4}",
backend.output_label(backend.label()),
metrics.rate,
metrics.p50_ns,
metrics.p99_ns,
metrics.committed_bytes / 1024,
metrics.reserved_bytes / (1024 * 1024),
metrics.local_free_spans,
metrics.reserved_run_activations,
metrics.run_territory_extensions,
metrics.run_territory_extension_extents,
metrics.snapshot_publishes,
metrics.steals,
metrics.run_local_window_refills,
metrics.run_local_window_hits,
metrics.run_local_window_releases,
metrics.global_candidate_probes,
metrics.jump_successes,
metrics.random_jumps,
metrics.post_jump_probes,
metrics.cas_failures,
metrics.tree_leaf_splits,
metrics.tree_parent_inserts,
metrics.tree_internal_splits,
metrics.tree_logical_removes,
metrics.tree_leaf_upserts,
metrics.tree_redirects,
metrics.tree_parent_cleanups,
metrics.tree_batch_replacements,
metrics.tree_help_calls,
metrics.tree_operation_aborts,
metrics.tree_candidate_discards,
);
}
}
fn main() {
let iterations = std::env::var("BOMALLOC_BENCH_ITERS")
.ok()
.and_then(|value| value.parse().ok())
.unwrap_or(500_000);
let selected = std::env::var("BOMALLOC_BENCH_WORKLOAD").unwrap_or_else(|_| "all".into());
if let Some(mode) = benchmark_affinity_mode() {
println!(
"# worker_affinity={mode:?} available_cpu_ids={:?}",
available_cpu_ids()
);
}
if selected == "all" || selected == "local" {
println!("# local hot allocate/deallocate pairs");
for backend in [Backend::System, Backend::Bomalloc] {
if !selected_allocator(backend.label()) {
continue;
}
for size in LOCAL_SIZES {
if selected_size(size) {
print_matrix("local_hot", backend, size, iterations, local_hot);
}
}
}
}
if selected == "tail" {
let size = std::env::var("BOMALLOC_BENCH_SIZE")
.ok()
.and_then(|value| value.parse::<usize>().ok())
.unwrap_or(96);
let samples_per_thread = std::env::var("BOMALLOC_BENCH_TAIL_SAMPLES")
.ok()
.and_then(|value| value.parse::<usize>().ok())
.unwrap_or(100_000);
assert!(
samples_per_thread >= 10_000,
"BOMALLOC_BENCH_TAIL_SAMPLES must be at least 10000 for p99.99"
);
println!(
"# individually timed local allocate/deallocate pairs; TSC/fallback-clock cost is included; p99.99 uses exact pooled samples"
);
for backend in [Backend::System, Backend::Bomalloc] {
if selected_allocator(backend.label()) {
print_tail_matrix(backend, size, samples_per_thread);
}
}
}
if selected == "headroom" {
let size = std::env::var("BOMALLOC_BENCH_SIZE")
.ok()
.and_then(|value| value.parse::<usize>().ok())
.unwrap_or(1024);
println!("# burst allocation with increasing stealable, uncommitted owner-territory caps");
for backend in [Backend::System, Backend::Bomalloc] {
if selected_allocator(backend.label()) {
print_headroom_matrix(backend, size, iterations);
}
}
}
if selected == "magazine" {
println!("# TLS-exclusive magazine A/B (shared bitmap vs adaptive max-capacity 8)");
for size in LOCAL_SIZES {
if selected_size(size) {
print_magazine_matrix("magazine", size, iterations, local_hot_with_config);
}
}
}
if selected == "magazine-burst" {
println!("# adaptive magazine burst-64 A/B");
for size in LOCAL_SIZES {
if selected_size(size) {
print_magazine_matrix("magazine64", size, iterations, magazine_burst_with_config);
}
}
}
if selected == "recent-free" {
println!("# four recycled holes in distinct bitmap words; TLS magazine disabled");
print_recent_free_matrix(iterations);
}
if selected == "all" || selected == "burst" {
println!("# burst-64 allocate then reverse free");
for backend in [Backend::System, Backend::Bomalloc] {
if !selected_allocator(backend.label()) {
continue;
}
for size in [
16,
96,
4096,
8 * 1024,
12 * 1024,
16 * 1024,
24 * 1024,
32 * 1024,
] {
if selected_size(size) {
print_matrix("burst64", backend, size, iterations, burst);
}
}
}
}
if selected == "all" || selected == "mixed" {
println!("# mixed burst-64 across 16B, 24B, 96B, 192B, 1KiB, 4KiB, 16KiB, and 32KiB");
for backend in [Backend::System, Backend::Bomalloc] {
if selected_allocator(backend.label()) {
print_mixed_matrix(backend, iterations);
}
}
}
if selected == "service" {
println!(
"# bidirectional actor request/response: mixed objects are allocated by one thread, touched/freed by its peer, with 1/16 surviving eight request waves"
);
for backend in [Backend::System, Backend::Bomalloc] {
if selected_allocator(backend.label()) {
print_service_matrix(backend, iterations);
}
}
}
if selected == "generation" || selected == "generation-young" {
println!("# generational waves: 95% die in their birth wave, 5% survive eight waves");
let profile = GenerationProfile {
label: "young-heavy",
survivor_stride: 20,
survivor_waves: 8,
};
for backend in [Backend::System, Backend::Bomalloc] {
if selected_allocator(backend.label()) {
print_generation_matrix(
backend,
iterations,
profile,
GenerationCollection::Immediate,
);
}
}
}
if selected == "generation" || selected == "generation-bimodal" {
println!("# generational waves: 80% die in their birth wave, 20% survive eight waves");
let profile = GenerationProfile {
label: "bimodal",
survivor_stride: 5,
survivor_waves: 8,
};
for backend in [Backend::System, Backend::Bomalloc] {
if selected_allocator(backend.label()) {
print_generation_matrix(
backend,
iterations,
profile,
GenerationCollection::Immediate,
);
}
}
}
if selected == "generation" || selected == "generation-adversarial" {
println!("# generational waves: 50% survive four waves (fragmentation guardrail)");
let profile = GenerationProfile {
label: "adversarial",
survivor_stride: 2,
survivor_waves: 4,
};
for backend in [Backend::System, Backend::Bomalloc] {
if selected_allocator(backend.label()) {
print_generation_matrix(
backend,
iterations,
profile,
GenerationCollection::Immediate,
);
}
}
}
if selected == "nursery" || selected == "nursery-young" {
println!(
"# nursery waves: complete birth cohort, then reclaim 95% from the hot frontier; 5% survive eight waves"
);
let profile = GenerationProfile {
label: "young-heavy",
survivor_stride: 20,
survivor_waves: 8,
};
for backend in [Backend::System, Backend::Bomalloc] {
if selected_allocator(backend.label()) {
print_generation_matrix(
backend,
iterations,
profile,
GenerationCollection::NurseryReverse,
);
}
}
}
if selected == "nursery" || selected == "nursery-bimodal" {
println!(
"# nursery waves: complete birth cohort, then reclaim 80% from the hot frontier; 20% survive eight waves"
);
let profile = GenerationProfile {
label: "bimodal",
survivor_stride: 5,
survivor_waves: 8,
};
for backend in [Backend::System, Backend::Bomalloc] {
if selected_allocator(backend.label()) {
print_generation_matrix(
backend,
iterations,
profile,
GenerationCollection::NurseryReverse,
);
}
}
}
if selected == "nursery" || selected == "nursery-adversarial" {
println!(
"# nursery waves: 50% survive four waves and dead objects are swept oldest-first (fragmentation guardrail)"
);
let profile = GenerationProfile {
label: "adversarial",
survivor_stride: 2,
survivor_waves: 4,
};
for backend in [Backend::System, Backend::Bomalloc] {
if selected_allocator(backend.label()) {
print_generation_matrix(
backend,
iterations,
profile,
GenerationCollection::NurseryForward,
);
}
}
}
if selected == "runtime-gc" {
println!(
"# runtime GC waves: initialized objects, survivor graph scans, hot-frontier reclamation, 10% surviving eight waves"
);
let profile = GenerationProfile {
label: "runtime",
survivor_stride: 10,
survivor_waves: 8,
};
for backend in [Backend::System, Backend::Bomalloc] {
if selected_allocator(backend.label()) {
print_generation_matrix(
backend,
iterations,
profile,
GenerationCollection::RuntimeReverse,
);
}
}
}
if selected == "runtime-gc-forward" {
println!(
"# runtime GC control: identical initialization, survivor scans, and promotion, with oldest-first reclamation"
);
let profile = GenerationProfile {
label: "forward",
survivor_stride: 10,
survivor_waves: 8,
};
for backend in [Backend::System, Backend::Bomalloc] {
if selected_allocator(backend.label()) {
print_generation_matrix(
backend,
iterations,
profile,
GenerationCollection::RuntimeForward,
);
}
}
}
if selected == "fragmentation" {
let objects_per_thread = (iterations / 256).clamp(1024, 4096);
println!(
"# touched eight-size fragmentation wave; rss=peak/half-free/excess-trim/empty/aggressive-trim"
);
for backend in [Backend::System, Backend::Bomalloc] {
if selected_allocator(backend.label()) {
print_fragmentation_matrix(backend, objects_per_thread);
}
}
}
if selected == "all" || selected == "locality" {
println!("# locality-window burst with one cached span per class");
for backend in [Backend::System, Backend::Bomalloc] {
if !selected_allocator(backend.label()) {
continue;
}
print_matrix("locality64", backend, 4096, iterations, locality_burst);
}
}
if selected == "all" || selected == "run" {
println!("# page-run burst-8 reuse path");
for backend in [Backend::System, Backend::Bomalloc] {
if !selected_allocator(backend.label()) {
continue;
}
print_matrix(
"run8",
backend,
64 * 1024,
iterations.min(100_000),
run_burst,
);
}
}
if selected == "all" || selected == "run-growth" {
println!("# cold concurrent page-run extent growth (64 live allocations per worker)");
for backend in [Backend::System, Backend::Bomalloc] {
if !selected_allocator(backend.label()) {
continue;
}
print_growth_matrix("run_growth", backend, 64 * 1024, 64);
}
}
if selected == "contention-growth" {
println!("# synchronized cold 96-byte growth for conflict/jump diagnostics");
for backend in [Backend::System, Backend::Bomalloc] {
if !selected_allocator(backend.label()) {
continue;
}
print_growth_matrix("contention", backend, 96, 2048);
}
}
if selected == "all" || selected == "run-sparse" {
println!("# sparse concurrent page-run territory (one live allocation per worker)");
for backend in [Backend::System, Backend::Bomalloc] {
if !selected_allocator(backend.label()) {
continue;
}
print_growth_matrix("run_sparse", backend, 64 * 1024, 1);
}
}
if selected == "all" || selected == "tree-churn" {
let waves = (iterations / 8192).clamp(8, 64);
println!(
"# synchronized 64 KiB growth/free/aggressive-trim waves ({waves} waves, 64 allocations per worker)"
);
for backend in [Backend::System, Backend::Bomalloc] {
if !selected_allocator(backend.label()) {
continue;
}
print_churn_matrix(backend, waves, 64);
}
}
if selected == "all" || selected == "remote" {
let remote_size = std::env::var("BOMALLOC_BENCH_SIZE")
.ok()
.and_then(|value| value.parse::<usize>().ok())
.unwrap_or(96);
println!(
"# producer/consumer remote-free pairs (channel included), size={remote_size} bytes"
);
if selected_allocator("none") {
for pairs in [1, 2, 4, 8] {
if selected_remote_thread_count(pairs * 2) {
let rate = channel_only(pairs, iterations / 2);
println!(
"workload=channel96 allocator=none pairs={pairs:>2} threads={:>2} pairs/s={rate:>12.0}",
pairs * 2,
);
}
}
}
for (backend, label, batching) in [
(Backend::System, "system", false),
(Backend::Bomalloc, "bomalloc", false),
(Backend::Bomalloc, "bomalloc-batched", true),
] {
if !selected_allocator(label) {
continue;
}
let output_label = backend.output_label(label);
let mut single_rate = 0.0;
for pairs in [1, 2, 4, 8] {
if !selected_remote_thread_count(pairs * 2) {
continue;
}
let metrics = remote_free(backend, remote_size, pairs, iterations / 2, batching);
if pairs == 1 {
single_rate = metrics.rate;
}
let speedup = metrics.rate / single_rate;
let efficiency = speedup / pairs as f64 * 100.0;
let average_batch = if metrics.batches == 0 {
0.0
} else {
metrics.blocks as f64 / metrics.batches as f64
};
println!(
"workload=remote{remote_size:<5} allocator={:<8} pairs={pairs:>2} threads={:>2} \
pairs/s={:>12.0} speedup={speedup:>5.2} efficiency={efficiency:>6.1}% \
batches={:>6} blocks={:>9} avg_batch={average_batch:>6.1} \
cas={:>8} locality={:>8} global={:>6} activations={:>5}",
output_label,
pairs * 2,
metrics.rate,
metrics.batches,
metrics.blocks,
metrics.cas_failures,
metrics.locality_hits,
metrics.global_searches,
metrics.reserved_activations,
);
}
}
}
if selected == "remote-hotspot" {
println!("# preallocated 96-byte objects, many consumers freeing one owner's spans");
for (backend, label, batching) in [
(Backend::System, "system", false),
(Backend::Bomalloc, "bomalloc", false),
(Backend::Bomalloc, "bomalloc-batched", true),
] {
if !selected_allocator(label) {
continue;
}
let output_label = backend.output_label(label);
let mut single_rate = 0.0;
for consumers in [1, 3, 7, 15] {
let threads = consumers + 1;
if !selected_remote_thread_count(threads) {
continue;
}
let metrics = remote_free_fanout(backend, consumers, iterations / 2, batching);
if consumers == 1 {
single_rate = metrics.rate;
}
let speedup = metrics.rate / single_rate;
let efficiency = speedup / consumers as f64 * 100.0;
let average_batch = if metrics.batches == 0 {
0.0
} else {
metrics.blocks as f64 / metrics.batches as f64
};
println!(
"workload=remote-hotspot96 allocator={:<15} consumers={consumers:>2} threads={threads:>2} \
frees/s={:>12.0} speedup={speedup:>5.2} efficiency={efficiency:>6.1}% \
batches={:>6} blocks={:>9} avg_batch={average_batch:>6.1} \
cas={:>8} locality={:>8} global={:>6} activations={:>5}",
output_label,
metrics.rate,
metrics.batches,
metrics.blocks,
metrics.cas_failures,
metrics.locality_hits,
metrics.global_searches,
metrics.reserved_activations,
);
}
}
}
}