use std::sync::atomic::Ordering;
#[cfg(unix)]
use std::collections::HashMap;
use crate::context::{
self, COVERAGE_BITMAP_PTR, ENERGY_BUDGET_PTR, EXPLORED_MAP_PTR, SHARED_RECIPE, SHARED_STATS,
};
#[cfg(unix)]
use crate::context::{BITMAP_POOL, BITMAP_POOL_SLOTS};
use crate::coverage::{COVERAGE_MAP_SIZE, CoverageBitmap, ExploredMap};
use crate::shared_stats::MAX_RECIPE_ENTRIES;
fn compute_child_seed(parent_seed: u64, mark_name: &str, child_idx: u32) -> u64 {
let mut hash: u64 = 0xcbf2_9ce4_8422_2325;
for &byte in mark_name.as_bytes() {
hash ^= u64::from(byte);
hash = hash.wrapping_mul(0x0100_0000_01b3);
}
hash ^= parent_seed;
hash = hash.wrapping_mul(0x0100_0000_01b3);
hash ^= u64::from(child_idx);
hash = hash.wrapping_mul(0x0100_0000_01b3);
hash
}
#[derive(Debug, Clone)]
pub enum Parallelism {
MaxCores,
HalfCores,
Cores(usize),
MaxCoresMinus(usize),
}
#[cfg(unix)]
fn resolve_parallelism(p: &Parallelism) -> usize {
let ncpus = unsafe { libc::sysconf(libc::_SC_NPROCESSORS_ONLN) };
let ncpus = if ncpus > 0 {
usize::try_from(ncpus).unwrap_or(1)
} else {
1
};
let n = match p {
Parallelism::MaxCores => ncpus,
Parallelism::HalfCores => ncpus / 2,
Parallelism::Cores(c) => *c,
Parallelism::MaxCoresMinus(minus) => ncpus.saturating_sub(*minus),
};
n.max(1) }
#[cfg(unix)]
fn get_or_init_pool(slot_count: usize) -> *mut u8 {
let existing = BITMAP_POOL.with(std::cell::Cell::get);
let existing_slots = BITMAP_POOL_SLOTS.with(std::cell::Cell::get);
if !existing.is_null() && existing_slots >= slot_count {
return existing;
}
if !existing.is_null() {
unsafe {
crate::shared_mem::free_shared(existing, existing_slots * COVERAGE_MAP_SIZE);
}
BITMAP_POOL.with(|c| c.set(std::ptr::null_mut()));
BITMAP_POOL_SLOTS.with(|c| c.set(0));
}
match crate::shared_mem::alloc_shared(slot_count * COVERAGE_MAP_SIZE) {
Ok(ptr) => {
BITMAP_POOL.with(|c| c.set(ptr));
BITMAP_POOL_SLOTS.with(|c| c.set(slot_count));
ptr
}
Err(_) => std::ptr::null_mut(),
}
}
#[cfg(unix)]
fn pool_slot(pool_base: *mut u8, idx: usize) -> *mut u8 {
unsafe { pool_base.add(idx * COVERAGE_MAP_SIZE) }
}
#[cfg(unix)]
fn setup_child(
child_seed: u64,
split_call_count: u64,
stats_ptr: *mut crate::shared_stats::SharedStats,
) {
context::rng_reseed(child_seed);
context::with_ctx_mut(|ctx| {
ctx.is_child = true;
ctx.depth += 1;
ctx.current_seed = child_seed;
ctx.recipe.push((split_call_count, child_seed));
});
if !stats_ptr.is_null() {
unsafe {
(*stats_ptr).total_timelines.fetch_add(1, Ordering::Relaxed);
}
}
BITMAP_POOL.with(|c| c.set(std::ptr::null_mut()));
BITMAP_POOL_SLOTS.with(|c| c.set(0));
crate::sancov::reset_bss_counters();
crate::sancov::SANCOV_POOL.with(|c| c.set(std::ptr::null_mut()));
crate::sancov::SANCOV_POOL_SLOTS.with(|c| c.set(0));
}
#[cfg(unix)]
struct ForkSharedState {
split_call_count: u64,
vm_ptr: *mut u8,
stats_ptr: *mut crate::shared_stats::SharedStats,
pool_base: *mut u8,
sancov_pool_base: *mut u8,
}
#[cfg(unix)]
struct ParallelState {
slot_count: usize,
pool_base: *mut u8,
sancov_pool_base: *mut u8,
parent_sancov_transfer: *mut u8,
parallel: bool,
}
#[cfg(unix)]
fn resolve_parallel_state() -> ParallelState {
let parallelism = context::with_ctx(|ctx| ctx.parallelism.clone());
let (slot_count, pool_base) = if let Some(ref p) = parallelism {
let sc = resolve_parallelism(p);
let pb = get_or_init_pool(sc);
if pb.is_null() {
(0, std::ptr::null_mut())
} else {
(sc, pb)
}
} else {
(0, std::ptr::null_mut())
};
let parallel = slot_count > 0;
let sancov_pool_base = if parallel {
crate::sancov::get_or_init_sancov_pool(slot_count)
} else {
std::ptr::null_mut()
};
let parent_sancov_transfer = if parallel && !sancov_pool_base.is_null() {
crate::sancov::SANCOV_TRANSFER.with(std::cell::Cell::get)
} else {
std::ptr::null_mut()
};
ParallelState {
slot_count,
pool_base,
sancov_pool_base,
parent_sancov_transfer,
parallel,
}
}
#[cfg(unix)]
fn drain_active_children(
active: &mut HashMap<libc::pid_t, (u64, usize)>,
free_slots: &mut Vec<usize>,
shared: &ForkSharedState,
batch_has_new: &mut bool,
) {
while !active.is_empty() {
reap_one(active, free_slots, shared, batch_has_new);
}
}
#[cfg(unix)]
fn restore_parent_state(state: &ParallelState, bm_ptr: *mut u8, parent_bitmap_backup: &[u8]) {
if state.parallel {
COVERAGE_BITMAP_PTR.with(|c| c.set(bm_ptr));
if !state.sancov_pool_base.is_null() {
crate::sancov::SANCOV_TRANSFER.with(|c| c.set(state.parent_sancov_transfer));
}
} else if !bm_ptr.is_null() {
unsafe {
std::ptr::copy_nonoverlapping(parent_bitmap_backup.as_ptr(), bm_ptr, COVERAGE_MAP_SIZE);
}
}
}
#[cfg(unix)]
enum SpawnOutcome {
Continued,
Stop,
InChild,
}
#[cfg(unix)]
fn spawn_parallel_child(
child_seed: u64,
shared: &ForkSharedState,
active: &mut HashMap<libc::pid_t, (u64, usize)>,
free_slots: &mut Vec<usize>,
batch_has_new: &mut bool,
) -> SpawnOutcome {
while free_slots.is_empty() {
reap_one(active, free_slots, shared, batch_has_new);
}
let Some(slot) = free_slots.pop() else {
return SpawnOutcome::Stop;
};
let slot_ptr = pool_slot(shared.pool_base, slot);
unsafe {
std::ptr::write_bytes(slot_ptr, 0, COVERAGE_MAP_SIZE);
}
COVERAGE_BITMAP_PTR.with(|c| c.set(slot_ptr));
if !shared.sancov_pool_base.is_null() {
let sancov_len = crate::sancov::sancov_edge_count();
unsafe {
let sancov_slot = crate::sancov::sancov_pool_slot(shared.sancov_pool_base, slot);
std::ptr::write_bytes(sancov_slot, 0, sancov_len);
crate::sancov::SANCOV_TRANSFER.with(|c| c.set(sancov_slot));
}
}
let pid = unsafe { libc::fork() };
match pid {
-1 => {
free_slots.push(slot);
SpawnOutcome::Stop
}
0 => {
setup_child(child_seed, shared.split_call_count, shared.stats_ptr);
SpawnOutcome::InChild
}
child_pid => {
active.insert(child_pid, (child_seed, slot));
SpawnOutcome::Continued
}
}
}
#[cfg(unix)]
fn spawn_sequential_child(
child_seed: u64,
bm_ptr: *mut u8,
shared: &ForkSharedState,
batch_has_new: &mut bool,
track_new_bits: bool,
) -> SpawnOutcome {
if !bm_ptr.is_null() {
let bm = unsafe { CoverageBitmap::new(bm_ptr) };
bm.clear();
}
crate::sancov::clear_transfer_buffer();
let pid = unsafe { libc::fork() };
match pid {
-1 => SpawnOutcome::Stop,
0 => {
setup_child(child_seed, shared.split_call_count, shared.stats_ptr);
SpawnOutcome::InChild
}
child_pid => {
let mut status: libc::c_int = 0;
unsafe { libc::waitpid(child_pid, &raw mut status, 0) };
if !bm_ptr.is_null() && !shared.vm_ptr.is_null() {
let bm = unsafe { CoverageBitmap::new(bm_ptr) };
let vm = unsafe { ExploredMap::new(shared.vm_ptr) };
if track_new_bits && vm.has_new_bits(&bm) {
*batch_has_new = true;
}
vm.merge_from(&bm);
}
*batch_has_new |= crate::sancov::has_new_sancov_coverage();
if libc::WIFEXITED(status) && libc::WEXITSTATUS(status) == 42 {
if !shared.stats_ptr.is_null() {
unsafe {
(*shared.stats_ptr)
.bug_found
.fetch_add(1, Ordering::Relaxed);
}
}
save_bug_recipe(shared.split_call_count, child_seed);
}
if !shared.stats_ptr.is_null() {
unsafe {
(*shared.stats_ptr)
.fork_points
.fetch_add(1, Ordering::Relaxed);
}
}
SpawnOutcome::Continued
}
}
}
#[cfg(unix)]
fn reap_one(
active: &mut HashMap<libc::pid_t, (u64, usize)>,
free_slots: &mut Vec<usize>,
shared: &ForkSharedState,
batch_has_new: &mut bool,
) {
let mut status: libc::c_int = 0;
let finished_pid = unsafe { libc::waitpid(-1, &raw mut status, 0) };
if finished_pid <= 0 {
return;
}
let Some((child_seed, slot)) = active.remove(&finished_pid) else {
return;
};
if !shared.vm_ptr.is_null() {
let child_bm = unsafe { CoverageBitmap::new(pool_slot(shared.pool_base, slot)) };
let vm = unsafe { ExploredMap::new(shared.vm_ptr) };
if vm.has_new_bits(&child_bm) {
*batch_has_new = true;
}
vm.merge_from(&child_bm);
}
if !shared.sancov_pool_base.is_null() {
let sancov_slot = unsafe { crate::sancov::sancov_pool_slot(shared.sancov_pool_base, slot) };
if crate::sancov::has_new_sancov_coverage_from(sancov_slot) {
*batch_has_new = true;
}
}
if libc::WIFEXITED(status) && libc::WEXITSTATUS(status) == 42 {
if !shared.stats_ptr.is_null() {
unsafe {
(*shared.stats_ptr)
.bug_found
.fetch_add(1, Ordering::Relaxed);
}
}
save_bug_recipe(shared.split_call_count, child_seed);
}
if !shared.stats_ptr.is_null() {
unsafe {
(*shared.stats_ptr)
.fork_points
.fetch_add(1, Ordering::Relaxed);
}
}
free_slots.push(slot);
}
#[derive(Debug, Clone)]
pub struct AdaptiveConfig {
pub batch_size: u32,
pub min_timelines: u32,
pub max_timelines: u32,
pub per_mark_energy: i64,
pub warm_min_timelines: Option<u32>,
}
#[cfg(unix)]
pub(crate) fn dispatch_split(mark_name: &str, slot_idx: usize) {
let has_adaptive = ENERGY_BUDGET_PTR.with(|c| !c.get().is_null());
if has_adaptive {
adaptive_split_on_discovery(mark_name, slot_idx);
} else {
split_on_discovery(mark_name);
}
}
#[cfg(not(unix))]
pub(crate) fn dispatch_split(_mark_name: &str, _slot_idx: usize) {}
#[cfg(unix)]
fn read_adaptive_batch_config() -> (u32, u32, u32) {
context::with_ctx(|ctx| {
let (batch_size, min_timelines, max_timelines) =
ctx.adaptive.as_ref().map_or((4, 1, 16), |a| {
(a.batch_size, a.min_timelines, a.max_timelines)
});
let warm_min = ctx
.adaptive
.as_ref()
.and_then(|a| a.warm_min_timelines)
.unwrap_or(batch_size);
let effective_min = if ctx.warm_start {
warm_min
} else {
min_timelines
};
(batch_size, max_timelines, effective_min)
})
}
#[cfg(unix)]
fn adaptive_split_on_discovery(mark_name: &str, slot_idx: usize) {
let (ctx_active, depth, max_depth, current_seed) =
context::with_ctx(|ctx| (ctx.active, ctx.depth, ctx.max_depth, ctx.current_seed));
if !ctx_active || depth >= max_depth {
return;
}
let budget_ptr = ENERGY_BUDGET_PTR.with(std::cell::Cell::get);
if budget_ptr.is_null() {
return;
}
unsafe {
crate::energy::init_mark_budget(budget_ptr, slot_idx);
}
let split_call_count = context::rng_get_count();
let bm_ptr = COVERAGE_BITMAP_PTR.with(std::cell::Cell::get);
let vm_ptr = EXPLORED_MAP_PTR.with(std::cell::Cell::get);
let stats_ptr = SHARED_STATS.with(std::cell::Cell::get);
let (batch_size, max_timelines, effective_min_timelines) = read_adaptive_batch_config();
let state = resolve_parallel_state();
let shared = ForkSharedState {
split_call_count,
vm_ptr,
stats_ptr,
pool_base: state.pool_base,
sancov_pool_base: state.sancov_pool_base,
};
let mut parent_bitmap_backup = [0u8; COVERAGE_MAP_SIZE];
if !state.parallel && !bm_ptr.is_null() {
unsafe {
std::ptr::copy_nonoverlapping(
bm_ptr,
parent_bitmap_backup.as_mut_ptr(),
COVERAGE_MAP_SIZE,
);
}
}
let mut timelines_spawned: u32 = 0;
let mut active: HashMap<libc::pid_t, (u64, usize)> = HashMap::new();
let mut free_slots: Vec<usize> = if state.parallel {
(0..state.slot_count).collect()
} else {
Vec::new()
};
loop {
let mut batch_has_new = false;
let batch_start = timelines_spawned;
while timelines_spawned - batch_start < batch_size {
if timelines_spawned >= max_timelines {
break;
}
if !unsafe { crate::energy::decrement_mark_energy(budget_ptr, slot_idx) } {
break;
}
let child_seed = compute_child_seed(current_seed, mark_name, timelines_spawned);
timelines_spawned += 1;
let outcome = if state.parallel {
spawn_parallel_child(
child_seed,
&shared,
&mut active,
&mut free_slots,
&mut batch_has_new,
)
} else {
spawn_sequential_child(child_seed, bm_ptr, &shared, &mut batch_has_new, true)
};
match outcome {
SpawnOutcome::Continued => {}
SpawnOutcome::Stop => break,
SpawnOutcome::InChild => return,
}
}
drain_active_children(&mut active, &mut free_slots, &shared, &mut batch_has_new);
if timelines_spawned >= max_timelines {
break;
}
if !batch_has_new && timelines_spawned >= effective_min_timelines {
unsafe {
crate::energy::return_mark_energy_to_pool(budget_ptr, slot_idx);
}
break;
}
if timelines_spawned - batch_start < batch_size && timelines_spawned < max_timelines {
break;
}
}
restore_parent_state(&state, bm_ptr, &parent_bitmap_backup);
}
#[cfg(unix)]
pub fn split_on_discovery(mark_name: &str) {
let (ctx_active, depth, max_depth, timelines_per_split, current_seed) =
context::with_ctx(|ctx| {
(
ctx.active,
ctx.depth,
ctx.max_depth,
ctx.timelines_per_split,
ctx.current_seed,
)
});
if !ctx_active || depth >= max_depth {
return;
}
let stats_ptr = SHARED_STATS.with(std::cell::Cell::get);
if stats_ptr.is_null() {
return;
}
if !unsafe { crate::shared_stats::decrement_energy(stats_ptr) } {
return;
}
let split_call_count = context::rng_get_count();
let bm_ptr = COVERAGE_BITMAP_PTR.with(std::cell::Cell::get);
let vm_ptr = EXPLORED_MAP_PTR.with(std::cell::Cell::get);
let state = resolve_parallel_state();
let shared = ForkSharedState {
split_call_count,
vm_ptr,
stats_ptr,
pool_base: state.pool_base,
sancov_pool_base: state.sancov_pool_base,
};
let mut parent_bitmap_backup = [0u8; COVERAGE_MAP_SIZE];
if !state.parallel && !bm_ptr.is_null() {
unsafe {
std::ptr::copy_nonoverlapping(
bm_ptr,
parent_bitmap_backup.as_mut_ptr(),
COVERAGE_MAP_SIZE,
);
}
}
let mut active: HashMap<libc::pid_t, (u64, usize)> = HashMap::new();
let mut free_slots: Vec<usize> = if state.parallel {
(0..state.slot_count).collect()
} else {
Vec::new()
};
let mut batch_has_new = false;
for child_idx in 0..timelines_per_split {
if child_idx > 0 {
if !unsafe { crate::shared_stats::decrement_energy(stats_ptr) } {
break;
}
}
let child_seed = compute_child_seed(current_seed, mark_name, child_idx);
let outcome = if state.parallel {
spawn_parallel_child(
child_seed,
&shared,
&mut active,
&mut free_slots,
&mut batch_has_new,
)
} else {
spawn_sequential_child(child_seed, bm_ptr, &shared, &mut batch_has_new, false)
};
match outcome {
SpawnOutcome::Continued => {}
SpawnOutcome::Stop => break,
SpawnOutcome::InChild => return,
}
}
drain_active_children(&mut active, &mut free_slots, &shared, &mut batch_has_new);
restore_parent_state(&state, bm_ptr, &parent_bitmap_backup);
}
#[cfg(not(unix))]
pub fn split_on_discovery(_mark_name: &str) {}
fn save_bug_recipe(split_call_count: u64, child_seed: u64) {
let recipe_ptr = SHARED_RECIPE.with(std::cell::Cell::get);
if recipe_ptr.is_null() {
return;
}
unsafe {
let recipe = &mut *recipe_ptr;
if recipe
.claimed
.compare_exchange(0, 1, Ordering::Relaxed, Ordering::Relaxed)
.is_ok()
{
context::with_ctx(|ctx| {
let total_entries = ctx.recipe.len() + 1;
let len = total_entries.min(MAX_RECIPE_ENTRIES);
for (i, &entry) in ctx.recipe.iter().take(len - 1).enumerate() {
recipe.entries[i] = entry;
}
if len > 0 {
recipe.entries[len - 1] = (split_call_count, child_seed);
}
recipe.len = u32::try_from(len).expect("len bounded by MAX_RECIPE_ENTRIES");
});
}
}
}
#[cfg(unix)]
pub fn exit_child(code: i32) -> ! {
crate::sancov::copy_counters_to_shared();
unsafe { libc::_exit(code) }
}
#[cfg(not(unix))]
pub fn exit_child(code: i32) -> ! {
std::process::exit(code)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_compute_child_seed_deterministic() {
let s1 = compute_child_seed(42, "test", 0);
let s2 = compute_child_seed(42, "test", 0);
assert_eq!(s1, s2);
}
#[test]
fn test_compute_child_seed_varies_by_index() {
let s0 = compute_child_seed(42, "test", 0);
let s1 = compute_child_seed(42, "test", 1);
let s2 = compute_child_seed(42, "test", 2);
assert_ne!(s0, s1);
assert_ne!(s1, s2);
assert_ne!(s0, s2);
}
#[test]
fn test_compute_child_seed_varies_by_name() {
let s1 = compute_child_seed(42, "alpha", 0);
let s2 = compute_child_seed(42, "beta", 0);
assert_ne!(s1, s2);
}
#[test]
fn test_compute_child_seed_varies_by_parent() {
let s1 = compute_child_seed(1, "test", 0);
let s2 = compute_child_seed(2, "test", 0);
assert_ne!(s1, s2);
}
}