1use std::sync::atomic::Ordering;
22
23#[cfg(unix)]
24use std::collections::HashMap;
25
26use crate::context::{
27 self, COVERAGE_BITMAP_PTR, ENERGY_BUDGET_PTR, EXPLORED_MAP_PTR, SHARED_RECIPE, SHARED_STATS,
28};
29#[cfg(unix)]
30use crate::context::{BITMAP_POOL, BITMAP_POOL_SLOTS};
31use crate::coverage::{COVERAGE_MAP_SIZE, CoverageBitmap, ExploredMap};
32use crate::shared_stats::MAX_RECIPE_ENTRIES;
33
34fn compute_child_seed(parent_seed: u64, mark_name: &str, child_idx: u32) -> u64 {
38 let mut hash: u64 = 0xcbf2_9ce4_8422_2325;
39 for &byte in mark_name.as_bytes() {
40 hash ^= u64::from(byte);
41 hash = hash.wrapping_mul(0x0100_0000_01b3);
42 }
43 hash ^= parent_seed;
44 hash = hash.wrapping_mul(0x0100_0000_01b3);
45 hash ^= u64::from(child_idx);
46 hash = hash.wrapping_mul(0x0100_0000_01b3);
47 hash
48}
49
50#[derive(Debug, Clone)]
56pub enum Parallelism {
57 MaxCores,
59 HalfCores,
61 Cores(usize),
63 MaxCoresMinus(usize),
65}
66
67#[cfg(unix)]
69fn resolve_parallelism(p: &Parallelism) -> usize {
70 let ncpus = unsafe { libc::sysconf(libc::_SC_NPROCESSORS_ONLN) };
73 let ncpus = if ncpus > 0 {
74 usize::try_from(ncpus).unwrap_or(1)
75 } else {
76 1
77 };
78 let n = match p {
79 Parallelism::MaxCores => ncpus,
80 Parallelism::HalfCores => ncpus / 2,
81 Parallelism::Cores(c) => *c,
82 Parallelism::MaxCoresMinus(minus) => ncpus.saturating_sub(*minus),
83 };
84 n.max(1) }
86
87#[cfg(unix)]
93fn get_or_init_pool(slot_count: usize) -> *mut u8 {
94 let existing = BITMAP_POOL.with(std::cell::Cell::get);
95 let existing_slots = BITMAP_POOL_SLOTS.with(std::cell::Cell::get);
96
97 if !existing.is_null() && existing_slots >= slot_count {
98 return existing;
99 }
100
101 if !existing.is_null() {
103 unsafe {
105 crate::shared_mem::free_shared(existing, existing_slots * COVERAGE_MAP_SIZE);
106 }
107 BITMAP_POOL.with(|c| c.set(std::ptr::null_mut()));
108 BITMAP_POOL_SLOTS.with(|c| c.set(0));
109 }
110
111 match crate::shared_mem::alloc_shared(slot_count * COVERAGE_MAP_SIZE) {
112 Ok(ptr) => {
113 BITMAP_POOL.with(|c| c.set(ptr));
114 BITMAP_POOL_SLOTS.with(|c| c.set(slot_count));
115 ptr
116 }
117 Err(_) => std::ptr::null_mut(),
118 }
119}
120
121#[cfg(unix)]
123fn pool_slot(pool_base: *mut u8, idx: usize) -> *mut u8 {
124 unsafe { pool_base.add(idx * COVERAGE_MAP_SIZE) }
126}
127
128#[cfg(unix)]
132fn setup_child(
133 child_seed: u64,
134 split_call_count: u64,
135 stats_ptr: *mut crate::shared_stats::SharedStats,
136) {
137 context::rng_reseed(child_seed);
138 context::with_ctx_mut(|ctx| {
139 ctx.is_child = true;
140 ctx.depth += 1;
141 ctx.current_seed = child_seed;
142 ctx.recipe.push((split_call_count, child_seed));
143 });
144 if !stats_ptr.is_null() {
145 unsafe {
147 (*stats_ptr).total_timelines.fetch_add(1, Ordering::Relaxed);
148 }
149 }
150 BITMAP_POOL.with(|c| c.set(std::ptr::null_mut()));
152 BITMAP_POOL_SLOTS.with(|c| c.set(0));
153
154 crate::sancov::reset_bss_counters();
156 crate::sancov::SANCOV_POOL.with(|c| c.set(std::ptr::null_mut()));
158 crate::sancov::SANCOV_POOL_SLOTS.with(|c| c.set(0));
159}
160
161#[cfg(unix)]
164struct ForkSharedState {
165 split_call_count: u64,
167 vm_ptr: *mut u8,
169 stats_ptr: *mut crate::shared_stats::SharedStats,
171 pool_base: *mut u8,
173 sancov_pool_base: *mut u8,
175}
176
177#[cfg(unix)]
179struct ParallelState {
180 slot_count: usize,
182 pool_base: *mut u8,
184 sancov_pool_base: *mut u8,
186 parent_sancov_transfer: *mut u8,
188 parallel: bool,
190}
191
192#[cfg(unix)]
194fn resolve_parallel_state() -> ParallelState {
195 let parallelism = context::with_ctx(|ctx| ctx.parallelism.clone());
196 let (slot_count, pool_base) = if let Some(ref p) = parallelism {
197 let sc = resolve_parallelism(p);
198 let pb = get_or_init_pool(sc);
199 if pb.is_null() {
200 (0, std::ptr::null_mut())
201 } else {
202 (sc, pb)
203 }
204 } else {
205 (0, std::ptr::null_mut())
206 };
207 let parallel = slot_count > 0;
208
209 let sancov_pool_base = if parallel {
210 crate::sancov::get_or_init_sancov_pool(slot_count)
211 } else {
212 std::ptr::null_mut()
213 };
214 let parent_sancov_transfer = if parallel && !sancov_pool_base.is_null() {
215 crate::sancov::SANCOV_TRANSFER.with(std::cell::Cell::get)
216 } else {
217 std::ptr::null_mut()
218 };
219
220 ParallelState {
221 slot_count,
222 pool_base,
223 sancov_pool_base,
224 parent_sancov_transfer,
225 parallel,
226 }
227}
228
229#[cfg(unix)]
231fn drain_active_children(
232 active: &mut HashMap<libc::pid_t, (u64, usize)>,
233 free_slots: &mut Vec<usize>,
234 shared: &ForkSharedState,
235 batch_has_new: &mut bool,
236) {
237 while !active.is_empty() {
238 reap_one(active, free_slots, shared, batch_has_new);
239 }
240}
241
242#[cfg(unix)]
244fn restore_parent_state(state: &ParallelState, bm_ptr: *mut u8, parent_bitmap_backup: &[u8]) {
245 if state.parallel {
246 COVERAGE_BITMAP_PTR.with(|c| c.set(bm_ptr));
248 if !state.sancov_pool_base.is_null() {
249 crate::sancov::SANCOV_TRANSFER.with(|c| c.set(state.parent_sancov_transfer));
250 }
251 } else if !bm_ptr.is_null() {
252 unsafe {
255 std::ptr::copy_nonoverlapping(parent_bitmap_backup.as_ptr(), bm_ptr, COVERAGE_MAP_SIZE);
256 }
257 }
258}
259
260#[cfg(unix)]
262enum SpawnOutcome {
263 Continued,
265 Stop,
267 InChild,
269}
270
271#[cfg(unix)]
274fn spawn_parallel_child(
275 child_seed: u64,
276 shared: &ForkSharedState,
277 active: &mut HashMap<libc::pid_t, (u64, usize)>,
278 free_slots: &mut Vec<usize>,
279 batch_has_new: &mut bool,
280) -> SpawnOutcome {
281 while free_slots.is_empty() {
282 reap_one(active, free_slots, shared, batch_has_new);
283 }
284 let Some(slot) = free_slots.pop() else {
285 return SpawnOutcome::Stop;
286 };
287 let slot_ptr = pool_slot(shared.pool_base, slot);
288
289 unsafe {
291 std::ptr::write_bytes(slot_ptr, 0, COVERAGE_MAP_SIZE);
292 }
293 COVERAGE_BITMAP_PTR.with(|c| c.set(slot_ptr));
294
295 if !shared.sancov_pool_base.is_null() {
296 let sancov_len = crate::sancov::sancov_edge_count();
297 unsafe {
299 let sancov_slot = crate::sancov::sancov_pool_slot(shared.sancov_pool_base, slot);
300 std::ptr::write_bytes(sancov_slot, 0, sancov_len);
301 crate::sancov::SANCOV_TRANSFER.with(|c| c.set(sancov_slot));
302 }
303 }
304
305 let pid = unsafe { libc::fork() };
307 match pid {
308 -1 => {
309 free_slots.push(slot);
310 SpawnOutcome::Stop
311 }
312 0 => {
313 setup_child(child_seed, shared.split_call_count, shared.stats_ptr);
314 SpawnOutcome::InChild
315 }
316 child_pid => {
317 active.insert(child_pid, (child_seed, slot));
318 SpawnOutcome::Continued
319 }
320 }
321}
322
323#[cfg(unix)]
328fn spawn_sequential_child(
329 child_seed: u64,
330 bm_ptr: *mut u8,
331 shared: &ForkSharedState,
332 batch_has_new: &mut bool,
333 track_new_bits: bool,
334) -> SpawnOutcome {
335 if !bm_ptr.is_null() {
336 let bm = unsafe { CoverageBitmap::new(bm_ptr) };
338 bm.clear();
339 }
340 crate::sancov::clear_transfer_buffer();
341
342 let pid = unsafe { libc::fork() };
344 match pid {
345 -1 => SpawnOutcome::Stop,
346 0 => {
347 setup_child(child_seed, shared.split_call_count, shared.stats_ptr);
348 SpawnOutcome::InChild
349 }
350 child_pid => {
351 let mut status: libc::c_int = 0;
352 unsafe { libc::waitpid(child_pid, &raw mut status, 0) };
354
355 if !bm_ptr.is_null() && !shared.vm_ptr.is_null() {
356 let bm = unsafe { CoverageBitmap::new(bm_ptr) };
358 let vm = unsafe { ExploredMap::new(shared.vm_ptr) };
359 if track_new_bits && vm.has_new_bits(&bm) {
360 *batch_has_new = true;
361 }
362 vm.merge_from(&bm);
363 }
364 *batch_has_new |= crate::sancov::has_new_sancov_coverage();
365
366 if libc::WIFEXITED(status) && libc::WEXITSTATUS(status) == 42 {
367 if !shared.stats_ptr.is_null() {
368 unsafe {
370 (*shared.stats_ptr)
371 .bug_found
372 .fetch_add(1, Ordering::Relaxed);
373 }
374 }
375 save_bug_recipe(shared.split_call_count, child_seed);
376 }
377
378 if !shared.stats_ptr.is_null() {
379 unsafe {
381 (*shared.stats_ptr)
382 .fork_points
383 .fetch_add(1, Ordering::Relaxed);
384 }
385 }
386 SpawnOutcome::Continued
387 }
388 }
389}
390
391#[cfg(unix)]
396fn reap_one(
397 active: &mut HashMap<libc::pid_t, (u64, usize)>,
398 free_slots: &mut Vec<usize>,
399 shared: &ForkSharedState,
400 batch_has_new: &mut bool,
401) {
402 let mut status: libc::c_int = 0;
403 let finished_pid = unsafe { libc::waitpid(-1, &raw mut status, 0) };
405 if finished_pid <= 0 {
406 return;
407 }
408
409 let Some((child_seed, slot)) = active.remove(&finished_pid) else {
410 return;
411 };
412
413 if !shared.vm_ptr.is_null() {
415 let child_bm = unsafe { CoverageBitmap::new(pool_slot(shared.pool_base, slot)) };
417 let vm = unsafe { ExploredMap::new(shared.vm_ptr) };
418 if vm.has_new_bits(&child_bm) {
419 *batch_has_new = true;
420 }
421 vm.merge_from(&child_bm);
422 }
423
424 if !shared.sancov_pool_base.is_null() {
426 let sancov_slot = unsafe { crate::sancov::sancov_pool_slot(shared.sancov_pool_base, slot) };
427 if crate::sancov::has_new_sancov_coverage_from(sancov_slot) {
428 *batch_has_new = true;
429 }
430 }
431
432 if libc::WIFEXITED(status) && libc::WEXITSTATUS(status) == 42 {
433 if !shared.stats_ptr.is_null() {
434 unsafe {
436 (*shared.stats_ptr)
437 .bug_found
438 .fetch_add(1, Ordering::Relaxed);
439 }
440 }
441 save_bug_recipe(shared.split_call_count, child_seed);
442 }
443
444 if !shared.stats_ptr.is_null() {
445 unsafe {
447 (*shared.stats_ptr)
448 .fork_points
449 .fetch_add(1, Ordering::Relaxed);
450 }
451 }
452
453 free_slots.push(slot);
454}
455
456#[derive(Debug, Clone)]
463pub struct AdaptiveConfig {
464 pub batch_size: u32,
466 pub min_timelines: u32,
468 pub max_timelines: u32,
470 pub per_mark_energy: i64,
472 pub warm_min_timelines: Option<u32>,
475}
476
477#[cfg(unix)]
482pub(crate) fn dispatch_split(mark_name: &str, slot_idx: usize) {
483 let has_adaptive = ENERGY_BUDGET_PTR.with(|c| !c.get().is_null());
484 if has_adaptive {
485 adaptive_split_on_discovery(mark_name, slot_idx);
486 } else {
487 split_on_discovery(mark_name);
488 }
489}
490
491#[cfg(not(unix))]
493pub(crate) fn dispatch_split(_mark_name: &str, _slot_idx: usize) {}
494
495#[cfg(unix)]
500fn read_adaptive_batch_config() -> (u32, u32, u32) {
501 context::with_ctx(|ctx| {
502 let (batch_size, min_timelines, max_timelines) =
503 ctx.adaptive.as_ref().map_or((4, 1, 16), |a| {
504 (a.batch_size, a.min_timelines, a.max_timelines)
505 });
506 let warm_min = ctx
507 .adaptive
508 .as_ref()
509 .and_then(|a| a.warm_min_timelines)
510 .unwrap_or(batch_size);
511 let effective_min = if ctx.warm_start {
512 warm_min
513 } else {
514 min_timelines
515 };
516 (batch_size, max_timelines, effective_min)
517 })
518}
519
520#[cfg(unix)]
525fn adaptive_split_on_discovery(mark_name: &str, slot_idx: usize) {
526 let (ctx_active, depth, max_depth, current_seed) =
528 context::with_ctx(|ctx| (ctx.active, ctx.depth, ctx.max_depth, ctx.current_seed));
529
530 if !ctx_active || depth >= max_depth {
531 return;
532 }
533
534 let budget_ptr = ENERGY_BUDGET_PTR.with(std::cell::Cell::get);
535 if budget_ptr.is_null() {
536 return;
537 }
538
539 unsafe {
542 crate::energy::init_mark_budget(budget_ptr, slot_idx);
543 }
544
545 let split_call_count = context::rng_get_count();
546
547 let bm_ptr = COVERAGE_BITMAP_PTR.with(std::cell::Cell::get);
548 let vm_ptr = EXPLORED_MAP_PTR.with(std::cell::Cell::get);
549 let stats_ptr = SHARED_STATS.with(std::cell::Cell::get);
550
551 let (batch_size, max_timelines, effective_min_timelines) = read_adaptive_batch_config();
552
553 let state = resolve_parallel_state();
554 let shared = ForkSharedState {
555 split_call_count,
556 vm_ptr,
557 stats_ptr,
558 pool_base: state.pool_base,
559 sancov_pool_base: state.sancov_pool_base,
560 };
561
562 let mut parent_bitmap_backup = [0u8; COVERAGE_MAP_SIZE];
564 if !state.parallel && !bm_ptr.is_null() {
565 unsafe {
567 std::ptr::copy_nonoverlapping(
568 bm_ptr,
569 parent_bitmap_backup.as_mut_ptr(),
570 COVERAGE_MAP_SIZE,
571 );
572 }
573 }
574
575 let mut timelines_spawned: u32 = 0;
576
577 let mut active: HashMap<libc::pid_t, (u64, usize)> = HashMap::new();
579 let mut free_slots: Vec<usize> = if state.parallel {
580 (0..state.slot_count).collect()
581 } else {
582 Vec::new()
583 };
584
585 loop {
587 let mut batch_has_new = false;
588 let batch_start = timelines_spawned;
589
590 while timelines_spawned - batch_start < batch_size {
591 if timelines_spawned >= max_timelines {
592 break;
593 }
594
595 if !unsafe { crate::energy::decrement_mark_energy(budget_ptr, slot_idx) } {
597 break;
598 }
599
600 let child_seed = compute_child_seed(current_seed, mark_name, timelines_spawned);
601 timelines_spawned += 1;
602
603 let outcome = if state.parallel {
604 spawn_parallel_child(
605 child_seed,
606 &shared,
607 &mut active,
608 &mut free_slots,
609 &mut batch_has_new,
610 )
611 } else {
612 spawn_sequential_child(child_seed, bm_ptr, &shared, &mut batch_has_new, true)
613 };
614 match outcome {
615 SpawnOutcome::Continued => {}
616 SpawnOutcome::Stop => break,
617 SpawnOutcome::InChild => return,
618 }
619 }
620
621 drain_active_children(&mut active, &mut free_slots, &shared, &mut batch_has_new);
622
623 if timelines_spawned >= max_timelines {
625 break;
626 }
627 if !batch_has_new && timelines_spawned >= effective_min_timelines {
628 unsafe {
631 crate::energy::return_mark_energy_to_pool(budget_ptr, slot_idx);
632 }
633 break;
634 }
635 if timelines_spawned - batch_start < batch_size && timelines_spawned < max_timelines {
637 break;
638 }
639 }
640
641 restore_parent_state(&state, bm_ptr, &parent_bitmap_backup);
642}
643
644#[cfg(unix)]
653pub fn split_on_discovery(mark_name: &str) {
654 let (ctx_active, depth, max_depth, timelines_per_split, current_seed) =
655 context::with_ctx(|ctx| {
656 (
657 ctx.active,
658 ctx.depth,
659 ctx.max_depth,
660 ctx.timelines_per_split,
661 ctx.current_seed,
662 )
663 });
664
665 if !ctx_active || depth >= max_depth {
666 return;
667 }
668
669 let stats_ptr = SHARED_STATS.with(std::cell::Cell::get);
670 if stats_ptr.is_null() {
671 return;
672 }
673 if !unsafe { crate::shared_stats::decrement_energy(stats_ptr) } {
675 return;
676 }
677
678 let split_call_count = context::rng_get_count();
679 let bm_ptr = COVERAGE_BITMAP_PTR.with(std::cell::Cell::get);
680 let vm_ptr = EXPLORED_MAP_PTR.with(std::cell::Cell::get);
681
682 let state = resolve_parallel_state();
683 let shared = ForkSharedState {
684 split_call_count,
685 vm_ptr,
686 stats_ptr,
687 pool_base: state.pool_base,
688 sancov_pool_base: state.sancov_pool_base,
689 };
690
691 let mut parent_bitmap_backup = [0u8; COVERAGE_MAP_SIZE];
693 if !state.parallel && !bm_ptr.is_null() {
694 unsafe {
696 std::ptr::copy_nonoverlapping(
697 bm_ptr,
698 parent_bitmap_backup.as_mut_ptr(),
699 COVERAGE_MAP_SIZE,
700 );
701 }
702 }
703
704 let mut active: HashMap<libc::pid_t, (u64, usize)> = HashMap::new();
706 let mut free_slots: Vec<usize> = if state.parallel {
707 (0..state.slot_count).collect()
708 } else {
709 Vec::new()
710 };
711 let mut batch_has_new = false;
712
713 for child_idx in 0..timelines_per_split {
714 if child_idx > 0 {
715 if !unsafe { crate::shared_stats::decrement_energy(stats_ptr) } {
717 break;
718 }
719 }
720
721 let child_seed = compute_child_seed(current_seed, mark_name, child_idx);
722
723 let outcome = if state.parallel {
724 spawn_parallel_child(
725 child_seed,
726 &shared,
727 &mut active,
728 &mut free_slots,
729 &mut batch_has_new,
730 )
731 } else {
732 spawn_sequential_child(child_seed, bm_ptr, &shared, &mut batch_has_new, false)
733 };
734 match outcome {
735 SpawnOutcome::Continued => {}
736 SpawnOutcome::Stop => break,
737 SpawnOutcome::InChild => return,
738 }
739 }
740
741 drain_active_children(&mut active, &mut free_slots, &shared, &mut batch_has_new);
742
743 restore_parent_state(&state, bm_ptr, &parent_bitmap_backup);
744}
745
746#[cfg(not(unix))]
748pub fn split_on_discovery(_mark_name: &str) {}
749
750fn save_bug_recipe(split_call_count: u64, child_seed: u64) {
752 let recipe_ptr = SHARED_RECIPE.with(std::cell::Cell::get);
753 if recipe_ptr.is_null() {
754 return;
755 }
756
757 unsafe {
759 let recipe = &mut *recipe_ptr;
760
761 if recipe
763 .claimed
764 .compare_exchange(0, 1, Ordering::Relaxed, Ordering::Relaxed)
765 .is_ok()
766 {
767 context::with_ctx(|ctx| {
769 let total_entries = ctx.recipe.len() + 1;
770 let len = total_entries.min(MAX_RECIPE_ENTRIES);
771
772 for (i, &entry) in ctx.recipe.iter().take(len - 1).enumerate() {
774 recipe.entries[i] = entry;
775 }
776 if len > 0 {
778 recipe.entries[len - 1] = (split_call_count, child_seed);
779 }
780 recipe.len = u32::try_from(len).expect("len bounded by MAX_RECIPE_ENTRIES");
781 });
782 }
783 }
784}
785
786#[cfg(unix)]
796pub fn exit_child(code: i32) -> ! {
797 crate::sancov::copy_counters_to_shared();
798 unsafe { libc::_exit(code) }
800}
801
802#[cfg(not(unix))]
804pub fn exit_child(code: i32) -> ! {
805 std::process::exit(code)
806}
807
808#[cfg(test)]
809mod tests {
810 use super::*;
811
812 #[test]
813 fn test_compute_child_seed_deterministic() {
814 let s1 = compute_child_seed(42, "test", 0);
815 let s2 = compute_child_seed(42, "test", 0);
816 assert_eq!(s1, s2);
817 }
818
819 #[test]
820 fn test_compute_child_seed_varies_by_index() {
821 let s0 = compute_child_seed(42, "test", 0);
822 let s1 = compute_child_seed(42, "test", 1);
823 let s2 = compute_child_seed(42, "test", 2);
824 assert_ne!(s0, s1);
825 assert_ne!(s1, s2);
826 assert_ne!(s0, s2);
827 }
828
829 #[test]
830 fn test_compute_child_seed_varies_by_name() {
831 let s1 = compute_child_seed(42, "alpha", 0);
832 let s2 = compute_child_seed(42, "beta", 0);
833 assert_ne!(s1, s2);
834 }
835
836 #[test]
837 fn test_compute_child_seed_varies_by_parent() {
838 let s1 = compute_child_seed(1, "test", 0);
839 let s2 = compute_child_seed(2, "test", 0);
840 assert_ne!(s1, s2);
841 }
842}