use core::sync::atomic::{AtomicBool, Ordering};
use std::sync::{Arc, OnceLock};
use std::thread::{self, JoinHandle};
use crate::sched::notify_ring::{NotifyHub, NotifySender};
pub type IoTask = Box<dyn FnOnce() + Send + 'static>;
pub struct IoPool {
hub: NotifyHub<IoTask>,
sender: NotifySender<IoTask>,
workers: Vec<Option<JoinHandle<()>>>,
shutdown: Arc<AtomicBool>,
}
impl std::fmt::Debug for IoPool {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("IoPool")
.field("n_workers", &self.workers.len())
.finish()
}
}
impl IoPool {
pub fn new(n_workers: usize) -> Arc<Self> {
let n = n_workers.max(1);
const IO_RING_CAPACITY: usize = 1024;
let hub = NotifyHub::<IoTask>::new(IO_RING_CAPACITY, n);
let sender = hub.sender();
let shutdown = Arc::new(AtomicBool::new(false));
let mut handles: Vec<Option<JoinHandle<()>>> = Vec::with_capacity(n);
for idx in 0..n {
let hub_for_worker = hub.clone();
let sd = Arc::clone(&shutdown);
let h = thread::Builder::new()
.name(format!("flynnel-io-{idx}"))
.spawn(move || io_worker_loop(idx, hub_for_worker, sd))
.expect("IO worker thread spawn must succeed");
handles.push(Some(h));
}
Arc::new(Self {
hub,
sender,
workers: handles,
shutdown,
})
}
pub fn submit<F: FnOnce() + Send + 'static>(&self, task: F) {
drop(self.sender.send(Box::new(task)));
}
pub fn worker_count(&self) -> usize {
self.workers.len()
}
}
impl Drop for IoPool {
fn drop(&mut self) {
self.shutdown.store(true, Ordering::Release);
self.hub.shutdown();
for slot in &mut self.workers {
if let Some(h) = slot.take() {
drop(h.join());
}
}
}
}
fn io_worker_loop(_idx: usize, hub: NotifyHub<IoTask>, shutdown: Arc<AtomicBool>) {
let rx = hub.register_consumer();
while !shutdown.load(Ordering::Acquire) {
match rx.recv() {
Some(task) => {
task();
}
None => {
break;
}
}
}
}
pub fn global_io_pool() -> Option<&'static Arc<IoPool>> {
static POOL: OnceLock<Option<Arc<IoPool>>> = OnceLock::new();
POOL.get_or_init(|| {
let enabled = std::env::var("FLYNNEL_SCHED_SMT_AS_IO")
.map(|v| {
let v = v.to_ascii_lowercase();
v == "on" || v == "1" || v == "true"
})
.unwrap_or(false);
if !enabled {
return None;
}
let phys = crate::cpu_info::cpu_info().physical_cores as usize;
Some(IoPool::new(phys))
})
.as_ref()
}
pub fn submit_io_or_inline<F: FnOnce() + Send + 'static>(task: F) {
match global_io_pool() {
Some(pool) => pool.submit(task),
None => task(),
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::sync::atomic::AtomicU32;
use std::time::{Duration, Instant};
#[test]
fn pool_with_one_worker_runs_task() {
let pool = IoPool::new(1);
let counter = Arc::new(AtomicU32::new(0));
let c = Arc::clone(&counter);
pool.submit(move || {
c.fetch_add(1, Ordering::SeqCst);
});
let deadline = Instant::now() + Duration::from_secs(5);
while counter.load(Ordering::SeqCst) == 0 {
if Instant::now() > deadline {
panic!("IO task did not run within 5s");
}
thread::yield_now();
}
drop(pool);
}
#[test]
fn pool_runs_many_tasks_across_workers() {
const N: u32 = 128;
let pool = IoPool::new(4);
let counter = Arc::new(AtomicU32::new(0));
for _ in 0..N {
let c = Arc::clone(&counter);
pool.submit(move || {
c.fetch_add(1, Ordering::SeqCst);
});
}
let deadline = Instant::now() + Duration::from_secs(10);
while counter.load(Ordering::SeqCst) < N {
if Instant::now() > deadline {
panic!(
"only {}/{N} IO tasks completed within 10s",
counter.load(Ordering::SeqCst)
);
}
thread::yield_now();
}
assert_eq!(counter.load(Ordering::SeqCst), N);
drop(pool);
}
#[test]
fn pool_drop_joins_workers_cleanly() {
let pool = IoPool::new(2);
let t0 = Instant::now();
drop(pool);
let elapsed = t0.elapsed();
assert!(
elapsed < Duration::from_secs(2),
"IoPool drop took {elapsed:?}, expected < 2s"
);
}
#[test]
#[ignore = "a stress repetition, read by hand"]
fn n_tasks_on_n_workers_all_run_together() {
use std::sync::atomic::AtomicUsize;
const ROUNDS: usize = 400;
let n = thread::available_parallelism()
.map(std::num::NonZeroUsize::get)
.expect("logical processor count");
let mut total_stranded = 0usize;
for gap_us in [0u64, 20, 200] {
let gap = Duration::from_micros(gap_us);
let (mut short, mut stranded, mut fewest) = (0usize, 0usize, n);
for _ in 0..ROUNDS {
let pool = IoPool::new(n);
let started = Arc::new(AtomicUsize::new(0));
let release = Arc::new(AtomicBool::new(false));
for _ in 0..n {
let (started, release) = (Arc::clone(&started), Arc::clone(&release));
pool.submit(move || {
started.fetch_add(1, Ordering::SeqCst);
while !release.load(Ordering::Acquire) {
thread::sleep(Duration::from_millis(1));
}
});
let until = Instant::now() + gap;
while Instant::now() < until {
std::hint::spin_loop();
}
}
let deadline = Instant::now() + Duration::from_secs(1);
while started.load(Ordering::SeqCst) < n && Instant::now() < deadline {
thread::sleep(Duration::from_millis(1));
}
let running = started.load(Ordering::SeqCst);
if running < n {
short += 1;
fewest = fewest.min(running);
let deadline = Instant::now() + Duration::from_secs(10);
while started.load(Ordering::SeqCst) < n && Instant::now() < deadline {
thread::sleep(Duration::from_millis(1));
}
if started.load(Ordering::SeqCst) < n {
stranded += 1;
}
}
release.store(true, Ordering::Release);
drop(pool);
}
total_stranded += stranded;
println!(
"STRAND pool gap_us={gap_us} rounds={ROUNDS} n={n} short_rounds={short} stranded_rounds={stranded} fewest_running={fewest}"
);
}
assert_eq!(
total_stranded, 0,
"some rounds left a task queued while a worker slept"
);
}
#[test]
fn submit_io_or_inline_runs_when_pool_disabled() {
let counter = Arc::new(AtomicU32::new(0));
let c = Arc::clone(&counter);
submit_io_or_inline(move || {
c.fetch_add(1, Ordering::SeqCst);
});
assert_eq!(counter.load(Ordering::SeqCst), 1);
}
}