use std::cell::Cell;
use std::collections::BTreeMap;
use std::io;
use std::mem;
#[cfg(unix)]
use std::os::unix::io::RawFd;
#[cfg(windows)]
use std::os::windows::io::RawSocket;
use std::panic;
use std::sync::atomic::{AtomicBool, AtomicU8, AtomicUsize, Ordering};
use std::sync::{Arc, Mutex, MutexGuard};
use std::task::{Context, Poll, Waker};
use std::thread;
use std::time::{Duration, Instant};
use concurrent_queue::ConcurrentQueue;
use futures_lite::*;
use once_cell::sync::Lazy;
use polling::{Event, Poller};
use vec_arena::Arena;
use waker_fn::waker_fn;
pub(crate) struct Reactor {
block_on_count: AtomicUsize,
thread_unparker: parking::Unparker,
poller: Poller,
ticker: AtomicUsize,
sources: Mutex<Arena<Arc<Source>>>,
events: Mutex<Vec<Event>>,
timers: Mutex<BTreeMap<(Instant, usize), Waker>>,
timer_ops: ConcurrentQueue<TimerOp>,
}
impl Reactor {
pub(crate) fn get() -> &'static Reactor {
static REACTOR: Lazy<Reactor> = Lazy::new(|| {
let (parker, unparker) = parking::pair();
thread::Builder::new()
.name("async-io".to_string())
.spawn(move || Reactor::get().main_loop(parker))
.expect("cannot spawn async-io thread");
Reactor {
block_on_count: AtomicUsize::new(0),
thread_unparker: unparker,
poller: Poller::new().expect("cannot initialize I/O event notification"),
ticker: AtomicUsize::new(0),
sources: Mutex::new(Arena::new()),
events: Mutex::new(Vec::new()),
timers: Mutex::new(BTreeMap::new()),
timer_ops: ConcurrentQueue::bounded(1000),
}
});
&REACTOR
}
fn main_loop(&self, parker: parking::Parker) {
let mut last_tick = 0;
let mut sleeps = 0u64;
loop {
let tick = self.ticker.load(Ordering::SeqCst);
if last_tick == tick {
let reactor_lock = if sleeps >= 10 {
Some(self.lock())
} else {
self.try_lock()
};
if let Some(mut reactor_lock) = reactor_lock {
let _ = reactor_lock.react(None);
last_tick = self.ticker.load(Ordering::SeqCst);
sleeps = 0;
}
} else {
last_tick = tick;
}
if self.block_on_count.load(Ordering::SeqCst) > 0 {
let delay_us = [50, 75, 100, 250, 500, 750, 1000, 2500, 5000]
.get(sleeps as usize)
.unwrap_or(&10_000);
if parker.park_timeout(Duration::from_micros(*delay_us)) {
last_tick = self.ticker.load(Ordering::SeqCst);
sleeps = 0;
} else {
sleeps += 1;
}
}
}
}
pub(crate) fn block_on<T>(&self, future: impl Future<Output = T>) -> T {
self.block_on_count.fetch_add(1, Ordering::SeqCst);
let _guard = CallOnDrop(|| {
Reactor::get().block_on_count.fetch_sub(1, Ordering::SeqCst);
Reactor::get().thread_unparker.unpark();
});
let (p, u) = parking::pair();
let io_blocked = Arc::new(AtomicBool::new(false));
thread_local! {
static IO_POLLING: Cell<bool> = Cell::new(false);
}
let waker = waker_fn({
let io_blocked = io_blocked.clone();
move || {
if u.unpark() {
if !IO_POLLING.with(Cell::get) && io_blocked.load(Ordering::SeqCst) {
Reactor::get().notify();
}
}
}
});
let cx = &mut Context::from_waker(&waker);
pin!(future);
loop {
if let Poll::Ready(t) = future.as_mut().poll(cx) {
return t;
}
if p.park_timeout(Duration::from_secs(0)) {
if let Some(mut reactor_lock) = Reactor::get().try_lock() {
IO_POLLING.with(|io| io.set(true));
let _guard = CallOnDrop(|| {
IO_POLLING.with(|io| io.set(false));
});
let _ = reactor_lock.react(Some(Duration::from_secs(0)));
}
continue;
}
if let Some(mut reactor_lock) = Reactor::get().try_lock() {
loop {
IO_POLLING.with(|io| io.set(true));
io_blocked.store(true, Ordering::SeqCst);
let _guard = CallOnDrop(|| {
IO_POLLING.with(|io| io.set(false));
io_blocked.store(false, Ordering::SeqCst);
});
if p.park_timeout(Duration::from_secs(0)) {
break;
}
let _ = reactor_lock.react(None);
if p.park_timeout(Duration::from_secs(0)) {
break;
}
}
} else {
p.park();
}
}
}
pub(crate) fn insert_io(
&self,
#[cfg(unix)] raw: RawFd,
#[cfg(windows)] raw: RawSocket,
) -> io::Result<Arc<Source>> {
self.poller.insert(raw)?;
let mut sources = self.sources.lock().unwrap();
let key = sources.next_vacant();
let source = Arc::new(Source {
raw,
key,
wakers: Mutex::new(Wakers {
tick_readable: 0,
tick_writable: 0,
readers: Vec::new(),
writers: Vec::new(),
}),
wakers_registered: AtomicU8::new(0),
});
sources.insert(source.clone());
Ok(source)
}
pub(crate) fn remove_io(&self, source: &Source) -> io::Result<()> {
let mut sources = self.sources.lock().unwrap();
sources.remove(source.key);
self.poller.remove(source.raw)
}
pub(crate) fn insert_timer(&self, when: Instant, waker: &Waker) -> usize {
static ID_GENERATOR: AtomicUsize = AtomicUsize::new(1);
let id = ID_GENERATOR.fetch_add(1, Ordering::Relaxed);
while self
.timer_ops
.push(TimerOp::Insert(when, id, waker.clone()))
.is_err()
{
let mut timers = self.timers.lock().unwrap();
self.process_timer_ops(&mut timers);
}
self.notify();
id
}
pub(crate) fn remove_timer(&self, when: Instant, id: usize) {
while self.timer_ops.push(TimerOp::Remove(when, id)).is_err() {
let mut timers = self.timers.lock().unwrap();
self.process_timer_ops(&mut timers);
}
}
fn notify(&self) {
self.poller.notify().expect("failed to notify reactor");
}
fn lock(&self) -> ReactorLock<'_> {
let reactor = self;
let events = self.events.lock().unwrap();
ReactorLock { reactor, events }
}
fn try_lock(&self) -> Option<ReactorLock<'_>> {
self.events.try_lock().ok().map(|events| {
let reactor = self;
ReactorLock { reactor, events }
})
}
fn process_timers(&self, wakers: &mut Vec<Waker>) -> Option<Duration> {
let mut timers = self.timers.lock().unwrap();
self.process_timer_ops(&mut timers);
let now = Instant::now();
let pending = timers.split_off(&(now, 0));
let ready = mem::replace(&mut *timers, pending);
let dur = if ready.is_empty() {
timers
.keys()
.next()
.map(|(when, _)| when.saturating_duration_since(now))
} else {
Some(Duration::from_secs(0))
};
drop(timers);
for (_, waker) in ready {
wakers.push(waker);
}
dur
}
fn process_timer_ops(&self, timers: &mut MutexGuard<'_, BTreeMap<(Instant, usize), Waker>>) {
for _ in 0..self.timer_ops.capacity().unwrap() {
match self.timer_ops.pop() {
Ok(TimerOp::Insert(when, id, waker)) => {
timers.insert((when, id), waker);
}
Ok(TimerOp::Remove(when, id)) => {
timers.remove(&(when, id));
}
Err(_) => break,
}
}
}
}
struct ReactorLock<'a> {
reactor: &'a Reactor,
events: MutexGuard<'a, Vec<Event>>,
}
impl ReactorLock<'_> {
fn react(&mut self, timeout: Option<Duration>) -> io::Result<()> {
let mut wakers = Vec::new();
let next_timer = self.reactor.process_timers(&mut wakers);
let timeout = match (next_timer, timeout) {
(None, None) => None,
(Some(t), None) | (None, Some(t)) => Some(t),
(Some(a), Some(b)) => Some(a.min(b)),
};
let tick = self
.reactor
.ticker
.fetch_add(1, Ordering::SeqCst)
.wrapping_add(1);
self.events.clear();
let res = match self.reactor.poller.wait(&mut self.events, timeout) {
Ok(0) => {
if timeout != Some(Duration::from_secs(0)) {
self.reactor.process_timers(&mut wakers);
}
Ok(())
}
Ok(_) => {
let sources = self.reactor.sources.lock().unwrap();
for ev in self.events.iter() {
if let Some(source) = sources.get(ev.key) {
let mut w = source.wakers.lock().unwrap();
if ev.readable {
w.tick_readable = tick;
wakers.append(&mut w.readers);
source
.wakers_registered
.fetch_and(!READERS_REGISTERED, Ordering::SeqCst);
}
if ev.writable {
w.tick_writable = tick;
wakers.append(&mut w.writers);
source
.wakers_registered
.fetch_and(!WRITERS_REGISTERED, Ordering::SeqCst);
}
if !(w.writers.is_empty() && w.readers.is_empty()) {
self.reactor.poller.interest(
source.raw,
Event {
key: source.key,
readable: !w.readers.is_empty(),
writable: !w.writers.is_empty(),
},
)?;
}
}
}
Ok(())
}
Err(err) if err.kind() == io::ErrorKind::Interrupted => Ok(()),
Err(err) => Err(err),
};
drop(self);
for waker in wakers {
let _ = panic::catch_unwind(|| waker.wake());
}
res
}
}
enum TimerOp {
Insert(Instant, usize, Waker),
Remove(Instant, usize),
}
#[derive(Debug)]
pub(crate) struct Source {
#[cfg(unix)]
pub(crate) raw: RawFd,
#[cfg(windows)]
pub(crate) raw: RawSocket,
key: usize,
wakers: Mutex<Wakers>,
wakers_registered: AtomicU8,
}
const READERS_REGISTERED: u8 = 1 << 0;
const WRITERS_REGISTERED: u8 = 1 << 1;
#[derive(Debug)]
struct Wakers {
tick_readable: usize,
tick_writable: usize,
readers: Vec<Waker>,
writers: Vec<Waker>,
}
impl Source {
pub(crate) async fn readable(&self) -> io::Result<()> {
let mut ticks = None;
future::poll_fn(|cx| {
let mut w = self.wakers.lock().unwrap();
if let Some((a, b)) = ticks {
if w.tick_readable != a && w.tick_readable != b {
return Poll::Ready(Ok(()));
}
}
if w.readers.is_empty() {
Reactor::get().poller.interest(
self.raw,
Event {
key: self.key,
readable: true,
writable: !w.writers.is_empty(),
},
)?;
self.wakers_registered
.fetch_or(READERS_REGISTERED, Ordering::SeqCst);
}
if w.readers.iter().all(|w| !w.will_wake(cx.waker())) {
w.readers.push(cx.waker().clone());
if limit_waker_list(&mut w.readers) {
self.wakers_registered
.fetch_and(!READERS_REGISTERED, Ordering::SeqCst);
}
}
if ticks.is_none() {
ticks = Some((
Reactor::get().ticker.load(Ordering::SeqCst),
w.tick_readable,
));
}
Poll::Pending
})
.await
}
pub(crate) fn readers_registered(&self) -> bool {
self.wakers_registered.load(Ordering::SeqCst) & READERS_REGISTERED != 0
}
pub(crate) async fn writable(&self) -> io::Result<()> {
let mut ticks = None;
future::poll_fn(|cx| {
let mut w = self.wakers.lock().unwrap();
if let Some((a, b)) = ticks {
if w.tick_writable != a && w.tick_writable != b {
return Poll::Ready(Ok(()));
}
}
if w.writers.is_empty() {
Reactor::get().poller.interest(
self.raw,
Event {
key: self.key,
readable: !w.readers.is_empty(),
writable: true,
},
)?;
self.wakers_registered
.fetch_or(WRITERS_REGISTERED, Ordering::SeqCst);
}
if w.writers.iter().all(|w| !w.will_wake(cx.waker())) {
w.writers.push(cx.waker().clone());
if limit_waker_list(&mut w.writers) {
self.wakers_registered
.fetch_and(!WRITERS_REGISTERED, Ordering::SeqCst);
}
}
if ticks.is_none() {
ticks = Some((
Reactor::get().ticker.load(Ordering::SeqCst),
w.tick_writable,
));
}
Poll::Pending
})
.await
}
pub(crate) fn writers_registered(&self) -> bool {
self.wakers_registered.load(Ordering::SeqCst) & WRITERS_REGISTERED != 0
}
}
fn limit_waker_list(wakers: &mut Vec<Waker>) -> bool {
if wakers.len() > 50 {
for waker in wakers.drain(..) {
let _ = panic::catch_unwind(|| waker.wake());
}
true
} else {
false
}
}
struct CallOnDrop<F: Fn()>(F);
impl<F: Fn()> Drop for CallOnDrop<F> {
fn drop(&mut self) {
(self.0)();
}
}