use std::sync::{Arc, Mutex, PoisonError};
use std::time::Duration;
#[cfg(not(net_io_loom))]
use std::sync::atomic::{AtomicU64, Ordering};
#[cfg(net_io_loom)]
use loom::sync::atomic::{AtomicU64, Ordering};
use super::{RawSock, Sock, Wake};
use crate::error::KillReason;
const ACTIVE: u64 = 1;
const CLOSING: u64 = 2;
const KILLED: u64 = 4;
const KILLING: u64 = 8;
const IDLE: u64 = 16;
const DRAINED: u64 = 32;
const SOFT: u64 = 64;
const SHED: u64 = 128;
const REASON_SHIFT: u32 = 16;
const GEN_SHIFT: u32 = 32;
fn gen_of(state: u64) -> u32 {
(state >> GEN_SHIFT) as u32
}
pub(crate) trait KillSink: Send + Sync + std::fmt::Debug {
fn killed(&self, index: u32);
}
#[derive(Debug)]
pub(crate) struct KillSlot {
state: AtomicU64,
raw: AtomicU64,
soft_bound: AtomicU64,
soft_closes: std::sync::OnceLock<Arc<crate::link::SoftCloses>>,
wake: Mutex<Option<Arc<Wake>>>,
sink: Mutex<Option<(Arc<dyn KillSink>, u32)>>,
}
impl KillSlot {
pub(crate) fn new() -> KillSlot {
KillSlot {
state: AtomicU64::new(0),
raw: AtomicU64::new(0),
soft_bound: AtomicU64::new(0),
soft_closes: std::sync::OnceLock::new(),
wake: Mutex::new(None),
sink: Mutex::new(None),
}
}
#[cfg_attr(not(feature = "reactor"), allow(dead_code))]
pub(crate) fn set_sink(&self, sink: Arc<dyn KillSink>, index: u32) {
*self.sink.lock().unwrap_or_else(PoisonError::into_inner) = Some((sink, index));
}
fn tell_sink(&self) {
let sink = self
.sink
.lock()
.unwrap_or_else(PoisonError::into_inner)
.clone();
if let Some((sink, index)) = sink {
sink.killed(index);
}
}
pub(crate) fn install(&self, sock: Sock<'_>) -> u32 {
self.raw
.store(super::raw_of(sock) as u64, Ordering::Relaxed);
let generation = gen_of(self.state.load(Ordering::Relaxed)).wrapping_add(1);
self.state.store(
(u64::from(generation) << GEN_SHIFT) | ACTIVE,
Ordering::Release,
);
generation
}
pub(crate) fn set_wake(&self, wake: Option<Arc<Wake>>) {
*self.wake.lock().unwrap_or_else(PoisonError::into_inner) = wake;
}
pub(crate) fn generation(&self) -> u32 {
gen_of(self.state.load(Ordering::Acquire))
}
pub(crate) fn is_open(&self, generation: u32) -> bool {
let s = self.state.load(Ordering::Acquire);
gen_of(s) == generation && s & ACTIVE != 0 && s & (CLOSING | KILLED) == 0
}
pub(crate) fn killed(&self) -> Option<KillReason> {
let s = self.state.load(Ordering::Acquire);
(s & KILLED != 0).then_some(KillReason(((s >> REASON_SHIFT) & 0xFFFF) as u16))
}
pub(crate) fn set_soft_closes(&self, places: &Arc<crate::link::SoftCloses>) {
let _ = self.soft_closes.get_or_init(|| places.clone());
}
pub(crate) fn soft_closes(&self) -> Option<&Arc<crate::link::SoftCloses>> {
self.soft_closes.get()
}
pub(crate) fn soft_kill(&self) -> Option<Duration> {
let s = self.state.load(Ordering::Acquire);
(s & (KILLED | SOFT) == KILLED | SOFT)
.then(|| Duration::from_micros(self.soft_bound.load(Ordering::Relaxed)))
}
pub(crate) fn kill_soft(&self, generation: u32, reason: KillReason, bound: Duration) -> bool {
let mut s = self.state.load(Ordering::Acquire);
loop {
if gen_of(s) != generation || s & ACTIVE == 0 || s & (CLOSING | KILLED) != 0 {
return false;
}
if s & KILLING != 0 {
std::hint::spin_loop();
s = self.state.load(Ordering::Acquire);
continue;
}
let micros = u64::try_from(bound.as_micros()).unwrap_or(u64::MAX);
self.soft_bound.store(micros, Ordering::Relaxed);
let new = (s & !(0xFFFF << REASON_SHIFT))
| KILLED
| KILLING
| SOFT
| (u64::from(reason.0) << REASON_SHIFT);
match self
.state
.compare_exchange_weak(s, new, Ordering::AcqRel, Ordering::Acquire)
{
Ok(_) => break,
Err(actual) => s = actual,
}
}
let raw = self.raw.load(Ordering::Relaxed) as RawSock;
let wake = self
.wake
.lock()
.unwrap_or_else(PoisonError::into_inner)
.clone();
let reactor = self
.sink
.lock()
.unwrap_or_else(PoisonError::into_inner)
.is_some();
let blocked = !super::WAIT_BEFORE_RECV && s & IDLE == 0;
if !reactor && (wake.is_none() || blocked) {
unsafe { super::shutdown_read_raw(raw) };
}
if let Some(w) = wake {
w.wake();
}
self.tell_sink();
self.state.fetch_and(!KILLING, Ordering::Release);
true
}
pub(crate) fn kill(&self, generation: u32, reason: KillReason) -> bool {
self.kill_with(generation, reason, |raw, wake| {
unsafe { super::shutdown_read_raw(raw) };
if let Some(w) = wake {
w.wake();
}
self.tell_sink();
})
}
pub(crate) fn kill_with(
&self,
generation: u32,
reason: KillReason,
action: impl FnOnce(RawSock, Option<Arc<Wake>>),
) -> bool {
let mut s = self.state.load(Ordering::Acquire);
loop {
let upgrade = s & (KILLED | SOFT) == KILLED | SOFT;
if gen_of(s) != generation
|| s & ACTIVE == 0
|| s & CLOSING != 0
|| (s & KILLED != 0 && !upgrade)
{
return false;
}
if s & KILLING != 0 {
std::hint::spin_loop();
s = self.state.load(Ordering::Acquire);
continue;
}
let new = (s & !((0xFFFF << REASON_SHIFT) | SOFT))
| KILLED
| KILLING
| (u64::from(reason.0) << REASON_SHIFT);
match self
.state
.compare_exchange_weak(s, new, Ordering::AcqRel, Ordering::Acquire)
{
Ok(_) => break,
Err(actual) => s = actual,
}
}
let raw = self.raw.load(Ordering::Relaxed) as RawSock;
let wake = self
.wake
.lock()
.unwrap_or_else(PoisonError::into_inner)
.clone();
action(raw, wake);
self.state.fetch_and(!KILLING, Ordering::Release);
true
}
#[inline]
pub(crate) fn set_idle(&self, idle: bool) {
if idle {
self.state.fetch_or(IDLE, Ordering::AcqRel);
} else {
self.state.fetch_and(!IDLE, Ordering::AcqRel);
}
}
pub(crate) fn drain_idle(&self, generation: u32) -> bool {
self.mark_idle(generation, DRAINED)
}
pub(crate) fn shed_idle(&self, generation: u32) -> bool {
self.mark_idle(generation, DRAINED | SHED)
}
fn mark_idle(&self, generation: u32, mark: u64) -> bool {
let mut s = self.state.load(Ordering::Acquire);
loop {
if gen_of(s) != generation
|| s & (ACTIVE | IDLE) != (ACTIVE | IDLE)
|| s & (CLOSING | KILLED | KILLING | DRAINED) != 0
{
return false;
}
match self.state.compare_exchange_weak(
s,
s | mark | KILLING,
Ordering::AcqRel,
Ordering::Acquire,
) {
Ok(_) => break,
Err(actual) => s = actual,
}
}
let raw = self.raw.load(Ordering::Relaxed) as RawSock;
let wake = self
.wake
.lock()
.unwrap_or_else(PoisonError::into_inner)
.clone();
if wake.is_none() {
#[cfg(unix)]
if unsafe { super::input_waiting_raw(raw) } {
self.state
.fetch_and(!(DRAINED | SHED | KILLING), Ordering::Release);
return false;
}
unsafe { super::shutdown_read_raw(raw) };
}
if let Some(w) = wake {
w.wake();
}
self.state.fetch_and(!KILLING, Ordering::Release);
true
}
#[inline]
pub(crate) fn undrain(&self) {
self.state.fetch_and(!(DRAINED | SHED), Ordering::AcqRel);
}
#[inline]
pub(crate) fn shed(&self) -> bool {
self.state.load(Ordering::Acquire) & SHED != 0
}
#[inline]
pub(crate) fn drained(&self) -> bool {
self.state.load(Ordering::Acquire) & DRAINED != 0
}
pub(crate) fn begin_close(&self) -> Option<KillReason> {
let s = self.state.fetch_or(CLOSING, Ordering::AcqRel);
let mut s = s | CLOSING;
let mut spins = 0u32;
while s & KILLING != 0 {
spins += 1;
if spins < 64 {
std::hint::spin_loop();
} else {
yield_now();
}
s = self.state.load(Ordering::Acquire);
}
(s & KILLED != 0).then_some(KillReason(((s >> REASON_SHIFT) & 0xFFFF) as u16))
}
pub(crate) fn uninstall(&self) {
let s = self.state.load(Ordering::Acquire);
self.state
.store(u64::from(gen_of(s)) << GEN_SHIFT, Ordering::Release);
}
fn release(&self, generation: u32) {
let s = self.state.load(Ordering::Acquire);
if gen_of(s) != generation || s & ACTIVE == 0 {
return;
}
let _ = self.begin_close();
self.uninstall();
}
}
#[derive(Debug)]
pub(crate) struct SlotLease {
slot: Arc<KillSlot>,
generation: u32,
}
impl SlotLease {
pub(crate) fn new(slot: Arc<KillSlot>, generation: u32) -> SlotLease {
SlotLease { slot, generation }
}
pub(crate) fn generation(&self) -> u32 {
self.generation
}
}
impl std::ops::Deref for SlotLease {
type Target = Arc<KillSlot>;
fn deref(&self) -> &Arc<KillSlot> {
&self.slot
}
}
impl Drop for SlotLease {
fn drop(&mut self) {
self.slot.release(self.generation);
}
}
#[cfg(not(net_io_loom))]
fn yield_now() {
std::thread::yield_now();
}
#[cfg(net_io_loom)]
fn yield_now() {
loom::thread::yield_now();
}
#[cfg(all(test, not(net_io_loom)))]
mod tests {
use super::*;
use std::net::TcpListener;
#[test]
fn test_kill_once_then_false() {
let l = TcpListener::bind("127.0.0.1:0").unwrap();
let slot = KillSlot::new();
let generation = slot.install(super::super::AsSock::sock(&l));
assert!(slot.is_open(generation));
let mut calls = 0;
assert!(slot.kill_with(generation, KillReason(7), |_, _| calls += 1));
assert!(!slot.kill_with(generation, KillReason(8), |_, _| calls += 1));
assert_eq!(calls, 1);
assert_eq!(slot.killed(), Some(KillReason(7)));
assert_eq!(slot.begin_close(), Some(KillReason(7)));
}
#[test]
fn test_soft_kill_is_upgraded_by_a_kill_once() {
let l = TcpListener::bind("127.0.0.1:0").unwrap();
let slot = KillSlot::new();
let generation = slot.install(super::super::AsSock::sock(&l));
slot.set_wake(Some(Arc::new(Wake::new().unwrap())));
assert!(slot.kill_soft(generation, KillReason(3), Duration::from_millis(250)));
assert!(!slot.kill_soft(generation, KillReason(4), Duration::from_secs(1)));
assert_eq!(slot.killed(), Some(KillReason(3)));
assert_eq!(slot.soft_kill(), Some(Duration::from_millis(250)));
assert!(!slot.is_open(generation));
let mut calls = 0;
assert!(slot.kill_with(generation, KillReason(5), |_, _| calls += 1));
assert!(!slot.kill_with(generation, KillReason(6), |_, _| calls += 1));
assert_eq!(calls, 1);
assert_eq!(slot.soft_kill(), None);
assert_eq!(slot.begin_close(), Some(KillReason(5)));
}
#[test]
fn test_kill_wrong_generation_or_closing_refused() {
let l = TcpListener::bind("127.0.0.1:0").unwrap();
let slot = KillSlot::new();
let generation = slot.install(super::super::AsSock::sock(&l));
assert!(!slot.kill_with(generation + 1, KillReason(1), |_, _| {}));
assert_eq!(slot.begin_close(), None);
assert!(!slot.kill_with(generation, KillReason(1), |_, _| {}));
slot.uninstall();
let next = slot.install(super::super::AsSock::sock(&l));
assert_eq!(next, generation + 1);
assert!(!slot.kill_with(generation, KillReason(1), |_, _| {}));
}
#[test]
fn test_drain_only_touches_an_idle_connection_once() {
let l = TcpListener::bind("127.0.0.1:0").unwrap();
let slot = KillSlot::new();
let generation = slot.install(super::super::AsSock::sock(&l));
assert!(!slot.drain_idle(generation), "busy: not drained");
slot.set_idle(true);
assert!(!slot.drain_idle(generation + 1), "wrong generation");
assert!(slot.drain_idle(generation));
assert!(!slot.drain_idle(generation), "once");
assert!(slot.drained());
assert_eq!(slot.killed(), None, "a drain is not a kill");
assert!(
slot.kill_with(generation, KillReason(2), |_, _| {}),
"a kill still works"
);
slot.set_idle(false);
assert_eq!(slot.begin_close(), Some(KillReason(2)));
}
#[cfg(unix)]
#[test]
fn test_drain_leaves_a_connection_with_input_waiting() {
use std::io::{Read, Write};
let l = TcpListener::bind("127.0.0.1:0").unwrap();
let mut peer = std::net::TcpStream::connect(l.local_addr().unwrap()).unwrap();
let (mut conn, _) = l.accept().unwrap();
let slot = KillSlot::new();
let generation = slot.install(super::super::AsSock::sock(&conn));
slot.set_idle(true);
peer.write_all(b"req").unwrap();
let raw = super::super::raw_of(super::super::AsSock::sock(&conn));
let waited = std::time::Instant::now();
while !unsafe { super::super::input_waiting_raw(raw) } {
assert!(waited.elapsed() < std::time::Duration::from_secs(5));
std::thread::yield_now();
}
assert!(!slot.drain_idle(generation), "input waiting: left alone");
assert!(!slot.drained());
let mut b = [0u8; 3];
conn.read_exact(&mut b).unwrap();
peer.write_all(b"rest").unwrap();
let mut rest = [0u8; 4];
conn.read_exact(&mut rest).expect("read side still open");
assert!(slot.drain_idle(generation), "nothing waiting: drained");
assert!(slot.drained());
}
#[test]
fn test_lease_dropped_without_a_close_ends_the_slot() {
let l = TcpListener::bind("127.0.0.1:0").unwrap();
let slot = Arc::new(KillSlot::new());
let generation = slot.install(super::super::AsSock::sock(&l));
drop(SlotLease::new(slot.clone(), generation));
assert!(!slot.is_open(generation));
assert!(!slot.kill_with(generation, KillReason(1), |_, _| {}));
let next = slot.install(super::super::AsSock::sock(&l));
let stale = SlotLease::new(slot.clone(), generation);
drop(stale);
assert!(
slot.is_open(next),
"an old lease does not end the next connection"
);
let lease = SlotLease::new(slot.clone(), next);
assert_eq!(slot.begin_close(), None);
slot.uninstall();
drop(lease);
assert!(!slot.is_open(next));
}
#[test]
fn test_kill_from_many_threads_exactly_one_wins() {
let l = TcpListener::bind("127.0.0.1:0").unwrap();
let slot = Arc::new(KillSlot::new());
let generation = slot.install(super::super::AsSock::sock(&l));
let winners: usize = (0..64)
.map(|i| {
let slot = slot.clone();
std::thread::spawn(move || {
usize::from(slot.kill_with(generation, KillReason(i), |_, _| {}))
})
})
.collect::<Vec<_>>()
.into_iter()
.map(|h| h.join().unwrap())
.sum();
assert_eq!(winners, 1);
}
}
#[cfg(all(test, net_io_loom))]
mod loom_tests {
use super::*;
use loom::sync::atomic::AtomicBool;
#[test]
fn loom_kill_never_acts_after_close() {
loom::model(|| {
let slot = loom::sync::Arc::new(KillSlot {
state: AtomicU64::new((1 << GEN_SHIFT) | ACTIVE),
raw: AtomicU64::new(3),
..KillSlot::new()
});
let closed = loom::sync::Arc::new(AtomicBool::new(false));
let (s2, c2) = (slot.clone(), closed.clone());
let killer = loom::thread::spawn(move || {
s2.kill_with(1, KillReason(1), |_, _| {
assert!(!c2.load(Ordering::SeqCst), "kill acted on a closed socket");
})
});
slot.begin_close();
closed.store(true, Ordering::SeqCst);
let _ = killer.join().unwrap();
});
}
#[test]
fn loom_unwound_lease_keeps_a_stale_kill_off_a_reused_number() {
loom::model(|| {
let slot = std::sync::Arc::new(KillSlot {
state: AtomicU64::new((1 << GEN_SHIFT) | ACTIVE),
raw: AtomicU64::new(3),
..KillSlot::new()
});
let owner = loom::sync::Arc::new(loom::sync::atomic::AtomicU32::new(1));
let (s2, o2) = (slot.clone(), owner.clone());
let killer = loom::thread::spawn(move || {
s2.kill_with(1, KillReason(1), |raw, _| {
assert_eq!(raw, 3);
assert_eq!(
o2.load(Ordering::SeqCst),
1,
"kill acted on a reused number"
);
})
});
drop(SlotLease::new(slot.clone(), 1));
owner.store(0, Ordering::SeqCst);
owner.store(2, Ordering::SeqCst);
slot.state
.store((2 << GEN_SHIFT) | ACTIVE, Ordering::Release);
let _ = killer.join().unwrap();
});
}
}