pub trait Atomics {
unsafe fn atomic_load(addr: u64) -> u64;
unsafe fn atomic_store(addr: u64, val: u64);
unsafe fn atomic_cas(addr: u64, current: u64, new: u64) -> u64;
#[inline]
unsafe fn atomic_cas_weak(addr: u64, current: u64, new: u64) -> (u64, bool) {
let read = unsafe { Self::atomic_cas(addr, current, new) };
(read, read == current)
}
#[inline]
unsafe fn atomic_swap(addr: u64, val: u64) -> u64 {
loop {
let cur = unsafe { Self::atomic_load(addr) };
if unsafe { Self::atomic_cas_weak(addr, cur, val) }.1 {
return cur;
}
}
}
#[inline]
unsafe fn atomic_add(addr: u64, val: u64) -> u64 {
loop {
let cur = unsafe { Self::atomic_load(addr) };
let new = cur.wrapping_add(val);
if unsafe { Self::atomic_cas_weak(addr, cur, new) }.1 {
return cur;
}
}
}
#[inline]
unsafe fn atomic_sub(addr: u64, val: u64) -> u64 {
loop {
let cur = unsafe { Self::atomic_load(addr) };
let new = cur.wrapping_sub(val);
if unsafe { Self::atomic_cas_weak(addr, cur, new) }.1 {
return cur;
}
}
}
#[inline]
unsafe fn atomic_and(addr: u64, val: u64) -> u64 {
loop {
let cur = unsafe { Self::atomic_load(addr) };
if unsafe { Self::atomic_cas_weak(addr, cur, cur & val) }.1 {
return cur;
}
}
}
#[inline]
unsafe fn atomic_or(addr: u64, val: u64) -> u64 {
loop {
let cur = unsafe { Self::atomic_load(addr) };
if unsafe { Self::atomic_cas_weak(addr, cur, cur | val) }.1 {
return cur;
}
}
}
#[inline]
unsafe fn atomic_inc(addr: u64) -> u64 {
unsafe { Self::atomic_add(addr, 1) }
}
#[inline]
unsafe fn atomic_dec(addr: u64) -> u64 {
unsafe { Self::atomic_sub(addr, 1) }
}
}
#[cfg(target_has_atomic = "64")]
#[derive(Clone, Copy, Debug, Default)]
pub struct CoreAtomics;
#[cfg(target_has_atomic = "64")]
impl CoreAtomics {
#[inline(always)]
unsafe fn at<'a>(addr: u64) -> &'a core::sync::atomic::AtomicU64 {
unsafe { &*(addr as usize as *const core::sync::atomic::AtomicU64) }
}
}
#[cfg(target_has_atomic = "64")]
impl Atomics for CoreAtomics {
#[inline(always)]
unsafe fn atomic_load(addr: u64) -> u64 {
unsafe { Self::at(addr) }.load(core::sync::atomic::Ordering::SeqCst)
}
#[inline(always)]
unsafe fn atomic_store(addr: u64, val: u64) {
unsafe { Self::at(addr) }.store(val, core::sync::atomic::Ordering::SeqCst);
}
#[inline(always)]
unsafe fn atomic_cas(addr: u64, current: u64, new: u64) -> u64 {
use core::sync::atomic::Ordering::SeqCst;
match unsafe { Self::at(addr) }.compare_exchange(current, new, SeqCst, SeqCst) {
Ok(prev) | Err(prev) => prev,
}
}
#[inline(always)]
unsafe fn atomic_cas_weak(addr: u64, current: u64, new: u64) -> (u64, bool) {
use core::sync::atomic::Ordering::SeqCst;
match unsafe { Self::at(addr) }.compare_exchange_weak(current, new, SeqCst, SeqCst) {
Ok(prev) => (prev, true),
Err(prev) => (prev, false),
}
}
#[inline(always)]
unsafe fn atomic_swap(addr: u64, val: u64) -> u64 {
unsafe { Self::at(addr) }.swap(val, core::sync::atomic::Ordering::SeqCst)
}
#[inline(always)]
unsafe fn atomic_add(addr: u64, val: u64) -> u64 {
unsafe { Self::at(addr) }.fetch_add(val, core::sync::atomic::Ordering::SeqCst)
}
#[inline(always)]
unsafe fn atomic_sub(addr: u64, val: u64) -> u64 {
unsafe { Self::at(addr) }.fetch_sub(val, core::sync::atomic::Ordering::SeqCst)
}
#[inline(always)]
unsafe fn atomic_and(addr: u64, val: u64) -> u64 {
unsafe { Self::at(addr) }.fetch_and(val, core::sync::atomic::Ordering::SeqCst)
}
#[inline(always)]
unsafe fn atomic_or(addr: u64, val: u64) -> u64 {
unsafe { Self::at(addr) }.fetch_or(val, core::sync::atomic::Ordering::SeqCst)
}
}