use core::sync::atomic::Ordering;
pub type IrqToken = crate::arch::CriticalToken;
pub const MAX_ACTIVE_CORES: usize = 8;
pub trait CpuArch {
fn current_core_id() -> usize;
fn max_cores() -> usize;
fn spinlock_acquire(lock: &LockWord);
fn spinlock_release(lock: &LockWord);
fn send_ipi(target_core: usize);
fn supports_smp() -> bool;
}
#[cfg(all(target_arch = "arm", target_has_atomic = "8"))]
pub type LockWord = core::sync::atomic::AtomicBool;
#[cfg(all(target_arch = "xtensa", target_has_atomic = "32"))]
pub type LockWord = core::sync::atomic::AtomicU32;
#[cfg(all(target_arch = "riscv32", target_has_atomic = "32"))]
pub type LockWord = core::sync::atomic::AtomicUsize;
#[cfg(all(feature = "std", target_has_atomic = "ptr"))]
pub type LockWord = core::sync::atomic::AtomicUsize;
#[cfg(not(any(
all(target_arch = "arm", target_has_atomic = "8"),
all(target_arch = "xtensa", target_has_atomic = "32"),
all(target_arch = "riscv32", target_has_atomic = "32"),
all(feature = "std", target_has_atomic = "ptr"),
)))]
pub type LockWord = SingleCoreLockWord;
#[cfg(not(any(
all(target_arch = "arm", target_has_atomic = "8"),
all(target_arch = "xtensa", target_has_atomic = "32"),
all(target_arch = "riscv32", target_has_atomic = "32"),
all(feature = "std", target_has_atomic = "ptr"),
)))]
pub struct SingleCoreLockWord(core::cell::UnsafeCell<bool>);
#[cfg(not(any(
all(target_arch = "arm", target_has_atomic = "8"),
all(target_arch = "xtensa", target_has_atomic = "32"),
all(target_arch = "riscv32", target_has_atomic = "32"),
all(feature = "std", target_has_atomic = "ptr"),
)))]
unsafe impl Sync for SingleCoreLockWord {}
#[cfg(not(any(
all(target_arch = "arm", target_has_atomic = "8"),
all(target_arch = "xtensa", target_has_atomic = "32"),
all(target_arch = "riscv32", target_has_atomic = "32"),
all(feature = "std", target_has_atomic = "ptr"),
)))]
impl SingleCoreLockWord {
pub const fn new(v: bool) -> Self {
SingleCoreLockWord(core::cell::UnsafeCell::new(v))
}
#[inline]
pub fn load(&self, _o: Ordering) -> bool {
unsafe { core::ptr::read_volatile(self.0.get()) }
}
#[inline]
pub fn store(&self, v: bool, _o: Ordering) {
unsafe { core::ptr::write_volatile(self.0.get(), v) }
}
}
#[cfg(any(
all(target_arch = "arm", target_has_atomic = "8"),
not(any(
all(target_arch = "arm", target_has_atomic = "8"),
all(target_arch = "xtensa", target_has_atomic = "32"),
all(target_arch = "riscv32", target_has_atomic = "32"),
all(feature = "std", target_has_atomic = "ptr"),
)),
))]
const fn lock_word_new() -> LockWord {
LockWord::new(false)
}
#[cfg(all(
not(all(target_arch = "arm", target_has_atomic = "8")),
any(
all(target_arch = "arm", target_has_atomic = "8"),
all(target_arch = "xtensa", target_has_atomic = "32"),
all(target_arch = "riscv32", target_has_atomic = "32"),
all(feature = "std", target_has_atomic = "ptr"),
),
))]
const fn lock_word_new() -> LockWord {
LockWord::new(0)
}
#[cfg(any(
all(target_arch = "arm", target_has_atomic = "8"),
not(any(
all(target_arch = "arm", target_has_atomic = "8"),
all(target_arch = "xtensa", target_has_atomic = "32"),
all(target_arch = "riscv32", target_has_atomic = "32"),
all(feature = "std", target_has_atomic = "ptr"),
)),
))]
mod word_ops {
use super::{LockWord, Ordering};
#[inline]
#[allow(dead_code)] pub fn lock_is_held(word: &LockWord) -> bool {
word.load(Ordering::Relaxed)
}
#[inline]
#[allow(dead_code)] pub fn lock_clear(word: &LockWord) {
word.store(false, Ordering::Relaxed)
}
}
#[cfg(all(
not(all(target_arch = "arm", target_has_atomic = "8")),
any(
all(target_arch = "arm", target_has_atomic = "8"),
all(target_arch = "xtensa", target_has_atomic = "32"),
all(target_arch = "riscv32", target_has_atomic = "32"),
all(feature = "std", target_has_atomic = "ptr"),
),
))]
mod word_ops {
use super::{LockWord, Ordering};
#[inline]
#[allow(dead_code)] pub fn lock_is_held(word: &LockWord) -> bool {
word.load(Ordering::Relaxed) != 0
}
#[inline]
#[allow(dead_code)] pub fn lock_clear(word: &LockWord) {
word.store(0, Ordering::Relaxed)
}
}
#[allow(unused_imports)]
use word_ops::{lock_clear, lock_is_held};
pub struct SpinLock {
word: LockWord,
}
unsafe impl Sync for SpinLock {}
impl SpinLock {
pub const fn new() -> Self {
SpinLock {
word: lock_word_new(),
}
}
#[inline]
pub fn lock(&self) -> SpinGuard<'_> {
let irq = unsafe { crate::arch::critical_enter() };
<Native as CpuArch>::spinlock_acquire(&self.word);
SpinGuard { lock: self, irq }
}
}
impl Default for SpinLock {
fn default() -> Self {
Self::new()
}
}
#[must_use = "a spinlock is held only for the lifetime of this guard"]
pub struct SpinGuard<'a> {
lock: &'a SpinLock,
irq: IrqToken,
}
impl Drop for SpinGuard<'_> {
#[inline]
fn drop(&mut self) {
<Native as CpuArch>::spinlock_release(&self.lock.word);
unsafe { crate::arch::critical_exit(self.irq) };
}
}
#[inline]
pub fn core_id() -> usize {
<Native as CpuArch>::current_core_id()
}
#[inline]
pub fn max_cores() -> usize {
<Native as CpuArch>::max_cores()
}
#[inline]
pub fn supports_smp() -> bool {
<Native as CpuArch>::supports_smp()
}
#[inline]
pub fn send_ipi(target_core: usize) {
<Native as CpuArch>::send_ipi(target_core)
}
#[cfg(all(not(feature = "std"), target_arch = "riscv32"))]
pub struct RiscvArch;
#[cfg(all(not(feature = "std"), target_arch = "riscv32"))]
impl CpuArch for RiscvArch {
fn current_core_id() -> usize {
let id: usize;
unsafe {
core::arch::asm!("csrr {0}, mhartid", out(reg) id, options(nomem, nostack));
}
id
}
fn max_cores() -> usize {
MAX_ACTIVE_CORES
}
fn spinlock_acquire(lock: &LockWord) {
while lock
.compare_exchange_weak(0, 1, Ordering::Acquire, Ordering::Relaxed)
.is_err()
{
while lock_is_held(lock) {
core::hint::spin_loop();
}
}
}
fn spinlock_release(lock: &LockWord) {
lock.store(0, Ordering::Release);
}
fn send_ipi(target_core: usize) {
const CLINT_MSIP: usize = 0x0200_0000;
unsafe {
core::ptr::write_volatile((CLINT_MSIP + 4 * target_core) as *mut u32, 1);
}
}
fn supports_smp() -> bool {
true
}
}
#[cfg(all(
not(feature = "std"),
target_arch = "arm",
not(target_feature = "mclass")
))]
pub struct ArmArch;
#[cfg(all(
not(feature = "std"),
target_arch = "arm",
not(target_feature = "mclass")
))]
impl CpuArch for ArmArch {
fn current_core_id() -> usize {
let mpidr: u32;
unsafe {
core::arch::asm!("mrc p15, 0, {0}, c0, c0, 5", out(reg) mpidr, options(nomem, nostack));
}
(mpidr & 0xFF) as usize
}
fn max_cores() -> usize {
MAX_ACTIVE_CORES
}
fn spinlock_acquire(lock: &LockWord) {
while lock
.compare_exchange_weak(false, true, Ordering::Acquire, Ordering::Relaxed)
.is_err()
{
while lock_is_held(lock) {
core::hint::spin_loop();
}
}
}
fn spinlock_release(lock: &LockWord) {
lock.store(false, Ordering::Release);
}
fn send_ipi(target_core: usize) {
const GICD_SGIR: usize = 0x0800_0F00;
let target_list = 1u32 << target_core;
unsafe {
core::ptr::write_volatile(GICD_SGIR as *mut u32, target_list << 16);
}
}
fn supports_smp() -> bool {
true
}
}
#[cfg(all(not(feature = "std"), target_arch = "xtensa"))]
pub struct XtensaArch;
#[cfg(all(not(feature = "std"), target_arch = "xtensa"))]
impl CpuArch for XtensaArch {
fn current_core_id() -> usize {
crate::arch::core_id() as usize
}
fn max_cores() -> usize {
2
}
fn spinlock_acquire(lock: &LockWord) {
while lock
.compare_exchange_weak(0, 1, Ordering::Acquire, Ordering::Relaxed)
.is_err()
{
while lock_is_held(lock) {
core::hint::spin_loop();
}
}
}
fn spinlock_release(lock: &LockWord) {
lock.store(0, Ordering::Release);
}
fn send_ipi(target_core: usize) {
const DPORT_CPU_INTR_FROM_CPU_0: usize = 0x3FF0_0000;
unsafe {
core::ptr::write_volatile((DPORT_CPU_INTR_FROM_CPU_0 + 4 * target_core) as *mut u32, 1);
}
}
fn supports_smp() -> bool {
true
}
}
#[cfg(all(not(feature = "std"), target_arch = "arm", target_feature = "mclass"))]
pub struct SingleCoreArch;
#[cfg(all(not(feature = "std"), target_arch = "arm", target_feature = "mclass"))]
impl CpuArch for SingleCoreArch {
fn current_core_id() -> usize {
0
}
fn max_cores() -> usize {
1
}
fn spinlock_acquire(lock: &LockWord) {
while lock_is_held(lock) {
core::hint::spin_loop();
}
lock.store(true, Ordering::Relaxed);
}
fn spinlock_release(lock: &LockWord) {
lock_clear(lock);
}
fn send_ipi(_target_core: usize) {}
fn supports_smp() -> bool {
false
}
}
#[cfg(feature = "std")]
pub struct HostArch;
#[cfg(feature = "std")]
impl CpuArch for HostArch {
fn current_core_id() -> usize {
std::thread_local! {
static CORE: core::cell::Cell<usize> = const { core::cell::Cell::new(usize::MAX) };
}
static NEXT: core::sync::atomic::AtomicUsize = core::sync::atomic::AtomicUsize::new(0);
CORE.with(|c| {
let id = c.get();
if id != usize::MAX {
return id;
}
let id = NEXT.fetch_add(1, Ordering::Relaxed);
c.set(id);
id
})
}
fn max_cores() -> usize {
std::thread::available_parallelism()
.map(|n| n.get())
.unwrap_or(1)
.min(MAX_ACTIVE_CORES)
}
fn spinlock_acquire(lock: &LockWord) {
while lock
.compare_exchange_weak(0, 1, Ordering::Acquire, Ordering::Relaxed)
.is_err()
{
std::thread::yield_now();
}
}
fn spinlock_release(lock: &LockWord) {
lock.store(0, Ordering::Release);
}
fn send_ipi(_target_core: usize) {}
fn supports_smp() -> bool {
true
}
}
#[cfg(feature = "std")]
pub type Native = HostArch;
#[cfg(all(not(feature = "std"), target_arch = "riscv32"))]
pub type Native = RiscvArch;
#[cfg(all(
not(feature = "std"),
target_arch = "arm",
not(target_feature = "mclass")
))]
pub type Native = ArmArch;
#[cfg(all(not(feature = "std"), target_arch = "xtensa"))]
pub type Native = XtensaArch;
#[cfg(all(not(feature = "std"), target_arch = "arm", target_feature = "mclass"))]
pub type Native = SingleCoreArch;