use alloc::{
alloc::{alloc, dealloc, handle_alloc_error},
boxed::Box,
};
use core::{
alloc::Layout,
cell::UnsafeCell,
iter::zip,
mem::MaybeUninit,
ptr::NonNull,
sync::atomic::{AtomicUsize, Ordering},
};
use ax_lazyinit::OnceLock;
use ax_percpu::CpuPin;
use ax_sync::PreemptGuard;
use crate::{
boxed::ItemBox,
item::{Item, Registry},
};
const SCOPE_GATE_WRITER: usize = 1 << (usize::BITS - 1);
const SCOPE_GATE_ACTIVE: usize = 1 << (usize::BITS - 2);
const SCOPE_GATE_READERS: usize = SCOPE_GATE_ACTIVE - 1;
struct ScopeGate {
state: AtomicUsize,
}
impl ScopeGate {
const fn new() -> Self {
Self {
state: AtomicUsize::new(0),
}
}
fn try_lock_shared(&self) -> bool {
let state = self.state.fetch_add(1, Ordering::Acquire);
self.finish_shared_reservation(state)
}
#[cfg(test)]
fn try_lock_shared_with(&self, interleave: impl FnOnce()) -> bool {
let state = self.state.fetch_add(1, Ordering::Acquire);
interleave();
self.finish_shared_reservation(state)
}
fn finish_shared_reservation(&self, state: usize) -> bool {
if state & SCOPE_GATE_WRITER != 0 || state & SCOPE_GATE_READERS == SCOPE_GATE_READERS {
self.state.fetch_sub(1, Ordering::Release);
return false;
}
true
}
fn try_lock_exclusive(&self) -> bool {
self.state
.compare_exchange(0, SCOPE_GATE_WRITER, Ordering::Acquire, Ordering::Relaxed)
.is_ok()
}
fn try_upgrade_active_shared_to_exclusive(&self) -> bool {
self.state
.compare_exchange(
SCOPE_GATE_ACTIVE,
SCOPE_GATE_WRITER,
Ordering::AcqRel,
Ordering::Acquire,
)
.is_ok()
}
unsafe fn downgrade_exclusive_to_active_shared(&self) {
self.state
.compare_exchange(
SCOPE_GATE_WRITER,
SCOPE_GATE_ACTIVE,
Ordering::Release,
Ordering::Relaxed,
)
.expect("scope downgrade requires one exclusive lease");
}
fn try_activate(&self) -> Result<(), ScopeActivationError> {
let mut state = self.state.load(Ordering::Acquire);
loop {
if state & SCOPE_GATE_WRITER != 0 {
return Err(ScopeActivationError::ExclusiveLease);
}
if state & SCOPE_GATE_ACTIVE != 0 {
return Err(ScopeActivationError::AlreadyActive);
}
match self.state.compare_exchange_weak(
state,
state | SCOPE_GATE_ACTIVE,
Ordering::AcqRel,
Ordering::Acquire,
) {
Ok(_) => return Ok(()),
Err(observed) => state = observed,
}
}
}
fn deactivate(&self) {
let old = self.state.fetch_and(!SCOPE_GATE_ACTIVE, Ordering::Release);
assert_ne!(
old & SCOPE_GATE_ACTIVE,
0,
"scope deactivation without a matching activation"
);
}
fn is_active(&self) -> bool {
self.state.load(Ordering::Acquire) & SCOPE_GATE_ACTIVE != 0
}
unsafe fn unlock_shared(&self) {
let old = self.state.fetch_sub(1, Ordering::Release);
assert_ne!(
old & SCOPE_GATE_READERS,
0,
"scope shared unlock without a matching lease"
);
}
unsafe fn unlock_exclusive(&self) {
let old = self.state.fetch_and(SCOPE_GATE_READERS, Ordering::Release);
assert_ne!(
old & SCOPE_GATE_WRITER,
0,
"scope exclusive unlock without a matching lease"
);
}
fn is_locked(&self) -> bool {
self.state.load(Ordering::Acquire) != 0
}
}
#[cfg(test)]
mod scope_gate_tests {
use std::sync::atomic::{AtomicBool, Ordering};
use super::{ScopeActivationError, ScopeCell, ScopeGate};
crate::scope_local! {
static GATE_TEST_ITEM: usize = 0;
}
#[test]
fn exclusive_attempt_is_bounded_by_live_readers() {
let gate = ScopeGate::new();
assert!(gate.try_lock_shared());
assert!(!gate.try_lock_exclusive());
assert!(gate.try_lock_shared());
unsafe {
gate.unlock_shared();
gate.unlock_shared();
}
assert!(gate.try_lock_exclusive());
unsafe { gate.unlock_exclusive() };
assert!(!gate.is_locked());
}
#[test]
fn active_mutation_publishes_writer_before_releasing_its_lease() {
let _retain_registry_entry = &GATE_TEST_ITEM;
let cell = ScopeCell::new();
assert_eq!(cell.try_acquire_active_lease(), Ok(()));
let barged = AtomicBool::new(false);
assert!(cell.try_withdraw_active_lease_for_writer(|| {
let admitted = cell.scope.inner().gate.try_lock_shared();
barged.store(admitted, Ordering::Relaxed);
if admitted {
unsafe { cell.scope.inner().unlock_shared() };
}
}));
unsafe { cell.scope.inner().unlock_exclusive() };
assert!(
!barged.load(Ordering::Relaxed),
"a new active lease entered after mutation began but before writer intent was visible"
);
}
#[test]
fn compatible_reader_interleave_does_not_report_busy() {
let gate = ScopeGate::new();
assert!(
gate.try_lock_shared_with(|| {
assert!(
gate.try_lock_shared(),
"the interleaved compatible reader must acquire its lease"
);
}),
"reader-count movement must not look like writer contention"
);
unsafe {
gate.unlock_shared();
gate.unlock_shared();
}
assert!(!gate.is_locked());
}
#[test]
fn activation_reports_an_exclusive_lease_separately() {
let cell = ScopeCell::new();
assert!(cell.scope.inner().gate.try_lock_exclusive());
assert_eq!(
cell.try_acquire_active_lease(),
Err(ScopeActivationError::ExclusiveLease)
);
unsafe { cell.scope.inner().gate.unlock_exclusive() };
}
}
pub struct Scope {
inner: Box<ScopeInner>,
}
struct ScopeInner {
gate: ScopeGate,
slots: NonNull<UnsafeCell<ItemSlot>>,
}
unsafe impl Send for Scope {}
unsafe impl Sync for Scope {}
impl Scope {
pub fn new() -> Self {
Self {
inner: Box::new(ScopeInner::new()),
}
}
fn inner(&self) -> &ScopeInner {
&self.inner
}
fn inner_ptr(&self) -> *const ScopeInner {
self.inner.as_ref()
}
pub(crate) fn read_item(&self, item: &'static Item) -> ScopeItemLease<'_> {
self.inner.read_item(item)
}
pub(crate) fn get_mut_unlocked(&mut self, item: &'static Item) -> &mut ItemBox {
unsafe { (&mut *self.inner.slot_ptr(item)).get_mut() }
}
}
impl Default for Scope {
fn default() -> Self {
Self::new()
}
}
impl ScopeInner {
fn len() -> usize {
Registry.len()
}
fn layout() -> Layout {
Layout::array::<UnsafeCell<ItemSlot>>(Self::len()).unwrap()
}
fn new() -> Self {
let layout = Self::layout();
let ptr = NonNull::new(unsafe { alloc(layout) })
.unwrap_or_else(|| handle_alloc_error(layout))
.cast();
let slice = unsafe {
core::slice::from_raw_parts_mut(ptr.cast::<MaybeUninit<_>>().as_ptr(), Registry.len())
};
for (item, d) in zip(&*Registry, slice) {
d.write(UnsafeCell::new(ItemSlot::new(item)));
}
Self {
gate: ScopeGate::new(),
slots: ptr,
}
}
fn try_lock_shared(&self) -> bool {
self.gate.try_lock_shared()
}
fn try_lock_exclusive(&self) -> bool {
self.gate.try_lock_exclusive()
}
unsafe fn unlock_shared(&self) {
unsafe { self.gate.unlock_shared() };
}
unsafe fn unlock_exclusive(&self) {
unsafe { self.gate.unlock_exclusive() };
}
pub(crate) fn read_item(&self, item: &'static Item) -> ScopeItemLease<'_> {
assert!(
self.try_lock_shared(),
"an exclusively borrowed scope cannot have a concurrent writer"
);
ScopeItemLease { inner: self, item }
}
fn get_shared(&self, item: &'static Item) -> &ItemBox {
let index = item.index();
unsafe { (&*self.slots.add(index).as_ref().get()).get() }
}
fn try_get_shared(&self, item: &'static Item) -> Option<&ItemBox> {
let index = item.index();
unsafe { (&*self.slots.add(index).as_ref().get()).try_get() }
}
fn slot_ptr(&self, item: &'static Item) -> *mut ItemSlot {
let index = item.index();
unsafe { self.slots.add(index).as_ref().get() }
}
}
pub(crate) struct ScopeItemLease<'scope> {
inner: &'scope ScopeInner,
item: &'static Item,
}
impl ScopeItemLease<'_> {
pub(crate) fn item(&self) -> &ItemBox {
self.inner.get_shared(self.item)
}
}
impl Drop for ScopeItemLease<'_> {
fn drop(&mut self) {
unsafe { self.inner.unlock_shared() };
}
}
impl Drop for ScopeInner {
fn drop(&mut self) {
let ptr = NonNull::slice_from_raw_parts(self.slots, Self::len());
unsafe {
ptr.drop_in_place();
dealloc(self.slots.cast().as_ptr(), Self::layout());
}
}
}
pub struct ScopeCell {
scope: Scope,
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub struct ScopeCellBusy;
impl core::fmt::Display for ScopeCellBusy {
fn fmt(&self, formatter: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
formatter.write_str("scope cell is busy")
}
}
impl core::error::Error for ScopeCellBusy {}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum ScopeActivationError {
ExclusiveLease,
AlreadyActive,
}
impl core::fmt::Display for ScopeActivationError {
fn fmt(&self, formatter: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
match self {
Self::ExclusiveLease => formatter.write_str("scope cell has an exclusive lease"),
Self::AlreadyActive => {
formatter.write_str("scope cell already has a scheduler activation")
}
}
}
}
impl core::error::Error for ScopeActivationError {}
impl ScopeCell {
pub fn new() -> Self {
Self::from_scope(Scope::new())
}
pub fn from_scope(scope: Scope) -> Self {
Self { scope }
}
pub fn try_read(&self) -> Result<ScopeCellReadGuard<'_>, ScopeCellBusy> {
let preempt = PreemptGuard::new();
if !self.scope.inner().gate.try_lock_shared() {
return Err(ScopeCellBusy);
}
Ok(ScopeCellReadGuard {
scope: &self.scope,
_preempt: preempt,
})
}
pub fn try_write(&self) -> Result<ScopeCellWriteGuard<'_>, ScopeCellBusy> {
let preempt = PreemptGuard::new();
let inner = self.scope.inner();
if !inner.try_lock_exclusive() {
return Err(ScopeCellBusy);
}
Ok(ScopeCellWriteGuard {
inner,
_preempt: Some(preempt),
owns_exclusive: true,
})
}
pub unsafe fn try_activate_pinned(&self, pin: &CpuPin<'_>) -> Result<(), ScopeActivationError> {
assert_eq!(
ActiveScope::current_scope_ptr_pinned(pin),
0,
"scope activation requires the global scope to be current"
);
self.try_acquire_active_lease()?;
unsafe { ActiveScope::set_pinned(&self.scope, pin) };
Ok(())
}
pub unsafe fn deactivate_pinned(&self, pin: &CpuPin<'_>) {
assert_eq!(
ActiveScope::current_scope_ptr_pinned(pin),
self.scope_ptr(),
"scope deactivation does not match the active scope"
);
unsafe { ActiveScope::set_global_pinned(pin) };
self.release_active_lease();
}
pub unsafe fn try_with_active_mut_pinned<R>(
&self,
pin: &CpuPin<'_>,
operation: impl for<'scope> FnOnce(&'scope mut ScopeCellWriteGuard<'_>) -> R,
) -> Result<R, ScopeCellBusy> {
assert_eq!(
ActiveScope::current_scope_ptr_pinned(pin),
self.scope_ptr(),
"active scope mutation does not match the current scope"
);
if !self.try_withdraw_active_lease_for_writer(|| {}) {
return Err(ScopeCellBusy);
}
unsafe { ActiveScope::set_global_pinned(pin) };
let inner = self.scope.inner();
let mut mutation = ActiveScopeMutation {
cell: self,
pin,
writer: Some(ScopeCellWriteGuard {
inner,
_preempt: None,
owns_exclusive: true,
}),
};
let result = operation(mutation.writer());
drop(mutation);
Ok(result)
}
fn scope_ptr(&self) -> usize {
self.scope.inner_ptr().expose_provenance()
}
fn try_acquire_active_lease(&self) -> Result<(), ScopeActivationError> {
self.scope.inner().gate.try_activate()
}
fn release_active_lease(&self) {
self.scope.inner().gate.deactivate();
}
fn try_withdraw_active_lease_for_writer(&self, writer_pending: impl FnOnce()) -> bool {
if !self
.scope
.inner()
.gate
.try_upgrade_active_shared_to_exclusive()
{
return false;
}
writer_pending();
true
}
fn restore_active_lease_from_writer(&self, pin: &CpuPin<'_>) {
unsafe {
ActiveScope::set_pinned(&self.scope, pin);
self.scope
.inner()
.gate
.downgrade_exclusive_to_active_shared();
}
}
}
struct ActiveScopeMutation<'cell, 'pin_ref, 'cpu> {
cell: &'cell ScopeCell,
pin: &'pin_ref CpuPin<'cpu>,
writer: Option<ScopeCellWriteGuard<'cell>>,
}
impl<'cell> ActiveScopeMutation<'cell, '_, '_> {
fn writer(&mut self) -> &mut ScopeCellWriteGuard<'cell> {
self.writer
.as_mut()
.expect("active scope mutation writer must be present")
}
}
impl Drop for ActiveScopeMutation<'_, '_, '_> {
fn drop(&mut self) {
let mut writer = self
.writer
.take()
.expect("active scope mutation writer must be present");
self.cell.restore_active_lease_from_writer(self.pin);
writer.owns_exclusive = false;
}
}
impl Default for ScopeCell {
fn default() -> Self {
Self::new()
}
}
impl Drop for ScopeCell {
fn drop(&mut self) {
assert!(
!self.scope.inner().gate.is_active(),
"cannot drop a scope with live scheduler activations"
);
assert!(
!self.scope.inner().gate.is_locked(),
"cannot drop a locked scope"
);
}
}
pub struct ScopeCellReadGuard<'a> {
scope: &'a Scope,
_preempt: PreemptGuard,
}
impl ScopeCellReadGuard<'_> {
pub(crate) fn get(&self, item: &'static Item) -> &ItemBox {
self.scope.inner().get_shared(item)
}
}
impl Drop for ScopeCellReadGuard<'_> {
fn drop(&mut self) {
unsafe { self.scope.inner().unlock_shared() };
}
}
pub struct ScopeCellWriteGuard<'a> {
inner: &'a ScopeInner,
_preempt: Option<PreemptGuard>,
owns_exclusive: bool,
}
impl ScopeCellWriteGuard<'_> {
pub(crate) fn get_mut(&mut self, item: &'static Item) -> &mut ItemBox {
unsafe { (&mut *self.inner.slot_ptr(item)).get_mut() }
}
}
impl Drop for ScopeCellWriteGuard<'_> {
fn drop(&mut self) {
if !self.owns_exclusive {
return;
}
unsafe { self.inner.unlock_exclusive() };
}
}
struct ItemSlot {
value: ItemBox,
}
impl ItemSlot {
fn new(item: &'static Item) -> Self {
Self {
value: ItemBox::new(item),
}
}
fn get(&self) -> &ItemBox {
&self.value
}
fn get_mut(&mut self) -> &mut ItemBox {
&mut self.value
}
fn try_get(&self) -> Option<&ItemBox> {
Some(&self.value)
}
}
static GLOBAL_SCOPE: OnceLock<Scope> = OnceLock::new();
static GLOBAL_SCOPE_STATE: AtomicUsize = AtomicUsize::new(GlobalScopeState::Uninitialized as usize);
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
#[repr(usize)]
enum GlobalScopeState {
Uninitialized,
Ready,
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
enum GlobalScopeAction {
Ready,
Recursive,
Claim,
Wait,
}
fn global_scope_action(state: usize, owner_context: usize) -> GlobalScopeAction {
if state == GlobalScopeState::Ready as usize {
GlobalScopeAction::Ready
} else if state == owner_context {
GlobalScopeAction::Recursive
} else if state == GlobalScopeState::Uninitialized as usize {
GlobalScopeAction::Claim
} else {
GlobalScopeAction::Wait
}
}
struct GlobalInitialization<'state> {
state: &'state AtomicUsize,
owner_context: usize,
published: bool,
}
impl<'state> GlobalInitialization<'state> {
fn begin(state: &'state AtomicUsize, owner_context: usize) -> Self {
Self {
state,
owner_context,
published: false,
}
}
fn publish(mut self, scope: Scope) {
GLOBAL_SCOPE.call_once(|| scope);
self.state
.store(GlobalScopeState::Ready as usize, Ordering::Release);
self.published = true;
}
}
impl Drop for GlobalInitialization<'_> {
fn drop(&mut self) {
if !self.published {
let _ = self.state.compare_exchange(
self.owner_context,
GlobalScopeState::Uninitialized as usize,
Ordering::Release,
Ordering::Relaxed,
);
}
}
}
#[ax_percpu::def_percpu]
pub(crate) static ACTIVE_SCOPE_PTR: usize = 0;
pub struct ActiveScope;
impl ActiveScope {
pub unsafe fn set(scope: &Scope) {
let _guard = PreemptGuard::new();
unsafe {
ax_percpu::with_cpu_pin(|pin| Self::set_pinned(scope, pin))
.expect("scope-local access requires an installed CPU area")
};
}
pub unsafe fn set_pinned(scope: &Scope, pin: &CpuPin<'_>) {
ACTIVE_SCOPE_PTR.write_current(pin, scope.inner_ptr().expose_provenance());
}
pub unsafe fn set_global() {
let _guard = PreemptGuard::new();
unsafe {
ax_percpu::with_cpu_pin(|pin| Self::set_global_pinned(pin))
.expect("scope-local access requires an installed CPU area")
};
}
pub unsafe fn set_global_pinned(pin: &CpuPin<'_>) {
ACTIVE_SCOPE_PTR.write_current(pin, 0);
}
pub fn is_global() -> bool {
let _guard = PreemptGuard::new();
unsafe { ax_percpu::with_cpu_pin(Self::is_global_pinned) }
.expect("scope-local access requires an installed CPU area")
}
pub fn is_global_pinned(pin: &CpuPin<'_>) -> bool {
ACTIVE_SCOPE_PTR.read_current(pin) == 0
}
pub fn is_pinned(scope: &Scope, pin: &CpuPin<'_>) -> bool {
Self::current_scope_ptr_pinned(pin) == scope.inner_ptr().expose_provenance()
}
pub(crate) fn with_item<'pin, R>(
item: &'static Item,
pin: &CpuPin<'pin>,
operation: impl for<'access> FnOnce(&'access ItemBox) -> R,
) -> R {
operation(Self::current_inner(pin).get_shared(item))
}
pub(crate) fn try_with_item<'pin, R>(
item: &'static Item,
pin: &CpuPin<'pin>,
operation: impl for<'access> FnOnce(&'access ItemBox) -> R,
) -> Option<R> {
Self::try_current_inner(pin)?
.try_get_shared(item)
.map(operation)
}
fn current_inner<'pin>(pin: &CpuPin<'pin>) -> &'pin ScopeInner {
let ptr = ACTIVE_SCOPE_PTR.read_current(pin);
let ptr = if ptr == 0 {
NonNull::from_ref(
GLOBAL_SCOPE
.get()
.expect("scope-local global scope must be initialized")
.inner(),
)
} else {
NonNull::new(core::ptr::with_exposed_provenance_mut::<ScopeInner>(ptr))
.expect("nonzero active scope address must reconstruct a pointer")
};
unsafe { ptr.as_ref() }
}
fn try_current_inner<'pin>(pin: &CpuPin<'pin>) -> Option<&'pin ScopeInner> {
let ptr = ACTIVE_SCOPE_PTR.read_current(pin);
let ptr = if ptr == 0 {
NonNull::from_ref(GLOBAL_SCOPE.get()?.inner())
} else {
NonNull::new(core::ptr::with_exposed_provenance_mut::<ScopeInner>(ptr))?
};
Some(unsafe { ptr.as_ref() })
}
pub(crate) fn initialize_global() {
let owner_context = current_context_identity();
loop {
match global_scope_action(GLOBAL_SCOPE_STATE.load(Ordering::Acquire), owner_context) {
GlobalScopeAction::Ready => return,
GlobalScopeAction::Recursive => {
panic!("scope-local global scope initialization is already in progress")
}
GlobalScopeAction::Claim => {
if GLOBAL_SCOPE_STATE
.compare_exchange(
GlobalScopeState::Uninitialized as usize,
owner_context,
Ordering::AcqRel,
Ordering::Acquire,
)
.is_ok()
{
let initialization =
GlobalInitialization::begin(&GLOBAL_SCOPE_STATE, owner_context);
initialization.publish(Scope::new());
return;
}
}
GlobalScopeAction::Wait => core::hint::spin_loop(),
}
}
}
fn current_scope_ptr_pinned(pin: &CpuPin<'_>) -> usize {
ACTIVE_SCOPE_PTR.read_current(pin)
}
}
#[cfg(test)]
mod global_scope_state_tests {
use core::sync::atomic::{AtomicUsize, Ordering};
use super::{GlobalInitialization, GlobalScopeAction, GlobalScopeState, global_scope_action};
#[test]
fn initialization_action_distinguishes_owner_and_competing_contexts() {
let owner = 17;
assert_eq!(
global_scope_action(GlobalScopeState::Uninitialized as usize, owner),
GlobalScopeAction::Claim
);
assert_eq!(
global_scope_action(owner, owner),
GlobalScopeAction::Recursive
);
assert_eq!(global_scope_action(29, owner), GlobalScopeAction::Wait);
assert_eq!(
global_scope_action(GlobalScopeState::Ready as usize, owner),
GlobalScopeAction::Ready
);
}
#[test]
fn abandoned_initialization_restores_the_retryable_state() {
let owner = 17;
let state = AtomicUsize::new(owner);
drop(GlobalInitialization::begin(&state, owner));
assert_eq!(
state.load(Ordering::Acquire),
GlobalScopeState::Uninitialized as usize
);
assert_eq!(
global_scope_action(state.load(Ordering::Acquire), owner),
GlobalScopeAction::Claim
);
}
#[test]
fn recursive_owner_unwind_restores_a_retryable_initialization() {
let owner = 17;
let state = AtomicUsize::new(GlobalScopeState::Uninitialized as usize);
assert!(
state
.compare_exchange(
GlobalScopeState::Uninitialized as usize,
owner,
Ordering::AcqRel,
Ordering::Acquire,
)
.is_ok()
);
let initialization = GlobalInitialization::begin(&state, owner);
assert_eq!(
global_scope_action(state.load(Ordering::Acquire), owner),
GlobalScopeAction::Recursive
);
drop(initialization);
assert_eq!(
global_scope_action(state.load(Ordering::Acquire), owner),
GlobalScopeAction::Claim
);
}
}
fn current_context_identity() -> usize {
let _guard = PreemptGuard::new();
let context = unsafe {
ax_percpu::with_cpu_pin(|pin| {
cpu_local::current_context(pin)
.expect("scope-local current context must be valid")
.as_ptr() as usize
})
.expect("scope-local access requires an installed CPU area")
};
assert!(
context > GlobalScopeState::Ready as usize,
"scope-local initialization requires a valid current context"
);
context
}