ax_task/sync/mutex/
pi_core.rs

1//! Lock-local state for scheduler-owned PI mutexes.
2
3use core::{
4    cell::{RefCell, UnsafeCell},
5    fmt,
6    mem::MaybeUninit,
7    ptr::NonNull,
8    sync::atomic::{AtomicU8, AtomicU64, Ordering},
9};
10
11use super::entry::{FastReleaseAttempt, try_release_current_owner_word};
12use crate::thread::{ParkTicket, ThreadHandle};
13
14static NEXT_PI_MUTEX_GENERATION: AtomicU64 = AtomicU64::new(1);
15const OWNER_HAS_WAITERS: u64 = 1 << 63;
16const OWNER_ID_MASK: u64 = !OWNER_HAS_WAITERS;
17const WAIT_STORAGE_UNINITIALIZED: u8 = 0;
18const WAIT_STORAGE_INITIALIZING: u8 = 1;
19const WAIT_STORAGE_READY: u8 = 2;
20
21/// Number of pointer-sized words reserved for provider-owned waiter metadata.
22///
23/// The ArceOS provider stores one ticket lock and one three-word ordered-tree
24/// root here. Keeping the storage in the physical mutex matches Linux
25/// `rt_mutex_base`: contention never allocates lock-local state.
26#[doc(hidden)]
27pub const PI_MUTEX_WAIT_STORAGE_WORDS: usize = 5;
28
29/// Inline storage for the scheduler-owned waiter tree.
30pub struct PiMutexWaitStorage {
31    state: AtomicU8,
32    words: UnsafeCell<[MaybeUninit<usize>; PI_MUTEX_WAIT_STORAGE_WORDS]>,
33}
34
35impl PiMutexWaitStorage {
36    const fn new() -> Self {
37        Self {
38            state: AtomicU8::new(WAIT_STORAGE_UNINITIALIZED),
39            words: UnsafeCell::new([MaybeUninit::uninit(); PI_MUTEX_WAIT_STORAGE_WORDS]),
40        }
41    }
42}
43
44/// Borrowed scheduler-owned waiter storage for one physical PI mutex.
45///
46/// Both native locks and fixed-layout external wrappers use this view, so the
47/// initialization and destruction state machine has exactly one owner.
48#[derive(Clone, Copy, Debug)]
49pub struct PiMutexWaitStorageView<'lock> {
50    state: &'lock AtomicU8,
51    words: &'lock UnsafeCell<[MaybeUninit<usize>; PI_MUTEX_WAIT_STORAGE_WORDS]>,
52}
53
54impl<'lock> PiMutexWaitStorageView<'lock> {
55    /// Creates a view over storage whose lifetime is owned by the physical lock.
56    #[doc(hidden)]
57    const fn from_parts(
58        state: &'lock AtomicU8,
59        words: &'lock UnsafeCell<[MaybeUninit<usize>; PI_MUTEX_WAIT_STORAGE_WORDS]>,
60    ) -> Self {
61        Self { state, words }
62    }
63
64    /// Returns the stable address of the provider-owned inline object.
65    #[doc(hidden)]
66    pub const fn as_ptr(self) -> *mut () {
67        self.words.get().cast()
68    }
69
70    /// Returns whether a provider object has been published.
71    #[doc(hidden)]
72    pub fn is_initialized(self) -> bool {
73        self.state.load(Ordering::Acquire) == WAIT_STORAGE_READY
74    }
75
76    /// Installs the provider waiter object exactly once without allocation.
77    ///
78    /// # Safety
79    ///
80    /// `T` must fit the published storage size and alignment. Every caller for
81    /// this storage must use the same `T`, and the provider must destroy it
82    /// through the task scheduler's waiter-handle destructor.
83    #[doc(hidden)]
84    pub unsafe fn get_or_init<T>(self, init: impl FnOnce() -> T) -> &'lock T {
85        assert!(
86            core::mem::size_of::<T>()
87                <= PI_MUTEX_WAIT_STORAGE_WORDS * core::mem::size_of::<usize>(),
88            "PI mutex provider waiter state exceeds inline storage"
89        );
90        assert!(
91            core::mem::align_of::<T>() <= core::mem::align_of::<usize>(),
92            "PI mutex provider waiter state exceeds inline alignment"
93        );
94
95        if self
96            .state
97            .compare_exchange(
98                WAIT_STORAGE_UNINITIALIZED,
99                WAIT_STORAGE_INITIALIZING,
100                Ordering::Acquire,
101                Ordering::Acquire,
102            )
103            .is_ok()
104        {
105            // SAFETY: this caller won exclusive initialization and validated
106            // the concrete object's size and alignment above.
107            unsafe { self.as_ptr().cast::<T>().write(init()) };
108            self.state.store(WAIT_STORAGE_READY, Ordering::Release);
109        } else {
110            while self.state.load(Ordering::Acquire) == WAIT_STORAGE_INITIALIZING {
111                core::hint::spin_loop();
112            }
113            assert_eq!(
114                self.state.load(Ordering::Acquire),
115                WAIT_STORAGE_READY,
116                "PI mutex waiter storage has an invalid lifecycle"
117            );
118        }
119
120        // SAFETY: READY publishes the unique initialized `T`; the containing
121        // mutex retains it until its final safe reference becomes unreachable.
122        unsafe { &*self.as_ptr().cast::<T>() }
123    }
124
125    /// Returns an already initialized provider object.
126    ///
127    /// # Safety
128    ///
129    /// `T` must be the same concrete type used by the successful initializer.
130    #[doc(hidden)]
131    pub unsafe fn get<T>(self) -> Option<&'lock T> {
132        if self.state.load(Ordering::Acquire) != WAIT_STORAGE_READY {
133            return None;
134        }
135        // SAFETY: READY publishes the initialized provider object.
136        Some(unsafe { &*self.as_ptr().cast::<T>() })
137    }
138}
139
140fn take_initialized_wait_storage(
141    state: &mut u8,
142    words: &mut [MaybeUninit<usize>; PI_MUTEX_WAIT_STORAGE_WORDS],
143) -> Option<*mut ()> {
144    match *state {
145        WAIT_STORAGE_UNINITIALIZED => None,
146        WAIT_STORAGE_READY => {
147            *state = WAIT_STORAGE_UNINITIALIZED;
148            Some(words.as_mut_ptr().cast())
149        }
150        _ => panic!("destroying PI mutex while waiter storage initializes"),
151    }
152}
153
154// SAFETY: initialization is published by `state`; concrete access remains
155// serialized by the provider object's own wait lock.
156unsafe impl Sync for PiMutexWaitStorage {}
157
158/// Non-zero scheduler identity stored in a PI mutex owner word.
159#[repr(transparent)]
160#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)]
161pub struct PiTaskId(u64);
162
163impl PiTaskId {
164    /// Creates an owner identity when `raw` fits the PI owner word.
165    pub const fn new(raw: u64) -> Option<Self> {
166        if raw == 0 || raw & OWNER_HAS_WAITERS != 0 {
167            None
168        } else {
169            Some(Self(raw))
170        }
171    }
172
173    /// Returns the scheduler-provided raw identity.
174    pub const fn get(self) -> u64 {
175        self.0
176    }
177}
178
179/// Stable identity of one physical PI mutex lifetime.
180#[repr(transparent)]
181#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)]
182pub struct PiMutexId(u64);
183
184impl PiMutexId {
185    /// Returns the globally unique generation allocated to this lock instance.
186    pub const fn get(self) -> u64 {
187        self.0
188    }
189}
190
191/// Failure of a lock-local PI owner-word transition.
192#[derive(Clone, Copy, Debug, Eq, PartialEq)]
193pub enum PiMutexStateError {
194    /// The current task attempted to acquire a mutex it already owns.
195    WaiterOwnsLock,
196    /// The owner word or lock generation violates the PI state machine.
197    InvalidState,
198}
199
200impl fmt::Display for PiMutexStateError {
201    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
202        formatter.write_str(match self {
203            Self::WaiterOwnsLock => "PI mutex waiter already owns the lock",
204            Self::InvalidState => "invalid PI mutex state",
205        })
206    }
207}
208
209impl core::error::Error for PiMutexStateError {}
210
211/// Lock-local owner word, identity, and scheduler-owned waiter handle.
212///
213/// The physical lock owns the opaque waiter handle. The task provider owns the
214/// object behind that handle and keeps its per-lock waiter tree alive until
215/// lock destruction transfers the unique inline object back to the scheduler.
216pub struct PiMutexCore {
217    owner: AtomicU64,
218    generation: AtomicU64,
219    wait_storage: PiMutexWaitStorage,
220}
221
222/// Borrowed storage of one native PI-mutex state machine.
223///
224/// The view keeps the algorithm independent from the physical wrapper. Native
225/// [`PiMutexCore`] values and OS-independent fixed-layout storage therefore
226/// execute the same owner, generation, and waiter-lifecycle transitions.
227#[derive(Clone, Copy, Debug)]
228pub struct PiMutexCoreView<'lock> {
229    owner: &'lock AtomicU64,
230    generation: &'lock AtomicU64,
231    wait_storage: PiMutexWaitStorageView<'lock>,
232}
233
234impl<'lock> PiMutexCoreView<'lock> {
235    /// Creates a PI core view over one physical lock's complete storage.
236    #[doc(hidden)]
237    pub(in crate::sync) const fn from_parts(
238        owner: &'lock AtomicU64,
239        generation: &'lock AtomicU64,
240        wait_state: &'lock AtomicU8,
241        wait_words: &'lock UnsafeCell<[MaybeUninit<usize>; PI_MUTEX_WAIT_STORAGE_WORDS]>,
242    ) -> Self {
243        Self {
244            owner,
245            generation,
246            wait_storage: PiMutexWaitStorageView::from_parts(wait_state, wait_words),
247        }
248    }
249
250    /// Attempts the atomic uncontended acquisition path.
251    pub fn try_acquire(self, current: PiTaskId) -> Result<PiMutexAcquire, PiMutexStateError> {
252        match self
253            .owner
254            .compare_exchange(0, current.get(), Ordering::Acquire, Ordering::Relaxed)
255        {
256            Ok(_) => Ok(PiMutexAcquire::Acquired),
257            Err(owner) if owner & OWNER_ID_MASK == current.get() => {
258                Err(PiMutexStateError::WaiterOwnsLock)
259            }
260            Err(_) => Ok(PiMutexAcquire::Contended),
261        }
262    }
263
264    /// Attempts acquisition for an explicitly scheduler-authorized identity.
265    ///
266    /// # Safety
267    ///
268    /// The caller must own the scheduler authority to establish `current` as
269    /// this physical mutex's executing owner.
270    #[doc(hidden)]
271    pub unsafe fn try_acquire_for_thread<T>(
272        self,
273        current: T,
274    ) -> Result<PiMutexAcquire, PiMutexStateError>
275    where
276        T: Into<PiTaskId>,
277    {
278        self.try_acquire(current.into())
279    }
280
281    /// Attempts release for an explicitly scheduler-authorized identity.
282    ///
283    /// # Safety
284    ///
285    /// The caller must own scheduler authority for `current` and serialize the
286    /// transition with the physical mutex owner.
287    #[doc(hidden)]
288    pub unsafe fn try_release_for_thread<T>(self, current: T) -> Result<bool, PiMutexStateError>
289    where
290        T: Into<PiTaskId>,
291    {
292        let current = current.into();
293        match try_release_current_owner_word(self.owner, current.get(), OWNER_ID_MASK) {
294            FastReleaseAttempt::Released => Ok(true),
295            FastReleaseAttempt::Contended => Ok(false),
296            FastReleaseAttempt::InvalidOwner => Err(PiMutexStateError::InvalidState),
297        }
298    }
299
300    /// Releases this raw mutex for the executing owner identity.
301    ///
302    /// # Safety
303    ///
304    /// The caller must own this mutex through a higher-level raw-mutex
305    /// contract, pass the executing scheduler identity as `current`, and
306    /// retain that authority through any contended handoff.
307    pub unsafe fn try_release_owned(
308        self,
309        current: PiTaskId,
310    ) -> Result<PiMutexOwnedRelease, PiMutexStateError> {
311        match try_release_current_owner_word(self.owner, current.get(), OWNER_ID_MASK) {
312            FastReleaseAttempt::Released => Ok(PiMutexOwnedRelease::Released),
313            FastReleaseAttempt::Contended => Ok(PiMutexOwnedRelease::Contended(current)),
314            FastReleaseAttempt::InvalidOwner => Err(PiMutexStateError::InvalidState),
315        }
316    }
317
318    /// Returns whether `current` is the physical owner.
319    pub fn is_owned_by(self, current: PiTaskId) -> bool {
320        owner_from_word(self.owner.load(Ordering::Acquire)) == Some(current)
321    }
322
323    /// Returns whether the mutex is owned or in an ownerless handoff window.
324    pub fn is_locked(self) -> bool {
325        self.owner.load(Ordering::Relaxed) != 0
326    }
327
328    /// Borrows this physical lock's generation-bearing scheduler identity.
329    pub fn mutex_ref(self) -> Result<PiMutexRef<'lock>, PiMutexStateError> {
330        let observed = self.generation.load(Ordering::Acquire);
331        if observed != 0 {
332            return Ok(PiMutexRef {
333                core: self,
334                id: PiMutexId(observed),
335            });
336        }
337
338        let allocated = NEXT_PI_MUTEX_GENERATION
339            .try_update(Ordering::AcqRel, Ordering::Acquire, |next| {
340                next.checked_add(1)
341            })
342            .map(PiMutexId)
343            .map_err(|_| PiMutexStateError::InvalidState)?;
344        match self
345            .generation
346            .compare_exchange(0, allocated.0, Ordering::AcqRel, Ordering::Acquire)
347        {
348            Ok(_) => Ok(PiMutexRef {
349                core: self,
350                id: allocated,
351            }),
352            Err(installed) if installed != 0 => Ok(PiMutexRef {
353                core: self,
354                id: PiMutexId(installed),
355            }),
356            Err(_) => Err(PiMutexStateError::InvalidState),
357        }
358    }
359
360    /// Returns the lock-local owner snapshot protected by a provider wait lock.
361    #[doc(hidden)]
362    pub fn owner_snapshot(self) -> PiMutexOwnerSnapshot {
363        let word = self.owner.load(Ordering::Acquire);
364        PiMutexOwnerSnapshot {
365            word,
366            owner: owner_from_word(word),
367        }
368    }
369
370    /// Attempts to acquire an unlocked snapshot while the wait lock is held.
371    #[doc(hidden)]
372    pub fn try_acquire_snapshot(self, snapshot: PiMutexOwnerSnapshot, current: PiTaskId) -> bool {
373        debug_assert_eq!(snapshot.word, 0);
374        self.owner
375            .compare_exchange(
376                snapshot.word,
377                current.get(),
378                Ordering::Acquire,
379                Ordering::Relaxed,
380            )
381            .is_ok()
382    }
383
384    /// Publishes the waiter bit while the provider wait lock is held.
385    #[doc(hidden)]
386    pub fn try_mark_waiters(self, snapshot: PiMutexOwnerSnapshot) -> bool {
387        if snapshot.has_waiters() {
388            return self.owner.load(Ordering::Acquire) == snapshot.word;
389        }
390        self.owner
391            .compare_exchange(
392                snapshot.word,
393                snapshot.word | OWNER_HAS_WAITERS,
394                Ordering::AcqRel,
395                Ordering::Acquire,
396            )
397            .is_ok()
398    }
399
400    /// Publishes an owned state after a serialized handoff claim.
401    #[doc(hidden)]
402    pub fn publish_owner(self, owner: PiTaskId, has_waiters: bool) {
403        self.owner.store(
404            owner.get() | if has_waiters { OWNER_HAS_WAITERS } else { 0 },
405            Ordering::Release,
406        );
407    }
408
409    /// Publishes the reserved ownerless handoff state.
410    #[doc(hidden)]
411    pub fn publish_ownerless(self) {
412        self.owner.store(OWNER_HAS_WAITERS, Ordering::Release);
413    }
414
415    /// Ends an ownerless handoff after its final waiter is removed.
416    #[doc(hidden)]
417    pub fn publish_unlocked(self) {
418        self.owner.store(0, Ordering::Release);
419    }
420
421    /// Clears the waiter bit while retaining an existing owner.
422    #[doc(hidden)]
423    pub fn clear_waiters_bit(self, owner: PiTaskId) {
424        self.owner.store(owner.get(), Ordering::Release);
425    }
426
427    /// Returns the inline scheduler-owned waiter storage.
428    #[doc(hidden)]
429    pub const fn wait_storage(self) -> PiMutexWaitStorageView<'lock> {
430        self.wait_storage
431    }
432}
433
434impl PiMutexCore {
435    /// Creates an unlocked PI mutex core without allocating waiter state.
436    pub const fn new() -> Self {
437        Self {
438            owner: AtomicU64::new(0),
439            generation: AtomicU64::new(0),
440            wait_storage: PiMutexWaitStorage::new(),
441        }
442    }
443
444    /// Returns a borrowed view over this physical lock's complete PI storage.
445    #[doc(hidden)]
446    pub const fn view(&self) -> PiMutexCoreView<'_> {
447        PiMutexCoreView::from_parts(
448            &self.owner,
449            &self.generation,
450            &self.wait_storage.state,
451            &self.wait_storage.words,
452        )
453    }
454}
455
456impl fmt::Debug for PiMutexCore {
457    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
458        formatter
459            .debug_struct("PiMutexCore")
460            .field(
461                "owner",
462                &owner_from_word(self.owner.load(Ordering::Relaxed)),
463            )
464            .field("generation", &self.generation.load(Ordering::Relaxed))
465            .finish_non_exhaustive()
466    }
467}
468
469impl Default for PiMutexCore {
470    fn default() -> Self {
471        Self::new()
472    }
473}
474
475impl Drop for PiMutexCore {
476    fn drop(&mut self) {
477        destroy_pi_mutex_storage(
478            &mut self.owner,
479            &mut self.generation,
480            &mut self.wait_storage.state,
481            &mut self.wait_storage.words,
482        );
483    }
484}
485
486pub(in crate::sync) fn destroy_pi_mutex_storage(
487    owner: &mut AtomicU64,
488    generation: &mut AtomicU64,
489    wait_state: &mut AtomicU8,
490    wait_words: &mut UnsafeCell<[MaybeUninit<usize>; PI_MUTEX_WAIT_STORAGE_WORDS]>,
491) {
492    *owner.get_mut() = 0;
493    *generation.get_mut() = 0;
494    if let Some(wait_handle) =
495        take_initialized_wait_storage(wait_state.get_mut(), wait_words.get_mut())
496    {
497        // SAFETY: mutable destruction makes every safe reference to this core
498        // unreachable, and the waiter handle verifies its tree is empty before
499        // releasing the inline object.
500        unsafe { crate::thread::drop_pi_mutex_wait_handle(wait_handle) };
501    }
502}
503
504/// Borrowed scheduler capability of one physical PI mutex.
505#[derive(Clone, Copy, Debug)]
506pub struct PiMutexRef<'lock> {
507    core: PiMutexCoreView<'lock>,
508    id: PiMutexId,
509}
510
511impl<'lock> PiMutexRef<'lock> {
512    /// Returns the stable generation-bearing lock identity.
513    pub const fn id(self) -> PiMutexId {
514        self.id
515    }
516
517    /// Returns the borrowed physical core.
518    #[doc(hidden)]
519    pub const fn core(self) -> PiMutexCoreView<'lock> {
520        self.core
521    }
522
523    /// Converts the borrow into a token-scoped raw capability.
524    #[doc(hidden)]
525    pub fn raw(self) -> PiMutexRaw {
526        PiMutexRaw {
527            owner: NonNull::from(self.core.owner),
528            generation: NonNull::from(self.core.generation),
529            wait_state: NonNull::from(self.core.wait_storage.state),
530            wait_words: NonNull::from(self.core.wait_storage.words),
531            id: self.id,
532        }
533    }
534}
535
536/// Raw generation-checked reference retained by a registered waiter.
537#[derive(Clone, Copy, Debug, Eq, PartialEq)]
538pub struct PiMutexRaw {
539    owner: NonNull<AtomicU64>,
540    generation: NonNull<AtomicU64>,
541    wait_state: NonNull<AtomicU8>,
542    wait_words: NonNull<UnsafeCell<[MaybeUninit<usize>; PI_MUTEX_WAIT_STORAGE_WORDS]>>,
543    id: PiMutexId,
544}
545
546impl PiMutexRaw {
547    /// Returns the stable lock identity.
548    pub const fn id(self) -> PiMutexId {
549        self.id
550    }
551
552    /// Recovers the physical lock core while its wait token is live.
553    ///
554    /// # Safety
555    ///
556    /// The caller must hold the registration whose token retained this raw
557    /// capability and must not outlive the physical mutex.
558    #[doc(hidden)]
559    pub unsafe fn core(self) -> PiMutexCoreView<'static> {
560        PiMutexCoreView {
561            // SAFETY: the live scheduler registration retains every storage
562            // field from the same physical mutex for this complete borrow.
563            owner: unsafe { self.owner.as_ref() },
564            // SAFETY: identical registration lifetime to `owner` above.
565            generation: unsafe { self.generation.as_ref() },
566            wait_storage: PiMutexWaitStorageView {
567                // SAFETY: identical registration lifetime to `owner` above.
568                state: unsafe { self.wait_state.as_ref() },
569                // SAFETY: identical registration lifetime to `owner` above.
570                words: unsafe { self.wait_words.as_ref() },
571            },
572        }
573    }
574}
575
576// SAFETY: provider code may move the raw identity only while a live waiter
577// registration keeps the physical mutex borrowed and its generation stable.
578unsafe impl Send for PiMutexRaw {}
579unsafe impl Sync for PiMutexRaw {}
580
581/// Atomic owner snapshot serialized with a provider's waiter-tree lock.
582#[derive(Clone, Copy, Debug, Eq, PartialEq)]
583pub struct PiMutexOwnerSnapshot {
584    word: u64,
585    owner: Option<PiTaskId>,
586}
587
588impl PiMutexOwnerSnapshot {
589    /// Returns the physical owner, if one exists.
590    pub const fn owner(self) -> Option<PiTaskId> {
591        self.owner
592    }
593
594    /// Returns whether the physical mutex is fully unlocked.
595    pub const fn is_unlocked(self) -> bool {
596        self.word == 0
597    }
598
599    /// Returns whether unlock reserved an ownerless waiter handoff.
600    pub const fn is_ownerless(self) -> bool {
601        self.word == OWNER_HAS_WAITERS
602    }
603
604    /// Returns whether the slow path owns waiter metadata.
605    pub const fn has_waiters(self) -> bool {
606        self.word & OWNER_HAS_WAITERS != 0
607    }
608}
609
610/// Result of the atomic PI mutex fast acquisition path.
611#[derive(Clone, Copy, Debug, Eq, PartialEq)]
612pub enum PiMutexAcquire {
613    /// The caller became the physical owner.
614    Acquired,
615    /// The caller must register in the task provider's waiter tree.
616    Contended,
617}
618
619/// Result of an owner-authorized PI mutex release.
620#[derive(Clone, Copy, Debug, Eq, PartialEq)]
621pub enum PiMutexOwnedRelease {
622    /// The uncontended owner word was released atomically.
623    Released,
624    /// Provider metadata must select and wake the next waiter.
625    Contended(PiTaskId),
626}
627
628/// Token joining one provider registration to the physical lock lifetime.
629#[must_use = "a PI wait token must be granted or explicitly cancelled"]
630#[derive(Debug)]
631pub struct PiWaitToken {
632    thread: PiTaskId,
633    initial_owner: Option<ThreadHandle>,
634    generation: u64,
635    lock: PiMutexRaw,
636    provider_waiter: NonNull<()>,
637    prepared_park: RefCell<Option<ParkTicket>>,
638}
639
640impl PiWaitToken {
641    /// Creates a token after the provider committed both waiter-tree edges.
642    ///
643    /// # Safety
644    ///
645    /// `lock`, `thread`, and `generation` must name one live registration, and
646    /// that registration must keep the physical mutex alive until cancellation
647    /// or handoff claim completes.
648    #[doc(hidden)]
649    pub unsafe fn from_registration(
650        lock: PiMutexRaw,
651        thread: PiTaskId,
652        initial_owner: Option<ThreadHandle>,
653        generation: u64,
654        provider_waiter: NonNull<()>,
655    ) -> Self {
656        Self {
657            thread,
658            initial_owner,
659            generation,
660            lock,
661            provider_waiter,
662            prepared_park: RefCell::new(None),
663        }
664    }
665
666    pub(crate) fn install_prepared_park(&self, ticket: ParkTicket) {
667        assert_eq!(
668            ticket.thread().as_u64(),
669            self.thread.get(),
670            "PI waiter park ticket must belong to the registered task"
671        );
672        assert!(
673            self.prepared_park.replace(Some(ticket)).is_none(),
674            "PI waiter may own only one prepared park"
675        );
676    }
677
678    pub(crate) fn take_prepared_park(&self) -> Option<ParkTicket> {
679        self.prepared_park.take()
680    }
681
682    /// Returns the registered task identity.
683    pub const fn thread_id(&self) -> PiTaskId {
684        self.thread
685    }
686
687    /// Returns the owner observed by the registration transaction.
688    pub fn initial_owner(&self) -> Option<PiTaskId> {
689        self.initial_owner
690            .as_ref()
691            .map(|owner| PiTaskId::from(owner.id()))
692    }
693
694    /// Borrows the generation-valid owner capability retained at registration.
695    #[doc(hidden)]
696    pub(crate) fn initial_owner_handle(&self) -> Option<&ThreadHandle> {
697        self.initial_owner.as_ref()
698    }
699
700    /// Returns the task-local waiter generation.
701    #[doc(hidden)]
702    pub const fn generation(&self) -> u64 {
703        self.generation
704    }
705
706    /// Returns the registered physical lock identity.
707    #[doc(hidden)]
708    pub const fn lock_raw(&self) -> PiMutexRaw {
709        self.lock
710    }
711
712    /// Returns the provider-owned task-local waiter capability.
713    ///
714    /// # Safety
715    ///
716    /// Only the provider that created this token may interpret the pointer,
717    /// and only while the waiter registration remains live.
718    #[doc(hidden)]
719    pub const unsafe fn provider_waiter(&self) -> NonNull<()> {
720        self.provider_waiter
721    }
722
723    /// Returns whether scheduler handoff completed for this generation.
724    pub fn is_granted(&self) -> bool {
725        crate::runtime::sync::pi_waiter_is_granted(self)
726    }
727
728    /// Returns whether this waiter is first and the mutex is ownerless.
729    pub fn can_claim(&self) -> bool {
730        self.is_top_waiter() && unsafe { self.lock.core() }.owner_snapshot().is_ownerless()
731    }
732
733    /// Returns whether this waiter is currently first in the lock tree.
734    pub fn is_top_waiter(&self) -> bool {
735        crate::runtime::sync::pi_waiter_is_top(self)
736    }
737
738    /// Returns whether the owner observed at registration still occupies a CPU.
739    pub fn initial_owner_is_on_cpu(&self) -> bool {
740        super::task_result(
741            crate::runtime::sync::pi_initial_owner_is_on_cpu(self),
742            "observe PI mutex owner execution state",
743        )
744    }
745}
746
747/// Result of entering the PI mutex slow path.
748#[must_use = "a registered PI waiter must be blocked, claimed, or cancelled"]
749#[derive(Debug)]
750pub enum PiMutexLockResult {
751    /// A racing fast unlock let this caller acquire the mutex directly.
752    Acquired,
753    /// The caller is linked in the mutex-owned waiter tree.
754    Waiting(PiWaitToken),
755}
756
757/// Result of serializing one ownerless PI-mutex claim.
758#[derive(Clone, Copy, Debug, Eq, PartialEq)]
759pub enum PiMutexClaimOutcome {
760    /// This waiter was still first and became the physical owner.
761    Claimed,
762    /// The owner or top waiter changed after the optimistic observation.
763    Retry,
764}
765
766/// Result of trying to cancel one committed PI waiter.
767#[derive(Clone, Copy, Debug, Eq, PartialEq)]
768pub enum PiWaitCancelOutcome {
769    /// The waiter and all inherited donations were removed.
770    Cancelled,
771    /// Unlock already published an ownerless handoff to this waiter.
772    HandoffPending,
773}
774
775fn owner_from_word(state: u64) -> Option<PiTaskId> {
776    PiTaskId::new(state & OWNER_ID_MASK)
777}
778
779#[cfg(test)]
780mod tests;
ax_task/sync/mutex/pi_core.rs

ax_task/sync/mutex/
pi_core.rs
