ax-task 0.7.0

OS-independent IRQ-safe SMP task scheduling core
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76

use core::sync::atomic::{AtomicU64, Ordering};

/// Result of one owner-word PI-mutex fast attempt.
pub(crate) enum FastLockAttempt {
    Acquired,
    Contended,
}

/// Result of entering either the acquired or contended lock path.
pub(crate) enum LockEntry<T> {
    Acquired,
    Contended(T),
}

/// Result of one current-identity owner-word release attempt.
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub(crate) enum FastReleaseAttempt {
    Released,
    Contended,
    InvalidOwner,
}

/// Captures current, performs the sole fast attempt, and prepares slow entry.
///
/// A contended result is not retried here. The slow path owns the waiter lock
/// that excludes new fast acquisitions and retries there, matching Linux
/// rtmutex ordering.
#[inline(always)]
pub(crate) fn capture_current_and_prepare_slow<T>(
    capture_current: impl FnOnce() -> T,
    try_fast: impl FnOnce(&T) -> FastLockAttempt,
    validate_blocking_context: impl FnOnce(),
) -> LockEntry<T> {
    let current = capture_current();
    match try_fast(&current) {
        FastLockAttempt::Acquired => LockEntry::Acquired,
        FastLockAttempt::Contended => {
            validate_blocking_context();
            LockEntry::Contended(current)
        }
    }
}

/// Attempts the Linux rtmutex current-owner release transition.
pub(crate) fn try_release_current_owner_word(
    owner: &AtomicU64,
    current: u64,
    owner_id_mask: u64,
) -> FastReleaseAttempt {
    if current == 0 || current & !owner_id_mask != 0 {
        return FastReleaseAttempt::InvalidOwner;
    }
    match owner.compare_exchange(current, 0, Ordering::Release, Ordering::Relaxed) {
        Ok(_) => FastReleaseAttempt::Released,
        Err(owner_word) if owner_word & owner_id_mask == current => FastReleaseAttempt::Contended,
        Err(_) => FastReleaseAttempt::InvalidOwner,
    }
}

/// Applies the Linux SMP gate before owner-progress observations.
pub(crate) fn owner_spin_eligible(
    cpu_count: usize,
    observe_progress_gates: impl FnOnce() -> bool,
) -> bool {
    cpu_count > 1 && observe_progress_gates()
}

/// Tests the Linux owner-progress gates after SMP eligibility is established.
pub(crate) fn owner_spin_progress_gates(
    same_owner: bool,
    owner_on_cpu: bool,
    waiter_is_top: bool,
    need_resched: bool,
) -> bool {
    same_owner && owner_on_cpu && waiter_is_top && !need_resched
}