1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
use super::*;
#[derive(Debug)]
pub(super) struct OwnerState {
claimed: AtomicBool,
idle_thread: AtomicU64,
busy_runtime_ns: AtomicU64,
}
impl OwnerState {
pub(super) const fn new() -> Self {
Self {
claimed: AtomicBool::new(false),
idle_thread: AtomicU64::new(0),
busy_runtime_ns: AtomicU64::new(0),
}
}
}
impl CpuRemote {
/// Returns the CPU that owns the corresponding runqueue.
pub const fn owner(&self) -> CpuId {
self.owner
}
/// Claims exclusive access to the corresponding owner-only scheduler object.
///
/// # Safety
///
/// `cpu` must identify the pinned, live [`CpuLocal`] associated with this
/// endpoint. After runtime publication, every access that can overlap this
/// claim must use the same endpoint rather than retaining an ungated borrow.
pub unsafe fn claim_local(
&self,
cpu: *mut CpuLocal,
) -> Result<CpuLocalOwnerBorrow<'_>, TaskError> {
let cpu = NonNull::new(cpu).ok_or(TaskError::InvalidRuntimeHandle)?;
self.owner_state
.claimed
.compare_exchange(false, true, Ordering::Acquire, Ordering::Relaxed)
.map_err(|_| TaskError::CpuOwnerBorrowed)?;
// SAFETY: the caller guarantees that this is the live pinned CpuLocal
// paired with this endpoint. The successful gate claim excludes every
// other runtime-derived reference while the identity is checked.
let actual = unsafe { cpu.as_ref() }.owner();
if actual != self.owner {
self.owner_state.claimed.store(false, Ordering::Release);
return Err(TaskError::CpuOwnerMismatch {
expected: self.owner.as_u32(),
actual: actual.as_u32(),
});
}
#[cfg(feature = "qperf-metrics")]
crate::diagnostics::counters::record_runtime_cpu_owner_claim();
Ok(CpuLocalOwnerBorrow {
remote: self,
cpu,
release_claim: true,
_not_send_or_sync: PhantomData,
})
}
/// Borrows the owner-only scheduler state under an existing scheduler baton.
///
/// # Safety
///
/// `cpu` must identify this endpoint's pinned [`CpuLocal`]. The caller must
/// own the CPU's IRQ-off scheduler frame for the complete returned borrow,
/// and no dynamically claimed owner borrow may overlap it.
pub unsafe fn borrow_local_in_scheduler_frame(
&self,
cpu: NonNull<CpuLocal>,
) -> CpuLocalOwnerBorrow<'_> {
CpuLocalOwnerBorrow {
remote: self,
cpu,
release_claim: false,
_not_send_or_sync: PhantomData,
}
}
/// Returns `rq->curr` under the authoritative runqueue lock.
pub fn current_thread(&self) -> Option<ThreadId> {
self.lock_run_queue(RunQueueGuardSource::OwnerCurrentThreadObservation)
.current_thread()
}
/// Returns the configured idle-thread snapshot.
pub fn idle_thread(&self) -> Option<ThreadId> {
decode_thread_id(self.owner_state.idle_thread.load(Ordering::Acquire))
}
pub(in crate::sched::system::cpu) fn publish_idle_thread(&self, idle: ThreadId) {
self.owner_state
.idle_thread
.store(idle.as_u64(), Ordering::Release);
}
/// Returns cumulative time this CPU has executed non-idle scheduler threads.
pub fn busy_runtime_ns(&self) -> u64 {
self.owner_state.busy_runtime_ns.load(Ordering::Relaxed)
}
pub(in crate::sched::system::cpu) fn charge_busy_runtime(&self, runtime_ns: u64) {
// Runtime charging is serialized by this CPU's owner rq lock. Other
// CPUs only sample the counter, so an atomic read/write publication is
// sufficient and avoids pretending there are concurrent writers.
let committed = self.owner_state.busy_runtime_ns.load(Ordering::Relaxed);
self.owner_state
.busy_runtime_ns
.store(committed.saturating_add(runtime_ns), Ordering::Relaxed);
}
}
/// Exclusive owner borrow of one pinned [`CpuLocal`].
///
/// Ordinary callers acquire the dynamic gate in the separately allocated
/// [`CpuRemote`] endpoint. A live IRQ-off scheduler frame may instead lend its
/// stronger CPU-owner baton to the same borrow type without another atomic
/// ownership transaction.
pub struct CpuLocalOwnerBorrow<'remote> {
remote: &'remote CpuRemote,
cpu: NonNull<CpuLocal>,
release_claim: bool,
_not_send_or_sync: PhantomData<*mut ()>,
}
impl CpuLocalOwnerBorrow<'_> {
/// Borrows the pinned owner state mutably for one audited call scope.
pub fn as_pin_mut(&mut self) -> Pin<&mut CpuLocal> {
// SAFETY: construction claimed the unique runtime owner gate, the
// pointer remains pinned, and the returned lifetime is bounded by the
// mutable borrow of this gate-owning wrapper.
unsafe { Pin::new_unchecked(self.cpu.as_mut()) }
}
}
impl Deref for CpuLocalOwnerBorrow<'_> {
type Target = CpuLocal;
fn deref(&self) -> &Self::Target {
// SAFETY: the wrapper owns the endpoint's exclusive claim and its
// lifetime is bounded by that claim.
unsafe { self.cpu.as_ref() }
}
}
impl Drop for CpuLocalOwnerBorrow<'_> {
fn drop(&mut self) {
if self.release_claim {
self.remote
.owner_state
.claimed
.store(false, Ordering::Release);
}
}
}
fn decode_thread_id(raw: u64) -> Option<ThreadId> {
(raw != 0).then(|| ThreadId::from_parts(raw as u32, (raw >> 32) as u32))
}