1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
//! Typed results published across the scheduler/runtime boundary.
use super::super::thread_sched::DeadlineActivity;
use crate::{
runtime::switch::{RuntimeSwitchPlan, ThreadRuntimeBinding},
sched::{CpuId, SchedulePolicy},
thread::{SwitchReason, ThreadCore, ThreadExtensionView, ThreadId},
};
/// Result of one scheduler safe-point decision.
#[derive(Debug)]
pub struct ScheduleDecision {
pub(super) previous: Option<ThreadId>,
pub(super) next: ThreadId,
pub(super) runtime_switch_plan: Option<RuntimeSwitchPlan>,
pub(super) switch_reason: SwitchReason,
pub(super) timestamp_ns: u64,
}
/// Result of an explicit scheduler yield.
#[derive(Debug)]
pub enum YieldOutcome {
/// The current scheduling class kept the same dispatch selected.
Unchanged,
/// The yield selected a different execution context.
Switch(ScheduleDecision),
}
impl YieldOutcome {
pub(crate) const fn decision_mut(&mut self) -> Option<&mut ScheduleDecision> {
match self {
Self::Unchanged => None,
Self::Switch(decision) => Some(decision),
}
}
}
/// Callback work that becomes valid only after the incoming thread is current.
///
/// The facade completes this work after releasing runqueue locks and its
/// CPU-local borrow, while retaining the scheduler's local IRQ exclusion.
#[doc(hidden)]
pub struct SwitchInCompletion {
thread: Option<ThreadId>,
policy: Option<SchedulePolicy>,
extension: Option<ThreadExtensionView>,
charged_runtime_ns: u64,
trace_wake: Option<fn()>,
}
impl SwitchInCompletion {
pub(crate) const NONE: Self = Self {
thread: None,
policy: None,
extension: None,
charged_runtime_ns: 0,
trace_wake: None,
};
pub(crate) fn for_core(
core: &ThreadCore,
policy: SchedulePolicy,
charged_runtime_ns: u64,
) -> Self {
Self {
thread: Some(core.id()),
policy: Some(policy),
extension: core.extension_view(),
charged_runtime_ns,
trace_wake: None,
}
}
pub(crate) fn with_trace_wake(mut self, wake: Option<fn()>) -> Self {
self.trace_wake = wake;
self
}
#[doc(hidden)]
pub fn finish(self) {
if let (Some(thread), Some(policy), Some(extension)) =
(self.thread, self.policy, self.extension)
{
// SAFETY: TaskSystem creates this token after current publication,
// previous-binding withdrawal and handoff consumption. The facade
// drops its CpuLocal borrow before finishing this token, while
// retaining the scheduler IRQ baton.
unsafe {
(extension.ops().on_switch_in)(
extension.data(),
thread,
policy,
self.charged_runtime_ns,
)
};
}
// Kernel-only incoming threads must also complete the notification.
// Capture retained no task pointer; this static callback may now wake
// its service thread without recursively acquiring the outgoing rq.
if let Some(wake) = self.trace_wake {
wake();
}
}
}
/// Result of one bounded scheduler safe point.
///
/// This type deliberately keeps lifecycle deferral and bounded owner work
/// separate from a scheduling decision. Callers must not infer either state
/// from a boolean `need_resched` value or an absent decision.
#[derive(Debug)]
pub enum SchedulerOutcome {
/// No context switch or owner-only work remains from this pass.
Quiescent,
/// The current thread owns an in-flight park token and must finish it.
ParkingDeferred,
/// One bounded owner batch completed, with more work retained.
OwnerWorkPending,
/// The scheduler selected a next thread.
Decision(ScheduleDecision),
}
impl SchedulerOutcome {
/// Returns the scheduler decision, if this pass selected a thread.
pub const fn decision(&self) -> Option<&ScheduleDecision> {
match self {
Self::Decision(decision) => Some(decision),
Self::Quiescent | Self::ParkingDeferred | Self::OwnerWorkPending => None,
}
}
pub(crate) const fn decision_mut(&mut self) -> Option<&mut ScheduleDecision> {
match self {
Self::Decision(decision) => Some(decision),
Self::Quiescent | Self::ParkingDeferred | Self::OwnerWorkPending => None,
}
}
/// Returns whether the caller must finish a pending park handshake before
/// scheduler task-work callbacks may execute.
pub const fn parking_deferred(&self) -> bool {
matches!(self, Self::ParkingDeferred)
}
/// Returns whether more owner-only work remains for a later bounded safe point.
pub const fn owner_work_pending(&self) -> bool {
matches!(self, Self::OwnerWorkPending)
}
}
impl ScheduleDecision {
/// Returns the thread that stopped running, if any.
pub const fn previous(&self) -> Option<ThreadId> {
self.previous
}
/// Returns the selected thread or CPU idle thread.
pub const fn next(&self) -> ThreadId {
self.next
}
/// Returns why the previous thread relinquished the CPU.
pub const fn switch_reason(&self) -> SwitchReason {
self.switch_reason
}
/// Returns the runqueue timestamp that committed this decision.
pub const fn timestamp_ns(&self) -> u64 {
self.timestamp_ns
}
/// Returns whether the architecture execution context must change.
pub fn requires_context_switch(&self) -> bool {
self.previous() != Some(self.next())
}
pub(crate) fn take_runtime_switch_plan(&mut self) -> Option<RuntimeSwitchPlan> {
self.runtime_switch_plan.take()
}
}
#[derive(Clone, Copy, Debug)]
pub(crate) struct SwitchEndpoint {
thread: ThreadId,
binding: ThreadRuntimeBinding,
address_space_identity: crate::runtime::resource::AddressSpaceMembarrierId,
}
impl SwitchEndpoint {
pub(crate) const fn new(
thread: ThreadId,
binding: ThreadRuntimeBinding,
address_space_identity: crate::runtime::resource::AddressSpaceMembarrierId,
) -> Self {
Self {
thread,
binding,
address_space_identity,
}
}
pub(crate) const fn thread(self) -> ThreadId {
self.thread
}
pub(crate) const fn binding(self) -> ThreadRuntimeBinding {
self.binding
}
pub(crate) const fn address_space_identity(
self,
) -> crate::runtime::resource::AddressSpaceMembarrierId {
self.address_space_identity
}
}
/// Result of charging one scheduler dispatch.
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub struct ChargeOutcome {
pub(super) slice_expired: bool,
pub(super) deadline_overrun: bool,
}
/// Snapshot of one Deadline reservation's CBS and PI state.
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub struct DeadlineRuntimeSnapshot {
pub(super) remaining_runtime_ns: u64,
pub(super) overruns: u64,
pub(super) pi_boosted: bool,
pub(super) donor: Option<ThreadId>,
}
/// Snapshot of a Deadline thread's GRUB ownership and zero-lag state.
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub struct DeadlineActivitySnapshot {
pub(super) activity: DeadlineActivity,
pub(super) bandwidth_cpu: Option<CpuId>,
pub(super) zero_lag_ns: Option<u64>,
}
impl DeadlineActivitySnapshot {
/// Returns the GRUB state.
pub const fn activity(self) -> DeadlineActivity {
self.activity
}
/// Returns the runqueue owning this reservation's `this_bw` contribution.
pub const fn bandwidth_cpu(self) -> Option<CpuId> {
self.bandwidth_cpu
}
/// Returns the pending zero-lag boundary.
pub const fn zero_lag_ns(self) -> Option<u64> {
self.zero_lag_ns
}
}
impl DeadlineRuntimeSnapshot {
/// Returns the remaining CBS runtime.
pub const fn remaining_runtime_ns(self) -> u64 {
self.remaining_runtime_ns
}
/// Returns observed CBS overruns.
pub const fn overruns(self) -> u64 {
self.overruns
}
/// Reports whether the task currently executes with a donated Deadline
/// reservation, equivalent to Linux `is_dl_boosted()`.
pub const fn pi_boosted(self) -> bool {
self.pi_boosted
}
/// Returns the original Deadline reservation currently donated to the thread.
pub const fn donor(self) -> Option<ThreadId> {
self.donor
}
}
/// Result of one bounded owner-control drain.
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub struct OwnerControlDrain {
pub(super) drained: usize,
pub(super) pending: bool,
}
impl OwnerControlDrain {
/// Returns the number of detached control messages consumed.
pub const fn drained(self) -> usize {
self.drained
}
/// Returns whether another bounded drain is required.
pub const fn pending(self) -> bool {
self.pending
}
}
impl ChargeOutcome {
/// Returns whether RR, fair service, or CBS budget reached its boundary.
pub const fn slice_expired(self) -> bool {
self.slice_expired
}
/// Returns whether CBS exhaustion entered a PI-critical rescue section.
pub const fn deadline_overrun(self) -> bool {
self.deadline_overrun
}
}