1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
//! On run queue under the owning scheduler transaction.
use super::*;
impl TaskSystem {
/// Completes Linux's `ttwu_runnable()` transaction without waiting for `on_cpu`.
///
/// If the task still owns `TASK_ON_RQ_QUEUED`, a delayed Fair task uses
/// `ENQUEUE_DELAYED` to cancel its pending dequeue while an ordinary task
/// stays linked. A concurrent dequeue instead falls through to the
/// already-reserved off-rq activation, matching `ttwu_runnable()` returning
/// false to `try_to_wake_up()`.
pub(super) fn wake_on_rq_locked(
&self,
core: &Arc<ThreadCore>,
mut sched_guard: crate::runtime::lock::IrqTicketGuard<'_, ThreadSchedState>,
target: CpuId,
intent: WakeIntent,
context: WakeTransactionContext,
) -> WakeResult {
let remote = &self.cpu_remotes[target.as_usize()];
remote.cancel_idle_pull_if_uncommitted();
let on_rq_publication = {
let (sched, irq_owner) = sched_guard.split_irq_owner();
if sched.lifecycle.state() != ThreadState::Blocked {
task_runtime::fatal_invariant(0x574b_0013, core.id().as_u64() as usize);
}
let mut run_queue = OwnerRqTxn::begin_nested(self, remote, &irq_owner);
let scheduling_state = run_queue.scheduling_state(core.id());
let revalidation = on_rq_revalidation(scheduling_state.is_some());
match revalidation {
OnRqRevalidation::ActivateOffRq => {
if sched.placement.queued_cpu().is_some()
|| sched.placement.committed_migration_target().is_some()
{
task_runtime::fatal_invariant(0x574b_0016, core.id().as_u64() as usize);
}
run_queue.commit();
None
}
OnRqRevalidation::CommitOnRq => {
let (policy, fair_wake) = wake_policy_for_revalidation(
scheduling_state.as_ref().map(|(policy, _entity)| policy),
revalidation,
)
.unwrap_or_else(|| {
task_runtime::fatal_invariant(0x574b_0016, core.id().as_u64() as usize)
});
if sched.placement.queued_cpu() != Some(target) {
task_runtime::fatal_invariant(0x574b_0016, core.id().as_u64() as usize);
}
let action = if fair_wake {
on_rq_wake_action(run_queue.is_delayed_fair(core.id()))
} else {
OnRqWakeAction::PublishAlreadyQueued
};
let on_cpu = match sched.placement.on_cpu() {
None => false,
Some(owner) if owner == target => true,
Some(owner) => {
task_runtime::fatal_invariant(0x574b_0018, owner.as_u32() as usize)
}
};
if fair_wake
&& run_queue.current().is_some_and(|current| {
matches!(current.schedule_policy(), SchedulePolicy::Fair { .. })
})
{
let _ = run_queue.settle_current(0);
}
let current_fair = fair_wake
.then(|| run_queue.current_fair_contender())
.flatten();
if fair_wake {
run_queue.update_fair_virtual_time(current_fair);
}
let (wakeup_entity, owner_work_required) = match action {
OnRqWakeAction::ReactivateDelayedFair => {
let enqueue = run_queue.reactivate_delayed_fair(
core.id(),
current_fair,
self.config.timing_granularity_ns(),
);
(
enqueue.entity().clone(),
enqueue.scheduler_deadline_refresh_required(),
)
}
OnRqWakeAction::PublishAlreadyQueued => (
scheduling_state
.map(|(_policy, entity)| entity)
.unwrap_or_else(|| {
task_runtime::fatal_invariant(
0x574b_0014,
core.id().as_u64() as usize,
)
}),
false,
),
};
if fair_wake && matches!(action, OnRqWakeAction::ReactivateDelayedFair) {
run_queue.update_fair_virtual_time(current_fair);
}
let fair_virtual_time = if fair_wake {
run_queue.virtual_time()
} else {
Default::default()
};
let reschedule_pending = remote.immediate_preemption_requested();
let preemption = if on_rq_wake_preemption_required(on_cpu) {
run_queue.wakeup_preempt_with_intent(
core.id(),
policy,
&wakeup_entity,
fair_virtual_time,
WakePreemptionContext::new(
intent,
EqualRtWakeAction::PreserveFifoOrder,
reschedule_pending,
),
)
} else {
WakePreemptionDecision::KeepCurrent
};
let reschedule = preemption.reschedule_kind(policy);
core.publish_effective_schedule(policy, &wakeup_entity);
core.set_wake_cpu_hint(target);
if sched.transition(core, ThreadState::Running).is_err() {
task_runtime::fatal_invariant(0x574b_0015, core.id().as_u64() as usize);
}
remote.publish_rq_scheduler_reasons(
reschedule,
owner_work_required,
context.producer,
&irq_owner,
);
run_queue.commit();
Some(())
}
}
};
let Some(()) = on_rq_publication else {
if sched_guard.transition(core, ThreadState::Waking).is_err() {
task_runtime::fatal_invariant(0x574b_0016, core.id().as_u64() as usize);
}
return self.activate_waking_thread_locked(core, sched_guard, target, intent, context);
};
drop(sched_guard);
WakeResult::Notified
}
}