ax_task/sched/system/cpu/remote/
mod.rs

1//! Remotely observable runqueue and owner-work publication state.
2
3use super::*;
4
5mod deadline;
6mod delivery;
7mod idle_pull;
8mod ktimer;
9mod lifecycle;
10mod load_summary;
11mod owner;
12mod run_queue;
13mod scheduler;
14
15pub(crate) use deadline::{
16    CpuDeadlineActivityGuard, CpuDeadlineBase, CpuDeadlinePublicationGuard, CpuDeadlineReadGuard,
17    CpuDeadlineState, DeadlineBaseGuardSource, KtimerClaimClass, SchedulerDeadlinePublicationState,
18    SchedulerNonTimerDeadlines,
19};
20pub(crate) use delivery::PreparedMigrationDelivery;
21pub(crate) use idle_pull::IdlePullReservation;
22pub use lifecycle::CpuLifecycleState;
23pub(crate) use lifecycle::{CpuRemotePublication, OwnedCpuRemotePublication};
24pub(crate) use load_summary::RunQueueLoadPublication;
25pub use owner::CpuLocalOwnerBorrow;
26pub(in crate::sched::system::cpu) use run_queue::RqCurrentUpdate;
27pub(crate) use run_queue::{
28    CpuRunQueueState, EqualRtWakeAction, OwnerRqEnqueue, RunQueueDomainPublication,
29    RunQueueGuardSource, WakePreemptionContext, WakePreemptionDecision,
30};
31pub(crate) use scheduler::{RescheduleKind, SchedulerRequestClaim, SchedulerRequestScope};
32
33/// Stable cross-CPU publication endpoint for one scheduler owner.
34///
35/// This object owns the IRQ-safe target runqueue, atomic delivery state, and
36/// intrusive owner-control inboxes. Owner-only runtime accounting and switch
37/// tail state remain in [`CpuLocal`].
38#[derive(Debug)]
39pub struct CpuRemote {
40    owner: CpuId,
41    run_queue: IrqTicketLock<CpuRunQueueState>,
42    rt_bandwidth: IrqTicketLock<RtRunQueueBandwidth>,
43    deadline: CpuDeadlineBase,
44    /// Linux `dl_rq.extra_bw`: root-domain bandwidth published for this rq.
45    deadline_extra_bw_scaled: AtomicU64,
46    owner_state: owner::OwnerState,
47    publication: lifecycle::CpuPublicationState,
48    scheduler_request: scheduler::SchedulerRequestState,
49    ktimer: ktimer::KtimerWorkerState,
50    load: load_summary::RemoteLoadState,
51    idle_pull: idle_pull::IdlePullState,
52    delivery: delivery::RemoteDeliveryState,
53}
54
55impl CpuRemote {
56    pub(crate) fn create(owner: CpuId, config: TaskSystemConfig) -> Arc<Self> {
57        let deadline_max_bw_scaled = u64::from(config.deadline_cap_percent())
58            * crate::sched::algorithm::DEADLINE_UTILIZATION_SCALE
59            / 100;
60        Arc::new(Self {
61            owner,
62            run_queue: IrqTicketLock::new(CpuRunQueueState::new(owner, config)),
63            rt_bandwidth: IrqTicketLock::new(RtRunQueueBandwidth::offline()),
64            deadline: CpuDeadlineBase::new(config),
65            deadline_extra_bw_scaled: AtomicU64::new(deadline_max_bw_scaled),
66            owner_state: owner::OwnerState::new(),
67            publication: lifecycle::CpuPublicationState::new(),
68            scheduler_request: scheduler::SchedulerRequestState::new(),
69            ktimer: ktimer::KtimerWorkerState::new(),
70            load: load_summary::RemoteLoadState::new(),
71            idle_pull: idle_pull::IdlePullState::new(),
72            delivery: delivery::RemoteDeliveryState::new(),
73        })
74    }
75
76    /// Acquires the target CPU runqueue with local IRQs disabled.
77    ///
78    /// Thread scheduler state must be acquired before this lock whenever one
79    /// transaction needs both. Owner-only switch-tail state is never protected
80    /// by this lock and must not escape its CPU-local scheduler baton.
81    pub(crate) fn lock_run_queue(
82        &self,
83        source: RunQueueGuardSource,
84    ) -> IrqTicketGuard<'_, CpuRunQueueState> {
85        self.run_queue.lock(source.irq_guard_source())
86    }
87
88    /// Acquires this rq below an already-held task scheduler IRQ owner.
89    pub(crate) fn lock_run_queue_nested<'a>(
90        &'a self,
91        owner: &'a IrqOwner<'_>,
92    ) -> IrqTicketGuard<'a, CpuRunQueueState> {
93        self.run_queue.lock_nested(owner)
94    }
95
96    /// Acquires the rq under an already-active IRQ-off CPU owner.
97    ///
98    /// # Safety
99    ///
100    /// The caller must retain either the scheduler baton or the offline boot
101    /// CPU's Linux-style `PREEMPT_DISABLED` ownership, with local IRQs disabled
102    /// for the complete guard lifetime. See
103    /// [`IrqTicketLock::lock_irq_disabled`].
104    pub(crate) unsafe fn lock_run_queue_irq_disabled(
105        &self,
106    ) -> IrqTicketGuard<'_, CpuRunQueueState> {
107        // SAFETY: forwarded unchanged to the caller's scheduler-baton contract.
108        unsafe { self.run_queue.lock_irq_disabled() }
109    }
110
111    /// Locks this CPU's hrtimer-style task-deadline base.
112    ///
113    /// The rq lock precedes this lock when both are required. Timer IRQ code
114    /// takes only this lock; soft-timer callbacks release it before acquiring a
115    /// task control lock or rq lock.
116    pub(crate) fn read_deadline_base(
117        &self,
118        source: DeadlineBaseGuardSource,
119    ) -> CpuDeadlineReadGuard<'_> {
120        self.deadline.read(source)
121    }
122
123    /// Skips the IRQ-disabled deadline-base read when no timer, expiration, or
124    /// softirq ownership has been published.
125    pub(crate) fn read_active_deadline_base(
126        &self,
127        source: DeadlineBaseGuardSource,
128    ) -> Option<CpuDeadlineReadGuard<'_>> {
129        self.deadline.read_if_active(source)
130    }
131
132    /// Locks physical clockevent publication metadata.
133    ///
134    /// Publication changes neither the logical timer queue nor expiry
135    /// ownership, so it must not rewrite the derived active bit.
136    pub(crate) fn lock_deadline_publication(&self) -> CpuDeadlinePublicationGuard<'_> {
137        self.deadline.lock_publication()
138    }
139
140    pub(crate) fn deadline_publication_snapshot_matches(
141        &self,
142        non_timer: SchedulerNonTimerDeadlines,
143    ) -> bool {
144        self.deadline.publication_snapshot_matches(non_timer)
145    }
146
147    /// Locks a transition that may change queue, buffered expiry, or softirq
148    /// ownership and republishes the derived active bit before unlock.
149    pub(crate) fn lock_deadline_activity(
150        &self,
151        source: DeadlineBaseGuardSource,
152    ) -> CpuDeadlineActivityGuard<'_> {
153        self.deadline.lock_activity(source)
154    }
155
156    /// Skips an expiry transition when the derived base publication is empty.
157    pub(crate) fn lock_active_deadline_activity(
158        &self,
159        source: DeadlineBaseGuardSource,
160    ) -> Option<CpuDeadlineActivityGuard<'_>> {
161        self.deadline.lock_activity_if_active(source)
162    }
163
164    /// Locks Linux `rt_rq::rt_runtime_lock` after the owner rq lock when both
165    /// are required. Fair-only rq transactions never enter this ledger.
166    pub(crate) fn lock_rt_bandwidth(&self) -> IrqTicketGuard<'_, RtRunQueueBandwidth> {
167        self.rt_bandwidth
168            .lock(crate::runtime::IrqGuardSource::CpuRtBandwidthTicket)
169    }
170
171    pub(crate) fn publish_deadline_extra_bw(&self, extra_bw_scaled: u64) {
172        self.deadline_extra_bw_scaled
173            .store(extra_bw_scaled, Ordering::Release);
174    }
175
176    pub(crate) fn deadline_extra_bw_scaled(&self) -> u64 {
177        self.deadline_extra_bw_scaled.load(Ordering::Acquire)
178    }
179}
ax_task/sched/system/cpu/remote/mod.rs

ax_task/sched/system/cpu/remote/
mod.rs
