ax_task/sched/system/cpu/remote/
mod.rs

1//! Remotely observable runqueue and owner-work publication state.
2
3use super::*;
4
5mod deadline;
6mod delivery;
7mod idle_pull;
8mod ktimer;
9mod lifecycle;
10mod load_summary;
11mod owner;
12mod run_queue;
13mod scheduler;
14
15pub(crate) use deadline::{
16    CpuDeadlineActivityGuard, CpuDeadlineBase, CpuDeadlinePublicationGuard, CpuDeadlineReadGuard,
17    CpuDeadlineState, DeadlineBaseGuardSource, KtimerClaimClass, SchedulerDeadlinePublicationState,
18    SchedulerNonTimerDeadlines,
19};
20pub(crate) use delivery::PreparedMigrationDelivery;
21pub(crate) use idle_pull::IdlePullReservation;
22pub use lifecycle::CpuLifecycleState;
23pub(crate) use lifecycle::{CpuRemotePublication, OwnedCpuRemotePublication};
24pub(crate) use load_summary::RunQueueLoadPublication;
25pub use owner::CpuLocalOwnerBorrow;
26pub(in crate::sched::system::cpu) use run_queue::RqCurrentUpdate;
27pub(crate) use run_queue::{
28    CpuRunQueueState, EqualRtWakeAction, OwnerRqEnqueue, RunQueueDomainPublication,
29    RunQueueGuardSource, WakePreemptionContext, WakePreemptionDecision,
30};
31pub(crate) use scheduler::{RescheduleKind, SchedulerRequestClaim, SchedulerRequestScope};
32
33/// Stable cross-CPU publication endpoint for one scheduler owner.
34///
35/// This object owns the IRQ-safe target runqueue, atomic delivery state, and
36/// intrusive owner-control inboxes. Owner-only runtime accounting and switch
37/// tail state remain in [`CpuLocal`].
38#[derive(Debug)]
39pub struct CpuRemote {
40    owner: CpuId,
41    pub(crate) migration_affinity: Arc<crate::sched::CpuSet>,
42    run_queue: IrqTicketLock<CpuRunQueueState>,
43    rt_bandwidth: IrqTicketLock<RtRunQueueBandwidth>,
44    deadline: CpuDeadlineBase,
45    /// Linux `dl_rq.extra_bw`: root-domain bandwidth published for this rq.
46    deadline_extra_bw_scaled: AtomicU64,
47    owner_state: owner::OwnerState,
48    publication: lifecycle::CpuPublicationState,
49    scheduler_request: scheduler::SchedulerRequestState,
50    ktimer: ktimer::KtimerWorkerState,
51    load: load_summary::RemoteLoadState,
52    idle_pull: idle_pull::IdlePullState,
53    delivery: delivery::RemoteDeliveryState,
54}
55
56impl CpuRemote {
57    pub(crate) fn create(
58        owner: CpuId,
59        config: TaskSystemConfig,
60    ) -> Result<Arc<Self>, crate::thread::TaskError> {
61        let deadline_max_bw_scaled = u64::from(config.deadline_cap_percent())
62            * crate::sched::algorithm::DEADLINE_UTILIZATION_SCALE
63            / 100;
64        let mut migration_affinity = crate::sched::CpuSet::empty(config.cpu_count());
65        assert!(migration_affinity.insert(owner));
66        crate::thread::allocation::try_arc(Self {
67            owner,
68            migration_affinity: Arc::new(migration_affinity),
69            run_queue: IrqTicketLock::new(CpuRunQueueState::new(owner, config)?),
70            rt_bandwidth: IrqTicketLock::new(RtRunQueueBandwidth::offline()),
71            deadline: CpuDeadlineBase::new(config),
72            deadline_extra_bw_scaled: AtomicU64::new(deadline_max_bw_scaled),
73            owner_state: owner::OwnerState::new(),
74            publication: lifecycle::CpuPublicationState::new(),
75            scheduler_request: scheduler::SchedulerRequestState::new(),
76            ktimer: ktimer::KtimerWorkerState::new(),
77            load: load_summary::RemoteLoadState::new(),
78            idle_pull: idle_pull::IdlePullState::new(),
79            delivery: delivery::RemoteDeliveryState::new(),
80        })
81    }
82
83    /// Acquires the target CPU runqueue with local IRQs disabled.
84    ///
85    /// Thread scheduler state must be acquired before this lock whenever one
86    /// transaction needs both. Owner-only switch-tail state is never protected
87    /// by this lock and must not escape its CPU-local scheduler baton.
88    pub(crate) fn lock_run_queue(
89        &self,
90        source: RunQueueGuardSource,
91    ) -> IrqTicketGuard<'_, CpuRunQueueState> {
92        self.run_queue.lock(source.irq_guard_source())
93    }
94
95    /// Acquires this rq below an already-held task scheduler IRQ owner.
96    pub(crate) fn lock_run_queue_nested<'a>(
97        &'a self,
98        owner: &'a IrqOwner<'_>,
99    ) -> IrqTicketGuard<'a, CpuRunQueueState> {
100        self.run_queue.lock_nested(owner)
101    }
102
103    /// Acquires the rq under an already-active IRQ-off CPU owner.
104    ///
105    /// # Safety
106    ///
107    /// The caller must retain either the scheduler baton or the offline boot
108    /// CPU's Linux-style `PREEMPT_DISABLED` ownership, with local IRQs disabled
109    /// for the complete guard lifetime. See
110    /// [`IrqTicketLock::lock_irq_disabled`].
111    pub(crate) unsafe fn lock_run_queue_irq_disabled(
112        &self,
113    ) -> IrqTicketGuard<'_, CpuRunQueueState> {
114        // SAFETY: forwarded unchanged to the caller's scheduler-baton contract.
115        unsafe { self.run_queue.lock_irq_disabled() }
116    }
117
118    /// Locks this CPU's hrtimer-style task-deadline base.
119    ///
120    /// The rq lock precedes this lock when both are required. Timer IRQ code
121    /// takes only this lock; soft-timer callbacks release it before acquiring a
122    /// task control lock or rq lock.
123    pub(crate) fn read_deadline_base(
124        &self,
125        source: DeadlineBaseGuardSource,
126    ) -> CpuDeadlineReadGuard<'_> {
127        self.deadline.read(source)
128    }
129
130    /// Skips the IRQ-disabled deadline-base read when no timer, expiration, or
131    /// softirq ownership has been published.
132    pub(crate) fn read_active_deadline_base(
133        &self,
134        source: DeadlineBaseGuardSource,
135    ) -> Option<CpuDeadlineReadGuard<'_>> {
136        self.deadline.read_if_active(source)
137    }
138
139    /// Locks physical clockevent publication metadata.
140    ///
141    /// Publication changes neither the logical timer queue nor expiry
142    /// ownership, so it must not rewrite the derived active bit.
143    pub(crate) fn lock_deadline_publication(&self) -> CpuDeadlinePublicationGuard<'_> {
144        self.deadline.lock_publication()
145    }
146
147    pub(crate) fn deadline_publication_snapshot_matches(
148        &self,
149        non_timer: SchedulerNonTimerDeadlines,
150    ) -> bool {
151        self.deadline.publication_snapshot_matches(non_timer)
152    }
153
154    /// Locks a transition that may change queue, buffered expiry, or softirq
155    /// ownership and republishes the derived active bit before unlock.
156    pub(crate) fn lock_deadline_activity(
157        &self,
158        source: DeadlineBaseGuardSource,
159    ) -> CpuDeadlineActivityGuard<'_> {
160        self.deadline.lock_activity(source)
161    }
162
163    /// Skips an expiry transition when the derived base publication is empty.
164    pub(crate) fn lock_active_deadline_activity(
165        &self,
166        source: DeadlineBaseGuardSource,
167    ) -> Option<CpuDeadlineActivityGuard<'_>> {
168        self.deadline.lock_activity_if_active(source)
169    }
170
171    /// Locks Linux `rt_rq::rt_runtime_lock` after the owner rq lock when both
172    /// are required. Fair-only rq transactions never enter this ledger.
173    pub(crate) fn lock_rt_bandwidth(&self) -> IrqTicketGuard<'_, RtRunQueueBandwidth> {
174        self.rt_bandwidth
175            .lock(crate::runtime::IrqGuardSource::CpuRtBandwidthTicket)
176    }
177
178    pub(crate) fn publish_deadline_extra_bw(&self, extra_bw_scaled: u64) {
179        self.deadline_extra_bw_scaled
180            .store(extra_bw_scaled, Ordering::Release);
181    }
182
183    pub(crate) fn deadline_extra_bw_scaled(&self) -> u64 {
184        self.deadline_extra_bw_scaled.load(Ordering::Acquire)
185    }
186}
ax_task/sched/system/cpu/remote/mod.rs

ax_task/sched/system/cpu/remote/
mod.rs
