ax_task/sched/system/cpu/remote/
mod.rs

1//! Remotely observable runqueue and owner-work publication state.
2
3use super::*;
4
5mod deadline;
6mod delivery;
7mod idle_pull;
8mod ktimer;
9mod lifecycle;
10mod load_summary;
11mod owner;
12mod run_queue;
13mod scheduler;
14
15pub(crate) use deadline::{
16    CpuDeadlineActivityGuard, CpuDeadlineBase, CpuDeadlinePublicationGuard, CpuDeadlineReadGuard,
17    CpuDeadlineState, DeadlineBaseGuardSource, KtimerClaimClass, SchedulerDeadlinePublicationState,
18    SchedulerNonTimerDeadlines,
19};
20pub(crate) use delivery::PreparedMigrationDelivery;
21pub(crate) use idle_pull::IdlePullReservation;
22pub use lifecycle::CpuLifecycleState;
23pub(crate) use lifecycle::{CpuRemotePublication, OwnedCpuRemotePublication};
24pub(crate) use load_summary::RunQueueLoadPublication;
25pub use owner::CpuLocalOwnerBorrow;
26pub(in crate::sched::system::cpu) use run_queue::RqCurrentUpdate;
27pub(crate) use run_queue::{
28    CpuRunQueueState, EqualRtWakeAction, OwnerRqEnqueue, RunQueueDomainPublication,
29    RunQueueGuardSource, WakePreemptionContext, WakePreemptionDecision,
30};
31pub(crate) use scheduler::{RescheduleKind, SchedulerRequestClaim, SchedulerRequestScope};
32
33/// Stable cross-CPU publication endpoint for one scheduler owner.
34///
35/// This object owns the IRQ-safe target runqueue, atomic delivery state, and
36/// intrusive owner-control inboxes. Owner-only runtime accounting and switch
37/// tail state remain in [`CpuLocal`].
38#[derive(Debug)]
39pub struct CpuRemote {
40    owner: CpuId,
41    pub(crate) migration_affinity: Arc<crate::sched::CpuSet>,
42    run_queue: IrqTicketLock<CpuRunQueueState>,
43    rt_bandwidth: IrqTicketLock<RtRunQueueBandwidth>,
44    deadline: CpuDeadlineBase,
45    /// Linux `dl_rq.extra_bw`: root-domain bandwidth published for this rq.
46    deadline_extra_bw_scaled: AtomicU64,
47    owner_state: owner::OwnerState,
48    publication: lifecycle::CpuPublicationState,
49    scheduler_request: scheduler::SchedulerRequestState,
50    ktimer: ktimer::KtimerWorkerState,
51    load: load_summary::RemoteLoadState,
52    idle_pull: idle_pull::IdlePullState,
53    delivery: delivery::RemoteDeliveryState,
54}
55
56impl CpuRemote {
57    pub(crate) fn create(
58        owner: CpuId,
59        config: TaskSystemConfig,
60    ) -> Result<Arc<Self>, crate::thread::TaskError> {
61        let deadline_max_bw_scaled = u64::from(config.deadline_cap_percent())
62            * crate::sched::algorithm::DEADLINE_UTILIZATION_SCALE
63            / 100;
64        let mut migration_affinity = crate::sched::CpuSet::empty(config.cpu_count());
65        assert!(migration_affinity.insert(owner));
66        crate::thread::allocation::try_arc(Self {
67            owner,
68            migration_affinity: Arc::new(migration_affinity),
69            run_queue: IrqTicketLock::new(CpuRunQueueState::new(owner, config)?),
70            rt_bandwidth: IrqTicketLock::new(RtRunQueueBandwidth::offline()),
71            deadline: CpuDeadlineBase::new(config),
72            deadline_extra_bw_scaled: AtomicU64::new(deadline_max_bw_scaled),
73            owner_state: owner::OwnerState::new(),
74            publication: lifecycle::CpuPublicationState::new(),
75            scheduler_request: scheduler::SchedulerRequestState::new(),
76            ktimer: ktimer::KtimerWorkerState::new(),
77            load: load_summary::RemoteLoadState::new(),
78            idle_pull: idle_pull::IdlePullState::new(),
79            delivery: delivery::RemoteDeliveryState::new(),
80        })
81    }
82
83    /// Acquires the target CPU runqueue with local IRQs disabled.
84    ///
85    /// Thread scheduler state must be acquired before this lock whenever one
86    /// transaction needs both. Owner-only switch-tail state is never protected
87    /// by this lock and must not escape its CPU-local scheduler baton.
88    pub(crate) fn lock_run_queue(
89        &self,
90        source: RunQueueGuardSource,
91    ) -> IrqTicketGuard<'_, CpuRunQueueState> {
92        self.run_queue.lock(source.irq_guard_source())
93    }
94
95    /// Acquires this rq below an already-held task scheduler IRQ owner.
96    pub(crate) fn lock_run_queue_nested<'a>(
97        &'a self,
98        owner: &'a IrqOwner<'_>,
99    ) -> IrqTicketGuard<'a, CpuRunQueueState> {
100        self.run_queue.lock_nested(owner)
101    }
102
103    /// Acquires the rq under an already-active IRQ-off CPU owner.
104    ///
105    /// # Safety
106    ///
107    /// The caller must retain either the scheduler baton or the offline boot
108    /// CPU's Linux-style `PREEMPT_DISABLED` ownership, with local IRQs disabled
109    /// for the complete guard lifetime. See
110    /// [`IrqTicketLock::lock_irq_disabled`].
111    pub(crate) unsafe fn lock_run_queue_irq_disabled(
112        &self,
113    ) -> IrqTicketGuard<'_, CpuRunQueueState> {
114        // SAFETY: forwarded unchanged to the caller's scheduler-baton contract.
115        unsafe { self.run_queue.lock_irq_disabled() }
116    }
117
118    /// Skips the IRQ-disabled deadline-base read when no timer, expiration, or
119    /// softirq ownership has been published.
120    /// The rq lock precedes this lock when both are required. Timer IRQ code
121    /// takes only this lock; soft-timer callbacks release it before acquiring
122    /// a task control lock or rq lock.
123    pub(crate) fn read_active_deadline_base(
124        &self,
125        source: DeadlineBaseGuardSource,
126    ) -> Option<CpuDeadlineReadGuard<'_>> {
127        self.deadline.read_if_active(source)
128    }
129
130    /// Locks physical clockevent publication metadata.
131    ///
132    /// Publication changes neither the logical timer queue nor expiry
133    /// ownership, so it must not rewrite the derived active bit.
134    pub(crate) fn lock_deadline_publication(&self) -> CpuDeadlinePublicationGuard<'_> {
135        self.deadline.lock_publication()
136    }
137
138    pub(crate) fn deadline_publication_snapshot_matches(
139        &self,
140        non_timer: SchedulerNonTimerDeadlines,
141    ) -> bool {
142        self.deadline.publication_snapshot_matches(non_timer)
143    }
144
145    /// Locks a transition that may change queue, buffered expiry, or softirq
146    /// ownership and republishes the derived active bit before unlock.
147    pub(crate) fn lock_deadline_activity(
148        &self,
149        source: DeadlineBaseGuardSource,
150    ) -> CpuDeadlineActivityGuard<'_> {
151        self.deadline.lock_activity(source)
152    }
153
154    /// Skips an expiry transition when the derived base publication is empty.
155    pub(crate) fn lock_active_deadline_activity(
156        &self,
157        source: DeadlineBaseGuardSource,
158    ) -> Option<CpuDeadlineActivityGuard<'_>> {
159        self.deadline.lock_activity_if_active(source)
160    }
161
162    /// Locks Linux `rt_rq::rt_runtime_lock` after the owner rq lock when both
163    /// are required. Fair-only rq transactions never enter this ledger.
164    pub(crate) fn lock_rt_bandwidth(&self) -> IrqTicketGuard<'_, RtRunQueueBandwidth> {
165        self.rt_bandwidth
166            .lock(crate::runtime::IrqGuardSource::CpuRtBandwidthTicket)
167    }
168
169    pub(crate) fn publish_deadline_extra_bw(&self, extra_bw_scaled: u64) {
170        self.deadline_extra_bw_scaled
171            .store(extra_bw_scaled, Ordering::Release);
172    }
173
174    pub(crate) fn deadline_extra_bw_scaled(&self) -> u64 {
175        self.deadline_extra_bw_scaled.load(Ordering::Acquire)
176    }
177}
ax_task/sched/system/cpu/remote/mod.rs

ax_task/sched/system/cpu/remote/
mod.rs
