ax_task/sched/system/cpu/remote/
mod.rs

1//! Remotely observable runqueue and owner-work publication state.
2
3use super::*;
4
5mod deadline;
6mod delivery;
7mod idle_pull;
8mod ktimer;
9mod lifecycle;
10mod load_summary;
11mod owner;
12mod run_queue;
13mod scheduler;
14
15pub(crate) use deadline::{
16    CpuDeadlineActivityGuard, CpuDeadlineBase, CpuDeadlinePublicationGuard, CpuDeadlineReadGuard,
17    CpuDeadlineState, DeadlineBaseGuardSource, KtimerClaimClass, SchedulerDeadlinePublicationState,
18    SchedulerNonTimerDeadlines,
19};
20pub(crate) use delivery::PreparedMigrationDelivery;
21pub(crate) use idle_pull::IdlePullReservation;
22pub use lifecycle::CpuLifecycleState;
23pub(crate) use lifecycle::{CpuRemotePublication, OwnedCpuRemotePublication};
24pub(crate) use load_summary::RunQueueLoadPublication;
25pub use owner::CpuLocalOwnerBorrow;
26pub(in crate::sched::system::cpu) use run_queue::RqCurrentUpdate;
27pub(crate) use run_queue::{
28    CpuRunQueueState, EqualRtWakeAction, OwnerRqEnqueue, RunQueueDomainPublication,
29    RunQueueGuardSource, WakePreemptionContext, WakePreemptionDecision,
30};
31pub(crate) use scheduler::{RescheduleKind, SchedulerRequestClaim, SchedulerRequestScope};
32
33/// Stable cross-CPU publication endpoint for one scheduler owner.
34///
35/// This object owns the IRQ-safe target runqueue, atomic delivery state, and
36/// intrusive owner-control inboxes. Owner-only runtime accounting and switch
37/// tail state remain in [`CpuLocal`].
38#[derive(Debug)]
39pub struct CpuRemote {
40    owner: CpuId,
41    pub(crate) cpu_capacity: u16,
42    pub(crate) migration_affinity: Arc<crate::sched::CpuSet>,
43    run_queue: IrqTicketLock<CpuRunQueueState>,
44    rt_bandwidth: IrqTicketLock<RtRunQueueBandwidth>,
45    deadline: CpuDeadlineBase,
46    /// Linux `dl_rq.extra_bw`: root-domain bandwidth published for this rq.
47    deadline_extra_bw_scaled: AtomicU64,
48    owner_state: owner::OwnerState,
49    publication: lifecycle::CpuPublicationState,
50    scheduler_request: scheduler::SchedulerRequestState,
51    ktimer: ktimer::KtimerWorkerState,
52    load: load_summary::RemoteLoadState,
53    idle_pull: idle_pull::IdlePullState,
54    delivery: delivery::RemoteDeliveryState,
55}
56
57impl CpuRemote {
58    pub(crate) fn create(
59        owner: CpuId,
60        config: TaskSystemConfig,
61        cpu_capacity: u16,
62    ) -> Result<Arc<Self>, crate::thread::TaskError> {
63        let deadline_max_bw_scaled = u64::from(config.deadline_cap_percent())
64            * crate::sched::algorithm::DEADLINE_UTILIZATION_SCALE
65            / 100;
66        let mut migration_affinity = crate::sched::CpuSet::empty(config.cpu_count());
67        assert!(migration_affinity.insert(owner));
68        crate::thread::allocation::try_arc(Self {
69            owner,
70            cpu_capacity,
71            migration_affinity: Arc::new(migration_affinity),
72            run_queue: IrqTicketLock::new(CpuRunQueueState::new(owner, config)?),
73            rt_bandwidth: IrqTicketLock::new(RtRunQueueBandwidth::offline()),
74            deadline: CpuDeadlineBase::new(config),
75            deadline_extra_bw_scaled: AtomicU64::new(deadline_max_bw_scaled),
76            owner_state: owner::OwnerState::new(),
77            publication: lifecycle::CpuPublicationState::new(),
78            scheduler_request: scheduler::SchedulerRequestState::new(),
79            ktimer: ktimer::KtimerWorkerState::new(),
80            load: load_summary::RemoteLoadState::new(),
81            idle_pull: idle_pull::IdlePullState::new(),
82            delivery: delivery::RemoteDeliveryState::new(),
83        })
84    }
85
86    /// Acquires the target CPU runqueue with local IRQs disabled.
87    ///
88    /// Thread scheduler state must be acquired before this lock whenever one
89    /// transaction needs both. Owner-only switch-tail state is never protected
90    /// by this lock and must not escape its CPU-local scheduler baton.
91    pub(crate) fn lock_run_queue(
92        &self,
93        source: RunQueueGuardSource,
94    ) -> IrqTicketGuard<'_, CpuRunQueueState> {
95        self.run_queue.lock(source.irq_guard_source())
96    }
97
98    /// Acquires this rq below an already-held task scheduler IRQ owner.
99    pub(crate) fn lock_run_queue_nested<'a>(
100        &'a self,
101        owner: &'a IrqOwner<'_>,
102    ) -> IrqTicketGuard<'a, CpuRunQueueState> {
103        self.run_queue.lock_nested(owner)
104    }
105
106    /// Acquires the rq under an already-active IRQ-off CPU owner.
107    ///
108    /// # Safety
109    ///
110    /// The caller must retain either the scheduler baton or the offline boot
111    /// CPU's Linux-style `PREEMPT_DISABLED` ownership, with local IRQs disabled
112    /// for the complete guard lifetime. See
113    /// [`IrqTicketLock::lock_irq_disabled`].
114    pub(crate) unsafe fn lock_run_queue_irq_disabled(
115        &self,
116    ) -> IrqTicketGuard<'_, CpuRunQueueState> {
117        // SAFETY: forwarded unchanged to the caller's scheduler-baton contract.
118        unsafe { self.run_queue.lock_irq_disabled() }
119    }
120
121    /// Skips the IRQ-disabled deadline-base read when no timer, expiration, or
122    /// softirq ownership has been published.
123    /// The rq lock precedes this lock when both are required. Timer IRQ code
124    /// takes only this lock; soft-timer callbacks release it before acquiring
125    /// a task control lock or rq lock.
126    pub(crate) fn read_active_deadline_base(
127        &self,
128        source: DeadlineBaseGuardSource,
129    ) -> Option<CpuDeadlineReadGuard<'_>> {
130        self.deadline.read_if_active(source)
131    }
132
133    /// Locks physical clockevent publication metadata.
134    ///
135    /// Publication changes neither the logical timer queue nor expiry
136    /// ownership, so it must not rewrite the derived active bit.
137    pub(crate) fn lock_deadline_publication(&self) -> CpuDeadlinePublicationGuard<'_> {
138        self.deadline.lock_publication()
139    }
140
141    pub(crate) fn deadline_publication_snapshot_matches(
142        &self,
143        non_timer: SchedulerNonTimerDeadlines,
144    ) -> bool {
145        self.deadline.publication_snapshot_matches(non_timer)
146    }
147
148    /// Locks a transition that may change queue, buffered expiry, or softirq
149    /// ownership and republishes the derived active bit before unlock.
150    pub(crate) fn lock_deadline_activity(
151        &self,
152        source: DeadlineBaseGuardSource,
153    ) -> CpuDeadlineActivityGuard<'_> {
154        self.deadline.lock_activity(source)
155    }
156
157    /// Skips an expiry transition when the derived base publication is empty.
158    pub(crate) fn lock_active_deadline_activity(
159        &self,
160        source: DeadlineBaseGuardSource,
161    ) -> Option<CpuDeadlineActivityGuard<'_>> {
162        self.deadline.lock_activity_if_active(source)
163    }
164
165    /// Locks Linux `rt_rq::rt_runtime_lock` after the owner rq lock when both
166    /// are required. Fair-only rq transactions never enter this ledger.
167    pub(crate) fn lock_rt_bandwidth(&self) -> IrqTicketGuard<'_, RtRunQueueBandwidth> {
168        self.rt_bandwidth
169            .lock(crate::runtime::IrqGuardSource::CpuRtBandwidthTicket)
170    }
171
172    pub(crate) fn publish_deadline_extra_bw(&self, extra_bw_scaled: u64) {
173        self.deadline_extra_bw_scaled
174            .store(extra_bw_scaled, Ordering::Release);
175    }
176
177    pub(crate) fn deadline_extra_bw_scaled(&self) -> u64 {
178        self.deadline_extra_bw_scaled.load(Ordering::Acquire)
179    }
180}
ax_task/sched/system/cpu/remote/mod.rs

ax_task/sched/system/cpu/remote/
mod.rs
