ax_task/sched/system/task_system/scheduling/
accounting.rs

1//! Accounting under the owning scheduler transaction.
2
3use super::*;
4
5impl TaskSystem {
6    /// Charges the current dispatch and reports class budget expiration.
7    pub fn charge_current(
8        &self,
9        cpu: Pin<&mut CpuLocal>,
10        runtime_ns: u64,
11        reclaimed_ns: u64,
12    ) -> Result<ChargeOutcome, TaskError> {
13        self.ensure_owner_cpu_context(&cpu)?;
14        if !cpu.is_online() {
15            return Err(TaskError::CpuOffline(cpu.owner().as_u32()));
16        }
17        // SAFETY: the owner borrow pins the CpuLocal and its immutable remote
18        // endpoint while this scheduling transaction and all dispatch-tail
19        // mutations are live.
20        let remote = unsafe { cpu.as_ref().get_ref().remote_for_owner() };
21        let mut transaction = OwnerRqTxn::begin(self, remote);
22        if transaction.current().is_none() {
23            transaction.commit();
24            return Err(TaskError::NoRunnableThread);
25        }
26        let charge = transaction.charge_current(runtime_ns, reclaimed_ns);
27        transaction.commit();
28        Ok(ChargeOutcome {
29            slice_expired: charge.slice_expired,
30            deadline_overrun: charge.deadline_overrun,
31        })
32    }
33
34    /// Charges exactly the unaccounted runtime since the current dispatch began
35    /// or was last sampled.
36    pub fn charge_current_until(
37        &self,
38        cpu: Pin<&mut CpuLocal>,
39        reclaimed_ns: u64,
40    ) -> Result<ChargeOutcome, TaskError> {
41        self.charge_current_until_with_clock(cpu, reclaimed_ns)
42            .map(|(charge, _clock, _thread, _rq_observation)| charge)
43    }
44
45    pub(crate) fn charge_current_until_with_clock(
46        &self,
47        cpu: Pin<&mut CpuLocal>,
48        reclaimed_ns: u64,
49    ) -> Result<
50        (
51            ChargeOutcome,
52            RunQueueClockSnapshot,
53            ThreadId,
54            SchedulerDeadlineRqObservation,
55        ),
56        TaskError,
57    > {
58        self.ensure_owner_cpu_context(&cpu)?;
59        if !cpu.is_online() {
60            return Err(TaskError::CpuOffline(cpu.owner().as_u32()));
61        }
62        // SAFETY: the owner borrow pins the CpuLocal and its immutable remote
63        // endpoint while this scheduling transaction and all dispatch-tail
64        // mutations are live.
65        let remote = unsafe { cpu.as_ref().get_ref().remote_for_owner() };
66        let mut transaction = OwnerRqTxn::begin(self, remote);
67        let clock = transaction.clock();
68        let Some(thread) = transaction.current_thread() else {
69            transaction.commit();
70            return Err(TaskError::NoRunnableThread);
71        };
72        let charge = transaction.settle_current(reclaimed_ns);
73        let rq_observation = transaction.scheduler_deadline_rq_observation(cpu.as_ref().get_ref());
74        transaction.commit();
75        Ok((
76            ChargeOutcome {
77                slice_expired: charge.slice_expired,
78                deadline_overrun: charge.deadline_overrun,
79            },
80            clock,
81            thread,
82            rq_observation,
83        ))
84    }
85
86    pub(crate) fn task_tick_current_until_with_clock(
87        &self,
88        cpu: Pin<&mut CpuLocal>,
89        reclaimed_ns: u64,
90        tick_ns: u64,
91    ) -> Result<
92        (
93            ChargeOutcome,
94            RunQueueClockSnapshot,
95            ThreadId,
96            SchedulerDeadlineRqObservation,
97        ),
98        TaskError,
99    > {
100        self.ensure_owner_cpu_context(&cpu)?;
101        if !cpu.is_online() {
102            return Err(TaskError::CpuOffline(cpu.owner().as_u32()));
103        }
104        // SAFETY: the owner borrow pins the CpuLocal and its immutable remote
105        // endpoint while this scheduling transaction and all dispatch-tail
106        // mutations are live.
107        let remote = unsafe { cpu.as_ref().get_ref().remote_for_owner() };
108        let mut transaction = OwnerRqTxn::begin(self, remote);
109        let clock = transaction.clock();
110        let Some(thread) = transaction.current_thread() else {
111            transaction.commit();
112            return Err(TaskError::NoRunnableThread);
113        };
114        let charge = transaction.task_tick_current_until(reclaimed_ns, tick_ns);
115        sample_current_realtime_tick(&transaction, tick_ns);
116        let rq_observation = transaction.scheduler_deadline_rq_observation(cpu.as_ref().get_ref());
117        transaction.commit();
118        Ok((
119            ChargeOutcome {
120                slice_expired: charge.slice_expired,
121                deadline_overrun: charge.deadline_overrun,
122            },
123            clock,
124            thread,
125            rq_observation,
126        ))
127    }
128
129    pub(crate) fn clock_event_current_until_with_clock(
130        &self,
131        cpu: Pin<&mut CpuLocal>,
132        reclaimed_ns: u64,
133    ) -> Result<
134        (
135            ChargeOutcome,
136            RunQueueClockSnapshot,
137            ThreadId,
138            SchedulerDeadlineRqObservation,
139        ),
140        TaskError,
141    > {
142        self.ensure_owner_cpu_context(&cpu)?;
143        if !cpu.is_online() {
144            return Err(TaskError::CpuOffline(cpu.owner().as_u32()));
145        }
146        // SAFETY: the owner borrow pins the CpuLocal and its immutable remote
147        // endpoint for the complete accounting transaction.
148        let remote = unsafe { cpu.as_ref().get_ref().remote_for_owner() };
149        let mut transaction = OwnerRqTxn::begin(self, remote);
150        let clock = transaction.clock();
151        let Some(thread) = transaction.current_thread() else {
152            transaction.commit();
153            return Err(TaskError::NoRunnableThread);
154        };
155        let charge = transaction.clock_event_current_until(reclaimed_ns);
156        let rq_observation = transaction.scheduler_deadline_rq_observation(cpu.as_ref().get_ref());
157        transaction.commit();
158        Ok((
159            ChargeOutcome {
160                slice_expired: charge.slice_expired,
161                deadline_overrun: charge.deadline_overrun,
162            },
163            clock,
164            thread,
165            rq_observation,
166        ))
167    }
168
169    pub(crate) fn task_tick_and_clock_event_current_until_with_clock(
170        &self,
171        cpu: Pin<&mut CpuLocal>,
172        reclaimed_ns: u64,
173        tick_ns: u64,
174    ) -> Result<
175        (
176            ChargeOutcome,
177            RunQueueClockSnapshot,
178            ThreadId,
179            SchedulerDeadlineRqObservation,
180        ),
181        TaskError,
182    > {
183        self.ensure_owner_cpu_context(&cpu)?;
184        if !cpu.is_online() {
185            return Err(TaskError::CpuOffline(cpu.owner().as_u32()));
186        }
187        // SAFETY: the owner borrow pins the CpuLocal and its immutable remote
188        // endpoint for the complete accounting transaction.
189        let remote = unsafe { cpu.as_ref().get_ref().remote_for_owner() };
190        let mut transaction = OwnerRqTxn::begin(self, remote);
191        let clock = transaction.clock();
192        let Some(thread) = transaction.current_thread() else {
193            transaction.commit();
194            return Err(TaskError::NoRunnableThread);
195        };
196        let charge = transaction.task_tick_and_clock_event_current_until(reclaimed_ns, tick_ns);
197        sample_current_realtime_tick(&transaction, tick_ns);
198        let rq_observation = transaction.scheduler_deadline_rq_observation(cpu.as_ref().get_ref());
199        transaction.commit();
200        Ok((
201            ChargeOutcome {
202                slice_expired: charge.slice_expired,
203                deadline_overrun: charge.deadline_overrun,
204            },
205            clock,
206            thread,
207            rq_observation,
208        ))
209    }
210
211    /// Reports Linux `!rt_rq_throttled(rq)` for the owner runqueue.
212    pub fn rt_run_queue_may_run(&self, cpu: Pin<&mut CpuLocal>) -> Result<bool, TaskError> {
213        self.ensure_owner_cpu_context(&cpu)?;
214        self.ensure_owner_cpu_online(&cpu)?;
215        let run_queue = cpu
216            .remote()
217            .lock_run_queue(RunQueueGuardSource::RtAccounting);
218        Ok(!run_queue.rt_is_throttled() || run_queue.has_exempt_rt())
219    }
220}
221
222fn sample_current_realtime_tick(transaction: &OwnerRqTxn<'_>, tick_ns: u64) {
223    if transaction.current().is_some_and(|current| {
224        matches!(
225            current.schedule_policy_ref(),
226            SchedulePolicy::Fifo { .. } | SchedulePolicy::RoundRobin { .. }
227        )
228    }) && let Some(core) = transaction.current_core_ref()
229    {
230        core.sample_realtime_tick(transaction.clock().wall().as_nanos(), tick_ns);
231    }
232}
ax_task/sched/system/task_system/scheduling/accounting.rs

ax_task/sched/system/task_system/scheduling/
accounting.rs
