ax_task/sched/system/task_system/scheduling/
accounting.rs

1//! Accounting under the owning scheduler transaction.
2
3use super::*;
4
5impl TaskSystem {
6    /// Charges the current dispatch and reports class budget expiration.
7    pub fn charge_current(
8        &self,
9        cpu: Pin<&mut CpuLocal>,
10        runtime_ns: u64,
11        reclaimed_ns: u64,
12    ) -> Result<ChargeOutcome, TaskError> {
13        self.ensure_owner_cpu_context(&cpu)?;
14        if !cpu.is_online() {
15            return Err(TaskError::CpuOffline(cpu.owner().as_u32()));
16        }
17        // SAFETY: the owner borrow pins the CpuLocal and its immutable remote
18        // endpoint while this scheduling transaction and all dispatch-tail
19        // mutations are live.
20        let remote = unsafe { cpu.as_ref().get_ref().remote_for_owner() };
21        let mut transaction = OwnerRqTxn::begin(self, remote);
22        if transaction.current().is_none() {
23            transaction.commit();
24            return Err(TaskError::NoRunnableThread);
25        }
26        let charge = transaction.charge_current(runtime_ns, reclaimed_ns);
27        transaction.commit();
28        Ok(ChargeOutcome {
29            slice_expired: charge.slice_expired,
30            deadline_overrun: charge.deadline_overrun,
31        })
32    }
33
34    /// Charges exactly the unaccounted runtime since the current dispatch began
35    /// or was last sampled.
36    pub fn charge_current_until(
37        &self,
38        cpu: Pin<&mut CpuLocal>,
39        reclaimed_ns: u64,
40    ) -> Result<ChargeOutcome, TaskError> {
41        self.charge_current_until_with_clock(cpu, reclaimed_ns)
42            .map(|(charge, _clock, _thread, _rq_observation)| charge)
43    }
44
45    pub(crate) fn charge_current_until_with_clock(
46        &self,
47        cpu: Pin<&mut CpuLocal>,
48        reclaimed_ns: u64,
49    ) -> Result<
50        (
51            ChargeOutcome,
52            RunQueueClockSnapshot,
53            ThreadId,
54            SchedulerDeadlineRqObservation,
55        ),
56        TaskError,
57    > {
58        self.ensure_owner_cpu_context(&cpu)?;
59        if !cpu.is_online() {
60            return Err(TaskError::CpuOffline(cpu.owner().as_u32()));
61        }
62        // SAFETY: the owner borrow pins the CpuLocal and its immutable remote
63        // endpoint while this scheduling transaction and all dispatch-tail
64        // mutations are live.
65        let remote = unsafe { cpu.as_ref().get_ref().remote_for_owner() };
66        let mut transaction = OwnerRqTxn::begin(self, remote);
67        let clock = transaction.clock();
68        let Some(thread) = transaction.current_thread() else {
69            transaction.commit();
70            return Err(TaskError::NoRunnableThread);
71        };
72        let charge = transaction.settle_current(reclaimed_ns);
73        let rq_observation = transaction.scheduler_deadline_rq_observation(cpu.as_ref().get_ref());
74        transaction.commit();
75        Ok((
76            ChargeOutcome {
77                slice_expired: charge.slice_expired,
78                deadline_overrun: charge.deadline_overrun,
79            },
80            clock,
81            thread,
82            rq_observation,
83        ))
84    }
85
86    pub(crate) fn task_tick_current_until_with_clock(
87        &self,
88        cpu: Pin<&mut CpuLocal>,
89        reclaimed_ns: u64,
90        tick_ns: u64,
91    ) -> Result<
92        (
93            ChargeOutcome,
94            RunQueueClockSnapshot,
95            ThreadId,
96            SchedulerDeadlineRqObservation,
97        ),
98        TaskError,
99    > {
100        self.ensure_owner_cpu_context(&cpu)?;
101        if !cpu.is_online() {
102            return Err(TaskError::CpuOffline(cpu.owner().as_u32()));
103        }
104        // SAFETY: the owner borrow pins the CpuLocal and its immutable remote
105        // endpoint while this scheduling transaction and all dispatch-tail
106        // mutations are live.
107        let remote = unsafe { cpu.as_ref().get_ref().remote_for_owner() };
108        let mut transaction = OwnerRqTxn::begin(self, remote);
109        let clock = transaction.clock();
110        let Some(thread) = transaction.current_thread() else {
111            transaction.commit();
112            return Err(TaskError::NoRunnableThread);
113        };
114        let charge = transaction.task_tick_current_until(reclaimed_ns, tick_ns);
115        let rq_observation = transaction.scheduler_deadline_rq_observation(cpu.as_ref().get_ref());
116        transaction.commit();
117        Ok((
118            ChargeOutcome {
119                slice_expired: charge.slice_expired,
120                deadline_overrun: charge.deadline_overrun,
121            },
122            clock,
123            thread,
124            rq_observation,
125        ))
126    }
127
128    pub(crate) fn clock_event_current_until_with_clock(
129        &self,
130        cpu: Pin<&mut CpuLocal>,
131        reclaimed_ns: u64,
132    ) -> Result<
133        (
134            ChargeOutcome,
135            RunQueueClockSnapshot,
136            ThreadId,
137            SchedulerDeadlineRqObservation,
138        ),
139        TaskError,
140    > {
141        self.ensure_owner_cpu_context(&cpu)?;
142        if !cpu.is_online() {
143            return Err(TaskError::CpuOffline(cpu.owner().as_u32()));
144        }
145        // SAFETY: the owner borrow pins the CpuLocal and its immutable remote
146        // endpoint for the complete accounting transaction.
147        let remote = unsafe { cpu.as_ref().get_ref().remote_for_owner() };
148        let mut transaction = OwnerRqTxn::begin(self, remote);
149        let clock = transaction.clock();
150        let Some(thread) = transaction.current_thread() else {
151            transaction.commit();
152            return Err(TaskError::NoRunnableThread);
153        };
154        let charge = transaction.clock_event_current_until(reclaimed_ns);
155        let rq_observation = transaction.scheduler_deadline_rq_observation(cpu.as_ref().get_ref());
156        transaction.commit();
157        Ok((
158            ChargeOutcome {
159                slice_expired: charge.slice_expired,
160                deadline_overrun: charge.deadline_overrun,
161            },
162            clock,
163            thread,
164            rq_observation,
165        ))
166    }
167
168    pub(crate) fn task_tick_and_clock_event_current_until_with_clock(
169        &self,
170        cpu: Pin<&mut CpuLocal>,
171        reclaimed_ns: u64,
172        tick_ns: u64,
173    ) -> Result<
174        (
175            ChargeOutcome,
176            RunQueueClockSnapshot,
177            ThreadId,
178            SchedulerDeadlineRqObservation,
179        ),
180        TaskError,
181    > {
182        self.ensure_owner_cpu_context(&cpu)?;
183        if !cpu.is_online() {
184            return Err(TaskError::CpuOffline(cpu.owner().as_u32()));
185        }
186        // SAFETY: the owner borrow pins the CpuLocal and its immutable remote
187        // endpoint for the complete accounting transaction.
188        let remote = unsafe { cpu.as_ref().get_ref().remote_for_owner() };
189        let mut transaction = OwnerRqTxn::begin(self, remote);
190        let clock = transaction.clock();
191        let Some(thread) = transaction.current_thread() else {
192            transaction.commit();
193            return Err(TaskError::NoRunnableThread);
194        };
195        let charge = transaction.task_tick_and_clock_event_current_until(reclaimed_ns, tick_ns);
196        let rq_observation = transaction.scheduler_deadline_rq_observation(cpu.as_ref().get_ref());
197        transaction.commit();
198        Ok((
199            ChargeOutcome {
200                slice_expired: charge.slice_expired,
201                deadline_overrun: charge.deadline_overrun,
202            },
203            clock,
204            thread,
205            rq_observation,
206        ))
207    }
208
209    /// Reports Linux `!rt_rq_throttled(rq)` for the owner runqueue.
210    pub fn rt_run_queue_may_run(&self, cpu: Pin<&mut CpuLocal>) -> Result<bool, TaskError> {
211        self.ensure_owner_cpu_context(&cpu)?;
212        self.ensure_owner_cpu_online(&cpu)?;
213        let run_queue = cpu
214            .remote()
215            .lock_run_queue(RunQueueGuardSource::RtAccounting);
216        Ok(!run_queue.rt_is_throttled() || run_queue.has_exempt_rt())
217    }
218}
ax_task/sched/system/task_system/scheduling/accounting.rs

ax_task/sched/system/task_system/scheduling/
accounting.rs
