rusty_rtos_kernel_core/kernel.rs
1//! The scheduler: `tasks.c` and `queue.c` as a state machine over indices.
2//!
3//! Every public function here is a FreeRTOS API function, and its body
4//! follows the C body statement for statement — including **where the
5//! critical sections open and close**, because on the sim that is where
6//! time passes (`ORACLES.md`, sim contract v1, rule 3), and including where
7//! the `trace*` macros fire, because the order of those lines is the gate.
8//!
9//! The subtlety that costs a diff if you miss it: FreeRTOS yields from
10//! *inside* the critical section in `vTaskResume`, `vTaskPrioritySet`,
11//! `xTaskResumeAll` and the queue calls, and from *outside* it in
12//! `vTaskSuspend` and `vTaskDelay`. Since `portYIELD()` on the Posix port
13//! is itself a critical section, an inside yield nests (and counts one
14//! outermost exit) where an outside yield counts two. Each site below says
15//! which it is.
16//!
17//! Where the C has a `configASSERT`, this has an [`Error`]: the C kernel
18//! stops the world on a bad handle, and the whole point of the remake is
19//! that we do not.
20
21use core::marker::PhantomData;
22
23use rusty_rtos_core::arena::Arena;
24use rusty_rtos_core::config::Config;
25use rusty_rtos_core::error::{Error, Result};
26use rusty_rtos_core::handle::{
27 EventGroup as EventGroupKind, EventGroupHandle, Queue as QueueKind, QueueHandle,
28 StreamBuffer as StreamKind, Task as TaskKind, TaskHandle, Timer as TimerKind, TimerHandle,
29};
30use rusty_rtos_core::hooks::TickHook;
31use rusty_rtos_core::isr::Woken;
32use rusty_rtos_core::list::{ItemId, ListId, Lists};
33use rusty_rtos_core::port::Port;
34use rusty_rtos_core::priority::Priority;
35use rusty_rtos_core::tick::TickWidth;
36use rusty_rtos_core::trace::{Event, Trace};
37
38use crate::events::EventGroup;
39use crate::name::Name;
40use crate::queue::Queue;
41use crate::queue::Wait;
42use crate::stream::StreamBuffer;
43use crate::timer::{MAX_TIMER_COMMANDS, Message, Timer};
44use crate::{OVERHEAD_LISTS, items_for, lists_for};
45
46/// A trace line owed by a task that was switched out before it could
47/// emit one. Only the queue failure paths can owe one: they are the only
48/// places FreeRTOS traces *after* leaving a critical section.
49/// Why `vTaskSwitchContext` could not choose a task to run.
50///
51/// # Law 3: no silent failures
52///
53/// Every variant here was, until 2026-09-11, a bare `return` — the scheduler
54/// declining to switch and saying nothing. Three of them are impossible in a
55/// healthy kernel and the fourth is what the C asserts on
56/// (`configASSERT( uxTopPriority )`), so reaching any of them means an
57/// invariant has already broken somewhere else.
58///
59/// They cost a multi-flash hardware hunt on the day the Xtensa port first
60/// ran a stacked task: every ready list had emptied, the kernel kept
61/// quietly running whoever was current, and nothing said so. A counter and
62/// a reason turn that into one printed line.
63#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
64pub enum Stall {
65 /// The scheduler chose normally.
66 #[default]
67 None,
68 /// No ready task at ANY priority. The idle task keeps priority 0
69 /// non-empty, so reaching this means the idle task is gone — which is
70 /// exactly what `configASSERT( uxTopPriority )` fires on.
71 NoReadyTask,
72 /// A ready list refused to answer at all.
73 ListError,
74 /// The chosen priority's list had nothing to rotate to, having just
75 /// reported itself non-empty.
76 EmptyRotation,
77 /// The chosen list item named no live task.
78 UnknownTask,
79}
80
81#[derive(Debug, Clone, Copy, PartialEq, Eq)]
82pub(crate) enum OwedTrace {
83 /// Nothing owed.
84 None,
85 /// `traceQUEUE_SEND_FAILED`.
86 SendFailed(QueueHandle),
87 /// `traceQUEUE_RECEIVE_FAILED`.
88 ReceiveFailed(QueueHandle),
89 /// `traceTIMER_COMMAND_SEND`, which the C runs on the line *after*
90 /// `xQueueSendToBack` — so a send that made the daemon ready traces
91 /// only once this task has the CPU back.
92 TimerCommandSend {
93 timer: TimerHandle,
94 name: Name,
95 command: i32,
96 value: u64,
97 },
98 /// `traceEVENT_GROUP_WAIT_BITS_END`, which the C runs after the
99 /// `xTaskResumeAll` that ends the wait — so a resume that switched away
100 /// traces once this task has the CPU back.
101 EventGroupWaitBitsEnd {
102 group: EventGroupHandle,
103 bits: u32,
104 timed_out: bool,
105 },
106 /// `prvAddNewTaskToReadyList`, the second half of `xTaskCreate`.
107 ///
108 /// A create costs three outermost exits before it reaches this — two
109 /// `pvPortMalloc`s and the port's `pthread_create` section — and a tick
110 /// can land on any of them. The C is a real thread, so it simply stops
111 /// there and the new task is not on a ready list until the creator runs
112 /// again. This is that pause, and it is the only owed item that is not
113 /// purely a trace line: it carries the ready-list insertion too.
114 AddNewTaskToReadyList { task: TaskHandle, priority: u8 },
115}
116
117/// `eTaskState`: what a task is doing, derived from the list it is in
118/// exactly as `eTaskGetState` derives it.
119#[derive(Debug, Clone, Copy, PartialEq, Eq)]
120pub enum TaskState {
121 /// The task `current` names.
122 Running,
123 /// On a ready list.
124 Ready,
125 /// On a delayed list, or on an event list with a timeout.
126 Blocked,
127 /// On the suspended list with no event pending.
128 Suspended,
129 /// The handle names no live task.
130 Deleted,
131}
132
133/// One task control block: the C `TCB_t` minus everything that is a
134/// pointer. No stack, no TLS, no `pxTopOfStack`.
135#[derive(Debug, Clone, Copy)]
136pub(crate) struct Tcb {
137 name: Name,
138 /// `uxPriority`, the possibly-inherited one the scheduler sorts by.
139 priority: u8,
140 /// `uxBasePriority`, what the task asked for.
141 base_priority: u8,
142 /// `uxMutexesHeld`: how many mutexes this task holds, which is what
143 /// decides whether giving one back may drop an inherited priority.
144 mutexes_held: u32,
145 /// The locals a blocking queue call keeps on the C stack across a
146 /// block. This kernel has no stack, so they live here — see
147 /// [`crate::queue`].
148 wait: WaitFrame,
149 /// `ulNotifiedValue[]`.
150 notified: [u32; MAX_NOTIFICATION_ENTRIES],
151 /// `ucNotifyState[]`.
152 notify_state: [NotifyState; MAX_NOTIFICATION_ENTRIES],
153 /// Whether a stream-buffer call this task made was preempted at the
154 /// critical-section exit that samples the buffer, and must resume
155 /// *after* that exit rather than repeat it.
156 stream_resume: bool,
157 /// The stack local that exit produced — `xBytesAvailable` on a receive,
158 /// `xSpace` on a send — kept because the frame it belonged to is gone.
159 stream_local: usize,
160 /// Whether this task has already blocked inside the notification wait
161 /// it is in. The C blocks inside `xTaskGenericNotifyWait`; a kernel
162 /// with no stacks returns `Blocked` and is called again, so it needs to
163 /// know the second call is a resumption rather than a fresh wait.
164 notify_blocked: bool,
165 /// The same, for the event-group wait this task is inside. Those two
166 /// functions have no retry loop — the C blocks once and then reads the
167 /// event item value — so the second call has to know it is the far side
168 /// of the switch and not a fresh wait.
169 event_blocked: bool,
170}
171
172/// How many notification slots a task has room for.
173///
174/// The C sizes its arrays from `configTASK_NOTIFICATION_ARRAY_ENTRIES`
175/// directly. A `Config` is a type here and stable Rust cannot size an array
176/// from one of its associated consts, so the arrays are the largest a
177/// configuration may ask for and [`Config::validate`] refuses more. The
178/// cost is a few bytes per task per unused slot; the alternative was a
179/// tenth const generic on every kernel type.
180pub const MAX_NOTIFICATION_ENTRIES: usize = 4;
181
182/// `ucNotifyState[]`: what a task's notification slot is doing.
183#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
184pub enum NotifyState {
185 /// `taskNOT_WAITING_NOTIFICATION`.
186 #[default]
187 NotWaiting,
188 /// `taskWAITING_NOTIFICATION`.
189 Waiting,
190 /// `taskNOTIFICATION_RECEIVED`.
191 Received,
192}
193
194/// `eNotifyAction`: what a notification does to the value already there.
195#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
196pub enum NotifyAction {
197 /// `eNoAction`: wake the task, leave the value alone.
198 #[default]
199 None,
200 /// `eSetBits`.
201 SetBits,
202 /// `eIncrement`.
203 Increment,
204 /// `eSetValueWithOverwrite`.
205 Overwrite,
206 /// `eSetValueWithoutOverwrite`: fails if one is already pending.
207 NoOverwrite,
208}
209
210/// What `xQueueGenericSend` and friends keep between passes of their
211/// `for(;;)`: the queue being waited on, the ticks left, and the timeout
212/// bookkeeping (`TimeOut_t`).
213#[derive(Debug, Clone, Copy, Default)]
214struct WaitFrame {
215 /// The queue this task is part-way through a blocking call on;
216 /// `TaskHandle::NULL`-equivalent when there is none.
217 queue: QueueHandle,
218 /// `xTicksToWait`, decremented by each `xTaskCheckForTimeOut`.
219 ticks: u64,
220 /// `xTimeOut.xTimeOnEntering`.
221 entering: u64,
222 /// `xTimeOut.xOverflowCount`.
223 overflows: u64,
224 /// `xEntryTimeSet`.
225 entry_set: bool,
226 /// `xInheritanceOccurred`.
227 inherited: bool,
228}
229
230/// What [`Kernel::start_scheduler`] created, so a runner can attach bodies.
231#[derive(Debug, Clone, Copy)]
232pub struct StartHandles {
233 /// The idle task.
234 pub idle: TaskHandle,
235 /// The timer service task.
236 pub timer: TaskHandle,
237 /// The timer command queue.
238 pub timer_queue: QueueHandle,
239}
240
241/// The scheduler.
242///
243/// `TASKS`, `ITEMS`, `LISTS`, `QUEUES`, `SLOTS`, `BUFFERS` and `BYTES` are
244/// the geometry: how many tasks, list items, lists, queues, queue item
245/// slots, stream buffers, and stream-buffer bytes this kernel has room
246/// for. See the crate docs and [`items_for`] / [`lists_for`].
247/// [`Kernel::new`] refuses a geometry that does not add up.
248pub struct Kernel<
249 C: Config,
250 P: Port,
251 T: Trace,
252 H,
253 const TASKS: usize,
254 const ITEMS: usize,
255 const LISTS: usize,
256 const QUEUES: usize,
257 const SLOTS: usize,
258 const BUFFERS: usize,
259 const BYTES: usize,
260 const TIMERS: usize,
261 const GROUPS: usize,
262> {
263 pub(crate) port: P,
264 pub(crate) trace: T,
265 pub(crate) tcbs: Arena<TaskKind, Tcb, TASKS>,
266 pub(crate) queues: Arena<QueueKind, Queue, QUEUES>,
267 pub(crate) lists: Lists<ITEMS, LISTS>,
268 pub(crate) slots: [u64; SLOTS],
269 pub(crate) buffers: Arena<StreamKind, StreamBuffer, BUFFERS>,
270 pub(crate) timers: Arena<TimerKind, Timer, TIMERS>,
271 pub(crate) groups: Arena<EventGroupKind, EventGroup, GROUPS>,
272 /// The ring of `DaemonTaskMessage_t`s the timer queue carries indices
273 /// into. It is exactly as long as the queue, so a message can only be
274 /// overwritten once the queue has already refused to hold its index.
275 pub(crate) timer_messages: [Message; MAX_TIMER_COMMANDS],
276 pub(crate) timer_message_next: usize,
277 /// Which of the two timer lists is `pxCurrentTimerList` right now.
278 pub(crate) timers_swapped: bool,
279 /// `xLastTime` in `prvSampleTimeNow`.
280 pub(crate) timer_last_time: u64,
281 /// `xTimerQueue`.
282 pub(crate) timer_queue: QueueHandle,
283 pub(crate) bytes: [u8; BYTES],
284 pub(crate) bytes_used: usize,
285 /// Blocks of the byte arena that a deleted stream buffer gave back,
286 /// as `(base, length)`, kept sorted and coalesced.
287 ///
288 /// This is the one allocator in the kernel, and it exists because the
289 /// C's stream buffers are heap objects: `MessageBufferDemo`'s echo
290 /// server creates one and deletes it again on every loop, and a bump
291 /// allocator would run out in a few hundred ticks. At most `BUFFERS`
292 /// buffers can be alive, so at most `BUFFERS` holes can exist between
293 /// them, which is why the list is that long and cannot overflow.
294 pub(crate) free_blocks: [(usize, usize); BUFFERS],
295 pub(crate) free_count: usize,
296 pub(crate) slots_used: usize,
297 /// `pxCurrentTCB`.
298 pub(crate) current: TaskHandle,
299 /// `uxTopReadyPriority`.
300 top_ready_priority: u8,
301 /// `xTickCount`, masked to the configuration's tick width.
302 pub(crate) tick: u64,
303 /// `xPendedTicks`.
304 pended_ticks: u64,
305 /// `uxSchedulerSuspended`.
306 suspended_depth: u32,
307 /// `xSchedulerRunning`.
308 running: bool,
309 /// `xNextTaskUnblockTime`.
310 next_unblock_time: u64,
311 /// `xYieldPendings[0]`.
312 yield_pending: bool,
313 /// `uxCurrentNumberOfTasks`.
314 task_count: usize,
315 /// How many times the scheduler could not choose a task, and why the
316 /// FIRST time. See [`Stall`].
317 stalls: u32,
318 first_stall: Stall,
319 /// A task that deleted ITSELF and whose slot the idle task has not
320 /// reclaimed yet — the C's `xTasksWaitingTermination`, which only ever
321 /// holds the running task, because any other task is freed on the spot.
322 ///
323 /// One slot is enough, and that is a property rather than a guess: a
324 /// self-deleting task is off every ready list and yields immediately, so
325 /// a second self-delete cannot happen until a switch has occurred, and
326 /// the switch is where this is reaped.
327 awaiting_reap: Option<TaskHandle>,
328 /// Which of the two delayed lists is `pxDelayedTaskList` right now.
329 delayed_swapped: bool,
330 /// `xNumOfOverflows`.
331 overflows: u64,
332 /// Which tasks have been switched in at least once. A task's first
333 /// switch-in is where a real port hands it a fresh stack, so the
334 /// outgoing task's open critical sections are abandoned there
335 /// (`Port::reset_nesting_for_first_start`).
336 started: [bool; TASKS],
337 /// A task that was switched out *in the middle of a kernel call* still
338 /// owes the tail of that call — in practice always the `portYIELD()`
339 /// that `vTaskDelay` and `vTaskSuspend` make after their critical
340 /// sections. On a real port the tail simply sits on the task's frozen
341 /// stack; here the stack is a program counter, so the debt is recorded
342 /// and paid by [`Kernel::resume_pending`] when the task runs again.
343 owes_yield: [bool; TASKS],
344 /// A trace line a task owes from a call it was switched out of.
345 ///
346 /// `xQueueReceive`'s failure path is `taskEXIT_CRITICAL();
347 /// traceQUEUE_RECEIVE_FAILED( pxQueue ); return errQUEUE_EMPTY;` — and
348 /// if the tick that the exit released switches the task away, those two
349 /// lines sit on a stack that is not running. The C emits the trace when
350 /// the task resumes, so this kernel does too.
351 owed_trace: [OwedTrace; TASKS],
352 /// How many outermost critical-section exits a task owes the clock.
353 ///
354 /// This is `uxSavedCriticalNesting` in the Posix port's
355 /// `prvSwitchThread` and then some. A thread stops at the switch; a
356 /// stackless call does not, so the abandoned frame runs on — closing
357 /// the sections it had open and, on some paths, opening one more. The
358 /// port tallies every exit that frame makes instead of counting it as
359 /// sim time, and the tally is paid here when the task runs again. That
360 /// is what puts a sim tick where the C one lands.
361 owed_exits: [u32; TASKS],
362 /// The task whose abandoned frame is running right now, if any.
363 unwinding: Option<TaskHandle>,
364 /// `vApplicationTickHook`, held by value so it can borrow the kernel.
365 pub(crate) tick_hook: H,
366 /// `ucDelayAborted`: the task was pulled out of the Blocked state by
367 /// [`Kernel::abort_delay`] rather than by its own block time running
368 /// out, so it must not re-evaluate that block time and block again.
369 delay_aborted: [bool; TASKS],
370 _config: PhantomData<C>,
371}
372
373impl<
374 C: Config,
375 P: Port,
376 T: Trace,
377 H,
378 const TASKS: usize,
379 const ITEMS: usize,
380 const LISTS: usize,
381 const QUEUES: usize,
382 const SLOTS: usize,
383 const BUFFERS: usize,
384 const BYTES: usize,
385 const TIMERS: usize,
386 const GROUPS: usize,
387> Kernel<C, P, T, H, TASKS, ITEMS, LISTS, QUEUES, SLOTS, BUFFERS, BYTES, TIMERS, GROUPS>
388where
389 H: TickHook<Self>,
390{
391 /// `portMAX_DELAY` at this configuration's tick width.
392 pub const MAX_DELAY: u64 = <C::Tick as TickWidth>::MAX;
393
394 // ------------------------------------------------------------ setup --
395
396 /// A kernel with no tasks, ready for [`Kernel::create_task`].
397 ///
398 /// # Errors
399 /// [`Error::InvalidArgument`] if the configuration is invalid
400 /// ([`Config::validate`]), if the name capacity cannot hold
401 /// `MAX_TASK_NAME_LEN`, or if the const geometry does not match the
402 /// configuration: `ITEMS` must be [`items_for`]`(TASKS)` and `LISTS`
403 /// must be [`lists_for`]`(MAX_PRIORITIES, QUEUES)`.
404 pub fn new(port: P, trace: T) -> Result<Self>
405 where
406 H: Default,
407 {
408 Self::with_tick_hook(port, trace, H::default())
409 }
410
411 /// [`Kernel::new`] with `vApplicationTickHook` installed.
412 ///
413 /// # Errors
414 /// As [`Kernel::new`].
415 pub fn with_tick_hook(port: P, trace: T, tick_hook: H) -> Result<Self> {
416 C::validate()?;
417 if ITEMS != items_for(TASKS, TIMERS)
418 || LISTS != lists_for(C::MAX_PRIORITIES, QUEUES, GROUPS)
419 || TASKS == 0
420 || C::MAX_TASK_NAME_LEN > crate::NAME_CAPACITY
421 {
422 return Err(Error::InvalidArgument);
423 }
424 Ok(Self {
425 port,
426 trace,
427 tcbs: Arena::new(),
428 queues: Arena::new(),
429 lists: Lists::new(),
430 slots: [0; SLOTS],
431 slots_used: 0,
432 buffers: Arena::new(),
433 timers: Arena::new(),
434 groups: Arena::new(),
435 timer_messages: [Message::default(); MAX_TIMER_COMMANDS],
436 timer_message_next: 0,
437 timers_swapped: false,
438 timer_last_time: 0,
439 timer_queue: QueueHandle::NULL,
440 bytes: [0; BYTES],
441 bytes_used: 0,
442 free_blocks: [(0, 0); BUFFERS],
443 free_count: 0,
444 current: TaskHandle::NULL,
445 top_ready_priority: 0,
446 tick: C::INITIAL_TICK_COUNT,
447 pended_ticks: 0,
448 suspended_depth: 0,
449 running: false,
450 next_unblock_time: Self::MAX_DELAY,
451 yield_pending: false,
452 task_count: 0,
453 stalls: 0,
454 first_stall: Stall::None,
455 awaiting_reap: None,
456 delayed_swapped: false,
457 overflows: 0,
458 started: [false; TASKS],
459 owes_yield: [false; TASKS],
460 owed_exits: [0; TASKS],
461 unwinding: None,
462 tick_hook,
463 delay_aborted: [false; TASKS],
464 owed_trace: [OwedTrace::None; TASKS],
465 _config: PhantomData,
466 })
467 }
468
469 /// The port, for a runner that needs its counters.
470 pub const fn port(&self) -> &P {
471 &self.port
472 }
473
474 /// The trace sink.
475 pub const fn trace(&self) -> &T {
476 &self.trace
477 }
478
479 /// The trace sink, mutably.
480 pub const fn trace_mut(&mut self) -> &mut T {
481 &mut self.trace
482 }
483
484 /// Take the trace sink back when the run is over, so a deliverable can
485 /// flush whatever it was writing to.
486 pub fn into_trace(self) -> T {
487 self.trace
488 }
489
490 /// `xNumOfOverflows`, for a scenario that reports counters.
491 #[must_use]
492 pub const fn overflows(&self) -> u64 {
493 self.overflows
494 }
495
496 // ------------------------------------------------------ list indices --
497
498 // `ListId` is a `u8` and `ItemId` a `u16` (the core's `list`): the
499 // whole scheduler addresses lists and items by small integers, which is
500 // what keeps a TCB free of pointers.
501
502 pub(crate) const fn ready_list(priority: u8) -> ListId {
503 priority
504 }
505
506 const fn delayed_list(&self) -> ListId {
507 if self.delayed_swapped {
508 C::MAX_PRIORITIES.saturating_add(1)
509 } else {
510 C::MAX_PRIORITIES
511 }
512 }
513
514 const fn overflow_delayed_list(&self) -> ListId {
515 if self.delayed_swapped {
516 C::MAX_PRIORITIES
517 } else {
518 C::MAX_PRIORITIES.saturating_add(1)
519 }
520 }
521
522 const fn pending_ready_list() -> ListId {
523 C::MAX_PRIORITIES.saturating_add(2)
524 }
525
526 const fn suspended_list() -> ListId {
527 C::MAX_PRIORITIES.saturating_add(3)
528 }
529
530 /// `xTasksWaitingToSend` of a queue.
531 pub(crate) fn queue_send_list(queue: QueueHandle) -> ListId {
532 let offset = u8::try_from(queue.index().saturating_mul(2)).unwrap_or(u8::MAX);
533 C::MAX_PRIORITIES
534 .saturating_add(u8::try_from(OVERHEAD_LISTS).unwrap_or(u8::MAX))
535 .saturating_add(offset)
536 }
537
538 /// `xTasksWaitingToReceive` of a queue.
539 pub(crate) fn queue_receive_list(queue: QueueHandle) -> ListId {
540 Self::queue_send_list(queue).saturating_add(1)
541 }
542
543 /// A task's `xStateListItem`: the task's own arena index.
544 pub(crate) const fn state_item(task: TaskHandle) -> ItemId {
545 task.index()
546 }
547
548 /// A task's `xEventListItem`: `TASKS` above its state item, so the two
549 /// never collide and either maps back to its task by arithmetic alone.
550 pub(crate) fn event_item(task: TaskHandle) -> ItemId {
551 Self::task_item_base().saturating_add(task.index())
552 }
553
554 fn task_item_base() -> ItemId {
555 u16::try_from(TASKS).unwrap_or(u16::MAX)
556 }
557
558 fn task_of_state_item(&self, item: ItemId) -> Result<TaskHandle> {
559 self.tcbs.handle_at(item).ok_or(Error::Gone)
560 }
561
562 fn task_of_event_item(&self, item: ItemId) -> Result<TaskHandle> {
563 let index = item
564 .checked_sub(Self::task_item_base())
565 .ok_or(Error::InvalidArgument)?;
566 self.tcbs.handle_at(index).ok_or(Error::Gone)
567 }
568
569 // ----------------------------------------------------------- getters --
570
571 /// `pxCurrentTCB`.
572 #[must_use]
573 pub const fn current(&self) -> TaskHandle {
574 self.current
575 }
576
577 /// `xTaskGetTickCount`.
578 #[must_use]
579 pub const fn tick_count(&self) -> u64 {
580 self.tick
581 }
582
583 /// `xSchedulerRunning`.
584 #[must_use]
585 pub const fn is_running(&self) -> bool {
586 self.running
587 }
588
589 /// `uxCurrentNumberOfTasks`.
590 #[must_use]
591 pub const fn task_count(&self) -> usize {
592 self.task_count
593 }
594
595 /// `uxSchedulerSuspended`, for a stepper reporting why a tick pended.
596 #[must_use]
597 pub const fn scheduler_suspended(&self) -> u32 {
598 self.suspended_depth
599 }
600
601 /// `xPendedTicks`, likewise.
602 #[must_use]
603 pub const fn pended_ticks(&self) -> u64 {
604 self.pended_ticks
605 }
606
607 /// The item ids in `priority`'s ready list, in order, into `out`.
608 ///
609 /// Returns how many were written. The companion to
610 /// [`Kernel::ready_cursor`]: when a ready task is never chosen, the
611 /// cursor says whether the rotation moved and this says what it was
612 /// moving THROUGH.
613 pub fn ready_items(&self, priority: u8, out: &mut [u16]) -> usize {
614 let mut n = 0;
615 for item in self.lists.iter(Self::ready_list(priority)) {
616 let Some(slot) = out.get_mut(n) else {
617 break;
618 };
619 *slot = item;
620 n = n.saturating_add(1);
621 }
622 n
623 }
624
625 /// Where the round-robin cursor sits in `priority`'s ready list.
626 ///
627 /// A diagnostic, paired with [`Kernel::ready_len`]: a ready task that
628 /// is never chosen is either not in the list the scheduler looks at,
629 /// or the cursor is not moving. These two answer both halves.
630 #[must_use]
631 pub fn ready_cursor(&self, priority: u8) -> u16 {
632 self.lists
633 .cursor_of(Self::ready_list(priority))
634 .unwrap_or(0)
635 }
636
637 /// The live handle in arena slot `index`, if that slot holds a task.
638 ///
639 /// The generation is the point: a bare index names a SLOT, and a slot
640 /// outlives the tasks that occupy it, so an index is not a task. This
641 /// answers the handle, which is.
642 ///
643 /// It exists so a diagnostic can walk every task without knowing any
644 /// of their names -- `task_get_handle` needs the name it is looking
645 /// for, and a crash report is exactly the case where you do not have
646 /// one.
647 #[must_use]
648 pub fn task_at(&self, index: usize) -> Option<TaskHandle> {
649 let index = u16::try_from(index).ok()?;
650 self.tcbs.handle_at(index)
651 }
652
653 /// A task's name.
654 ///
655 /// # Errors
656 /// [`Error::Gone`] for a stale or null handle.
657 pub fn name_of(&self, task: TaskHandle) -> Result<Name> {
658 self.tcbs.resolve(task).map(|t| t.name)
659 }
660
661 /// `uxTaskPriorityGet`. `None` means the current task, as in C.
662 ///
663 /// # Errors
664 /// [`Error::Gone`] for a stale handle.
665 pub fn priority_of(&self, task: Option<TaskHandle>) -> Result<u8> {
666 let h = task.unwrap_or(self.current);
667 self.tcbs.resolve(h).map(|t| t.priority)
668 }
669
670 pub(crate) fn current_priority(&self) -> u8 {
671 self.priority_of(None).unwrap_or(0)
672 }
673
674 /// `uxTaskPriorityGet`: the C API, which takes a critical section.
675 ///
676 /// The distinction matters: on the sim a critical-section exit is where
677 /// time passes, so reading a priority through the API costs a tick
678 /// sixteen calls later, exactly as it does in C
679 /// (`portBASE_TYPE_ENTER_CRITICAL` is `taskENTER_CRITICAL` on the Posix
680 /// port). The kernel's own reads use the field directly, as C's do.
681 ///
682 /// # Errors
683 /// [`Error::Gone`] for a stale handle.
684 pub fn task_priority_get(&mut self, task: Option<TaskHandle>) -> Result<u8> {
685 self.enter_critical();
686 let result = self.priority_of(task);
687 self.exit_critical();
688 result
689 }
690
691 /// `eTaskGetState`: the C API, which takes a critical section for any
692 /// task but the running one.
693 ///
694 /// # Errors
695 /// A list error surfaces as itself.
696 pub fn task_state_get(&mut self, task: TaskHandle) -> Result<TaskState> {
697 if task == self.current {
698 return Ok(TaskState::Running);
699 }
700 self.enter_critical();
701 let result = self.state_of(task);
702 self.exit_critical();
703 result
704 }
705
706 /// `eTaskGetState`, derived from the list the task is in.
707 ///
708 /// # Errors
709 /// A list error surfaces as itself.
710 pub fn state_of(&self, task: TaskHandle) -> Result<TaskState> {
711 if !self.tcbs.contains(task) {
712 return Ok(TaskState::Deleted);
713 }
714 if task == self.current {
715 return Ok(TaskState::Running);
716 }
717 let Some(list) = self.lists.container(Self::state_item(task))? else {
718 return Ok(TaskState::Deleted);
719 };
720 if list == Self::suspended_list() {
721 if self.lists.container(Self::event_item(task))?.is_some() {
722 return Ok(TaskState::Blocked);
723 }
724 // A task blocked indefinitely on a notification is in the
725 // suspended list and on no event list at all, so the C scans
726 // the notification array before it calls such a task suspended.
727 // Miss this and a stream buffer's reader looks suspended to
728 // `eTaskGetState` while it is plainly waiting.
729 let waiting_notification = self
730 .tcbs
731 .resolve(task)
732 .map(|t| {
733 t.notify_state
734 .iter()
735 .take(C::NOTIFICATION_ARRAY_ENTRIES)
736 .any(|s| *s == NotifyState::Waiting)
737 })
738 .unwrap_or(false);
739 return Ok(if waiting_notification {
740 TaskState::Blocked
741 } else {
742 TaskState::Suspended
743 });
744 }
745 if list == self.delayed_list() || list == self.overflow_delayed_list() {
746 return Ok(TaskState::Blocked);
747 }
748 Ok(TaskState::Ready)
749 }
750
751 /// How many tasks sit on a priority's ready list — what the idle task
752 /// checks before yielding (`configIDLE_SHOULD_YIELD`).
753 ///
754 /// # Errors
755 /// [`Error::InvalidPriority`] for a priority outside the configuration.
756 pub fn ready_len(&self, priority: u8) -> Result<usize> {
757 if priority >= C::MAX_PRIORITIES {
758 return Err(Error::InvalidPriority);
759 }
760 self.lists.len(Self::ready_list(priority))
761 }
762
763 /// Record that the scheduler could not choose.
764 ///
765 /// Counts every occurrence and keeps the FIRST reason: the first is the
766 /// diagnosis and the rest are consequences.
767 fn note_stall(&mut self, why: Stall) {
768 self.stalls = self.stalls.saturating_add(1);
769 if self.first_stall == Stall::None {
770 self.first_stall = why;
771 }
772 }
773
774 /// How many times the scheduler could not choose a task.
775 ///
776 /// **Non-zero means an invariant broke.** A healthy kernel never stalls:
777 /// the idle task is always ready. A firmware that prints this alongside
778 /// its own counters turns a hang into a sentence.
779 #[must_use]
780 pub const fn stalls(&self) -> u32 {
781 self.stalls
782 }
783
784 /// Why the scheduler first failed to choose. [`Stall::None`] if it never
785 /// has.
786 #[must_use]
787 pub const fn first_stall(&self) -> Stall {
788 self.first_stall
789 }
790
791 // ------------------------------------------------------------ tracing --
792
793 pub(crate) fn trace_task<F>(&mut self, task: TaskHandle, make: F)
794 where
795 F: for<'a> FnOnce(TaskHandle, &'a str) -> Event<'a>,
796 {
797 let name = self.tcbs.get(task).map(|t| t.name).unwrap_or_default();
798 let tick = self.tick;
799 self.trace.note_exits(self.port.exits());
800 self.trace.event(tick, make(task, name.as_str()));
801 }
802
803 pub(crate) fn priority_value(raw: u8) -> Priority {
804 Priority::new(raw, C::MAX_PRIORITIES).unwrap_or(Priority::IDLE)
805 }
806
807 // --------------------------------------------------- critical section --
808
809 /// What `pvPortMalloc` costs in sim time on a heap-backed C kernel:
810 /// `vTaskSuspendAll()` around the allocation, then `xTaskResumeAll()`,
811 /// whose critical section is one outermost exit.
812 ///
813 /// This kernel allocates nothing, so the call is the whole of it. On a
814 /// configuration with [`Config::DYNAMIC_ALLOCATION`] off it compiles
815 /// away to nothing at all.
816 pub(crate) fn account_for_allocation(&mut self) {
817 if C::DYNAMIC_ALLOCATION {
818 self.suspend_all();
819 let _ = self.resume_all();
820 }
821 }
822
823 /// `taskENTER_CRITICAL()`.
824 pub fn enter_critical(&mut self) {
825 self.port.enter_critical();
826 }
827
828 /// `taskEXIT_CRITICAL()`.
829 ///
830 /// On the sim this is where time passes: the port raises a tick on
831 /// every sixteenth outermost exit and the kernel takes it here, which
832 /// is exactly where the C kernel's `SIGALRM` handler would have run.
833 pub fn exit_critical(&mut self) {
834 self.port.exit_critical();
835 if self.port.take_pending_tick() {
836 self.tick_from_isr();
837 }
838 }
839
840 /// `vPortSystemTickHandler`: the tick, from interrupt context.
841 ///
842 /// A silicon port calls this from its timer interrupt. The sim reaches
843 /// it from [`Kernel::exit_critical`] and from the idle hook.
844 pub fn tick_from_isr(&mut self) {
845 self.port.set_in_tick_entry(true);
846 self.port.count_tick();
847 // The C handler bumps `uxCriticalNesting` by hand and drops it the
848 // same way — a raw `--`, never `vPortExitCritical` — so the bump is
849 // invisible to the exit count and to anything that unwinds later.
850 // Modelling it as a real critical section would invent an exit at
851 // every tick that caused a switch.
852 if self.increment_tick() {
853 self.switch_context();
854 }
855 self.port.set_in_tick_entry(false);
856 }
857
858 /// The idle hook's tick, `vPortKairosTick()` on the C side: a critical
859 /// section around one unconditional tick. The enclosing section counts
860 /// toward rule 3 exactly as the C one does.
861 pub fn idle_hook_tick(&mut self) {
862 self.enter_critical();
863 self.tick_from_isr();
864 self.exit_critical();
865 }
866
867 /// `portYIELD()` as the Posix port spells it: a critical section around
868 /// the context switch, so a tick raised during it lands on the way out.
869 pub(crate) fn port_yield(&mut self) {
870 self.enter_critical();
871 self.port.count_yield();
872 if P::COMMITS_SWITCH {
873 // A STACKED port. Raise its switching exception and leave
874 // `current` where it is: the exception will call
875 // `switch_context` itself, so the decision and the register
876 // swap happen together and nothing runs in between as a task
877 // the kernel has already moved on from.
878 //
879 // The exception cannot fire yet -- `exit_critical` below is
880 // what unmasks -- so the kernel is consistent at the moment it
881 // is taken.
882 self.port.yield_now();
883 } else {
884 // A STACKLESS kernel: moving `current` IS the switch, because
885 // no task owns a stack. This is the path every corpus
886 // architecture takes, and it is unchanged.
887 self.switch_context();
888 }
889 self.exit_critical();
890 }
891
892 /// `taskYIELD()` from a task body (the idle task's yield).
893 pub fn task_yield(&mut self) {
894 self.port_yield();
895 }
896
897 // -------------------------------------------------------- task create --
898
899 /// `xTaskCreate`.
900 ///
901 /// # Errors
902 /// [`Error::Full`] when the task arena is full (the C
903 /// `errCOULD_NOT_ALLOCATE_REQUIRED_MEMORY`).
904 pub fn create_task(&mut self, name: &str, priority: u8) -> Result<TaskHandle> {
905 let caller = self.current;
906 // configASSERT( uxPriority < configMAX_PRIORITIES ), then C clamps.
907 let priority = priority.min(C::MAX_PRIORITIES.saturating_sub(1));
908 // `prvCreateTask` takes the stack and then the TCB from the heap
909 // before anything is initialised — `portSTACK_GROWTH` is negative on
910 // the oracle's port, which fixes that order — and `pvPortMallocStack`
911 // *is* `pvPortMalloc` with the MPU wrappers off. On heap_3 each one
912 // suspends the scheduler, so a create costs two outermost exits.
913 //
914 // Every scenario before `death` created its tasks BEFORE
915 // `vTaskStartScheduler`, and the sim port counts no exits there (the
916 // C patch counts only on a FreeRTOS thread), so this cost was
917 // invisible to the whole corpus. `death` is the first scenario that
918 // creates a task with the scheduler running, and the first that can
919 // tell the difference.
920 self.account_for_allocation();
921 self.account_for_allocation();
922 let tcb = Tcb {
923 name: Name::new(name, C::MAX_TASK_NAME_LEN),
924 priority,
925 base_priority: priority,
926 mutexes_held: 0,
927 wait: WaitFrame::default(),
928 notified: [0; MAX_NOTIFICATION_ENTRIES],
929 notify_state: [NotifyState::NotWaiting; MAX_NOTIFICATION_ENTRIES],
930 notify_blocked: false,
931 event_blocked: false,
932 stream_resume: false,
933 stream_local: 0,
934 };
935 let handle = match self.tcbs.try_insert(tcb) {
936 Ok(h) => h,
937 Err(_) => {
938 let tick = self.tick;
939 self.trace.note_exits(self.port.exits());
940 self.trace.event(tick, Event::TaskCreateFailed);
941 return Err(Error::Full);
942 }
943 };
944 // The event item sorts by `configMAX_PRIORITIES - uxPriority`, so a
945 // higher-priority task waits nearer the head of an event list.
946 let event_value = u64::from(C::MAX_PRIORITIES.saturating_sub(priority));
947 self.lists
948 .set_value(Self::event_item(handle), event_value)?;
949 // `prvInitialiseNewTask` ends by calling `pxPortInitialiseStack`,
950 // which on the oracle's Posix port wraps `pthread_create` in a
951 // critical section — one more exit, and a PORT cost rather than a
952 // kernel one, which is why it has a flag of its own.
953 if C::PORT_STACK_INIT_CRITICAL {
954 self.enter_critical();
955 self.exit_critical();
956 }
957 // Any of the three exits above can release a tick that switches the
958 // creator out. In the C the creator is a thread and simply STOPS
959 // there: `prvAddNewTaskToReadyList` has not run, so the new task is
960 // on no ready list and `traceTASK_CREATE` has not fired. Running it
961 // now would put both events on the wrong side of the switch — which
962 // is precisely how `death` first diverged.
963 if self.current != caller {
964 if let Some(slot) = self.owed_trace.get_mut(usize::from(caller.index())) {
965 *slot = OwedTrace::AddNewTaskToReadyList {
966 task: handle,
967 priority,
968 };
969 }
970 return Ok(handle);
971 }
972 self.add_new_task_to_ready_list(handle, priority)?;
973 Ok(handle)
974 }
975
976 /// `prvAddNewTaskToReadyList`.
977 fn add_new_task_to_ready_list(&mut self, task: TaskHandle, priority: u8) -> Result<()> {
978 self.enter_critical();
979 {
980 self.task_count = self.task_count.saturating_add(1);
981 if self.current.is_null() {
982 self.current = task;
983 } else if !self.running {
984 // `<=`, so the last-created task of the highest priority is
985 // the one that runs first.
986 if self.current_priority() <= priority {
987 self.current = task;
988 }
989 }
990 self.trace_task(task, |task, name| Event::TaskCreate {
991 task,
992 name,
993 priority: Self::priority_value(priority),
994 });
995 self.add_task_to_ready_list(task)?;
996 }
997 self.exit_critical();
998 // taskYIELD_IF_USING_PREEMPTION(), outside the section.
999 if self.running && C::USE_PREEMPTION && self.current_priority() < priority {
1000 self.port_yield();
1001 }
1002 Ok(())
1003 }
1004
1005 /// `prvAddTaskToReadyList`.
1006 pub(crate) fn add_task_to_ready_list(&mut self, task: TaskHandle) -> Result<()> {
1007 let priority = self.tcbs.resolve(task)?.priority;
1008 self.trace_task(task, |task, name| Event::MovedTaskToReadyState {
1009 task,
1010 name,
1011 });
1012 // taskRECORD_READY_PRIORITY
1013 if priority > self.top_ready_priority {
1014 self.top_ready_priority = priority;
1015 }
1016 self.lists
1017 .insert_end(Self::ready_list(priority), Self::state_item(task))
1018 }
1019
1020 // ---------------------------------------------------- scheduler start --
1021
1022 /// `vTaskStartScheduler`, minus the part that never returns.
1023 ///
1024 /// Creates the idle task, the timer command queue and the timer service
1025 /// task, fires `TASK_SWITCHED_IN` for whichever task will run first and
1026 /// then `STARTING_SCHEDULER`, and returns the handles so the runner can
1027 /// attach their bodies.
1028 ///
1029 /// # Errors
1030 /// As [`Kernel::create_task`].
1031 pub fn start_scheduler(&mut self) -> Result<StartHandles> {
1032 let idle = self.create_task("IDLE", 0)?;
1033 // `xTimerCreateTimerTask` opens with `prvCheckForValidListAndQueue`,
1034 // which makes the command queue only if no `xTimerCreate` has made
1035 // it already.
1036 self.check_for_valid_list_and_queue()?;
1037 let timer_queue = self.timer_queue;
1038 let timer = self.create_task("Tmr Svc", C::TIMER_TASK_PRIORITY)?;
1039 self.next_unblock_time = Self::MAX_DELAY;
1040 self.running = true;
1041 self.tick = C::INITIAL_TICK_COUNT;
1042 self.port.scheduler_started();
1043 // `vPortStartFirstTask` runs the first task through the same
1044 // first-start path as every other.
1045 let current = self.current;
1046 self.note_first_start(current);
1047 self.trace_task(current, |task, name| Event::TaskSwitchedIn { task, name });
1048 let tick = self.tick;
1049 self.trace.note_exits(self.port.exits());
1050 self.trace.event(tick, Event::StartingScheduler);
1051 Ok(StartHandles {
1052 idle,
1053 timer,
1054 timer_queue,
1055 })
1056 }
1057
1058 // -------------------------------------------------------- the switch --
1059
1060 /// `vTaskSwitchContext`.
1061 pub fn switch_context(&mut self) {
1062 if self.suspended_depth != 0 {
1063 self.yield_pending = true;
1064 return;
1065 }
1066 self.yield_pending = false;
1067 let current = self.current;
1068 self.trace_task(current, |task, name| Event::TaskSwitchedOut { task, name });
1069 // taskSELECT_HIGHEST_PRIORITY_TASK
1070 let mut top = self.top_ready_priority;
1071 loop {
1072 match self.lists.is_empty(Self::ready_list(top)) {
1073 Ok(false) => break,
1074 Ok(true) => {
1075 if top == 0 {
1076 // The C `configASSERT( uxTopPriority )`. The idle
1077 // task keeps priority 0 non-empty, so reaching here
1078 // means it is gone. The kernel keeps the current
1079 // task rather than indexing out of range -- and now
1080 // SAYS so, because carrying on quietly is how this
1081 // costs an afternoon on a board.
1082 self.note_stall(Stall::NoReadyTask);
1083 return;
1084 }
1085 top = top.saturating_sub(1);
1086 }
1087 Err(_) => {
1088 self.note_stall(Stall::ListError);
1089 return;
1090 }
1091 }
1092 }
1093 let Ok(Some(item)) = self.lists.next_round_robin(Self::ready_list(top)) else {
1094 self.note_stall(Stall::EmptyRotation);
1095 return;
1096 };
1097 let Ok(next) = self.task_of_state_item(item) else {
1098 self.note_stall(Stall::UnknownTask);
1099 return;
1100 };
1101 self.top_ready_priority = top;
1102 if next != current {
1103 self.hand_over(current, next);
1104 }
1105 self.current = next;
1106 self.trace_task(next, |task, name| Event::TaskSwitchedIn { task, name });
1107 }
1108
1109 // ----------------------------------------------------------- the tick --
1110
1111 /// Pay whatever the running task still owes from a call it was
1112 /// preempted inside. The runner calls this before stepping a body;
1113 /// `true` means work was done and the current task may have changed
1114 /// again, so nothing else should be assumed.
1115 pub fn resume_pending(&mut self) -> bool {
1116 self.settle_unwind();
1117 let index = usize::from(self.current.index());
1118 // First the stack unwinds — the sections the task had open when it
1119 // was switched out, and whatever its abandoned frame opened after
1120 // that — and only then does the statement after the yield run.
1121 // Reversing the two moves every tick.
1122 let owed = self.owed_exits.get(index).copied().unwrap_or(0);
1123 if owed > 0 {
1124 if let Some(slot) = self.owed_exits.get_mut(index) {
1125 *slot = 0;
1126 }
1127 // One counted exit each: the tally is already in outermost
1128 // exits, so replaying it as nesting would lose the sections the
1129 // tail opened and closed on its own.
1130 for _ in 0..owed {
1131 self.enter_critical();
1132 self.exit_critical();
1133 // A replayed exit can release a tick that switches this
1134 // task straight back out. The rest of the loop is then the
1135 // tail of *this* frame, which the port tallies and the next
1136 // resume pays — so there is nothing to unwind by hand.
1137 }
1138 return true;
1139 }
1140 // Then the line the abandoned frame had not reached yet.
1141 match self.owed_trace.get(index).copied() {
1142 Some(OwedTrace::SendFailed(queue)) => {
1143 if let Some(slot) = self.owed_trace.get_mut(index) {
1144 *slot = OwedTrace::None;
1145 }
1146 let tick = self.tick;
1147 self.trace.note_exits(self.port.exits());
1148 self.trace
1149 .event(tick, Event::QueueSendFailed { queue, name: "" });
1150 return true;
1151 }
1152 Some(OwedTrace::ReceiveFailed(queue)) => {
1153 if let Some(slot) = self.owed_trace.get_mut(index) {
1154 *slot = OwedTrace::None;
1155 }
1156 let tick = self.tick;
1157 self.trace.note_exits(self.port.exits());
1158 self.trace
1159 .event(tick, Event::QueueReceiveFailed { queue, name: "" });
1160 return true;
1161 }
1162 Some(OwedTrace::EventGroupWaitBitsEnd {
1163 group,
1164 bits,
1165 timed_out,
1166 }) => {
1167 if let Some(slot) = self.owed_trace.get_mut(index) {
1168 *slot = OwedTrace::None;
1169 }
1170 let tick = self.tick;
1171 self.trace.note_exits(self.port.exits());
1172 self.trace.event(
1173 tick,
1174 Event::EventGroupWaitBitsEnd {
1175 group,
1176 bits,
1177 timed_out,
1178 },
1179 );
1180 return true;
1181 }
1182 Some(OwedTrace::TimerCommandSend {
1183 timer,
1184 name,
1185 command,
1186 value,
1187 }) => {
1188 if let Some(slot) = self.owed_trace.get_mut(index) {
1189 *slot = OwedTrace::None;
1190 }
1191 let tick = self.tick;
1192 self.trace.note_exits(self.port.exits());
1193 self.trace.event(
1194 tick,
1195 Event::TimerCommandSend {
1196 timer,
1197 name: name.as_str(),
1198 command,
1199 value,
1200 },
1201 );
1202 return true;
1203 }
1204 Some(OwedTrace::AddNewTaskToReadyList { task, priority }) => {
1205 if let Some(slot) = self.owed_trace.get_mut(index) {
1206 *slot = OwedTrace::None;
1207 }
1208 let _ = self.add_new_task_to_ready_list(task, priority);
1209 return true;
1210 }
1211 Some(OwedTrace::None) | None => {}
1212 }
1213 if self.owes_yield.get(index).copied() != Some(true) {
1214 return false;
1215 }
1216 if let Some(flag) = self.owes_yield.get_mut(index) {
1217 *flag = false;
1218 }
1219 self.port_yield();
1220 true
1221 }
1222
1223 /// Collect the tally of an abandoned frame that has finished running.
1224 ///
1225 /// The frame runs to its end before control comes back to the runner,
1226 /// so this is called once, at the top of [`Kernel::resume_pending`],
1227 /// which is the first thing the runner does.
1228 fn settle_unwind(&mut self) {
1229 let Some(task) = self.unwinding.take() else {
1230 return;
1231 };
1232 let owed = self.port.end_unwind();
1233 if let Some(slot) = self.owed_exits.get_mut(usize::from(task.index())) {
1234 // Accumulate: a replay that is itself interrupted leaves the
1235 // rest of its own loop as a tail, and that tail is more of the
1236 // same debt.
1237 *slot = slot.saturating_add(owed);
1238 }
1239 }
1240
1241 /// Emit a queue failure line now, or owe it if a tick switched the
1242 /// caller out of the call that was about to emit it.
1243 pub(crate) fn trace_failure_or_owe(&mut self, caller: TaskHandle, owed: OwedTrace) {
1244 if self.current == caller {
1245 let tick = self.tick;
1246 self.trace.note_exits(self.port.exits());
1247 match owed {
1248 OwedTrace::SendFailed(queue) => self
1249 .trace
1250 .event(tick, Event::QueueSendFailed { queue, name: "" }),
1251 OwedTrace::ReceiveFailed(queue) => self
1252 .trace
1253 .event(tick, Event::QueueReceiveFailed { queue, name: "" }),
1254 OwedTrace::TimerCommandSend {
1255 timer,
1256 name,
1257 command,
1258 value,
1259 } => self.trace.event(
1260 tick,
1261 Event::TimerCommandSend {
1262 timer,
1263 name: name.as_str(),
1264 command,
1265 value,
1266 },
1267 ),
1268 OwedTrace::EventGroupWaitBitsEnd {
1269 group,
1270 bits,
1271 timed_out,
1272 } => self.trace.event(
1273 tick,
1274 Event::EventGroupWaitBitsEnd {
1275 group,
1276 bits,
1277 timed_out,
1278 },
1279 ),
1280 // Not reachable through this function: `create_task` owes
1281 // its second half directly, because that half is not a
1282 // trace line — it inserts into a ready list — and it must
1283 // run INSIDE the section this one has already left.
1284 OwedTrace::AddNewTaskToReadyList { .. } | OwedTrace::None => {}
1285 }
1286 return;
1287 }
1288 if let Some(slot) = self.owed_trace.get_mut(usize::from(caller.index())) {
1289 *slot = owed;
1290 }
1291 }
1292
1293 /// Record that `task` was preempted before the `portYIELD()` at the end
1294 /// of the call it is inside.
1295 pub(crate) fn owe_yield(&mut self, task: TaskHandle) {
1296 if let Some(flag) = self.owes_yield.get_mut(usize::from(task.index())) {
1297 *flag = true;
1298 }
1299 }
1300
1301 /// Hand the CPU from `outgoing` to `incoming`: what `prvSwitchThread`
1302 /// does to `uxCriticalNesting`.
1303 ///
1304 /// Everything the outgoing task's call still does from here belongs to
1305 /// that task at the time it runs again, so the port stops counting the
1306 /// frame's exits as sim time and starts tallying them;
1307 /// [`Kernel::settle_unwind`] collects the tally once the frame has
1308 /// finished. A task being switched in for the first time has a fresh
1309 /// stack and owes nothing.
1310 fn hand_over(&mut self, outgoing: TaskHandle, incoming: TaskHandle) {
1311 // A tail that switches again is still the first frame's tail: the
1312 // code after the second switch is on the same abandoned stack.
1313 if self.unwinding.is_none() {
1314 self.unwinding = Some(outgoing);
1315 self.port.begin_unwind();
1316 }
1317 let index = usize::from(incoming.index());
1318 if let Some(flag) = self.started.get_mut(index) {
1319 if !*flag {
1320 *flag = true;
1321 if let Some(slot) = self.owed_exits.get_mut(index) {
1322 *slot = 0;
1323 }
1324 if let Some(flag) = self.owes_yield.get_mut(index) {
1325 *flag = false;
1326 }
1327 if let Some(slot) = self.owed_trace.get_mut(index) {
1328 *slot = OwedTrace::None;
1329 }
1330 }
1331 }
1332 }
1333
1334 /// Mark a task as having run, without a hand-over: the first task the
1335 /// scheduler starts has no predecessor.
1336 fn note_first_start(&mut self, task: TaskHandle) {
1337 if let Some(flag) = self.started.get_mut(usize::from(task.index())) {
1338 *flag = true;
1339 }
1340 }
1341
1342 /// `xTaskIncrementTick`; `true` when a context switch is required.
1343 pub fn increment_tick(&mut self) -> bool {
1344 let tick = self.tick;
1345 self.trace.note_exits(self.port.exits());
1346 self.trace.event(tick, Event::TaskIncrementTick { tick });
1347 let mut switch_required = false;
1348 if self.suspended_depth == 0 {
1349 let next = self.tick.wrapping_add(1) & Self::MAX_DELAY;
1350 self.tick = next;
1351 if next == 0 {
1352 self.switch_delayed_lists();
1353 }
1354 if next >= self.next_unblock_time {
1355 switch_required = self.wake_due_tasks(next);
1356 }
1357 // `!switch_required` first: when the tick already woke a
1358 // task, this can only set what is already set, and asking
1359 // costs a TCB resolve for the running priority plus a list
1360 // read. Setting a `true` to `true` is not worth either.
1361 if C::USE_PREEMPTION
1362 && C::USE_TIME_SLICING
1363 && !switch_required
1364 && self
1365 .lists
1366 .len(Self::ready_list(self.current_priority()))
1367 .unwrap_or(0)
1368 > 1
1369 {
1370 switch_required = true;
1371 }
1372 // The C guards this site with `xPendedTicks == 0`, so a hook
1373 // does not fire again for each tick being unwound by
1374 // `xTaskResumeAll`. It sits after the time-slicing test and
1375 // before the yield-pending one, and a `FromISR` call inside it
1376 // can set `yield_pending`, so the order is load-bearing.
1377 if self.pended_ticks == 0 {
1378 self.run_tick_hook();
1379 }
1380 if C::USE_PREEMPTION && self.yield_pending {
1381 switch_required = true;
1382 }
1383 } else {
1384 self.pended_ticks = self.pended_ticks.wrapping_add(1);
1385 // "The tick hook gets called at regular intervals, even if the
1386 // scheduler is locked" — and here with no guard, because this
1387 // tick is being pended rather than unwound.
1388 self.run_tick_hook();
1389 }
1390 switch_required
1391 }
1392
1393 // ------------------------------------------- task notifications --
1394 //
1395 // A notification is the lightest thing a task can block on: no object,
1396 // no event list, just a slot in the TCB and the delayed list. That is
1397 // why the stream buffers use one rather than a queue, and why waking a
1398 // task this way is a list move rather than an event-list walk.
1399
1400 /// `xTaskGenericNotifyWait`.
1401 ///
1402 /// `true` is `pdTRUE`: a notification was received. `Blocked` means the
1403 /// caller must call again — the C blocks inside this function, and a
1404 /// kernel with no stacks returns instead.
1405 ///
1406 /// # Errors
1407 /// [`Error::InvalidArgument`] for an index past the configured array.
1408 pub fn notify_wait(
1409 &mut self,
1410 index: usize,
1411 clear_on_entry: u32,
1412 clear_on_exit: u32,
1413 ticks: u64,
1414 ) -> Result<Wait<(bool, u32)>> {
1415 if index >= C::NOTIFICATION_ARRAY_ENTRIES {
1416 return Err(Error::InvalidArgument);
1417 }
1418 let caller = self.current;
1419 let state = self.notify_state_of(caller, index);
1420 if state != NotifyState::Received && ticks > 0 && !self.notify_blocked(caller) {
1421 self.suspend_all();
1422 self.enter_critical();
1423 let mut should_block = false;
1424 if self.notify_state_of(caller, index) != NotifyState::Received {
1425 if let Ok(tcb) = self.tcbs.resolve_mut(caller) {
1426 if let Some(slot) = tcb.notified.get_mut(index) {
1427 *slot &= !clear_on_entry;
1428 }
1429 if let Some(slot) = tcb.notify_state.get_mut(index) {
1430 *slot = NotifyState::Waiting;
1431 }
1432 }
1433 should_block = true;
1434 }
1435 self.exit_critical();
1436 if should_block {
1437 let tick = self.tick;
1438 self.trace.note_exits(self.port.exits());
1439 self.trace_task(caller, |task, name| Event::TaskNotifyWaitBlock {
1440 task,
1441 name,
1442 index,
1443 });
1444 let _ = tick;
1445 self.add_current_task_to_delayed_list(ticks, true)?;
1446 }
1447 let already_yielded = self.resume_all();
1448 if should_block && !already_yielded {
1449 if self.current == caller {
1450 self.port_yield();
1451 } else {
1452 self.owe_yield(caller);
1453 }
1454 }
1455 // The C blocks here; this kernel comes back and is called again.
1456 self.set_notify_blocked(caller, true);
1457 return Ok(Wait::Blocked);
1458 }
1459 self.set_notify_blocked(caller, false);
1460 self.enter_critical();
1461 self.trace.note_exits(self.port.exits());
1462 self.trace_task(caller, |task, name| Event::TaskNotifyWait {
1463 task,
1464 name,
1465 index,
1466 });
1467 let value = self.notified_value_of(caller, index);
1468 let received = self.notify_state_of(caller, index) == NotifyState::Received;
1469 if let Ok(tcb) = self.tcbs.resolve_mut(caller) {
1470 if received {
1471 if let Some(slot) = tcb.notified.get_mut(index) {
1472 *slot &= !clear_on_exit;
1473 }
1474 }
1475 if let Some(slot) = tcb.notify_state.get_mut(index) {
1476 *slot = NotifyState::NotWaiting;
1477 }
1478 }
1479 self.exit_critical();
1480 Ok(Wait::Ready((received, value)))
1481 }
1482
1483 /// `ulTaskGenericNotifyTake`: the notification used as a lightweight
1484 /// counting semaphore — block until the value is non-zero, then either
1485 /// zero it or decrement it by one.
1486 ///
1487 /// This is the half of the notification API the kernel was missing.
1488 /// `notify_wait` waits on *bits*; this waits on a *count*, and the two
1489 /// are separate calls in the C with their own trace events. Its
1490 /// partner, `xTaskNotifyGive`, needs no method of its own: the C
1491 /// defines it as `xTaskGenericNotify( .., 0, eIncrement, NULL )` and
1492 /// it traces as an ordinary notify, so
1493 /// `notify(task, index, 0, NotifyAction::Increment)` already is it.
1494 ///
1495 /// # The shape, and why it is not `notify_wait`'s
1496 ///
1497 /// The C blocks on `ulNotifiedValue == 0`, **not** on the notify
1498 /// state, and it does no clear-on-entry. Both halves run on the way
1499 /// out: `traceTASK_NOTIFY_TAKE` fires whether or not the call ever
1500 /// blocked, which is what makes a take that finds a count already
1501 /// waiting still emit one line.
1502 ///
1503 /// This kernel has no stack, so the block is a return: the first call
1504 /// answers [`Wait::Blocked`] and the caller is entered again when the
1505 /// scheduler runs it, taking the second half. `notify_wait_pending`
1506 /// answers for this call too — a task is only ever inside one of the
1507 /// two.
1508 ///
1509 /// # Errors
1510 /// [`Error::InvalidArgument`] for an index past the configured array.
1511 pub fn notify_take(
1512 &mut self,
1513 index: usize,
1514 clear_on_exit: bool,
1515 ticks: u64,
1516 ) -> Result<Wait<u32>> {
1517 if index >= C::NOTIFICATION_ARRAY_ENTRIES {
1518 return Err(Error::InvalidArgument);
1519 }
1520 let caller = self.current;
1521 if self.notified_value_of(caller, index) == 0 && ticks > 0 && !self.notify_blocked(caller) {
1522 self.suspend_all();
1523 self.enter_critical();
1524 let mut should_block = false;
1525 // Re-checked inside the section, as the C does: a notify from
1526 // an ISR between the two reads would otherwise be lost.
1527 if self.notified_value_of(caller, index) == 0 {
1528 if let Ok(tcb) = self.tcbs.resolve_mut(caller) {
1529 if let Some(slot) = tcb.notify_state.get_mut(index) {
1530 *slot = NotifyState::Waiting;
1531 }
1532 }
1533 should_block = true;
1534 }
1535 self.exit_critical();
1536 if should_block {
1537 // The C traces BEFORE it adds itself to the delayed list.
1538 self.trace.note_exits(self.port.exits());
1539 self.trace_task(caller, |task, name| Event::TaskNotifyTakeBlock {
1540 task,
1541 name,
1542 index,
1543 });
1544 self.add_current_task_to_delayed_list(ticks, true)?;
1545 }
1546 let already_yielded = self.resume_all();
1547 if should_block && !already_yielded {
1548 if self.current == caller {
1549 self.port_yield();
1550 } else {
1551 self.owe_yield(caller);
1552 }
1553 }
1554 self.set_notify_blocked(caller, true);
1555 return Ok(Wait::Blocked);
1556 }
1557 self.set_notify_blocked(caller, false);
1558 self.enter_critical();
1559 self.trace.note_exits(self.port.exits());
1560 self.trace_task(caller, |task, name| Event::TaskNotifyTake {
1561 task,
1562 name,
1563 index,
1564 });
1565 let value = self.notified_value_of(caller, index);
1566 if let Ok(tcb) = self.tcbs.resolve_mut(caller) {
1567 if value != 0 {
1568 if let Some(slot) = tcb.notified.get_mut(index) {
1569 // Guarded by `value != 0` above, so the saturation
1570 // never fires; it is here because the workspace forbids
1571 // bare arithmetic, and a decrement is exactly the place
1572 // an unguarded one would wrap.
1573 *slot = if clear_on_exit {
1574 0
1575 } else {
1576 value.saturating_sub(1)
1577 };
1578 }
1579 }
1580 if let Some(slot) = tcb.notify_state.get_mut(index) {
1581 *slot = NotifyState::NotWaiting;
1582 }
1583 }
1584 self.exit_critical();
1585 Ok(Wait::Ready(value))
1586 }
1587
1588 /// `xTaskGetHandle`: the task registered under `name`.
1589 ///
1590 /// # It is not free, and that is the point
1591 ///
1592 /// The C walks every ready list, then the delayed lists, then the
1593 /// suspended and waiting-termination lists, all inside
1594 /// `vTaskSuspendAll` / `xTaskResumeAll` — and `xTaskResumeAll` takes a
1595 /// critical section. So a lookup that emits **no trace event** still
1596 /// costs **one critical-section exit**, which under the sim contract is
1597 /// one unit of time.
1598 ///
1599 /// That is why this exists rather than a scenario keeping the handle it
1600 /// was given at creation. `AbortDelay`'s remake did exactly that, on the
1601 /// reasoning that a lookup emitting no event could not change the trace,
1602 /// and the corpus caught it: every event still agreed and the
1603 /// exit column was one short at the first block.
1604 ///
1605 /// The arena is searched instead of five lists, which cannot change the
1606 /// answer — a live task is in exactly one of them — and does not change
1607 /// the accounting either, because the cost is the suspend/resume pair.
1608 ///
1609 /// # Errors
1610 /// [`Error::Gone`] when no live task carries that name.
1611 pub fn task_get_handle(&mut self, name: &str) -> Result<TaskHandle> {
1612 self.suspend_all();
1613 let mut found = None;
1614 for index in 0..u16::try_from(self.tcbs.capacity()).unwrap_or(u16::MAX) {
1615 let Some(handle) = self.tcbs.handle_at(index) else {
1616 continue;
1617 };
1618 if self
1619 .tcbs
1620 .resolve(handle)
1621 .is_ok_and(|tcb| tcb.name.as_str() == name)
1622 {
1623 found = Some(handle);
1624 break;
1625 }
1626 }
1627 let _ = self.resume_all();
1628 found.ok_or(Error::Gone)
1629 }
1630
1631 /// `xTaskGenericNotify`.
1632 ///
1633 /// # Errors
1634 /// [`Error::InvalidArgument`] for a bad index; [`Error::Gone`] for a
1635 /// stale handle.
1636 pub fn notify(
1637 &mut self,
1638 task: TaskHandle,
1639 index: usize,
1640 value: u32,
1641 action: NotifyAction,
1642 ) -> Result<bool> {
1643 if index >= C::NOTIFICATION_ARRAY_ENTRIES {
1644 return Err(Error::InvalidArgument);
1645 }
1646 self.enter_critical();
1647 let result = self.notify_locked(task, index, value, action, false);
1648 self.exit_critical();
1649 result.map(|(ok, _)| ok)
1650 }
1651
1652 /// `xTaskGenericNotifyFromISR`: the same, without a critical section
1653 /// that costs anything, and putting a woken task on the pending-ready
1654 /// list when the scheduler is suspended.
1655 ///
1656 /// # Errors
1657 /// As [`Kernel::notify`].
1658 pub fn notify_from_isr(
1659 &mut self,
1660 task: TaskHandle,
1661 index: usize,
1662 value: u32,
1663 action: NotifyAction,
1664 ) -> Result<(bool, Woken)> {
1665 if index >= C::NOTIFICATION_ARRAY_ENTRIES {
1666 return Err(Error::InvalidArgument);
1667 }
1668 let mask = self.port.enter_critical_from_isr();
1669 let result = self.notify_locked(task, index, value, action, true);
1670 self.port.exit_critical_from_isr(mask);
1671 result.map(|(ok, woken)| (ok, if woken { Woken::YES } else { Woken::NO }))
1672 }
1673
1674 /// The body both notify paths share.
1675 fn notify_locked(
1676 &mut self,
1677 task: TaskHandle,
1678 index: usize,
1679 value: u32,
1680 action: NotifyAction,
1681 from_isr: bool,
1682 ) -> Result<(bool, bool)> {
1683 let original = self.notify_state_of(task, index);
1684 let mut ok = true;
1685 {
1686 let tcb = self.tcbs.resolve_mut(task)?;
1687 if let Some(slot) = tcb.notify_state.get_mut(index) {
1688 *slot = NotifyState::Received;
1689 }
1690 if let Some(slot) = tcb.notified.get_mut(index) {
1691 match action {
1692 NotifyAction::SetBits => *slot |= value,
1693 NotifyAction::Increment => *slot = slot.wrapping_add(1),
1694 NotifyAction::Overwrite => *slot = value,
1695 NotifyAction::NoOverwrite => {
1696 if original == NotifyState::Received {
1697 ok = false;
1698 } else {
1699 *slot = value;
1700 }
1701 }
1702 NotifyAction::None => {}
1703 }
1704 }
1705 }
1706 // `traceTASK_NOTIFY` is hooked; `traceTASK_NOTIFY_FROM_ISR` is not,
1707 // so an interrupt's notification says nothing on either side.
1708 if !from_isr {
1709 self.trace.note_exits(self.port.exits());
1710 self.trace_task(task, |t, name| Event::TaskNotify {
1711 task: t,
1712 name,
1713 index,
1714 });
1715 }
1716 let mut woke_higher = false;
1717 if original == NotifyState::Waiting {
1718 let item = Self::state_item(task);
1719 if from_isr && self.suspended_depth != 0 {
1720 let _ = self.lists.remove(Self::event_item(task));
1721 let value = self.lists.value(Self::event_item(task))?;
1722 self.lists
1723 .insert(Self::pending_ready_list(), Self::event_item(task), value)?;
1724 } else {
1725 if self.lists.container(item)?.is_some() {
1726 let _ = self.lists.remove(item);
1727 }
1728 self.add_task_to_ready_list(task)?;
1729 }
1730 let woken = self.tcbs.resolve(task).map(|t| t.priority).unwrap_or(0);
1731 if woken > self.current_priority() {
1732 woke_higher = true;
1733 if from_isr {
1734 // The C sets `xYieldPendings[ 0 ]` here as well as
1735 // reporting the wake through the caller's flag — so an
1736 // interrupt that discards the flag still gets the
1737 // switch, on the way out of `xTaskIncrementTick`. That
1738 // is why the tick hook runs before the yield-pending
1739 // test and not after it.
1740 self.yield_pending = true;
1741 } else {
1742 // taskYIELD_ANY_CORE_IF_USING_PREEMPTION, inside the
1743 // section.
1744 self.port_yield();
1745 }
1746 }
1747 }
1748 Ok((ok, woke_higher))
1749 }
1750
1751 /// `xTaskGenericNotifyStateClear`. `None` is the calling task.
1752 ///
1753 /// # Errors
1754 /// As [`Kernel::notify`].
1755 pub fn notify_state_clear(&mut self, task: Option<TaskHandle>, index: usize) -> Result<bool> {
1756 if index >= C::NOTIFICATION_ARRAY_ENTRIES {
1757 return Err(Error::InvalidArgument);
1758 }
1759 let target = task.unwrap_or(self.current);
1760 self.enter_critical();
1761 let cleared = self.notify_state_of(target, index) == NotifyState::Received;
1762 if cleared {
1763 if let Ok(tcb) = self.tcbs.resolve_mut(target) {
1764 if let Some(slot) = tcb.notify_state.get_mut(index) {
1765 *slot = NotifyState::NotWaiting;
1766 }
1767 }
1768 }
1769 self.exit_critical();
1770 Ok(cleared)
1771 }
1772
1773 /// `ulNotifiedValue[ index ]` for a task. `None` is the calling task.
1774 ///
1775 /// The C reaches this through `xTaskNotifyAndQuery`, which hands back
1776 /// the value a notification is about to REPLACE, and through
1777 /// `ulTaskNotifyValueClear`. Neither could be implemented without it:
1778 /// `xTaskNotifyAndQuery` is `xTaskGenericNotify` with a
1779 /// `pulPreviousNotificationValue` out-parameter, and `TaskNotify.c:373`
1780 /// asserts on what comes back. The field existed; only the way to read
1781 /// it was missing, which is the kind of gap a C ABI finds and a Rust
1782 /// face never does.
1783 ///
1784 /// # Errors
1785 /// `InvalidArgument` if `index` is outside the notification array, or
1786 /// the resolve error if the handle is stale.
1787 pub fn notify_value(&mut self, task: Option<TaskHandle>, index: usize) -> Result<u32> {
1788 if index >= C::NOTIFICATION_ARRAY_ENTRIES {
1789 return Err(Error::InvalidArgument);
1790 }
1791 let target = task.unwrap_or(self.current);
1792 self.enter_critical();
1793 let value = self
1794 .tcbs
1795 .resolve(target)
1796 .map(|tcb| tcb.notified.get(index).copied().unwrap_or(0));
1797 self.exit_critical();
1798 value
1799 }
1800
1801 /// `ulTaskGenericNotifyValueClear`: clear `bits` in the task's
1802 /// notification value and answer what it was BEFORE the clear.
1803 ///
1804 /// # Errors
1805 /// As [`Kernel::notify_value`].
1806 pub fn notify_value_clear(
1807 &mut self,
1808 task: Option<TaskHandle>,
1809 index: usize,
1810 bits: u32,
1811 ) -> Result<u32> {
1812 if index >= C::NOTIFICATION_ARRAY_ENTRIES {
1813 return Err(Error::InvalidArgument);
1814 }
1815 let target = task.unwrap_or(self.current);
1816 self.enter_critical();
1817 let before = match self.tcbs.resolve_mut(target) {
1818 Ok(tcb) => match tcb.notified.get_mut(index) {
1819 Some(slot) => {
1820 let was = *slot;
1821 *slot &= !bits;
1822 Ok(was)
1823 }
1824 None => Err(Error::InvalidArgument),
1825 },
1826 Err(e) => Err(e),
1827 };
1828 self.exit_critical();
1829 before
1830 }
1831
1832 /// Whether `task` is part-way through a notification wait — it blocked
1833 /// once and has not yet run the half of `xTaskGenericNotifyWait` that
1834 /// happens when a task wakes.
1835 ///
1836 /// A caller that blocks on a notification needs this: the C's wait
1837 /// returns *after* the wake, so its second half always runs, and a
1838 /// stackless caller that decided not to block again would otherwise
1839 /// skip it and leave the slot marked as received for ever.
1840 #[must_use]
1841 pub fn notify_wait_pending(&self, task: TaskHandle) -> bool {
1842 self.notify_blocked(task)
1843 }
1844
1845 /// Whether `task` has a stream-buffer call to resume, and the local it
1846 /// left behind. Taking it clears the marker.
1847 pub(crate) fn take_stream_resume(&mut self, task: TaskHandle) -> Option<usize> {
1848 let tcb = self.tcbs.resolve_mut(task).ok()?;
1849 if !tcb.stream_resume {
1850 return None;
1851 }
1852 tcb.stream_resume = false;
1853 Some(tcb.stream_local)
1854 }
1855
1856 /// Remember where a preempted stream-buffer call has to start again.
1857 pub(crate) fn set_stream_resume(&mut self, task: TaskHandle, local: usize) {
1858 if let Ok(tcb) = self.tcbs.resolve_mut(task) {
1859 tcb.stream_resume = true;
1860 tcb.stream_local = local;
1861 }
1862 }
1863
1864 fn notify_blocked(&self, task: TaskHandle) -> bool {
1865 self.tcbs
1866 .resolve(task)
1867 .map(|t| t.notify_blocked)
1868 .unwrap_or(false)
1869 }
1870
1871 fn set_notify_blocked(&mut self, task: TaskHandle, blocked: bool) {
1872 if let Ok(tcb) = self.tcbs.resolve_mut(task) {
1873 tcb.notify_blocked = blocked;
1874 }
1875 }
1876
1877 fn notify_state_of(&self, task: TaskHandle, index: usize) -> NotifyState {
1878 self.tcbs
1879 .resolve(task)
1880 .ok()
1881 .and_then(|t| t.notify_state.get(index).copied())
1882 .unwrap_or(NotifyState::NotWaiting)
1883 }
1884
1885 fn notified_value_of(&self, task: TaskHandle, index: usize) -> u32 {
1886 self.tcbs
1887 .resolve(task)
1888 .ok()
1889 .and_then(|t| t.notified.get(index).copied())
1890 .unwrap_or(0)
1891 }
1892
1893 /// `xTaskGetTickCountFromISR`.
1894 ///
1895 /// No critical section, exactly as `xTaskGetTickCount` takes none: the
1896 /// Posix port sets `portTICK_TYPE_IS_ATOMIC`, so reading the count is
1897 /// a single load on both sides and costs no sim time.
1898 #[must_use]
1899 pub const fn tick_count_from_isr(&self) -> u64 {
1900 self.tick
1901 }
1902
1903 /// `vTaskMissedYield`: a yield that could not be taken where it was
1904 /// asked for — because a task is walking an event list, or because the
1905 /// scheduler is suspended — and must happen on the way out instead.
1906 pub(crate) fn missed_yield(&mut self) {
1907 self.yield_pending = true;
1908 }
1909
1910 /// `vApplicationTickHook()`, when `configUSE_TICK_HOOK` is 1.
1911 ///
1912 /// The hook is copied out, run, and the result stored: that is what
1913 /// lets it borrow the kernel mutably while the kernel owns it. A hook
1914 /// that reads the kernel's own copy of itself therefore sees the value
1915 /// from before this call.
1916 fn run_tick_hook(&mut self) {
1917 if !C::USE_TICK_HOOK {
1918 return;
1919 }
1920 let hook = self.tick_hook;
1921 self.tick_hook = hook.tick(self);
1922 }
1923
1924 /// The tick hook, as it now stands.
1925 ///
1926 /// A scenario's `xAre...StillRunning()` reads the status its interrupt
1927 /// half latched, so the owner of the kernel needs to see it.
1928 pub const fn tick_hook(&self) -> &H {
1929 &self.tick_hook
1930 }
1931
1932 /// The tick hook, to install or reset one after construction.
1933 pub const fn tick_hook_mut(&mut self) -> &mut H {
1934 &mut self.tick_hook
1935 }
1936
1937 /// The wake loop inside `xTaskIncrementTick`.
1938 fn wake_due_tasks(&mut self, now: u64) -> bool {
1939 let mut switch_required = false;
1940 loop {
1941 let delayed = self.delayed_list();
1942 if self.lists.is_empty(delayed) != Ok(false) {
1943 self.next_unblock_time = Self::MAX_DELAY;
1944 return switch_required;
1945 }
1946 let Ok(Some(item)) = self.lists.head(delayed) else {
1947 self.next_unblock_time = Self::MAX_DELAY;
1948 return switch_required;
1949 };
1950 let Ok(wake_at) = self.lists.value(item) else {
1951 return switch_required;
1952 };
1953 if now < wake_at {
1954 self.next_unblock_time = wake_at;
1955 return switch_required;
1956 }
1957 let Ok(task) = self.task_of_state_item(item) else {
1958 return switch_required;
1959 };
1960 let _ = self.lists.remove(item);
1961 if self
1962 .lists
1963 .container(Self::event_item(task))
1964 .unwrap_or(None)
1965 .is_some()
1966 {
1967 let _ = self.lists.remove(Self::event_item(task));
1968 }
1969 if self.add_task_to_ready_list(task).is_err() {
1970 return switch_required;
1971 }
1972 if C::USE_PREEMPTION {
1973 let woken = self.tcbs.resolve(task).map(|t| t.priority).unwrap_or(0);
1974 if woken > self.current_priority() {
1975 switch_required = true;
1976 }
1977 }
1978 }
1979 }
1980
1981 /// `taskSWITCH_DELAYED_LISTS()`.
1982 fn switch_delayed_lists(&mut self) {
1983 self.delayed_swapped = !self.delayed_swapped;
1984 self.overflows = self.overflows.wrapping_add(1);
1985 self.reset_next_task_unblock_time();
1986 }
1987
1988 /// `prvResetNextTaskUnblockTime`.
1989 fn reset_next_task_unblock_time(&mut self) {
1990 let delayed = self.delayed_list();
1991 self.next_unblock_time = self.lists.head_value(delayed).unwrap_or(Self::MAX_DELAY);
1992 }
1993
1994 // ------------------------------------------------------------- delay --
1995
1996 /// `vTaskDelay`.
1997 ///
1998 /// # Errors
1999 /// A list error surfaces as itself.
2000 pub fn delay(&mut self, ticks: u64) -> Result<()> {
2001 let caller = self.current;
2002 let mut already_yielded = false;
2003 if ticks > 0 {
2004 self.suspend_all();
2005 {
2006 let current = self.current;
2007 self.trace_task(current, |task, name| Event::TaskDelay { task, name, ticks });
2008 self.add_current_task_to_delayed_list(ticks, false)?;
2009 }
2010 already_yielded = self.resume_all();
2011 }
2012 // taskYIELD_WITHIN_API(), outside any critical section — unless a
2013 // tick switched us out somewhere above, in which case this line is
2014 // on a stack that is not running, and runs when the task does.
2015 if !already_yielded {
2016 if self.current == caller {
2017 self.port_yield();
2018 } else {
2019 self.owe_yield(caller);
2020 }
2021 }
2022 Ok(())
2023 }
2024
2025 /// `xTaskAbortDelay`: pull a task out of the Blocked state early.
2026 ///
2027 /// `true` is `pdPASS` — the task really was blocked and is now ready.
2028 /// A task that was not blocked is left alone and `false` comes back,
2029 /// which is the C's `pdFAIL`.
2030 ///
2031 /// The shape matters as much as the effect: the whole thing runs with
2032 /// the scheduler suspended, the event list is touched inside a critical
2033 /// section of its own because an interrupt can reach it, and a yield
2034 /// that the higher priority of the woken task calls for is *pended*
2035 /// rather than taken, so it happens on the way out of `xTaskResumeAll`.
2036 ///
2037 /// # Errors
2038 /// [`Error::Gone`] for a stale handle.
2039 pub fn abort_delay(&mut self, task: TaskHandle) -> Result<bool> {
2040 if !self.tcbs.contains(task) {
2041 return Err(Error::Gone);
2042 }
2043 self.suspend_all();
2044 // eTaskGetState: a critical section of its own on any task but the
2045 // running one, and the running one is never Blocked.
2046 let blocked = self.task_state_get(task)? == TaskState::Blocked;
2047 if !blocked {
2048 let _ = self.resume_all();
2049 return Ok(false);
2050 }
2051 let item = Self::state_item(task);
2052 if self.lists.container(item)?.is_some() {
2053 let _ = self.lists.remove(item);
2054 }
2055 self.enter_critical();
2056 {
2057 let event = Self::event_item(task);
2058 if self.lists.container(event)?.is_some() {
2059 let _ = self.lists.remove(event);
2060 if let Some(flag) = self.delay_aborted.get_mut(usize::from(task.index())) {
2061 *flag = true;
2062 }
2063 }
2064 }
2065 self.exit_critical();
2066 self.add_task_to_ready_list(task)?;
2067 // configUSE_PREEMPTION, one core: pend the yield rather than take
2068 // it, so it lands where `xTaskResumeAll` puts it.
2069 let woken = self.tcbs.resolve(task).map(|t| t.priority).unwrap_or(0);
2070 if woken > self.current_priority() {
2071 self.yield_pending = true;
2072 }
2073 let _ = self.resume_all();
2074 Ok(true)
2075 }
2076
2077 /// `prvAddCurrentTaskToDelayedList`.
2078 pub(crate) fn add_current_task_to_delayed_list(
2079 &mut self,
2080 ticks: u64,
2081 can_block_indefinitely: bool,
2082 ) -> Result<()> {
2083 let now = self.tick;
2084 let current = self.current;
2085 // About to enter a delayed list, so the abort flag is cleared here
2086 // and can only be seen set by a task that really was aborted.
2087 if let Some(flag) = self.delay_aborted.get_mut(usize::from(current.index())) {
2088 *flag = false;
2089 }
2090 let item = Self::state_item(current);
2091 // `uxListRemove` already answers `NotActive` for an item that is
2092 // in no list, and both it and the container test are one read of
2093 // the same node — so asking first was reading it twice. This is
2094 // the running task's state item, so it is in the ready list and
2095 // the test passed anyway.
2096 let _ = self.lists.remove(item);
2097 if ticks == Self::MAX_DELAY && can_block_indefinitely {
2098 self.lists.insert_end(Self::suspended_list(), item)?;
2099 return Ok(());
2100 }
2101 let wake_at = now.wrapping_add(ticks) & Self::MAX_DELAY;
2102 // `vListInsert` sets the item's value from the one it sorts by, so
2103 // both arms below write `wake_at` themselves. Setting it here as
2104 // well was a second lookup of the same item to store the same
2105 // number into it.
2106 if wake_at < now {
2107 self.trace_task(current, |task, name| {
2108 Event::MovedTaskToOverflowDelayedList { task, name }
2109 });
2110 let list = self.overflow_delayed_list();
2111 self.lists.insert(list, item, wake_at)?;
2112 } else {
2113 self.trace_task(current, |task, name| Event::MovedTaskToDelayedList {
2114 task,
2115 name,
2116 });
2117 let list = self.delayed_list();
2118 self.lists.insert(list, item, wake_at)?;
2119 if wake_at < self.next_unblock_time {
2120 self.next_unblock_time = wake_at;
2121 }
2122 }
2123 Ok(())
2124 }
2125
2126 // --------------------------------------------------- suspend / resume --
2127
2128 /// `vTaskSuspend`. `None` suspends the calling task.
2129 ///
2130 /// # Errors
2131 /// [`Error::Gone`] for a stale handle.
2132 pub fn suspend(&mut self, task: Option<TaskHandle>) -> Result<()> {
2133 let caller = self.current;
2134 let target = task.unwrap_or(caller);
2135 self.enter_critical();
2136 {
2137 if !self.tcbs.contains(target) {
2138 self.exit_critical();
2139 return Err(Error::Gone);
2140 }
2141 self.trace_task(target, |task, name| Event::TaskSuspend { task, name });
2142 let item = Self::state_item(target);
2143 if self.lists.container(item)?.is_some() {
2144 let _ = self.lists.remove(item);
2145 }
2146 let event = Self::event_item(target);
2147 if self.lists.container(event)?.is_some() {
2148 let _ = self.lists.remove(event);
2149 }
2150 self.lists.insert_end(Self::suspended_list(), item)?;
2151 // `vTaskSuspend`, `tasks.c`:
2152 //
2153 // ```c
2154 // if( pxTCB->ucNotifyState[ x ] == taskWAITING_NOTIFICATION )
2155 // {
2156 // /* The task was blocked to wait for a notification, but
2157 // * is now suspended, so no notification was received. */
2158 // pxTCB->ucNotifyState[ x ] = taskNOT_WAITING_NOTIFICATION;
2159 // }
2160 // ```
2161 //
2162 // Not tidying: `eTaskGetState` calls a task on the suspended
2163 // list BLOCKED rather than SUSPENDED if any of its notification
2164 // slots is still waiting, so leaving the flag set makes a
2165 // suspended task report as blocked for ever.
2166 // `TaskNotify.c:498` asserts exactly that, in a timer callback
2167 // that suspends a task which is waiting for a notification --
2168 // and a task that reports the wrong state there is a task the
2169 // rest of that test never resumes correctly.
2170 if let Ok(tcb) = self.tcbs.resolve_mut(target) {
2171 for slot in &mut tcb.notify_state {
2172 if *slot == NotifyState::Waiting {
2173 *slot = NotifyState::NotWaiting;
2174 }
2175 }
2176 }
2177 }
2178 self.exit_critical();
2179 // A second, separate critical section — two outermost exits, which
2180 // is what the C does and therefore what the tick count depends on.
2181 if self.running {
2182 self.enter_critical();
2183 self.reset_next_task_unblock_time();
2184 self.exit_critical();
2185 }
2186 if target == caller {
2187 if self.running {
2188 // portYIELD_WITHIN_API(), outside the section.
2189 if self.current == caller {
2190 self.port_yield();
2191 } else {
2192 self.owe_yield(caller);
2193 }
2194 } else if self.lists.len(Self::suspended_list()).unwrap_or(0) != self.task_count {
2195 self.switch_context();
2196 }
2197 }
2198 Ok(())
2199 }
2200
2201 /// `vTaskResume`.
2202 ///
2203 /// # Errors
2204 /// [`Error::Gone`] for a stale handle.
2205 pub fn resume(&mut self, task: TaskHandle) -> Result<()> {
2206 if task == self.current || task.is_null() {
2207 return Ok(());
2208 }
2209 if !self.tcbs.contains(task) {
2210 return Err(Error::Gone);
2211 }
2212 self.enter_critical();
2213 {
2214 if self.task_is_suspended(task)? {
2215 self.trace_task(task, |task, name| Event::TaskResume { task, name });
2216 let _ = self.lists.remove(Self::state_item(task));
2217 self.add_task_to_ready_list(task)?;
2218 // taskYIELD_ANY_CORE_IF_USING_PREEMPTION: inside the
2219 // section, and only when the resumed task outranks the
2220 // running one.
2221 let resumed = self.tcbs.resolve(task)?.priority;
2222 if C::USE_PREEMPTION && self.current_priority() < resumed {
2223 self.port_yield();
2224 }
2225 }
2226 }
2227 self.exit_critical();
2228 Ok(())
2229 }
2230
2231 /// `prvTaskIsTaskSuspended`: on the suspended list, not on the pending
2232 /// ready list, and not waiting on an event.
2233 fn task_is_suspended(&self, task: TaskHandle) -> Result<bool> {
2234 if self.lists.container(Self::state_item(task))? != Some(Self::suspended_list()) {
2235 return Ok(false);
2236 }
2237 let event_list = self.lists.container(Self::event_item(task))?;
2238 if event_list == Some(Self::pending_ready_list()) {
2239 return Ok(false);
2240 }
2241 Ok(event_list.is_none())
2242 }
2243
2244 // -------------------------------------------------------- priorities --
2245
2246 /// `vTaskDelete`. `None` deletes the calling task.
2247 ///
2248 /// # Two paths, and the C picks between them for a reason
2249 ///
2250 /// Deleting *another* task frees its TCB on the spot. Deleting the
2251 /// **running** task cannot: the caller is still executing out of it. The
2252 /// C parks such a task on `xTasksWaitingTermination` and lets the idle
2253 /// task reclaim it, which is what `prvCheckTasksWaitingTermination` is
2254 /// for — so [`Kernel::check_tasks_waiting_termination`] is where the
2255 /// deferred half lands here, called from the idle body at the same point
2256 /// in the same loop.
2257 ///
2258 /// # The frees are outside the critical section, and they cost exits
2259 ///
2260 /// `prvDeleteTCB` runs **after** `taskEXIT_CRITICAL()` and frees twice —
2261 /// `vPortFreeStack( pxStack )` then `vPortFree( pxTCB )`. On the oracle's
2262 /// heap every free suspends the scheduler, so a delete is one exit for
2263 /// its own section plus two for the frees, in that order. Folding the
2264 /// frees into the section, or forgetting them, moves every later tick.
2265 ///
2266 /// # Errors
2267 /// [`Error::Gone`] for a stale handle.
2268 pub fn task_delete(&mut self, task: Option<TaskHandle>) -> Result<()> {
2269 // `prvGetTCBFromHandle`: NULL is the calling task.
2270 let target = task.unwrap_or(self.current);
2271 self.tcbs.resolve(target)?;
2272
2273 self.enter_critical();
2274 // `uxListRemove( &( pxTCB->xStateListItem ) )`. The C follows this
2275 // with `taskRESET_READY_PRIORITY`, which is an EMPTY MACRO under the
2276 // oracle's `configUSE_PORT_OPTIMISED_TASK_SELECTION 0` — there is no
2277 // priority bitmap to clear, and `vTaskSwitchContext` rediscovers the
2278 // top priority by walking down. Nothing to model.
2279 let state = Self::state_item(target);
2280 if matches!(self.lists.container(state), Ok(Some(_))) {
2281 let _ = self.lists.remove(state);
2282 }
2283 // `if( listLIST_ITEM_CONTAINER( &( pxTCB->xEventListItem ) ) != NULL )`
2284 // — a task blocked on a queue is on two lists, and both must let go.
2285 let event = Self::event_item(target);
2286 if matches!(self.lists.container(event), Ok(Some(_))) {
2287 let _ = self.lists.remove(event);
2288 }
2289 // `uxTaskNumber++` is not modelled: it exists so kernel-aware
2290 // debuggers can tell the task lists changed, and nothing the trace
2291 // records reads it.
2292
2293 // `xSchedulerRunning != pdFALSE && taskTASK_IS_RUNNING_OR_SCHEDULED_
2294 // _TO_YIELD( pxTCB )`, which at `configNUMBER_OF_CORES 1` is exactly
2295 // `pxTCB == pxCurrentTCB`.
2296 let defer = self.running && target == self.current;
2297 if defer {
2298 // `vListInsertEnd( &xTasksWaitingTermination, ... )` and
2299 // `++uxDeletedTasksWaitingCleanUp`. The count is NOT decremented
2300 // here — the idle task does that when it reaps.
2301 self.awaiting_reap = Some(target);
2302 self.trace_task(target, |task, name| Event::TaskDelete { task, name });
2303 } else {
2304 self.task_count = self.task_count.saturating_sub(1);
2305 self.trace_task(target, |task, name| Event::TaskDelete { task, name });
2306 self.reset_next_task_unblock_time();
2307 }
2308 self.exit_critical();
2309
2310 // `if( xDeleteTCBInIdleTask != pdTRUE ) { prvDeleteTCB( pxTCB ); }`,
2311 // outside the section.
2312 if !defer {
2313 self.delete_tcb(target);
2314 }
2315
2316 // `taskYIELD_WITHIN_API()`. The C guards this with
2317 // `xSchedulerRunning && pxTCB == pxCurrentTCB`, which is the same
2318 // condition that chose the deferred path.
2319 if defer {
2320 self.port_yield();
2321 }
2322 Ok(())
2323 }
2324
2325 /// `prvDeleteTCB`: two frees, and on the oracle's heap each one suspends
2326 /// the scheduler, so each is one outermost exit.
2327 fn delete_tcb(&mut self, task: TaskHandle) {
2328 // `vPortFreeStack( pxTCB->pxStack )`.
2329 self.account_for_allocation();
2330 // `vPortFree( pxTCB )`.
2331 self.account_for_allocation();
2332 let _ = self.tcbs.remove(task);
2333
2334 // The index is now free for reuse, so mark the task UNSTARTED.
2335 //
2336 // The C frees the TCB and a later `xTaskCreate` gets fresh memory,
2337 // so a new task cannot inherit anything. Our arena hands back the
2338 // same index, and the per-index bookkeeping — owed exits, an owed
2339 // yield, an owed trace line — is keyed on that index, not on the
2340 // TCB. Without this a task deleted while it still owed the tail of
2341 // a call bequeaths that debt to its successor, and the successor
2342 // pays an exit it never incurred: exactly how `death`'s second
2343 // cycle first diverged, one exit early, 1,100 lines after the first
2344 // cycle had matched perfectly.
2345 //
2346 // Clearing this ONE flag is the whole fix, and deliberately so.
2347 // `hand_over` already empties all three of those on a task's first
2348 // switch-in, and a reused index is a first switch-in again. Zeroing
2349 // them here as well would work and would be worse: two places that
2350 // must agree about what a fresh task owes, and a gate that cannot
2351 // catch either one being dropped because the other still covers it.
2352 if let Some(slot) = self.started.get_mut(usize::from(task.index())) {
2353 *slot = false;
2354 }
2355 }
2356
2357 /// `prvCheckTasksWaitingTermination`, called from the idle task.
2358 ///
2359 /// # This must cost NOTHING when nothing is pending
2360 ///
2361 /// The C's body is `while( uxDeletedTasksWaitingCleanUp > 0 )`, so an
2362 /// idle loop with no deleted task takes no critical section and pays no
2363 /// exit. That is what lets this be wired into the idle body without
2364 /// disturbing a single scenario in the corpus — and it is the property
2365 /// the corpus is checking when it stays byte-identical.
2366 ///
2367 /// At most one task can be pending: only the *running* task defers, and
2368 /// it yields before it can return, so a second deferred delete cannot
2369 /// happen until this has run.
2370 pub fn check_tasks_waiting_termination(&mut self) {
2371 let Some(task) = self.awaiting_reap else {
2372 return;
2373 };
2374 // The C takes the section per reaped task, decrements both counts
2375 // inside it, and calls `prvDeleteTCB` outside.
2376 self.enter_critical();
2377 self.awaiting_reap = None;
2378 self.task_count = self.task_count.saturating_sub(1);
2379 self.exit_critical();
2380 self.delete_tcb(task);
2381 }
2382
2383 /// `vTaskPrioritySet`. `None` is the calling task. /// `vTaskPrioritySet`. `None` is the calling task.
2384 ///
2385 /// # Errors
2386 /// [`Error::Gone`] for a stale handle.
2387 pub fn set_priority(&mut self, task: Option<TaskHandle>, new_priority: u8) -> Result<()> {
2388 let new_priority = new_priority.min(C::MAX_PRIORITIES.saturating_sub(1));
2389 let target = task.unwrap_or(self.current);
2390 let mut yield_required = false;
2391 self.enter_critical();
2392 {
2393 if !self.tcbs.contains(target) {
2394 self.exit_critical();
2395 return Err(Error::Gone);
2396 }
2397 self.trace_task(target, |task, name| Event::TaskPrioritySet {
2398 task,
2399 name,
2400 priority: Self::priority_value(new_priority),
2401 });
2402 let base = self.tcbs.resolve(target)?.base_priority;
2403 if base != new_priority {
2404 if new_priority > base {
2405 if target != self.current && new_priority > self.current_priority() {
2406 yield_required = true;
2407 }
2408 } else if target == self.current {
2409 yield_required = true;
2410 }
2411 let used_on_entry = self.tcbs.resolve(target)?.priority;
2412 {
2413 let tcb = self.tcbs.resolve_mut(target)?;
2414 if tcb.base_priority == tcb.priority || new_priority > tcb.priority {
2415 tcb.priority = new_priority;
2416 }
2417 tcb.base_priority = new_priority;
2418 }
2419 let event_value = u64::from(C::MAX_PRIORITIES.saturating_sub(new_priority));
2420 self.lists
2421 .set_value(Self::event_item(target), event_value)?;
2422 let item = Self::state_item(target);
2423 if self.lists.container(item)? == Some(Self::ready_list(used_on_entry)) {
2424 let _ = self.lists.remove(item);
2425 self.add_task_to_ready_list(target)?;
2426 }
2427 // taskYIELD_TASK_CORE_IF_USING_PREEMPTION: inside the
2428 // section.
2429 if yield_required && C::USE_PREEMPTION {
2430 self.port_yield();
2431 }
2432 }
2433 }
2434 self.exit_critical();
2435 Ok(())
2436 }
2437
2438 // ------------------------------------------------- scheduler suspend --
2439
2440 /// `vTaskSuspendAll`.
2441 pub fn suspend_all(&mut self) {
2442 self.suspended_depth = self.suspended_depth.saturating_add(1);
2443 }
2444
2445 /// `xTaskResumeAll`; `true` when it yielded on the caller's behalf.
2446 pub fn resume_all(&mut self) -> bool {
2447 let mut already_yielded = false;
2448 self.enter_critical();
2449 {
2450 self.suspended_depth = self.suspended_depth.saturating_sub(1);
2451 if self.suspended_depth == 0 && self.task_count > 0 {
2452 let mut moved_any = false;
2453 while self.lists.is_empty(Self::pending_ready_list()) == Ok(false) {
2454 let Ok(Some(item)) = self.lists.head(Self::pending_ready_list()) else {
2455 break;
2456 };
2457 let Ok(task) = self.task_of_event_item(item) else {
2458 break;
2459 };
2460 let _ = self.lists.remove(Self::event_item(task));
2461 let _ = self.lists.remove(Self::state_item(task));
2462 if self.add_task_to_ready_list(task).is_err() {
2463 break;
2464 }
2465 moved_any = true;
2466 let woken = self.tcbs.resolve(task).map(|t| t.priority).unwrap_or(0);
2467 if woken > self.current_priority() {
2468 self.yield_pending = true;
2469 }
2470 }
2471 if moved_any {
2472 self.reset_next_task_unblock_time();
2473 }
2474 if self.pended_ticks > 0 {
2475 let mut pended = self.pended_ticks;
2476 while pended > 0 {
2477 if self.increment_tick() {
2478 self.yield_pending = true;
2479 }
2480 pended = pended.saturating_sub(1);
2481 }
2482 self.pended_ticks = 0;
2483 }
2484 if self.yield_pending {
2485 if C::USE_PREEMPTION {
2486 already_yielded = true;
2487 }
2488 // taskYIELD_TASK_CORE_IF_USING_PREEMPTION: inside the
2489 // section.
2490 self.port_yield();
2491 }
2492 }
2493 }
2494 self.exit_critical();
2495 already_yielded
2496 }
2497
2498 // ------------------------------------------- the blocking-call frame --
2499
2500 /// Start (or continue) a blocking call's `for(;;)`.
2501 ///
2502 /// The first pass records the block time and the entry timestamp, the
2503 /// way `xEntryTimeSet` / `vTaskInternalSetTimeOutState` do; later
2504 /// passes leave the running total alone, because the `ticks` the caller
2505 /// passes is the original block time and the kernel is holding what is
2506 /// left of it.
2507 pub(crate) fn begin_wait(
2508 &mut self,
2509 task: TaskHandle,
2510 queue: QueueHandle,
2511 ticks: u64,
2512 ) -> Result<()> {
2513 let tick = self.tick;
2514 let overflows = self.overflows;
2515 let Ok(tcb) = self.tcbs.resolve_mut(task) else {
2516 // A queue call made before the scheduler started, from what the
2517 // C would call `main()`: there is no task to keep a frame for,
2518 // and such a call never blocks — `xSemaphoreGive` on a fresh
2519 // semaphore is the usual one.
2520 return Ok(());
2521 };
2522 if tcb.wait.queue != queue || !tcb.wait.entry_set {
2523 tcb.wait = WaitFrame {
2524 queue,
2525 ticks,
2526 entering: tick,
2527 overflows,
2528 entry_set: true,
2529 inherited: false,
2530 };
2531 }
2532 Ok(())
2533 }
2534
2535 /// The call finished, one way or the other.
2536 pub(crate) fn end_wait(&mut self, task: TaskHandle) {
2537 if let Ok(tcb) = self.tcbs.resolve_mut(task) {
2538 // Now that the frame is only set on the path that blocks, most
2539 // calls reach here with nothing to clear, and this was writing
2540 // a default frame over a default frame. `entry_set` is the
2541 // sentinel the C calls `xEntryTimeSet`, and it is the first
2542 // thing set — `inherited` is only reached far inside the
2543 // blocking path, so it cannot be true while this is false.
2544 if tcb.wait.entry_set {
2545 tcb.wait = WaitFrame::default();
2546 }
2547 }
2548 }
2549
2550 /// `xTicksToWait` as it now stands.
2551 pub(crate) fn remaining_ticks(&self, task: TaskHandle) -> u64 {
2552 self.tcbs.resolve(task).map(|t| t.wait.ticks).unwrap_or(0)
2553 }
2554
2555 /// `xInheritanceOccurred`.
2556 pub(crate) fn wait_inherited(&self, task: TaskHandle) -> bool {
2557 self.tcbs
2558 .resolve(task)
2559 .map(|t| t.wait.inherited)
2560 .unwrap_or(false)
2561 }
2562
2563 pub(crate) fn set_wait_inherited(&mut self, task: TaskHandle) {
2564 if let Ok(tcb) = self.tcbs.resolve_mut(task) {
2565 tcb.wait.inherited = true;
2566 }
2567 }
2568
2569 /// `xTaskCheckForTimeOut`: `true` when the block time has run out.
2570 /// Takes a critical section, as the C does, and charges the elapsed
2571 /// time against what is left.
2572 pub(crate) fn check_for_timeout(&mut self, task: TaskHandle) -> bool {
2573 self.enter_critical();
2574 // An aborted delay is not the same as a time out, but it has the
2575 // same result: stop waiting.
2576 if self.delay_aborted.get(usize::from(task.index())).copied() == Some(true) {
2577 if let Some(flag) = self.delay_aborted.get_mut(usize::from(task.index())) {
2578 *flag = false;
2579 }
2580 self.exit_critical();
2581 return true;
2582 }
2583 let now = self.tick;
2584 let overflows = self.overflows;
2585 let result = match self.tcbs.resolve_mut(task) {
2586 Ok(tcb) => {
2587 let elapsed = now.wrapping_sub(tcb.wait.entering) & Self::MAX_DELAY;
2588 if tcb.wait.ticks == Self::MAX_DELAY {
2589 // An indefinite block never times out.
2590 false
2591 } else if overflows != tcb.wait.overflows && now >= tcb.wait.entering {
2592 tcb.wait.ticks = 0;
2593 true
2594 } else if elapsed < tcb.wait.ticks {
2595 tcb.wait.ticks = tcb.wait.ticks.saturating_sub(elapsed);
2596 tcb.wait.entering = now;
2597 tcb.wait.overflows = overflows;
2598 false
2599 } else {
2600 tcb.wait.ticks = 0;
2601 true
2602 }
2603 }
2604 Err(_) => true,
2605 };
2606 self.exit_critical();
2607 result
2608 }
2609
2610 /// `vTaskPlaceOnUnorderedEventList`: the item value carries the
2611 /// condition rather than the priority, so the item goes on the end and
2612 /// the list is never sorted.
2613 ///
2614 /// `taskEVENT_LIST_ITEM_VALUE_IN_USE` is not set here the way the C
2615 /// sets it. The C needs it because the same `ListItem_t` is a
2616 /// priority-ordered event item the rest of the time and the flag says
2617 /// which; the value is only ever read back by the event-group code,
2618 /// which masks the control byte off, so setting it would change
2619 /// nothing but the arithmetic in the doc comment.
2620 pub(crate) fn place_on_unordered_event_list(
2621 &mut self,
2622 list: ListId,
2623 value: u64,
2624 ticks: u64,
2625 ) -> Result<()> {
2626 let item = Self::event_item(self.current);
2627 self.lists.set_value(item, value)?;
2628 self.lists.insert_end(list, item)?;
2629 self.add_current_task_to_delayed_list(ticks, true)
2630 }
2631
2632 /// `vTaskRemoveFromUnorderedEventList`: write the answer into the item
2633 /// value, then wake the task.
2634 ///
2635 /// Unlike `xTaskRemoveFromEventList` this always goes straight to the
2636 /// ready list. It is only ever called with the scheduler suspended —
2637 /// `xEventGroupSetBits` holds it across the whole walk — and the C
2638 /// still bypasses the pending-ready list, because the value it just
2639 /// wrote is the task's return value and a second pass would overwrite
2640 /// it.
2641 pub(crate) fn remove_from_unordered_event_list(
2642 &mut self,
2643 item: ItemId,
2644 value: u64,
2645 ) -> Result<()> {
2646 self.lists.set_value(item, value)?;
2647 let task = self.task_of_event_item(item)?;
2648 let _ = self.lists.remove(item);
2649 let _ = self.lists.remove(Self::state_item(task));
2650 self.add_task_to_ready_list(task)?;
2651 if self.tcbs.resolve(task)?.priority > self.current_priority() {
2652 self.yield_pending = true;
2653 }
2654 Ok(())
2655 }
2656
2657 /// `uxTaskResetEventItemValue`: read the answer the unblocker left, and
2658 /// put the item back to the priority order an ordinary event list wants.
2659 pub(crate) fn reset_event_item_value(&mut self, task: TaskHandle) -> Result<u64> {
2660 let item = Self::event_item(task);
2661 let value = self.lists.value(item)?;
2662 let priority = self.tcbs.resolve(task)?.priority;
2663 self.lists
2664 .set_value(item, u64::from(C::MAX_PRIORITIES.saturating_sub(priority)))?;
2665 Ok(value)
2666 }
2667
2668 /// Whether `task` is resuming an event-group wait rather than starting
2669 /// one. Taking it clears the marker.
2670 pub(crate) fn take_event_resume(&mut self, task: TaskHandle) -> bool {
2671 match self.tcbs.resolve_mut(task) {
2672 Ok(tcb) if tcb.event_blocked => {
2673 tcb.event_blocked = false;
2674 true
2675 }
2676 _ => false,
2677 }
2678 }
2679
2680 /// Mark that `task` blocked inside an event-group wait, so the call it
2681 /// is inside resumes rather than restarts.
2682 pub(crate) fn set_event_resume(&mut self, task: TaskHandle) {
2683 if let Ok(tcb) = self.tcbs.resolve_mut(task) {
2684 tcb.event_blocked = true;
2685 }
2686 }
2687
2688 /// `vTaskPlaceOnEventList`: sorted by priority, then blocked.
2689 pub(crate) fn place_on_event_list(&mut self, list: ListId, ticks: u64) -> Result<()> {
2690 let current = self.current;
2691 let item = Self::event_item(current);
2692 // The item keeps the value its call set; reading it out only to
2693 // hand it back made the list read the item twice and write it once
2694 // for no change.
2695 self.lists.insert_keeping_value(list, item)?;
2696 self.add_current_task_to_delayed_list(ticks, true)
2697 }
2698
2699 /// Take the trailing `portYIELD()` now, or owe it if a tick already
2700 /// switched us out of this call.
2701 pub(crate) fn yield_or_owe(&mut self, caller: TaskHandle) {
2702 if self.current == caller {
2703 self.port_yield();
2704 } else {
2705 self.owe_yield(caller);
2706 }
2707 }
2708
2709 /// `pvTaskIncrementMutexHeldCount`.
2710 pub(crate) fn increment_mutexes_held(&mut self, task: TaskHandle) {
2711 if let Ok(tcb) = self.tcbs.resolve_mut(task) {
2712 tcb.mutexes_held = tcb.mutexes_held.saturating_add(1);
2713 }
2714 }
2715
2716 // ------------------------------------------------ priority inheritance --
2717
2718 /// `xTaskPriorityInherit`: lift the mutex holder to the waiter's
2719 /// priority. `true` when the holder is (or already was) lifted, which
2720 /// is what the waiter remembers so it can undo it on a timeout.
2721 pub(crate) fn priority_inherit(&mut self, holder: TaskHandle) -> Result<bool> {
2722 if holder.is_null() || !self.tcbs.contains(holder) {
2723 return Ok(false);
2724 }
2725 let waiter_priority = self.current_priority();
2726 let (holder_priority, holder_base) = {
2727 let tcb = self.tcbs.resolve(holder)?;
2728 (tcb.priority, tcb.base_priority)
2729 };
2730 if holder_priority >= waiter_priority {
2731 // Already at least as urgent; the waiter still records that the
2732 // holder is running above its base, if it is.
2733 return Ok(holder_base < waiter_priority);
2734 }
2735 let event_value = u64::from(C::MAX_PRIORITIES.saturating_sub(waiter_priority));
2736 self.lists
2737 .set_value(Self::event_item(holder), event_value)?;
2738 let item = Self::state_item(holder);
2739 if self.lists.container(item)? == Some(Self::ready_list(holder_priority)) {
2740 let _ = self.lists.remove(item);
2741 self.tcbs.resolve_mut(holder)?.priority = waiter_priority;
2742 self.add_task_to_ready_list(holder)?;
2743 } else {
2744 self.tcbs.resolve_mut(holder)?.priority = waiter_priority;
2745 }
2746 self.trace_task(holder, |task, name| Event::TaskPriorityInherit {
2747 task,
2748 name,
2749 priority: Self::priority_value(waiter_priority),
2750 });
2751 Ok(true)
2752 }
2753
2754 /// `xTaskPriorityDisinherit`: giving the mutex back drops the
2755 /// inherited priority. `true` when a yield is wanted.
2756 pub(crate) fn priority_disinherit(&mut self, holder: TaskHandle) -> Result<bool> {
2757 if holder.is_null() || !self.tcbs.contains(holder) {
2758 return Ok(false);
2759 }
2760 let (priority, base, held) = {
2761 let tcb = self.tcbs.resolve(holder)?;
2762 (tcb.priority, tcb.base_priority, tcb.mutexes_held)
2763 };
2764 let remaining = held.saturating_sub(1);
2765 if let Ok(tcb) = self.tcbs.resolve_mut(holder) {
2766 tcb.mutexes_held = remaining;
2767 }
2768 if priority == base || remaining != 0 {
2769 return Ok(false);
2770 }
2771 let item = Self::state_item(holder);
2772 if self.lists.remove(item).is_err() {
2773 return Ok(false);
2774 }
2775 self.trace_task(holder, |task, name| Event::TaskPriorityDisinherit {
2776 task,
2777 name,
2778 priority: Self::priority_value(base),
2779 });
2780 {
2781 let tcb = self.tcbs.resolve_mut(holder)?;
2782 tcb.priority = base;
2783 }
2784 let event_value = u64::from(C::MAX_PRIORITIES.saturating_sub(base));
2785 self.lists
2786 .set_value(Self::event_item(holder), event_value)?;
2787 self.add_task_to_ready_list(holder)?;
2788 Ok(true)
2789 }
2790
2791 /// `vTaskPriorityDisinheritAfterTimeout`: a waiter gave up, so the
2792 /// holder keeps only what the tasks still waiting justify.
2793 pub(crate) fn priority_disinherit_after_timeout(
2794 &mut self,
2795 holder: TaskHandle,
2796 highest_waiting: u8,
2797 ) -> Result<()> {
2798 if holder.is_null() || !self.tcbs.contains(holder) {
2799 return Ok(());
2800 }
2801 let (priority, base, held) = {
2802 let tcb = self.tcbs.resolve(holder)?;
2803 (tcb.priority, tcb.base_priority, tcb.mutexes_held)
2804 };
2805 if priority == base || held != 1 {
2806 return Ok(());
2807 }
2808 let target = if highest_waiting > base {
2809 highest_waiting
2810 } else {
2811 base
2812 };
2813 if priority == target {
2814 return Ok(());
2815 }
2816 self.trace_task(holder, |task, name| Event::TaskPriorityDisinherit {
2817 task,
2818 name,
2819 priority: Self::priority_value(target),
2820 });
2821 self.tcbs.resolve_mut(holder)?.priority = target;
2822 let event_value = u64::from(C::MAX_PRIORITIES.saturating_sub(target));
2823 self.lists
2824 .set_value(Self::event_item(holder), event_value)?;
2825 let item = Self::state_item(holder);
2826 if self.lists.container(item)? == Some(Self::ready_list(priority)) {
2827 let _ = self.lists.remove(item);
2828 self.add_task_to_ready_list(holder)?;
2829 }
2830 Ok(())
2831 }
2832
2833 // ------------------------------------------------------- delay until --
2834
2835 /// `xTaskDelayUntil`: wake at `*previous_wake + period`, whatever the
2836 /// task did in between. `false` when the deadline has already passed,
2837 /// in which case nothing blocks — exactly as the C returns `pdFALSE`.
2838 ///
2839 /// # Errors
2840 /// A list error surfaces as itself.
2841 pub fn delay_until(&mut self, previous_wake: &mut u64, period: u64) -> Result<bool> {
2842 let caller = self.current;
2843 let mut should_delay = false;
2844 self.suspend_all();
2845 {
2846 let now = self.tick;
2847 let wake_at = previous_wake.wrapping_add(period) & Self::MAX_DELAY;
2848 if now < *previous_wake {
2849 // The tick count overflowed since the last wake.
2850 if wake_at < *previous_wake && wake_at > now {
2851 should_delay = true;
2852 }
2853 } else if wake_at < *previous_wake || wake_at > now {
2854 should_delay = true;
2855 }
2856 *previous_wake = wake_at;
2857 if should_delay {
2858 self.trace_task(caller, |task, name| Event::TaskDelayUntil {
2859 task,
2860 name,
2861 wake_at,
2862 });
2863 let ticks = wake_at.wrapping_sub(now) & Self::MAX_DELAY;
2864 self.add_current_task_to_delayed_list(ticks, false)?;
2865 }
2866 }
2867 if !self.resume_all() {
2868 self.yield_or_owe(caller);
2869 }
2870 Ok(should_delay)
2871 }
2872
2873 /// `xTaskRemoveFromEventList`; `true` when the woken task outranks the
2874 /// running one and a yield is therefore required.
2875 pub(crate) fn remove_from_event_list(&mut self, list: ListId) -> Result<bool> {
2876 let Some(item) = self.lists.head(list)? else {
2877 return Ok(false);
2878 };
2879 let task = self.task_of_event_item(item)?;
2880 let _ = self.lists.remove(item);
2881 if self.suspended_depth == 0 {
2882 let _ = self.lists.remove(Self::state_item(task));
2883 self.add_task_to_ready_list(task)?;
2884 } else {
2885 self.lists.insert_end(Self::pending_ready_list(), item)?;
2886 }
2887 let woken = self.tcbs.resolve(task)?.priority;
2888 if woken > self.current_priority() {
2889 self.yield_pending = true;
2890 return Ok(true);
2891 }
2892 Ok(false)
2893 }
2894}