Skip to main content

rusty_rtos_kernel_core/
kernel.rs

1//! The scheduler: `tasks.c` and `queue.c` as a state machine over indices.
2//!
3//! Every public function here is a FreeRTOS API function, and its body
4//! follows the C body statement for statement — including **where the
5//! critical sections open and close**, because on the sim that is where
6//! time passes (`ORACLES.md`, sim contract v1, rule 3), and including where
7//! the `trace*` macros fire, because the order of those lines is the gate.
8//!
9//! The subtlety that costs a diff if you miss it: FreeRTOS yields from
10//! *inside* the critical section in `vTaskResume`, `vTaskPrioritySet`,
11//! `xTaskResumeAll` and the queue calls, and from *outside* it in
12//! `vTaskSuspend` and `vTaskDelay`. Since `portYIELD()` on the Posix port
13//! is itself a critical section, an inside yield nests (and counts one
14//! outermost exit) where an outside yield counts two. Each site below says
15//! which it is.
16//!
17//! Where the C has a `configASSERT`, this has an [`Error`]: the C kernel
18//! stops the world on a bad handle, and the whole point of the remake is
19//! that we do not.
20
21use core::marker::PhantomData;
22
23use rusty_rtos_core::arena::Arena;
24use rusty_rtos_core::config::Config;
25use rusty_rtos_core::error::{Error, Result};
26use rusty_rtos_core::handle::{
27    EventGroup as EventGroupKind, EventGroupHandle, Queue as QueueKind, QueueHandle,
28    StreamBuffer as StreamKind, Task as TaskKind, TaskHandle, Timer as TimerKind, TimerHandle,
29};
30use rusty_rtos_core::hooks::TickHook;
31use rusty_rtos_core::isr::Woken;
32use rusty_rtos_core::list::{ItemId, ListId, Lists};
33use rusty_rtos_core::port::Port;
34use rusty_rtos_core::priority::Priority;
35use rusty_rtos_core::tick::TickWidth;
36use rusty_rtos_core::trace::{Event, Trace};
37
38use crate::events::EventGroup;
39use crate::name::Name;
40use crate::queue::Queue;
41use crate::queue::Wait;
42use crate::stream::StreamBuffer;
43use crate::timer::{MAX_TIMER_COMMANDS, Message, Timer};
44use crate::{OVERHEAD_LISTS, items_for, lists_for};
45
46/// A trace line owed by a task that was switched out before it could
47/// emit one. Only the queue failure paths can owe one: they are the only
48/// places FreeRTOS traces *after* leaving a critical section.
49/// Why `vTaskSwitchContext` could not choose a task to run.
50///
51/// # Law 3: no silent failures
52///
53/// Every variant here was, until 2026-09-11, a bare `return` — the scheduler
54/// declining to switch and saying nothing. Three of them are impossible in a
55/// healthy kernel and the fourth is what the C asserts on
56/// (`configASSERT( uxTopPriority )`), so reaching any of them means an
57/// invariant has already broken somewhere else.
58///
59/// They cost a multi-flash hardware hunt on the day the Xtensa port first
60/// ran a stacked task: every ready list had emptied, the kernel kept
61/// quietly running whoever was current, and nothing said so. A counter and
62/// a reason turn that into one printed line.
63#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
64pub enum Stall {
65    /// The scheduler chose normally.
66    #[default]
67    None,
68    /// No ready task at ANY priority. The idle task keeps priority 0
69    /// non-empty, so reaching this means the idle task is gone — which is
70    /// exactly what `configASSERT( uxTopPriority )` fires on.
71    NoReadyTask,
72    /// A ready list refused to answer at all.
73    ListError,
74    /// The chosen priority's list had nothing to rotate to, having just
75    /// reported itself non-empty.
76    EmptyRotation,
77    /// The chosen list item named no live task.
78    UnknownTask,
79}
80
81#[derive(Debug, Clone, Copy, PartialEq, Eq)]
82pub(crate) enum OwedTrace {
83    /// Nothing owed.
84    None,
85    /// `traceQUEUE_SEND_FAILED`.
86    SendFailed(QueueHandle),
87    /// `traceQUEUE_RECEIVE_FAILED`.
88    ReceiveFailed(QueueHandle),
89    /// `traceTIMER_COMMAND_SEND`, which the C runs on the line *after*
90    /// `xQueueSendToBack` — so a send that made the daemon ready traces
91    /// only once this task has the CPU back.
92    TimerCommandSend {
93        timer: TimerHandle,
94        name: Name,
95        command: i32,
96        value: u64,
97    },
98    /// `traceEVENT_GROUP_WAIT_BITS_END`, which the C runs after the
99    /// `xTaskResumeAll` that ends the wait — so a resume that switched away
100    /// traces once this task has the CPU back.
101    EventGroupWaitBitsEnd {
102        group: EventGroupHandle,
103        bits: u32,
104        timed_out: bool,
105    },
106    /// `prvAddNewTaskToReadyList`, the second half of `xTaskCreate`.
107    ///
108    /// A create costs three outermost exits before it reaches this — two
109    /// `pvPortMalloc`s and the port's `pthread_create` section — and a tick
110    /// can land on any of them. The C is a real thread, so it simply stops
111    /// there and the new task is not on a ready list until the creator runs
112    /// again. This is that pause, and it is the only owed item that is not
113    /// purely a trace line: it carries the ready-list insertion too.
114    AddNewTaskToReadyList { task: TaskHandle, priority: u8 },
115}
116
117/// `eTaskState`: what a task is doing, derived from the list it is in
118/// exactly as `eTaskGetState` derives it.
119#[derive(Debug, Clone, Copy, PartialEq, Eq)]
120pub enum TaskState {
121    /// The task `current` names.
122    Running,
123    /// On a ready list.
124    Ready,
125    /// On a delayed list, or on an event list with a timeout.
126    Blocked,
127    /// On the suspended list with no event pending.
128    Suspended,
129    /// The handle names no live task.
130    Deleted,
131}
132
133/// One task control block: the C `TCB_t` minus everything that is a
134/// pointer. No stack, no TLS, no `pxTopOfStack`.
135#[derive(Debug, Clone, Copy)]
136pub(crate) struct Tcb {
137    name: Name,
138    /// `uxPriority`, the possibly-inherited one the scheduler sorts by.
139    priority: u8,
140    /// `uxBasePriority`, what the task asked for.
141    base_priority: u8,
142    /// `uxMutexesHeld`: how many mutexes this task holds, which is what
143    /// decides whether giving one back may drop an inherited priority.
144    mutexes_held: u32,
145    /// The locals a blocking queue call keeps on the C stack across a
146    /// block. This kernel has no stack, so they live here — see
147    /// [`crate::queue`].
148    wait: WaitFrame,
149    /// `ulNotifiedValue[]`.
150    notified: [u32; MAX_NOTIFICATION_ENTRIES],
151    /// `ucNotifyState[]`.
152    notify_state: [NotifyState; MAX_NOTIFICATION_ENTRIES],
153    /// Whether a stream-buffer call this task made was preempted at the
154    /// critical-section exit that samples the buffer, and must resume
155    /// *after* that exit rather than repeat it.
156    stream_resume: bool,
157    /// The stack local that exit produced — `xBytesAvailable` on a receive,
158    /// `xSpace` on a send — kept because the frame it belonged to is gone.
159    stream_local: usize,
160    /// Whether this task has already blocked inside the notification wait
161    /// it is in. The C blocks inside `xTaskGenericNotifyWait`; a kernel
162    /// with no stacks returns `Blocked` and is called again, so it needs to
163    /// know the second call is a resumption rather than a fresh wait.
164    notify_blocked: bool,
165    /// The same, for the event-group wait this task is inside. Those two
166    /// functions have no retry loop — the C blocks once and then reads the
167    /// event item value — so the second call has to know it is the far side
168    /// of the switch and not a fresh wait.
169    event_blocked: bool,
170}
171
172/// How many notification slots a task has room for.
173///
174/// The C sizes its arrays from `configTASK_NOTIFICATION_ARRAY_ENTRIES`
175/// directly. A `Config` is a type here and stable Rust cannot size an array
176/// from one of its associated consts, so the arrays are the largest a
177/// configuration may ask for and [`Config::validate`] refuses more. The
178/// cost is a few bytes per task per unused slot; the alternative was a
179/// tenth const generic on every kernel type.
180pub const MAX_NOTIFICATION_ENTRIES: usize = 4;
181
182/// `ucNotifyState[]`: what a task's notification slot is doing.
183#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
184pub enum NotifyState {
185    /// `taskNOT_WAITING_NOTIFICATION`.
186    #[default]
187    NotWaiting,
188    /// `taskWAITING_NOTIFICATION`.
189    Waiting,
190    /// `taskNOTIFICATION_RECEIVED`.
191    Received,
192}
193
194/// `eNotifyAction`: what a notification does to the value already there.
195#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
196pub enum NotifyAction {
197    /// `eNoAction`: wake the task, leave the value alone.
198    #[default]
199    None,
200    /// `eSetBits`.
201    SetBits,
202    /// `eIncrement`.
203    Increment,
204    /// `eSetValueWithOverwrite`.
205    Overwrite,
206    /// `eSetValueWithoutOverwrite`: fails if one is already pending.
207    NoOverwrite,
208}
209
210/// What `xQueueGenericSend` and friends keep between passes of their
211/// `for(;;)`: the queue being waited on, the ticks left, and the timeout
212/// bookkeeping (`TimeOut_t`).
213#[derive(Debug, Clone, Copy, Default)]
214struct WaitFrame {
215    /// The queue this task is part-way through a blocking call on;
216    /// `TaskHandle::NULL`-equivalent when there is none.
217    queue: QueueHandle,
218    /// `xTicksToWait`, decremented by each `xTaskCheckForTimeOut`.
219    ticks: u64,
220    /// `xTimeOut.xTimeOnEntering`.
221    entering: u64,
222    /// `xTimeOut.xOverflowCount`.
223    overflows: u64,
224    /// `xEntryTimeSet`.
225    entry_set: bool,
226    /// `xInheritanceOccurred`.
227    inherited: bool,
228}
229
230/// What [`Kernel::start_scheduler`] created, so a runner can attach bodies.
231#[derive(Debug, Clone, Copy)]
232pub struct StartHandles {
233    /// The idle task.
234    pub idle: TaskHandle,
235    /// The timer service task.
236    pub timer: TaskHandle,
237    /// The timer command queue.
238    pub timer_queue: QueueHandle,
239}
240
241/// The scheduler.
242///
243/// `TASKS`, `ITEMS`, `LISTS`, `QUEUES`, `SLOTS`, `BUFFERS` and `BYTES` are
244/// the geometry: how many tasks, list items, lists, queues, queue item
245/// slots, stream buffers, and stream-buffer bytes this kernel has room
246/// for. See the crate docs and [`items_for`] / [`lists_for`].
247/// [`Kernel::new`] refuses a geometry that does not add up.
248pub struct Kernel<
249    C: Config,
250    P: Port,
251    T: Trace,
252    H,
253    const TASKS: usize,
254    const ITEMS: usize,
255    const LISTS: usize,
256    const QUEUES: usize,
257    const SLOTS: usize,
258    const BUFFERS: usize,
259    const BYTES: usize,
260    const TIMERS: usize,
261    const GROUPS: usize,
262> {
263    pub(crate) port: P,
264    pub(crate) trace: T,
265    pub(crate) tcbs: Arena<TaskKind, Tcb, TASKS>,
266    pub(crate) queues: Arena<QueueKind, Queue, QUEUES>,
267    pub(crate) lists: Lists<ITEMS, LISTS>,
268    pub(crate) slots: [u64; SLOTS],
269    pub(crate) buffers: Arena<StreamKind, StreamBuffer, BUFFERS>,
270    pub(crate) timers: Arena<TimerKind, Timer, TIMERS>,
271    pub(crate) groups: Arena<EventGroupKind, EventGroup, GROUPS>,
272    /// The ring of `DaemonTaskMessage_t`s the timer queue carries indices
273    /// into. It is exactly as long as the queue, so a message can only be
274    /// overwritten once the queue has already refused to hold its index.
275    pub(crate) timer_messages: [Message; MAX_TIMER_COMMANDS],
276    pub(crate) timer_message_next: usize,
277    /// Which of the two timer lists is `pxCurrentTimerList` right now.
278    pub(crate) timers_swapped: bool,
279    /// `xLastTime` in `prvSampleTimeNow`.
280    pub(crate) timer_last_time: u64,
281    /// `xTimerQueue`.
282    pub(crate) timer_queue: QueueHandle,
283    pub(crate) bytes: [u8; BYTES],
284    pub(crate) bytes_used: usize,
285    /// Blocks of the byte arena that a deleted stream buffer gave back,
286    /// as `(base, length)`, kept sorted and coalesced.
287    ///
288    /// This is the one allocator in the kernel, and it exists because the
289    /// C's stream buffers are heap objects: `MessageBufferDemo`'s echo
290    /// server creates one and deletes it again on every loop, and a bump
291    /// allocator would run out in a few hundred ticks. At most `BUFFERS`
292    /// buffers can be alive, so at most `BUFFERS` holes can exist between
293    /// them, which is why the list is that long and cannot overflow.
294    pub(crate) free_blocks: [(usize, usize); BUFFERS],
295    pub(crate) free_count: usize,
296    pub(crate) slots_used: usize,
297    /// `pxCurrentTCB`.
298    pub(crate) current: TaskHandle,
299    /// `uxTopReadyPriority`.
300    top_ready_priority: u8,
301    /// `xTickCount`, masked to the configuration's tick width.
302    pub(crate) tick: u64,
303    /// `xPendedTicks`.
304    pended_ticks: u64,
305    /// `uxSchedulerSuspended`.
306    suspended_depth: u32,
307    /// `xSchedulerRunning`.
308    running: bool,
309    /// `xNextTaskUnblockTime`.
310    next_unblock_time: u64,
311    /// `xYieldPendings[0]`.
312    yield_pending: bool,
313    /// `uxCurrentNumberOfTasks`.
314    task_count: usize,
315    /// How many times the scheduler could not choose a task, and why the
316    /// FIRST time. See [`Stall`].
317    stalls: u32,
318    first_stall: Stall,
319    /// A task that deleted ITSELF and whose slot the idle task has not
320    /// reclaimed yet — the C's `xTasksWaitingTermination`, which only ever
321    /// holds the running task, because any other task is freed on the spot.
322    ///
323    /// One slot is enough, and that is a property rather than a guess: a
324    /// self-deleting task is off every ready list and yields immediately, so
325    /// a second self-delete cannot happen until a switch has occurred, and
326    /// the switch is where this is reaped.
327    awaiting_reap: Option<TaskHandle>,
328    /// Which of the two delayed lists is `pxDelayedTaskList` right now.
329    delayed_swapped: bool,
330    /// `xNumOfOverflows`.
331    overflows: u64,
332    /// Which tasks have been switched in at least once. A task's first
333    /// switch-in is where a real port hands it a fresh stack, so the
334    /// outgoing task's open critical sections are abandoned there
335    /// (`Port::reset_nesting_for_first_start`).
336    started: [bool; TASKS],
337    /// A task that was switched out *in the middle of a kernel call* still
338    /// owes the tail of that call — in practice always the `portYIELD()`
339    /// that `vTaskDelay` and `vTaskSuspend` make after their critical
340    /// sections. On a real port the tail simply sits on the task's frozen
341    /// stack; here the stack is a program counter, so the debt is recorded
342    /// and paid by [`Kernel::resume_pending`] when the task runs again.
343    owes_yield: [bool; TASKS],
344    /// A trace line a task owes from a call it was switched out of.
345    ///
346    /// `xQueueReceive`'s failure path is `taskEXIT_CRITICAL();
347    /// traceQUEUE_RECEIVE_FAILED( pxQueue ); return errQUEUE_EMPTY;` — and
348    /// if the tick that the exit released switches the task away, those two
349    /// lines sit on a stack that is not running. The C emits the trace when
350    /// the task resumes, so this kernel does too.
351    owed_trace: [OwedTrace; TASKS],
352    /// How many outermost critical-section exits a task owes the clock.
353    ///
354    /// This is `uxSavedCriticalNesting` in the Posix port's
355    /// `prvSwitchThread` and then some. A thread stops at the switch; a
356    /// stackless call does not, so the abandoned frame runs on — closing
357    /// the sections it had open and, on some paths, opening one more. The
358    /// port tallies every exit that frame makes instead of counting it as
359    /// sim time, and the tally is paid here when the task runs again. That
360    /// is what puts a sim tick where the C one lands.
361    owed_exits: [u32; TASKS],
362    /// The task whose abandoned frame is running right now, if any.
363    unwinding: Option<TaskHandle>,
364    /// `vApplicationTickHook`, held by value so it can borrow the kernel.
365    pub(crate) tick_hook: H,
366    /// `ucDelayAborted`: the task was pulled out of the Blocked state by
367    /// [`Kernel::abort_delay`] rather than by its own block time running
368    /// out, so it must not re-evaluate that block time and block again.
369    delay_aborted: [bool; TASKS],
370    _config: PhantomData<C>,
371}
372
373impl<
374    C: Config,
375    P: Port,
376    T: Trace,
377    H,
378    const TASKS: usize,
379    const ITEMS: usize,
380    const LISTS: usize,
381    const QUEUES: usize,
382    const SLOTS: usize,
383    const BUFFERS: usize,
384    const BYTES: usize,
385    const TIMERS: usize,
386    const GROUPS: usize,
387> Kernel<C, P, T, H, TASKS, ITEMS, LISTS, QUEUES, SLOTS, BUFFERS, BYTES, TIMERS, GROUPS>
388where
389    H: TickHook<Self>,
390{
391    /// `portMAX_DELAY` at this configuration's tick width.
392    pub const MAX_DELAY: u64 = <C::Tick as TickWidth>::MAX;
393
394    // ------------------------------------------------------------ setup --
395
396    /// A kernel with no tasks, ready for [`Kernel::create_task`].
397    ///
398    /// # Errors
399    /// [`Error::InvalidArgument`] if the configuration is invalid
400    /// ([`Config::validate`]), if the name capacity cannot hold
401    /// `MAX_TASK_NAME_LEN`, or if the const geometry does not match the
402    /// configuration: `ITEMS` must be [`items_for`]`(TASKS)` and `LISTS`
403    /// must be [`lists_for`]`(MAX_PRIORITIES, QUEUES)`.
404    pub fn new(port: P, trace: T) -> Result<Self>
405    where
406        H: Default,
407    {
408        Self::with_tick_hook(port, trace, H::default())
409    }
410
411    /// [`Kernel::new`] with `vApplicationTickHook` installed.
412    ///
413    /// # Errors
414    /// As [`Kernel::new`].
415    pub fn with_tick_hook(port: P, trace: T, tick_hook: H) -> Result<Self> {
416        C::validate()?;
417        if ITEMS != items_for(TASKS, TIMERS)
418            || LISTS != lists_for(C::MAX_PRIORITIES, QUEUES, GROUPS)
419            || TASKS == 0
420            || C::MAX_TASK_NAME_LEN > crate::NAME_CAPACITY
421        {
422            return Err(Error::InvalidArgument);
423        }
424        Ok(Self {
425            port,
426            trace,
427            tcbs: Arena::new(),
428            queues: Arena::new(),
429            lists: Lists::new(),
430            slots: [0; SLOTS],
431            slots_used: 0,
432            buffers: Arena::new(),
433            timers: Arena::new(),
434            groups: Arena::new(),
435            timer_messages: [Message::default(); MAX_TIMER_COMMANDS],
436            timer_message_next: 0,
437            timers_swapped: false,
438            timer_last_time: 0,
439            timer_queue: QueueHandle::NULL,
440            bytes: [0; BYTES],
441            bytes_used: 0,
442            free_blocks: [(0, 0); BUFFERS],
443            free_count: 0,
444            current: TaskHandle::NULL,
445            top_ready_priority: 0,
446            tick: C::INITIAL_TICK_COUNT,
447            pended_ticks: 0,
448            suspended_depth: 0,
449            running: false,
450            next_unblock_time: Self::MAX_DELAY,
451            yield_pending: false,
452            task_count: 0,
453            stalls: 0,
454            first_stall: Stall::None,
455            awaiting_reap: None,
456            delayed_swapped: false,
457            overflows: 0,
458            started: [false; TASKS],
459            owes_yield: [false; TASKS],
460            owed_exits: [0; TASKS],
461            unwinding: None,
462            tick_hook,
463            delay_aborted: [false; TASKS],
464            owed_trace: [OwedTrace::None; TASKS],
465            _config: PhantomData,
466        })
467    }
468
469    /// The port, for a runner that needs its counters.
470    pub const fn port(&self) -> &P {
471        &self.port
472    }
473
474    /// The trace sink.
475    pub const fn trace(&self) -> &T {
476        &self.trace
477    }
478
479    /// The trace sink, mutably.
480    pub const fn trace_mut(&mut self) -> &mut T {
481        &mut self.trace
482    }
483
484    /// Take the trace sink back when the run is over, so a deliverable can
485    /// flush whatever it was writing to.
486    pub fn into_trace(self) -> T {
487        self.trace
488    }
489
490    /// `xNumOfOverflows`, for a scenario that reports counters.
491    #[must_use]
492    pub const fn overflows(&self) -> u64 {
493        self.overflows
494    }
495
496    // ------------------------------------------------------ list indices --
497
498    // `ListId` is a `u8` and `ItemId` a `u16` (the core's `list`): the
499    // whole scheduler addresses lists and items by small integers, which is
500    // what keeps a TCB free of pointers.
501
502    pub(crate) const fn ready_list(priority: u8) -> ListId {
503        priority
504    }
505
506    const fn delayed_list(&self) -> ListId {
507        if self.delayed_swapped {
508            C::MAX_PRIORITIES.saturating_add(1)
509        } else {
510            C::MAX_PRIORITIES
511        }
512    }
513
514    const fn overflow_delayed_list(&self) -> ListId {
515        if self.delayed_swapped {
516            C::MAX_PRIORITIES
517        } else {
518            C::MAX_PRIORITIES.saturating_add(1)
519        }
520    }
521
522    const fn pending_ready_list() -> ListId {
523        C::MAX_PRIORITIES.saturating_add(2)
524    }
525
526    const fn suspended_list() -> ListId {
527        C::MAX_PRIORITIES.saturating_add(3)
528    }
529
530    /// `xTasksWaitingToSend` of a queue.
531    pub(crate) fn queue_send_list(queue: QueueHandle) -> ListId {
532        let offset = u8::try_from(queue.index().saturating_mul(2)).unwrap_or(u8::MAX);
533        C::MAX_PRIORITIES
534            .saturating_add(u8::try_from(OVERHEAD_LISTS).unwrap_or(u8::MAX))
535            .saturating_add(offset)
536    }
537
538    /// `xTasksWaitingToReceive` of a queue.
539    pub(crate) fn queue_receive_list(queue: QueueHandle) -> ListId {
540        Self::queue_send_list(queue).saturating_add(1)
541    }
542
543    /// A task's `xStateListItem`: the task's own arena index.
544    pub(crate) const fn state_item(task: TaskHandle) -> ItemId {
545        task.index()
546    }
547
548    /// A task's `xEventListItem`: `TASKS` above its state item, so the two
549    /// never collide and either maps back to its task by arithmetic alone.
550    pub(crate) fn event_item(task: TaskHandle) -> ItemId {
551        Self::task_item_base().saturating_add(task.index())
552    }
553
554    fn task_item_base() -> ItemId {
555        u16::try_from(TASKS).unwrap_or(u16::MAX)
556    }
557
558    fn task_of_state_item(&self, item: ItemId) -> Result<TaskHandle> {
559        self.tcbs.handle_at(item).ok_or(Error::Gone)
560    }
561
562    fn task_of_event_item(&self, item: ItemId) -> Result<TaskHandle> {
563        let index = item
564            .checked_sub(Self::task_item_base())
565            .ok_or(Error::InvalidArgument)?;
566        self.tcbs.handle_at(index).ok_or(Error::Gone)
567    }
568
569    // ----------------------------------------------------------- getters --
570
571    /// `pxCurrentTCB`.
572    #[must_use]
573    pub const fn current(&self) -> TaskHandle {
574        self.current
575    }
576
577    /// `xTaskGetTickCount`.
578    #[must_use]
579    pub const fn tick_count(&self) -> u64 {
580        self.tick
581    }
582
583    /// `xSchedulerRunning`.
584    #[must_use]
585    pub const fn is_running(&self) -> bool {
586        self.running
587    }
588
589    /// `uxCurrentNumberOfTasks`.
590    #[must_use]
591    pub const fn task_count(&self) -> usize {
592        self.task_count
593    }
594
595    /// `uxSchedulerSuspended`, for a stepper reporting why a tick pended.
596    #[must_use]
597    pub const fn scheduler_suspended(&self) -> u32 {
598        self.suspended_depth
599    }
600
601    /// `xPendedTicks`, likewise.
602    #[must_use]
603    pub const fn pended_ticks(&self) -> u64 {
604        self.pended_ticks
605    }
606
607    /// The item ids in `priority`'s ready list, in order, into `out`.
608    ///
609    /// Returns how many were written. The companion to
610    /// [`Kernel::ready_cursor`]: when a ready task is never chosen, the
611    /// cursor says whether the rotation moved and this says what it was
612    /// moving THROUGH.
613    pub fn ready_items(&self, priority: u8, out: &mut [u16]) -> usize {
614        let mut n = 0;
615        for item in self.lists.iter(Self::ready_list(priority)) {
616            let Some(slot) = out.get_mut(n) else {
617                break;
618            };
619            *slot = item;
620            n = n.saturating_add(1);
621        }
622        n
623    }
624
625    /// Where the round-robin cursor sits in `priority`'s ready list.
626    ///
627    /// A diagnostic, paired with [`Kernel::ready_len`]: a ready task that
628    /// is never chosen is either not in the list the scheduler looks at,
629    /// or the cursor is not moving. These two answer both halves.
630    #[must_use]
631    pub fn ready_cursor(&self, priority: u8) -> u16 {
632        self.lists
633            .cursor_of(Self::ready_list(priority))
634            .unwrap_or(0)
635    }
636
637    /// The live handle in arena slot `index`, if that slot holds a task.
638    ///
639    /// The generation is the point: a bare index names a SLOT, and a slot
640    /// outlives the tasks that occupy it, so an index is not a task. This
641    /// answers the handle, which is.
642    ///
643    /// It exists so a diagnostic can walk every task without knowing any
644    /// of their names -- `task_get_handle` needs the name it is looking
645    /// for, and a crash report is exactly the case where you do not have
646    /// one.
647    #[must_use]
648    pub fn task_at(&self, index: usize) -> Option<TaskHandle> {
649        let index = u16::try_from(index).ok()?;
650        self.tcbs.handle_at(index)
651    }
652
653    /// A task's name.
654    ///
655    /// # Errors
656    /// [`Error::Gone`] for a stale or null handle.
657    pub fn name_of(&self, task: TaskHandle) -> Result<Name> {
658        self.tcbs.resolve(task).map(|t| t.name)
659    }
660
661    /// `uxTaskPriorityGet`. `None` means the current task, as in C.
662    ///
663    /// # Errors
664    /// [`Error::Gone`] for a stale handle.
665    pub fn priority_of(&self, task: Option<TaskHandle>) -> Result<u8> {
666        let h = task.unwrap_or(self.current);
667        self.tcbs.resolve(h).map(|t| t.priority)
668    }
669
670    pub(crate) fn current_priority(&self) -> u8 {
671        self.priority_of(None).unwrap_or(0)
672    }
673
674    /// `uxTaskPriorityGet`: the C API, which takes a critical section.
675    ///
676    /// The distinction matters: on the sim a critical-section exit is where
677    /// time passes, so reading a priority through the API costs a tick
678    /// sixteen calls later, exactly as it does in C
679    /// (`portBASE_TYPE_ENTER_CRITICAL` is `taskENTER_CRITICAL` on the Posix
680    /// port). The kernel's own reads use the field directly, as C's do.
681    ///
682    /// # Errors
683    /// [`Error::Gone`] for a stale handle.
684    pub fn task_priority_get(&mut self, task: Option<TaskHandle>) -> Result<u8> {
685        self.enter_critical();
686        let result = self.priority_of(task);
687        self.exit_critical();
688        result
689    }
690
691    /// `eTaskGetState`: the C API, which takes a critical section for any
692    /// task but the running one.
693    ///
694    /// # Errors
695    /// A list error surfaces as itself.
696    pub fn task_state_get(&mut self, task: TaskHandle) -> Result<TaskState> {
697        if task == self.current {
698            return Ok(TaskState::Running);
699        }
700        self.enter_critical();
701        let result = self.state_of(task);
702        self.exit_critical();
703        result
704    }
705
706    /// `eTaskGetState`, derived from the list the task is in.
707    ///
708    /// # Errors
709    /// A list error surfaces as itself.
710    pub fn state_of(&self, task: TaskHandle) -> Result<TaskState> {
711        if !self.tcbs.contains(task) {
712            return Ok(TaskState::Deleted);
713        }
714        if task == self.current {
715            return Ok(TaskState::Running);
716        }
717        let Some(list) = self.lists.container(Self::state_item(task))? else {
718            return Ok(TaskState::Deleted);
719        };
720        if list == Self::suspended_list() {
721            if self.lists.container(Self::event_item(task))?.is_some() {
722                return Ok(TaskState::Blocked);
723            }
724            // A task blocked indefinitely on a notification is in the
725            // suspended list and on no event list at all, so the C scans
726            // the notification array before it calls such a task suspended.
727            // Miss this and a stream buffer's reader looks suspended to
728            // `eTaskGetState` while it is plainly waiting.
729            let waiting_notification = self
730                .tcbs
731                .resolve(task)
732                .map(|t| {
733                    t.notify_state
734                        .iter()
735                        .take(C::NOTIFICATION_ARRAY_ENTRIES)
736                        .any(|s| *s == NotifyState::Waiting)
737                })
738                .unwrap_or(false);
739            return Ok(if waiting_notification {
740                TaskState::Blocked
741            } else {
742                TaskState::Suspended
743            });
744        }
745        if list == self.delayed_list() || list == self.overflow_delayed_list() {
746            return Ok(TaskState::Blocked);
747        }
748        Ok(TaskState::Ready)
749    }
750
751    /// How many tasks sit on a priority's ready list — what the idle task
752    /// checks before yielding (`configIDLE_SHOULD_YIELD`).
753    ///
754    /// # Errors
755    /// [`Error::InvalidPriority`] for a priority outside the configuration.
756    pub fn ready_len(&self, priority: u8) -> Result<usize> {
757        if priority >= C::MAX_PRIORITIES {
758            return Err(Error::InvalidPriority);
759        }
760        self.lists.len(Self::ready_list(priority))
761    }
762
763    /// Record that the scheduler could not choose.
764    ///
765    /// Counts every occurrence and keeps the FIRST reason: the first is the
766    /// diagnosis and the rest are consequences.
767    fn note_stall(&mut self, why: Stall) {
768        self.stalls = self.stalls.saturating_add(1);
769        if self.first_stall == Stall::None {
770            self.first_stall = why;
771        }
772    }
773
774    /// How many times the scheduler could not choose a task.
775    ///
776    /// **Non-zero means an invariant broke.** A healthy kernel never stalls:
777    /// the idle task is always ready. A firmware that prints this alongside
778    /// its own counters turns a hang into a sentence.
779    #[must_use]
780    pub const fn stalls(&self) -> u32 {
781        self.stalls
782    }
783
784    /// Why the scheduler first failed to choose. [`Stall::None`] if it never
785    /// has.
786    #[must_use]
787    pub const fn first_stall(&self) -> Stall {
788        self.first_stall
789    }
790
791    // ------------------------------------------------------------ tracing --
792
793    pub(crate) fn trace_task<F>(&mut self, task: TaskHandle, make: F)
794    where
795        F: for<'a> FnOnce(TaskHandle, &'a str) -> Event<'a>,
796    {
797        let name = self.tcbs.get(task).map(|t| t.name).unwrap_or_default();
798        let tick = self.tick;
799        self.trace.note_exits(self.port.exits());
800        self.trace.event(tick, make(task, name.as_str()));
801    }
802
803    pub(crate) fn priority_value(raw: u8) -> Priority {
804        Priority::new(raw, C::MAX_PRIORITIES).unwrap_or(Priority::IDLE)
805    }
806
807    // --------------------------------------------------- critical section --
808
809    /// What `pvPortMalloc` costs in sim time on a heap-backed C kernel:
810    /// `vTaskSuspendAll()` around the allocation, then `xTaskResumeAll()`,
811    /// whose critical section is one outermost exit.
812    ///
813    /// This kernel allocates nothing, so the call is the whole of it. On a
814    /// configuration with [`Config::DYNAMIC_ALLOCATION`] off it compiles
815    /// away to nothing at all.
816    pub(crate) fn account_for_allocation(&mut self) {
817        if C::DYNAMIC_ALLOCATION {
818            self.suspend_all();
819            let _ = self.resume_all();
820        }
821    }
822
823    /// `taskENTER_CRITICAL()`.
824    pub fn enter_critical(&mut self) {
825        self.port.enter_critical();
826    }
827
828    /// `taskEXIT_CRITICAL()`.
829    ///
830    /// On the sim this is where time passes: the port raises a tick on
831    /// every sixteenth outermost exit and the kernel takes it here, which
832    /// is exactly where the C kernel's `SIGALRM` handler would have run.
833    pub fn exit_critical(&mut self) {
834        self.port.exit_critical();
835        if self.port.take_pending_tick() {
836            self.tick_from_isr();
837        }
838    }
839
840    /// `vPortSystemTickHandler`: the tick, from interrupt context.
841    ///
842    /// A silicon port calls this from its timer interrupt. The sim reaches
843    /// it from [`Kernel::exit_critical`] and from the idle hook.
844    pub fn tick_from_isr(&mut self) {
845        self.port.set_in_tick_entry(true);
846        self.port.count_tick();
847        // The C handler bumps `uxCriticalNesting` by hand and drops it the
848        // same way — a raw `--`, never `vPortExitCritical` — so the bump is
849        // invisible to the exit count and to anything that unwinds later.
850        // Modelling it as a real critical section would invent an exit at
851        // every tick that caused a switch.
852        if self.increment_tick() {
853            self.switch_context();
854        }
855        self.port.set_in_tick_entry(false);
856    }
857
858    /// The idle hook's tick, `vPortKairosTick()` on the C side: a critical
859    /// section around one unconditional tick. The enclosing section counts
860    /// toward rule 3 exactly as the C one does.
861    pub fn idle_hook_tick(&mut self) {
862        self.enter_critical();
863        self.tick_from_isr();
864        self.exit_critical();
865    }
866
867    /// `portYIELD()` as the Posix port spells it: a critical section around
868    /// the context switch, so a tick raised during it lands on the way out.
869    pub(crate) fn port_yield(&mut self) {
870        self.enter_critical();
871        self.port.count_yield();
872        if P::COMMITS_SWITCH {
873            // A STACKED port. Raise its switching exception and leave
874            // `current` where it is: the exception will call
875            // `switch_context` itself, so the decision and the register
876            // swap happen together and nothing runs in between as a task
877            // the kernel has already moved on from.
878            //
879            // The exception cannot fire yet -- `exit_critical` below is
880            // what unmasks -- so the kernel is consistent at the moment it
881            // is taken.
882            self.port.yield_now();
883        } else {
884            // A STACKLESS kernel: moving `current` IS the switch, because
885            // no task owns a stack. This is the path every corpus
886            // architecture takes, and it is unchanged.
887            self.switch_context();
888        }
889        self.exit_critical();
890    }
891
892    /// `taskYIELD()` from a task body (the idle task's yield).
893    pub fn task_yield(&mut self) {
894        self.port_yield();
895    }
896
897    // -------------------------------------------------------- task create --
898
899    /// `xTaskCreate`.
900    ///
901    /// # Errors
902    /// [`Error::Full`] when the task arena is full (the C
903    /// `errCOULD_NOT_ALLOCATE_REQUIRED_MEMORY`).
904    pub fn create_task(&mut self, name: &str, priority: u8) -> Result<TaskHandle> {
905        let caller = self.current;
906        // configASSERT( uxPriority < configMAX_PRIORITIES ), then C clamps.
907        let priority = priority.min(C::MAX_PRIORITIES.saturating_sub(1));
908        // `prvCreateTask` takes the stack and then the TCB from the heap
909        // before anything is initialised — `portSTACK_GROWTH` is negative on
910        // the oracle's port, which fixes that order — and `pvPortMallocStack`
911        // *is* `pvPortMalloc` with the MPU wrappers off. On heap_3 each one
912        // suspends the scheduler, so a create costs two outermost exits.
913        //
914        // Every scenario before `death` created its tasks BEFORE
915        // `vTaskStartScheduler`, and the sim port counts no exits there (the
916        // C patch counts only on a FreeRTOS thread), so this cost was
917        // invisible to the whole corpus. `death` is the first scenario that
918        // creates a task with the scheduler running, and the first that can
919        // tell the difference.
920        self.account_for_allocation();
921        self.account_for_allocation();
922        let tcb = Tcb {
923            name: Name::new(name, C::MAX_TASK_NAME_LEN),
924            priority,
925            base_priority: priority,
926            mutexes_held: 0,
927            wait: WaitFrame::default(),
928            notified: [0; MAX_NOTIFICATION_ENTRIES],
929            notify_state: [NotifyState::NotWaiting; MAX_NOTIFICATION_ENTRIES],
930            notify_blocked: false,
931            event_blocked: false,
932            stream_resume: false,
933            stream_local: 0,
934        };
935        let handle = match self.tcbs.try_insert(tcb) {
936            Ok(h) => h,
937            Err(_) => {
938                let tick = self.tick;
939                self.trace.note_exits(self.port.exits());
940                self.trace.event(tick, Event::TaskCreateFailed);
941                return Err(Error::Full);
942            }
943        };
944        // The event item sorts by `configMAX_PRIORITIES - uxPriority`, so a
945        // higher-priority task waits nearer the head of an event list.
946        let event_value = u64::from(C::MAX_PRIORITIES.saturating_sub(priority));
947        self.lists
948            .set_value(Self::event_item(handle), event_value)?;
949        // `prvInitialiseNewTask` ends by calling `pxPortInitialiseStack`,
950        // which on the oracle's Posix port wraps `pthread_create` in a
951        // critical section — one more exit, and a PORT cost rather than a
952        // kernel one, which is why it has a flag of its own.
953        if C::PORT_STACK_INIT_CRITICAL {
954            self.enter_critical();
955            self.exit_critical();
956        }
957        // Any of the three exits above can release a tick that switches the
958        // creator out. In the C the creator is a thread and simply STOPS
959        // there: `prvAddNewTaskToReadyList` has not run, so the new task is
960        // on no ready list and `traceTASK_CREATE` has not fired. Running it
961        // now would put both events on the wrong side of the switch — which
962        // is precisely how `death` first diverged.
963        if self.current != caller {
964            if let Some(slot) = self.owed_trace.get_mut(usize::from(caller.index())) {
965                *slot = OwedTrace::AddNewTaskToReadyList {
966                    task: handle,
967                    priority,
968                };
969            }
970            return Ok(handle);
971        }
972        self.add_new_task_to_ready_list(handle, priority)?;
973        Ok(handle)
974    }
975
976    /// `prvAddNewTaskToReadyList`.
977    fn add_new_task_to_ready_list(&mut self, task: TaskHandle, priority: u8) -> Result<()> {
978        self.enter_critical();
979        {
980            self.task_count = self.task_count.saturating_add(1);
981            if self.current.is_null() {
982                self.current = task;
983            } else if !self.running {
984                // `<=`, so the last-created task of the highest priority is
985                // the one that runs first.
986                if self.current_priority() <= priority {
987                    self.current = task;
988                }
989            }
990            self.trace_task(task, |task, name| Event::TaskCreate {
991                task,
992                name,
993                priority: Self::priority_value(priority),
994            });
995            self.add_task_to_ready_list(task)?;
996        }
997        self.exit_critical();
998        // taskYIELD_IF_USING_PREEMPTION(), outside the section.
999        if self.running && C::USE_PREEMPTION && self.current_priority() < priority {
1000            self.port_yield();
1001        }
1002        Ok(())
1003    }
1004
1005    /// `prvAddTaskToReadyList`.
1006    pub(crate) fn add_task_to_ready_list(&mut self, task: TaskHandle) -> Result<()> {
1007        let priority = self.tcbs.resolve(task)?.priority;
1008        self.trace_task(task, |task, name| Event::MovedTaskToReadyState {
1009            task,
1010            name,
1011        });
1012        // taskRECORD_READY_PRIORITY
1013        if priority > self.top_ready_priority {
1014            self.top_ready_priority = priority;
1015        }
1016        self.lists
1017            .insert_end(Self::ready_list(priority), Self::state_item(task))
1018    }
1019
1020    // ---------------------------------------------------- scheduler start --
1021
1022    /// `vTaskStartScheduler`, minus the part that never returns.
1023    ///
1024    /// Creates the idle task, the timer command queue and the timer service
1025    /// task, fires `TASK_SWITCHED_IN` for whichever task will run first and
1026    /// then `STARTING_SCHEDULER`, and returns the handles so the runner can
1027    /// attach their bodies.
1028    ///
1029    /// # Errors
1030    /// As [`Kernel::create_task`].
1031    pub fn start_scheduler(&mut self) -> Result<StartHandles> {
1032        let idle = self.create_task("IDLE", 0)?;
1033        // `xTimerCreateTimerTask` opens with `prvCheckForValidListAndQueue`,
1034        // which makes the command queue only if no `xTimerCreate` has made
1035        // it already.
1036        self.check_for_valid_list_and_queue()?;
1037        let timer_queue = self.timer_queue;
1038        let timer = self.create_task("Tmr Svc", C::TIMER_TASK_PRIORITY)?;
1039        self.next_unblock_time = Self::MAX_DELAY;
1040        self.running = true;
1041        self.tick = C::INITIAL_TICK_COUNT;
1042        self.port.scheduler_started();
1043        // `vPortStartFirstTask` runs the first task through the same
1044        // first-start path as every other.
1045        let current = self.current;
1046        self.note_first_start(current);
1047        self.trace_task(current, |task, name| Event::TaskSwitchedIn { task, name });
1048        let tick = self.tick;
1049        self.trace.note_exits(self.port.exits());
1050        self.trace.event(tick, Event::StartingScheduler);
1051        Ok(StartHandles {
1052            idle,
1053            timer,
1054            timer_queue,
1055        })
1056    }
1057
1058    // -------------------------------------------------------- the switch --
1059
1060    /// `vTaskSwitchContext`.
1061    pub fn switch_context(&mut self) {
1062        if self.suspended_depth != 0 {
1063            self.yield_pending = true;
1064            return;
1065        }
1066        self.yield_pending = false;
1067        let current = self.current;
1068        self.trace_task(current, |task, name| Event::TaskSwitchedOut { task, name });
1069        // taskSELECT_HIGHEST_PRIORITY_TASK
1070        let mut top = self.top_ready_priority;
1071        loop {
1072            match self.lists.is_empty(Self::ready_list(top)) {
1073                Ok(false) => break,
1074                Ok(true) => {
1075                    if top == 0 {
1076                        // The C `configASSERT( uxTopPriority )`. The idle
1077                        // task keeps priority 0 non-empty, so reaching here
1078                        // means it is gone. The kernel keeps the current
1079                        // task rather than indexing out of range -- and now
1080                        // SAYS so, because carrying on quietly is how this
1081                        // costs an afternoon on a board.
1082                        self.note_stall(Stall::NoReadyTask);
1083                        return;
1084                    }
1085                    top = top.saturating_sub(1);
1086                }
1087                Err(_) => {
1088                    self.note_stall(Stall::ListError);
1089                    return;
1090                }
1091            }
1092        }
1093        let Ok(Some(item)) = self.lists.next_round_robin(Self::ready_list(top)) else {
1094            self.note_stall(Stall::EmptyRotation);
1095            return;
1096        };
1097        let Ok(next) = self.task_of_state_item(item) else {
1098            self.note_stall(Stall::UnknownTask);
1099            return;
1100        };
1101        self.top_ready_priority = top;
1102        if next != current {
1103            self.hand_over(current, next);
1104        }
1105        self.current = next;
1106        self.trace_task(next, |task, name| Event::TaskSwitchedIn { task, name });
1107    }
1108
1109    // ----------------------------------------------------------- the tick --
1110
1111    /// Pay whatever the running task still owes from a call it was
1112    /// preempted inside. The runner calls this before stepping a body;
1113    /// `true` means work was done and the current task may have changed
1114    /// again, so nothing else should be assumed.
1115    pub fn resume_pending(&mut self) -> bool {
1116        self.settle_unwind();
1117        let index = usize::from(self.current.index());
1118        // First the stack unwinds — the sections the task had open when it
1119        // was switched out, and whatever its abandoned frame opened after
1120        // that — and only then does the statement after the yield run.
1121        // Reversing the two moves every tick.
1122        let owed = self.owed_exits.get(index).copied().unwrap_or(0);
1123        if owed > 0 {
1124            if let Some(slot) = self.owed_exits.get_mut(index) {
1125                *slot = 0;
1126            }
1127            // One counted exit each: the tally is already in outermost
1128            // exits, so replaying it as nesting would lose the sections the
1129            // tail opened and closed on its own.
1130            for _ in 0..owed {
1131                self.enter_critical();
1132                self.exit_critical();
1133                // A replayed exit can release a tick that switches this
1134                // task straight back out. The rest of the loop is then the
1135                // tail of *this* frame, which the port tallies and the next
1136                // resume pays — so there is nothing to unwind by hand.
1137            }
1138            return true;
1139        }
1140        // Then the line the abandoned frame had not reached yet.
1141        match self.owed_trace.get(index).copied() {
1142            Some(OwedTrace::SendFailed(queue)) => {
1143                if let Some(slot) = self.owed_trace.get_mut(index) {
1144                    *slot = OwedTrace::None;
1145                }
1146                let tick = self.tick;
1147                self.trace.note_exits(self.port.exits());
1148                self.trace
1149                    .event(tick, Event::QueueSendFailed { queue, name: "" });
1150                return true;
1151            }
1152            Some(OwedTrace::ReceiveFailed(queue)) => {
1153                if let Some(slot) = self.owed_trace.get_mut(index) {
1154                    *slot = OwedTrace::None;
1155                }
1156                let tick = self.tick;
1157                self.trace.note_exits(self.port.exits());
1158                self.trace
1159                    .event(tick, Event::QueueReceiveFailed { queue, name: "" });
1160                return true;
1161            }
1162            Some(OwedTrace::EventGroupWaitBitsEnd {
1163                group,
1164                bits,
1165                timed_out,
1166            }) => {
1167                if let Some(slot) = self.owed_trace.get_mut(index) {
1168                    *slot = OwedTrace::None;
1169                }
1170                let tick = self.tick;
1171                self.trace.note_exits(self.port.exits());
1172                self.trace.event(
1173                    tick,
1174                    Event::EventGroupWaitBitsEnd {
1175                        group,
1176                        bits,
1177                        timed_out,
1178                    },
1179                );
1180                return true;
1181            }
1182            Some(OwedTrace::TimerCommandSend {
1183                timer,
1184                name,
1185                command,
1186                value,
1187            }) => {
1188                if let Some(slot) = self.owed_trace.get_mut(index) {
1189                    *slot = OwedTrace::None;
1190                }
1191                let tick = self.tick;
1192                self.trace.note_exits(self.port.exits());
1193                self.trace.event(
1194                    tick,
1195                    Event::TimerCommandSend {
1196                        timer,
1197                        name: name.as_str(),
1198                        command,
1199                        value,
1200                    },
1201                );
1202                return true;
1203            }
1204            Some(OwedTrace::AddNewTaskToReadyList { task, priority }) => {
1205                if let Some(slot) = self.owed_trace.get_mut(index) {
1206                    *slot = OwedTrace::None;
1207                }
1208                let _ = self.add_new_task_to_ready_list(task, priority);
1209                return true;
1210            }
1211            Some(OwedTrace::None) | None => {}
1212        }
1213        if self.owes_yield.get(index).copied() != Some(true) {
1214            return false;
1215        }
1216        if let Some(flag) = self.owes_yield.get_mut(index) {
1217            *flag = false;
1218        }
1219        self.port_yield();
1220        true
1221    }
1222
1223    /// Collect the tally of an abandoned frame that has finished running.
1224    ///
1225    /// The frame runs to its end before control comes back to the runner,
1226    /// so this is called once, at the top of [`Kernel::resume_pending`],
1227    /// which is the first thing the runner does.
1228    fn settle_unwind(&mut self) {
1229        let Some(task) = self.unwinding.take() else {
1230            return;
1231        };
1232        let owed = self.port.end_unwind();
1233        if let Some(slot) = self.owed_exits.get_mut(usize::from(task.index())) {
1234            // Accumulate: a replay that is itself interrupted leaves the
1235            // rest of its own loop as a tail, and that tail is more of the
1236            // same debt.
1237            *slot = slot.saturating_add(owed);
1238        }
1239    }
1240
1241    /// Emit a queue failure line now, or owe it if a tick switched the
1242    /// caller out of the call that was about to emit it.
1243    pub(crate) fn trace_failure_or_owe(&mut self, caller: TaskHandle, owed: OwedTrace) {
1244        if self.current == caller {
1245            let tick = self.tick;
1246            self.trace.note_exits(self.port.exits());
1247            match owed {
1248                OwedTrace::SendFailed(queue) => self
1249                    .trace
1250                    .event(tick, Event::QueueSendFailed { queue, name: "" }),
1251                OwedTrace::ReceiveFailed(queue) => self
1252                    .trace
1253                    .event(tick, Event::QueueReceiveFailed { queue, name: "" }),
1254                OwedTrace::TimerCommandSend {
1255                    timer,
1256                    name,
1257                    command,
1258                    value,
1259                } => self.trace.event(
1260                    tick,
1261                    Event::TimerCommandSend {
1262                        timer,
1263                        name: name.as_str(),
1264                        command,
1265                        value,
1266                    },
1267                ),
1268                OwedTrace::EventGroupWaitBitsEnd {
1269                    group,
1270                    bits,
1271                    timed_out,
1272                } => self.trace.event(
1273                    tick,
1274                    Event::EventGroupWaitBitsEnd {
1275                        group,
1276                        bits,
1277                        timed_out,
1278                    },
1279                ),
1280                // Not reachable through this function: `create_task` owes
1281                // its second half directly, because that half is not a
1282                // trace line — it inserts into a ready list — and it must
1283                // run INSIDE the section this one has already left.
1284                OwedTrace::AddNewTaskToReadyList { .. } | OwedTrace::None => {}
1285            }
1286            return;
1287        }
1288        if let Some(slot) = self.owed_trace.get_mut(usize::from(caller.index())) {
1289            *slot = owed;
1290        }
1291    }
1292
1293    /// Record that `task` was preempted before the `portYIELD()` at the end
1294    /// of the call it is inside.
1295    pub(crate) fn owe_yield(&mut self, task: TaskHandle) {
1296        if let Some(flag) = self.owes_yield.get_mut(usize::from(task.index())) {
1297            *flag = true;
1298        }
1299    }
1300
1301    /// Hand the CPU from `outgoing` to `incoming`: what `prvSwitchThread`
1302    /// does to `uxCriticalNesting`.
1303    ///
1304    /// Everything the outgoing task's call still does from here belongs to
1305    /// that task at the time it runs again, so the port stops counting the
1306    /// frame's exits as sim time and starts tallying them;
1307    /// [`Kernel::settle_unwind`] collects the tally once the frame has
1308    /// finished. A task being switched in for the first time has a fresh
1309    /// stack and owes nothing.
1310    fn hand_over(&mut self, outgoing: TaskHandle, incoming: TaskHandle) {
1311        // A tail that switches again is still the first frame's tail: the
1312        // code after the second switch is on the same abandoned stack.
1313        if self.unwinding.is_none() {
1314            self.unwinding = Some(outgoing);
1315            self.port.begin_unwind();
1316        }
1317        let index = usize::from(incoming.index());
1318        if let Some(flag) = self.started.get_mut(index) {
1319            if !*flag {
1320                *flag = true;
1321                if let Some(slot) = self.owed_exits.get_mut(index) {
1322                    *slot = 0;
1323                }
1324                if let Some(flag) = self.owes_yield.get_mut(index) {
1325                    *flag = false;
1326                }
1327                if let Some(slot) = self.owed_trace.get_mut(index) {
1328                    *slot = OwedTrace::None;
1329                }
1330            }
1331        }
1332    }
1333
1334    /// Mark a task as having run, without a hand-over: the first task the
1335    /// scheduler starts has no predecessor.
1336    fn note_first_start(&mut self, task: TaskHandle) {
1337        if let Some(flag) = self.started.get_mut(usize::from(task.index())) {
1338            *flag = true;
1339        }
1340    }
1341
1342    /// `xTaskIncrementTick`; `true` when a context switch is required.
1343    pub fn increment_tick(&mut self) -> bool {
1344        let tick = self.tick;
1345        self.trace.note_exits(self.port.exits());
1346        self.trace.event(tick, Event::TaskIncrementTick { tick });
1347        let mut switch_required = false;
1348        if self.suspended_depth == 0 {
1349            let next = self.tick.wrapping_add(1) & Self::MAX_DELAY;
1350            self.tick = next;
1351            if next == 0 {
1352                self.switch_delayed_lists();
1353            }
1354            if next >= self.next_unblock_time {
1355                switch_required = self.wake_due_tasks(next);
1356            }
1357            // `!switch_required` first: when the tick already woke a
1358            // task, this can only set what is already set, and asking
1359            // costs a TCB resolve for the running priority plus a list
1360            // read. Setting a `true` to `true` is not worth either.
1361            if C::USE_PREEMPTION
1362                && C::USE_TIME_SLICING
1363                && !switch_required
1364                && self
1365                    .lists
1366                    .len(Self::ready_list(self.current_priority()))
1367                    .unwrap_or(0)
1368                    > 1
1369            {
1370                switch_required = true;
1371            }
1372            // The C guards this site with `xPendedTicks == 0`, so a hook
1373            // does not fire again for each tick being unwound by
1374            // `xTaskResumeAll`. It sits after the time-slicing test and
1375            // before the yield-pending one, and a `FromISR` call inside it
1376            // can set `yield_pending`, so the order is load-bearing.
1377            if self.pended_ticks == 0 {
1378                self.run_tick_hook();
1379            }
1380            if C::USE_PREEMPTION && self.yield_pending {
1381                switch_required = true;
1382            }
1383        } else {
1384            self.pended_ticks = self.pended_ticks.wrapping_add(1);
1385            // "The tick hook gets called at regular intervals, even if the
1386            // scheduler is locked" — and here with no guard, because this
1387            // tick is being pended rather than unwound.
1388            self.run_tick_hook();
1389        }
1390        switch_required
1391    }
1392
1393    // ------------------------------------------- task notifications --
1394    //
1395    // A notification is the lightest thing a task can block on: no object,
1396    // no event list, just a slot in the TCB and the delayed list. That is
1397    // why the stream buffers use one rather than a queue, and why waking a
1398    // task this way is a list move rather than an event-list walk.
1399
1400    /// `xTaskGenericNotifyWait`.
1401    ///
1402    /// `true` is `pdTRUE`: a notification was received. `Blocked` means the
1403    /// caller must call again — the C blocks inside this function, and a
1404    /// kernel with no stacks returns instead.
1405    ///
1406    /// # Errors
1407    /// [`Error::InvalidArgument`] for an index past the configured array.
1408    pub fn notify_wait(
1409        &mut self,
1410        index: usize,
1411        clear_on_entry: u32,
1412        clear_on_exit: u32,
1413        ticks: u64,
1414    ) -> Result<Wait<(bool, u32)>> {
1415        if index >= C::NOTIFICATION_ARRAY_ENTRIES {
1416            return Err(Error::InvalidArgument);
1417        }
1418        let caller = self.current;
1419        let state = self.notify_state_of(caller, index);
1420        if state != NotifyState::Received && ticks > 0 && !self.notify_blocked(caller) {
1421            self.suspend_all();
1422            self.enter_critical();
1423            let mut should_block = false;
1424            if self.notify_state_of(caller, index) != NotifyState::Received {
1425                if let Ok(tcb) = self.tcbs.resolve_mut(caller) {
1426                    if let Some(slot) = tcb.notified.get_mut(index) {
1427                        *slot &= !clear_on_entry;
1428                    }
1429                    if let Some(slot) = tcb.notify_state.get_mut(index) {
1430                        *slot = NotifyState::Waiting;
1431                    }
1432                }
1433                should_block = true;
1434            }
1435            self.exit_critical();
1436            if should_block {
1437                let tick = self.tick;
1438                self.trace.note_exits(self.port.exits());
1439                self.trace_task(caller, |task, name| Event::TaskNotifyWaitBlock {
1440                    task,
1441                    name,
1442                    index,
1443                });
1444                let _ = tick;
1445                self.add_current_task_to_delayed_list(ticks, true)?;
1446            }
1447            let already_yielded = self.resume_all();
1448            if should_block && !already_yielded {
1449                if self.current == caller {
1450                    self.port_yield();
1451                } else {
1452                    self.owe_yield(caller);
1453                }
1454            }
1455            // The C blocks here; this kernel comes back and is called again.
1456            self.set_notify_blocked(caller, true);
1457            return Ok(Wait::Blocked);
1458        }
1459        self.set_notify_blocked(caller, false);
1460        self.enter_critical();
1461        self.trace.note_exits(self.port.exits());
1462        self.trace_task(caller, |task, name| Event::TaskNotifyWait {
1463            task,
1464            name,
1465            index,
1466        });
1467        let value = self.notified_value_of(caller, index);
1468        let received = self.notify_state_of(caller, index) == NotifyState::Received;
1469        if let Ok(tcb) = self.tcbs.resolve_mut(caller) {
1470            if received {
1471                if let Some(slot) = tcb.notified.get_mut(index) {
1472                    *slot &= !clear_on_exit;
1473                }
1474            }
1475            if let Some(slot) = tcb.notify_state.get_mut(index) {
1476                *slot = NotifyState::NotWaiting;
1477            }
1478        }
1479        self.exit_critical();
1480        Ok(Wait::Ready((received, value)))
1481    }
1482
1483    /// `ulTaskGenericNotifyTake`: the notification used as a lightweight
1484    /// counting semaphore — block until the value is non-zero, then either
1485    /// zero it or decrement it by one.
1486    ///
1487    /// This is the half of the notification API the kernel was missing.
1488    /// `notify_wait` waits on *bits*; this waits on a *count*, and the two
1489    /// are separate calls in the C with their own trace events. Its
1490    /// partner, `xTaskNotifyGive`, needs no method of its own: the C
1491    /// defines it as `xTaskGenericNotify( .., 0, eIncrement, NULL )` and
1492    /// it traces as an ordinary notify, so
1493    /// `notify(task, index, 0, NotifyAction::Increment)` already is it.
1494    ///
1495    /// # The shape, and why it is not `notify_wait`'s
1496    ///
1497    /// The C blocks on `ulNotifiedValue == 0`, **not** on the notify
1498    /// state, and it does no clear-on-entry. Both halves run on the way
1499    /// out: `traceTASK_NOTIFY_TAKE` fires whether or not the call ever
1500    /// blocked, which is what makes a take that finds a count already
1501    /// waiting still emit one line.
1502    ///
1503    /// This kernel has no stack, so the block is a return: the first call
1504    /// answers [`Wait::Blocked`] and the caller is entered again when the
1505    /// scheduler runs it, taking the second half. `notify_wait_pending`
1506    /// answers for this call too — a task is only ever inside one of the
1507    /// two.
1508    ///
1509    /// # Errors
1510    /// [`Error::InvalidArgument`] for an index past the configured array.
1511    pub fn notify_take(
1512        &mut self,
1513        index: usize,
1514        clear_on_exit: bool,
1515        ticks: u64,
1516    ) -> Result<Wait<u32>> {
1517        if index >= C::NOTIFICATION_ARRAY_ENTRIES {
1518            return Err(Error::InvalidArgument);
1519        }
1520        let caller = self.current;
1521        if self.notified_value_of(caller, index) == 0 && ticks > 0 && !self.notify_blocked(caller) {
1522            self.suspend_all();
1523            self.enter_critical();
1524            let mut should_block = false;
1525            // Re-checked inside the section, as the C does: a notify from
1526            // an ISR between the two reads would otherwise be lost.
1527            if self.notified_value_of(caller, index) == 0 {
1528                if let Ok(tcb) = self.tcbs.resolve_mut(caller) {
1529                    if let Some(slot) = tcb.notify_state.get_mut(index) {
1530                        *slot = NotifyState::Waiting;
1531                    }
1532                }
1533                should_block = true;
1534            }
1535            self.exit_critical();
1536            if should_block {
1537                // The C traces BEFORE it adds itself to the delayed list.
1538                self.trace.note_exits(self.port.exits());
1539                self.trace_task(caller, |task, name| Event::TaskNotifyTakeBlock {
1540                    task,
1541                    name,
1542                    index,
1543                });
1544                self.add_current_task_to_delayed_list(ticks, true)?;
1545            }
1546            let already_yielded = self.resume_all();
1547            if should_block && !already_yielded {
1548                if self.current == caller {
1549                    self.port_yield();
1550                } else {
1551                    self.owe_yield(caller);
1552                }
1553            }
1554            self.set_notify_blocked(caller, true);
1555            return Ok(Wait::Blocked);
1556        }
1557        self.set_notify_blocked(caller, false);
1558        self.enter_critical();
1559        self.trace.note_exits(self.port.exits());
1560        self.trace_task(caller, |task, name| Event::TaskNotifyTake {
1561            task,
1562            name,
1563            index,
1564        });
1565        let value = self.notified_value_of(caller, index);
1566        if let Ok(tcb) = self.tcbs.resolve_mut(caller) {
1567            if value != 0 {
1568                if let Some(slot) = tcb.notified.get_mut(index) {
1569                    // Guarded by `value != 0` above, so the saturation
1570                    // never fires; it is here because the workspace forbids
1571                    // bare arithmetic, and a decrement is exactly the place
1572                    // an unguarded one would wrap.
1573                    *slot = if clear_on_exit {
1574                        0
1575                    } else {
1576                        value.saturating_sub(1)
1577                    };
1578                }
1579            }
1580            if let Some(slot) = tcb.notify_state.get_mut(index) {
1581                *slot = NotifyState::NotWaiting;
1582            }
1583        }
1584        self.exit_critical();
1585        Ok(Wait::Ready(value))
1586    }
1587
1588    /// `xTaskGetHandle`: the task registered under `name`.
1589    ///
1590    /// # It is not free, and that is the point
1591    ///
1592    /// The C walks every ready list, then the delayed lists, then the
1593    /// suspended and waiting-termination lists, all inside
1594    /// `vTaskSuspendAll` / `xTaskResumeAll` — and `xTaskResumeAll` takes a
1595    /// critical section. So a lookup that emits **no trace event** still
1596    /// costs **one critical-section exit**, which under the sim contract is
1597    /// one unit of time.
1598    ///
1599    /// That is why this exists rather than a scenario keeping the handle it
1600    /// was given at creation. `AbortDelay`'s remake did exactly that, on the
1601    /// reasoning that a lookup emitting no event could not change the trace,
1602    /// and the corpus caught it: every event still agreed and the
1603    /// exit column was one short at the first block.
1604    ///
1605    /// The arena is searched instead of five lists, which cannot change the
1606    /// answer — a live task is in exactly one of them — and does not change
1607    /// the accounting either, because the cost is the suspend/resume pair.
1608    ///
1609    /// # Errors
1610    /// [`Error::Gone`] when no live task carries that name.
1611    pub fn task_get_handle(&mut self, name: &str) -> Result<TaskHandle> {
1612        self.suspend_all();
1613        let mut found = None;
1614        for index in 0..u16::try_from(self.tcbs.capacity()).unwrap_or(u16::MAX) {
1615            let Some(handle) = self.tcbs.handle_at(index) else {
1616                continue;
1617            };
1618            if self
1619                .tcbs
1620                .resolve(handle)
1621                .is_ok_and(|tcb| tcb.name.as_str() == name)
1622            {
1623                found = Some(handle);
1624                break;
1625            }
1626        }
1627        let _ = self.resume_all();
1628        found.ok_or(Error::Gone)
1629    }
1630
1631    /// `xTaskGenericNotify`.
1632    ///
1633    /// # Errors
1634    /// [`Error::InvalidArgument`] for a bad index; [`Error::Gone`] for a
1635    /// stale handle.
1636    pub fn notify(
1637        &mut self,
1638        task: TaskHandle,
1639        index: usize,
1640        value: u32,
1641        action: NotifyAction,
1642    ) -> Result<bool> {
1643        if index >= C::NOTIFICATION_ARRAY_ENTRIES {
1644            return Err(Error::InvalidArgument);
1645        }
1646        self.enter_critical();
1647        let result = self.notify_locked(task, index, value, action, false);
1648        self.exit_critical();
1649        result.map(|(ok, _)| ok)
1650    }
1651
1652    /// `xTaskGenericNotifyFromISR`: the same, without a critical section
1653    /// that costs anything, and putting a woken task on the pending-ready
1654    /// list when the scheduler is suspended.
1655    ///
1656    /// # Errors
1657    /// As [`Kernel::notify`].
1658    pub fn notify_from_isr(
1659        &mut self,
1660        task: TaskHandle,
1661        index: usize,
1662        value: u32,
1663        action: NotifyAction,
1664    ) -> Result<(bool, Woken)> {
1665        if index >= C::NOTIFICATION_ARRAY_ENTRIES {
1666            return Err(Error::InvalidArgument);
1667        }
1668        let mask = self.port.enter_critical_from_isr();
1669        let result = self.notify_locked(task, index, value, action, true);
1670        self.port.exit_critical_from_isr(mask);
1671        result.map(|(ok, woken)| (ok, if woken { Woken::YES } else { Woken::NO }))
1672    }
1673
1674    /// The body both notify paths share.
1675    fn notify_locked(
1676        &mut self,
1677        task: TaskHandle,
1678        index: usize,
1679        value: u32,
1680        action: NotifyAction,
1681        from_isr: bool,
1682    ) -> Result<(bool, bool)> {
1683        let original = self.notify_state_of(task, index);
1684        let mut ok = true;
1685        {
1686            let tcb = self.tcbs.resolve_mut(task)?;
1687            if let Some(slot) = tcb.notify_state.get_mut(index) {
1688                *slot = NotifyState::Received;
1689            }
1690            if let Some(slot) = tcb.notified.get_mut(index) {
1691                match action {
1692                    NotifyAction::SetBits => *slot |= value,
1693                    NotifyAction::Increment => *slot = slot.wrapping_add(1),
1694                    NotifyAction::Overwrite => *slot = value,
1695                    NotifyAction::NoOverwrite => {
1696                        if original == NotifyState::Received {
1697                            ok = false;
1698                        } else {
1699                            *slot = value;
1700                        }
1701                    }
1702                    NotifyAction::None => {}
1703                }
1704            }
1705        }
1706        // `traceTASK_NOTIFY` is hooked; `traceTASK_NOTIFY_FROM_ISR` is not,
1707        // so an interrupt's notification says nothing on either side.
1708        if !from_isr {
1709            self.trace.note_exits(self.port.exits());
1710            self.trace_task(task, |t, name| Event::TaskNotify {
1711                task: t,
1712                name,
1713                index,
1714            });
1715        }
1716        let mut woke_higher = false;
1717        if original == NotifyState::Waiting {
1718            let item = Self::state_item(task);
1719            if from_isr && self.suspended_depth != 0 {
1720                let _ = self.lists.remove(Self::event_item(task));
1721                let value = self.lists.value(Self::event_item(task))?;
1722                self.lists
1723                    .insert(Self::pending_ready_list(), Self::event_item(task), value)?;
1724            } else {
1725                if self.lists.container(item)?.is_some() {
1726                    let _ = self.lists.remove(item);
1727                }
1728                self.add_task_to_ready_list(task)?;
1729            }
1730            let woken = self.tcbs.resolve(task).map(|t| t.priority).unwrap_or(0);
1731            if woken > self.current_priority() {
1732                woke_higher = true;
1733                if from_isr {
1734                    // The C sets `xYieldPendings[ 0 ]` here as well as
1735                    // reporting the wake through the caller's flag — so an
1736                    // interrupt that discards the flag still gets the
1737                    // switch, on the way out of `xTaskIncrementTick`. That
1738                    // is why the tick hook runs before the yield-pending
1739                    // test and not after it.
1740                    self.yield_pending = true;
1741                } else {
1742                    // taskYIELD_ANY_CORE_IF_USING_PREEMPTION, inside the
1743                    // section.
1744                    self.port_yield();
1745                }
1746            }
1747        }
1748        Ok((ok, woke_higher))
1749    }
1750
1751    /// `xTaskGenericNotifyStateClear`. `None` is the calling task.
1752    ///
1753    /// # Errors
1754    /// As [`Kernel::notify`].
1755    pub fn notify_state_clear(&mut self, task: Option<TaskHandle>, index: usize) -> Result<bool> {
1756        if index >= C::NOTIFICATION_ARRAY_ENTRIES {
1757            return Err(Error::InvalidArgument);
1758        }
1759        let target = task.unwrap_or(self.current);
1760        self.enter_critical();
1761        let cleared = self.notify_state_of(target, index) == NotifyState::Received;
1762        if cleared {
1763            if let Ok(tcb) = self.tcbs.resolve_mut(target) {
1764                if let Some(slot) = tcb.notify_state.get_mut(index) {
1765                    *slot = NotifyState::NotWaiting;
1766                }
1767            }
1768        }
1769        self.exit_critical();
1770        Ok(cleared)
1771    }
1772
1773    /// `ulNotifiedValue[ index ]` for a task. `None` is the calling task.
1774    ///
1775    /// The C reaches this through `xTaskNotifyAndQuery`, which hands back
1776    /// the value a notification is about to REPLACE, and through
1777    /// `ulTaskNotifyValueClear`. Neither could be implemented without it:
1778    /// `xTaskNotifyAndQuery` is `xTaskGenericNotify` with a
1779    /// `pulPreviousNotificationValue` out-parameter, and `TaskNotify.c:373`
1780    /// asserts on what comes back. The field existed; only the way to read
1781    /// it was missing, which is the kind of gap a C ABI finds and a Rust
1782    /// face never does.
1783    ///
1784    /// # Errors
1785    /// `InvalidArgument` if `index` is outside the notification array, or
1786    /// the resolve error if the handle is stale.
1787    pub fn notify_value(&mut self, task: Option<TaskHandle>, index: usize) -> Result<u32> {
1788        if index >= C::NOTIFICATION_ARRAY_ENTRIES {
1789            return Err(Error::InvalidArgument);
1790        }
1791        let target = task.unwrap_or(self.current);
1792        self.enter_critical();
1793        let value = self
1794            .tcbs
1795            .resolve(target)
1796            .map(|tcb| tcb.notified.get(index).copied().unwrap_or(0));
1797        self.exit_critical();
1798        value
1799    }
1800
1801    /// `ulTaskGenericNotifyValueClear`: clear `bits` in the task's
1802    /// notification value and answer what it was BEFORE the clear.
1803    ///
1804    /// # Errors
1805    /// As [`Kernel::notify_value`].
1806    pub fn notify_value_clear(
1807        &mut self,
1808        task: Option<TaskHandle>,
1809        index: usize,
1810        bits: u32,
1811    ) -> Result<u32> {
1812        if index >= C::NOTIFICATION_ARRAY_ENTRIES {
1813            return Err(Error::InvalidArgument);
1814        }
1815        let target = task.unwrap_or(self.current);
1816        self.enter_critical();
1817        let before = match self.tcbs.resolve_mut(target) {
1818            Ok(tcb) => match tcb.notified.get_mut(index) {
1819                Some(slot) => {
1820                    let was = *slot;
1821                    *slot &= !bits;
1822                    Ok(was)
1823                }
1824                None => Err(Error::InvalidArgument),
1825            },
1826            Err(e) => Err(e),
1827        };
1828        self.exit_critical();
1829        before
1830    }
1831
1832    /// Whether `task` is part-way through a notification wait — it blocked
1833    /// once and has not yet run the half of `xTaskGenericNotifyWait` that
1834    /// happens when a task wakes.
1835    ///
1836    /// A caller that blocks on a notification needs this: the C's wait
1837    /// returns *after* the wake, so its second half always runs, and a
1838    /// stackless caller that decided not to block again would otherwise
1839    /// skip it and leave the slot marked as received for ever.
1840    #[must_use]
1841    pub fn notify_wait_pending(&self, task: TaskHandle) -> bool {
1842        self.notify_blocked(task)
1843    }
1844
1845    /// Whether `task` has a stream-buffer call to resume, and the local it
1846    /// left behind. Taking it clears the marker.
1847    pub(crate) fn take_stream_resume(&mut self, task: TaskHandle) -> Option<usize> {
1848        let tcb = self.tcbs.resolve_mut(task).ok()?;
1849        if !tcb.stream_resume {
1850            return None;
1851        }
1852        tcb.stream_resume = false;
1853        Some(tcb.stream_local)
1854    }
1855
1856    /// Remember where a preempted stream-buffer call has to start again.
1857    pub(crate) fn set_stream_resume(&mut self, task: TaskHandle, local: usize) {
1858        if let Ok(tcb) = self.tcbs.resolve_mut(task) {
1859            tcb.stream_resume = true;
1860            tcb.stream_local = local;
1861        }
1862    }
1863
1864    fn notify_blocked(&self, task: TaskHandle) -> bool {
1865        self.tcbs
1866            .resolve(task)
1867            .map(|t| t.notify_blocked)
1868            .unwrap_or(false)
1869    }
1870
1871    fn set_notify_blocked(&mut self, task: TaskHandle, blocked: bool) {
1872        if let Ok(tcb) = self.tcbs.resolve_mut(task) {
1873            tcb.notify_blocked = blocked;
1874        }
1875    }
1876
1877    fn notify_state_of(&self, task: TaskHandle, index: usize) -> NotifyState {
1878        self.tcbs
1879            .resolve(task)
1880            .ok()
1881            .and_then(|t| t.notify_state.get(index).copied())
1882            .unwrap_or(NotifyState::NotWaiting)
1883    }
1884
1885    fn notified_value_of(&self, task: TaskHandle, index: usize) -> u32 {
1886        self.tcbs
1887            .resolve(task)
1888            .ok()
1889            .and_then(|t| t.notified.get(index).copied())
1890            .unwrap_or(0)
1891    }
1892
1893    /// `xTaskGetTickCountFromISR`.
1894    ///
1895    /// No critical section, exactly as `xTaskGetTickCount` takes none: the
1896    /// Posix port sets `portTICK_TYPE_IS_ATOMIC`, so reading the count is
1897    /// a single load on both sides and costs no sim time.
1898    #[must_use]
1899    pub const fn tick_count_from_isr(&self) -> u64 {
1900        self.tick
1901    }
1902
1903    /// `vTaskMissedYield`: a yield that could not be taken where it was
1904    /// asked for — because a task is walking an event list, or because the
1905    /// scheduler is suspended — and must happen on the way out instead.
1906    pub(crate) fn missed_yield(&mut self) {
1907        self.yield_pending = true;
1908    }
1909
1910    /// `vApplicationTickHook()`, when `configUSE_TICK_HOOK` is 1.
1911    ///
1912    /// The hook is copied out, run, and the result stored: that is what
1913    /// lets it borrow the kernel mutably while the kernel owns it. A hook
1914    /// that reads the kernel's own copy of itself therefore sees the value
1915    /// from before this call.
1916    fn run_tick_hook(&mut self) {
1917        if !C::USE_TICK_HOOK {
1918            return;
1919        }
1920        let hook = self.tick_hook;
1921        self.tick_hook = hook.tick(self);
1922    }
1923
1924    /// The tick hook, as it now stands.
1925    ///
1926    /// A scenario's `xAre...StillRunning()` reads the status its interrupt
1927    /// half latched, so the owner of the kernel needs to see it.
1928    pub const fn tick_hook(&self) -> &H {
1929        &self.tick_hook
1930    }
1931
1932    /// The tick hook, to install or reset one after construction.
1933    pub const fn tick_hook_mut(&mut self) -> &mut H {
1934        &mut self.tick_hook
1935    }
1936
1937    /// The wake loop inside `xTaskIncrementTick`.
1938    fn wake_due_tasks(&mut self, now: u64) -> bool {
1939        let mut switch_required = false;
1940        loop {
1941            let delayed = self.delayed_list();
1942            if self.lists.is_empty(delayed) != Ok(false) {
1943                self.next_unblock_time = Self::MAX_DELAY;
1944                return switch_required;
1945            }
1946            let Ok(Some(item)) = self.lists.head(delayed) else {
1947                self.next_unblock_time = Self::MAX_DELAY;
1948                return switch_required;
1949            };
1950            let Ok(wake_at) = self.lists.value(item) else {
1951                return switch_required;
1952            };
1953            if now < wake_at {
1954                self.next_unblock_time = wake_at;
1955                return switch_required;
1956            }
1957            let Ok(task) = self.task_of_state_item(item) else {
1958                return switch_required;
1959            };
1960            let _ = self.lists.remove(item);
1961            if self
1962                .lists
1963                .container(Self::event_item(task))
1964                .unwrap_or(None)
1965                .is_some()
1966            {
1967                let _ = self.lists.remove(Self::event_item(task));
1968            }
1969            if self.add_task_to_ready_list(task).is_err() {
1970                return switch_required;
1971            }
1972            if C::USE_PREEMPTION {
1973                let woken = self.tcbs.resolve(task).map(|t| t.priority).unwrap_or(0);
1974                if woken > self.current_priority() {
1975                    switch_required = true;
1976                }
1977            }
1978        }
1979    }
1980
1981    /// `taskSWITCH_DELAYED_LISTS()`.
1982    fn switch_delayed_lists(&mut self) {
1983        self.delayed_swapped = !self.delayed_swapped;
1984        self.overflows = self.overflows.wrapping_add(1);
1985        self.reset_next_task_unblock_time();
1986    }
1987
1988    /// `prvResetNextTaskUnblockTime`.
1989    fn reset_next_task_unblock_time(&mut self) {
1990        let delayed = self.delayed_list();
1991        self.next_unblock_time = self.lists.head_value(delayed).unwrap_or(Self::MAX_DELAY);
1992    }
1993
1994    // ------------------------------------------------------------- delay --
1995
1996    /// `vTaskDelay`.
1997    ///
1998    /// # Errors
1999    /// A list error surfaces as itself.
2000    pub fn delay(&mut self, ticks: u64) -> Result<()> {
2001        let caller = self.current;
2002        let mut already_yielded = false;
2003        if ticks > 0 {
2004            self.suspend_all();
2005            {
2006                let current = self.current;
2007                self.trace_task(current, |task, name| Event::TaskDelay { task, name, ticks });
2008                self.add_current_task_to_delayed_list(ticks, false)?;
2009            }
2010            already_yielded = self.resume_all();
2011        }
2012        // taskYIELD_WITHIN_API(), outside any critical section — unless a
2013        // tick switched us out somewhere above, in which case this line is
2014        // on a stack that is not running, and runs when the task does.
2015        if !already_yielded {
2016            if self.current == caller {
2017                self.port_yield();
2018            } else {
2019                self.owe_yield(caller);
2020            }
2021        }
2022        Ok(())
2023    }
2024
2025    /// `xTaskAbortDelay`: pull a task out of the Blocked state early.
2026    ///
2027    /// `true` is `pdPASS` — the task really was blocked and is now ready.
2028    /// A task that was not blocked is left alone and `false` comes back,
2029    /// which is the C's `pdFAIL`.
2030    ///
2031    /// The shape matters as much as the effect: the whole thing runs with
2032    /// the scheduler suspended, the event list is touched inside a critical
2033    /// section of its own because an interrupt can reach it, and a yield
2034    /// that the higher priority of the woken task calls for is *pended*
2035    /// rather than taken, so it happens on the way out of `xTaskResumeAll`.
2036    ///
2037    /// # Errors
2038    /// [`Error::Gone`] for a stale handle.
2039    pub fn abort_delay(&mut self, task: TaskHandle) -> Result<bool> {
2040        if !self.tcbs.contains(task) {
2041            return Err(Error::Gone);
2042        }
2043        self.suspend_all();
2044        // eTaskGetState: a critical section of its own on any task but the
2045        // running one, and the running one is never Blocked.
2046        let blocked = self.task_state_get(task)? == TaskState::Blocked;
2047        if !blocked {
2048            let _ = self.resume_all();
2049            return Ok(false);
2050        }
2051        let item = Self::state_item(task);
2052        if self.lists.container(item)?.is_some() {
2053            let _ = self.lists.remove(item);
2054        }
2055        self.enter_critical();
2056        {
2057            let event = Self::event_item(task);
2058            if self.lists.container(event)?.is_some() {
2059                let _ = self.lists.remove(event);
2060                if let Some(flag) = self.delay_aborted.get_mut(usize::from(task.index())) {
2061                    *flag = true;
2062                }
2063            }
2064        }
2065        self.exit_critical();
2066        self.add_task_to_ready_list(task)?;
2067        // configUSE_PREEMPTION, one core: pend the yield rather than take
2068        // it, so it lands where `xTaskResumeAll` puts it.
2069        let woken = self.tcbs.resolve(task).map(|t| t.priority).unwrap_or(0);
2070        if woken > self.current_priority() {
2071            self.yield_pending = true;
2072        }
2073        let _ = self.resume_all();
2074        Ok(true)
2075    }
2076
2077    /// `prvAddCurrentTaskToDelayedList`.
2078    pub(crate) fn add_current_task_to_delayed_list(
2079        &mut self,
2080        ticks: u64,
2081        can_block_indefinitely: bool,
2082    ) -> Result<()> {
2083        let now = self.tick;
2084        let current = self.current;
2085        // About to enter a delayed list, so the abort flag is cleared here
2086        // and can only be seen set by a task that really was aborted.
2087        if let Some(flag) = self.delay_aborted.get_mut(usize::from(current.index())) {
2088            *flag = false;
2089        }
2090        let item = Self::state_item(current);
2091        // `uxListRemove` already answers `NotActive` for an item that is
2092        // in no list, and both it and the container test are one read of
2093        // the same node — so asking first was reading it twice. This is
2094        // the running task's state item, so it is in the ready list and
2095        // the test passed anyway.
2096        let _ = self.lists.remove(item);
2097        if ticks == Self::MAX_DELAY && can_block_indefinitely {
2098            self.lists.insert_end(Self::suspended_list(), item)?;
2099            return Ok(());
2100        }
2101        let wake_at = now.wrapping_add(ticks) & Self::MAX_DELAY;
2102        // `vListInsert` sets the item's value from the one it sorts by, so
2103        // both arms below write `wake_at` themselves. Setting it here as
2104        // well was a second lookup of the same item to store the same
2105        // number into it.
2106        if wake_at < now {
2107            self.trace_task(current, |task, name| {
2108                Event::MovedTaskToOverflowDelayedList { task, name }
2109            });
2110            let list = self.overflow_delayed_list();
2111            self.lists.insert(list, item, wake_at)?;
2112        } else {
2113            self.trace_task(current, |task, name| Event::MovedTaskToDelayedList {
2114                task,
2115                name,
2116            });
2117            let list = self.delayed_list();
2118            self.lists.insert(list, item, wake_at)?;
2119            if wake_at < self.next_unblock_time {
2120                self.next_unblock_time = wake_at;
2121            }
2122        }
2123        Ok(())
2124    }
2125
2126    // --------------------------------------------------- suspend / resume --
2127
2128    /// `vTaskSuspend`. `None` suspends the calling task.
2129    ///
2130    /// # Errors
2131    /// [`Error::Gone`] for a stale handle.
2132    pub fn suspend(&mut self, task: Option<TaskHandle>) -> Result<()> {
2133        let caller = self.current;
2134        let target = task.unwrap_or(caller);
2135        self.enter_critical();
2136        {
2137            if !self.tcbs.contains(target) {
2138                self.exit_critical();
2139                return Err(Error::Gone);
2140            }
2141            self.trace_task(target, |task, name| Event::TaskSuspend { task, name });
2142            let item = Self::state_item(target);
2143            if self.lists.container(item)?.is_some() {
2144                let _ = self.lists.remove(item);
2145            }
2146            let event = Self::event_item(target);
2147            if self.lists.container(event)?.is_some() {
2148                let _ = self.lists.remove(event);
2149            }
2150            self.lists.insert_end(Self::suspended_list(), item)?;
2151            // `vTaskSuspend`, `tasks.c`:
2152            //
2153            // ```c
2154            // if( pxTCB->ucNotifyState[ x ] == taskWAITING_NOTIFICATION )
2155            // {
2156            //     /* The task was blocked to wait for a notification, but
2157            //      * is now suspended, so no notification was received. */
2158            //     pxTCB->ucNotifyState[ x ] = taskNOT_WAITING_NOTIFICATION;
2159            // }
2160            // ```
2161            //
2162            // Not tidying: `eTaskGetState` calls a task on the suspended
2163            // list BLOCKED rather than SUSPENDED if any of its notification
2164            // slots is still waiting, so leaving the flag set makes a
2165            // suspended task report as blocked for ever.
2166            // `TaskNotify.c:498` asserts exactly that, in a timer callback
2167            // that suspends a task which is waiting for a notification --
2168            // and a task that reports the wrong state there is a task the
2169            // rest of that test never resumes correctly.
2170            if let Ok(tcb) = self.tcbs.resolve_mut(target) {
2171                for slot in &mut tcb.notify_state {
2172                    if *slot == NotifyState::Waiting {
2173                        *slot = NotifyState::NotWaiting;
2174                    }
2175                }
2176            }
2177        }
2178        self.exit_critical();
2179        // A second, separate critical section — two outermost exits, which
2180        // is what the C does and therefore what the tick count depends on.
2181        if self.running {
2182            self.enter_critical();
2183            self.reset_next_task_unblock_time();
2184            self.exit_critical();
2185        }
2186        if target == caller {
2187            if self.running {
2188                // portYIELD_WITHIN_API(), outside the section.
2189                if self.current == caller {
2190                    self.port_yield();
2191                } else {
2192                    self.owe_yield(caller);
2193                }
2194            } else if self.lists.len(Self::suspended_list()).unwrap_or(0) != self.task_count {
2195                self.switch_context();
2196            }
2197        }
2198        Ok(())
2199    }
2200
2201    /// `vTaskResume`.
2202    ///
2203    /// # Errors
2204    /// [`Error::Gone`] for a stale handle.
2205    pub fn resume(&mut self, task: TaskHandle) -> Result<()> {
2206        if task == self.current || task.is_null() {
2207            return Ok(());
2208        }
2209        if !self.tcbs.contains(task) {
2210            return Err(Error::Gone);
2211        }
2212        self.enter_critical();
2213        {
2214            if self.task_is_suspended(task)? {
2215                self.trace_task(task, |task, name| Event::TaskResume { task, name });
2216                let _ = self.lists.remove(Self::state_item(task));
2217                self.add_task_to_ready_list(task)?;
2218                // taskYIELD_ANY_CORE_IF_USING_PREEMPTION: inside the
2219                // section, and only when the resumed task outranks the
2220                // running one.
2221                let resumed = self.tcbs.resolve(task)?.priority;
2222                if C::USE_PREEMPTION && self.current_priority() < resumed {
2223                    self.port_yield();
2224                }
2225            }
2226        }
2227        self.exit_critical();
2228        Ok(())
2229    }
2230
2231    /// `prvTaskIsTaskSuspended`: on the suspended list, not on the pending
2232    /// ready list, and not waiting on an event.
2233    fn task_is_suspended(&self, task: TaskHandle) -> Result<bool> {
2234        if self.lists.container(Self::state_item(task))? != Some(Self::suspended_list()) {
2235            return Ok(false);
2236        }
2237        let event_list = self.lists.container(Self::event_item(task))?;
2238        if event_list == Some(Self::pending_ready_list()) {
2239            return Ok(false);
2240        }
2241        Ok(event_list.is_none())
2242    }
2243
2244    // -------------------------------------------------------- priorities --
2245
2246    /// `vTaskDelete`. `None` deletes the calling task.
2247    ///
2248    /// # Two paths, and the C picks between them for a reason
2249    ///
2250    /// Deleting *another* task frees its TCB on the spot. Deleting the
2251    /// **running** task cannot: the caller is still executing out of it. The
2252    /// C parks such a task on `xTasksWaitingTermination` and lets the idle
2253    /// task reclaim it, which is what `prvCheckTasksWaitingTermination` is
2254    /// for — so [`Kernel::check_tasks_waiting_termination`] is where the
2255    /// deferred half lands here, called from the idle body at the same point
2256    /// in the same loop.
2257    ///
2258    /// # The frees are outside the critical section, and they cost exits
2259    ///
2260    /// `prvDeleteTCB` runs **after** `taskEXIT_CRITICAL()` and frees twice —
2261    /// `vPortFreeStack( pxStack )` then `vPortFree( pxTCB )`. On the oracle's
2262    /// heap every free suspends the scheduler, so a delete is one exit for
2263    /// its own section plus two for the frees, in that order. Folding the
2264    /// frees into the section, or forgetting them, moves every later tick.
2265    ///
2266    /// # Errors
2267    /// [`Error::Gone`] for a stale handle.
2268    pub fn task_delete(&mut self, task: Option<TaskHandle>) -> Result<()> {
2269        // `prvGetTCBFromHandle`: NULL is the calling task.
2270        let target = task.unwrap_or(self.current);
2271        self.tcbs.resolve(target)?;
2272
2273        self.enter_critical();
2274        // `uxListRemove( &( pxTCB->xStateListItem ) )`. The C follows this
2275        // with `taskRESET_READY_PRIORITY`, which is an EMPTY MACRO under the
2276        // oracle's `configUSE_PORT_OPTIMISED_TASK_SELECTION 0` — there is no
2277        // priority bitmap to clear, and `vTaskSwitchContext` rediscovers the
2278        // top priority by walking down. Nothing to model.
2279        let state = Self::state_item(target);
2280        if matches!(self.lists.container(state), Ok(Some(_))) {
2281            let _ = self.lists.remove(state);
2282        }
2283        // `if( listLIST_ITEM_CONTAINER( &( pxTCB->xEventListItem ) ) != NULL )`
2284        // — a task blocked on a queue is on two lists, and both must let go.
2285        let event = Self::event_item(target);
2286        if matches!(self.lists.container(event), Ok(Some(_))) {
2287            let _ = self.lists.remove(event);
2288        }
2289        // `uxTaskNumber++` is not modelled: it exists so kernel-aware
2290        // debuggers can tell the task lists changed, and nothing the trace
2291        // records reads it.
2292
2293        // `xSchedulerRunning != pdFALSE && taskTASK_IS_RUNNING_OR_SCHEDULED_
2294        // _TO_YIELD( pxTCB )`, which at `configNUMBER_OF_CORES 1` is exactly
2295        // `pxTCB == pxCurrentTCB`.
2296        let defer = self.running && target == self.current;
2297        if defer {
2298            // `vListInsertEnd( &xTasksWaitingTermination, ... )` and
2299            // `++uxDeletedTasksWaitingCleanUp`. The count is NOT decremented
2300            // here — the idle task does that when it reaps.
2301            self.awaiting_reap = Some(target);
2302            self.trace_task(target, |task, name| Event::TaskDelete { task, name });
2303        } else {
2304            self.task_count = self.task_count.saturating_sub(1);
2305            self.trace_task(target, |task, name| Event::TaskDelete { task, name });
2306            self.reset_next_task_unblock_time();
2307        }
2308        self.exit_critical();
2309
2310        // `if( xDeleteTCBInIdleTask != pdTRUE ) { prvDeleteTCB( pxTCB ); }`,
2311        // outside the section.
2312        if !defer {
2313            self.delete_tcb(target);
2314        }
2315
2316        // `taskYIELD_WITHIN_API()`. The C guards this with
2317        // `xSchedulerRunning && pxTCB == pxCurrentTCB`, which is the same
2318        // condition that chose the deferred path.
2319        if defer {
2320            self.port_yield();
2321        }
2322        Ok(())
2323    }
2324
2325    /// `prvDeleteTCB`: two frees, and on the oracle's heap each one suspends
2326    /// the scheduler, so each is one outermost exit.
2327    fn delete_tcb(&mut self, task: TaskHandle) {
2328        // `vPortFreeStack( pxTCB->pxStack )`.
2329        self.account_for_allocation();
2330        // `vPortFree( pxTCB )`.
2331        self.account_for_allocation();
2332        let _ = self.tcbs.remove(task);
2333
2334        // The index is now free for reuse, so mark the task UNSTARTED.
2335        //
2336        // The C frees the TCB and a later `xTaskCreate` gets fresh memory,
2337        // so a new task cannot inherit anything. Our arena hands back the
2338        // same index, and the per-index bookkeeping — owed exits, an owed
2339        // yield, an owed trace line — is keyed on that index, not on the
2340        // TCB. Without this a task deleted while it still owed the tail of
2341        // a call bequeaths that debt to its successor, and the successor
2342        // pays an exit it never incurred: exactly how `death`'s second
2343        // cycle first diverged, one exit early, 1,100 lines after the first
2344        // cycle had matched perfectly.
2345        //
2346        // Clearing this ONE flag is the whole fix, and deliberately so.
2347        // `hand_over` already empties all three of those on a task's first
2348        // switch-in, and a reused index is a first switch-in again. Zeroing
2349        // them here as well would work and would be worse: two places that
2350        // must agree about what a fresh task owes, and a gate that cannot
2351        // catch either one being dropped because the other still covers it.
2352        if let Some(slot) = self.started.get_mut(usize::from(task.index())) {
2353            *slot = false;
2354        }
2355    }
2356
2357    /// `prvCheckTasksWaitingTermination`, called from the idle task.
2358    ///
2359    /// # This must cost NOTHING when nothing is pending
2360    ///
2361    /// The C's body is `while( uxDeletedTasksWaitingCleanUp > 0 )`, so an
2362    /// idle loop with no deleted task takes no critical section and pays no
2363    /// exit. That is what lets this be wired into the idle body without
2364    /// disturbing a single scenario in the corpus — and it is the property
2365    /// the corpus is checking when it stays byte-identical.
2366    ///
2367    /// At most one task can be pending: only the *running* task defers, and
2368    /// it yields before it can return, so a second deferred delete cannot
2369    /// happen until this has run.
2370    pub fn check_tasks_waiting_termination(&mut self) {
2371        let Some(task) = self.awaiting_reap else {
2372            return;
2373        };
2374        // The C takes the section per reaped task, decrements both counts
2375        // inside it, and calls `prvDeleteTCB` outside.
2376        self.enter_critical();
2377        self.awaiting_reap = None;
2378        self.task_count = self.task_count.saturating_sub(1);
2379        self.exit_critical();
2380        self.delete_tcb(task);
2381    }
2382
2383    /// `vTaskPrioritySet`. `None` is the calling task.    /// `vTaskPrioritySet`. `None` is the calling task.
2384    ///
2385    /// # Errors
2386    /// [`Error::Gone`] for a stale handle.
2387    pub fn set_priority(&mut self, task: Option<TaskHandle>, new_priority: u8) -> Result<()> {
2388        let new_priority = new_priority.min(C::MAX_PRIORITIES.saturating_sub(1));
2389        let target = task.unwrap_or(self.current);
2390        let mut yield_required = false;
2391        self.enter_critical();
2392        {
2393            if !self.tcbs.contains(target) {
2394                self.exit_critical();
2395                return Err(Error::Gone);
2396            }
2397            self.trace_task(target, |task, name| Event::TaskPrioritySet {
2398                task,
2399                name,
2400                priority: Self::priority_value(new_priority),
2401            });
2402            let base = self.tcbs.resolve(target)?.base_priority;
2403            if base != new_priority {
2404                if new_priority > base {
2405                    if target != self.current && new_priority > self.current_priority() {
2406                        yield_required = true;
2407                    }
2408                } else if target == self.current {
2409                    yield_required = true;
2410                }
2411                let used_on_entry = self.tcbs.resolve(target)?.priority;
2412                {
2413                    let tcb = self.tcbs.resolve_mut(target)?;
2414                    if tcb.base_priority == tcb.priority || new_priority > tcb.priority {
2415                        tcb.priority = new_priority;
2416                    }
2417                    tcb.base_priority = new_priority;
2418                }
2419                let event_value = u64::from(C::MAX_PRIORITIES.saturating_sub(new_priority));
2420                self.lists
2421                    .set_value(Self::event_item(target), event_value)?;
2422                let item = Self::state_item(target);
2423                if self.lists.container(item)? == Some(Self::ready_list(used_on_entry)) {
2424                    let _ = self.lists.remove(item);
2425                    self.add_task_to_ready_list(target)?;
2426                }
2427                // taskYIELD_TASK_CORE_IF_USING_PREEMPTION: inside the
2428                // section.
2429                if yield_required && C::USE_PREEMPTION {
2430                    self.port_yield();
2431                }
2432            }
2433        }
2434        self.exit_critical();
2435        Ok(())
2436    }
2437
2438    // ------------------------------------------------- scheduler suspend --
2439
2440    /// `vTaskSuspendAll`.
2441    pub fn suspend_all(&mut self) {
2442        self.suspended_depth = self.suspended_depth.saturating_add(1);
2443    }
2444
2445    /// `xTaskResumeAll`; `true` when it yielded on the caller's behalf.
2446    pub fn resume_all(&mut self) -> bool {
2447        let mut already_yielded = false;
2448        self.enter_critical();
2449        {
2450            self.suspended_depth = self.suspended_depth.saturating_sub(1);
2451            if self.suspended_depth == 0 && self.task_count > 0 {
2452                let mut moved_any = false;
2453                while self.lists.is_empty(Self::pending_ready_list()) == Ok(false) {
2454                    let Ok(Some(item)) = self.lists.head(Self::pending_ready_list()) else {
2455                        break;
2456                    };
2457                    let Ok(task) = self.task_of_event_item(item) else {
2458                        break;
2459                    };
2460                    let _ = self.lists.remove(Self::event_item(task));
2461                    let _ = self.lists.remove(Self::state_item(task));
2462                    if self.add_task_to_ready_list(task).is_err() {
2463                        break;
2464                    }
2465                    moved_any = true;
2466                    let woken = self.tcbs.resolve(task).map(|t| t.priority).unwrap_or(0);
2467                    if woken > self.current_priority() {
2468                        self.yield_pending = true;
2469                    }
2470                }
2471                if moved_any {
2472                    self.reset_next_task_unblock_time();
2473                }
2474                if self.pended_ticks > 0 {
2475                    let mut pended = self.pended_ticks;
2476                    while pended > 0 {
2477                        if self.increment_tick() {
2478                            self.yield_pending = true;
2479                        }
2480                        pended = pended.saturating_sub(1);
2481                    }
2482                    self.pended_ticks = 0;
2483                }
2484                if self.yield_pending {
2485                    if C::USE_PREEMPTION {
2486                        already_yielded = true;
2487                    }
2488                    // taskYIELD_TASK_CORE_IF_USING_PREEMPTION: inside the
2489                    // section.
2490                    self.port_yield();
2491                }
2492            }
2493        }
2494        self.exit_critical();
2495        already_yielded
2496    }
2497
2498    // ------------------------------------------- the blocking-call frame --
2499
2500    /// Start (or continue) a blocking call's `for(;;)`.
2501    ///
2502    /// The first pass records the block time and the entry timestamp, the
2503    /// way `xEntryTimeSet` / `vTaskInternalSetTimeOutState` do; later
2504    /// passes leave the running total alone, because the `ticks` the caller
2505    /// passes is the original block time and the kernel is holding what is
2506    /// left of it.
2507    pub(crate) fn begin_wait(
2508        &mut self,
2509        task: TaskHandle,
2510        queue: QueueHandle,
2511        ticks: u64,
2512    ) -> Result<()> {
2513        let tick = self.tick;
2514        let overflows = self.overflows;
2515        let Ok(tcb) = self.tcbs.resolve_mut(task) else {
2516            // A queue call made before the scheduler started, from what the
2517            // C would call `main()`: there is no task to keep a frame for,
2518            // and such a call never blocks — `xSemaphoreGive` on a fresh
2519            // semaphore is the usual one.
2520            return Ok(());
2521        };
2522        if tcb.wait.queue != queue || !tcb.wait.entry_set {
2523            tcb.wait = WaitFrame {
2524                queue,
2525                ticks,
2526                entering: tick,
2527                overflows,
2528                entry_set: true,
2529                inherited: false,
2530            };
2531        }
2532        Ok(())
2533    }
2534
2535    /// The call finished, one way or the other.
2536    pub(crate) fn end_wait(&mut self, task: TaskHandle) {
2537        if let Ok(tcb) = self.tcbs.resolve_mut(task) {
2538            // Now that the frame is only set on the path that blocks, most
2539            // calls reach here with nothing to clear, and this was writing
2540            // a default frame over a default frame. `entry_set` is the
2541            // sentinel the C calls `xEntryTimeSet`, and it is the first
2542            // thing set — `inherited` is only reached far inside the
2543            // blocking path, so it cannot be true while this is false.
2544            if tcb.wait.entry_set {
2545                tcb.wait = WaitFrame::default();
2546            }
2547        }
2548    }
2549
2550    /// `xTicksToWait` as it now stands.
2551    pub(crate) fn remaining_ticks(&self, task: TaskHandle) -> u64 {
2552        self.tcbs.resolve(task).map(|t| t.wait.ticks).unwrap_or(0)
2553    }
2554
2555    /// `xInheritanceOccurred`.
2556    pub(crate) fn wait_inherited(&self, task: TaskHandle) -> bool {
2557        self.tcbs
2558            .resolve(task)
2559            .map(|t| t.wait.inherited)
2560            .unwrap_or(false)
2561    }
2562
2563    pub(crate) fn set_wait_inherited(&mut self, task: TaskHandle) {
2564        if let Ok(tcb) = self.tcbs.resolve_mut(task) {
2565            tcb.wait.inherited = true;
2566        }
2567    }
2568
2569    /// `xTaskCheckForTimeOut`: `true` when the block time has run out.
2570    /// Takes a critical section, as the C does, and charges the elapsed
2571    /// time against what is left.
2572    pub(crate) fn check_for_timeout(&mut self, task: TaskHandle) -> bool {
2573        self.enter_critical();
2574        // An aborted delay is not the same as a time out, but it has the
2575        // same result: stop waiting.
2576        if self.delay_aborted.get(usize::from(task.index())).copied() == Some(true) {
2577            if let Some(flag) = self.delay_aborted.get_mut(usize::from(task.index())) {
2578                *flag = false;
2579            }
2580            self.exit_critical();
2581            return true;
2582        }
2583        let now = self.tick;
2584        let overflows = self.overflows;
2585        let result = match self.tcbs.resolve_mut(task) {
2586            Ok(tcb) => {
2587                let elapsed = now.wrapping_sub(tcb.wait.entering) & Self::MAX_DELAY;
2588                if tcb.wait.ticks == Self::MAX_DELAY {
2589                    // An indefinite block never times out.
2590                    false
2591                } else if overflows != tcb.wait.overflows && now >= tcb.wait.entering {
2592                    tcb.wait.ticks = 0;
2593                    true
2594                } else if elapsed < tcb.wait.ticks {
2595                    tcb.wait.ticks = tcb.wait.ticks.saturating_sub(elapsed);
2596                    tcb.wait.entering = now;
2597                    tcb.wait.overflows = overflows;
2598                    false
2599                } else {
2600                    tcb.wait.ticks = 0;
2601                    true
2602                }
2603            }
2604            Err(_) => true,
2605        };
2606        self.exit_critical();
2607        result
2608    }
2609
2610    /// `vTaskPlaceOnUnorderedEventList`: the item value carries the
2611    /// condition rather than the priority, so the item goes on the end and
2612    /// the list is never sorted.
2613    ///
2614    /// `taskEVENT_LIST_ITEM_VALUE_IN_USE` is not set here the way the C
2615    /// sets it. The C needs it because the same `ListItem_t` is a
2616    /// priority-ordered event item the rest of the time and the flag says
2617    /// which; the value is only ever read back by the event-group code,
2618    /// which masks the control byte off, so setting it would change
2619    /// nothing but the arithmetic in the doc comment.
2620    pub(crate) fn place_on_unordered_event_list(
2621        &mut self,
2622        list: ListId,
2623        value: u64,
2624        ticks: u64,
2625    ) -> Result<()> {
2626        let item = Self::event_item(self.current);
2627        self.lists.set_value(item, value)?;
2628        self.lists.insert_end(list, item)?;
2629        self.add_current_task_to_delayed_list(ticks, true)
2630    }
2631
2632    /// `vTaskRemoveFromUnorderedEventList`: write the answer into the item
2633    /// value, then wake the task.
2634    ///
2635    /// Unlike `xTaskRemoveFromEventList` this always goes straight to the
2636    /// ready list. It is only ever called with the scheduler suspended —
2637    /// `xEventGroupSetBits` holds it across the whole walk — and the C
2638    /// still bypasses the pending-ready list, because the value it just
2639    /// wrote is the task's return value and a second pass would overwrite
2640    /// it.
2641    pub(crate) fn remove_from_unordered_event_list(
2642        &mut self,
2643        item: ItemId,
2644        value: u64,
2645    ) -> Result<()> {
2646        self.lists.set_value(item, value)?;
2647        let task = self.task_of_event_item(item)?;
2648        let _ = self.lists.remove(item);
2649        let _ = self.lists.remove(Self::state_item(task));
2650        self.add_task_to_ready_list(task)?;
2651        if self.tcbs.resolve(task)?.priority > self.current_priority() {
2652            self.yield_pending = true;
2653        }
2654        Ok(())
2655    }
2656
2657    /// `uxTaskResetEventItemValue`: read the answer the unblocker left, and
2658    /// put the item back to the priority order an ordinary event list wants.
2659    pub(crate) fn reset_event_item_value(&mut self, task: TaskHandle) -> Result<u64> {
2660        let item = Self::event_item(task);
2661        let value = self.lists.value(item)?;
2662        let priority = self.tcbs.resolve(task)?.priority;
2663        self.lists
2664            .set_value(item, u64::from(C::MAX_PRIORITIES.saturating_sub(priority)))?;
2665        Ok(value)
2666    }
2667
2668    /// Whether `task` is resuming an event-group wait rather than starting
2669    /// one. Taking it clears the marker.
2670    pub(crate) fn take_event_resume(&mut self, task: TaskHandle) -> bool {
2671        match self.tcbs.resolve_mut(task) {
2672            Ok(tcb) if tcb.event_blocked => {
2673                tcb.event_blocked = false;
2674                true
2675            }
2676            _ => false,
2677        }
2678    }
2679
2680    /// Mark that `task` blocked inside an event-group wait, so the call it
2681    /// is inside resumes rather than restarts.
2682    pub(crate) fn set_event_resume(&mut self, task: TaskHandle) {
2683        if let Ok(tcb) = self.tcbs.resolve_mut(task) {
2684            tcb.event_blocked = true;
2685        }
2686    }
2687
2688    /// `vTaskPlaceOnEventList`: sorted by priority, then blocked.
2689    pub(crate) fn place_on_event_list(&mut self, list: ListId, ticks: u64) -> Result<()> {
2690        let current = self.current;
2691        let item = Self::event_item(current);
2692        // The item keeps the value its call set; reading it out only to
2693        // hand it back made the list read the item twice and write it once
2694        // for no change.
2695        self.lists.insert_keeping_value(list, item)?;
2696        self.add_current_task_to_delayed_list(ticks, true)
2697    }
2698
2699    /// Take the trailing `portYIELD()` now, or owe it if a tick already
2700    /// switched us out of this call.
2701    pub(crate) fn yield_or_owe(&mut self, caller: TaskHandle) {
2702        if self.current == caller {
2703            self.port_yield();
2704        } else {
2705            self.owe_yield(caller);
2706        }
2707    }
2708
2709    /// `pvTaskIncrementMutexHeldCount`.
2710    pub(crate) fn increment_mutexes_held(&mut self, task: TaskHandle) {
2711        if let Ok(tcb) = self.tcbs.resolve_mut(task) {
2712            tcb.mutexes_held = tcb.mutexes_held.saturating_add(1);
2713        }
2714    }
2715
2716    // ------------------------------------------------ priority inheritance --
2717
2718    /// `xTaskPriorityInherit`: lift the mutex holder to the waiter's
2719    /// priority. `true` when the holder is (or already was) lifted, which
2720    /// is what the waiter remembers so it can undo it on a timeout.
2721    pub(crate) fn priority_inherit(&mut self, holder: TaskHandle) -> Result<bool> {
2722        if holder.is_null() || !self.tcbs.contains(holder) {
2723            return Ok(false);
2724        }
2725        let waiter_priority = self.current_priority();
2726        let (holder_priority, holder_base) = {
2727            let tcb = self.tcbs.resolve(holder)?;
2728            (tcb.priority, tcb.base_priority)
2729        };
2730        if holder_priority >= waiter_priority {
2731            // Already at least as urgent; the waiter still records that the
2732            // holder is running above its base, if it is.
2733            return Ok(holder_base < waiter_priority);
2734        }
2735        let event_value = u64::from(C::MAX_PRIORITIES.saturating_sub(waiter_priority));
2736        self.lists
2737            .set_value(Self::event_item(holder), event_value)?;
2738        let item = Self::state_item(holder);
2739        if self.lists.container(item)? == Some(Self::ready_list(holder_priority)) {
2740            let _ = self.lists.remove(item);
2741            self.tcbs.resolve_mut(holder)?.priority = waiter_priority;
2742            self.add_task_to_ready_list(holder)?;
2743        } else {
2744            self.tcbs.resolve_mut(holder)?.priority = waiter_priority;
2745        }
2746        self.trace_task(holder, |task, name| Event::TaskPriorityInherit {
2747            task,
2748            name,
2749            priority: Self::priority_value(waiter_priority),
2750        });
2751        Ok(true)
2752    }
2753
2754    /// `xTaskPriorityDisinherit`: giving the mutex back drops the
2755    /// inherited priority. `true` when a yield is wanted.
2756    pub(crate) fn priority_disinherit(&mut self, holder: TaskHandle) -> Result<bool> {
2757        if holder.is_null() || !self.tcbs.contains(holder) {
2758            return Ok(false);
2759        }
2760        let (priority, base, held) = {
2761            let tcb = self.tcbs.resolve(holder)?;
2762            (tcb.priority, tcb.base_priority, tcb.mutexes_held)
2763        };
2764        let remaining = held.saturating_sub(1);
2765        if let Ok(tcb) = self.tcbs.resolve_mut(holder) {
2766            tcb.mutexes_held = remaining;
2767        }
2768        if priority == base || remaining != 0 {
2769            return Ok(false);
2770        }
2771        let item = Self::state_item(holder);
2772        if self.lists.remove(item).is_err() {
2773            return Ok(false);
2774        }
2775        self.trace_task(holder, |task, name| Event::TaskPriorityDisinherit {
2776            task,
2777            name,
2778            priority: Self::priority_value(base),
2779        });
2780        {
2781            let tcb = self.tcbs.resolve_mut(holder)?;
2782            tcb.priority = base;
2783        }
2784        let event_value = u64::from(C::MAX_PRIORITIES.saturating_sub(base));
2785        self.lists
2786            .set_value(Self::event_item(holder), event_value)?;
2787        self.add_task_to_ready_list(holder)?;
2788        Ok(true)
2789    }
2790
2791    /// `vTaskPriorityDisinheritAfterTimeout`: a waiter gave up, so the
2792    /// holder keeps only what the tasks still waiting justify.
2793    pub(crate) fn priority_disinherit_after_timeout(
2794        &mut self,
2795        holder: TaskHandle,
2796        highest_waiting: u8,
2797    ) -> Result<()> {
2798        if holder.is_null() || !self.tcbs.contains(holder) {
2799            return Ok(());
2800        }
2801        let (priority, base, held) = {
2802            let tcb = self.tcbs.resolve(holder)?;
2803            (tcb.priority, tcb.base_priority, tcb.mutexes_held)
2804        };
2805        if priority == base || held != 1 {
2806            return Ok(());
2807        }
2808        let target = if highest_waiting > base {
2809            highest_waiting
2810        } else {
2811            base
2812        };
2813        if priority == target {
2814            return Ok(());
2815        }
2816        self.trace_task(holder, |task, name| Event::TaskPriorityDisinherit {
2817            task,
2818            name,
2819            priority: Self::priority_value(target),
2820        });
2821        self.tcbs.resolve_mut(holder)?.priority = target;
2822        let event_value = u64::from(C::MAX_PRIORITIES.saturating_sub(target));
2823        self.lists
2824            .set_value(Self::event_item(holder), event_value)?;
2825        let item = Self::state_item(holder);
2826        if self.lists.container(item)? == Some(Self::ready_list(priority)) {
2827            let _ = self.lists.remove(item);
2828            self.add_task_to_ready_list(holder)?;
2829        }
2830        Ok(())
2831    }
2832
2833    // ------------------------------------------------------- delay until --
2834
2835    /// `xTaskDelayUntil`: wake at `*previous_wake + period`, whatever the
2836    /// task did in between. `false` when the deadline has already passed,
2837    /// in which case nothing blocks — exactly as the C returns `pdFALSE`.
2838    ///
2839    /// # Errors
2840    /// A list error surfaces as itself.
2841    pub fn delay_until(&mut self, previous_wake: &mut u64, period: u64) -> Result<bool> {
2842        let caller = self.current;
2843        let mut should_delay = false;
2844        self.suspend_all();
2845        {
2846            let now = self.tick;
2847            let wake_at = previous_wake.wrapping_add(period) & Self::MAX_DELAY;
2848            if now < *previous_wake {
2849                // The tick count overflowed since the last wake.
2850                if wake_at < *previous_wake && wake_at > now {
2851                    should_delay = true;
2852                }
2853            } else if wake_at < *previous_wake || wake_at > now {
2854                should_delay = true;
2855            }
2856            *previous_wake = wake_at;
2857            if should_delay {
2858                self.trace_task(caller, |task, name| Event::TaskDelayUntil {
2859                    task,
2860                    name,
2861                    wake_at,
2862                });
2863                let ticks = wake_at.wrapping_sub(now) & Self::MAX_DELAY;
2864                self.add_current_task_to_delayed_list(ticks, false)?;
2865            }
2866        }
2867        if !self.resume_all() {
2868            self.yield_or_owe(caller);
2869        }
2870        Ok(should_delay)
2871    }
2872
2873    /// `xTaskRemoveFromEventList`; `true` when the woken task outranks the
2874    /// running one and a yield is therefore required.
2875    pub(crate) fn remove_from_event_list(&mut self, list: ListId) -> Result<bool> {
2876        let Some(item) = self.lists.head(list)? else {
2877            return Ok(false);
2878        };
2879        let task = self.task_of_event_item(item)?;
2880        let _ = self.lists.remove(item);
2881        if self.suspended_depth == 0 {
2882            let _ = self.lists.remove(Self::state_item(task));
2883            self.add_task_to_ready_list(task)?;
2884        } else {
2885            self.lists.insert_end(Self::pending_ready_list(), item)?;
2886        }
2887        let woken = self.tcbs.resolve(task)?.priority;
2888        if woken > self.current_priority() {
2889            self.yield_pending = true;
2890            return Ok(true);
2891        }
2892        Ok(false)
2893    }
2894}