Skip to main content

rusty_rtos_kernel_core/
kernel.rs

1//! The scheduler: `tasks.c` and `queue.c` as a state machine over indices.
2//!
3//! Every public function here is a FreeRTOS API function, and its body
4//! follows the C body statement for statement — including **where the
5//! critical sections open and close**, because on the sim that is where
6//! time passes (`ORACLES.md`, sim contract v1, rule 3), and including where
7//! the `trace*` macros fire, because the order of those lines is the gate.
8//!
9//! The subtlety that costs a diff if you miss it: FreeRTOS yields from
10//! *inside* the critical section in `vTaskResume`, `vTaskPrioritySet`,
11//! `xTaskResumeAll` and the queue calls, and from *outside* it in
12//! `vTaskSuspend` and `vTaskDelay`. Since `portYIELD()` on the Posix port
13//! is itself a critical section, an inside yield nests (and counts one
14//! outermost exit) where an outside yield counts two. Each site below says
15//! which it is.
16//!
17//! Where the C has a `configASSERT`, this has an [`Error`]: the C kernel
18//! stops the world on a bad handle, and the whole point of the remake is
19//! that we do not.
20
21use core::marker::PhantomData;
22
23use core::num::NonZeroU64;
24use rusty_rtos_core::arena::Arena;
25use rusty_rtos_core::config::Config;
26
27use rusty_rtos_core::error::{Error, Result};
28use rusty_rtos_core::handle::{
29    EventGroup as EventGroupKind, EventGroupHandle, Queue as QueueKind, QueueHandle,
30    StreamBuffer as StreamKind, StreamBufferHandle, Task as TaskKind, TaskHandle,
31    Timer as TimerKind, TimerHandle,
32};
33use rusty_rtos_core::hooks::TickHook;
34use rusty_rtos_core::isr::Woken;
35use rusty_rtos_core::list::{ItemId, ListId, Lists};
36use rusty_rtos_core::port::Port;
37use rusty_rtos_core::priority::Priority;
38use rusty_rtos_core::tick::TickWidth;
39use rusty_rtos_core::trace::{Event, Trace};
40
41use crate::events::EventGroup;
42use crate::name::Name;
43use crate::queue::Queue;
44use crate::queue::Wait;
45use crate::stream::StreamBuffer;
46use crate::timer::{Message, Timer};
47use crate::{OVERHEAD_LISTS, list_slots_for, lists_for};
48
49/// How many times [`Kernel::trace_failure_or_owe`] has been entered.
50///
51/// A reachability anchor, not a measurement. Every row of
52/// `riscv32-qemu-tick-work` read **zero** here until the failure rows were
53/// added, which is why a change to the `OwedTrace` machinery measured as pure
54/// code layout. It is behind its own feature because the `fetch_add` sits
55/// INSIDE the measured bracket, so a build that quotes instruction counts must
56/// not carry it.
57#[cfg(feature = "census")]
58pub static CENSUS_OWE: core::sync::atomic::AtomicU32 = core::sync::atomic::AtomicU32::new(0);
59
60/// A trace line owed by a task that was switched out before it could
61/// emit one. Only the queue failure paths can owe one: they are the only
62/// places FreeRTOS traces *after* leaving a critical section.
63/// Why `vTaskSwitchContext` could not choose a task to run.
64///
65/// # Law 3: no silent failures
66///
67/// Every variant here was, until 2026-09-11, a bare `return` — the scheduler
68/// declining to switch and saying nothing. Three of them are impossible in a
69/// healthy kernel and the fourth is what the C asserts on
70/// (`configASSERT( uxTopPriority )`), so reaching any of them means an
71/// invariant has already broken somewhere else.
72///
73/// They cost a multi-flash hardware hunt on the day the Xtensa port first
74/// ran a stacked task: every ready list had emptied, the kernel kept
75/// quietly running whoever was current, and nothing said so. A counter and
76/// a reason turn that into one printed line.
77#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
78pub enum Stall {
79    /// The scheduler chose normally.
80    #[default]
81    None,
82    /// No ready task at ANY priority. The idle task keeps priority 0
83    /// non-empty, so reaching this means the idle task is gone — which is
84    /// exactly what `configASSERT( uxTopPriority )` fires on.
85    NoReadyTask,
86    /// A ready list refused to answer at all.
87    ListError,
88    /// The chosen priority's list had nothing to rotate to, having just
89    /// reported itself non-empty.
90    EmptyRotation,
91    /// The chosen list item named no live task.
92    UnknownTask,
93}
94
95#[derive(Debug, Clone, Copy, PartialEq, Eq)]
96pub(crate) enum OwedTrace {
97    /// Nothing owed.
98    None,
99    /// `traceQUEUE_SEND_FAILED`.
100    SendFailed(QueueHandle),
101    /// `traceQUEUE_RECEIVE_FAILED`.
102    ReceiveFailed(QueueHandle),
103    /// `traceTIMER_COMMAND_SEND`, which the C runs on the line *after*
104    /// `xQueueSendToBack` — so a send that made the daemon ready traces
105    /// only once this task has the CPU back.
106    TimerCommandSend {
107        timer: TimerHandle,
108        name: Name,
109        command: i32,
110        value: u64,
111    },
112    /// `traceSTREAM_BUFFER_CREATE`, which the C runs after the
113    /// `pvPortMalloc` that made the buffer.
114    ///
115    /// `heap_4` brackets that malloc with `vTaskSuspendAll` /
116    /// `xTaskResumeAll`, so a tick can land on its exit -- and the C's
117    /// thread then stops inside the allocator, with the trace still ahead
118    /// of it. Emitting it anyway puts the line in another task's run of the
119    /// trace. `StreamBufferDemo`'s echo client showed it at tick 429, one
120    /// line early and a whole task's work out of place.
121    StreamBufferCreate {
122        buffer: StreamBufferHandle,
123        is_message_buffer: bool,
124    },
125    /// `traceEVENT_GROUP_WAIT_BITS_END`, which the C runs after the
126    /// `xTaskResumeAll` that ends the wait — so a resume that switched away
127    /// traces once this task has the CPU back.
128    EventGroupWaitBitsEnd {
129        group: EventGroupHandle,
130        bits: u32,
131        timed_out: bool,
132    },
133    /// `prvAddNewTaskToReadyList`, the second half of `xTaskCreate`.
134    ///
135    /// A create costs three outermost exits before it reaches this — two
136    /// `pvPortMalloc`s and the port's `pthread_create` section — and a tick
137    /// can land on any of them. The C is a real thread, so it simply stops
138    /// there and the new task is not on a ready list until the creator runs
139    /// again. This is that pause, and it is the only owed item that is not
140    /// purely a trace line: it carries the ready-list insertion too.
141    AddNewTaskToReadyList { task: TaskHandle, priority: u8 },
142}
143
144/// `eTaskState`: what a task is doing, derived from the list it is in
145/// exactly as `eTaskGetState` derives it.
146#[derive(Debug, Clone, Copy, PartialEq, Eq)]
147pub enum TaskState {
148    /// The task `current` names.
149    Running,
150    /// On a ready list.
151    Ready,
152    /// On a delayed list, or on an event list with a timeout.
153    Blocked,
154    /// On the suspended list with no event pending.
155    Suspended,
156    /// The handle names no live task.
157    Deleted,
158}
159
160/// Where a preempted queue call has to start again.
161///
162/// A queue call has SEVERAL critical-section exits that can release a tick
163/// and switch the caller away, and the C's thread resumes below whichever
164/// one it stopped at. One marker is not enough to say which.
165#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
166pub(crate) enum QueueResume {
167    /// Nothing to resume; enter at the top.
168    #[default]
169    No,
170    /// Below the section that samples the queue, in the caller.
171    BelowSample,
172    /// Below the `xTaskResumeAll` of the TIMED-OUT branch, which is the one
173    /// `IntQueue` reaches at tick 16,260: the tick pended inside the
174    /// scheduler suspension is replayed there, and replaying it switches
175    /// the caller away with `prvIsQueueEmpty` still to run.
176    BelowTimedOutResume,
177}
178
179/// One task control block: the C `TCB_t` minus everything that is a
180/// pointer. No stack, no TLS, no `pxTopOfStack`.
181#[derive(Debug, Clone, Copy, Default)]
182pub(crate) struct Tcb {
183    name: Name,
184    /// `uxPriority`, the possibly-inherited one the scheduler sorts by.
185    priority: u8,
186    /// `uxBasePriority`, what the task asked for.
187    base_priority: u8,
188    /// `uxMutexesHeld`: how many mutexes this task holds, which is what
189    /// decides whether giving one back may drop an inherited priority.
190    mutexes_held: u32,
191    /// The locals a blocking queue call keeps on the C stack across a
192    /// block. This kernel has no stack, so they live here — see
193    /// [`crate::queue`].
194    wait: WaitFrame,
195    /// `ulNotifiedValue[]`.
196    notified: [u32; MAX_NOTIFICATION_ENTRIES],
197    /// `ucNotifyState[]`.
198    notify_state: [NotifyState; MAX_NOTIFICATION_ENTRIES],
199    /// Whether a stream-buffer call this task made was preempted at the
200    /// critical-section exit that samples the buffer, and must resume
201    /// *after* that exit rather than repeat it.
202    stream_resume: bool,
203    /// Where a preempted queue call resumes. See [`QueueResume`].
204    ///
205    /// `xQueueGenericSend` and `xQueueReceive` exit their sampling section
206    /// before suspending the scheduler to block, and THAT exit can release
207    /// a tick which switches the caller away. The C's thread stops inside
208    /// the exit; everything below it -- `traceBLOCKING_ON_QUEUE_SEND` and
209    /// `vTaskPlaceOnEventList` -- runs when the task is next scheduled, and
210    /// so names the right task. Without this marker the trace attributes
211    /// the block to whoever the tick switched TO, which `IntQueue` catches
212    /// at tick 5 and no other scenario reaches.
213    queue_resume: QueueResume,
214    /// Whether a blocking `xStreamBufferSend` has already run its
215    /// `vTaskSetTimeOutState`. See [`Kernel::take_stream_timed`].
216    stream_timed: bool,
217    /// Whether a stream-buffer call's notify wait has already finished.
218    /// See [`Kernel::take_stream_waited`].
219    stream_waited: bool,
220    /// The stack local that exit produced — `xBytesAvailable` on a receive,
221    /// `xSpace` on a send — kept because the frame it belonged to is gone.
222    stream_local: usize,
223    /// Whether this task has already blocked inside the notification wait
224    /// it is in. The C blocks inside `xTaskGenericNotifyWait`; a kernel
225    /// with no stacks returns `Blocked` and is called again, so it needs to
226    /// know the second call is a resumption rather than a fresh wait.
227    notify_blocked: bool,
228    /// The same, for the event-group wait this task is inside. Those two
229    /// functions have no retry loop — the C blocks once and then reads the
230    /// event item value — so the second call has to know it is the far side
231    /// of the switch and not a fresh wait.
232    event_blocked: bool,
233    /// Padding that rounds `Slot<Tcb>` up to a POWER OF TWO (128 bytes), so
234    /// `index * size_of::<Slot<Tcb>>()` is a shift and not a multiply.
235    ///
236    /// `bench/kernel-ram` pins the stride at 128 and `bench/kernel-flash` pins
237    /// `mul` at 0; without this the stride is 92 and every TCB resolve pays a
238    /// `mul`. It is a deliberate flash-for-RAM trade: 254 bytes of flash
239    /// against 36 bytes of RAM per task, so it pays below roughly eight tasks
240    /// and is the owner's call above that.
241    _stride_pad: [u32; STRIDE_PAD_WORDS],
242}
243
244/// How many `u32`s of [`Tcb::_stride_pad`] it takes to make `Slot<Tcb>` a power
245/// of two, which is what keeps `index * stride` a shift instead of a multiply.
246///
247/// It differs by pointer width because the fields it pads out do: `usize` and
248/// the `Name` alignment (see [`crate::name::Name`], aligned to eight on a
249/// 64-bit host only) both change size. Nine words gives 128 bytes on a 32-bit
250/// target -- the number `bench/kernel-ram` pins and `bench/kernel-flash`'s
251/// `mul = 0` depends on -- and four gives 128 on the host.
252///
253/// Before this was split, the host slot was 144 bytes and every TCB index cost
254/// a `lea`+`shl` where a power-of-two stride costs one `shl`.
255#[cfg(target_pointer_width = "64")]
256pub(crate) const STRIDE_PAD_WORDS: usize = 4;
257
258/// See the 64-bit case above; nine words is what makes an rv32 `Slot<Tcb>` 128
259/// bytes, and that number is pinned by two benches.
260#[cfg(not(target_pointer_width = "64"))]
261pub(crate) const STRIDE_PAD_WORDS: usize = 9;
262
263/// How many notification slots a task has room for.
264///
265/// The C sizes its arrays from `configTASK_NOTIFICATION_ARRAY_ENTRIES`
266/// directly. A `Config` is a type here and stable Rust cannot size an array
267/// from one of its associated consts, so the arrays are the largest a
268/// configuration may ask for and [`Config::validate`] refuses more. The
269/// cost is a few bytes per task per unused slot; the alternative was a
270/// tenth const generic on every kernel type.
271pub const MAX_NOTIFICATION_ENTRIES: usize = 4;
272
273/// `ucNotifyState[]`: what a task's notification slot is doing.
274#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
275pub enum NotifyState {
276    /// `taskNOT_WAITING_NOTIFICATION`.
277    #[default]
278    NotWaiting,
279    /// `taskWAITING_NOTIFICATION`.
280    Waiting,
281    /// `taskNOTIFICATION_RECEIVED`.
282    Received,
283}
284
285/// `eNotifyAction`: what a notification does to the value already there.
286#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
287pub enum NotifyAction {
288    /// `eNoAction`: wake the task, leave the value alone.
289    #[default]
290    None,
291    /// `eSetBits`.
292    SetBits,
293    /// `eIncrement`.
294    Increment,
295    /// `eSetValueWithOverwrite`.
296    Overwrite,
297    /// `eSetValueWithoutOverwrite`: fails if one is already pending.
298    NoOverwrite,
299}
300
301/// What `xQueueGenericSend` and friends keep between passes of their
302/// `for(;;)`: the queue being waited on, the ticks left, and the timeout
303/// bookkeeping (`TimeOut_t`).
304#[derive(Debug, Clone, Copy, Default)]
305struct WaitFrame {
306    /// The queue this task is part-way through a blocking call on;
307    /// `TaskHandle::NULL`-equivalent when there is none.
308    queue: QueueHandle,
309    /// `xTicksToWait`, decremented by each `xTaskCheckForTimeOut`.
310    ///
311    /// All three are SPLIT (`timer::Split64`), so a `WaitFrame` does not align
312    /// a `Tcb` to eight. An arena slot pays its value's alignment as padding,
313    /// and `Tcb` is one per task.
314    ticks: crate::timer::Split64,
315    /// `xTimeOut.xTimeOnEntering`.
316    entering: crate::timer::Split64,
317    /// `xTimeOut.xOverflowCount`.
318    /// Counted in a `u32`: it advances once per delayed-list swap, so four
319    /// billion of them is not a bound anything reaches, and a `u64` here was
320    /// two instructions an operation on rv32.
321    overflows: u32,
322    /// `xEntryTimeSet`.
323    entry_set: bool,
324    /// `xInheritanceOccurred`.
325    inherited: bool,
326}
327
328/// What [`Kernel::start_scheduler`] created, so a runner can attach bodies.
329#[derive(Debug, Clone, Copy)]
330pub struct StartHandles {
331    /// The idle task (core 0's, on an SMP build).
332    pub idle: TaskHandle,
333    /// SMP: core 1's passive idle task (`IDLE1`); null on one core.
334    pub passive_idle: TaskHandle,
335    /// The timer service task.
336    pub timer: TaskHandle,
337    /// The timer command queue.
338    pub timer_queue: QueueHandle,
339}
340
341/// The most cores a kernel can be configured for (`configNUMBER_OF_CORES`).
342///
343/// Two, because the part SMP exists for here is the ESP32-S3. The per-core
344/// state -- `current`, its cached priority, `xYieldPendings` -- is an array
345/// of this length, and on a one-core build every index into it is the
346/// CONSTANT zero ([`Kernel::core`]), so the arrays cost one slot of
347/// `TaskHandle` and two bytes and no instruction: `self.current[0]` is the
348/// same load the scalar field was.
349pub const MAX_CORES: usize = 2;
350// [`Kernel::core`] MASKS a core id with `MAX_CORES - 1`. Out here rather
351// than in that body: a `const` block inside an `#[inline(always)]` accessor
352// is MIR the one-core build weighs at every call site.
353const _: () = assert!(MAX_CORES.is_power_of_two());
354
355/// The scheduler.
356///
357/// `TASKS`, `ITEMS`, `LISTS`, `QUEUES`, `SLOTS`, `BUFFERS` and `BYTES` are
358/// the geometry: how many tasks, list items, lists, queues, queue item
359/// slots, stream buffers, and stream-buffer bytes this kernel has room
360/// for. See the crate docs and [`items_for`] / [`lists_for`].
361/// [`Kernel::new`] refuses a geometry that does not add up.
362#[repr(C)]
363pub struct Kernel<
364    C: Config,
365    P: Port,
366    T: Trace,
367    H,
368    const TASKS: usize,
369    const ITEMS: usize,
370    const LISTS: usize,
371    const QUEUES: usize,
372    const SLOTS: usize,
373    const BUFFERS: usize,
374    const BYTES: usize,
375    const TIMERS: usize,
376    const GROUPS: usize,
377    const TIMER_CMDS: usize,
378> {
379    /// `xTickCount`, masked to the configuration's tick width.
380    pub(crate) tick: u64,
381    /// `xPendedTicks`.
382    /// Counted in a `u32`, for the reason `overflows` is: it counts ticks
383    /// that arrived while the scheduler was suspended, which a suspension
384    /// long enough to overflow four billion would have to outlive.
385    pended_ticks: u32,
386    /// `xNextTaskUnblockTime`.
387    next_unblock_time: u64,
388    /// `xNumOfOverflows`.
389    overflows: u32,
390    /// `xLastTime` in `prvSampleTimeNow`.
391    pub(crate) timer_last_time: u64,
392    /// `pxCurrentTCB`.
393    pub(crate) current: [TaskHandle; MAX_CORES],
394    /// `uxSchedulerSuspended`.
395    suspended_depth: u32,
396    /// `uxCurrentNumberOfTasks`.
397    task_count: usize,
398    /// How many times the scheduler could not choose a task, and why the
399    /// FIRST time. See [`Stall`].
400    stalls: u32,
401    pub(crate) timer_message_next: usize,
402    pub(crate) bytes_used: usize,
403    pub(crate) free_count: usize,
404    pub(crate) free_slot_count: usize,
405    pub(crate) slots_used: usize,
406    /// The task whose abandoned frame is running right now, if any.
407    /// `TaskHandle::NULL` for "nobody", not `Option<TaskHandle>`.
408    ///
409    /// A `Handle` is two `u16`s with no niche, so the `Option` was eight bytes
410    /// and a real discriminant — and `resume_pending` tests this field on
411    /// every step the runner takes, which is the most-called entry in the
412    /// kernel. `NULL` is already the sentinel (`generation == 0`), so the
413    /// representation carries the fact the discriminant was carrying.
414    unwinding: TaskHandle,
415    /// A task that deleted ITSELF and whose slot the idle task has not
416    /// reclaimed yet — the C's `xTasksWaitingTermination`, which only ever
417    /// holds the running task, because any other task is freed on the spot.
418    ///
419    /// One slot is enough, and that is a property rather than a guess: a
420    /// self-deleting task is off every ready list and yields immediately, so
421    /// a second self-delete cannot happen until a switch has occurred, and
422    /// the switch is where this is reaped.
423    awaiting_reap: TaskHandle,
424    /// `xTimerQueue`.
425    pub(crate) timer_queue: QueueHandle,
426    /// How many tasks are parked at a queue call's sampling exit.
427    ///
428    /// A summary, so that [`Kernel::take_queue_resume`] — which runs at the
429    /// top of every queue send and every queue receive — can answer "nothing
430    /// to resume" from one field instead of resolving the caller's TCB out of
431    /// the arena. Only the three sites that park a task raise it, and only a
432    /// take that finds a slot set lowers it, so it counts exactly the slots
433    /// that are not `QueueResume::No`.
434    queue_resumes: u16,
435    /// How many tasks are parked inside an event-group wait, for the same
436    /// reason [`Kernel::queue_resumes`] exists: `take_event_resume` runs at
437    /// the top of every `event_group_wait_bits` and every `event_group_sync`,
438    /// and on all but the parked few the answer is no.
439    event_resumes: u16,
440    /// `uxTopReadyPriority`.
441    top_ready_priority: u8,
442    /// The running task's scheduling priority, kept beside `current`.
443    ///
444    /// The C reads `pxCurrentTCB->uxPriority`: one pointer dereference,
445    /// because the pointer IS the task. A handle is not a pointer, so the
446    /// same read went through `Arena::resolve` — a bounds check, a generation
447    /// compare and an `Option` — **every tick**, on the one handle the kernel
448    /// itself maintains and cannot have wrong.
449    ///
450    /// Measured: that resolve was **43 of the tick's 56 instructions, 77 %**.
451    /// Stubbing it to a constant took `increment_tick` from 56 to 13 against
452    /// the C's 15 — from 3.73x against us to 0.87x. This field is that
453    /// finding, done correctly.
454    ///
455    /// **It is written in exactly two places** — [`Self::set_current`] and
456    /// [`Self::set_task_priority`] — so that a call site cannot forget it.
457    /// `current_priority` debug-asserts it against the resolved value, so the
458    /// kernel's own tests and the conformance corpus fail loudly if it ever
459    /// drifts.
460    current_priority: [u8; MAX_CORES],
461    /// `xSchedulerRunning`.
462    running: bool,
463    /// `xYieldPendings`, per core.
464    yield_pending: [bool; MAX_CORES],
465    /// Which of the two delayed lists is `pxDelayedTaskList` right now.
466    delayed_swapped: bool,
467    /// Which of the two timer lists is `pxCurrentTimerList` right now.
468    pub(crate) timers_swapped: bool,
469    first_stall: Stall,
470
471    pub(crate) port: P,
472    pub(crate) trace: T,
473    /// `vApplicationTickHook`, held by value so it can borrow the kernel.
474    pub(crate) tick_hook: H,
475    /// How many outermost critical-section exits a task owes the clock.
476    ///
477    /// This is `uxSavedCriticalNesting` in the Posix port's
478    /// `prvSwitchThread` and then some. A thread stops at the switch; a
479    /// stackless call does not, so the abandoned frame runs on — closing
480    /// the sections it had open and, on some paths, opening one more. The
481    /// port tallies every exit that frame makes instead of counting it as
482    /// sim time, and the tally is paid here when the task runs again. That
483    /// is what puts a sim tick where the C one lands.
484    /// Five per-task booleans, one bit each, in ONE array.
485    ///
486    /// They were five `[bool; TASKS]`. Each one is a separate base address, so
487    /// a function touching two of them — and the owed body touches three —
488    /// computed two bases and did two loads to read two bits that now share a
489    /// byte. Packed they cost one byte a task instead of five, and one base
490    /// instead of five across the whole kernel.
491    flags: [u8; TASKS],
492    /// `started`, kept OUT of `flags` on purpose.
493    ///
494    /// It is the only one of the five that `switch_context` reads — through
495    /// `hand_over`, on every switch — and packing it cost the selection row
496    /// one instruction for the mask. The other four are never on that path,
497    /// so they stay packed. Per-call-site, not per-idea.
498    started: [bool; TASKS],
499    /// `owes_anything`, kept OUT of `flags` for the reason `started` is: it is
500    /// the single most-read byte in the kernel — `resume_pending` tests it
501    /// before every step the runner takes — and packing it cost that row three
502    /// instructions for the mask. Per call site, not per idea.
503    owes_anything: [bool; TASKS],
504    /// `delay_aborted`, kept OUT of `flags`: `check_for_timeout` reads it on
505    /// every pass of every blocking call.
506    delay_aborted: [bool; TASKS],
507    pub(crate) lists: Lists<ITEMS, LISTS>,
508    pub(crate) tcbs: Arena<TaskKind, Tcb, TASKS>,
509    owed_exits: [u32; TASKS],
510    /// A trace line a task owes from a call it was switched out of.
511    ///
512    /// `xQueueReceive`'s failure path is `taskEXIT_CRITICAL();
513    /// traceQUEUE_RECEIVE_FAILED( pxQueue ); return errQUEUE_EMPTY;` — and
514    /// if the tick that the exit released switches the task away, those two
515    /// lines sit on a stack that is not running. The C emits the trace when
516    /// the task resumes, so this kernel does too.
517    owed_trace: [OwedTrace; TASKS],
518    pub(crate) queues: Arena<QueueKind, Queue, QUEUES>,
519    pub(crate) timers: Arena<TimerKind, Timer, TIMERS>,
520    pub(crate) groups: Arena<EventGroupKind, EventGroup, GROUPS>,
521    pub(crate) buffers: Arena<StreamKind, StreamBuffer, BUFFERS>,
522    /// Blocks of the byte arena that a deleted stream buffer gave back,
523    /// as `(base, length)`, kept sorted and coalesced.
524    ///
525    /// This is the one allocator in the kernel, and it exists because the
526    /// C's stream buffers are heap objects: `MessageBufferDemo`'s echo
527    /// server creates one and deletes it again on every loop, and a bump
528    /// allocator would run out in a few hundred ticks. At most `BUFFERS`
529    /// buffers can be alive, so at most `BUFFERS` holes can exist between
530    /// them, which is why the list is that long and cannot overflow.
531    pub(crate) free_blocks: [(usize, usize); BUFFERS],
532    /// Extents of the SLOT arena that a deleted queue gave back, as
533    /// `(base, length)`, coalesced the same way `free_blocks` is.
534    ///
535    /// The byte arena has had this since `MessageBufferDemo` needed it. The
536    /// slot arena did not, so `queue_delete` returned the descriptor while
537    /// the storage stayed spent and a create/delete loop exhausted `SLOTS`
538    /// permanently — found through `AbortDelay` on 2026-09-21, after nine
539    /// hypotheses and eight refutations, because the descriptor arena works
540    /// perfectly and that is where everyone looked.
541    ///
542    /// At most `QUEUES` queues are alive, so at most `QUEUES` holes can sit
543    /// between them and the list cannot overflow.
544    pub(crate) free_slots: [(usize, usize); QUEUES],
545    /// The ring of `DaemonTaskMessage_t`s the timer queue carries indices
546    /// into. It is exactly as long as the queue, so a message can only be
547    /// overwritten once the queue has already refused to hold its index.
548    /// The timer daemon's command mailbox, sized by the configuration.
549    ///
550    /// It used to be `[Message; MAX_TIMER_COMMANDS]` — a hardcoded 32 — while
551    /// `Config::TIMER_QUEUE_LENGTH` already said how many the configuration
552    /// wanted and defaulted to ten. Because it scaled with **nothing**, it was
553    /// a fixed 768 bytes at every geometry, which is **39 % of a two-task
554    /// kernel's entire static footprint** and 31 % of a four-task one.
555    ///
556    /// `TIMER_CMDS` is that number, checked against the config in
557    /// [`Self::with_tick_hook`] exactly as `ITEMS` and `LISTS` are.
558    pub(crate) timer_messages: [Message; TIMER_CMDS],
559    pub(crate) slots: [u64; SLOTS],
560    pub(crate) bytes: [u8; BYTES],
561    _config: PhantomData<C>,
562    // ---- SMP state, deliberately LAST ------------------------------------
563    //
564    // `Kernel` is `#[repr(C)]`, so where a field sits is a layout decision.
565    // These three were first added beside `yield_pending` and cost the
566    // ONE-core scheduling bench **+24,000 Ir (+1.6 %)** without a line of SMP
567    // logic running: they moved every hot field behind them. At the end of
568    // the struct they cost nothing (`bench/ksched-ir`, 2026-10-01).
569    /// SMP: `pxCurrentTCBs[ core ]->xTaskRunState == taskTASK_SCHEDULED_TO_YIELD`.
570    ///
571    /// The C keeps a run state in every TCB. Here "running on core `c`" IS
572    /// `current[c]`, so the one state that is not derivable -- "running, but
573    /// another core has asked it to yield" -- is kept per core instead, and
574    /// the TCB is untouched (a one-core build does not pay for it).
575    yield_requested: [bool; MAX_CORES],
576    /// SMP: the cores this kernel has asked to yield and the port has not yet
577    /// interrupted, one bit per core (`portYIELD_CORE`). Drained by
578    /// [`Kernel::take_core_yields`], because raising another core's
579    /// interrupt is the PORT's act and the kernel is `forbid(unsafe)`.
580    core_yields: u8,
581    /// SMP: `xIdleTaskHandles`, so a running idle task can be ranked below
582    /// every real task of priority 0 (`prvYieldForTask`).
583    idle: [TaskHandle; MAX_CORES],
584    /// SMP: a second task deleted while a core still held it. One core can
585    /// owe at most one deferred reap (its own task); two cores can owe two,
586    /// so SMP needs a second slot -- here at the end for the same layout
587    /// reason as the fields above.
588    awaiting_reap_smp: TaskHandle,
589}
590
591impl<
592    C: Config,
593    P: Port,
594    T: Trace,
595    H,
596    const TASKS: usize,
597    const ITEMS: usize,
598    const LISTS: usize,
599    const QUEUES: usize,
600    const SLOTS: usize,
601    const BUFFERS: usize,
602    const BYTES: usize,
603    const TIMERS: usize,
604    const GROUPS: usize,
605    const TIMER_CMDS: usize,
606> Kernel<C, P, T, H, TASKS, ITEMS, LISTS, QUEUES, SLOTS, BUFFERS, BYTES, TIMERS, GROUPS, TIMER_CMDS>
607where
608    H: TickHook<Self>,
609{
610    /// `portMAX_DELAY` at this configuration's tick width.
611    pub const MAX_DELAY: u64 = <C::Tick as TickWidth>::MAX;
612
613    // ------------------------------------------------------- footprint --
614
615    /// The kernel's own static size, and the same number `size_of` gives.
616    ///
617    /// The consts below decompose it. They exist because a footprint you
618    /// cannot decompose to the byte is a footprint you cannot optimise: the
619    /// arena fields are `pub(crate)` and their types are private, so nothing
620    /// outside this crate can measure them, and a decomposition assembled
621    /// from per-dimension SLOPES is not one — `ITEMS` and `LISTS` are derived
622    /// from `TASKS`, `QUEUES` and `GROUPS`, so the function is not linear and
623    /// slopes do not sum to the total. That was tried; it produced 696 bytes
624    /// of nonsense.
625    ///
626    /// Every one is `size_of` on whatever target this is compiled for, which
627    /// is the only place the number means anything: a host `usize` is eight
628    /// bytes and rv32's is four.
629    ///
630    /// [`FOOTPRINT_ACCOUNTED`](Self::FOOTPRINT_ACCOUNTED) sums them, and the
631    /// difference from [`FOOTPRINT`](Self::FOOTPRINT) is padding plus the
632    /// scalar tail — print both, never one. `bench/kernel-ram` does.
633    pub const FOOTPRINT: usize = core::mem::size_of::<Self>();
634
635    /// The task control blocks.
636    pub const FOOTPRINT_TCBS: usize = core::mem::size_of::<Arena<TaskKind, Tcb, TASKS>>();
637    /// The queue descriptors, and NOT their item storage.
638    pub const FOOTPRINT_QUEUES: usize = core::mem::size_of::<Arena<QueueKind, Queue, QUEUES>>();
639    /// Every intrusive list: ready, delayed, and the per-object waiters.
640    pub const FOOTPRINT_LISTS: usize = core::mem::size_of::<Lists<ITEMS, LISTS>>();
641    /// Queue item storage, as `u64` slots.
642    pub const FOOTPRINT_SLOTS: usize = core::mem::size_of::<[u64; SLOTS]>();
643    /// The stream-buffer descriptors, and NOT their bytes.
644    pub const FOOTPRINT_BUFFERS: usize =
645        core::mem::size_of::<Arena<StreamKind, StreamBuffer, BUFFERS>>();
646    /// The software timers.
647    pub const FOOTPRINT_TIMERS: usize = core::mem::size_of::<Arena<TimerKind, Timer, TIMERS>>();
648    /// The event groups.
649    pub const FOOTPRINT_GROUPS: usize =
650        core::mem::size_of::<Arena<EventGroupKind, EventGroup, GROUPS>>();
651    /// The timer daemon's command mailbox.
652    ///
653    /// It scales with `TIMER_CMDS` now, which the geometry check pins to
654    /// `Config::TIMER_QUEUE_LENGTH`. It used to be a hardcoded 32 and so
655    /// scaled with nothing — a fixed 768 bytes, 39 % of a two-task kernel.
656    /// The line stays separate because that history is worth being able to
657    /// read off a decomposition.
658    pub const FOOTPRINT_TIMER_MESSAGES: usize = core::mem::size_of::<[Message; TIMER_CMDS]>();
659    /// The byte arena, for stream-buffer contents.
660    pub const FOOTPRINT_BYTES: usize = core::mem::size_of::<[u8; BYTES]>();
661    /// The two free lists, over the byte arena and the slot arena.
662    pub const FOOTPRINT_FREE_LISTS: usize = core::mem::size_of::<[(usize, usize); BUFFERS]>()
663        + core::mem::size_of::<[(usize, usize); QUEUES]>();
664
665    /// The seven per-task side arrays, together.
666    ///
667    /// Seven arrays indexed by the same task, five of them `[bool; TASKS]`.
668    /// Whether that wants to be one array of a flags byte is a question this
669    /// number is here to make askable.
670    pub const FOOTPRINT_PER_TASK_SIDE: usize = core::mem::size_of::<[u8; TASKS]>()
671        + core::mem::size_of::<[bool; TASKS]>() * 3
672        + core::mem::size_of::<[OwedTrace; TASKS]>()
673        + core::mem::size_of::<[u32; TASKS]>();
674
675    /// What the lines above account for.
676    ///
677    /// `FOOTPRINT - FOOTPRINT_ACCOUNTED` is the unattributed remainder: the
678    /// scalar tail (`tick`, `current`, the counters) plus whatever padding
679    /// the layout inserts. It was 140 bytes of 6,784 when this was written. A
680    /// remainder that grows is a field somebody added without adding a line
681    /// here.
682    pub const FOOTPRINT_ACCOUNTED: usize = Self::FOOTPRINT_TCBS
683        + Self::FOOTPRINT_QUEUES
684        + Self::FOOTPRINT_LISTS
685        + Self::FOOTPRINT_SLOTS
686        + Self::FOOTPRINT_BUFFERS
687        + Self::FOOTPRINT_TIMERS
688        + Self::FOOTPRINT_GROUPS
689        + Self::FOOTPRINT_TIMER_MESSAGES
690        + Self::FOOTPRINT_BYTES
691        + Self::FOOTPRINT_FREE_LISTS
692        + Self::FOOTPRINT_PER_TASK_SIDE;
693
694    // ------------------------------------------------------------ setup --
695
696    /// A kernel with no tasks, ready for [`Kernel::create_task`].
697    ///
698    /// # Errors
699    /// [`Error::InvalidArgument`] if the configuration is invalid
700    /// ([`Config::validate`]), if the name capacity cannot hold
701    /// `MAX_TASK_NAME_LEN`, or if the const geometry does not match the
702    /// configuration: `ITEMS` must be [`items_for`]`(TASKS)` and `LISTS`
703    /// must be [`lists_for`]`(MAX_PRIORITIES, QUEUES)`.
704    pub fn new(port: P, trace: T) -> Result<Self>
705    where
706        H: Default,
707    {
708        Self::with_tick_hook(port, trace, H::default())
709    }
710
711    /// [`Kernel::new`] with `vApplicationTickHook` installed.
712    ///
713    /// # Errors
714    /// As [`Kernel::new`].
715    pub fn with_tick_hook(port: P, trace: T, tick_hook: H) -> Result<Self> {
716        C::validate()?;
717        if ITEMS != list_slots_for(TASKS, TIMERS, LISTS)
718            || LISTS != lists_for(C::MAX_PRIORITIES, QUEUES, GROUPS)
719            || TASKS == 0
720            || C::MAX_TASK_NAME_LEN > crate::NAME_CAPACITY
721            // The mailbox is a declared dimension now, so a declaration that
722            // disagrees with its own config is refused rather than silently
723            // sized to whichever one the type happened to carry.
724            || TIMER_CMDS != C::TIMER_QUEUE_LENGTH
725        {
726            return Err(Error::InvalidArgument);
727        }
728        Ok(Self {
729            port,
730            trace,
731            tcbs: Arena::new(),
732            queues: Arena::new(),
733            lists: Lists::new(),
734            slots: [0; SLOTS],
735            slots_used: 0,
736            buffers: Arena::new(),
737            timers: Arena::new(),
738            groups: Arena::new(),
739            timer_messages: [Message::default(); TIMER_CMDS],
740            timer_message_next: 0,
741            timers_swapped: false,
742            timer_last_time: 0,
743            timer_queue: QueueHandle::NULL,
744            bytes: [0; BYTES],
745            bytes_used: 0,
746            free_blocks: [(0, 0); BUFFERS],
747            free_count: 0,
748            free_slots: [(0, 0); QUEUES],
749            free_slot_count: 0,
750            current: [TaskHandle::NULL; MAX_CORES],
751            top_ready_priority: 0,
752            current_priority: [0; MAX_CORES],
753            tick: C::INITIAL_TICK_COUNT,
754            pended_ticks: 0,
755            suspended_depth: 0,
756            running: false,
757            next_unblock_time: Self::MAX_DELAY,
758            yield_pending: [false; MAX_CORES],
759            yield_requested: [false; MAX_CORES],
760            core_yields: 0,
761            idle: [TaskHandle::NULL; MAX_CORES],
762            awaiting_reap_smp: TaskHandle::NULL,
763            task_count: 0,
764            stalls: 0,
765            first_stall: Stall::None,
766            awaiting_reap: TaskHandle::NULL,
767            delayed_swapped: false,
768            overflows: 0,
769            flags: [0; TASKS],
770            started: [false; TASKS],
771            owes_anything: [false; TASKS],
772            delay_aborted: [false; TASKS],
773            owed_exits: [0; TASKS],
774            unwinding: TaskHandle::NULL,
775            queue_resumes: 0,
776            event_resumes: 0,
777            tick_hook,
778            owed_trace: [OwedTrace::None; TASKS],
779            _config: PhantomData,
780        })
781    }
782
783    /// The port, for a runner that needs its counters.
784    pub const fn port(&self) -> &P {
785        &self.port
786    }
787
788    /// The trace sink.
789    pub const fn trace(&self) -> &T {
790        &self.trace
791    }
792
793    /// The trace sink, mutably.
794    pub const fn trace_mut(&mut self) -> &mut T {
795        &mut self.trace
796    }
797
798    /// Take the trace sink back when the run is over, so a deliverable can
799    /// flush whatever it was writing to.
800    pub fn into_trace(self) -> T {
801        self.trace
802    }
803
804    /// `xNumOfOverflows`, for a scenario that reports counters.
805    #[must_use]
806    pub const fn overflows(&self) -> u64 {
807        self.overflows as u64
808    }
809
810    // ------------------------------------------------------ list indices --
811
812    // `ListId` is a `u8` and `ItemId` a `u16` (the core's `list`): the
813    // whole scheduler addresses lists and items by small integers, which is
814    // what keeps a TCB free of pointers.
815
816    pub(crate) const fn ready_list(priority: u8) -> ListId {
817        priority
818    }
819
820    const fn delayed_list(&self) -> ListId {
821        if self.delayed_swapped {
822            C::MAX_PRIORITIES.saturating_add(1)
823        } else {
824            C::MAX_PRIORITIES
825        }
826    }
827
828    const fn overflow_delayed_list(&self) -> ListId {
829        if self.delayed_swapped {
830            C::MAX_PRIORITIES
831        } else {
832            C::MAX_PRIORITIES.saturating_add(1)
833        }
834    }
835
836    const fn pending_ready_list() -> ListId {
837        C::MAX_PRIORITIES.saturating_add(2)
838    }
839
840    const fn suspended_list() -> ListId {
841        C::MAX_PRIORITIES.saturating_add(3)
842    }
843
844    /// `xTasksWaitingToSend` of a queue.
845    pub(crate) fn queue_send_list(queue: QueueHandle) -> ListId {
846        let offset = u8::try_from(queue.index().saturating_mul(2)).unwrap_or(u8::MAX);
847        C::MAX_PRIORITIES
848            .saturating_add(u8::try_from(OVERHEAD_LISTS).unwrap_or(u8::MAX))
849            .saturating_add(offset)
850    }
851
852    /// `xTasksWaitingToReceive` of a queue.
853    pub(crate) fn queue_receive_list(queue: QueueHandle) -> ListId {
854        Self::queue_send_list(queue).saturating_add(1)
855    }
856
857    /// A task's `xStateListItem`: the task's own arena index.
858    pub(crate) const fn state_item(task: TaskHandle) -> ItemId {
859        task.index() as u16
860    }
861
862    /// A task's `xEventListItem`: `TASKS` above its state item, so the two
863    /// never collide and either maps back to its task by arithmetic alone.
864    pub(crate) fn event_item(task: TaskHandle) -> ItemId {
865        Self::task_item_base().saturating_add(task.index() as u16)
866    }
867
868    fn task_item_base() -> ItemId {
869        u16::try_from(TASKS).unwrap_or(u16::MAX)
870    }
871
872    fn task_of_state_item(&self, item: ItemId) -> Result<TaskHandle> {
873        self.tcbs.handle_at(item).ok_or(Error::Gone)
874    }
875
876    fn task_of_event_item(&self, item: ItemId) -> Result<TaskHandle> {
877        let index = item
878            .checked_sub(Self::task_item_base())
879            .ok_or(Error::InvalidArgument)?;
880        self.tcbs.handle_at(index).ok_or(Error::Gone)
881    }
882
883    // ----------------------------------------------------------- getters --
884
885    /// `pxCurrentTCB`: the task running on the calling core.
886    ///
887    /// No longer `const`: on an SMP build the answer depends on which core
888    /// asks (`portGET_CORE_ID()`), which is the port's to say.
889    #[must_use]
890    pub fn current(&self) -> TaskHandle {
891        self.cur()
892    }
893
894    /// `pxCurrentTCBs[ core ]`: the task running on `core`, or the null
895    /// handle for a core this configuration does not have.
896    #[must_use]
897    pub fn current_on(&self, core: usize) -> TaskHandle {
898        if core < Self::cores() {
899            self.current_of(core)
900        } else {
901            TaskHandle::NULL
902        }
903    }
904
905    /// `xTaskGetTickCount`.
906    #[must_use]
907    pub const fn tick_count(&self) -> u64 {
908        self.tick
909    }
910
911    /// `xSchedulerRunning`.
912    #[must_use]
913    pub const fn is_running(&self) -> bool {
914        self.running
915    }
916
917    /// `uxCurrentNumberOfTasks`.
918    #[must_use]
919    pub const fn task_count(&self) -> usize {
920        self.task_count
921    }
922
923    /// `uxSchedulerSuspended`, for a stepper reporting why a tick pended.
924    #[must_use]
925    pub const fn scheduler_suspended(&self) -> u32 {
926        self.suspended_depth
927    }
928
929    /// `xPendedTicks`, likewise.
930    #[must_use]
931    pub const fn pended_ticks(&self) -> u64 {
932        self.pended_ticks as u64
933    }
934
935    /// The item ids in `priority`'s ready list, in order, into `out`.
936    ///
937    /// Returns how many were written. The companion to
938    /// [`Kernel::ready_cursor`]: when a ready task is never chosen, the
939    /// cursor says whether the rotation moved and this says what it was
940    /// moving THROUGH.
941    pub fn ready_items(&self, priority: u8, out: &mut [u16]) -> usize {
942        let mut n = 0;
943        for item in self.lists.iter(Self::ready_list(priority)) {
944            let Some(slot) = out.get_mut(n) else {
945                break;
946            };
947            *slot = item;
948            n = n.wrapping_add(1);
949        }
950        n
951    }
952
953    /// Where the round-robin cursor sits in `priority`'s ready list.
954    ///
955    /// A diagnostic, paired with [`Kernel::ready_len`]: a ready task that
956    /// is never chosen is either not in the list the scheduler looks at,
957    /// or the cursor is not moving. These two answer both halves.
958    #[must_use]
959    pub fn ready_cursor(&self, priority: u8) -> u16 {
960        self.lists
961            .cursor_of(Self::ready_list(priority))
962            .unwrap_or(0)
963    }
964
965    /// The live handle in arena slot `index`, if that slot holds a task.
966    ///
967    /// The generation is the point: a bare index names a SLOT, and a slot
968    /// outlives the tasks that occupy it, so an index is not a task. This
969    /// answers the handle, which is.
970    ///
971    /// It exists so a diagnostic can walk every task without knowing any
972    /// of their names -- `task_get_handle` needs the name it is looking
973    /// for, and a crash report is exactly the case where you do not have
974    /// one.
975    #[must_use]
976    pub fn task_at(&self, index: usize) -> Option<TaskHandle> {
977        let index = u16::try_from(index).ok()?;
978        self.tcbs.handle_at(index)
979    }
980
981    /// A task's name.
982    ///
983    /// # Errors
984    /// [`Error::Gone`] for a stale or null handle.
985    pub fn name_of(&self, task: TaskHandle) -> Result<Name> {
986        self.tcbs.resolve(task).map(|t| t.name)
987    }
988
989    /// `uxTaskPriorityGet`. `None` means the current task, as in C.
990    ///
991    /// # Errors
992    /// [`Error::Gone`] for a stale handle.
993    pub fn priority_of(&self, task: Option<TaskHandle>) -> Result<u8> {
994        let h = task.unwrap_or(self.cur());
995        self.tcbs.resolve(h).map(|t| t.priority)
996    }
997
998    pub(crate) fn current_priority(&self) -> u8 {
999        // The cache is the whole point; this is the proof it is honest. It
1000        // costs nothing in release and fails the kernel's own tests, the
1001        // Kani proofs and the conformance corpus the moment a new write to
1002        // `current` or to a priority forgets its helper.
1003        debug_assert_eq!(
1004            self.cur_priority(),
1005            self.priority_of(None).unwrap_or(0),
1006            "the cached current priority went stale -- a write to `current` or              to a task's priority bypassed set_current/set_task_priority"
1007        );
1008        self.cur_priority()
1009    }
1010
1011    /// Make `task` the running task, keeping [`Self::current_priority`] with
1012    /// it, where the caller already knows the priority.
1013    ///
1014    /// One of the two places that field is written; assigning `self.current`
1015    /// directly is what this exists to prevent.
1016    ///
1017    /// **The priority is a parameter and not resolved here**, because every
1018    /// caller already has it. `switch_context`'s search loop finds the highest
1019    /// non-empty ready list and takes `next` out of it, so `next`'s priority
1020    /// IS that list's index; task creation has it as an argument. Resolving
1021    /// the TCB again to learn what the caller just proved cost **9
1022    /// instructions on the switch row** — measured, by landing this without
1023    /// the parameter and watching `switch_select` go 79 → 88, then 88 → 82
1024    /// when it was threaded through.
1025    fn set_current_at(&mut self, task: TaskHandle, priority: u8) {
1026        let core = self.core();
1027        if let Some(slot) = self.current.get_mut(core) {
1028            *slot = task;
1029        }
1030        if let Some(slot) = self.current_priority.get_mut(core) {
1031            *slot = priority;
1032        }
1033    }
1034
1035    /// `pxCurrentTCB` for the calling core.
1036    ///
1037    /// ★ On one core this DESTRUCTURES element 0 rather than calling
1038    /// `.get(0).copied().unwrap_or(..)`. The two should fold to the same
1039    /// load and measurably do not: with the `Option` form on every per-core
1040    /// accessor the scheduling bench (`bench/ksched-ir`) read **+14,985 Ir
1041    /// (+0.98 %)** against the scalar field; destructuring took that to
1042    /// +2,985. `.get` stays for the SMP arm, where the index is a runtime
1043    /// core id and the kernel stays free of panicking indexing.
1044    #[inline(always)]
1045    pub(crate) fn cur(&self) -> TaskHandle {
1046        if C::NUMBER_OF_CORES <= 1 {
1047            let [first, ..] = self.current;
1048            return first;
1049        }
1050        self.current
1051            .get(self.core())
1052            .copied()
1053            .unwrap_or(TaskHandle::NULL)
1054    }
1055
1056    /// The calling core's cached running priority (see `current_priority`).
1057    #[inline(always)]
1058    fn cur_priority(&self) -> u8 {
1059        if C::NUMBER_OF_CORES <= 1 {
1060            let [first, ..] = self.current_priority;
1061            return first;
1062        }
1063        self.current_priority.get(self.core()).copied().unwrap_or(0)
1064    }
1065
1066    /// `xYieldPendings[ portGET_CORE_ID() ]`.
1067    #[inline(always)]
1068    fn pending_here(&self) -> bool {
1069        if C::NUMBER_OF_CORES <= 1 {
1070            let [first, ..] = self.yield_pending;
1071            return first;
1072        }
1073        self.yield_pending
1074            .get(self.core())
1075            .copied()
1076            .unwrap_or(false)
1077    }
1078
1079    /// Set `xYieldPendings[ portGET_CORE_ID() ]`.
1080    #[inline(always)]
1081    fn set_pending_here(&mut self, value: bool) {
1082        if C::NUMBER_OF_CORES <= 1 {
1083            let [first, ..] = &mut self.yield_pending;
1084            *first = value;
1085            return;
1086        }
1087        let core = self.core();
1088        if let Some(slot) = self.yield_pending.get_mut(core) {
1089            *slot = value;
1090        }
1091    }
1092
1093    /// `portGET_CORE_ID()`, as an index into the per-core state.
1094    ///
1095    /// On a one-core build this is the CONSTANT zero -- the branch is on a
1096    /// `const` and folds -- so every `self.cur()` is the same
1097    /// load the scalar field was, and the port is never asked.
1098    #[inline(always)]
1099    pub(crate) fn core(&self) -> usize {
1100        if C::NUMBER_OF_CORES <= 1 {
1101            0
1102        } else {
1103            // A MASK, not a clamp. `portGET_CORE_ID()` is below the core
1104            // count by definition, and for every such id the two agree; the
1105            // clamp compiled to a compare and a `setne` at every kernel entry
1106            // where the mask is one `and`. An id outside the contract lands
1107            // on SOME core's slot either way, never outside the arrays.
1108            //
1109            // The `.min` is for the ONE-core build, and is folded away here
1110            // (the masked id is already in range). Without it this body is
1111            // lighter MIR than the clamp was, and rustc's MIR inliner -- which
1112            // weighs it before `C::NUMBER_OF_CORES` folds -- inlined the
1113            // one-core kernel differently: +63,896 Ir on `bench/kernel-ir`'s
1114            // kernel rows (step, suspend, delay) for a branch that never runs.
1115            (usize::from(self.port.core_id()) & (MAX_CORES - 1)).min(MAX_CORES - 1)
1116        }
1117    }
1118
1119    /// Write a task's scheduling priority, keeping [`Self::current_priority`]
1120    /// with it when that task is the running one.
1121    ///
1122    /// The other of the two. Note it is the SCHEDULING priority only:
1123    /// `base_priority` does not decide which ready list a task sits on, so it
1124    /// is written directly at its call sites.
1125    ///
1126    /// # Errors
1127    /// [`Error::InvalidHandle`] or [`Error::Gone`] for a handle the arena
1128    /// refuses, exactly as `resolve_mut` does.
1129    fn set_task_priority(&mut self, task: TaskHandle, priority: u8) -> Result<()> {
1130        self.tcbs.resolve_mut(task)?.priority = priority;
1131        // Every core that is running `task` caches its priority; on one core
1132        // the loop is the single compare it always was.
1133        for core in 0..Self::cores() {
1134            if task == self.current_of(core) {
1135                if let Some(slot) = self.current_priority.get_mut(core) {
1136                    *slot = priority;
1137                }
1138            }
1139        }
1140        Ok(())
1141    }
1142
1143    /// `uxTaskPriorityGet`: the C API, which takes a critical section.
1144    ///
1145    /// The distinction matters: on the sim a critical-section exit is where
1146    /// time passes, so reading a priority through the API costs a tick
1147    /// sixteen calls later, exactly as it does in C
1148    /// (`portBASE_TYPE_ENTER_CRITICAL` is `taskENTER_CRITICAL` on the Posix
1149    /// port). The kernel's own reads use the field directly, as C's do.
1150    ///
1151    /// # Errors
1152    /// [`Error::Gone`] for a stale handle.
1153    pub fn task_priority_get(&mut self, task: Option<TaskHandle>) -> Result<u8> {
1154        self.enter_critical();
1155        let result = self.priority_of(task);
1156        self.exit_critical();
1157        result
1158    }
1159
1160    /// `eTaskGetState`: the C API, which takes a critical section for any
1161    /// task but the running one.
1162    ///
1163    /// # Errors
1164    /// A list error surfaces as itself.
1165    pub fn task_state_get(&mut self, task: TaskHandle) -> Result<TaskState> {
1166        if task == self.cur() {
1167            return Ok(TaskState::Running);
1168        }
1169        self.enter_critical();
1170        let result = self.state_of(task);
1171        self.exit_critical();
1172        result
1173    }
1174
1175    /// `eTaskGetState`, derived from the list the task is in.
1176    ///
1177    /// # Errors
1178    /// A list error surfaces as itself.
1179    pub fn state_of(&self, task: TaskHandle) -> Result<TaskState> {
1180        if !self.tcbs.contains(task) {
1181            return Ok(TaskState::Deleted);
1182        }
1183        if task == self.cur() {
1184            return Ok(TaskState::Running);
1185        }
1186        let Some(list) = self.lists.container(Self::state_item(task))? else {
1187            return Ok(TaskState::Deleted);
1188        };
1189        if list == Self::suspended_list() {
1190            if self.lists.container(Self::event_item(task))?.is_some() {
1191                return Ok(TaskState::Blocked);
1192            }
1193            // A task blocked indefinitely on a notification is in the
1194            // suspended list and on no event list at all, so the C scans
1195            // the notification array before it calls such a task suspended.
1196            // Miss this and a stream buffer's reader looks suspended to
1197            // `eTaskGetState` while it is plainly waiting.
1198            let waiting_notification = self
1199                .tcbs
1200                .resolve(task)
1201                .map(|t| {
1202                    t.notify_state
1203                        .iter()
1204                        .take(C::NOTIFICATION_ARRAY_ENTRIES)
1205                        .any(|s| *s == NotifyState::Waiting)
1206                })
1207                .unwrap_or(false);
1208            return Ok(if waiting_notification {
1209                TaskState::Blocked
1210            } else {
1211                TaskState::Suspended
1212            });
1213        }
1214        if list == self.delayed_list() || list == self.overflow_delayed_list() {
1215            return Ok(TaskState::Blocked);
1216        }
1217        Ok(TaskState::Ready)
1218    }
1219
1220    /// How many tasks sit on a priority's ready list — what the idle task
1221    /// checks before yielding (`configIDLE_SHOULD_YIELD`).
1222    ///
1223    /// # Errors
1224    /// [`Error::InvalidPriority`] for a priority outside the configuration.
1225    pub fn ready_len(&self, priority: u8) -> Result<usize> {
1226        if priority >= C::MAX_PRIORITIES {
1227            return Err(Error::InvalidPriority);
1228        }
1229        self.lists.len(Self::ready_list(priority))
1230    }
1231
1232    /// Record that the scheduler could not choose.
1233    ///
1234    /// Counts every occurrence and keeps the FIRST reason: the first is the
1235    /// diagnosis and the rest are consequences.
1236    #[cold]
1237    #[inline(never)]
1238    fn note_stall(&mut self, why: Stall) {
1239        self.stalls = self.stalls.saturating_add(1);
1240        if self.first_stall == Stall::None {
1241            self.first_stall = why;
1242        }
1243    }
1244
1245    /// How many times the scheduler could not choose a task.
1246    ///
1247    /// **Non-zero means an invariant broke.** A healthy kernel never stalls:
1248    /// the idle task is always ready. A firmware that prints this alongside
1249    /// its own counters turns a hang into a sentence.
1250    #[must_use]
1251    pub const fn stalls(&self) -> u32 {
1252        self.stalls
1253    }
1254
1255    /// Why the scheduler first failed to choose. [`Stall::None`] if it never
1256    /// has.
1257    #[must_use]
1258    pub const fn first_stall(&self) -> Stall {
1259        self.first_stall
1260    }
1261
1262    // ------------------------------------------------------------ tracing --
1263
1264    pub(crate) fn trace_task<F>(&mut self, task: TaskHandle, make: F)
1265    where
1266        F: for<'a> FnOnce(TaskHandle, &'a str) -> Event<'a>,
1267    {
1268        // `tcbs` and `trace` are different fields, so the name can be read
1269        // where it lives rather than copied onto the stack to end a borrow
1270        // that never had to end.
1271        //
1272        // A sink that never reads the name pays for neither the lookup nor
1273        // the UTF-8 validation that turning one into a `&str` costs.
1274        let Self {
1275            tcbs,
1276            trace,
1277            port,
1278            tick,
1279            ..
1280        } = self;
1281        let name = if T::WANTS_NAMES {
1282            tcbs.get(task).map(|t| t.name.as_str()).unwrap_or("")
1283        } else {
1284            ""
1285        };
1286        let tick = *tick;
1287        trace.note_exits(port.exits());
1288        trace.event(tick, make(task, name));
1289    }
1290
1291    pub(crate) fn priority_value(raw: u8) -> Priority {
1292        Priority::new(raw, C::MAX_PRIORITIES).unwrap_or(Priority::IDLE)
1293    }
1294
1295    // --------------------------------------------------- critical section --
1296
1297    /// What `pvPortMalloc` costs in sim time on a heap-backed C kernel:
1298    /// `vTaskSuspendAll()` around the allocation, then `xTaskResumeAll()`,
1299    /// whose critical section is one outermost exit.
1300    ///
1301    /// This kernel allocates nothing, so the call is the whole of it. On a
1302    /// configuration with [`Config::DYNAMIC_ALLOCATION`] off it compiles
1303    /// away to nothing at all.
1304    // Out of line: 12 cold create/delete sites, each otherwise inlining
1305    // suspend_all + resume_all. Worth -46 B, and the Ir cost is nil because
1306    // every caller is a create or a delete.
1307    #[inline(never)]
1308    pub(crate) fn account_for_allocation(&mut self) {
1309        if C::DYNAMIC_ALLOCATION {
1310            self.suspend_all();
1311            let _ = self.resume_all();
1312        }
1313    }
1314
1315    /// `taskENTER_CRITICAL()`.
1316    pub fn enter_critical(&mut self) {
1317        self.port.enter_critical();
1318    }
1319
1320    /// `taskEXIT_CRITICAL()`.
1321    ///
1322    /// On the sim this is where time passes: the port raises a tick on
1323    /// every sixteenth outermost exit and the kernel takes it here, which
1324    /// is exactly where the C kernel's `SIGALRM` handler would have run.
1325    ///
1326    /// NOT in line, though it is three statements called 118,640 times in
1327    /// a 20,000-tick `StreamBufferDemo`. `#[inline(always)]` was measured:
1328    /// it saves that scenario 210,292 and costs `BlockQ` 200,912 and
1329    /// `GenQTest` 280,540, for a net loss of 271,962 over six scenarios.
1330    /// Whatever the hot sites gain, the switch-heavy ones lose more.
1331    pub fn exit_critical(&mut self) {
1332        self.port.exit_critical();
1333        if self.port.take_pending_tick() {
1334            self.tick_on_exit();
1335        }
1336    }
1337
1338    /// The tick as [`Kernel::exit_critical`] reaches it, deliberately out of
1339    /// line.
1340    ///
1341    /// `exit_critical` is inlined into every queue walker and every demo
1342    /// step, and the tick is its rare arm -- 8,000 of 48,000 sends in
1343    /// `khot-ir` raise one, and 38,013 ticks across 18 scenarios in
1344    /// `kernel-ir`. Left in line it put `increment_tick` and
1345    /// `switch_context` inside all of those frames.
1346    ///
1347    /// Taking it out is worth 233,207 on `kernel-ir` and costs the four
1348    /// kernel micro-instruments 9,650 between them, which is about what one
1349    /// extra call layer costs at their call counts.
1350    ///
1351    /// The other half was refuted: `#[inline(always)]` on
1352    /// [`Kernel::tick_from_isr`] itself costs `khot-ir` 178,245 while saving
1353    /// the three lighter instruments 82,642, and stacked on top of this
1354    /// outlining it still costs `khot-ir` 12,001, `ksched-ir` 7,555 and
1355    /// `kipc-ir` 2,000 for nothing `kernel-ir` did not already have. So the
1356    /// body keeps LLVM's own choice and only this call site is pinned.
1357    #[cold]
1358    #[inline(never)]
1359    fn tick_on_exit(&mut self) {
1360        self.tick_from_isr();
1361    }
1362
1363    /// `vPortSystemTickHandler`: the tick, from interrupt context.
1364    ///
1365    /// A silicon port calls this from its timer interrupt. The sim reaches
1366    /// it from [`Kernel::exit_critical`] -- through [`Kernel::tick_on_exit`],
1367    /// which is where the reason for this attribute is written down -- and
1368    /// from the idle hook.
1369    pub fn tick_from_isr(&mut self) {
1370        self.port.set_in_tick_entry(true);
1371        self.port.count_tick();
1372        // The C handler bumps `uxCriticalNesting` by hand and drops it the
1373        // same way — a raw `--`, never `vPortExitCritical` — so the bump is
1374        // invisible to the exit count and to anything that unwinds later.
1375        // Modelling it as a real critical section would invent an exit at
1376        // every tick that caused a switch.
1377        if self.increment_tick() {
1378            self.switch_context();
1379        }
1380        self.port.set_in_tick_entry(false);
1381    }
1382
1383    /// The idle hook's tick, `vPortKairosTick()` on the C side: a critical
1384    /// section around one unconditional tick. The enclosing section counts
1385    /// toward rule 3 exactly as the C one does.
1386    pub fn idle_hook_tick(&mut self) {
1387        self.enter_critical();
1388        self.tick_from_isr();
1389        self.exit_critical();
1390    }
1391
1392    // ---- tickless idle --------------------------------------------------
1393
1394    /// `tskIDLE_PRIORITY`.
1395    const IDLE_PRIORITY: u8 = 0;
1396
1397    /// `prvGetExpectedIdleTime`: how long nothing is due, in ticks.
1398    ///
1399    /// Zero whenever the answer would be unsafe to sleep on -- something
1400    /// above the idle priority is running, another idle-priority task is
1401    /// ready to share the slice, or a higher-priority task is ready. The
1402    /// last of those cannot happen with preemption on, and the C keeps the
1403    /// test anyway because a cooperative build reaches it.
1404    fn expected_idle_time(&self) -> u64 {
1405        // The C spells these as three arms so each carries its own reason;
1406        // they answer zero alike, so here they are one short-circuit chain
1407        // in the same order:
1408        //
1409        //  - something above the idle priority is running;
1410        //  - another idle-priority task is ready to share the slice, so the
1411        //    very next tick has to be processed;
1412        //  - a higher-priority task is ready, which only a cooperative
1413        //    build can reach and which the C tests for anyway.
1414        let due = self.current_priority() > Self::IDLE_PRIORITY
1415            || self.ready_len(Self::IDLE_PRIORITY).unwrap_or(0) > 1
1416            || self.top_ready_priority > Self::IDLE_PRIORITY;
1417
1418        if due {
1419            0
1420        } else {
1421            self.next_unblock_time.saturating_sub(self.tick)
1422        }
1423    }
1424
1425    /// `vTaskStepTick`: wind the tick count forward over a suppressed period.
1426    ///
1427    /// Note this does *not* call the tick hook for each stepped tick, which
1428    /// is the C's own note and the reason a tickless trace has no heartbeat
1429    /// in it.
1430    ///
1431    /// The C `configASSERT`s that the jump does not pass the next unblock
1432    /// time. This clamps instead: a port that overslept is a port bug, and
1433    /// winding the clock past a task's wake would lose the wake entirely,
1434    /// where losing the extra sleep is recoverable. `forbid(unsafe)` buys
1435    /// nothing if the kernel panics on a misbehaving port.
1436    ///
1437    /// Landing exactly on the next unblock time leaves the last tick pended
1438    /// rather than stepped, so `increment_tick` runs it when the scheduler
1439    /// resumes and the delayed task is woken by the same code that would
1440    /// have woken it.
1441    pub fn step_tick(&mut self, ticks_to_jump: u64) {
1442        let room = self.next_unblock_time.saturating_sub(self.tick);
1443        let mut jump = ticks_to_jump.min(room);
1444
1445        if self.tick.saturating_add(jump) == self.next_unblock_time && jump > 0 {
1446            self.pended_ticks = self.pended_ticks.saturating_add(1);
1447            jump = jump.wrapping_sub(1);
1448        }
1449
1450        self.tick = self.tick.wrapping_add(jump) & Self::MAX_DELAY;
1451    }
1452
1453    /// The `configUSE_TICKLESS_IDLE` block of `prvIdleTask`.
1454    ///
1455    /// Compiled away entirely when the configuration leaves tickless idle
1456    /// off, which is the default and is why every existing scenario is
1457    /// byte-identical with this in the tree.
1458    ///
1459    /// The shape is the C's, including the part that looks redundant: the
1460    /// expected idle time is sampled once WITHOUT the scheduler suspended,
1461    /// and again with it. The first answer is not necessarily valid and the
1462    /// C takes it anyway, because suspending and resuming the scheduler on
1463    /// every pass of the idle task costs more than a sample that is
1464    /// sometimes wasted.
1465    pub fn idle_suppress_ticks(&mut self) {
1466        if !C::USE_TICKLESS_IDLE {
1467            return;
1468        }
1469
1470        if self.expected_idle_time() < C::EXPECTED_IDLE_TIME_BEFORE_SLEEP {
1471            return;
1472        }
1473
1474        self.suspend_all();
1475        let expected = self.expected_idle_time();
1476        if expected >= C::EXPECTED_IDLE_TIME_BEFORE_SLEEP {
1477            let tick = self.tick;
1478            self.note_exits();
1479            self.trace.event(tick, Event::LowPowerIdleBegin);
1480
1481            let slept = self.port.suppress_ticks_and_sleep(expected);
1482            self.step_tick(slept);
1483
1484            let tick = self.tick;
1485            self.note_exits();
1486            self.trace.event(tick, Event::LowPowerIdleEnd);
1487        }
1488        let _ = self.resume_all();
1489    }
1490
1491    /// `portYIELD()` as the Posix port spells it: a critical section around
1492    /// the context switch, so a tick raised during it lands on the way out.
1493    /// `#[cold]` because the call is GUARDED and reached from many sites on
1494    /// one hot path, which is the shape that pays: it lets LLVM keep the
1495    /// caller's frame setup out of the likely route. Measured on
1496    /// `riscv32-qemu-tick-work`, one attribute at a time.
1497    /// Worth queue -7, group -3 on its own.
1498    #[cold]
1499    pub(crate) fn port_yield(&mut self) {
1500        self.enter_critical();
1501        self.port.count_yield();
1502        if P::COMMITS_SWITCH {
1503            // A STACKED port. Raise its switching exception and leave
1504            // `current` where it is: the exception will call
1505            // `switch_context` itself, so the decision and the register
1506            // swap happen together and nothing runs in between as a task
1507            // the kernel has already moved on from.
1508            //
1509            // The exception cannot fire yet -- `exit_critical` below is
1510            // what unmasks -- so the kernel is consistent at the moment it
1511            // is taken.
1512            self.port.yield_now();
1513        } else {
1514            // A STACKLESS kernel: moving `current` IS the switch, because
1515            // no task owns a stack. This is the path every corpus
1516            // architecture takes, and it is unchanged.
1517            self.switch_context();
1518        }
1519        self.exit_critical();
1520    }
1521
1522    /// `taskYIELD()` from a task body (the idle task's yield).
1523    pub fn task_yield(&mut self) {
1524        self.port_yield();
1525    }
1526
1527    // -------------------------------------------------------- task create --
1528
1529    /// `xTaskCreate`.
1530    ///
1531    /// Like the C, **the new task becomes current if its priority is at or
1532    /// above the running one's**. On a hosted firmware that is the sharp
1533    /// edge: the CPU is still on the caller's stack, so creating a
1534    /// higher-priority task hands `current` to something that is not
1535    /// running. See [`Kernel::start_scheduler`], which creates the timer
1536    /// daemon at [`Config::TIMER_TASK_PRIORITY`] for exactly this reason and
1537    /// documents the failure it produces — a clean run reporting zero laps.
1538    ///
1539    /// # Errors
1540    /// [`Error::Full`] when the task arena is full (the C
1541    /// `errCOULD_NOT_ALLOCATE_REQUIRED_MEMORY`).
1542    pub fn create_task(&mut self, name: &str, priority: u8) -> Result<TaskHandle> {
1543        let caller = self.cur();
1544        // configASSERT( uxPriority < configMAX_PRIORITIES ), then C clamps.
1545        let priority = priority.min(C::MAX_PRIORITIES.saturating_sub(1));
1546        // `prvCreateTask` takes the stack and then the TCB from the heap
1547        // before anything is initialised — `portSTACK_GROWTH` is negative on
1548        // the oracle's port, which fixes that order — and `pvPortMallocStack`
1549        // *is* `pvPortMalloc` with the MPU wrappers off. On heap_3 each one
1550        // suspends the scheduler, so a create costs two outermost exits.
1551        //
1552        // Every scenario before `death` created its tasks BEFORE
1553        // `vTaskStartScheduler`, and the sim port counts no exits there (the
1554        // C patch counts only on a FreeRTOS thread), so this cost was
1555        // invisible to the whole corpus. `death` is the first scenario that
1556        // creates a task with the scheduler running, and the first that can
1557        // tell the difference.
1558        self.account_for_allocation();
1559        self.account_for_allocation();
1560        let tcb = Tcb {
1561            name: Name::new(name, C::MAX_TASK_NAME_LEN),
1562            priority,
1563            base_priority: priority,
1564            mutexes_held: 0,
1565            wait: WaitFrame::default(),
1566            notified: [0; MAX_NOTIFICATION_ENTRIES],
1567            notify_state: [NotifyState::NotWaiting; MAX_NOTIFICATION_ENTRIES],
1568            notify_blocked: false,
1569            event_blocked: false,
1570            stream_resume: false,
1571            queue_resume: QueueResume::No,
1572            stream_timed: false,
1573            stream_waited: false,
1574            stream_local: 0,
1575            _stride_pad: [0; STRIDE_PAD_WORDS],
1576        };
1577        let handle = match self.tcbs.try_insert(tcb) {
1578            Ok(h) => h,
1579            Err(_) => {
1580                let tick = self.tick;
1581                self.note_exits();
1582                self.trace.event(tick, Event::TaskCreateFailed);
1583                return Err(Error::Full);
1584            }
1585        };
1586        // A slot an idle task once held must not carry its mark into the
1587        // task that reuses it. SMP only: nothing else reads the bit.
1588        if C::NUMBER_OF_CORES > 1 {
1589            self.set_flag(handle.index() as usize, Self::F_IDLE, false);
1590        }
1591        // The event item sorts by `configMAX_PRIORITIES - uxPriority`, so a
1592        // higher-priority task waits nearer the head of an event list.
1593        let event_value = u64::from(C::MAX_PRIORITIES.saturating_sub(priority));
1594        self.lists
1595            .set_value(Self::event_item(handle), event_value)?;
1596        // `prvInitialiseNewTask` ends by calling `pxPortInitialiseStack`,
1597        // which on the oracle's Posix port wraps `pthread_create` in a
1598        // critical section — one more exit, and a PORT cost rather than a
1599        // kernel one, which is why it has a flag of its own.
1600        if C::PORT_STACK_INIT_CRITICAL {
1601            self.enter_critical();
1602            self.exit_critical();
1603        }
1604        // Any of the three exits above can release a tick that switches the
1605        // creator out. In the C the creator is a thread and simply STOPS
1606        // there: `prvAddNewTaskToReadyList` has not run, so the new task is
1607        // on no ready list and `traceTASK_CREATE` has not fired. Running it
1608        // now would put both events on the wrong side of the switch — which
1609        // is precisely how `death` first diverged.
1610        if self.cur() != caller {
1611            if let Some(slot) = self.owed_trace.get_mut(caller.index() as usize) {
1612                *slot = OwedTrace::AddNewTaskToReadyList {
1613                    task: handle,
1614                    priority,
1615                };
1616            }
1617            self.owe(caller);
1618            return Ok(handle);
1619        }
1620        self.add_new_task_to_ready_list(handle, priority)?;
1621        Ok(handle)
1622    }
1623
1624    /// `prvAddNewTaskToReadyList`.
1625    fn add_new_task_to_ready_list(&mut self, task: TaskHandle, priority: u8) -> Result<()> {
1626        self.enter_critical();
1627        {
1628            self.task_count = self.task_count.wrapping_add(1);
1629            if C::NUMBER_OF_CORES > 1 {
1630                // SMP never assigns `current` at creation: the idle tasks
1631                // hold every core from `start_scheduler` on.
1632            } else if self.cur().is_null() {
1633                self.set_current_at(task, priority);
1634            } else if !self.running {
1635                // `<=`, so the last-created task of the highest priority is
1636                // the one that runs first.
1637                if self.current_priority() <= priority {
1638                    self.set_current_at(task, priority);
1639                }
1640            }
1641            self.trace_task(task, |task, name| Event::TaskCreate {
1642                task,
1643                name,
1644                priority: Self::priority_value(priority),
1645            });
1646            self.add_task_to_ready_list(task)?;
1647        }
1648        self.exit_critical();
1649        if C::NUMBER_OF_CORES > 1 {
1650            // taskYIELD_ANY_CORE_IF_USING_PREEMPTION( pxNewTCB ): the new task
1651            // may displace this core's task or another core's.
1652            if self.running && C::USE_PREEMPTION {
1653                self.yield_for_task(task, priority);
1654                if self.pending_here() {
1655                    self.port_yield();
1656                }
1657            }
1658            return Ok(());
1659        }
1660        // taskYIELD_IF_USING_PREEMPTION(), outside the section.
1661        if self.running && C::USE_PREEMPTION && self.current_priority() < priority {
1662            self.port_yield();
1663        }
1664        Ok(())
1665    }
1666
1667    /// `prvAddTaskToReadyList`.
1668    /// Returns `Result<()>`, and that is a measured decision, not an
1669    /// oversight.
1670    ///
1671    /// Its two hot callers -- the tick's wake loop and the pending-ready
1672    /// drain -- resolve the SAME TCB again on the next line for the SAME
1673    /// field, to ask whether the woken task outranks the running one. Having
1674    /// this hand the priority back instead costs **+3.69% on kdelay-ir and
1675    /// +3.10% on ksched-ir**.
1676    ///
1677    /// Wrapping it -- narrow signature for the eleven callers that ignore the
1678    /// value, a wide inner for the two that want it -- does NOT help: that
1679    /// measured bit-for-bit identical to the unwrapped version on all four
1680    /// arms, because LLVM inlines through the wrapper and propagates the wide
1681    /// return anyway.
1682    ///
1683    /// The likely mechanism is the tail call. At `Result<()>` this function
1684    /// ends by handing back `insert_end`'s own result; at `Result<u8>` it has
1685    /// to capture that result, test it, and build a new one, at every site.
1686    pub(crate) fn add_task_to_ready_list(&mut self, task: TaskHandle) -> Result<()> {
1687        let priority = self.tcbs.resolve(task)?.priority;
1688        self.trace_task(task, |task, name| Event::MovedTaskToReadyState {
1689            task,
1690            name,
1691        });
1692        // taskRECORD_READY_PRIORITY
1693        if priority > self.top_ready_priority {
1694            self.top_ready_priority = priority;
1695        }
1696        self.lists
1697            .insert_end(Self::ready_list(priority), Self::state_item(task))
1698    }
1699
1700    // ---------------------------------------------------- scheduler start --
1701
1702    /// `vTaskStartScheduler`, minus the part that never returns.
1703    ///
1704    /// Creates the idle task, the timer command queue and the timer service
1705    /// task, fires `TASK_SWITCHED_IN` for whichever task will run first and
1706    /// then `STARTING_SCHEDULER`, and returns the handles so the runner can
1707    /// attach their bodies.
1708    ///
1709    /// # ★ The timer daemon is created at [`Config::TIMER_TASK_PRIORITY`],
1710    /// and it will outrank a task you created below it
1711    ///
1712    /// This is the C's behaviour and it is deliberate, but on a hosted
1713    /// firmware it has a failure mode that looks like nothing at all.
1714    ///
1715    /// `Tmr Svc` is created here **unconditionally**, and
1716    /// [`Kernel::create_task`] makes the highest-priority task current. So a
1717    /// firmware whose own `main` runs below `TIMER_TASK_PRIORITY` leaves the
1718    /// kernel believing a **stackless** task is running: `current` names the
1719    /// daemon while the CPU is on `main`'s stack. Every
1720    /// [`Kernel::switch_context`] then declines as `from == to`, the workers
1721    /// never start, and the symptom is a clean run reporting **zero laps**
1722    /// rather than a fault.
1723    ///
1724    /// Nothing here can detect it — a stackless kernel cannot see whose stack
1725    /// the CPU is actually on — so it is a caller's invariant:
1726    ///
1727    /// > **Give the task that owns the CPU a priority at or above
1728    /// > [`Config::TIMER_TASK_PRIORITY`]**, or lower `TIMER_TASK_PRIORITY`
1729    /// > below it.
1730    ///
1731    /// Reported from the Janus side, where it cost a board run
1732    /// (`docs/plans/janus-rtos.md` §5b), and seen again in the radio cell.
1733    ///
1734    /// # Errors
1735    /// As [`Kernel::create_task`].
1736    pub fn start_scheduler(&mut self) -> Result<StartHandles> {
1737        if C::NUMBER_OF_CORES > 1 {
1738            return self.start_scheduler_smp();
1739        }
1740        let idle = self.create_task("IDLE", 0)?;
1741        // `xTimerCreateTimerTask` opens with `prvCheckForValidListAndQueue`,
1742        // which makes the command queue only if no `xTimerCreate` has made
1743        // it already.
1744        self.check_for_valid_list_and_queue()?;
1745        let timer_queue = self.timer_queue;
1746        let timer = self.create_task("Tmr Svc", C::TIMER_TASK_PRIORITY)?;
1747        self.next_unblock_time = Self::MAX_DELAY;
1748        self.running = true;
1749        self.tick = C::INITIAL_TICK_COUNT;
1750        self.port.scheduler_started();
1751        // `vPortStartFirstTask` runs the first task through the same
1752        // first-start path as every other.
1753        let current = self.cur();
1754        self.note_first_start(current);
1755        self.trace_task(current, |task, name| Event::TaskSwitchedIn { task, name });
1756        let tick = self.tick;
1757        self.note_exits();
1758        self.trace.event(tick, Event::StartingScheduler);
1759        Ok(StartHandles {
1760            idle,
1761            passive_idle: TaskHandle::NULL,
1762            timer,
1763            timer_queue,
1764        })
1765    }
1766
1767    /// `vTaskStartScheduler` with `configNUMBER_OF_CORES > 1`.
1768    ///
1769    /// `prvCreateIdleTasks` makes one idle task per core -- `IDLE0` runs
1770    /// `prvIdleTask`, the rest `prvPassiveIdleTask` -- and makes each the
1771    /// CURRENT task of its core, which is why SMP task creation never
1772    /// touches `current`: before the scheduler starts no core is assigned,
1773    /// and after it every core is running something. The first switch on
1774    /// each core then picks the highest ready task.
1775    #[cold]
1776    #[inline(never)]
1777    fn start_scheduler_smp(&mut self) -> Result<StartHandles> {
1778        let cores = Self::cores();
1779        for core in 0..cores {
1780            let name = if core == 0 { "IDLE0" } else { "IDLE1" };
1781            let idle = self.create_task(name, Self::IDLE_PRIORITY)?;
1782            if let Some(slot) = self.idle.get_mut(core) {
1783                *slot = idle;
1784            }
1785            self.set_flag(idle.index() as usize, Self::F_IDLE, true);
1786            self.set_current_of(core, idle, Self::IDLE_PRIORITY);
1787        }
1788        self.check_for_valid_list_and_queue()?;
1789        let timer_queue = self.timer_queue;
1790        let timer = self.create_task("Tmr Svc", C::TIMER_TASK_PRIORITY)?;
1791        self.next_unblock_time = Self::MAX_DELAY;
1792        self.running = true;
1793        self.tick = C::INITIAL_TICK_COUNT;
1794        self.port.scheduler_started();
1795        for core in 0..cores {
1796            let idle = self.current_of(core);
1797            self.note_first_start(idle);
1798        }
1799        let current = self.cur();
1800        self.trace_task(current, |task, name| Event::TaskSwitchedIn { task, name });
1801        let tick = self.tick;
1802        self.note_exits();
1803        self.trace.event(tick, Event::StartingScheduler);
1804        Ok(StartHandles {
1805            idle: self.idle_of(0),
1806            passive_idle: if cores > 1 {
1807                self.idle_of(1)
1808            } else {
1809                TaskHandle::NULL
1810            },
1811            timer,
1812            timer_queue,
1813        })
1814    }
1815
1816    // ------------------------------------------------------------------- SMP --
1817
1818    /// `configNUMBER_OF_CORES`, clamped to what this kernel can hold.
1819    #[inline(always)]
1820    const fn cores() -> usize {
1821        let n = C::NUMBER_OF_CORES as usize;
1822        if n < 1 {
1823            1
1824        } else if n > MAX_CORES {
1825            MAX_CORES
1826        } else {
1827            n
1828        }
1829    }
1830
1831    /// `pxCurrentTCBs[ core ]`, or null for a core out of range.
1832    #[inline(always)]
1833    fn current_of(&self, core: usize) -> TaskHandle {
1834        self.current.get(core).copied().unwrap_or(TaskHandle::NULL)
1835    }
1836
1837    /// The cached running priority of `core`.
1838    #[inline(always)]
1839    fn priority_on(&self, core: usize) -> u8 {
1840        self.current_priority.get(core).copied().unwrap_or(0)
1841    }
1842
1843    /// Make `task` (of `priority`) the running task of `core`.
1844    fn set_current_of(&mut self, core: usize, task: TaskHandle, priority: u8) {
1845        if let Some(slot) = self.current.get_mut(core) {
1846            *slot = task;
1847        }
1848        if let Some(slot) = self.current_priority.get_mut(core) {
1849            *slot = priority;
1850        }
1851    }
1852
1853    fn idle_of(&self, core: usize) -> TaskHandle {
1854        self.idle.get(core).copied().unwrap_or(TaskHandle::NULL)
1855    }
1856
1857    fn pending_on(&self, core: usize) -> bool {
1858        self.yield_pending.get(core).copied().unwrap_or(false)
1859    }
1860
1861    fn set_pending_on(&mut self, core: usize, value: bool) {
1862        if let Some(slot) = self.yield_pending.get_mut(core) {
1863            *slot = value;
1864        }
1865    }
1866
1867    fn requested_on(&self, core: usize) -> bool {
1868        self.yield_requested.get(core).copied().unwrap_or(false)
1869    }
1870
1871    fn set_requested_on(&mut self, core: usize, value: bool) {
1872        if let Some(slot) = self.yield_requested.get_mut(core) {
1873            *slot = value;
1874        }
1875    }
1876
1877    /// `taskTASK_IS_RUNNING( pxTCB )`: the core `task` is running on, if any,
1878    /// NOT counting a core that has been asked to yield it.
1879    fn running_core(&self, task: TaskHandle) -> Option<usize> {
1880        (0..Self::cores()).find(|&c| self.current_of(c) == task && !self.requested_on(c))
1881    }
1882
1883    /// `xTaskRunState != taskTASK_NOT_RUNNING`: whether ANY core holds `task`,
1884    /// including one asked to yield it -- which is what selection must skip.
1885    fn held_by_any_core(&self, task: TaskHandle) -> bool {
1886        (0..Self::cores()).any(|c| self.current_of(c) == task)
1887    }
1888
1889    /// `prvYieldCore( xCoreID )`.
1890    ///
1891    /// The calling core only records the yield -- it is inside a critical
1892    /// section and acts on it on the way out. Another core is asked once:
1893    /// its bit goes into [`Kernel::take_core_yields`] for the port to raise,
1894    /// and its task is marked `taskTASK_SCHEDULED_TO_YIELD` so a second
1895    /// request does not interrupt it twice.
1896    fn yield_core(&mut self, core: usize) {
1897        if core >= Self::cores() {
1898            return;
1899        }
1900        if core == self.core() {
1901            self.set_pending_on(core, true);
1902        } else if !self.requested_on(core) {
1903            self.core_yields |= 1u8.checked_shl(core as u32).unwrap_or(0);
1904            self.set_requested_on(core, true);
1905        }
1906    }
1907
1908    /// `prvYieldForTask( pxTCB )`, for `configRUN_MULTIPLE_PRIORITIES == 1`
1909    /// and no core affinity.
1910    ///
1911    /// `task` has just become ready. Find the core running the LOWEST
1912    /// priority task that `task` outranks -- a running idle task counts as
1913    /// one below priority 0, so a real priority-0 task displaces an idle
1914    /// one -- and yield it. Ties go to the HIGHER core number, because the
1915    /// C walks upwards with `<=`. A core already owing a yield, or already
1916    /// asked for one, is not a candidate.
1917    #[cold]
1918    #[inline(never)]
1919    fn yield_for_task(&mut self, task: TaskHandle, priority: u8) {
1920        // `taskTASK_IS_RUNNING`, by slot index. Before the scheduler starts
1921        // every core's current task is null (slot 0), so no core can be
1922        // running `task` and the test is skipped; after it, a core's current
1923        // task and the task just readied are both live, and a slot holds one
1924        // live task at a time -- so the index IS the handle.
1925        let slot = task.index();
1926        if self.running
1927            && (0..Self::cores())
1928                .any(|c| self.current_of(c).index() == slot && !self.requested_on(c))
1929        {
1930            return;
1931        }
1932        let mut lowest = i16::from(priority).saturating_sub(1);
1933        let mut lowest_core = None;
1934        for core in 0..Self::cores() {
1935            // The two flags first: a core that already owes or was asked for
1936            // a yield is no candidate.
1937            if self.requested_on(core) || self.pending_on(core) {
1938                continue;
1939            }
1940            let running = self.current_of(core).index();
1941            let mut p = i16::from(self.priority_on(core));
1942            // `taskATTRIBUTE_IS_IDLE`, as the C keeps it: one bit of the
1943            // running task's own flags, where comparing against the idle
1944            // tasks' handles rebuilt their slots on every call. Before the
1945            // scheduler starts no task carries the bit, so a core's null
1946            // current (slot 0) answers "not idle", as the handle test did.
1947            if self.flag(running as usize, Self::F_IDLE, false) {
1948                p = p.saturating_sub(1);
1949            }
1950            if p <= lowest {
1951                lowest = p;
1952                lowest_core = Some(core);
1953            }
1954        }
1955        if let Some(core) = lowest_core {
1956            self.yield_core(core);
1957        }
1958    }
1959
1960    /// SMP: `task` (now at `priority`) has just become ready, so run
1961    /// `prvYieldForTask` -- `taskYIELD_ANY_CORE_IF_USING_PREEMPTION` -- and
1962    /// answer whether THIS core now owes a yield.
1963    ///
1964    /// Every one-core "did the woken task outrank the running one?" test has
1965    /// this as its SMP arm.
1966    ///
1967    /// In line: the body is one call (`yield_for_task`, out of line itself)
1968    /// and a flag read, so out of line it was a call to make a call. The
1969    /// MIR-inliner risk that keeps the SMP tick bodies out of line was
1970    /// measured here and is absent: `bench/kernel-ir`'s one-core kernel rows
1971    /// are unchanged.
1972    #[inline(always)]
1973    fn smp_readied(&mut self, task: TaskHandle, priority: u8) -> bool {
1974        if C::USE_PREEMPTION {
1975            self.yield_for_task(task, priority);
1976        }
1977        self.pending_here()
1978    }
1979
1980    /// The core holding `task` -- running it, or asked to yield it.
1981    fn held_core(&self, task: TaskHandle) -> Option<usize> {
1982        (0..Self::cores()).find(|&c| self.current_of(c) == task)
1983    }
1984
1985    /// The OTHER cores this kernel has asked to yield since the last call,
1986    /// one bit per core, cleared by the read.
1987    ///
1988    /// A port calls this on its way out of the kernel lock and raises the
1989    /// inter-processor interrupt of each core named. Always zero on one core.
1990    pub fn take_core_yields(&mut self) -> u8 {
1991        core::mem::take(&mut self.core_yields)
1992    }
1993
1994    /// `prvSelectHighestPriorityTask( xCoreID )` for
1995    /// `configRUN_MULTIPLE_PRIORITIES == 1` and no core affinity.
1996    ///
1997    /// Answers whether a task was scheduled. Read from the pinned `tasks.c`
1998    /// (V11.3.1), not remembered:
1999    ///
2000    /// 1. The core's current task, if it is still in its ready list, goes to
2001    ///    the END of that list first -- so a task that yields is not picked
2002    ///    again ahead of tasks that have waited longer.
2003    /// 2. From `uxTopReadyPriority` down, each ready list is walked from its
2004    ///    HEAD (not the round-robin cursor): the first task no core holds is
2005    ///    swapped in; this core's own current task may be kept; a task held
2006    ///    by another core is skipped.
2007    /// 3. `uxTopReadyPriority` comes down only past levels that are EMPTY.
2008    ///
2009    /// In line: its one caller is [`Kernel::switch_context_smp`], itself the
2010    /// out-of-line SMP body, so this costs no flash and saves a call, a
2011    /// return and two sets of saved registers on every switch.
2012    #[inline(always)]
2013    fn select_for_core(&mut self, core: usize) -> bool {
2014        let current = self.current_of(core);
2015        let current_list = Self::ready_list(self.priority_on(core));
2016        // `if listIS_CONTAINED_WITHIN(...) { uxListRemove; vListInsertEnd }`,
2017        // as one list operation that does nothing when the task is not in its
2018        // ready list.
2019        let _ = self
2020            .lists
2021            .move_to_end(current_list, Self::state_item(current));
2022        // Which state items the cores hold, by item rather than by handle: a
2023        // ready list holds only live tasks and a slot holds one live task at a
2024        // time, so the item IS the task, and the walk can skip a held one
2025        // without resolving it through the arena. Only the task chosen is
2026        // resolved. A core holding nothing names an item no list can contain.
2027        //
2028        // Once the scheduler runs, no core's current task is null: the idle
2029        // tasks take every core before `running` is set, and only a pick ever
2030        // replaces one. So the null test -- a compare and a select per core,
2031        // on every switch -- is for a switch made before the start.
2032        let held: [ItemId; MAX_CORES] = if self.running {
2033            core::array::from_fn(|c| {
2034                if c < Self::cores() {
2035                    Self::state_item(self.current_of(c))
2036                } else {
2037                    ItemId::MAX
2038                }
2039            })
2040        } else {
2041            core::array::from_fn(|c| {
2042                let task = self.current_of(c);
2043                if c < Self::cores() && !task.is_null() {
2044                    Self::state_item(task)
2045                } else {
2046                    ItemId::MAX
2047                }
2048            })
2049        };
2050        let mine = held.get(core).copied().unwrap_or(ItemId::MAX);
2051        let mut priority = self.top_ready_priority;
2052        let mut decrement_top = true;
2053        loop {
2054            let list = Self::ready_list(priority);
2055            if !self.lists.is_empty_of(list) {
2056                decrement_top = false;
2057                // The list's own iterator follows the links with none of the
2058                // per-step container and bounds checks `head`/`next` make. The
2059                // first item no other core holds wins: a free task (swapped
2060                // in) or this core's own (kept).
2061                //
2062                // A `for` with a `break`, not `find_map`: handing the pick
2063                // back through the closure's nested `Option` cost one register
2064                // copy of the chosen slot on every switch that swaps a task
2065                // in (`mov %r11d,%r12d`, 53,176 times in semtest).
2066                let mut pick = None;
2067                for item in self.lists.iter(list) {
2068                    if !held.contains(&item) {
2069                        if let Some(task) = self.tcbs.handle_at(item) {
2070                            pick = Some(Some(task));
2071                            break;
2072                        }
2073                    } else if item == mine {
2074                        pick = Some(None);
2075                        break;
2076                    }
2077                }
2078                match pick {
2079                    Some(Some(task)) => {
2080                        self.set_requested_on(core, false);
2081                        self.set_current_of(core, task, priority);
2082                        return true;
2083                    }
2084                    Some(None) => {
2085                        self.set_requested_on(core, false);
2086                        return true;
2087                    }
2088                    None => {}
2089                }
2090            }
2091            if decrement_top {
2092                self.top_ready_priority = self.top_ready_priority.saturating_sub(1);
2093            }
2094            if priority == 0 {
2095                return false;
2096            }
2097            priority = priority.saturating_sub(1);
2098        }
2099    }
2100
2101    /// `xTaskIncrementTick`'s SMP time slicing: EVERY core whose running
2102    /// priority has a second ready task owes a yield (`xYieldPendings`).
2103    ///
2104    /// Out of line, like every SMP body: rustc's MIR inliner weighs a
2105    /// function before monomorphisation folds `C::NUMBER_OF_CORES`, so an
2106    /// SMP loop written inline in `increment_tick` changes how a ONE-core
2107    /// build inlines the tick even though it never runs there.
2108    #[cold]
2109    #[inline(never)]
2110    fn tick_time_slice_smp(&mut self) {
2111        if C::USE_PREEMPTION && C::USE_TIME_SLICING {
2112            for core in 0..Self::cores() {
2113                let list = Self::ready_list(self.priority_on(core));
2114                if self.lists.len(list).unwrap_or(0) > 1 {
2115                    self.set_pending_on(core, true);
2116                }
2117            }
2118        }
2119    }
2120
2121    /// `xTaskIncrementTick`'s SMP preemption: this core switches on the way
2122    /// out (answered `true`); every other core that owes a yield is
2123    /// interrupted (`prvYieldCore`).
2124    #[cold]
2125    #[inline(never)]
2126    fn tick_yields_smp(&mut self) -> bool {
2127        let mut here = false;
2128        if C::USE_PREEMPTION {
2129            let me = self.core();
2130            for core in 0..Self::cores() {
2131                if self.pending_on(core) {
2132                    if core == me {
2133                        here = true;
2134                    } else {
2135                        self.yield_core(core);
2136                    }
2137                }
2138            }
2139        }
2140        here
2141    }
2142
2143    /// `vTaskSwitchContext( xCoreID )`, for the calling core.
2144    #[cold]
2145    #[inline(never)]
2146    fn switch_context_smp(&mut self) {
2147        let core = self.core();
2148        if self.suspended_depth != 0 {
2149            self.set_pending_on(core, true);
2150            return;
2151        }
2152        self.set_pending_on(core, false);
2153        let outgoing = self.current_of(core);
2154        self.trace_task(outgoing, |task, name| Event::TaskSwitchedOut { task, name });
2155        if !self.select_for_core(core) {
2156            self.note_stall(Stall::NoReadyTask);
2157            return;
2158        }
2159        let incoming = self.current_of(core);
2160        if incoming.index() != outgoing.index() {
2161            self.hand_over(incoming);
2162        }
2163        self.trace_task(incoming, |task, name| Event::TaskSwitchedIn { task, name });
2164    }
2165
2166    // -------------------------------------------------------- the switch --
2167
2168    /// `vTaskSwitchContext`.
2169    pub fn switch_context(&mut self) {
2170        if C::NUMBER_OF_CORES > 1 {
2171            return self.switch_context_smp();
2172        }
2173        if self.suspended_depth != 0 {
2174            self.set_pending_here(true);
2175            return;
2176        }
2177        self.set_pending_here(false);
2178        let current = self.cur();
2179        self.trace_task(current, |task, name| Event::TaskSwitchedOut { task, name });
2180        // taskSELECT_HIGHEST_PRIORITY_TASK
2181        //
2182        // ONE call per priority level, not two. The search used to ask
2183        // `is_empty` — a bounded read of the list's metadata — and then hand
2184        // the winning level to `next_round_robin`, which reads that same
2185        // metadata again to find the cursor. `next_round_robin` already
2186        // answers `None` for an empty list, and does so BEFORE it touches the
2187        // cursor, so it is its own emptiness probe and the level that loses
2188        // is left exactly as it was found.
2189        let mut top = self.top_ready_priority;
2190        // ★ `Err` is folded into "that level is empty" below, and the fold IS
2191        // the win: with three arms the walk carried a packed
2192        // `Result<Option<ItemId>>` across the back edge -- a `setb`/`shl`/`or`
2193        // to build it and a `test`/`jne` to take it apart, on every one of the
2194        // 1.58 levels a call walks. Two arms took this function from 168.0 to
2195        // 157.7 Ir per call, -937,965 on `bench/kernel-ir`, all of it here.
2196        //
2197        // But an out-of-range list id is a different fault from an empty level,
2198        // and the fold alone would have reported it as `NoReadyTask`. So it is
2199        // proved HERE instead, once, where it can still name itself:
2200        // `ready_list` is the identity, so the only way the walk can index out
2201        // of range is a `top_ready_priority` above the configured ceiling.
2202        // That is the diagnostic bought back, and the cost of buying it is in
2203        // the commit message beside the fold's.
2204        if top >= C::MAX_PRIORITIES {
2205            self.note_stall(Stall::ListError);
2206            return;
2207        }
2208        let item = loop {
2209            match self.lists.next_round_robin(Self::ready_list(top)) {
2210                Ok(Some(item)) => break item,
2211                Ok(None) | Err(_) => {
2212                    if top == 0 {
2213                        // The C `configASSERT( uxTopPriority )`. The idle
2214                        // task keeps priority 0 non-empty, so reaching here
2215                        // means it is gone. The kernel keeps the current
2216                        // task rather than indexing out of range -- and now
2217                        // SAYS so, because carrying on quietly is how this
2218                        // costs an afternoon on a board.
2219                        self.note_stall(Stall::NoReadyTask);
2220                        return;
2221                    }
2222                    // At least 1: the arm above returns at zero.
2223                    top = top.wrapping_sub(1);
2224                }
2225            }
2226        };
2227        let Ok(next) = self.task_of_state_item(item) else {
2228            self.note_stall(Stall::UnknownTask);
2229            return;
2230        };
2231        self.top_ready_priority = top;
2232        // INDICES, and `hand_over` takes only the incoming handle: together
2233        // those two delete the load of `current`'s generation word entirely.
2234        // Either alone leaves it live -- the full-handle `!=` compares both
2235        // halves, and a `hand_over` taking the outgoing handle needs both -- so
2236        // the index compare measured 0 on its own. `rusty-compiler-leverage`
2237        // A2: price the SET, not the halves.
2238        //
2239        // Comparing indices is sound because both handles are LIVE: `next` came
2240        // out of a slot whose generation was just matched, and `current` names
2241        // the running task. The arena issues one live handle per slot at a time,
2242        // so two live handles sharing an index share a generation. That is the
2243        // same argument `Handle::is_null` makes in the other direction.
2244        if next.index() != current.index() {
2245            self.hand_over(next);
2246        }
2247        // `top` is `next`'s priority by construction: the loop above found
2248        // the highest non-empty ready list and `next` came out of it.
2249        self.set_current_at(next, top);
2250        self.trace_task(next, |task, name| Event::TaskSwitchedIn { task, name });
2251    }
2252
2253    // ----------------------------------------------------------- the tick --
2254
2255    /// Pay whatever the running task still owes from a call it was
2256    /// preempted inside. The runner calls this before stepping a body;
2257    /// `true` means work was done and the current task may have changed
2258    /// again, so nothing else should be assumed.
2259    #[inline(always)]
2260    pub fn resume_pending(&mut self) -> bool {
2261        // The fast path, in line at the runner's one call site.
2262        //
2263        // `Runner::step_once` asks this before every step -- 214,488 times
2264        // in a 20,000-tick `StreamBufferDemo` -- and the answer is "nothing
2265        // owed" almost every time. Two loads and two branches decide it,
2266        // and out of line they cost a call and a frame as well.
2267        //
2268        // The two tests are the entry conditions of the two things the
2269        // body does: `settle_unwind` returns at once unless `unwinding` is
2270        // set, and the rest returns at once unless this task owes
2271        // something. Neither is duplicated -- the cold half still runs
2272        // both in full.
2273        // On a `COMMITS_SWITCH` port `unwinding` is never set (see `hand_over`),
2274        // and saying so with the const lets LLVM drop the load and the branch.
2275        if P::COMMITS_SWITCH || self.unwinding.is_null() {
2276            // Nothing to settle, which is the shape of almost every call.
2277            // `settle_unwind` would test `unwinding` and return at once, and
2278            // the cold half would then re-read the very byte this line reads
2279            // — so on this shape both are skipped and the owed body is
2280            // entered directly.
2281            let index = self.cur().index() as usize;
2282            // B1, the shape win 7 established: `self.cur()` is read RAW here -- no
2283            // bound-checking accessor between the field and the use -- and `index` then
2284            // feeds `owes_anything.get(index)`. Contrast `remove_from_event_list`, where
2285            // the same proof measured EXACTLY +0 because its handle comes back from
2286            // `handle_at`, which bound-checks, so the fact was already local.
2287            if index >= TASKS {
2288                return false;
2289            }
2290            if !self.owes_anything.get(index).copied().unwrap_or(true) {
2291                return false;
2292            }
2293            return self.resume_pending_owed(index);
2294        }
2295        self.resume_pending_cold()
2296    }
2297
2298    /// Everything [`Kernel::resume_pending`] does when something IS owed.
2299    ///
2300    /// Out of line on purpose: it is reached on a small fraction of steps,
2301    /// and inlining it would put the whole `OwedTrace` match at the hot
2302    /// site. Same body/symbol split the trace sink's `num` measured.
2303    #[inline(never)]
2304    fn resume_pending_cold(&mut self) -> bool {
2305        self.settle_unwind();
2306        let index = self.cur().index() as usize;
2307        // B1, the shape win 7 established: `self.cur()` is read RAW here -- no
2308        // bound-checking accessor between the field and the use -- and `index` then
2309        // feeds `owes_anything.get(index)`. Contrast `remove_from_event_list`, where
2310        // the same proof measured EXACTLY +0 because its handle comes back from
2311        // `handle_at`, which bound-checks, so the fact was already local.
2312        if index >= TASKS {
2313            return false;
2314        }
2315
2316        // One load and one test for the overwhelmingly common case. The
2317        // combined check below is the fallback, and it is what clears the hint
2318        // -- so a hint left standing costs one slow path and then goes away.
2319        //
2320        // This read cannot be threaded in from `resume_pending`: settling an
2321        // unwind calls `owe` itself, so the answer here is not always the
2322        // answer there. That is exactly why the no-unwind shape gets its own
2323        // entry rather than this one being made cheaper.
2324        if !self.owes_anything.get(index).copied().unwrap_or(true) {
2325            return false;
2326        }
2327        self.resume_pending_owed(index)
2328    }
2329
2330    /// Everything owed to the current task, once it is known that something
2331    /// is.
2332    ///
2333    /// Its own symbol so that both shapes above can reach it without either
2334    /// paying for the other's entry conditions.
2335    #[inline(never)]
2336    fn resume_pending_owed(&mut self, index: usize) -> bool {
2337        let did = self.resume_pending_owed_inner(index);
2338        if did {
2339            // Every branch below that did work returned with the hint still
2340            // standing, so the runner came straight back and spent a second
2341            // entry re-deriving "nothing owed". Clearing it here covers all
2342            // of them at one site.
2343            self.clear_owe_if_settled(index);
2344        }
2345        did
2346    }
2347
2348    /// The body of [`Kernel::resume_pending_owed`].
2349    fn resume_pending_owed_inner(&mut self, index: usize) -> bool {
2350        // Almost every call answers "nothing owed", and the sequence below
2351        // reaches that answer in three stages -- a compare, a match over
2352        // `OwedTrace`, then another compare -- each reachable only after the
2353        // one before. The same three loads, but one branch instead of three.
2354        // No `.copied()` on `owed_trace`: that materialises an
2355        // `Option<OwedTrace>`, and `OwedTrace` is forty bytes wide because of
2356        // `TimerCommandSend`'s `Name`. Matching through the reference tests
2357        // the discriminant where the copy moved the whole variant to test it.
2358        if self.owed_exits.get(index).copied().unwrap_or(0) == 0
2359            && matches!(self.owed_trace.get(index), Some(OwedTrace::None) | None)
2360            && !self.flag(index, Self::F_YIELD, false)
2361        {
2362            if let Some(f) = self.owes_anything.get_mut(index) {
2363                *f = false;
2364            }
2365            return false;
2366        }
2367        // First the stack unwinds — the sections the task had open when it
2368        // was switched out, and whatever its abandoned frame opened after
2369        // that — and only then does the statement after the yield run.
2370        // Reversing the two moves every tick.
2371        let owed = self.owed_exits.get(index).copied().unwrap_or(0);
2372        if owed > 0 {
2373            if let Some(slot) = self.owed_exits.get_mut(index) {
2374                *slot = 0;
2375            }
2376            // One counted exit each: the tally is already in outermost
2377            // exits, so replaying it as nesting would lose the sections the
2378            // tail opened and closed on its own.
2379            for _ in 0..owed {
2380                self.enter_critical();
2381                self.exit_critical();
2382                // A replayed exit can release a tick that switches this
2383                // task straight back out. The rest of the loop is then the
2384                // tail of *this* frame, which the port tallies and the next
2385                // resume pays — so there is nothing to unwind by hand.
2386            }
2387            return true;
2388        }
2389        // Then the line the abandoned frame had not reached yet.
2390        match self.owed_trace.get(index).copied() {
2391            Some(OwedTrace::SendFailed(queue)) => {
2392                if let Some(slot) = self.owed_trace.get_mut(index) {
2393                    *slot = crate::kernel::OwedTrace::None;
2394                }
2395                let tick = self.tick;
2396                self.note_exits();
2397                self.trace
2398                    .event(tick, Event::QueueSendFailed { queue, name: "" });
2399                return true;
2400            }
2401            Some(OwedTrace::ReceiveFailed(queue)) => {
2402                if let Some(slot) = self.owed_trace.get_mut(index) {
2403                    *slot = crate::kernel::OwedTrace::None;
2404                }
2405                let tick = self.tick;
2406                self.note_exits();
2407                self.trace
2408                    .event(tick, Event::QueueReceiveFailed { queue, name: "" });
2409                return true;
2410            }
2411            Some(OwedTrace::EventGroupWaitBitsEnd {
2412                group,
2413                bits,
2414                timed_out,
2415            }) => {
2416                if let Some(slot) = self.owed_trace.get_mut(index) {
2417                    *slot = crate::kernel::OwedTrace::None;
2418                }
2419                let tick = self.tick;
2420                self.note_exits();
2421                self.trace.event(
2422                    tick,
2423                    Event::EventGroupWaitBitsEnd {
2424                        group,
2425                        bits,
2426                        timed_out,
2427                    },
2428                );
2429                return true;
2430            }
2431            Some(OwedTrace::StreamBufferCreate {
2432                buffer,
2433                is_message_buffer,
2434            }) => {
2435                if let Some(slot) = self.owed_trace.get_mut(index) {
2436                    *slot = crate::kernel::OwedTrace::None;
2437                }
2438                let tick = self.tick;
2439                self.note_exits();
2440                self.trace.event(
2441                    tick,
2442                    Event::StreamBufferCreate {
2443                        buffer,
2444                        is_message_buffer,
2445                    },
2446                );
2447                return true;
2448            }
2449            Some(OwedTrace::TimerCommandSend {
2450                timer,
2451                name,
2452                command,
2453                value,
2454            }) => {
2455                if let Some(slot) = self.owed_trace.get_mut(index) {
2456                    *slot = crate::kernel::OwedTrace::None;
2457                }
2458                let tick = self.tick;
2459                self.note_exits();
2460                self.trace.event(
2461                    tick,
2462                    Event::TimerCommandSend {
2463                        timer,
2464                        name: name.as_str(),
2465                        command,
2466                        value,
2467                    },
2468                );
2469                return true;
2470            }
2471            Some(OwedTrace::AddNewTaskToReadyList { task, priority }) => {
2472                if let Some(slot) = self.owed_trace.get_mut(index) {
2473                    *slot = crate::kernel::OwedTrace::None;
2474                }
2475                let _ = self.add_new_task_to_ready_list(task, priority);
2476                return true;
2477            }
2478            Some(OwedTrace::None) | None => {}
2479        }
2480        if !self.flag(index, Self::F_YIELD, false) {
2481            return false;
2482        }
2483        self.set_flag(index, Self::F_YIELD, false);
2484        self.port_yield();
2485        true
2486    }
2487
2488    /// Collect the tally of an abandoned frame that has finished running.
2489    ///
2490    /// The frame runs to its end before control comes back to the runner,
2491    /// so this is called once, at the top of [`Kernel::resume_pending`],
2492    /// which is the first thing the runner does.
2493    fn settle_unwind(&mut self) {
2494        // Never anything to settle on a port that commits its own switches.
2495        if P::COMMITS_SWITCH {
2496            return;
2497        }
2498        // Nothing to settle is the common case by a wide margin, and `take()`
2499        // writes `None` back even then -- a store, on every call, to clear a
2500        // slot that was already clear. Peeking first leaves the store for the
2501        // calls that actually have something to clear.
2502        if self.unwinding.is_null() {
2503            return;
2504        }
2505        let task = core::mem::replace(&mut self.unwinding, TaskHandle::NULL);
2506        if task.is_null() {
2507            return;
2508        };
2509        let owed = self.port.end_unwind();
2510        if let Some(slot) = self.owed_exits.get_mut(task.index() as usize) {
2511            // Accumulate: a replay that is itself interrupted leaves the
2512            // rest of its own loop as a tail, and that tail is more of the
2513            // same debt.
2514            *slot = slot.saturating_add(owed);
2515        }
2516        if owed > 0 {
2517            self.owe(task);
2518        }
2519    }
2520
2521    /// Emit a queue failure line now, or owe it if a tick switched the
2522    /// caller out of the call that was about to emit it.
2523    /// In line on purpose, at all seven call sites. It is reached 46,671
2524    /// times in kernel-ir and every one of them paid a call and a frame to
2525    /// choose between two events the caller had already decided between.
2526    #[inline(always)]
2527    pub(crate) fn trace_failure_or_owe(&mut self, caller: TaskHandle, owed: OwedTrace) {
2528        #[cfg(feature = "census")]
2529        CENSUS_OWE.fetch_add(1, core::sync::atomic::Ordering::Relaxed);
2530        // Unlike every other trace site, this one has a side effect a no-op
2531        // sink cannot delete: the miss path STORES an `OwedTrace` and raises
2532        // `owes_anything`, which sends the task down the owed body on its next
2533        // run to reproduce a line the sink will drop.
2534        if !T::EMITS {
2535            return;
2536        }
2537        if self.cur() == caller {
2538            let tick = self.tick;
2539            self.note_exits();
2540            match owed {
2541                OwedTrace::SendFailed(queue) => self
2542                    .trace
2543                    .event(tick, Event::QueueSendFailed { queue, name: "" }),
2544                OwedTrace::ReceiveFailed(queue) => self
2545                    .trace
2546                    .event(tick, Event::QueueReceiveFailed { queue, name: "" }),
2547                OwedTrace::StreamBufferCreate {
2548                    buffer,
2549                    is_message_buffer,
2550                } => self.trace.event(
2551                    tick,
2552                    Event::StreamBufferCreate {
2553                        buffer,
2554                        is_message_buffer,
2555                    },
2556                ),
2557                OwedTrace::TimerCommandSend {
2558                    timer,
2559                    name,
2560                    command,
2561                    value,
2562                } => self.trace.event(
2563                    tick,
2564                    Event::TimerCommandSend {
2565                        timer,
2566                        name: name.as_str(),
2567                        command,
2568                        value,
2569                    },
2570                ),
2571                OwedTrace::EventGroupWaitBitsEnd {
2572                    group,
2573                    bits,
2574                    timed_out,
2575                } => self.trace.event(
2576                    tick,
2577                    Event::EventGroupWaitBitsEnd {
2578                        group,
2579                        bits,
2580                        timed_out,
2581                    },
2582                ),
2583                // Not reachable through this function: `create_task` owes
2584                // its second half directly, because that half is not a
2585                // trace line — it inserts into a ready list — and it must
2586                // run INSIDE the section this one has already left.
2587                OwedTrace::AddNewTaskToReadyList { .. } | OwedTrace::None => {}
2588            }
2589            return;
2590        }
2591        if let Some(slot) = self.owed_trace.get_mut(caller.index() as usize) {
2592            *slot = owed;
2593        }
2594        self.owe(caller);
2595    }
2596
2597    /// Raise the hint that `task` owes something.
2598    ///
2599    /// Called by every writer of `owed_trace`, `owes_yield` and `owed_exits`.
2600    /// Missing one would let `resume_pending` skip work that was really owed,
2601    /// which the conformance differential would show as a missing trace line.
2602    /// `owes_yield`: a `portYIELD()` the task was preempted before reaching.
2603    const F_YIELD: u8 = 1 << 1;
2604    /// `wait_set`: a blocking call's `WaitFrame` is live in the TCB.
2605    const F_WAIT: u8 = 1 << 2;
2606    /// `taskATTRIBUTE_IS_IDLE`, SMP only: set on each core's idle task by
2607    /// `start_scheduler_smp`, cleared whenever a slot gets a new task.
2608    ///
2609    /// The SIGN bit on purpose. `yield_for_task` subtracts it as 0 or 1,
2610    /// which LLVM builds as a sign mask: from bit 3 that took `shl $4`,
2611    /// `sar $7` and a sign-extension per core; from bit 7 the `shl` is gone.
2612    const F_IDLE: u8 = 1 << 7;
2613
2614    /// Read one per-task flag.
2615    ///
2616    /// `missing` is what an index past the end answers. Each caller chooses it
2617    /// so an impossible index takes the SLOW path rather than the fast one,
2618    /// which is what the five separate arrays did with `unwrap_or`.
2619    #[inline(always)]
2620    fn flag(&self, index: usize, bit: u8, missing: bool) -> bool {
2621        match self.flags.get(index) {
2622            Some(f) => f & bit != 0,
2623            None => missing,
2624        }
2625    }
2626
2627    /// Write one per-task flag. An index past the end writes nothing, as
2628    /// `get_mut` did.
2629    #[inline(always)]
2630    fn set_flag(&mut self, index: usize, bit: u8, on: bool) {
2631        if let Some(f) = self.flags.get_mut(index) {
2632            if on {
2633                *f |= bit;
2634            } else {
2635                *f &= !bit;
2636            }
2637        }
2638    }
2639
2640    /// Drop the hint if nothing is owed any more.
2641    ///
2642    /// Worth one whole entry to [`Kernel::resume_pending_owed`] per owed item.
2643    /// A census over three scenarios found every raise producing exactly TWO
2644    /// entries — one that did the work and returned `true`, and a second that
2645    /// re-derived "nothing owed" and cleared the hint on its way out (BlockQ:
2646    /// 43,110 raises, 86,207 entries, 43,103 of them finding nothing). Doing
2647    /// it at the end of the branch that did the work turns that second entry
2648    /// into a `resume_pending` fast path, which is a few instructions instead
2649    /// of forty-odd.
2650    ///
2651    /// This cannot clear a live hint. Everything that opens new debt raises
2652    /// the hint itself — a replay that gets switched out leaves its tail to
2653    /// `settle_unwind`, which raises it again when it collects the tally.
2654    fn clear_owe_if_settled(&mut self, index: usize) {
2655        if self.owed_exits.get(index).copied().unwrap_or(0) == 0
2656            && matches!(self.owed_trace.get(index), Some(OwedTrace::None) | None)
2657            && !self.flag(index, Self::F_YIELD, false)
2658        {
2659            if let Some(f) = self.owes_anything.get_mut(index) {
2660                *f = false;
2661            }
2662        }
2663    }
2664
2665    fn owe(&mut self, task: TaskHandle) {
2666        if let Some(f) = self.owes_anything.get_mut(task.index() as usize) {
2667            *f = true;
2668        }
2669    }
2670
2671    /// Record that `task` was preempted before the `portYIELD()` at the end
2672    /// of the call it is inside.
2673    pub(crate) fn owe_yield(&mut self, task: TaskHandle) {
2674        self.set_flag(task.index() as usize, Self::F_YIELD, true);
2675        self.owe(task);
2676    }
2677
2678    /// Hand the CPU from `outgoing` to `incoming`: what `prvSwitchThread`
2679    /// does to `uxCriticalNesting`.
2680    ///
2681    /// Everything the outgoing task's call still does from here belongs to
2682    /// that task at the time it runs again, so the port stops counting the
2683    /// frame's exits as sim time and starts tallying them;
2684    /// [`Kernel::settle_unwind`] collects the tally once the frame has
2685    /// finished. A task being switched in for the first time has a fresh
2686    /// stack and owes nothing.
2687    /// Takes only the INCOMING handle. The outgoing one is `self.cur()`, which
2688    /// this reads for itself -- `set_current_at` has not run yet, so the field
2689    /// still names the task being left. Passing it in made `switch_context` load
2690    /// BOTH words of the handle at the top of the function, and the generation
2691    /// half is read only on the unwind arm below, which is skipped on every
2692    /// switch after the first.
2693    fn hand_over(&mut self, incoming: TaskHandle) {
2694        // A tail that switches again is still the first frame's tail: the
2695        // code after the second switch is on the same abandoned stack.
2696        // The unwinding marker exists for a port that does NOT commit the switch:
2697        // there `switch_context` returns into the OUTGOING task's Rust frame,
2698        // whose tail then runs on an abandoned stack and must be tallied rather
2699        // than counted. A `COMMITS_SWITCH` port saves the registers itself and
2700        // resumes the incoming task in ITS frame; nothing is abandoned, its
2701        // `begin_unwind` is a no-op and its `end_unwind` answers 0 -- so on
2702        // those ports this only forced `resume_pending` down its cold path
2703        // once per switch to settle a tally of zero. The const folds the whole
2704        // thing away on silicon and leaves the sim untouched.
2705        if !P::COMMITS_SWITCH && self.unwinding.is_null() {
2706            self.unwinding = self.cur();
2707            self.port.begin_unwind();
2708        }
2709        let index = incoming.index() as usize;
2710        // ONE access to the byte, not two: the test and the two bit writes all
2711        // work on the same `&mut`. STARTED goes on and YIELD goes off together,
2712        // which is the whole point of their sharing a byte.
2713        let first = match self.started.get_mut(index) {
2714            Some(f) if !*f => {
2715                *f = true;
2716                true
2717            }
2718            _ => false,
2719        };
2720        if first {
2721            self.set_flag(index, Self::F_YIELD, false);
2722            if let Some(slot) = self.owed_exits.get_mut(index) {
2723                *slot = 0;
2724            }
2725            if let Some(slot) = self.owed_trace.get_mut(index) {
2726                *slot = crate::kernel::OwedTrace::None;
2727            }
2728        }
2729    }
2730
2731    /// Mark a task as having run, without a hand-over: the first task the
2732    /// scheduler starts has no predecessor.
2733    fn note_first_start(&mut self, task: TaskHandle) {
2734        if let Some(f) = self.started.get_mut(task.index() as usize) {
2735            *f = true;
2736        }
2737    }
2738
2739    /// `xTaskIncrementTick`; `true` when a context switch is required.
2740    pub fn increment_tick(&mut self) -> bool {
2741        let tick = self.tick;
2742        self.note_exits();
2743        self.trace.event(tick, Event::TaskIncrementTick { tick });
2744        let mut switch_required = false;
2745        if self.suspended_depth == 0 {
2746            let next = self.tick.wrapping_add(1) & Self::MAX_DELAY;
2747            self.tick = next;
2748            if next == 0 {
2749                self.switch_delayed_lists();
2750            }
2751            // The mask is not arithmetic, it is INFORMATION. Both sides are
2752            // tick values and every writer of either has already masked with
2753            // `MAX_DELAY` -- `next` two lines above, `next_unblock_time` from a
2754            // `wake_at` that `add_current_task_to_delayed_list` masked, or from
2755            // `MAX_DELAY` itself. So at a 32-bit tick width the high half of
2756            // both is invariantly zero.
2757            //
2758            // LLVM proves that for `next`, which it just computed, and cannot
2759            // for `next_unblock_time`, which it loaded -- so the comparison was
2760            // emitted as a full 64-bit one: a second `lw` for the high word,
2761            // then `sltu`/`snez`/`or` to combine. Masking says the high half is
2762            // zero and the compare collapses to the single `bltu` the values
2763            // deserve. At a 64-bit width `MAX_DELAY` is `u64::MAX` and the mask
2764            // folds to nothing, so this costs those configurations no
2765            // instruction at all.
2766            // The mask is only sound while every writer of the field has
2767            // already applied it. All six do -- three store `MAX_DELAY`, two
2768            // store a delayed list's head value, and the one place that MINTS
2769            // such a value masks it (`add_current_task_to_delayed_list`'s
2770            // `wake_at`). This is the proof, and it costs nothing in release:
2771            // a writer that forgets fails the kernel's own tests, the Kani
2772            // proofs and the conformance corpus instead of silently changing a
2773            // comparison. Same guard shape as `current_priority`'s.
2774            debug_assert_eq!(
2775                self.next_unblock_time,
2776                self.next_unblock_time & Self::MAX_DELAY,
2777                "next_unblock_time holds a value wider than the tick, so masking                  it in the comparison below would change the answer"
2778            );
2779            if next >= self.next_unblock_time & Self::MAX_DELAY {
2780                switch_required = self.wake_due_tasks(next);
2781            }
2782            // `!switch_required` first: when the tick already woke a
2783            // task, this can only set what is already set, and asking
2784            // costs a TCB resolve for the running priority plus a list
2785            // read. Setting a `true` to `true` is not worth either.
2786            if C::NUMBER_OF_CORES > 1 {
2787                self.tick_time_slice_smp();
2788            } else if C::USE_PREEMPTION
2789                && C::USE_TIME_SLICING
2790                && !switch_required
2791                && self
2792                    .lists
2793                    .len(Self::ready_list(self.current_priority()))
2794                    .unwrap_or(0)
2795                    > 1
2796            {
2797                switch_required = true;
2798            }
2799            // The C guards this site with `xPendedTicks == 0`, so a hook
2800            // does not fire again for each tick being unwound by
2801            // `xTaskResumeAll`. It sits after the time-slicing test and
2802            // before the yield-pending one, and a `FromISR` call inside it
2803            // can set `yield_pending`, so the order is load-bearing.
2804            if self.pended_ticks == 0 {
2805                self.run_tick_hook();
2806            }
2807            if C::NUMBER_OF_CORES > 1 {
2808                switch_required |= self.tick_yields_smp();
2809            } else if C::USE_PREEMPTION && self.pending_here() {
2810                switch_required = true;
2811            }
2812        } else {
2813            self.pended_ticks = self.pended_ticks.wrapping_add(1);
2814            // "The tick hook gets called at regular intervals, even if the
2815            // scheduler is locked" — and here with no guard, because this
2816            // tick is being pended rather than unwound.
2817            self.run_tick_hook();
2818        }
2819        switch_required
2820    }
2821
2822    // ------------------------------------------- task notifications --
2823    //
2824    // A notification is the lightest thing a task can block on: no object,
2825    // no event list, just a slot in the TCB and the delayed list. That is
2826    // why the stream buffers use one rather than a queue, and why waking a
2827    // task this way is a list move rather than an event-list walk.
2828
2829    /// `xTaskGenericNotifyWait`.
2830    ///
2831    /// `true` is `pdTRUE`: a notification was received. `Blocked` means the
2832    /// caller must call again — the C blocks inside this function, and a
2833    /// kernel with no stacks returns instead.
2834    ///
2835    /// # Errors
2836    /// [`Error::InvalidArgument`] for an index past the configured array.
2837    pub fn notify_wait(
2838        &mut self,
2839        index: usize,
2840        clear_on_entry: u32,
2841        clear_on_exit: u32,
2842        ticks: u64,
2843    ) -> Result<Wait<(bool, u32)>> {
2844        if index >= C::NOTIFICATION_ARRAY_ENTRIES {
2845            return Err(Error::InvalidArgument);
2846        }
2847        let caller = self.cur();
2848        let state = self.notify_state_of(caller, index);
2849        if state != NotifyState::Received && ticks > 0 && !self.notify_blocked(caller) {
2850            self.suspend_all();
2851            self.enter_critical();
2852            let mut should_block = false;
2853            if self.notify_state_of(caller, index) != NotifyState::Received {
2854                if let Ok(tcb) = self.tcbs.resolve_mut(caller) {
2855                    if let Some(slot) = tcb.notified.get_mut(index) {
2856                        *slot &= !clear_on_entry;
2857                    }
2858                    if let Some(slot) = tcb.notify_state.get_mut(index) {
2859                        *slot = NotifyState::Waiting;
2860                    }
2861                }
2862                should_block = true;
2863            }
2864            self.exit_critical();
2865            if should_block {
2866                let tick = self.tick;
2867                self.note_exits();
2868                self.trace_task(caller, |task, name| Event::TaskNotifyWaitBlock {
2869                    task,
2870                    name,
2871                    index,
2872                });
2873                let _ = tick;
2874                self.add_current_task_to_delayed_list(ticks, true)?;
2875            }
2876            let already_yielded = self.resume_all();
2877            if should_block && !already_yielded {
2878                if self.cur() == caller {
2879                    self.port_yield();
2880                } else {
2881                    self.owe_yield(caller);
2882                }
2883            }
2884            // The C blocks here; this kernel comes back and is called again.
2885            self.set_notify_blocked(caller, true);
2886            return Ok(Wait::Blocked);
2887        }
2888        self.set_notify_blocked(caller, false);
2889        self.enter_critical();
2890        self.note_exits();
2891        self.trace_task(caller, |task, name| Event::TaskNotifyWait {
2892            task,
2893            name,
2894            index,
2895        });
2896        let value = self.notified_value_of(caller, index);
2897        let received = self.notify_state_of(caller, index) == NotifyState::Received;
2898        if let Ok(tcb) = self.tcbs.resolve_mut(caller) {
2899            if received {
2900                if let Some(slot) = tcb.notified.get_mut(index) {
2901                    *slot &= !clear_on_exit;
2902                }
2903            }
2904            if let Some(slot) = tcb.notify_state.get_mut(index) {
2905                *slot = NotifyState::NotWaiting;
2906            }
2907        }
2908        self.exit_critical();
2909        Ok(Wait::Ready((received, value)))
2910    }
2911
2912    /// `ulTaskGenericNotifyTake`: the notification used as a lightweight
2913    /// counting semaphore — block until the value is non-zero, then either
2914    /// zero it or decrement it by one.
2915    ///
2916    /// This is the half of the notification API the kernel was missing.
2917    /// `notify_wait` waits on *bits*; this waits on a *count*, and the two
2918    /// are separate calls in the C with their own trace events. Its
2919    /// partner, `xTaskNotifyGive`, needs no method of its own: the C
2920    /// defines it as `xTaskGenericNotify( .., 0, eIncrement, NULL )` and
2921    /// it traces as an ordinary notify, so
2922    /// `notify(task, index, 0, NotifyAction::Increment)` already is it.
2923    ///
2924    /// # The shape, and why it is not `notify_wait`'s
2925    ///
2926    /// The C blocks on `ulNotifiedValue == 0`, **not** on the notify
2927    /// state, and it does no clear-on-entry. Both halves run on the way
2928    /// out: `traceTASK_NOTIFY_TAKE` fires whether or not the call ever
2929    /// blocked, which is what makes a take that finds a count already
2930    /// waiting still emit one line.
2931    ///
2932    /// This kernel has no stack, so the block is a return: the first call
2933    /// answers [`Wait::Blocked`] and the caller is entered again when the
2934    /// scheduler runs it, taking the second half. `notify_wait_pending`
2935    /// answers for this call too — a task is only ever inside one of the
2936    /// two.
2937    ///
2938    /// # Errors
2939    /// [`Error::InvalidArgument`] for an index past the configured array.
2940    pub fn notify_take(
2941        &mut self,
2942        index: usize,
2943        clear_on_exit: bool,
2944        ticks: u64,
2945    ) -> Result<Wait<u32>> {
2946        if index >= C::NOTIFICATION_ARRAY_ENTRIES {
2947            return Err(Error::InvalidArgument);
2948        }
2949        let caller = self.cur();
2950        if self.notified_value_of(caller, index) == 0 && ticks > 0 && !self.notify_blocked(caller) {
2951            self.suspend_all();
2952            self.enter_critical();
2953            let mut should_block = false;
2954            // Re-checked inside the section, as the C does: a notify from
2955            // an ISR between the two reads would otherwise be lost.
2956            if self.notified_value_of(caller, index) == 0 {
2957                if let Ok(tcb) = self.tcbs.resolve_mut(caller) {
2958                    if let Some(slot) = tcb.notify_state.get_mut(index) {
2959                        *slot = NotifyState::Waiting;
2960                    }
2961                }
2962                should_block = true;
2963            }
2964            self.exit_critical();
2965            if should_block {
2966                // The C traces BEFORE it adds itself to the delayed list.
2967                self.note_exits();
2968                self.trace_task(caller, |task, name| Event::TaskNotifyTakeBlock {
2969                    task,
2970                    name,
2971                    index,
2972                });
2973                self.add_current_task_to_delayed_list(ticks, true)?;
2974            }
2975            let already_yielded = self.resume_all();
2976            if should_block && !already_yielded {
2977                if self.cur() == caller {
2978                    self.port_yield();
2979                } else {
2980                    self.owe_yield(caller);
2981                }
2982            }
2983            self.set_notify_blocked(caller, true);
2984            return Ok(Wait::Blocked);
2985        }
2986        self.set_notify_blocked(caller, false);
2987        self.enter_critical();
2988        self.note_exits();
2989        self.trace_task(caller, |task, name| Event::TaskNotifyTake {
2990            task,
2991            name,
2992            index,
2993        });
2994        // One resolve for the read and both writes. `notified_value_of`
2995        // resolved the TCB to read the value and this line resolved it again
2996        // to write it back -- the same trade `unlock_queue` makes, and a
2997        // `&mut self` call sits between this and the read at the top of the
2998        // function, so the two cannot be folded by the compiler.
2999        let mut value = 0;
3000        if let Ok(tcb) = self.tcbs.resolve_mut(caller) {
3001            value = tcb.notified.get(index).copied().unwrap_or(0);
3002            if value != 0 {
3003                if let Some(slot) = tcb.notified.get_mut(index) {
3004                    // Guarded by `value != 0` above, so the saturation
3005                    // never fires; it is here because the workspace forbids
3006                    // bare arithmetic, and a decrement is exactly the place
3007                    // an unguarded one would wrap.
3008                    *slot = if clear_on_exit {
3009                        0
3010                    } else {
3011                        value.saturating_sub(1)
3012                    };
3013                }
3014            }
3015            if let Some(slot) = tcb.notify_state.get_mut(index) {
3016                *slot = NotifyState::NotWaiting;
3017            }
3018        }
3019        self.exit_critical();
3020        Ok(Wait::Ready(value))
3021    }
3022
3023    /// `xTaskGetHandle`: the task registered under `name`.
3024    ///
3025    /// # It is not free, and that is the point
3026    ///
3027    /// The C walks every ready list, then the delayed lists, then the
3028    /// suspended and waiting-termination lists, all inside
3029    /// `vTaskSuspendAll` / `xTaskResumeAll` — and `xTaskResumeAll` takes a
3030    /// critical section. So a lookup that emits **no trace event** still
3031    /// costs **one critical-section exit**, which under the sim contract is
3032    /// one unit of time.
3033    ///
3034    /// That is why this exists rather than a scenario keeping the handle it
3035    /// was given at creation. `AbortDelay`'s remake did exactly that, on the
3036    /// reasoning that a lookup emitting no event could not change the trace,
3037    /// and the corpus caught it: every event still agreed and the
3038    /// exit column was one short at the first block.
3039    ///
3040    /// The arena is searched instead of five lists, which cannot change the
3041    /// answer — a live task is in exactly one of them — and does not change
3042    /// the accounting either, because the cost is the suspend/resume pair.
3043    ///
3044    /// # Errors
3045    /// [`Error::Gone`] when no live task carries that name.
3046    pub fn task_get_handle(&mut self, name: &str) -> Result<TaskHandle> {
3047        self.suspend_all();
3048        let mut found = None;
3049        for index in 0..u16::try_from(self.tcbs.capacity()).unwrap_or(u16::MAX) {
3050            let Some(handle) = self.tcbs.handle_at(index) else {
3051                continue;
3052            };
3053            if self
3054                .tcbs
3055                .resolve(handle)
3056                .is_ok_and(|tcb| tcb.name.matches(name))
3057            {
3058                found = Some(handle);
3059                break;
3060            }
3061        }
3062        let _ = self.resume_all();
3063        found.ok_or(Error::Gone)
3064    }
3065
3066    /// `xTaskGenericNotify`.
3067    ///
3068    /// # Errors
3069    /// [`Error::InvalidArgument`] for a bad index; [`Error::Gone`] for a
3070    /// stale handle.
3071    pub fn notify(
3072        &mut self,
3073        task: TaskHandle,
3074        index: usize,
3075        value: u32,
3076        action: NotifyAction,
3077    ) -> Result<bool> {
3078        if index >= C::NOTIFICATION_ARRAY_ENTRIES {
3079            return Err(Error::InvalidArgument);
3080        }
3081        self.enter_critical();
3082        let result = self.notify_locked(task, index, value, action, false);
3083        self.exit_critical();
3084        result.map(|(ok, _)| ok)
3085    }
3086
3087    /// `xTaskGenericNotifyFromISR`: the same, without a critical section
3088    /// that costs anything, and putting a woken task on the pending-ready
3089    /// list when the scheduler is suspended.
3090    ///
3091    /// # Errors
3092    /// As [`Kernel::notify`].
3093    pub fn notify_from_isr(
3094        &mut self,
3095        task: TaskHandle,
3096        index: usize,
3097        value: u32,
3098        action: NotifyAction,
3099    ) -> Result<(bool, Woken)> {
3100        if index >= C::NOTIFICATION_ARRAY_ENTRIES {
3101            return Err(Error::InvalidArgument);
3102        }
3103        let mask = self.port.enter_critical_from_isr();
3104        let result = self.notify_locked(task, index, value, action, true);
3105        self.port.exit_critical_from_isr(mask);
3106        result.map(|(ok, woken)| (ok, if woken { Woken::YES } else { Woken::NO }))
3107    }
3108
3109    /// `xTaskGenericNotify` with its `pulPreviousNotificationValue`
3110    /// out-parameter filled: `xTaskNotifyAndQuery`.
3111    ///
3112    /// It exists because the answer has to come out of the SAME critical
3113    /// section as the notification. Reading [`Kernel::notify_value`] first
3114    /// and then notifying yields the same two numbers and costs a second
3115    /// critical section -- and on the sim a critical-section exit is a
3116    /// tick opportunity, so the extra one moves the tick and the trace
3117    /// stops matching the C. `TaskNotify.c` calls this eleven times.
3118    ///
3119    /// # Errors
3120    /// As [`Kernel::notify`].
3121    pub fn notify_and_query(
3122        &mut self,
3123        task: TaskHandle,
3124        index: usize,
3125        value: u32,
3126        action: NotifyAction,
3127    ) -> Result<(bool, u32)> {
3128        if index >= C::NOTIFICATION_ARRAY_ENTRIES {
3129            return Err(Error::InvalidArgument);
3130        }
3131        self.enter_critical();
3132        let previous = self
3133            .tcbs
3134            .resolve(task)
3135            .map(|tcb| tcb.notified.get(index).copied().unwrap_or(0));
3136        let result = self.notify_locked(task, index, value, action, false);
3137        self.exit_critical();
3138        Ok((result?.0, previous?))
3139    }
3140
3141    /// `xTaskGenericNotifyAndQueryFromISR`.
3142    ///
3143    /// # Errors
3144    /// As [`Kernel::notify`].
3145    pub fn notify_and_query_from_isr(
3146        &mut self,
3147        task: TaskHandle,
3148        index: usize,
3149        value: u32,
3150        action: NotifyAction,
3151    ) -> Result<(bool, u32, Woken)> {
3152        if index >= C::NOTIFICATION_ARRAY_ENTRIES {
3153            return Err(Error::InvalidArgument);
3154        }
3155        let mask = self.port.enter_critical_from_isr();
3156        let previous = self
3157            .tcbs
3158            .resolve(task)
3159            .map(|tcb| tcb.notified.get(index).copied().unwrap_or(0));
3160        let result = self.notify_locked(task, index, value, action, true);
3161        self.port.exit_critical_from_isr(mask);
3162        let (ok, woken) = result?;
3163        Ok((ok, previous?, if woken { Woken::YES } else { Woken::NO }))
3164    }
3165
3166    /// The body both notify paths share.
3167    fn notify_locked(
3168        &mut self,
3169        task: TaskHandle,
3170        index: usize,
3171        value: u32,
3172        action: NotifyAction,
3173        from_isr: bool,
3174    ) -> Result<(bool, bool)> {
3175        let original = self.notify_state_of(task, index);
3176        let mut ok = true;
3177        {
3178            let tcb = self.tcbs.resolve_mut(task)?;
3179            if let Some(slot) = tcb.notify_state.get_mut(index) {
3180                *slot = NotifyState::Received;
3181            }
3182            if let Some(slot) = tcb.notified.get_mut(index) {
3183                match action {
3184                    NotifyAction::SetBits => *slot |= value,
3185                    NotifyAction::Increment => *slot = slot.wrapping_add(1),
3186                    NotifyAction::Overwrite => *slot = value,
3187                    NotifyAction::NoOverwrite => {
3188                        if original == NotifyState::Received {
3189                            ok = false;
3190                        } else {
3191                            *slot = value;
3192                        }
3193                    }
3194                    NotifyAction::None => {}
3195                }
3196            }
3197        }
3198        // `traceTASK_NOTIFY` is hooked; `traceTASK_NOTIFY_FROM_ISR` is not,
3199        // so an interrupt's notification says nothing on either side.
3200        if !from_isr {
3201            self.note_exits();
3202            self.trace_task(task, |t, name| Event::TaskNotify {
3203                task: t,
3204                name,
3205                index,
3206            });
3207        }
3208        let mut woke_higher = false;
3209        if original == NotifyState::Waiting {
3210            let item = Self::state_item(task);
3211            if from_isr && self.suspended_depth != 0 {
3212                let _ = self.lists.remove(Self::event_item(task));
3213                let value = self.lists.value(Self::event_item(task))?;
3214                self.lists
3215                    .insert(Self::pending_ready_list(), Self::event_item(task), value)?;
3216            } else {
3217                if self.lists.container(item)?.is_some() {
3218                    let _ = self.lists.remove(item);
3219                }
3220                self.add_task_to_ready_list(task)?;
3221            }
3222            let woken = self.tcbs.resolve(task).map(|t| t.priority).unwrap_or(0);
3223            if C::NUMBER_OF_CORES > 1 {
3224                // `prvYieldForTask`; the caller's flag is whether THIS core
3225                // now owes a yield, and a task-side notify takes it here.
3226                woke_higher = self.smp_readied(task, woken);
3227                if woke_higher && !from_isr {
3228                    self.port_yield();
3229                }
3230            } else if woken > self.current_priority() {
3231                woke_higher = true;
3232                if from_isr {
3233                    // The C sets `xYieldPendings[ 0 ]` here as well as
3234                    // reporting the wake through the caller's flag — so an
3235                    // interrupt that discards the flag still gets the
3236                    // switch, on the way out of `xTaskIncrementTick`. That
3237                    // is why the tick hook runs before the yield-pending
3238                    // test and not after it.
3239                    self.set_pending_here(true);
3240                } else {
3241                    // taskYIELD_ANY_CORE_IF_USING_PREEMPTION, inside the
3242                    // section.
3243                    self.port_yield();
3244                }
3245            }
3246        }
3247        Ok((ok, woke_higher))
3248    }
3249
3250    /// `xTaskGenericNotifyStateClear`. `None` is the calling task.
3251    ///
3252    /// # Errors
3253    /// As [`Kernel::notify`].
3254    pub fn notify_state_clear(&mut self, task: Option<TaskHandle>, index: usize) -> Result<bool> {
3255        if index >= C::NOTIFICATION_ARRAY_ENTRIES {
3256            return Err(Error::InvalidArgument);
3257        }
3258        let target = task.unwrap_or(self.cur());
3259        self.enter_critical();
3260        let cleared = self.notify_state_of(target, index) == NotifyState::Received;
3261        if cleared {
3262            if let Ok(tcb) = self.tcbs.resolve_mut(target) {
3263                if let Some(slot) = tcb.notify_state.get_mut(index) {
3264                    *slot = NotifyState::NotWaiting;
3265                }
3266            }
3267        }
3268        self.exit_critical();
3269        Ok(cleared)
3270    }
3271
3272    /// `ulNotifiedValue[ index ]` for a task. `None` is the calling task.
3273    ///
3274    /// The C reaches this through `xTaskNotifyAndQuery`, which hands back
3275    /// the value a notification is about to REPLACE, and through
3276    /// `ulTaskNotifyValueClear`. Neither could be implemented without it:
3277    /// `xTaskNotifyAndQuery` is `xTaskGenericNotify` with a
3278    /// `pulPreviousNotificationValue` out-parameter, and `TaskNotify.c:373`
3279    /// asserts on what comes back. The field existed; only the way to read
3280    /// it was missing, which is the kind of gap a C ABI finds and a Rust
3281    /// face never does.
3282    ///
3283    /// # Errors
3284    /// `InvalidArgument` if `index` is outside the notification array, or
3285    /// the resolve error if the handle is stale.
3286    pub fn notify_value(&mut self, task: Option<TaskHandle>, index: usize) -> Result<u32> {
3287        if index >= C::NOTIFICATION_ARRAY_ENTRIES {
3288            return Err(Error::InvalidArgument);
3289        }
3290        let target = task.unwrap_or(self.cur());
3291        self.enter_critical();
3292        let value = self
3293            .tcbs
3294            .resolve(target)
3295            .map(|tcb| tcb.notified.get(index).copied().unwrap_or(0));
3296        self.exit_critical();
3297        value
3298    }
3299
3300    /// `ulTaskGenericNotifyValueClear`: clear `bits` in the task's
3301    /// notification value and answer what it was BEFORE the clear.
3302    ///
3303    /// # Errors
3304    /// As [`Kernel::notify_value`].
3305    pub fn notify_value_clear(
3306        &mut self,
3307        task: Option<TaskHandle>,
3308        index: usize,
3309        bits: u32,
3310    ) -> Result<u32> {
3311        if index >= C::NOTIFICATION_ARRAY_ENTRIES {
3312            return Err(Error::InvalidArgument);
3313        }
3314        let target = task.unwrap_or(self.cur());
3315        self.enter_critical();
3316        let before = match self.tcbs.resolve_mut(target) {
3317            Ok(tcb) => match tcb.notified.get_mut(index) {
3318                Some(slot) => {
3319                    let was = *slot;
3320                    *slot &= !bits;
3321                    Ok(was)
3322                }
3323                None => Err(Error::InvalidArgument),
3324            },
3325            Err(e) => Err(e),
3326        };
3327        self.exit_critical();
3328        before
3329    }
3330
3331    /// Whether `task` is part-way through a notification wait — it blocked
3332    /// once and has not yet run the half of `xTaskGenericNotifyWait` that
3333    /// happens when a task wakes.
3334    ///
3335    /// A caller that blocks on a notification needs this: the C's wait
3336    /// returns *after* the wake, so its second half always runs, and a
3337    /// stackless caller that decided not to block again would otherwise
3338    /// skip it and leave the slot marked as received for ever.
3339    #[must_use]
3340    pub fn notify_wait_pending(&self, task: TaskHandle) -> bool {
3341        self.notify_blocked(task)
3342    }
3343
3344    /// Whether `task` has a stream-buffer call to resume, and the local it
3345    /// left behind. Taking it clears the marker.
3346    pub(crate) fn take_stream_resume(&mut self, task: TaskHandle) -> Option<usize> {
3347        let tcb = self.tcbs.resolve_mut(task).ok()?;
3348        if !tcb.stream_resume {
3349            return None;
3350        }
3351        tcb.stream_resume = false;
3352        Some(tcb.stream_local)
3353    }
3354
3355    /// Remember where a preempted stream-buffer call has to start again.
3356    pub(crate) fn set_stream_resume(&mut self, task: TaskHandle, local: usize) {
3357        if let Ok(tcb) = self.tcbs.resolve_mut(task) {
3358            tcb.stream_resume = true;
3359            tcb.stream_local = local;
3360        }
3361    }
3362
3363    /// Whether `task` has a queue call to resume below its sampling exit.
3364    /// Taking it clears the marker.
3365    pub(crate) fn take_queue_resume(&mut self, task: TaskHandle) -> QueueResume {
3366        // Nobody is parked at a sampling exit, so there is nothing to read.
3367        // This is the answer on all but a handful of queue calls, and it costs
3368        // one field load where resolving the TCB costs an arena lookup.
3369        if self.queue_resumes == 0 {
3370            return QueueResume::No;
3371        }
3372        let Ok(tcb) = self.tcbs.resolve_mut(task) else {
3373            return QueueResume::No;
3374        };
3375        let resume = tcb.queue_resume;
3376        // Peek before clearing, the same trade `settle_unwind` makes: every
3377        // queue send and every queue receive passes through here, and on all
3378        // but the handful that were preempted at a sampling exit the slot is
3379        // already `No` — so the unconditional write was a store, on every
3380        // queue call, to clear something already clear.
3381        if resume != QueueResume::No {
3382            tcb.queue_resume = QueueResume::No;
3383            self.queue_resumes = self.queue_resumes.saturating_sub(1);
3384        }
3385        resume
3386    }
3387
3388    /// Remember that a queue call was preempted at its sampling exit.
3389    ///
3390    /// No local travels with it, unlike [`Kernel::set_stream_resume`]: the C
3391    /// re-reads the queue under `prvLockQueue` on the other side of the
3392    /// exit, so there is nothing sampled to carry.
3393    pub(crate) fn set_queue_resume(&mut self, task: TaskHandle, at: QueueResume) {
3394        debug_assert_ne!(
3395            at,
3396            QueueResume::No,
3397            "set_queue_resume only ever parks a task; clearing is take's job, and the summary count depends on that"
3398        );
3399        if let Ok(tcb) = self.tcbs.resolve_mut(task) {
3400            // Count the slot, not the call: re-parking a task already parked
3401            // replaces its slot and must not be counted twice.
3402            if tcb.queue_resume == QueueResume::No {
3403                self.queue_resumes = self.queue_resumes.wrapping_add(1);
3404            }
3405            tcb.queue_resume = at;
3406        }
3407    }
3408
3409    /// Has this task's blocking send already paid for `vTaskSetTimeOutState`?
3410    ///
3411    /// A one-shot, like [`Kernel::take_stream_resume`], and for the same
3412    /// reason: `vTaskSetTimeOutState`'s own exit can release a tick and
3413    /// switch the caller away, and the C's thread then resumes **below** it,
3414    /// in the sampling loop. Running it again on re-entry would spend a
3415    /// second exit the C never spends -- which is a sixteenth of a tick, and
3416    /// `StreamBufferDemo`'s echo client hit it at tick 419.
3417    pub(crate) fn take_stream_timed(&mut self, task: TaskHandle) -> bool {
3418        let Ok(tcb) = self.tcbs.resolve_mut(task) else {
3419            return false;
3420        };
3421        let was = tcb.stream_timed;
3422        tcb.stream_timed = false;
3423        was
3424    }
3425
3426    /// Remember that the timeout state is set and must not be set again.
3427    pub(crate) fn set_stream_timed(&mut self, task: TaskHandle) {
3428        if let Ok(tcb) = self.tcbs.resolve_mut(task) {
3429            tcb.stream_timed = true;
3430        }
3431    }
3432
3433    /// Has this task's stream-buffer call already finished its wait?
3434    ///
3435    /// The other half of [`Kernel::take_stream_resume`]. A call can be
3436    /// preempted at two different exits, and the two resume in different
3437    /// places: at the SAMPLING exit the C goes on to block, and at the
3438    /// WAIT's own exit it goes on to move the bytes and trace them. Without
3439    /// this the second case is indistinguishable from the first, and a call
3440    /// that had already waited would wait again.
3441    pub(crate) fn take_stream_waited(&mut self, task: TaskHandle) -> bool {
3442        let Ok(tcb) = self.tcbs.resolve_mut(task) else {
3443            return false;
3444        };
3445        let was = tcb.stream_waited;
3446        tcb.stream_waited = false;
3447        was
3448    }
3449
3450    /// Remember that the wait is done and only the transfer is left.
3451    pub(crate) fn set_stream_waited(&mut self, task: TaskHandle) {
3452        if let Ok(tcb) = self.tcbs.resolve_mut(task) {
3453            tcb.stream_waited = true;
3454        }
3455    }
3456
3457    fn notify_blocked(&self, task: TaskHandle) -> bool {
3458        self.tcbs
3459            .resolve(task)
3460            .map(|t| t.notify_blocked)
3461            .unwrap_or(false)
3462    }
3463
3464    fn set_notify_blocked(&mut self, task: TaskHandle, blocked: bool) {
3465        if let Ok(tcb) = self.tcbs.resolve_mut(task) {
3466            tcb.notify_blocked = blocked;
3467        }
3468    }
3469
3470    fn notify_state_of(&self, task: TaskHandle, index: usize) -> NotifyState {
3471        self.tcbs
3472            .resolve(task)
3473            .ok()
3474            .and_then(|t| t.notify_state.get(index).copied())
3475            .unwrap_or(NotifyState::NotWaiting)
3476    }
3477
3478    fn notified_value_of(&self, task: TaskHandle, index: usize) -> u32 {
3479        self.tcbs
3480            .resolve(task)
3481            .ok()
3482            .and_then(|t| t.notified.get(index).copied())
3483            .unwrap_or(0)
3484    }
3485
3486    /// `xTaskGetTickCountFromISR`.
3487    ///
3488    /// No critical section, exactly as `xTaskGetTickCount` takes none: the
3489    /// Posix port sets `portTICK_TYPE_IS_ATOMIC`, so reading the count is
3490    /// a single load on both sides and costs no sim time.
3491    #[must_use]
3492    pub const fn tick_count_from_isr(&self) -> u64 {
3493        self.tick
3494    }
3495
3496    /// `vTaskMissedYield`: a yield that could not be taken where it was
3497    /// asked for — because a task is walking an event list, or because the
3498    /// scheduler is suspended — and must happen on the way out instead.
3499    pub(crate) fn missed_yield(&mut self) {
3500        self.set_pending_here(true);
3501    }
3502
3503    /// `vApplicationTickHook()`, when `configUSE_TICK_HOOK` is 1.
3504    ///
3505    /// The hook is copied out, run, and the result stored: that is what
3506    /// lets it borrow the kernel mutably while the kernel owns it. A hook
3507    /// that reads the kernel's own copy of itself therefore sees the value
3508    /// from before this call.
3509    fn run_tick_hook(&mut self) {
3510        if !C::USE_TICK_HOOK {
3511            return;
3512        }
3513        let hook = self.tick_hook;
3514        self.tick_hook = hook.tick(self);
3515    }
3516
3517    /// The tick hook, as it now stands.
3518    ///
3519    /// A scenario's `xAre...StillRunning()` reads the status its interrupt
3520    /// half latched, so the owner of the kernel needs to see it.
3521    pub const fn tick_hook(&self) -> &H {
3522        &self.tick_hook
3523    }
3524
3525    /// The tick hook, to install or reset one after construction.
3526    pub const fn tick_hook_mut(&mut self) -> &mut H {
3527        &mut self.tick_hook
3528    }
3529
3530    /// The wake loop inside `xTaskIncrementTick`.
3531    // Out of line: this walks the delayed list and runs only when a tick
3532    // reaches the next unblock time, but `increment_tick` runs on EVERY tick
3533    // and was carrying a frame sized for the walk regardless.
3534    #[inline(never)]
3535    fn wake_due_tasks(&mut self, now: u64) -> bool {
3536        let mut switch_required = false;
3537        // Hoisted: see the note at the bottom of the loop.
3538        let running = if C::USE_PREEMPTION {
3539            self.current_priority()
3540        } else {
3541            0
3542        };
3543        loop {
3544            let delayed = self.delayed_list();
3545            if self.lists.is_empty_of(delayed) {
3546                self.next_unblock_time = Self::MAX_DELAY;
3547                return switch_required;
3548            }
3549            let Ok(Some(item)) = self.lists.head(delayed) else {
3550                self.next_unblock_time = Self::MAX_DELAY;
3551                return switch_required;
3552            };
3553            let Ok(wake_at) = self.lists.value(item) else {
3554                return switch_required;
3555            };
3556            if now < wake_at {
3557                self.next_unblock_time = wake_at;
3558                return switch_required;
3559            }
3560            let Ok(task) = self.task_of_state_item(item) else {
3561                return switch_required;
3562            };
3563            let _ = self.lists.remove(item);
3564            if self
3565                .lists
3566                .container(Self::event_item(task))
3567                .unwrap_or(None)
3568                .is_some()
3569            {
3570                let _ = self.lists.remove(Self::event_item(task));
3571            }
3572            if self.add_task_to_ready_list(task).is_err() {
3573                return switch_required;
3574            }
3575            // `!switch_required` FIRST, and the running priority read once
3576            // for the whole drain.
3577            //
3578            // Both of these cost a TCB resolve -- a generation check, a
3579            // bounds check and an `Option` -- and this loop runs once per
3580            // task the tick wakes. `self.cur()` cannot move while the
3581            // drain is running (nothing here switches), so the running
3582            // priority is a loop invariant that was being re-derived on
3583            // every lap. And once a switch is already required, asking again
3584            // can only set a `true` to `true`, which is not worth the
3585            // resolve it costs -- the same reasoning `increment_tick`
3586            // already applies one level up.
3587            if C::NUMBER_OF_CORES > 1 {
3588                // SMP: the tick's yield decision is made per core at the end
3589                // of `increment_tick`, from `xYieldPendings`.
3590                if C::USE_PREEMPTION {
3591                    let woken = self.tcbs.resolve(task).map(|t| t.priority).unwrap_or(0);
3592                    self.yield_for_task(task, woken);
3593                }
3594            } else if C::USE_PREEMPTION && !switch_required {
3595                let woken = self.tcbs.resolve(task).map(|t| t.priority).unwrap_or(0);
3596                if woken > running {
3597                    switch_required = true;
3598                }
3599            }
3600        }
3601    }
3602
3603    /// `taskSWITCH_DELAYED_LISTS()`.
3604    // Out of line for the same reason: the overflow swap happens when the
3605    // tick count wraps, which is once in a very long while.
3606    // In line at its ONE caller (A3): win 26. It carried #[inline(never)] AND a
3607    // later #[inline] together, so the A3 change was dead from the day it was
3608    // written -- and rustc said so on every build.
3609    #[inline(always)]
3610    fn switch_delayed_lists(&mut self) {
3611        self.delayed_swapped = !self.delayed_swapped;
3612        self.overflows = self.overflows.wrapping_add(1);
3613        self.reset_next_task_unblock_time();
3614    }
3615
3616    /// `prvResetNextTaskUnblockTime`.
3617    ///
3618    /// ★ The EMPTY case is `MAX_DELAY`, not the list's own maximum.
3619    ///
3620    /// `head_value` cannot answer "empty": on an empty list the head IS the end
3621    /// marker, whose value is `u64::MAX` so that sorted inserts always find a
3622    /// larger value to stop at. So `unwrap_or` never fired and a reset over an
3623    /// empty delayed list stored `u64::MAX`, where the C stores `portMAX_DELAY`:
3624    ///
3625    /// ```c
3626    /// if( listLIST_IS_EMPTY( pxDelayedTaskList ) != pdFALSE ) {
3627    ///     xNextTaskUnblockTime = portMAX_DELAY;
3628    /// ```
3629    ///
3630    /// Unobservable in the corpus -- the difference only shows at a tick equal
3631    /// to `MAX_DELAY`, and `switch_delayed_lists` resets at the wrap anyway --
3632    /// which is why a test had PINNED the divergence rather than a gate
3633    /// catching it. Asking the emptiness question directly matches the C and
3634    /// makes `next_unblock_time <= MAX_DELAY` an invariant of the field.
3635    ///
3636    /// That invariant is worth five instructions per TICK: `increment_tick`
3637    /// compares `next` against this field, and while the field could hold a
3638    /// value wider than the tick that comparison had to be a full 64-bit one on
3639    /// a 32-bit target -- a second load, a `snez` and an `or` on top of the
3640    /// ordinary compare. With the invariant true, the comparison masks to the
3641    /// tick width and collapses. `bench/tick-work`: tick_idle and tick_delayed
3642    /// 14 -> 9 against the C's 15, 0.93x -> 0.60x.
3643    /// # Out of line, and that is load-bearing
3644    ///
3645    /// `switch_delayed_lists` is `#[inline(always)]` (win 26) and it calls this,
3646    /// so this body lands inside `increment_tick` -- on the arm taken when the
3647    /// tick WRAPS, i.e. once in `MAX_DELAY` ticks. Asking the emptiness question
3648    /// above made the body bigger, and in line that cost the tick row
3649    /// **14 -> 34** on `bench/tick-work`: twenty instructions of register
3650    /// pressure on every tick, to serve an arm that had not run once in the
3651    /// measurement. `rusty-compiler-leverage` A2, and the largest single
3652    /// instance of it this kernel has produced.
3653    #[cold]
3654    #[inline(never)]
3655    fn reset_next_task_unblock_time(&mut self) {
3656        let delayed = self.delayed_list();
3657        self.next_unblock_time = if self.lists.is_empty_of(delayed) {
3658            Self::MAX_DELAY
3659        } else {
3660            self.lists.head_value(delayed).unwrap_or(Self::MAX_DELAY)
3661        };
3662    }
3663
3664    // ------------------------------------------------------------- delay --
3665
3666    /// `vTaskDelay`.
3667    ///
3668    /// # Errors
3669    /// A list error surfaces as itself.
3670    pub fn delay(&mut self, ticks: u64) -> Result<()> {
3671        let caller = self.cur();
3672        let mut already_yielded = false;
3673        if ticks > 0 {
3674            self.suspend_all();
3675            {
3676                let current = self.cur();
3677                self.trace_task(current, |task, name| Event::TaskDelay { task, name, ticks });
3678                self.add_current_task_to_delayed_list(ticks, false)?;
3679            }
3680            already_yielded = self.resume_all();
3681        }
3682        // taskYIELD_WITHIN_API(), outside any critical section — unless a
3683        // tick switched us out somewhere above, in which case this line is
3684        // on a stack that is not running, and runs when the task does.
3685        if !already_yielded {
3686            if self.cur() == caller {
3687                self.port_yield();
3688            } else {
3689                self.owe_yield(caller);
3690            }
3691        }
3692        Ok(())
3693    }
3694
3695    /// `xTaskAbortDelay`: pull a task out of the Blocked state early.
3696    ///
3697    /// `true` is `pdPASS` — the task really was blocked and is now ready.
3698    /// A task that was not blocked is left alone and `false` comes back,
3699    /// which is the C's `pdFAIL`.
3700    ///
3701    /// The shape matters as much as the effect: the whole thing runs with
3702    /// the scheduler suspended, the event list is touched inside a critical
3703    /// section of its own because an interrupt can reach it, and a yield
3704    /// that the higher priority of the woken task calls for is *pended*
3705    /// rather than taken, so it happens on the way out of `xTaskResumeAll`.
3706    ///
3707    /// # Errors
3708    /// [`Error::Gone`] for a stale handle.
3709    pub fn abort_delay(&mut self, task: TaskHandle) -> Result<bool> {
3710        if !self.tcbs.contains(task) {
3711            return Err(Error::Gone);
3712        }
3713        self.suspend_all();
3714        // eTaskGetState: a critical section of its own on any task but the
3715        // running one, and the running one is never Blocked.
3716        let blocked = self.task_state_get(task)? == TaskState::Blocked;
3717        if !blocked {
3718            let _ = self.resume_all();
3719            return Ok(false);
3720        }
3721        let item = Self::state_item(task);
3722        // `remove` is its own guard, as above.
3723        let _ = self.lists.remove(item);
3724        self.enter_critical();
3725        {
3726            let event = Self::event_item(task);
3727            // `remove` answers `NotActive` when the item is in no list, so
3728            // it is its own guard -- and `is_ok` is exactly "something was
3729            // removed", which is what the flag below depends on.
3730            if self.lists.remove(event).is_ok() {
3731                if let Some(f) = self.delay_aborted.get_mut(task.index() as usize) {
3732                    *f = true;
3733                }
3734            }
3735        }
3736        self.exit_critical();
3737        self.add_task_to_ready_list(task)?;
3738        // configUSE_PREEMPTION, one core: pend the yield rather than take
3739        // it, so it lands where `xTaskResumeAll` puts it.
3740        let woken = self.tcbs.resolve(task).map(|t| t.priority).unwrap_or(0);
3741        if C::NUMBER_OF_CORES > 1 {
3742            let _ = self.smp_readied(task, woken);
3743        } else if woken > self.current_priority() {
3744            self.set_pending_here(true);
3745        }
3746        let _ = self.resume_all();
3747        Ok(true)
3748    }
3749
3750    /// `prvAddCurrentTaskToDelayedList`.
3751    pub(crate) fn add_current_task_to_delayed_list(
3752        &mut self,
3753        ticks: u64,
3754        can_block_indefinitely: bool,
3755    ) -> Result<()> {
3756        // Both masked, and for the reason `increment_tick`'s comparison is: at a
3757        // 32-bit tick width the high half of every tick value is invariantly
3758        // zero, LLVM cannot know that of a field it loaded or a parameter it was
3759        // handed, and without being told it emits full 64-bit compares on a
3760        // 32-bit target. Three of them live below -- `ticks == MAX_DELAY`,
3761        // `wake_at < now`, and `wake_at < next_unblock_time`.
3762        //
3763        // `self.tick` is masked by every writer (`increment_tick` and
3764        // `step_tick` both `& MAX_DELAY`). `ticks` reaches here from
3765        // `check_for_timeout`, whose answers are all at most `MAX_DELAY`, and
3766        // from `begin_wait`, which truncates what it stores. At a 64-bit width
3767        // `MAX_DELAY` is `u64::MAX` and both masks fold to nothing.
3768        debug_assert_eq!(
3769            self.tick,
3770            self.tick & Self::MAX_DELAY,
3771            "the tick is wider than its own width"
3772        );
3773        let now = self.tick & Self::MAX_DELAY;
3774        let ticks = ticks & Self::MAX_DELAY;
3775        let current = self.cur();
3776        // B1, and this function is where it pays most: by its own note below it
3777        // is the single largest consumer of the blocking workload, and it
3778        // derives FOUR bound-checked accesses from an index it read out of
3779        // memory -- `delay_aborted`, then `state_item` feeding `lists.remove`
3780        // and one of three inserts. `self.cur()` is a live task on every path
3781        // that reaches here, but it is a FIELD, so nothing carries that across
3782        // the calls above.
3783        if current.index() as usize >= TASKS {
3784            return Err(Error::Gone);
3785        }
3786        // About to enter a delayed list, so the abort flag is cleared here
3787        // and can only be seen set by a task that really was aborted.
3788        if let Some(f) = self.delay_aborted.get_mut(current.index() as usize) {
3789            *f = false;
3790        }
3791        let item = Self::state_item(current);
3792        // `uxListRemove` already answers `NotActive` for an item that is
3793        // in no list, and both it and the container test are one read of
3794        // the same node — so asking first was reading it twice. This is
3795        // the running task's state item, so it is in the ready list and
3796        // the test passed anyway.
3797        let _ = self.lists.remove(item);
3798        if ticks == Self::MAX_DELAY && can_block_indefinitely {
3799            self.lists.insert_end(Self::suspended_list(), item)?;
3800            return Ok(());
3801        }
3802        let wake_at = now.wrapping_add(ticks) & Self::MAX_DELAY;
3803        // `vListInsert` sets the item's value from the one it sorts by, so
3804        // both arms below write `wake_at` themselves. Setting it here as
3805        // well was a second lookup of the same item to store the same
3806        // number into it.
3807        if wake_at < now {
3808            self.trace_task(current, |task, name| {
3809                Event::MovedTaskToOverflowDelayedList { task, name }
3810            });
3811            let list = self.overflow_delayed_list();
3812            self.lists.insert(list, item, wake_at)?;
3813        } else {
3814            self.trace_task(current, |task, name| Event::MovedTaskToDelayedList {
3815                task,
3816                name,
3817            });
3818            let list = self.delayed_list();
3819            // ---- append, when the wake time says we may ------------------
3820            //
3821            // `vListInsert` walks from the head until it finds a larger
3822            // value, so a delayed list `n` deep costs `n` steps -- and this
3823            // function is the single largest consumer of the blocking
3824            // workload, most of it that walk.
3825            //
3826            // The walk is almost always finding the same answer: the END.
3827            // `wake_at` is `now + ticks` and `now` only increases, so tasks
3828            // blocking with comparable timeouts produce ASCENDING wake times
3829            // and belong after everything already there.
3830            //
3831            // `insert_sorted` takes that shortcut, and it asks this caller
3832            // for exactly ONE thing: that the list is already sorted. The
3833            // delayed list is, because it is only ever built by sorted
3834            // inserts and by this append -- so its last item carries its
3835            // largest value.
3836            //
3837            // The other two promises this call site used to make are gone
3838            // into the list, where they are written once instead of at
3839            // every future call site: the `>=` tie rule, and the fact that
3840            // `insert_end` only appends while the cursor is at the marker.
3841            // Both were silent when wrong. See `ListsOf::insert_sorted`,
3842            // and `ListsOf::is_sorted` for checking the one that is left.
3843            self.lists.insert_sorted(list, item, wake_at)?;
3844            // Masked for the same reason `increment_tick`'s comparison is, and
3845            // it is the second site of that win: `next_unblock_time <=
3846            // MAX_DELAY` is an invariant of the field (see
3847            // `reset_next_task_unblock_time`, which is what made it one), and
3848            // `wake_at` was masked where it was computed above. So at a 32-bit
3849            // tick width the high half of both is invariantly zero, and without
3850            // saying so LLVM emits a full 64-bit compare on a 32-bit target --
3851            // a second load, a `snez` and an `or` on top of the ordinary one.
3852            //
3853            // At a 64-bit width `MAX_DELAY` is `u64::MAX` and the mask folds to
3854            // nothing, so that configuration pays no instruction for this.
3855            debug_assert_eq!(
3856                self.next_unblock_time,
3857                self.next_unblock_time & Self::MAX_DELAY,
3858                "next_unblock_time holds a value wider than the tick"
3859            );
3860            if wake_at < self.next_unblock_time & Self::MAX_DELAY {
3861                self.next_unblock_time = wake_at;
3862            }
3863        }
3864        Ok(())
3865    }
3866
3867    // --------------------------------------------------- suspend / resume --
3868
3869    /// `vTaskSuspend`. `None` suspends the calling task.
3870    ///
3871    /// # Errors
3872    /// [`Error::Gone`] for a stale handle.
3873    pub fn suspend(&mut self, task: Option<TaskHandle>) -> Result<()> {
3874        let caller = self.cur();
3875        let target = task.unwrap_or(caller);
3876        self.enter_critical();
3877        {
3878            if !self.tcbs.contains(target) {
3879                self.exit_critical();
3880                return Err(Error::Gone);
3881            }
3882            self.trace_task(target, |task, name| Event::TaskSuspend { task, name });
3883            let item = Self::state_item(target);
3884            // `remove` is its own guard: it answers `NotActive` for an item
3885            // in no list, and the result was discarded either way.
3886            let _ = self.lists.remove(item);
3887            let event = Self::event_item(target);
3888            if self.lists.container(event)?.is_some() {
3889                let _ = self.lists.remove(event);
3890            }
3891            self.lists.insert_end(Self::suspended_list(), item)?;
3892            // `vTaskSuspend`, `tasks.c`:
3893            //
3894            // ```c
3895            // if( pxTCB->ucNotifyState[ x ] == taskWAITING_NOTIFICATION )
3896            // {
3897            //     /* The task was blocked to wait for a notification, but
3898            //      * is now suspended, so no notification was received. */
3899            //     pxTCB->ucNotifyState[ x ] = taskNOT_WAITING_NOTIFICATION;
3900            // }
3901            // ```
3902            //
3903            // Not tidying: `eTaskGetState` calls a task on the suspended
3904            // list BLOCKED rather than SUSPENDED if any of its notification
3905            // slots is still waiting, so leaving the flag set makes a
3906            // suspended task report as blocked for ever.
3907            // `TaskNotify.c:498` asserts exactly that, in a timer callback
3908            // that suspends a task which is waiting for a notification --
3909            // and a task that reports the wrong state there is a task the
3910            // rest of that test never resumes correctly.
3911            if let Ok(tcb) = self.tcbs.resolve_mut(target) {
3912                for slot in &mut tcb.notify_state {
3913                    if *slot == NotifyState::Waiting {
3914                        *slot = NotifyState::NotWaiting;
3915                    }
3916                }
3917            }
3918        }
3919        self.exit_critical();
3920        // A second, separate critical section — two outermost exits, which
3921        // is what the C does and therefore what the tick count depends on.
3922        if self.running {
3923            self.enter_critical();
3924            self.reset_next_task_unblock_time();
3925            self.exit_critical();
3926        }
3927        if C::NUMBER_OF_CORES > 1 {
3928            // `vTaskSuspend`: the core running `target` switches away -- this
3929            // one now, another by `prvYieldCore`.
3930            if self.running {
3931                // By slot index, as `yield_for_task`: `target` was found
3932                // live above, and so is every core's current task once the
3933                // scheduler runs.
3934                let slot = target.index();
3935                let core = (0..Self::cores())
3936                    .find(|&c| self.current_of(c).index() == slot && !self.requested_on(c));
3937                match core {
3938                    Some(core) if core == self.core() => self.port_yield(),
3939                    Some(core) => self.yield_core(core),
3940                    None => {}
3941                }
3942            }
3943            return Ok(());
3944        }
3945        if target == caller {
3946            if self.running {
3947                // portYIELD_WITHIN_API(), outside the section.
3948                if self.cur() == caller {
3949                    self.port_yield();
3950                } else {
3951                    self.owe_yield(caller);
3952                }
3953            } else if self.lists.len(Self::suspended_list()).unwrap_or(0) != self.task_count {
3954                self.switch_context();
3955            }
3956        }
3957        Ok(())
3958    }
3959
3960    /// `vTaskResume`.
3961    ///
3962    /// # Errors
3963    /// [`Error::Gone`] for a stale handle.
3964    pub fn resume(&mut self, task: TaskHandle) -> Result<()> {
3965        if task == self.cur() || task.is_null() {
3966            return Ok(());
3967        }
3968        if !self.tcbs.contains(task) {
3969            return Err(Error::Gone);
3970        }
3971        self.enter_critical();
3972        {
3973            if self.task_is_suspended(task)? {
3974                self.trace_task(task, |task, name| Event::TaskResume { task, name });
3975                let _ = self.lists.remove(Self::state_item(task));
3976                self.add_task_to_ready_list(task)?;
3977                // taskYIELD_ANY_CORE_IF_USING_PREEMPTION: inside the
3978                // section, and only when the resumed task outranks the
3979                // running one.
3980                let resumed = self.tcbs.resolve(task)?.priority;
3981                if C::NUMBER_OF_CORES > 1 {
3982                    if self.smp_readied(task, resumed) {
3983                        self.port_yield();
3984                    }
3985                } else if C::USE_PREEMPTION && self.current_priority() < resumed {
3986                    self.port_yield();
3987                }
3988            }
3989        }
3990        self.exit_critical();
3991        Ok(())
3992    }
3993
3994    /// `prvTaskIsTaskSuspended`: on the suspended list, not on the pending
3995    /// ready list, and not waiting on an event.
3996    fn task_is_suspended(&self, task: TaskHandle) -> Result<bool> {
3997        if self.lists.container(Self::state_item(task))? != Some(Self::suspended_list()) {
3998            return Ok(false);
3999        }
4000        let event_list = self.lists.container(Self::event_item(task))?;
4001        if event_list == Some(Self::pending_ready_list()) {
4002            return Ok(false);
4003        }
4004        Ok(event_list.is_none())
4005    }
4006
4007    // -------------------------------------------------------- priorities --
4008
4009    /// `vTaskDelete`. `None` deletes the calling task.
4010    ///
4011    /// # Two paths, and the C picks between them for a reason
4012    ///
4013    /// Deleting *another* task frees its TCB on the spot. Deleting the
4014    /// **running** task cannot: the caller is still executing out of it. The
4015    /// C parks such a task on `xTasksWaitingTermination` and lets the idle
4016    /// task reclaim it, which is what `prvCheckTasksWaitingTermination` is
4017    /// for — so [`Kernel::check_tasks_waiting_termination`] is where the
4018    /// deferred half lands here, called from the idle body at the same point
4019    /// in the same loop.
4020    ///
4021    /// # The frees are outside the critical section, and they cost exits
4022    ///
4023    /// `prvDeleteTCB` runs **after** `taskEXIT_CRITICAL()` and frees twice —
4024    /// `vPortFreeStack( pxStack )` then `vPortFree( pxTCB )`. On the oracle's
4025    /// heap every free suspends the scheduler, so a delete is one exit for
4026    /// its own section plus two for the frees, in that order. Folding the
4027    /// frees into the section, or forgetting them, moves every later tick.
4028    ///
4029    /// # Errors
4030    /// [`Error::Gone`] for a stale handle.
4031    pub fn task_delete(&mut self, task: Option<TaskHandle>) -> Result<()> {
4032        // `prvGetTCBFromHandle`: NULL is the calling task.
4033        let target = task.unwrap_or(self.cur());
4034        self.tcbs.resolve(target)?;
4035
4036        self.enter_critical();
4037        // `uxListRemove( &( pxTCB->xStateListItem ) )`. The C follows this
4038        // with `taskRESET_READY_PRIORITY`, which is an EMPTY MACRO under the
4039        // oracle's `configUSE_PORT_OPTIMISED_TASK_SELECTION 0` — there is no
4040        // priority bitmap to clear, and `vTaskSwitchContext` rediscovers the
4041        // top priority by walking down. Nothing to model.
4042        let state = Self::state_item(target);
4043        if matches!(self.lists.container(state), Ok(Some(_))) {
4044            let _ = self.lists.remove(state);
4045        }
4046        // `if( listLIST_ITEM_CONTAINER( &( pxTCB->xEventListItem ) ) != NULL )`
4047        // — a task blocked on a queue is on two lists, and both must let go.
4048        let event = Self::event_item(target);
4049        if matches!(self.lists.container(event), Ok(Some(_))) {
4050            let _ = self.lists.remove(event);
4051        }
4052        // `uxTaskNumber++` is not modelled: it exists so kernel-aware
4053        // debuggers can tell the task lists changed, and nothing the trace
4054        // records reads it.
4055
4056        // `xSchedulerRunning != pdFALSE && taskTASK_IS_RUNNING_OR_SCHEDULED_
4057        // _TO_YIELD( pxTCB )`, which at `configNUMBER_OF_CORES 1` is exactly
4058        // `pxTCB == pxCurrentTCB`.
4059        // SMP: a task any core still holds -- running, or asked to yield --
4060        // cannot have its TCB freed under that core (`vTaskDelete`'s
4061        // `taskTASK_IS_RUNNING_OR_SCHEDULED_TO_YIELD`).
4062        let holder = if C::NUMBER_OF_CORES > 1 {
4063            self.held_core(target)
4064        } else {
4065            None
4066        };
4067        let defer = self.running
4068            && if C::NUMBER_OF_CORES > 1 {
4069                holder.is_some()
4070            } else {
4071                target == self.cur()
4072            };
4073        if defer {
4074            // `vListInsertEnd( &xTasksWaitingTermination, ... )` and
4075            // `++uxDeletedTasksWaitingCleanUp`. The count is NOT decremented
4076            // here — the idle task does that when it reaps.
4077            if C::NUMBER_OF_CORES > 1 && !self.awaiting_reap.is_null() {
4078                self.awaiting_reap_smp = target;
4079            } else {
4080                self.awaiting_reap = target;
4081            }
4082            self.trace_task(target, |task, name| Event::TaskDelete { task, name });
4083        } else {
4084            self.task_count = self.task_count.saturating_sub(1);
4085            self.trace_task(target, |task, name| Event::TaskDelete { task, name });
4086            self.reset_next_task_unblock_time();
4087        }
4088        self.exit_critical();
4089
4090        // `if( xDeleteTCBInIdleTask != pdTRUE ) { prvDeleteTCB( pxTCB ); }`,
4091        // outside the section.
4092        if !defer {
4093            self.delete_tcb(target);
4094        }
4095
4096        // `taskYIELD_WITHIN_API()`. The C guards this with
4097        // `xSchedulerRunning && pxTCB == pxCurrentTCB`, which is the same
4098        // condition that chose the deferred path.
4099        if defer {
4100            if C::NUMBER_OF_CORES > 1 {
4101                // The holder switches away: this core now, another core by
4102                // `prvYieldCore` -- unless it was already asked.
4103                match holder {
4104                    Some(core) if core == self.core() => self.port_yield(),
4105                    Some(core) if self.running_core(target) == Some(core) => {
4106                        self.yield_core(core);
4107                    }
4108                    _ => {}
4109                }
4110            } else {
4111                self.port_yield();
4112            }
4113        }
4114        Ok(())
4115    }
4116
4117    /// `prvDeleteTCB`: two frees, and on the oracle's heap each one suspends
4118    /// the scheduler, so each is one outermost exit.
4119    /// `prvCheckTasksWaitingTermination` on SMP: free a deleted task only
4120    /// once NO core holds it -- the other core may not have switched away
4121    /// yet.
4122    #[cold]
4123    #[inline(never)]
4124    fn reap_smp(&mut self) {
4125        for smp_slot in [false, true] {
4126            let task = if smp_slot {
4127                self.awaiting_reap_smp
4128            } else {
4129                self.awaiting_reap
4130            };
4131            if task.is_null() || self.held_by_any_core(task) {
4132                continue;
4133            }
4134            self.enter_critical();
4135            if smp_slot {
4136                self.awaiting_reap_smp = TaskHandle::NULL;
4137            } else {
4138                self.awaiting_reap = TaskHandle::NULL;
4139            }
4140            self.task_count = self.task_count.saturating_sub(1);
4141            self.exit_critical();
4142            self.delete_tcb(task);
4143        }
4144    }
4145
4146    fn delete_tcb(&mut self, task: TaskHandle) {
4147        // `vPortFreeStack( pxTCB->pxStack )`.
4148        self.account_for_allocation();
4149        // `vPortFree( pxTCB )`.
4150        self.account_for_allocation();
4151        let _ = self.tcbs.remove(task);
4152
4153        // The index is now free for reuse, so mark the task UNSTARTED.
4154        //
4155        // The C frees the TCB and a later `xTaskCreate` gets fresh memory,
4156        // so a new task cannot inherit anything. Our arena hands back the
4157        // same index, and the per-index bookkeeping — owed exits, an owed
4158        // yield, an owed trace line — is keyed on that index, not on the
4159        // TCB. Without this a task deleted while it still owed the tail of
4160        // a call bequeaths that debt to its successor, and the successor
4161        // pays an exit it never incurred: exactly how `death`'s second
4162        // cycle first diverged, one exit early, 1,100 lines after the first
4163        // cycle had matched perfectly.
4164        //
4165        // Clearing this ONE flag is the whole fix, and deliberately so.
4166        // `hand_over` already empties all three of those on a task's first
4167        // switch-in, and a reused index is a first switch-in again. Zeroing
4168        // them here as well would work and would be worse: two places that
4169        // must agree about what a fresh task owes, and a gate that cannot
4170        // catch either one being dropped because the other still covers it.
4171        if let Some(f) = self.started.get_mut(task.index() as usize) {
4172            *f = false;
4173        }
4174    }
4175
4176    /// `prvCheckTasksWaitingTermination`, called from the idle task.
4177    ///
4178    /// # This must cost NOTHING when nothing is pending
4179    ///
4180    /// The C's body is `while( uxDeletedTasksWaitingCleanUp > 0 )`, so an
4181    /// idle loop with no deleted task takes no critical section and pays no
4182    /// exit. That is what lets this be wired into the idle body without
4183    /// disturbing a single scenario in the corpus — and it is the property
4184    /// the corpus is checking when it stays byte-identical.
4185    ///
4186    /// At most one task can be pending: only the *running* task defers, and
4187    /// it yields before it can return, so a second deferred delete cannot
4188    /// happen until this has run.
4189    pub fn check_tasks_waiting_termination(&mut self) {
4190        if C::NUMBER_OF_CORES > 1 {
4191            // Nothing deleted is the answer on almost every idle pass: say so
4192            // here rather than in a call to the out-of-line body.
4193            if !self.awaiting_reap.is_null() || !self.awaiting_reap_smp.is_null() {
4194                self.reap_smp();
4195            }
4196            return;
4197        }
4198        let task = self.awaiting_reap;
4199        if task.is_null() {
4200            return;
4201        };
4202        // The C takes the section per reaped task, decrements both counts
4203        // inside it, and calls `prvDeleteTCB` outside.
4204        self.enter_critical();
4205        self.awaiting_reap = TaskHandle::NULL;
4206        self.task_count = self.task_count.saturating_sub(1);
4207        self.exit_critical();
4208        self.delete_tcb(task);
4209    }
4210
4211    /// `vTaskPrioritySet`. `None` is the calling task.    /// `vTaskPrioritySet`. `None` is the calling task.
4212    ///
4213    /// # Errors
4214    /// [`Error::Gone`] for a stale handle.
4215    pub fn set_priority(&mut self, task: Option<TaskHandle>, new_priority: u8) -> Result<()> {
4216        let new_priority = new_priority.min(C::MAX_PRIORITIES.saturating_sub(1));
4217        let target = task.unwrap_or(self.cur());
4218        let mut yield_required = false;
4219        self.enter_critical();
4220        {
4221            if !self.tcbs.contains(target) {
4222                self.exit_critical();
4223                return Err(Error::Gone);
4224            }
4225            self.trace_task(target, |task, name| Event::TaskPrioritySet {
4226                task,
4227                name,
4228                priority: Self::priority_value(new_priority),
4229            });
4230            let base = self.tcbs.resolve(target)?.base_priority;
4231            // SMP (`vTaskPrioritySet`): raising readies a contender anywhere
4232            // (`taskYIELD_ANY_CORE`); lowering a RUNNING task yields the core
4233            // running it (`taskYIELD_TASK_CORE`), which may not be this one.
4234            let mut smp_yield_for = C::NUMBER_OF_CORES > 1 && new_priority > base;
4235            let smp_yield_core = if C::NUMBER_OF_CORES > 1 && new_priority < base {
4236                self.running_core(target)
4237            } else {
4238                None
4239            };
4240            if base != new_priority {
4241                if new_priority > base {
4242                    if target != self.cur() && new_priority > self.current_priority() {
4243                        yield_required = true;
4244                    }
4245                } else if target == self.cur() {
4246                    yield_required = true;
4247                }
4248                let used_on_entry = self.tcbs.resolve(target)?.priority;
4249                {
4250                    let tcb = self.tcbs.resolve(target)?;
4251                    let raise = tcb.base_priority == tcb.priority || new_priority > tcb.priority;
4252                    if raise {
4253                        self.set_task_priority(target, new_priority)?;
4254                    }
4255                    self.tcbs.resolve_mut(target)?.base_priority = new_priority;
4256                }
4257                let event_value = u64::from(C::MAX_PRIORITIES.saturating_sub(new_priority));
4258                self.lists
4259                    .set_value(Self::event_item(target), event_value)?;
4260                let item = Self::state_item(target);
4261                if self.lists.container(item)? == Some(Self::ready_list(used_on_entry)) {
4262                    let _ = self.lists.remove(item);
4263                    self.add_task_to_ready_list(target)?;
4264                } else {
4265                    // Not ready (blocked or suspended): nothing to yield for.
4266                    smp_yield_for = false;
4267                }
4268                // taskYIELD_TASK_CORE_IF_USING_PREEMPTION: inside the
4269                // section.
4270                if C::NUMBER_OF_CORES > 1 {
4271                    if C::USE_PREEMPTION {
4272                        if let Some(core) = smp_yield_core {
4273                            self.yield_core(core);
4274                        } else if smp_yield_for {
4275                            let now = self.tcbs.resolve(target)?.priority;
4276                            self.yield_for_task(target, now);
4277                        }
4278                        if self.pending_here() {
4279                            self.port_yield();
4280                        }
4281                    }
4282                } else if yield_required && C::USE_PREEMPTION {
4283                    self.port_yield();
4284                }
4285            }
4286        }
4287        self.exit_critical();
4288        Ok(())
4289    }
4290
4291    // ------------------------------------------------- scheduler suspend --
4292
4293    /// `vTaskSuspendAll`.
4294    pub fn suspend_all(&mut self) {
4295        self.suspended_depth = self.suspended_depth.wrapping_add(1);
4296    }
4297
4298    /// Move everything an ISR readied onto the ready lists.
4299    ///
4300    /// Out of line because it is the rare half of [`Kernel::resume_all`]: the
4301    /// pending list is empty on essentially every call, and inlining this made
4302    /// every one of those calls carry a frame sized for the walk.
4303    ///
4304    /// **The GUARD is inlined and only the WALK is out of line.** Outlining
4305    /// the whole thing — which is what this used to do — meant every call paid
4306    /// a call and a frame to discover there was nothing to do, on a path taken
4307    /// three times inside `queue_send_blocking` and three more inside
4308    /// `queue_take_blocking`. Asking "is the list empty" is one list read; the
4309    /// walk behind it is what wanted outlining.
4310    #[inline(always)]
4311    fn drain_pending_ready(&mut self) {
4312        if self.lists.is_empty_of(Self::pending_ready_list()) {
4313            // Empty, or a list error: either way the walk below moves nothing,
4314            // so `moved_any` would stay false and the reset would not happen.
4315            return;
4316        }
4317        self.drain_pending_ready_walk();
4318    }
4319
4320    /// The walk itself. See [`Kernel::drain_pending_ready`].
4321    #[cold]
4322    #[inline(never)]
4323    fn drain_pending_ready_walk(&mut self) {
4324        let mut moved_any = false;
4325        while !self.lists.is_empty_of(Self::pending_ready_list()) {
4326            let Ok(Some(item)) = self.lists.head(Self::pending_ready_list()) else {
4327                break;
4328            };
4329            let Ok(task) = self.task_of_event_item(item) else {
4330                break;
4331            };
4332            let _ = self.lists.remove(Self::event_item(task));
4333            let _ = self.lists.remove(Self::state_item(task));
4334            if self.add_task_to_ready_list(task).is_err() {
4335                break;
4336            }
4337            moved_any = true;
4338            let woken = self.tcbs.resolve(task).map(|t| t.priority).unwrap_or(0);
4339            if C::NUMBER_OF_CORES > 1 {
4340                let _ = self.smp_readied(task, woken);
4341            } else if woken > self.current_priority() {
4342                self.set_pending_here(true);
4343            }
4344        }
4345        if moved_any {
4346            self.reset_next_task_unblock_time();
4347        }
4348    }
4349
4350    /// Run the ticks that landed while the scheduler was suspended.
4351    ///
4352    /// Out of line for the same reason as its sibling: `pended_ticks` is zero
4353    /// on essentially every call — and, as there, **only the loop is out of
4354    /// line now**. The guard is a field read, and it also skips the
4355    /// `pended_ticks = 0` store that the zero case was writing over a zero.
4356    #[inline(always)]
4357    fn unwind_pended_ticks(&mut self) {
4358        if self.pended_ticks == 0 {
4359            return;
4360        }
4361        self.unwind_pended_ticks_loop();
4362    }
4363
4364    /// The loop itself. See [`Kernel::unwind_pended_ticks`].
4365    #[cold]
4366    #[inline(never)]
4367    fn unwind_pended_ticks_loop(&mut self) {
4368        let mut pended = self.pended_ticks;
4369        while pended > 0 {
4370            if self.increment_tick() {
4371                self.set_pending_here(true);
4372            }
4373            pended = pended.saturating_sub(1);
4374        }
4375        self.pended_ticks = 0;
4376    }
4377
4378    /// Hand the port's critical-section exit tally to the trace sink.
4379    ///
4380    /// Gated on `T::EMITS` because `Port::exits` reads an ATOMIC counter, and
4381    /// LLVM may not delete an atomic load even when the value it produces is
4382    /// unused. With `NoTrace` the body folds to nothing while the `lw` behind it
4383    /// would stay, so every site was paying a load of a counter no sink was
4384    /// going to read. The gate is a compile-time constant, so an emitting sink
4385    /// is unchanged and a silent one loses the load entirely.
4386    #[inline(always)]
4387    pub(crate) fn note_exits(&mut self) {
4388        if T::EMITS {
4389            self.trace.note_exits(self.port.exits());
4390        }
4391    }
4392
4393    /// `xTaskResumeAll`; `true` when it yielded on the caller's behalf.
4394    ///
4395    /// Out of line for the cold callers; the body is [`Kernel::resume_all_inline`].
4396    #[inline(never)]
4397    pub fn resume_all(&mut self) -> bool {
4398        self.resume_all_inline()
4399    }
4400
4401    /// The body of [`Kernel::resume_all`], in line for the hot callers
4402    /// (one shared body under the `small` feature).
4403    #[cfg_attr(not(feature = "small"), inline(always))]
4404    #[cfg_attr(feature = "small", inline(never))]
4405    pub(crate) fn resume_all_inline(&mut self) -> bool {
4406        let mut already_yielded = false;
4407        self.enter_critical();
4408        {
4409            self.suspended_depth = self.suspended_depth.saturating_sub(1);
4410            if self.suspended_depth == 0 && self.task_count > 0 {
4411                if !self.lists.is_empty_of(Self::pending_ready_list()) {
4412                    self.drain_pending_ready();
4413                }
4414                if self.pended_ticks > 0 {
4415                    self.unwind_pended_ticks();
4416                }
4417                if self.pending_here() {
4418                    if C::USE_PREEMPTION {
4419                        already_yielded = true;
4420                    }
4421                    // taskYIELD_TASK_CORE_IF_USING_PREEMPTION: inside the
4422                    // section.
4423                    self.port_yield();
4424                }
4425            }
4426        }
4427        self.exit_critical();
4428        already_yielded
4429    }
4430
4431    // ------------------------------------------- the blocking-call frame --
4432
4433    /// Start (or continue) a blocking call's `for(;;)`.
4434    ///
4435    /// The first pass records the block time and the entry timestamp, the
4436    /// way `xEntryTimeSet` / `vTaskInternalSetTimeOutState` do; later
4437    /// passes leave the running total alone, because the `ticks` the caller
4438    /// passes is the original block time and the kernel is holding what is
4439    /// left of it.
4440    ///
4441    /// Answers the block time that is LEFT. Both call sites wanted it right
4442    /// afterwards and used to resolve the same TCB a second time to read a
4443    /// field this function is already holding a `&mut` to. The answer is exact
4444    /// rather than merely equal to `ticks`, because the frame may be an earlier
4445    /// pass's — in which case the remaining time is what the kernel has been
4446    /// counting down, not what the caller passed.
4447    ///
4448    /// `None` means no frame was written at all, so the caller can skip the
4449    /// `end_wait` that would tear one down.
4450    pub(crate) fn begin_wait(
4451        &mut self,
4452        task: TaskHandle,
4453        queue: QueueHandle,
4454        ticks: u64,
4455    ) -> Option<u64> {
4456        // ★ TRUNCATE TO THE TICK WIDTH, at the boundary, because that is what
4457        // the C does and it is what makes every later comparison 32-bit.
4458        //
4459        // C's `xTicksToWait` is a `TickType_t`. At a 32-bit configuration a
4460        // caller CANNOT express a wait longer than `MAX_DELAY` -- the value is
4461        // truncated by the assignment, so `2^32` ticks becomes 0 and
4462        // `u64::MAX` becomes `portMAX_DELAY`, i.e. forever. This API takes a
4463        // `u64` so the kernel is width-independent, which means it must do that
4464        // truncation itself rather than let a value the C could not represent
4465        // reach a wait frame.
4466        //
4467        // The masking is BEFORE the zero test on purpose: C's truncation
4468        // happens at the assignment, so a caller asking for exactly `2^32`
4469        // ticks gets C's answer -- do not block -- and not a very long wait.
4470        //
4471        // What it buys: every field of a `WaitFrame` is then provably no wider
4472        // than the tick, so `check_for_timeout`'s comparisons against them stop
4473        // being 64-bit ones on a 32-bit target. That is the same lever that took
4474        // `tick_idle` 14 -> 9; see `reset_next_task_unblock_time`.
4475        let ticks = ticks & Self::MAX_DELAY;
4476        // The zero-block-time exit, taken before anything is read. `wait_set`
4477        // mirrors the TCB's `entry_set` in one byte, which is the same trade
4478        // `end_wait` documents and makes — so the frame can be ruled out
4479        // without an arena lookup, and without reading the clock either.
4480        // Out of range reads as SET, so an impossible index still goes the
4481        // long way round.
4482        if ticks == 0 && !self.flag(task.index() as usize, Self::F_WAIT, true) {
4483            // `None`, not `Some(0)`: no frame was written, so the caller can
4484            // skip the `end_wait` that would tear one down. It reads the same
4485            // `wait_set` byte to find nothing to do.
4486            return None;
4487        }
4488        let tick = self.tick;
4489        let overflows = self.overflows;
4490        let Ok(tcb) = self.tcbs.resolve_mut(task) else {
4491            // A queue call made before the scheduler started, from what the
4492            // C would call `main()`: there is no task to keep a frame for,
4493            // and such a call never blocks — `xSemaphoreGive` on a fresh
4494            // semaphore is the usual one. The old read answered 0 here
4495            // (`unwrap_or(0)` on the same failed resolve), so 0 is what this
4496            // must answer to keep the call sites reading the same.
4497            return Some(0);
4498        };
4499        if tcb.wait.queue != queue || !tcb.wait.entry_set {
4500            tcb.wait = WaitFrame {
4501                queue,
4502                ticks: crate::timer::Split64::new(ticks),
4503                entering: crate::timer::Split64::new(tick),
4504                overflows,
4505                entry_set: true,
4506                inherited: false,
4507            };
4508        }
4509        let remaining = tcb.wait.ticks.get();
4510        // The frame is set either way on this path -- either it was written
4511        // just now, or it was already this queue's.
4512        self.set_flag(task.index() as usize, Self::F_WAIT, true);
4513        Some(remaining)
4514    }
4515
4516    /// The call finished, one way or the other.
4517    pub(crate) fn end_wait(&mut self, task: TaskHandle) {
4518        // Almost every call arrives with no frame to clear, and the mirror
4519        // says so in one byte where resolving the TCB to read `entry_set`
4520        // cost an arena lookup. Out of range reads as SET, so an impossible
4521        // index still takes the slow path below.
4522        let at = task.index() as usize;
4523        if !self.flag(at, Self::F_WAIT, true) {
4524            return;
4525        }
4526        self.set_flag(at, Self::F_WAIT, false);
4527        if let Ok(tcb) = self.tcbs.resolve_mut(task) {
4528            // Now that the frame is only set on the path that blocks, most
4529            // calls reach here with nothing to clear, and this was writing
4530            // a default frame over a default frame. `entry_set` is the
4531            // sentinel the C calls `xEntryTimeSet`, and it is the first
4532            // thing set — `inherited` is only reached far inside the
4533            // blocking path, so it cannot be true while this is false.
4534            if tcb.wait.entry_set {
4535                tcb.wait = WaitFrame::default();
4536            }
4537        }
4538    }
4539
4540    /// `xInheritanceOccurred`.
4541    pub(crate) fn wait_inherited(&self, task: TaskHandle) -> bool {
4542        self.tcbs
4543            .resolve(task)
4544            .map(|t| t.wait.inherited)
4545            .unwrap_or(false)
4546    }
4547
4548    pub(crate) fn set_wait_inherited(&mut self, task: TaskHandle) {
4549        if let Ok(tcb) = self.tcbs.resolve_mut(task) {
4550            tcb.wait.inherited = true;
4551        }
4552    }
4553
4554    /// `xTaskCheckForTimeOut`: `true` when the block time has run out.
4555    /// Takes a critical section, as the C does, and charges the elapsed
4556    /// time against what is left.
4557    /// `None` is `pdTRUE` — the block time has run out. `Some(left)` is
4558    /// `pdFALSE`, and carries what is left of it.
4559    ///
4560    /// The block time is threaded out because both call sites asked for it
4561    /// immediately afterwards by resolving the TCB again, which
4562    /// the same TCB out of the arena again to read the very field the match
4563    /// below has just written.
4564    /// ★ `Option<NonZeroU64>`, not `Option<u64>`, and the reason is the
4565    /// CALLER's register budget.
4566    ///
4567    /// `Option<u64>` has no niche, so it is sixteen bytes — two registers —
4568    /// and `queue_take_blocking` holds it live across the branch that tests
4569    /// it. That function sits at a register cliff: six callee-saved pushes, a
4570    /// 72-byte frame, `caller`'s index spilled and re-read five times, and
4571    /// four separate attempts to make it cheaper all LOST because each added a
4572    /// live value or a branch (+200,668, +11,619, +254,429, +624,386; see
4573    /// `docs/LEDGER.md`). The one shape left is to make an existing value
4574    /// smaller, which is `rusty-compiler-leverage` B5 — the representation
4575    /// change that won −987,806 elsewhere by turning a 16-byte `Option` into
4576    /// one word.
4577    ///
4578    /// `Some(0)` is unreachable, which is what makes this free: the indefinite
4579    /// arm answers `MAX_DELAY` (non-zero at either tick width), and the
4580    /// still-waiting arm answers `held - elapsed` under `elapsed < held`, so at
4581    /// least one. `NonZeroU64::new` therefore maps every reachable `Some` to
4582    /// itself; a zero would become `None`, which is the same verdict a caller
4583    /// draws from zero ticks left anyway.
4584    pub(crate) fn check_for_timeout(&mut self, task: TaskHandle) -> Option<NonZeroU64> {
4585        self.enter_critical();
4586        // Prove the index in range ONCE. Three bounded reads below key on the
4587        // same task -- the abort flag, its clear, and the TCB resolve -- and an
4588        // out-of-range task already answers `None` through every one of them, so
4589        // saying so up front is free of behaviour and lets their checks fold.
4590        // `rusty-compiler-leverage` B1, the lever that took `switch_context`
4591        // -969,669 and the tick 14 -> 9 today.
4592        //
4593        // AFTER `enter_critical`, and the exit below is not optional: on the sim
4594        // an outermost critical-section exit IS the clock, so a return that
4595        // skipped it would move every later event in the trace and `conform`
4596        // would fail on all twenty-six scenarios.
4597        let index = task.index() as usize;
4598        if index >= TASKS {
4599            self.exit_critical();
4600            return None;
4601        }
4602        // An aborted delay is not the same as a time out, but it has the
4603        // same result: stop waiting.
4604        if self.delay_aborted.get(index).copied().unwrap_or(false) {
4605            if let Some(f) = self.delay_aborted.get_mut(index) {
4606                *f = false;
4607            }
4608            self.exit_critical();
4609            return None;
4610        }
4611        let now = self.tick;
4612        let overflows = self.overflows;
4613        let result = match self.tcbs.resolve_mut(task) {
4614            Ok(tcb) => {
4615                // Masked: `begin_wait` truncates what it stores to the tick
4616                // width, so these are no-ops on the VALUE and information to
4617                // LLVM, which cannot otherwise know a field it loaded is
4618                // narrow. Without them every comparison below is a 64-bit one.
4619                debug_assert_eq!(
4620                    tcb.wait.entering.get(),
4621                    tcb.wait.entering.get() & Self::MAX_DELAY,
4622                    "a wait frame's `entering` is wider than the tick"
4623                );
4624                debug_assert_eq!(
4625                    tcb.wait.ticks.get(),
4626                    tcb.wait.ticks.get() & Self::MAX_DELAY,
4627                    "a wait frame's `ticks` is wider than the tick"
4628                );
4629                let entering = tcb.wait.entering.get() & Self::MAX_DELAY;
4630                let held = tcb.wait.ticks.get() & Self::MAX_DELAY;
4631                let elapsed = now.wrapping_sub(entering) & Self::MAX_DELAY;
4632                if held == Self::MAX_DELAY {
4633                    // An indefinite block never times out. `MAX_DELAY` is
4634                    // non-zero at either tick width, so this is always `Some`.
4635                    debug_assert_ne!(Self::MAX_DELAY, 0, "MAX_DELAY must be non-zero");
4636                    NonZeroU64::new(Self::MAX_DELAY)
4637                } else if overflows != tcb.wait.overflows && now >= entering {
4638                    tcb.wait.ticks = crate::timer::Split64::new(0);
4639                    None
4640                } else if elapsed < held {
4641                    let left = held.wrapping_sub(elapsed);
4642                    tcb.wait.ticks = crate::timer::Split64::new(left);
4643                    tcb.wait.entering = crate::timer::Split64::new(now);
4644                    tcb.wait.overflows = overflows;
4645                    // `elapsed < held` on this arm, so `left` is at least one
4646                    // and this is always `Some`. The assert is what keeps the
4647                    // niche honest if that ever stops being true.
4648                    debug_assert_ne!(left, 0, "left must be non-zero on the still-waiting arm");
4649                    NonZeroU64::new(left)
4650                } else {
4651                    tcb.wait.ticks = crate::timer::Split64::new(0);
4652                    None
4653                }
4654            }
4655            Err(_) => None,
4656        };
4657        self.exit_critical();
4658        result
4659    }
4660
4661    /// `vTaskPlaceOnUnorderedEventList`: the item value carries the
4662    /// condition rather than the priority, so the item goes on the end and
4663    /// the list is never sorted.
4664    ///
4665    /// `taskEVENT_LIST_ITEM_VALUE_IN_USE` is not set here the way the C
4666    /// sets it. The C needs it because the same `ListItem_t` is a
4667    /// priority-ordered event item the rest of the time and the flag says
4668    /// which; the value is only ever read back by the event-group code,
4669    /// which masks the control byte off, so setting it would change
4670    /// nothing but the arithmetic in the doc comment.
4671    pub(crate) fn place_on_unordered_event_list(
4672        &mut self,
4673        list: ListId,
4674        value: u64,
4675        ticks: u64,
4676    ) -> Result<()> {
4677        let item = Self::event_item(self.cur());
4678        self.lists.set_value(item, value)?;
4679        self.lists.insert_end(list, item)?;
4680        self.add_current_task_to_delayed_list(ticks, true)
4681    }
4682
4683    /// `vTaskRemoveFromUnorderedEventList`: write the answer into the item
4684    /// value, then wake the task.
4685    ///
4686    /// Unlike `xTaskRemoveFromEventList` this always goes straight to the
4687    /// ready list. It is only ever called with the scheduler suspended —
4688    /// `xEventGroupSetBits` holds it across the whole walk — and the C
4689    /// still bypasses the pending-ready list, because the value it just
4690    /// wrote is the task's return value and a second pass would overwrite
4691    /// it.
4692    pub(crate) fn remove_from_unordered_event_list(
4693        &mut self,
4694        item: ItemId,
4695        value: u64,
4696    ) -> Result<()> {
4697        self.lists.set_value(item, value)?;
4698        let task = self.task_of_event_item(item)?;
4699        let _ = self.lists.remove(item);
4700        let _ = self.lists.remove(Self::state_item(task));
4701        self.add_task_to_ready_list(task)?;
4702        let woken = self.tcbs.resolve(task)?.priority;
4703        if C::NUMBER_OF_CORES > 1 {
4704            let _ = self.smp_readied(task, woken);
4705        } else if woken > self.current_priority() {
4706            self.set_pending_here(true);
4707        }
4708        Ok(())
4709    }
4710
4711    /// `uxTaskResetEventItemValue`: read the answer the unblocker left, and
4712    /// put the item back to the priority order an ordinary event list wants.
4713    pub(crate) fn reset_event_item_value(&mut self, task: TaskHandle) -> Result<u64> {
4714        let item = Self::event_item(task);
4715        let value = self.lists.value(item)?;
4716        let priority = self.tcbs.resolve(task)?.priority;
4717        self.lists
4718            .set_value(item, u64::from(C::MAX_PRIORITIES.saturating_sub(priority)))?;
4719        Ok(value)
4720    }
4721
4722    /// Whether `task` is resuming an event-group wait rather than starting
4723    /// one. Taking it clears the marker.
4724    pub(crate) fn take_event_resume(&mut self, task: TaskHandle) -> bool {
4725        // Nobody is parked in a wait, so there is nothing to read — one field
4726        // load in place of an arena lookup.
4727        if self.event_resumes == 0 {
4728            return false;
4729        }
4730        match self.tcbs.resolve_mut(task) {
4731            Ok(tcb) if tcb.event_blocked => {
4732                tcb.event_blocked = false;
4733                self.event_resumes = self.event_resumes.wrapping_sub(1);
4734                true
4735            }
4736            _ => false,
4737        }
4738    }
4739
4740    /// Mark that `task` blocked inside an event-group wait, so the call it
4741    /// is inside resumes rather than restarts.
4742    pub(crate) fn set_event_resume(&mut self, task: TaskHandle) {
4743        if let Ok(tcb) = self.tcbs.resolve_mut(task) {
4744            // Count the slot, not the call.
4745            if !tcb.event_blocked {
4746                self.event_resumes = self.event_resumes.wrapping_add(1);
4747            }
4748            tcb.event_blocked = true;
4749        }
4750    }
4751
4752    /// `vTaskPlaceOnEventList`: sorted by priority, then blocked.
4753    // The attribute is load-bearing and the measurement is the reason. The
4754    // bound proof below grew this function past LLVM's own inlining threshold,
4755    // and outlining it turned a -668,718 row win in `queue_take_blocking` into
4756    // a **+154,439 LOSS on the program** -- A3: the frame and the argument
4757    // marshalling cost more than the proof saves. Asking for the inline puts it
4758    // back to -96,574.
4759    //
4760    // `#[inline]` and `#[inline(always)]` measured IDENTICALLY here, on both
4761    // axes, so the weaker word is the honest one: the 22 bytes of rv32 flash
4762    // this costs are the bound check duplicated at each site, not the
4763    // attribute's doing.
4764    #[inline]
4765    pub(crate) fn place_on_event_list(&mut self, list: ListId, ticks: u64) -> Result<()> {
4766        let current = self.cur();
4767        // B1, and the bound is the whole point: `event_item` is
4768        // `TASKS.saturating_add(index)`, which tells LLVM only that the result
4769        // fits a `u16` -- so `insert_keeping_value` still emits its own check
4770        // that the item names a real node, and the saturate costs a `mov` and a
4771        // `cmov` besides. Proving the index here instead makes the sum provably
4772        // below `2 * TASKS`, and both of those fold.
4773        //
4774        // It has to be proved HERE rather than relied on: `self.cur()` is a
4775        // live task on every path that reaches this function, but the calls in
4776        // between take `&mut self`, so LLVM cannot carry that across them.
4777        if current.index() as usize >= TASKS {
4778            self.note_stall(Stall::ListError);
4779            return Err(Error::Gone);
4780        }
4781        let item = Self::event_item(current);
4782        // The item keeps the value its call set; reading it out only to
4783        // hand it back made the list read the item twice and write it once
4784        // for no change.
4785        self.lists.insert_keeping_value(list, item)?;
4786        self.add_current_task_to_delayed_list(ticks, true)
4787    }
4788
4789    /// Take the trailing `portYIELD()` now, or owe it if a tick already
4790    /// switched us out of this call.
4791    pub(crate) fn yield_or_owe(&mut self, caller: TaskHandle) {
4792        if self.cur() == caller {
4793            self.port_yield();
4794        } else {
4795            self.owe_yield(caller);
4796        }
4797    }
4798
4799    /// `pvTaskIncrementMutexHeldCount`.
4800    pub(crate) fn increment_mutexes_held(&mut self, task: TaskHandle) {
4801        if let Ok(tcb) = self.tcbs.resolve_mut(task) {
4802            tcb.mutexes_held = tcb.mutexes_held.wrapping_add(1);
4803        }
4804    }
4805
4806    // ------------------------------------------------ priority inheritance --
4807
4808    /// `xTaskPriorityInherit`: lift the mutex holder to the waiter's
4809    /// priority. `true` when the holder is (or already was) lifted, which
4810    /// is what the waiter remembers so it can undo it on a timeout.
4811    #[cold]
4812    pub(crate) fn priority_inherit(&mut self, holder: TaskHandle) -> Result<bool> {
4813        if holder.is_null() || !self.tcbs.contains(holder) {
4814            return Ok(false);
4815        }
4816        let waiter_priority = self.current_priority();
4817        let (holder_priority, holder_base) = {
4818            let tcb = self.tcbs.resolve(holder)?;
4819            (tcb.priority, tcb.base_priority)
4820        };
4821        if holder_priority >= waiter_priority {
4822            // Already at least as urgent; the waiter still records that the
4823            // holder is running above its base, if it is.
4824            return Ok(holder_base < waiter_priority);
4825        }
4826        let event_value = u64::from(C::MAX_PRIORITIES.saturating_sub(waiter_priority));
4827        self.lists
4828            .set_value(Self::event_item(holder), event_value)?;
4829        let item = Self::state_item(holder);
4830        if self.lists.container(item)? == Some(Self::ready_list(holder_priority)) {
4831            let _ = self.lists.remove(item);
4832            self.set_task_priority(holder, waiter_priority)?;
4833            self.add_task_to_ready_list(holder)?;
4834            // SMP: "The priority of the task is raised. Yield for this task
4835            // if it is not running." (`xTaskPriorityInherit`)
4836            if C::NUMBER_OF_CORES > 1 && C::USE_PREEMPTION && self.running_core(holder).is_none() {
4837                self.yield_for_task(holder, waiter_priority);
4838            }
4839        } else {
4840            self.set_task_priority(holder, waiter_priority)?;
4841        }
4842        self.trace_task(holder, |task, name| Event::TaskPriorityInherit {
4843            task,
4844            name,
4845            priority: Self::priority_value(waiter_priority),
4846        });
4847        Ok(true)
4848    }
4849
4850    /// `xTaskPriorityDisinherit`: giving the mutex back drops the
4851    /// inherited priority. `true` when a yield is wanted.
4852    #[cold]
4853    pub(crate) fn priority_disinherit(&mut self, holder: TaskHandle) -> Result<bool> {
4854        if holder.is_null() {
4855            return Ok(false);
4856        }
4857        // One lookup where there were three: `contains` asked the arena
4858        // whether the holder is there, `resolve` asked again to read its
4859        // three fields, and `resolve_mut` asked a third time to put one back
4860        // -- with nothing but a subtraction in between. A holder the arena
4861        // does not have still answers `Ok(false)`, which is what `contains`
4862        // was for.
4863        let (priority, base, remaining) = match self.tcbs.resolve_mut(holder) {
4864            Ok(tcb) => {
4865                let remaining = tcb.mutexes_held.saturating_sub(1);
4866                tcb.mutexes_held = remaining;
4867                (tcb.priority, tcb.base_priority, remaining)
4868            }
4869            Err(_) => return Ok(false),
4870        };
4871        if priority == base || remaining != 0 {
4872            return Ok(false);
4873        }
4874        let item = Self::state_item(holder);
4875        if self.lists.remove(item).is_err() {
4876            return Ok(false);
4877        }
4878        self.trace_task(holder, |task, name| Event::TaskPriorityDisinherit {
4879            task,
4880            name,
4881            priority: Self::priority_value(base),
4882        });
4883        self.set_task_priority(holder, base)?;
4884        let event_value = u64::from(C::MAX_PRIORITIES.saturating_sub(base));
4885        self.lists
4886            .set_value(Self::event_item(holder), event_value)?;
4887        self.add_task_to_ready_list(holder)?;
4888        // SMP: "The priority of the task is dropped. Yield the core on which
4889        // the task is running." (`xTaskPriorityDisinherit`) Found by the
4890        // two-core corpus: without it a give that disinherits AND wakes a
4891        // waiter sends the waiter to the other core, where the C runs it on
4892        // the giver's.
4893        if C::NUMBER_OF_CORES > 1 {
4894            if let Some(core) = self.running_core(holder) {
4895                self.yield_core(core);
4896            }
4897        }
4898        Ok(true)
4899    }
4900
4901    /// `vTaskPriorityDisinheritAfterTimeout`: a waiter gave up, so the
4902    /// holder keeps only what the tasks still waiting justify.
4903    #[cold]
4904    pub(crate) fn priority_disinherit_after_timeout(
4905        &mut self,
4906        holder: TaskHandle,
4907        highest_waiting: u8,
4908    ) -> Result<()> {
4909        if holder.is_null() || !self.tcbs.contains(holder) {
4910            return Ok(());
4911        }
4912        let (priority, base, held) = {
4913            let tcb = self.tcbs.resolve(holder)?;
4914            (tcb.priority, tcb.base_priority, tcb.mutexes_held)
4915        };
4916        if priority == base || held != 1 {
4917            return Ok(());
4918        }
4919        let target = if highest_waiting > base {
4920            highest_waiting
4921        } else {
4922            base
4923        };
4924        if priority == target {
4925            return Ok(());
4926        }
4927        self.trace_task(holder, |task, name| Event::TaskPriorityDisinherit {
4928            task,
4929            name,
4930            priority: Self::priority_value(target),
4931        });
4932        self.set_task_priority(holder, target)?;
4933        let event_value = u64::from(C::MAX_PRIORITIES.saturating_sub(target));
4934        self.lists
4935            .set_value(Self::event_item(holder), event_value)?;
4936        let item = Self::state_item(holder);
4937        if self.lists.container(item)? == Some(Self::ready_list(priority)) {
4938            let _ = self.lists.remove(item);
4939            self.add_task_to_ready_list(holder)?;
4940            // SMP: as `priority_disinherit` (`vTaskPriorityDisinheritAfterTimeout`).
4941            if C::NUMBER_OF_CORES > 1 {
4942                if let Some(core) = self.running_core(holder) {
4943                    self.yield_core(core);
4944                }
4945            }
4946        }
4947        Ok(())
4948    }
4949
4950    // ------------------------------------------------------- delay until --
4951
4952    /// `xTaskDelayUntil`: wake at `*previous_wake + period`, whatever the
4953    /// task did in between. `false` when the deadline has already passed,
4954    /// in which case nothing blocks — exactly as the C returns `pdFALSE`.
4955    ///
4956    /// # Errors
4957    /// A list error surfaces as itself.
4958    pub fn delay_until(&mut self, previous_wake: &mut u64, period: u64) -> Result<bool> {
4959        let caller = self.cur();
4960        let mut should_delay = false;
4961        self.suspend_all();
4962        {
4963            let now = self.tick;
4964            let wake_at = previous_wake.wrapping_add(period) & Self::MAX_DELAY;
4965            if now < *previous_wake {
4966                // The tick count overflowed since the last wake.
4967                if wake_at < *previous_wake && wake_at > now {
4968                    should_delay = true;
4969                }
4970            } else if wake_at < *previous_wake || wake_at > now {
4971                should_delay = true;
4972            }
4973            *previous_wake = wake_at;
4974            if should_delay {
4975                self.trace_task(caller, |task, name| Event::TaskDelayUntil {
4976                    task,
4977                    name,
4978                    wake_at,
4979                });
4980                let ticks = wake_at.wrapping_sub(now) & Self::MAX_DELAY;
4981                self.add_current_task_to_delayed_list(ticks, false)?;
4982            }
4983        }
4984        if !self.resume_all() {
4985            self.yield_or_owe(caller);
4986        }
4987        Ok(should_delay)
4988    }
4989
4990    /// Which error `task_of_event_item` would have produced, re-derived out of
4991    /// line so its two discriminants do not sit in the caller's entry block.
4992    // `#[inline(never)]` is load-bearing, not decoration: with `#[cold]` alone
4993    // LLVM inlines this back and re-materialises the constants in the caller's
4994    // entry block -- measured EXACTLY +0 Ir for -2 B. The hint only reweights
4995    // the branch; MOVING the code is what sinks the constants.
4996    #[cold]
4997    #[inline(never)]
4998    fn event_item_error(&self, item: ItemId) -> Error {
4999        match item.checked_sub(Self::task_item_base()) {
5000            None => Error::InvalidArgument,
5001            Some(_) => Error::Gone,
5002        }
5003    }
5004
5005    /// `xTaskRemoveFromEventList`; `true` when the woken task outranks the
5006    /// running one and a yield is therefore required.
5007    /// `#[cold]` because the call is GUARDED and reached from many sites on
5008    /// one hot path, which is the shape that pays: it lets LLVM keep the
5009    /// caller's frame setup out of the likely route. Measured on
5010    /// `riscv32-qemu-tick-work`, one attribute at a time.
5011    /// Worth queue -7 on its own.
5012    #[cold]
5013    pub(crate) fn remove_from_event_list(&mut self, list: ListId) -> Result<bool> {
5014        let Some(item) = self.lists.head(list)? else {
5015            return Ok(false);
5016        };
5017        // The `?` put FIVE constants in the entry block -- `xor %eax,%eax`,
5018        // `mov $0x0,%edx`, `mov $0x1,%al`, `mov $0x8,%dl`, `mov $0x5,%dl` --
5019        // the Ok/Err tags and both error discriminants, materialised before the
5020        // tests that would use them. The census says ALL 21,487 BlockQ calls
5021        // pass every one of those tests, so that is five instructions of dead
5022        // setup on the only path anybody takes. Producing the error inside a
5023        // `#[cold]` callee sinks them out of the entry block.
5024        let Ok(task) = self.task_of_event_item(item) else {
5025            return Err(self.event_item_error(item));
5026        };
5027        let _ = self.lists.remove(item);
5028        if self.suspended_depth == 0 {
5029            let _ = self.lists.remove(Self::state_item(task));
5030            self.add_task_to_ready_list(task)?;
5031        } else {
5032            self.lists.insert_end(Self::pending_ready_list(), item)?;
5033        }
5034        let woken = self.tcbs.resolve(task)?.priority;
5035        if C::NUMBER_OF_CORES > 1 {
5036            // `prvYieldForTask`, and the answer is whether THIS core now owes
5037            // a yield -- the C's `xYieldPendings[ portGET_CORE_ID() ]`.
5038            if C::USE_PREEMPTION {
5039                self.yield_for_task(task, woken);
5040            }
5041            return Ok(self.pending_here());
5042        }
5043        if woken > self.current_priority() {
5044            self.set_pending_here(true);
5045            return Ok(true);
5046        }
5047        Ok(false)
5048    }
5049}
5050
5051// ================================================================ tests ==
5052
5053/// `kernel.rs` is corpus-oracled BY DESIGN (`docs/HOLES.md` H3), and these
5054/// do not change that. They exist for the mutants that survived BOTH
5055/// oracles — measured, 80 of 415 viable — because the corpus and the unit
5056/// suite each cannot reach them:
5057///
5058/// * the corpus runs `PosixDemoConfig` for 2,000 ticks, so the TICK
5059///   OVERFLOW arms of `delay_until` never execute. Nine survivors sat
5060///   there, and the only way to reach them is to choose a
5061///   `previous_wake` near the maximum rather than to simulate 2^32 ticks;
5062/// * the diagnostic accessors (`task_at`, `ready_items`, `ready_cursor`)
5063///   and the geometry guard in `with_tick_hook` are called by no scenario
5064///   at all — they are also on H2's list of APIs with no differential, so
5065///   two independent methods agree on the same surface.
5066#[cfg(test)]
5067#[allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
5068mod tests {
5069    use rusty_rtos_core::config::Config;
5070    use rusty_rtos_core::hooks::NoTickHook;
5071
5072    use crate::system::tests::{NoTrace, TestConfig, TestPort};
5073
5074    type K = crate::Kernel<
5075        TestConfig,
5076        TestPort,
5077        NoTrace,
5078        NoTickHook,
5079        4,
5080        { crate::list_slots_for(4, 0, crate::lists_for(TestConfig::MAX_PRIORITIES, 1, 0)) },
5081        { crate::lists_for(TestConfig::MAX_PRIORITIES, 1, 0) },
5082        1,
5083        1,
5084        0,
5085        0,
5086        0,
5087        0,
5088        { <TestConfig as ::rusty_rtos_core::config::Config>::TIMER_QUEUE_LENGTH },
5089    >;
5090
5091    fn kernel() -> K {
5092        K::new(TestPort::default(), NoTrace).expect("the declared geometry adds up")
5093    }
5094
5095    /// A kernel with one task of our own running, so the calls that act on
5096    /// "the current task" have one.
5097    fn running() -> K {
5098        let mut k = kernel();
5099        k.create_task("t", 1).expect("a task");
5100        let h = k.start_scheduler().expect("start");
5101        k.suspend(Some(h.timer)).expect("park the daemon");
5102        for _ in 0..50_u32 {
5103            if k.name_of(k.current()).expect("a name").as_str() == "t" {
5104                return k;
5105            }
5106            k.switch_context();
5107        }
5108        panic!("the task never reached the CPU");
5109    }
5110
5111    /// `vTaskDelayUntil` with no overflow: a deadline still ahead delays,
5112    /// and one already passed does NOT.
5113    ///
5114    /// ```c
5115    /// else if( ( xTimeIncrement > ( xConstTickCount - *pxPreviousWakeTime ) ) ... )
5116    /// ```
5117    ///
5118    /// The second half is the one that matters and the one a naive test
5119    /// misses: a task that overran its period must NOT sleep a whole extra
5120    /// one, it must return immediately and say it did not delay.
5121    #[test]
5122    fn delay_until_skips_a_deadline_that_has_already_passed() {
5123        let mut k = running();
5124
5125        // now is 0; the deadline is 20 ticks away, so it delays.
5126        let mut previous = 0_u64;
5127        assert_eq!(k.delay_until(&mut previous, 20), Ok(true));
5128        assert_eq!(previous, 20, "the wake time advances by the period");
5129
5130        // Run the clock well past the next deadline.
5131        for _ in 0..60_u32 {
5132            k.tick_from_isr();
5133        }
5134
5135        // now is 60 and the deadline was 40: already gone.
5136        let mut previous = 20_u64;
5137        assert_eq!(
5138            k.delay_until(&mut previous, 20),
5139            Ok(false),
5140            "the period was overrun, so it must not sleep another one"
5141        );
5142        assert_eq!(
5143            previous, 40,
5144            "and the wake time STILL advances, or the task never catches up"
5145        );
5146    }
5147
5148    /// The tick-overflow arm, which the corpus cannot reach in 2,000 ticks.
5149    ///
5150    /// ```c
5151    /// if( xConstTickCount < *pxPreviousWakeTime ) {
5152    ///     if( ( xTimeToWake < *pxPreviousWakeTime ) && ( xTimeToWake > xConstTickCount ) )
5153    /// ```
5154    ///
5155    /// Both halves are needed: the wake time must have wrapped TOO, and it
5156    /// must still be ahead of the wrapped-around now. Reaching it needs a
5157    /// `previous_wake` near the maximum, not 2^32 ticks of simulation.
5158    #[test]
5159    fn delay_until_handles_a_wake_time_that_wrapped_with_the_tick() {
5160        let mut k = running();
5161        let max = K::MAX_DELAY;
5162
5163        // A few ticks, so `now` is small and positive.
5164        for _ in 0..5_u32 {
5165            k.tick_from_isr();
5166        }
5167
5168        // previous_wake sits just below the maximum; the period carries the
5169        // wake time over the top, to 9. now is 5, so 9 is still ahead.
5170        let mut previous = max.wrapping_sub(10);
5171        assert_eq!(
5172            k.delay_until(&mut previous, 20),
5173            Ok(true),
5174            "wrapped, and still in the future: it must delay"
5175        );
5176        assert_eq!(previous, 9, "the wake time wrapped with the tick");
5177
5178        // Same wrap, but the wake time is already behind a larger now.
5179        let mut k = running();
5180        for _ in 0..40_u32 {
5181            k.tick_from_isr();
5182        }
5183        let mut previous = max.wrapping_sub(10);
5184        assert_eq!(
5185            k.delay_until(&mut previous, 20),
5186            Ok(false),
5187            "wrapped, and now is already past it: it must NOT delay"
5188        );
5189    }
5190
5191    /// `task_at` walks the arena by SLOT, which is how a debugger or a
5192    /// health check enumerates tasks without holding handles.
5193    #[test]
5194    fn task_at_enumerates_live_slots_and_stops_at_the_end() {
5195        let mut k = kernel();
5196        let a = k.create_task("a", 1).expect("a task");
5197
5198        assert_eq!(k.task_at(0), Some(a), "slot zero is the first task");
5199        assert_eq!(
5200            k.task_at(usize::from(u16::MAX)),
5201            None,
5202            "past the arena is None, not a handle"
5203        );
5204        assert_eq!(
5205            k.task_at(usize::from(u16::MAX) + 1),
5206            None,
5207            "and an index that does not even fit a u16 is None, not a panic"
5208        );
5209    }
5210
5211    /// `ready_items` fills the caller's buffer and answers how many it
5212    /// wrote, stopping at the buffer's end rather than running past it.
5213    #[test]
5214    fn ready_items_fills_what_it_can_and_counts_what_it_filled() {
5215        let mut k = kernel();
5216        k.create_task("a", 1).expect("a");
5217        k.create_task("b", 1).expect("b");
5218
5219        let mut out = [0_u16; 8];
5220        let n = k.ready_items(1, &mut out);
5221        assert_eq!(n, 2, "two tasks are ready at priority 1");
5222
5223        // A buffer too small must be filled to its end and no further.
5224        let mut small = [0_u16; 1];
5225        assert_eq!(
5226            k.ready_items(1, &mut small),
5227            1,
5228            "it wrote one, because one is all there was room for"
5229        );
5230
5231        // An empty priority writes nothing.
5232        let mut out = [0_u16; 8];
5233        assert_eq!(k.ready_items(0, &mut out), 0);
5234    }
5235
5236    /// `ready_cursor` is the diagnostic that answers "did the round robin
5237    /// move", so it must report the list it was asked about.
5238    #[test]
5239    fn ready_cursor_reports_the_list_it_was_asked_about() {
5240        let mut k = kernel();
5241        k.create_task("a", 1).expect("a");
5242        k.create_task("b", 1).expect("b");
5243
5244        let before = k.ready_cursor(1);
5245        k.switch_context();
5246        let after = k.ready_cursor(1);
5247        assert_ne!(after, before, "a switch moved the cursor at priority 1");
5248        assert_eq!(
5249            k.ready_cursor(0),
5250            k.ready_cursor(0),
5251            "and an untouched priority answers the same thing twice"
5252        );
5253    }
5254
5255    /// `with_tick_hook`'s geometry guard, which is the only thing standing
5256    /// between a mis-declared kernel and a silently wrong one.
5257    ///
5258    /// `ITEMS` and `LISTS` are derived constants; if a caller writes them
5259    /// by hand and gets them wrong, every list index is off. The guard
5260    /// refuses, and these are the arms of it.
5261    #[test]
5262    fn the_geometry_guard_refuses_a_kernel_that_does_not_add_up() {
5263        // ITEMS too small for the declared tasks and timers.
5264        type WrongItems = crate::Kernel<
5265            TestConfig,
5266            TestPort,
5267            NoTrace,
5268            NoTickHook,
5269            4,
5270            // Legal as a geometry — a power of two above `LISTS` — but not
5271            // the derived value, which is 32. A grosser error (2 slots for
5272            // 12 lists) is now a COMPILE error in `SIZES_FIT`, so it cannot
5273            // be used to exercise the runtime guard any more. That the
5274            // check moved earlier is the improvement; this keeps the later
5275            // one honest.
5276            16,
5277            { crate::lists_for(TestConfig::MAX_PRIORITIES, 1, 0) },
5278            1,
5279            1,
5280            0,
5281            0,
5282            0,
5283            0,
5284            { <TestConfig as ::rusty_rtos_core::config::Config>::TIMER_QUEUE_LENGTH },
5285        >;
5286        assert!(
5287            WrongItems::new(TestPort::default(), NoTrace).is_err(),
5288            "ITEMS must be list_slots_for(TASKS, TIMERS, LISTS)"
5289        );
5290
5291        // LISTS not derived from the priorities, queues and groups.
5292        type WrongLists = crate::Kernel<
5293            TestConfig,
5294            TestPort,
5295            NoTrace,
5296            NoTickHook,
5297            4,
5298            // Correct for the two lists declared below, so the refusal
5299            // below is attributable to `LISTS` and nothing else.
5300            { crate::list_slots_for(4, 0, 2) },
5301            2,
5302            1,
5303            1,
5304            0,
5305            0,
5306            0,
5307            0,
5308            { <TestConfig as ::rusty_rtos_core::config::Config>::TIMER_QUEUE_LENGTH },
5309        >;
5310        assert!(
5311            WrongLists::new(TestPort::default(), NoTrace).is_err(),
5312            "LISTS must be lists_for(MAX_PRIORITIES, QUEUES, GROUPS)"
5313        );
5314
5315        // A kernel with no room for a single task.
5316        type NoTasks = crate::Kernel<
5317            TestConfig,
5318            TestPort,
5319            NoTrace,
5320            NoTickHook,
5321            0,
5322            { crate::list_slots_for(0, 0, crate::lists_for(TestConfig::MAX_PRIORITIES, 1, 0)) },
5323            { crate::lists_for(TestConfig::MAX_PRIORITIES, 1, 0) },
5324            1,
5325            1,
5326            0,
5327            0,
5328            0,
5329            0,
5330            { <TestConfig as ::rusty_rtos_core::config::Config>::TIMER_QUEUE_LENGTH },
5331        >;
5332        assert!(
5333            NoTasks::new(TestPort::default(), NoTrace).is_err(),
5334            "TASKS == 0 cannot even hold the idle task"
5335        );
5336
5337        // And the geometry that DOES add up is accepted, or the three
5338        // assertions above would pass for the wrong reason.
5339        assert!(K::new(TestPort::default(), NoTrace).is_ok());
5340    }
5341    /// `prvGetExpectedIdleTime`, arm one: something above the idle
5342    /// priority is running, so there is no window to sleep through.
5343    ///
5344    /// The three arms are a short-circuit chain and each is separately
5345    /// constructible, which is how they get pinned one at a time. A `>`
5346    /// turned `>=` in any of them would let the kernel sleep through work
5347    /// that is ready NOW.
5348    #[test]
5349    fn expected_idle_time_is_zero_while_anything_outranks_the_idle_task() {
5350        let mut k = kernel();
5351        k.create_task("hi", 1).expect("a task above idle");
5352        k.start_scheduler().expect("start");
5353
5354        assert!(
5355            k.current_priority() > 0,
5356            "the task above idle is the one running"
5357        );
5358        assert_eq!(
5359            k.expected_idle_time(),
5360            0,
5361            "something outranks idle, so the window is zero"
5362        );
5363    }
5364
5365    /// Arm two: another task shares the idle priority, so the very next
5366    /// tick has to be processed to give it its slice.
5367    #[test]
5368    fn expected_idle_time_is_zero_when_another_idle_task_wants_the_slice() {
5369        let mut k = kernel();
5370        k.create_task("a", 0).expect("a task AT the idle priority");
5371        k.start_scheduler().expect("start");
5372
5373        assert!(
5374            k.ready_len(0).unwrap_or(0) > 1,
5375            "the idle task and ours are both ready at priority 0"
5376        );
5377        assert_eq!(
5378            k.expected_idle_time(),
5379            0,
5380            "a shared slice cannot be slept through"
5381        );
5382    }
5383
5384    /// And with none of the three arms true, the window is the time to the
5385    /// next unblock -- which is the number the whole tickless feature
5386    /// rests on.
5387    #[test]
5388    fn expected_idle_time_is_the_window_to_the_next_unblock() {
5389        let mut k = running();
5390        k.delay(30).expect("park our task for thirty ticks");
5391
5392        // Only the idle task is ready now, and nothing outranks it.
5393        assert_eq!(k.current_priority(), 0, "the idle task is running");
5394        assert_eq!(k.ready_len(0).unwrap_or(0), 1, "and it is alone at zero");
5395        assert_eq!(
5396            k.expected_idle_time(),
5397            30,
5398            "so the port may sleep right up to the next unblock"
5399        );
5400    }
5401
5402    /// `ulTaskNotifyValueClear`'s reader half, and `xTaskNotifyAndQuery`'s:
5403    /// it answers the slot it was ASKED for, for the task it was asked
5404    /// about.
5405    ///
5406    /// Pinned with a value that is neither 0 nor 1, because those are what
5407    /// the surviving mutants replaced it with.
5408    #[test]
5409    fn notify_value_answers_the_slot_it_was_asked_for() {
5410        use crate::kernel::NotifyAction;
5411
5412        let mut k = running();
5413        let me = k.current();
5414        k.notify(me, 0, 0xabcd_1234, NotifyAction::Overwrite)
5415            .expect("notify");
5416
5417        assert_eq!(
5418            k.notify_value(Some(me), 0),
5419            Ok(0xabcd_1234),
5420            "the value that was set, not zero and not one"
5421        );
5422        assert_eq!(
5423            k.notify_value(None, 0),
5424            Ok(0xabcd_1234),
5425            "None means the CURRENT task"
5426        );
5427        assert!(
5428            k.notify_value(Some(me), TestConfig::NOTIFICATION_ARRAY_ENTRIES)
5429                .is_err(),
5430            "an index past the configured array is an error, not a zero"
5431        );
5432    }
5433    /// `delay_until`'s BOUNDARIES, which is where the four mutants that
5434    /// survived both oracles live.
5435    ///
5436    /// Each comparison in it has an edge the ordinary tests never touch,
5437    /// and each edge is reachable once you notice that `previous_wake` is
5438    /// a CALLER variable and the period may be zero:
5439    ///
5440    /// | line | comparison | the edge |
5441    /// |---|---|---|
5442    /// | 3228 | `wake_at < *previous_wake` | equal, via a period of 0 |
5443    /// | 3228 | `wake_at > now` | equal, by ticking to the wake time |
5444    /// | 3231 | `wake_at < *previous_wake` | equal, via a period of 0 |
5445    ///
5446    /// All three answers must be "do not delay". A `<=` or a `>=` in any of
5447    /// them turns a zero-length wait into a full period of sleep.
5448    #[test]
5449    fn delay_until_does_not_delay_on_any_of_its_boundaries() {
5450        let max = K::MAX_DELAY;
5451
5452        // (1) OVERFLOW arm, period zero: the wake time equals the previous
5453        // one, so there is nothing to wait for.
5454        let mut k = running();
5455        for _ in 0..5_u32 {
5456            k.tick_from_isr();
5457        }
5458        let mut previous = max.wrapping_sub(10);
5459        assert_eq!(
5460            k.delay_until(&mut previous, 0),
5461            Ok(false),
5462            "a period of zero is not a period of MAX_DELAY"
5463        );
5464
5465        // (2) OVERFLOW arm, wake time exactly equal to now: already due.
5466        // (max - 10) + 20 wraps to 9, so tick to 9.
5467        let mut k = running();
5468        for _ in 0..9_u32 {
5469            k.tick_from_isr();
5470        }
5471        let mut previous = max.wrapping_sub(10);
5472        assert_eq!(
5473            previous.wrapping_add(20) & max,
5474            9,
5475            "the wake time wraps to 9"
5476        );
5477        assert_eq!(
5478            k.delay_until(&mut previous, 20),
5479            Ok(false),
5480            "the wrapped wake time is NOW, so it is not in the future"
5481        );
5482
5483        // (3) NON-overflow arm, period zero with now already at the wake
5484        // time: neither disjunct holds.
5485        let mut k = running();
5486        for _ in 0..10_u32 {
5487            k.tick_from_isr();
5488        }
5489        let mut previous = 10_u64;
5490        assert_eq!(
5491            k.delay_until(&mut previous, 0),
5492            Ok(false),
5493            "nothing to wait for, and no period to sleep"
5494        );
5495        assert_eq!(previous, 10, "and the wake time does not move");
5496    }
5497    /// The tick-wrap bookkeeping, which `xTaskIncrementTick` runs once
5498    /// every `MAX_DELAY + 1` ticks.
5499    ///
5500    /// Neither oracle can reach it: the corpus runs 2,000 ticks and the
5501    /// unit suite does not wrap either, so every mutation of it survived
5502    /// both. It is a private `fn` and this module is inside `kernel.rs`,
5503    /// so the test calls it directly rather than trying to simulate 2^32
5504    /// ticks — which is the right answer when the trigger is unreachable
5505    /// but the behaviour is not.
5506    #[test]
5507    fn switching_the_delayed_lists_flips_the_pair_and_counts_the_wrap() {
5508        let mut k = running();
5509        let swapped = k.delayed_swapped;
5510        let overflows = k.overflows;
5511
5512        k.switch_delayed_lists();
5513        assert_ne!(
5514            k.delayed_swapped, swapped,
5515            "the delayed and overflow lists change places"
5516        );
5517        assert_eq!(
5518            k.overflows,
5519            overflows.wrapping_add(1),
5520            "and the wrap is counted, because `overflows` is how a trace \
5521             tells one epoch from the next"
5522        );
5523
5524        k.switch_delayed_lists();
5525        assert_eq!(k.delayed_swapped, swapped, "flipping twice is the identity");
5526        assert_eq!(k.overflows, overflows.wrapping_add(2));
5527    }
5528
5529    /// `prvResetNextTaskUnblockTime`, which the wrap calls: after the swap
5530    /// the next unblock time has to be recomputed from the list that is
5531    /// NOW the delayed one.
5532    ///
5533    /// That list is empty immediately after a wrap, so the answer must be
5534    /// `MAX_DELAY` — a stale value from the old list would make the
5535    /// kernel think a task is due and spin, or sleep through one that is.
5536    #[test]
5537    fn switching_the_delayed_lists_recomputes_the_next_unblock_time() {
5538        let mut k = running();
5539        let due = k.tick_count().wrapping_add(30);
5540        k.delay(30).expect("park our task on the delayed list");
5541        assert_eq!(k.next_unblock_time, due, "the parked task is what is next");
5542
5543        k.switch_delayed_lists();
5544        assert_eq!(
5545            k.next_unblock_time,
5546            K::MAX_DELAY,
5547            "the list that is now 'delayed' is empty, so nothing is due"
5548        );
5549    }
5550
5551    /// The sentinel for "nothing is due" is the SAME value in both places this
5552    /// kernel writes it, and this pins that.
5553    ///
5554    /// ★ It used to differ, and this test used to pin the difference. A reset
5555    /// over an empty delayed list stored `u64::MAX` because `head_value` cannot
5556    /// answer "empty" -- on an empty list the head IS the end marker, whose
5557    /// value is `u64::MAX` so sorted inserts always find something larger to
5558    /// stop at -- so the `unwrap_or(MAX_DELAY)` never fired. The C stores
5559    /// `portMAX_DELAY` there.
5560    ///
5561    /// The divergence was unobservable (it shows only at a tick equal to
5562    /// `MAX_DELAY`, and the wrap resets anyway), which is why a test pinned it
5563    /// instead of a gate catching it -- and the test one screen up had said in
5564    /// prose that `MAX_DELAY` "must" be the answer while asserting `u64::MAX`.
5565    /// The prose was right.
5566    ///
5567    /// Worth five instructions per tick: with the field provably no wider than
5568    /// the tick, `increment_tick`'s comparison against it stops being a 64-bit
5569    /// one on a 32-bit target. See `reset_next_task_unblock_time`.
5570    ///
5571    /// `new()` starts it at `MAX_DELAY` -- the tick mask, and what the C's
5572    /// `portMAX_DELAY` is. `reset_next_task_unblock_time` instead takes
5573    /// `head_value` of an empty list, which answers `ListsOf::MAX_VALUE`
5574    /// = `u64::MAX`, so the `unwrap_or(MAX_DELAY)` beside it never fires
5575    /// for emptiness.
5576    ///
5577    /// Neither is reachable as a tick, because the tick is masked to
5578    /// `MAX_DELAY`, so no scheduling decision can tell them apart and the
5579    /// corpus is green either way. It IS visible to the tickless path:
5580    /// `expected_idle_time` answers `next_unblock_time - tick`, which is
5581    /// the window the port is handed, and the two sentinels differ there
5582    /// by four billion.
5583    ///
5584    /// Recorded rather than changed: making them agree is an owner
5585    /// decision, because `expected_idle_time` feeds a port and any change
5586    /// to it moves what a tickless build asks for.
5587    #[test]
5588    fn the_nothing_is_due_sentinel_differs_between_init_and_reset() {
5589        let k = kernel();
5590        assert_eq!(
5591            k.next_unblock_time,
5592            K::MAX_DELAY,
5593            "a fresh kernel uses the tick mask, which is the C's portMAX_DELAY"
5594        );
5595
5596        let mut k = running();
5597        k.reset_next_task_unblock_time();
5598        assert_eq!(
5599            k.next_unblock_time,
5600            K::MAX_DELAY,
5601            "and a reset over an empty list uses the SAME sentinel, as the C does"
5602        );
5603        assert_ne!(
5604            K::MAX_DELAY,
5605            u64::MAX,
5606            "MAX_DELAY must be narrower than u64::MAX at this width, or the              invariant this test defends is vacuous"
5607        );
5608        assert_eq!(
5609            k.next_unblock_time & K::MAX_DELAY,
5610            k.next_unblock_time,
5611            "the field must never hold a value wider than the tick: that is what              lets increment_tick's comparison stay 32-bit"
5612        );
5613    }
5614
5615    /// The owe hint must not outlive the debt it stands for.
5616    ///
5617    /// A census over three scenarios found every raise producing exactly TWO
5618    /// entries to `resume_pending_owed` — one that did the work and returned
5619    /// `true`, and a second that re-derived "nothing owed" and cleared the
5620    /// hint on its way out. BlockQ: 43,110 raises, 86,207 entries. Clearing
5621    /// the hint at the end of the work made entries equal raises.
5622    ///
5623    /// Both directions are asserted. A `clear_owe_if_settled` that cleared
5624    /// unconditionally would pass the first half and lose a real owed item,
5625    /// which is a missing trace line in the differential — so the second half
5626    /// is the half that matters.
5627    #[test]
5628    fn the_owe_hint_does_not_outlive_the_debt() {
5629        let mut k = running();
5630        let me = k.current();
5631        let i = me.index() as usize;
5632        fn hint(k: &K, i: usize) -> bool {
5633            k.owes_anything.get(i).copied().unwrap_or(false)
5634        }
5635
5636        k.owe(me);
5637        assert!(hint(&k, i), "the hint was raised");
5638        k.clear_owe_if_settled(i);
5639        assert!(
5640            !hint(&k, i),
5641            "nothing is owed, so the hint is dead and must go -- leaving it standing costs one whole entry to the owed body per owed item"
5642        );
5643
5644        // The negative case, once per kind of debt. A `clear_owe_if_settled`
5645        // that cleared unconditionally would pass the half above and lose a
5646        // real owed item, which is a missing line in the differential.
5647        for kind in 0..3 {
5648            match kind {
5649                0 => *k.owed_exits.get_mut(i).expect("in range") = 1,
5650                1 => k.set_flag(i, K::F_YIELD, true),
5651                _ => {
5652                    *k.owed_trace.get_mut(i).expect("in range") =
5653                        crate::kernel::OwedTrace::AddNewTaskToReadyList {
5654                            task: me,
5655                            priority: 1,
5656                        }
5657                }
5658            }
5659            k.owe(me);
5660            k.clear_owe_if_settled(i);
5661            assert!(hint(&k, i), "kind {kind} is live debt; the hint must stay");
5662            *k.owed_exits.get_mut(i).expect("in range") = 0;
5663            k.set_flag(i, K::F_YIELD, false);
5664            *k.owed_trace.get_mut(i).expect("in range") = crate::kernel::OwedTrace::None;
5665        }
5666    }
5667
5668    /// The stream-buffer resume flag is a ONE-SHOT: the take clears it, so
5669    /// a second take answers None. A `take` that did not clear would let a
5670    /// blocked reader wake twice on one send.
5671    #[test]
5672    fn a_stream_resume_is_taken_once_and_then_gone() {
5673        let mut k = running();
5674        let me = k.current();
5675        assert_eq!(k.take_stream_resume(me), None, "nothing set yet");
5676
5677        k.set_stream_resume(me, 7);
5678        assert_eq!(k.take_stream_resume(me), Some(7), "the value that was set");
5679        assert_eq!(k.take_stream_resume(me), None, "and it is consumed");
5680    }
5681
5682    /// `vTaskMissedYield`: a yield asked for while the scheduler is
5683    /// suspended has to be remembered, or it is simply lost.
5684    #[test]
5685    fn a_missed_yield_is_remembered_rather_than_dropped() {
5686        let mut k = running();
5687        k.yield_pending = [false; super::MAX_CORES];
5688        k.missed_yield();
5689        assert!(
5690            k.yield_pending[0],
5691            "the yield survives until something can act on it"
5692        );
5693    }
5694}