Skip to main content

rusty_rtos_kernel_core/
kernel.rs

1//! The scheduler: `tasks.c` and `queue.c` as a state machine over indices.
2//!
3//! Every public function here is a FreeRTOS API function, and its body
4//! follows the C body statement for statement — including **where the
5//! critical sections open and close**, because on the sim that is where
6//! time passes (`ORACLES.md`, sim contract v1, rule 3), and including where
7//! the `trace*` macros fire, because the order of those lines is the gate.
8//!
9//! The subtlety that costs a diff if you miss it: FreeRTOS yields from
10//! *inside* the critical section in `vTaskResume`, `vTaskPrioritySet`,
11//! `xTaskResumeAll` and the queue calls, and from *outside* it in
12//! `vTaskSuspend` and `vTaskDelay`. Since `portYIELD()` on the Posix port
13//! is itself a critical section, an inside yield nests (and counts one
14//! outermost exit) where an outside yield counts two. Each site below says
15//! which it is.
16//!
17//! Where the C has a `configASSERT`, this has an [`Error`]: the C kernel
18//! stops the world on a bad handle, and the whole point of the remake is
19//! that we do not.
20
21use core::marker::PhantomData;
22
23use core::num::NonZeroU64;
24use rusty_rtos_core::arena::Arena;
25use rusty_rtos_core::config::Config;
26
27use rusty_rtos_core::error::{Error, Result};
28use rusty_rtos_core::handle::{
29    EventGroup as EventGroupKind, EventGroupHandle, Queue as QueueKind, QueueHandle,
30    StreamBuffer as StreamKind, StreamBufferHandle, Task as TaskKind, TaskHandle,
31    Timer as TimerKind, TimerHandle,
32};
33use rusty_rtos_core::hooks::TickHook;
34use rusty_rtos_core::isr::Woken;
35use rusty_rtos_core::list::{ItemId, ListId, Lists};
36use rusty_rtos_core::port::Port;
37use rusty_rtos_core::priority::Priority;
38use rusty_rtos_core::tick::TickWidth;
39use rusty_rtos_core::trace::{Event, Trace};
40
41use crate::events::EventGroup;
42use crate::name::Name;
43use crate::queue::Queue;
44use crate::queue::Wait;
45use crate::stream::StreamBuffer;
46use crate::timer::{Message, Timer};
47use crate::{OVERHEAD_LISTS, list_slots_for, lists_for};
48
49/// How many times [`Kernel::trace_failure_or_owe`] has been entered.
50///
51/// A reachability anchor, not a measurement. Every row of
52/// `riscv32-qemu-tick-work` read **zero** here until the failure rows were
53/// added, which is why a change to the `OwedTrace` machinery measured as pure
54/// code layout. It is behind its own feature because the `fetch_add` sits
55/// INSIDE the measured bracket, so a build that quotes instruction counts must
56/// not carry it.
57#[cfg(feature = "census")]
58pub static CENSUS_OWE: core::sync::atomic::AtomicU32 = core::sync::atomic::AtomicU32::new(0);
59
60/// A trace line owed by a task that was switched out before it could
61/// emit one. Only the queue failure paths can owe one: they are the only
62/// places FreeRTOS traces *after* leaving a critical section.
63/// Why `vTaskSwitchContext` could not choose a task to run.
64///
65/// # Law 3: no silent failures
66///
67/// Every variant here was, until 2026-09-11, a bare `return` — the scheduler
68/// declining to switch and saying nothing. Three of them are impossible in a
69/// healthy kernel and the fourth is what the C asserts on
70/// (`configASSERT( uxTopPriority )`), so reaching any of them means an
71/// invariant has already broken somewhere else.
72///
73/// They cost a multi-flash hardware hunt on the day the Xtensa port first
74/// ran a stacked task: every ready list had emptied, the kernel kept
75/// quietly running whoever was current, and nothing said so. A counter and
76/// a reason turn that into one printed line.
77#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
78pub enum Stall {
79    /// The scheduler chose normally.
80    #[default]
81    None,
82    /// No ready task at ANY priority. The idle task keeps priority 0
83    /// non-empty, so reaching this means the idle task is gone — which is
84    /// exactly what `configASSERT( uxTopPriority )` fires on.
85    NoReadyTask,
86    /// A ready list refused to answer at all.
87    ListError,
88    /// The chosen priority's list had nothing to rotate to, having just
89    /// reported itself non-empty.
90    EmptyRotation,
91    /// The chosen list item named no live task.
92    UnknownTask,
93}
94
95#[derive(Debug, Clone, Copy, PartialEq, Eq)]
96pub(crate) enum OwedTrace {
97    /// Nothing owed.
98    None,
99    /// `traceQUEUE_SEND_FAILED`.
100    SendFailed(QueueHandle),
101    /// `traceQUEUE_RECEIVE_FAILED`.
102    ReceiveFailed(QueueHandle),
103    /// `traceTIMER_COMMAND_SEND`, which the C runs on the line *after*
104    /// `xQueueSendToBack` — so a send that made the daemon ready traces
105    /// only once this task has the CPU back.
106    TimerCommandSend {
107        timer: TimerHandle,
108        name: Name,
109        command: i32,
110        value: u64,
111    },
112    /// `traceSTREAM_BUFFER_CREATE`, which the C runs after the
113    /// `pvPortMalloc` that made the buffer.
114    ///
115    /// `heap_4` brackets that malloc with `vTaskSuspendAll` /
116    /// `xTaskResumeAll`, so a tick can land on its exit -- and the C's
117    /// thread then stops inside the allocator, with the trace still ahead
118    /// of it. Emitting it anyway puts the line in another task's run of the
119    /// trace. `StreamBufferDemo`'s echo client showed it at tick 429, one
120    /// line early and a whole task's work out of place.
121    StreamBufferCreate {
122        buffer: StreamBufferHandle,
123        is_message_buffer: bool,
124    },
125    /// `traceEVENT_GROUP_WAIT_BITS_END`, which the C runs after the
126    /// `xTaskResumeAll` that ends the wait — so a resume that switched away
127    /// traces once this task has the CPU back.
128    EventGroupWaitBitsEnd {
129        group: EventGroupHandle,
130        bits: u32,
131        timed_out: bool,
132    },
133    /// `prvAddNewTaskToReadyList`, the second half of `xTaskCreate`.
134    ///
135    /// A create costs three outermost exits before it reaches this — two
136    /// `pvPortMalloc`s and the port's `pthread_create` section — and a tick
137    /// can land on any of them. The C is a real thread, so it simply stops
138    /// there and the new task is not on a ready list until the creator runs
139    /// again. This is that pause, and it is the only owed item that is not
140    /// purely a trace line: it carries the ready-list insertion too.
141    AddNewTaskToReadyList { task: TaskHandle, priority: u8 },
142}
143
144/// `eTaskState`: what a task is doing, derived from the list it is in
145/// exactly as `eTaskGetState` derives it.
146#[derive(Debug, Clone, Copy, PartialEq, Eq)]
147pub enum TaskState {
148    /// The task `current` names.
149    Running,
150    /// On a ready list.
151    Ready,
152    /// On a delayed list, or on an event list with a timeout.
153    Blocked,
154    /// On the suspended list with no event pending.
155    Suspended,
156    /// The handle names no live task.
157    Deleted,
158}
159
160/// Where a preempted queue call has to start again.
161///
162/// A queue call has SEVERAL critical-section exits that can release a tick
163/// and switch the caller away, and the C's thread resumes below whichever
164/// one it stopped at. One marker is not enough to say which.
165#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
166pub(crate) enum QueueResume {
167    /// Nothing to resume; enter at the top.
168    #[default]
169    No,
170    /// Below the section that samples the queue, in the caller.
171    BelowSample,
172    /// Below the `xTaskResumeAll` of the TIMED-OUT branch, which is the one
173    /// `IntQueue` reaches at tick 16,260: the tick pended inside the
174    /// scheduler suspension is replayed there, and replaying it switches
175    /// the caller away with `prvIsQueueEmpty` still to run.
176    BelowTimedOutResume,
177}
178
179/// One task control block: the C `TCB_t` minus everything that is a
180/// pointer. No stack, no TLS, no `pxTopOfStack`.
181#[derive(Debug, Clone, Copy, Default)]
182pub(crate) struct Tcb {
183    name: Name,
184    /// `uxPriority`, the possibly-inherited one the scheduler sorts by.
185    priority: u8,
186    /// `uxBasePriority`, what the task asked for.
187    base_priority: u8,
188    /// `uxMutexesHeld`: how many mutexes this task holds, which is what
189    /// decides whether giving one back may drop an inherited priority.
190    mutexes_held: u32,
191    /// The locals a blocking queue call keeps on the C stack across a
192    /// block. This kernel has no stack, so they live here — see
193    /// [`crate::queue`].
194    wait: WaitFrame,
195    /// `ulNotifiedValue[]`.
196    notified: [u32; MAX_NOTIFICATION_ENTRIES],
197    /// `ucNotifyState[]`.
198    notify_state: [NotifyState; MAX_NOTIFICATION_ENTRIES],
199    /// Whether a stream-buffer call this task made was preempted at the
200    /// critical-section exit that samples the buffer, and must resume
201    /// *after* that exit rather than repeat it.
202    stream_resume: bool,
203    /// Where a preempted queue call resumes. See [`QueueResume`].
204    ///
205    /// `xQueueGenericSend` and `xQueueReceive` exit their sampling section
206    /// before suspending the scheduler to block, and THAT exit can release
207    /// a tick which switches the caller away. The C's thread stops inside
208    /// the exit; everything below it -- `traceBLOCKING_ON_QUEUE_SEND` and
209    /// `vTaskPlaceOnEventList` -- runs when the task is next scheduled, and
210    /// so names the right task. Without this marker the trace attributes
211    /// the block to whoever the tick switched TO, which `IntQueue` catches
212    /// at tick 5 and no other scenario reaches.
213    queue_resume: QueueResume,
214    /// Whether a blocking `xStreamBufferSend` has already run its
215    /// `vTaskSetTimeOutState`. See [`Kernel::take_stream_timed`].
216    stream_timed: bool,
217    /// Whether a stream-buffer call's notify wait has already finished.
218    /// See [`Kernel::take_stream_waited`].
219    stream_waited: bool,
220    /// The stack local that exit produced — `xBytesAvailable` on a receive,
221    /// `xSpace` on a send — kept because the frame it belonged to is gone.
222    stream_local: usize,
223    /// Whether this task has already blocked inside the notification wait
224    /// it is in. The C blocks inside `xTaskGenericNotifyWait`; a kernel
225    /// with no stacks returns `Blocked` and is called again, so it needs to
226    /// know the second call is a resumption rather than a fresh wait.
227    notify_blocked: bool,
228    /// The same, for the event-group wait this task is inside. Those two
229    /// functions have no retry loop — the C blocks once and then reads the
230    /// event item value — so the second call has to know it is the far side
231    /// of the switch and not a fresh wait.
232    event_blocked: bool,
233    /// What a timer command that blocked on a full command queue is sending:
234    /// for `xTimerStart` and `xTimerReset`, the tick count the C read ONCE,
235    /// before the send, and carries through the block in its message. See
236    /// [`Kernel::timer_command`]. Two words taken out of `_stride_pad`, so it
237    /// costs no RAM.
238    held: crate::timer::Split64,
239    /// Padding that rounds `Slot<Tcb>` up to a POWER OF TWO (128 bytes), so
240    /// `index * size_of::<Slot<Tcb>>()` is a shift and not a multiply.
241    ///
242    /// `bench/kernel-ram` pins the stride at 128 and `bench/kernel-flash` pins
243    /// `mul` at 0; without this the stride is 92 and every TCB resolve pays a
244    /// `mul`. It is a deliberate flash-for-RAM trade: 254 bytes of flash
245    /// against 36 bytes of RAM per task, so it pays below roughly eight tasks
246    /// and is the owner's call above that.
247    /// Its place in `xTasksWaitingTermination`: the deletion number it was
248    /// given when it deleted itself while running, 0 when it is not waiting.
249    /// One of the stride's padding words, so the slot is still 128 bytes.
250    reap_seq: u32,
251    _stride_pad: [u32; STRIDE_PAD_WORDS],
252}
253
254/// How many `u32`s of [`Tcb::_stride_pad`] it takes to make `Slot<Tcb>` a power
255/// of two, which is what keeps `index * stride` a shift instead of a multiply.
256///
257/// It differs by pointer width because the fields it pads out do: `usize` and
258/// the `Name` alignment (see [`crate::name::Name`], aligned to eight on a
259/// 64-bit host only) both change size. Nine words gives 128 bytes on a 32-bit
260/// target -- the number `bench/kernel-ram` pins and `bench/kernel-flash`'s
261/// `mul = 0` depends on -- and four gives 128 on the host. Two of each went to
262/// [`Tcb::held`] (2026-10-02) and one of each to [`Tcb::reap_seq`]
263/// (2026-10-04), so it is six and one now.
264///
265/// Before this was split, the host slot was 144 bytes and every TCB index cost
266/// a `lea`+`shl` where a power-of-two stride costs one `shl`.
267#[cfg(target_pointer_width = "64")]
268pub(crate) const STRIDE_PAD_WORDS: usize = 1;
269
270/// See the 64-bit case above; seven words is what makes an rv32 `Slot<Tcb>`
271/// 128 bytes, and that number is pinned by two benches.
272#[cfg(not(target_pointer_width = "64"))]
273pub(crate) const STRIDE_PAD_WORDS: usize = 6;
274
275/// How many notification slots a task has room for.
276///
277/// The C sizes its arrays from `configTASK_NOTIFICATION_ARRAY_ENTRIES`
278/// directly. A `Config` is a type here and stable Rust cannot size an array
279/// from one of its associated consts, so the arrays are the largest a
280/// configuration may ask for and [`Config::validate`] refuses more. The
281/// cost is a few bytes per task per unused slot; the alternative was a
282/// tenth const generic on every kernel type.
283pub const MAX_NOTIFICATION_ENTRIES: usize = 4;
284
285/// `ucNotifyState[]`: what a task's notification slot is doing.
286#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
287pub enum NotifyState {
288    /// `taskNOT_WAITING_NOTIFICATION`.
289    #[default]
290    NotWaiting,
291    /// `taskWAITING_NOTIFICATION`.
292    Waiting,
293    /// `taskNOTIFICATION_RECEIVED`.
294    Received,
295}
296
297/// `eNotifyAction`: what a notification does to the value already there.
298#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
299pub enum NotifyAction {
300    /// `eNoAction`: wake the task, leave the value alone.
301    #[default]
302    None,
303    /// `eSetBits`.
304    SetBits,
305    /// `eIncrement`.
306    Increment,
307    /// `eSetValueWithOverwrite`.
308    Overwrite,
309    /// `eSetValueWithoutOverwrite`: fails if one is already pending.
310    NoOverwrite,
311}
312
313/// What `xQueueGenericSend` and friends keep between passes of their
314/// `for(;;)`: the queue being waited on, the ticks left, and the timeout
315/// bookkeeping (`TimeOut_t`).
316#[derive(Debug, Clone, Copy, Default)]
317struct WaitFrame {
318    /// The queue this task is part-way through a blocking call on;
319    /// `TaskHandle::NULL`-equivalent when there is none.
320    queue: QueueHandle,
321    /// `xTicksToWait`, decremented by each `xTaskCheckForTimeOut`.
322    ///
323    /// All three are SPLIT (`timer::Split64`), so a `WaitFrame` does not align
324    /// a `Tcb` to eight. An arena slot pays its value's alignment as padding,
325    /// and `Tcb` is one per task.
326    ticks: crate::timer::Split64,
327    /// `xTimeOut.xTimeOnEntering`.
328    entering: crate::timer::Split64,
329    /// `xTimeOut.xOverflowCount`.
330    /// Counted in a `u32`: it advances once per delayed-list swap, so four
331    /// billion of them is not a bound anything reaches, and a `u64` here was
332    /// two instructions an operation on rv32.
333    overflows: u32,
334    /// `xEntryTimeSet`.
335    entry_set: bool,
336    /// `xInheritanceOccurred`.
337    inherited: bool,
338}
339
340/// What [`Kernel::start_scheduler`] created, so a runner can attach bodies.
341#[derive(Debug, Clone, Copy)]
342pub struct StartHandles {
343    /// The idle task (core 0's, on an SMP build).
344    pub idle: TaskHandle,
345    /// SMP: core 1's passive idle task (`IDLE1`); null on one core.
346    pub passive_idle: TaskHandle,
347    /// The timer service task.
348    pub timer: TaskHandle,
349    /// The timer command queue.
350    pub timer_queue: QueueHandle,
351}
352
353/// The most cores a kernel can be configured for (`configNUMBER_OF_CORES`).
354///
355/// Two, because the part SMP exists for here is the ESP32-S3. The per-core
356/// state -- `current`, its cached priority, `xYieldPendings` -- is an array
357/// of this length, and on a one-core build every index into it is the
358/// CONSTANT zero ([`Kernel::core`]), so the arrays cost one slot of
359/// `TaskHandle` and two bytes and no instruction: `self.current[0]` is the
360/// same load the scalar field was.
361pub const MAX_CORES: usize = 2;
362
363/// `taskEVENT_LIST_ITEM_VALUE_IN_USE` at a 32-bit `EventBits_t`: the top bit
364/// of the control byte events.rs masks off. Set on an event item that holds
365/// an event-group wait's condition instead of a priority.
366pub(crate) const EVENT_LIST_ITEM_VALUE_IN_USE: u64 = 0x8000_0000;
367// [`Kernel::core`] MASKS a core id with `MAX_CORES - 1`. Out here rather
368// than in that body: a `const` block inside an `#[inline(always)]` accessor
369// is MIR the one-core build weighs at every call site.
370const _: () = assert!(MAX_CORES.is_power_of_two());
371
372/// The scheduler.
373///
374/// `TASKS`, `ITEMS`, `LISTS`, `QUEUES`, `SLOTS`, `BUFFERS` and `BYTES` are
375/// the geometry: how many tasks, list items, lists, queues, queue item
376/// slots, stream buffers, and stream-buffer bytes this kernel has room
377/// for. See the crate docs and [`items_for`] / [`lists_for`].
378/// [`Kernel::new`] refuses a geometry that does not add up.
379#[repr(C)]
380pub struct Kernel<
381    C: Config,
382    P: Port,
383    T: Trace,
384    H,
385    const TASKS: usize,
386    const ITEMS: usize,
387    const LISTS: usize,
388    const QUEUES: usize,
389    const SLOTS: usize,
390    const BUFFERS: usize,
391    const BYTES: usize,
392    const TIMERS: usize,
393    const GROUPS: usize,
394    const TIMER_CMDS: usize,
395> {
396    /// `xTickCount`, masked to the configuration's tick width.
397    pub(crate) tick: u64,
398    /// `xPendedTicks`.
399    /// Counted in a `u32`, for the reason `overflows` is: it counts ticks
400    /// that arrived while the scheduler was suspended, which a suspension
401    /// long enough to overflow four billion would have to outlive.
402    pended_ticks: u32,
403    /// `xNextTaskUnblockTime`.
404    next_unblock_time: u64,
405    /// `xNumOfOverflows`.
406    overflows: u32,
407    /// `xLastTime` in `prvSampleTimeNow`.
408    pub(crate) timer_last_time: u64,
409    /// `pxCurrentTCB`.
410    pub(crate) current: [TaskHandle; MAX_CORES],
411    /// `uxSchedulerSuspended`.
412    suspended_depth: u32,
413    /// `uxCurrentNumberOfTasks`.
414    task_count: usize,
415    /// How many times the scheduler could not choose a task, and why the
416    /// FIRST time. See [`Stall`].
417    stalls: u32,
418    pub(crate) timer_message_next: usize,
419    pub(crate) bytes_used: usize,
420    pub(crate) free_count: usize,
421    pub(crate) free_slot_count: usize,
422    pub(crate) slots_used: usize,
423    /// The task whose abandoned frame is running right now, if any.
424    /// `TaskHandle::NULL` for "nobody", not `Option<TaskHandle>`.
425    ///
426    /// A `Handle` is two `u16`s with no niche, so the `Option` was eight bytes
427    /// and a real discriminant — and `resume_pending` tests this field on
428    /// every step the runner takes, which is the most-called entry in the
429    /// kernel. `NULL` is already the sentinel (`generation == 0`), so the
430    /// representation carries the fact the discriminant was carrying.
431    unwinding: TaskHandle,
432    /// `uxDeletedTasksWaitingCleanUp`: tasks deleted while running (on any
433    /// core) whose slots the idle task has not reclaimed yet. Which ones, in
434    /// which order, is each TCB's [`Tcb::reap_seq`] -- the C's
435    /// `xTasksWaitingTermination`, a FIFO list.
436    ///
437    /// It was ONE slot (two on two cores), on the stated property that a
438    /// second self-delete could not happen before the first was reaped. It can:
439    /// the deleting task yields to the next ready task, not to idle, and that
440    /// task may delete itself too. The second overwrote the first, whose slot
441    /// was then never freed and whose count was never taken off -- found while
442    /// proving the two-core gates of the P5 survey (`task_delete`'s
443    /// `NUMBER_OF_CORES > 1` relaxed), 2026-10-04.
444    reap_waiting: u16,
445    /// The deletion number the next deferred delete takes; 0 is "not
446    /// waiting", so it starts at 1 and skips 0 if it ever wraps.
447    reap_next: u32,
448    /// `xTimerQueue`.
449    pub(crate) timer_queue: QueueHandle,
450    /// How many tasks are parked at a queue call's sampling exit.
451    ///
452    /// A summary, so that [`Kernel::take_queue_resume`] — which runs at the
453    /// top of every queue send and every queue receive — can answer "nothing
454    /// to resume" from one field instead of resolving the caller's TCB out of
455    /// the arena. Only the three sites that park a task raise it, and only a
456    /// take that finds a slot set lowers it, so it counts exactly the slots
457    /// that are not `QueueResume::No`.
458    queue_resumes: u16,
459    /// How many tasks are parked inside an event-group wait, for the same
460    /// reason [`Kernel::queue_resumes`] exists: `take_event_resume` runs at
461    /// the top of every `event_group_wait_bits` and every `event_group_sync`,
462    /// and on all but the parked few the answer is no.
463    event_resumes: u16,
464    /// `uxTopReadyPriority`.
465    top_ready_priority: u8,
466    /// The running task's scheduling priority, kept beside `current`.
467    ///
468    /// The C reads `pxCurrentTCB->uxPriority`: one pointer dereference,
469    /// because the pointer IS the task. A handle is not a pointer, so the
470    /// same read went through `Arena::resolve` — a bounds check, a generation
471    /// compare and an `Option` — **every tick**, on the one handle the kernel
472    /// itself maintains and cannot have wrong.
473    ///
474    /// Measured: that resolve was **43 of the tick's 56 instructions, 77 %**.
475    /// Stubbing it to a constant took `increment_tick` from 56 to 13 against
476    /// the C's 15 — from 3.73x against us to 0.87x. This field is that
477    /// finding, done correctly.
478    ///
479    /// **It is written in exactly two places** — [`Self::set_current`] and
480    /// [`Self::set_task_priority`] — so that a call site cannot forget it.
481    /// `current_priority` debug-asserts it against the resolved value, so the
482    /// kernel's own tests and the conformance corpus fail loudly if it ever
483    /// drifts.
484    current_priority: [u8; MAX_CORES],
485    /// `xSchedulerRunning`.
486    running: bool,
487    /// `xYieldPendings`, per core.
488    yield_pending: [bool; MAX_CORES],
489    /// `portGET_CRITICAL_NESTING_COUNT( xCoreID )`, kept by the kernel on
490    /// two cores and more, where the outermost exit is a yield point (see
491    /// [`Kernel::exit_critical`]). Unused, and never written, on one core.
492    nesting: [u8; MAX_CORES],
493    /// Which of the two delayed lists is `pxDelayedTaskList` right now.
494    delayed_swapped: bool,
495    /// Which of the two timer lists is `pxCurrentTimerList` right now.
496    pub(crate) timers_swapped: bool,
497    first_stall: Stall,
498
499    pub(crate) port: P,
500    pub(crate) trace: T,
501    /// `vApplicationTickHook`, held by value so it can borrow the kernel.
502    pub(crate) tick_hook: H,
503    /// How many outermost critical-section exits a task owes the clock.
504    ///
505    /// This is `uxSavedCriticalNesting` in the Posix port's
506    /// `prvSwitchThread` and then some. A thread stops at the switch; a
507    /// stackless call does not, so the abandoned frame runs on — closing
508    /// the sections it had open and, on some paths, opening one more. The
509    /// port tallies every exit that frame makes instead of counting it as
510    /// sim time, and the tally is paid here when the task runs again. That
511    /// is what puts a sim tick where the C one lands.
512    /// Five per-task booleans, one bit each, in ONE array.
513    ///
514    /// They were five `[bool; TASKS]`. Each one is a separate base address, so
515    /// a function touching two of them — and the owed body touches three —
516    /// computed two bases and did two loads to read two bits that now share a
517    /// byte. Packed they cost one byte a task instead of five, and one base
518    /// instead of five across the whole kernel.
519    flags: [u8; TASKS],
520    /// `started`, kept OUT of `flags` on purpose.
521    ///
522    /// It is the only one of the five that `switch_context` reads — through
523    /// `hand_over`, on every switch — and packing it cost the selection row
524    /// one instruction for the mask. The other four are never on that path,
525    /// so they stay packed. Per-call-site, not per-idea.
526    started: [bool; TASKS],
527    /// `owes_anything`, kept OUT of `flags` for the reason `started` is: it is
528    /// the single most-read byte in the kernel — `resume_pending` tests it
529    /// before every step the runner takes — and packing it cost that row three
530    /// instructions for the mask. Per call site, not per idea.
531    owes_anything: [bool; TASKS],
532    /// `delay_aborted`, kept OUT of `flags`: `check_for_timeout` reads it on
533    /// every pass of every blocking call.
534    delay_aborted: [bool; TASKS],
535    pub(crate) lists: Lists<ITEMS, LISTS>,
536    pub(crate) tcbs: Arena<TaskKind, Tcb, TASKS>,
537    owed_exits: [u32; TASKS],
538    /// A trace line a task owes from a call it was switched out of.
539    ///
540    /// `xQueueReceive`'s failure path is `taskEXIT_CRITICAL();
541    /// traceQUEUE_RECEIVE_FAILED( pxQueue ); return errQUEUE_EMPTY;` — and
542    /// if the tick that the exit released switches the task away, those two
543    /// lines sit on a stack that is not running. The C emits the trace when
544    /// the task resumes, so this kernel does too.
545    owed_trace: [OwedTrace; TASKS],
546    pub(crate) queues: Arena<QueueKind, Queue, QUEUES>,
547    pub(crate) timers: Arena<TimerKind, Timer, TIMERS>,
548    pub(crate) groups: Arena<EventGroupKind, EventGroup, GROUPS>,
549    pub(crate) buffers: Arena<StreamKind, StreamBuffer, BUFFERS>,
550    /// Blocks of the byte arena that a deleted stream buffer gave back,
551    /// as `(base, length)`, kept sorted and coalesced.
552    ///
553    /// This is the one allocator in the kernel, and it exists because the
554    /// C's stream buffers are heap objects: `MessageBufferDemo`'s echo
555    /// server creates one and deletes it again on every loop, and a bump
556    /// allocator would run out in a few hundred ticks. At most `BUFFERS`
557    /// buffers can be alive, so at most `BUFFERS` holes can exist between
558    /// them, which is why the list is that long and cannot overflow.
559    pub(crate) free_blocks: [(usize, usize); BUFFERS],
560    /// Extents of the SLOT arena that a deleted queue gave back, as
561    /// `(base, length)`, coalesced the same way `free_blocks` is.
562    ///
563    /// The byte arena has had this since `MessageBufferDemo` needed it. The
564    /// slot arena did not, so `queue_delete` returned the descriptor while
565    /// the storage stayed spent and a create/delete loop exhausted `SLOTS`
566    /// permanently — found through `AbortDelay` on 2026-09-21, after nine
567    /// hypotheses and eight refutations, because the descriptor arena works
568    /// perfectly and that is where everyone looked.
569    ///
570    /// At most `QUEUES` queues are alive, so at most `QUEUES` holes can sit
571    /// between them and the list cannot overflow.
572    pub(crate) free_slots: [(usize, usize); QUEUES],
573    /// The ring of `DaemonTaskMessage_t`s the timer queue carries indices
574    /// into. It is exactly as long as the queue, so a message can only be
575    /// overwritten once the queue has already refused to hold its index.
576    /// The timer daemon's command mailbox, sized by the configuration.
577    ///
578    /// It used to be `[Message; MAX_TIMER_COMMANDS]` — a hardcoded 32 — while
579    /// `Config::TIMER_QUEUE_LENGTH` already said how many the configuration
580    /// wanted and defaulted to ten. Because it scaled with **nothing**, it was
581    /// a fixed 768 bytes at every geometry, which is **39 % of a two-task
582    /// kernel's entire static footprint** and 31 % of a four-task one.
583    ///
584    /// `TIMER_CMDS` is that number, checked against the config in
585    /// [`Self::with_tick_hook`] exactly as `ITEMS` and `LISTS` are.
586    pub(crate) timer_messages: [Message; TIMER_CMDS],
587    pub(crate) slots: [u64; SLOTS],
588    pub(crate) bytes: [u8; BYTES],
589    _config: PhantomData<C>,
590    // ---- SMP state, deliberately LAST ------------------------------------
591    //
592    // `Kernel` is `#[repr(C)]`, so where a field sits is a layout decision.
593    // These three were first added beside `yield_pending` and cost the
594    // ONE-core scheduling bench **+24,000 Ir (+1.6 %)** without a line of SMP
595    // logic running: they moved every hot field behind them. At the end of
596    // the struct they cost nothing (`bench/ksched-ir`, 2026-10-01).
597    /// SMP: `pxCurrentTCBs[ core ]->xTaskRunState == taskTASK_SCHEDULED_TO_YIELD`.
598    ///
599    /// The C keeps a run state in every TCB. Here "running on core `c`" IS
600    /// `current[c]`, so the one state that is not derivable -- "running, but
601    /// another core has asked it to yield" -- is kept per core instead, and
602    /// the TCB is untouched (a one-core build does not pay for it).
603    yield_requested: [bool; MAX_CORES],
604    /// SMP: the cores this kernel has asked to yield and the port has not yet
605    /// interrupted, one bit per core (`portYIELD_CORE`). Drained by
606    /// [`Kernel::take_core_yields`], because raising another core's
607    /// interrupt is the PORT's act and the kernel is `forbid(unsafe)`.
608    core_yields: u8,
609    /// SMP: `xIdleTaskHandles`, so a running idle task can be ranked below
610    /// every real task of priority 0 (`prvYieldForTask`).
611    idle: [TaskHandle; MAX_CORES],
612}
613
614impl<
615    C: Config,
616    P: Port,
617    T: Trace,
618    H,
619    const TASKS: usize,
620    const ITEMS: usize,
621    const LISTS: usize,
622    const QUEUES: usize,
623    const SLOTS: usize,
624    const BUFFERS: usize,
625    const BYTES: usize,
626    const TIMERS: usize,
627    const GROUPS: usize,
628    const TIMER_CMDS: usize,
629> Kernel<C, P, T, H, TASKS, ITEMS, LISTS, QUEUES, SLOTS, BUFFERS, BYTES, TIMERS, GROUPS, TIMER_CMDS>
630where
631    H: TickHook<Self>,
632{
633    /// `portMAX_DELAY` at this configuration's tick width.
634    pub const MAX_DELAY: u64 = <C::Tick as TickWidth>::MAX;
635
636    // ------------------------------------------------------- footprint --
637
638    /// The kernel's own static size, and the same number `size_of` gives.
639    ///
640    /// The consts below decompose it. They exist because a footprint you
641    /// cannot decompose to the byte is a footprint you cannot optimise: the
642    /// arena fields are `pub(crate)` and their types are private, so nothing
643    /// outside this crate can measure them, and a decomposition assembled
644    /// from per-dimension SLOPES is not one — `ITEMS` and `LISTS` are derived
645    /// from `TASKS`, `QUEUES` and `GROUPS`, so the function is not linear and
646    /// slopes do not sum to the total. That was tried; it produced 696 bytes
647    /// of nonsense.
648    ///
649    /// Every one is `size_of` on whatever target this is compiled for, which
650    /// is the only place the number means anything: a host `usize` is eight
651    /// bytes and rv32's is four.
652    ///
653    /// [`FOOTPRINT_ACCOUNTED`](Self::FOOTPRINT_ACCOUNTED) sums them, and the
654    /// difference from [`FOOTPRINT`](Self::FOOTPRINT) is padding plus the
655    /// scalar tail — print both, never one. `bench/kernel-ram` does.
656    pub const FOOTPRINT: usize = core::mem::size_of::<Self>();
657
658    /// The task control blocks.
659    pub const FOOTPRINT_TCBS: usize = core::mem::size_of::<Arena<TaskKind, Tcb, TASKS>>();
660    /// The queue descriptors, and NOT their item storage.
661    pub const FOOTPRINT_QUEUES: usize = core::mem::size_of::<Arena<QueueKind, Queue, QUEUES>>();
662    /// Every intrusive list: ready, delayed, and the per-object waiters.
663    pub const FOOTPRINT_LISTS: usize = core::mem::size_of::<Lists<ITEMS, LISTS>>();
664    /// Queue item storage, as `u64` slots.
665    pub const FOOTPRINT_SLOTS: usize = core::mem::size_of::<[u64; SLOTS]>();
666    /// The stream-buffer descriptors, and NOT their bytes.
667    pub const FOOTPRINT_BUFFERS: usize =
668        core::mem::size_of::<Arena<StreamKind, StreamBuffer, BUFFERS>>();
669    /// The software timers.
670    pub const FOOTPRINT_TIMERS: usize = core::mem::size_of::<Arena<TimerKind, Timer, TIMERS>>();
671    /// The event groups.
672    pub const FOOTPRINT_GROUPS: usize =
673        core::mem::size_of::<Arena<EventGroupKind, EventGroup, GROUPS>>();
674    /// The timer daemon's command mailbox.
675    ///
676    /// It scales with `TIMER_CMDS` now, which the geometry check pins to
677    /// `Config::TIMER_QUEUE_LENGTH`. It used to be a hardcoded 32 and so
678    /// scaled with nothing — a fixed 768 bytes, 39 % of a two-task kernel.
679    /// The line stays separate because that history is worth being able to
680    /// read off a decomposition.
681    pub const FOOTPRINT_TIMER_MESSAGES: usize = core::mem::size_of::<[Message; TIMER_CMDS]>();
682    /// The byte arena, for stream-buffer contents.
683    pub const FOOTPRINT_BYTES: usize = core::mem::size_of::<[u8; BYTES]>();
684    /// The two free lists, over the byte arena and the slot arena.
685    pub const FOOTPRINT_FREE_LISTS: usize = core::mem::size_of::<[(usize, usize); BUFFERS]>()
686        + core::mem::size_of::<[(usize, usize); QUEUES]>();
687
688    /// The seven per-task side arrays, together.
689    ///
690    /// Seven arrays indexed by the same task, five of them `[bool; TASKS]`.
691    /// Whether that wants to be one array of a flags byte is a question this
692    /// number is here to make askable.
693    pub const FOOTPRINT_PER_TASK_SIDE: usize = core::mem::size_of::<[u8; TASKS]>()
694        + core::mem::size_of::<[bool; TASKS]>() * 3
695        + core::mem::size_of::<[OwedTrace; TASKS]>()
696        + core::mem::size_of::<[u32; TASKS]>();
697
698    /// What the lines above account for.
699    ///
700    /// `FOOTPRINT - FOOTPRINT_ACCOUNTED` is the unattributed remainder: the
701    /// scalar tail (`tick`, `current`, the counters) plus whatever padding
702    /// the layout inserts. It was 140 bytes of 6,784 when this was written. A
703    /// remainder that grows is a field somebody added without adding a line
704    /// here.
705    pub const FOOTPRINT_ACCOUNTED: usize = Self::FOOTPRINT_TCBS
706        + Self::FOOTPRINT_QUEUES
707        + Self::FOOTPRINT_LISTS
708        + Self::FOOTPRINT_SLOTS
709        + Self::FOOTPRINT_BUFFERS
710        + Self::FOOTPRINT_TIMERS
711        + Self::FOOTPRINT_GROUPS
712        + Self::FOOTPRINT_TIMER_MESSAGES
713        + Self::FOOTPRINT_BYTES
714        + Self::FOOTPRINT_FREE_LISTS
715        + Self::FOOTPRINT_PER_TASK_SIDE;
716
717    // ------------------------------------------------------------ setup --
718
719    /// A kernel with no tasks, ready for [`Kernel::create_task`].
720    ///
721    /// # Errors
722    /// [`Error::InvalidArgument`] if the configuration is invalid
723    /// ([`Config::validate`]), if the name capacity cannot hold
724    /// `MAX_TASK_NAME_LEN`, or if the const geometry does not match the
725    /// configuration: `ITEMS` must be [`items_for`]`(TASKS)` and `LISTS`
726    /// must be [`lists_for`]`(MAX_PRIORITIES, QUEUES)`.
727    pub fn new(port: P, trace: T) -> Result<Self>
728    where
729        H: Default,
730    {
731        Self::with_tick_hook(port, trace, H::default())
732    }
733
734    /// [`Kernel::new`] with `vApplicationTickHook` installed.
735    ///
736    /// # Errors
737    /// As [`Kernel::new`].
738    pub fn with_tick_hook(port: P, trace: T, tick_hook: H) -> Result<Self> {
739        C::validate()?;
740        if ITEMS != list_slots_for(TASKS, TIMERS, LISTS)
741            || LISTS != lists_for(C::MAX_PRIORITIES, QUEUES, GROUPS)
742            || TASKS == 0
743            || C::MAX_TASK_NAME_LEN > crate::NAME_CAPACITY
744            // The mailbox is a declared dimension now, so a declaration that
745            // disagrees with its own config is refused rather than silently
746            // sized to whichever one the type happened to carry.
747            || TIMER_CMDS != C::TIMER_QUEUE_LENGTH
748            // `configMESSAGE_BUFFER_LENGTH_TYPE`: one to eight bytes. Wider
749            // than eight cannot hold a length any target can send.
750            || C::MESSAGE_LENGTH_BYTES == 0
751            || C::MESSAGE_LENGTH_BYTES > 8
752            // Two cores need a port that COMMITS its own switches. The
753            // stackless model -- `switch_context` returning into the
754            // outgoing task's frame, whose tail is tallied as owed -- keeps
755            // ONE abandoned frame (`unwinding`), which is one core's; two
756            // cores switching at once would share it. Every two-core port
757            // there is commits (sim, Cortex-M, RISC-V, Xtensa, host); this
758            // says so instead of leaving it to be discovered.
759            || (C::NUMBER_OF_CORES > 1 && !P::COMMITS_SWITCH)
760        {
761            return Err(Error::InvalidArgument);
762        }
763        Ok(Self {
764            port,
765            trace,
766            tcbs: Arena::new(),
767            queues: Arena::new(),
768            lists: Lists::new(),
769            slots: [0; SLOTS],
770            slots_used: 0,
771            buffers: Arena::new(),
772            timers: Arena::new(),
773            groups: Arena::new(),
774            timer_messages: [Message::default(); TIMER_CMDS],
775            timer_message_next: 0,
776            timers_swapped: false,
777            timer_last_time: 0,
778            timer_queue: QueueHandle::NULL,
779            bytes: [0; BYTES],
780            bytes_used: 0,
781            free_blocks: [(0, 0); BUFFERS],
782            free_count: 0,
783            free_slots: [(0, 0); QUEUES],
784            free_slot_count: 0,
785            current: [TaskHandle::NULL; MAX_CORES],
786            top_ready_priority: 0,
787            current_priority: [0; MAX_CORES],
788            tick: C::INITIAL_TICK_COUNT,
789            pended_ticks: 0,
790            suspended_depth: 0,
791            running: false,
792            next_unblock_time: Self::MAX_DELAY,
793            yield_pending: [false; MAX_CORES],
794            nesting: [0; MAX_CORES],
795            yield_requested: [false; MAX_CORES],
796            core_yields: 0,
797            idle: [TaskHandle::NULL; MAX_CORES],
798            task_count: 0,
799            stalls: 0,
800            first_stall: Stall::None,
801            reap_waiting: 0,
802            reap_next: 0,
803            delayed_swapped: false,
804            overflows: 0,
805            flags: [0; TASKS],
806            started: [false; TASKS],
807            owes_anything: [false; TASKS],
808            delay_aborted: [false; TASKS],
809            owed_exits: [0; TASKS],
810            unwinding: TaskHandle::NULL,
811            queue_resumes: 0,
812            event_resumes: 0,
813            tick_hook,
814            owed_trace: [OwedTrace::None; TASKS],
815            _config: PhantomData,
816        })
817    }
818
819    /// The port, for a runner that needs its counters.
820    pub const fn port(&self) -> &P {
821        &self.port
822    }
823
824    /// The trace sink.
825    pub const fn trace(&self) -> &T {
826        &self.trace
827    }
828
829    /// The trace sink, mutably.
830    pub const fn trace_mut(&mut self) -> &mut T {
831        &mut self.trace
832    }
833
834    /// Take the trace sink back when the run is over, so a deliverable can
835    /// flush whatever it was writing to.
836    pub fn into_trace(self) -> T {
837        self.trace
838    }
839
840    /// `xNumOfOverflows`, for a scenario that reports counters.
841    #[must_use]
842    pub const fn overflows(&self) -> u64 {
843        self.overflows as u64
844    }
845
846    // ------------------------------------------------------ list indices --
847
848    // `ListId` is a `u8` and `ItemId` a `u16` (the core's `list`): the
849    // whole scheduler addresses lists and items by small integers, which is
850    // what keeps a TCB free of pointers.
851
852    pub(crate) const fn ready_list(priority: u8) -> ListId {
853        priority
854    }
855
856    const fn delayed_list(&self) -> ListId {
857        if self.delayed_swapped {
858            C::MAX_PRIORITIES.saturating_add(1)
859        } else {
860            C::MAX_PRIORITIES
861        }
862    }
863
864    const fn overflow_delayed_list(&self) -> ListId {
865        if self.delayed_swapped {
866            C::MAX_PRIORITIES
867        } else {
868            C::MAX_PRIORITIES.saturating_add(1)
869        }
870    }
871
872    const fn pending_ready_list() -> ListId {
873        C::MAX_PRIORITIES.saturating_add(2)
874    }
875
876    const fn suspended_list() -> ListId {
877        C::MAX_PRIORITIES.saturating_add(3)
878    }
879
880    /// `xTasksWaitingToSend` of a queue.
881    pub(crate) fn queue_send_list(queue: QueueHandle) -> ListId {
882        let offset = u8::try_from(queue.index().saturating_mul(2)).unwrap_or(u8::MAX);
883        C::MAX_PRIORITIES
884            .saturating_add(u8::try_from(OVERHEAD_LISTS).unwrap_or(u8::MAX))
885            .saturating_add(offset)
886    }
887
888    /// `xTasksWaitingToReceive` of a queue.
889    pub(crate) fn queue_receive_list(queue: QueueHandle) -> ListId {
890        Self::queue_send_list(queue).saturating_add(1)
891    }
892
893    /// A task's `xStateListItem`: the task's own arena index.
894    pub(crate) const fn state_item(task: TaskHandle) -> ItemId {
895        task.index() as u16
896    }
897
898    /// A task's `xEventListItem`: `TASKS` above its state item, so the two
899    /// never collide and either maps back to its task by arithmetic alone.
900    pub(crate) fn event_item(task: TaskHandle) -> ItemId {
901        Self::task_item_base().saturating_add(task.index() as u16)
902    }
903
904    fn task_item_base() -> ItemId {
905        u16::try_from(TASKS).unwrap_or(u16::MAX)
906    }
907
908    fn task_of_state_item(&self, item: ItemId) -> Result<TaskHandle> {
909        self.tcbs.handle_at(item).ok_or(Error::Gone)
910    }
911
912    fn task_of_event_item(&self, item: ItemId) -> Result<TaskHandle> {
913        let index = item
914            .checked_sub(Self::task_item_base())
915            .ok_or(Error::InvalidArgument)?;
916        self.tcbs.handle_at(index).ok_or(Error::Gone)
917    }
918
919    // ----------------------------------------------------------- getters --
920
921    /// `pxCurrentTCB`: the task running on the calling core.
922    ///
923    /// No longer `const`: on an SMP build the answer depends on which core
924    /// asks (`portGET_CORE_ID()`), which is the port's to say.
925    #[must_use]
926    pub fn current(&self) -> TaskHandle {
927        self.cur()
928    }
929
930    /// `pxCurrentTCBs[ core ]`: the task running on `core`, or the null
931    /// handle for a core this configuration does not have.
932    #[must_use]
933    pub fn current_on(&self, core: usize) -> TaskHandle {
934        if core < Self::cores() {
935            self.current_of(core)
936        } else {
937            TaskHandle::NULL
938        }
939    }
940
941    /// `xTaskGetTickCount`.
942    #[must_use]
943    pub const fn tick_count(&self) -> u64 {
944        self.tick
945    }
946
947    /// `xSchedulerRunning`.
948    #[must_use]
949    pub const fn is_running(&self) -> bool {
950        self.running
951    }
952
953    /// `uxCurrentNumberOfTasks`.
954    #[must_use]
955    pub const fn task_count(&self) -> usize {
956        self.task_count
957    }
958
959    /// `uxSchedulerSuspended`, for a stepper reporting why a tick pended.
960    #[must_use]
961    pub const fn scheduler_suspended(&self) -> u32 {
962        self.suspended_depth
963    }
964
965    /// `xPendedTicks`, likewise.
966    #[must_use]
967    pub const fn pended_ticks(&self) -> u64 {
968        self.pended_ticks as u64
969    }
970
971    /// The item ids in `priority`'s ready list, in order, into `out`.
972    ///
973    /// Returns how many were written. The companion to
974    /// [`Kernel::ready_cursor`]: when a ready task is never chosen, the
975    /// cursor says whether the rotation moved and this says what it was
976    /// moving THROUGH.
977    pub fn ready_items(&self, priority: u8, out: &mut [u16]) -> usize {
978        let mut n = 0;
979        for item in self.lists.iter(Self::ready_list(priority)) {
980            let Some(slot) = out.get_mut(n) else {
981                break;
982            };
983            *slot = item;
984            n = n.wrapping_add(1);
985        }
986        n
987    }
988
989    /// Where the round-robin cursor sits in `priority`'s ready list.
990    ///
991    /// A diagnostic, paired with [`Kernel::ready_len`]: a ready task that
992    /// is never chosen is either not in the list the scheduler looks at,
993    /// or the cursor is not moving. These two answer both halves.
994    #[must_use]
995    pub fn ready_cursor(&self, priority: u8) -> u16 {
996        self.lists
997            .cursor_of(Self::ready_list(priority))
998            .unwrap_or(0)
999    }
1000
1001    /// The live handle in arena slot `index`, if that slot holds a task.
1002    ///
1003    /// The generation is the point: a bare index names a SLOT, and a slot
1004    /// outlives the tasks that occupy it, so an index is not a task. This
1005    /// answers the handle, which is.
1006    ///
1007    /// It exists so a diagnostic can walk every task without knowing any
1008    /// of their names -- `task_get_handle` needs the name it is looking
1009    /// for, and a crash report is exactly the case where you do not have
1010    /// one.
1011    #[must_use]
1012    pub fn task_at(&self, index: usize) -> Option<TaskHandle> {
1013        let index = u16::try_from(index).ok()?;
1014        self.tcbs.handle_at(index)
1015    }
1016
1017    /// A task's name.
1018    ///
1019    /// # Errors
1020    /// [`Error::Gone`] for a stale or null handle.
1021    pub fn name_of(&self, task: TaskHandle) -> Result<Name> {
1022        self.tcbs.resolve(task).map(|t| t.name)
1023    }
1024
1025    /// `uxTaskPriorityGet`. `None` means the current task, as in C.
1026    ///
1027    /// # Errors
1028    /// [`Error::Gone`] for a stale handle.
1029    pub fn priority_of(&self, task: Option<TaskHandle>) -> Result<u8> {
1030        let h = task.unwrap_or(self.cur());
1031        self.tcbs.resolve(h).map(|t| t.priority)
1032    }
1033
1034    pub(crate) fn current_priority(&self) -> u8 {
1035        // The cache is the whole point; this is the proof it is honest. It
1036        // costs nothing in release and fails the kernel's own tests, the
1037        // Kani proofs and the conformance corpus the moment a new write to
1038        // `current` or to a priority forgets its helper.
1039        debug_assert_eq!(
1040            self.cur_priority(),
1041            self.priority_of(None).unwrap_or(0),
1042            "the cached current priority went stale -- a write to `current` or              to a task's priority bypassed set_current/set_task_priority"
1043        );
1044        self.cur_priority()
1045    }
1046
1047    /// Make `task` the running task, keeping [`Self::current_priority`] with
1048    /// it, where the caller already knows the priority.
1049    ///
1050    /// One of the two places that field is written; assigning `self.current`
1051    /// directly is what this exists to prevent.
1052    ///
1053    /// **The priority is a parameter and not resolved here**, because every
1054    /// caller already has it. `switch_context`'s search loop finds the highest
1055    /// non-empty ready list and takes `next` out of it, so `next`'s priority
1056    /// IS that list's index; task creation has it as an argument. Resolving
1057    /// the TCB again to learn what the caller just proved cost **9
1058    /// instructions on the switch row** — measured, by landing this without
1059    /// the parameter and watching `switch_select` go 79 → 88, then 88 → 82
1060    /// when it was threaded through.
1061    fn set_current_at(&mut self, task: TaskHandle, priority: u8) {
1062        let core = self.core();
1063        if let Some(slot) = self.current.get_mut(core) {
1064            *slot = task;
1065        }
1066        if let Some(slot) = self.current_priority.get_mut(core) {
1067            *slot = priority;
1068        }
1069    }
1070
1071    /// `pxCurrentTCB` for the calling core.
1072    ///
1073    /// ★ On one core this DESTRUCTURES element 0 rather than calling
1074    /// `.get(0).copied().unwrap_or(..)`. The two should fold to the same
1075    /// load and measurably do not: with the `Option` form on every per-core
1076    /// accessor the scheduling bench (`bench/ksched-ir`) read **+14,985 Ir
1077    /// (+0.98 %)** against the scalar field; destructuring took that to
1078    /// +2,985. `.get` stays for the SMP arm, where the index is a runtime
1079    /// core id and the kernel stays free of panicking indexing.
1080    #[inline(always)]
1081    pub(crate) fn cur(&self) -> TaskHandle {
1082        if C::NUMBER_OF_CORES <= 1 {
1083            let [first, ..] = self.current;
1084            return first;
1085        }
1086        self.current
1087            .get(self.core())
1088            .copied()
1089            .unwrap_or(TaskHandle::NULL)
1090    }
1091
1092    /// The calling core's cached running priority (see `current_priority`).
1093    #[inline(always)]
1094    fn cur_priority(&self) -> u8 {
1095        if C::NUMBER_OF_CORES <= 1 {
1096            let [first, ..] = self.current_priority;
1097            return first;
1098        }
1099        self.current_priority.get(self.core()).copied().unwrap_or(0)
1100    }
1101
1102    /// `xYieldPendings[ portGET_CORE_ID() ]`.
1103    #[inline(always)]
1104    fn pending_here(&self) -> bool {
1105        if C::NUMBER_OF_CORES <= 1 {
1106            let [first, ..] = self.yield_pending;
1107            return first;
1108        }
1109        self.yield_pending
1110            .get(self.core())
1111            .copied()
1112            .unwrap_or(false)
1113    }
1114
1115    /// Set `xYieldPendings[ portGET_CORE_ID() ]`.
1116    #[inline(always)]
1117    fn set_pending_here(&mut self, value: bool) {
1118        if C::NUMBER_OF_CORES <= 1 {
1119            let [first, ..] = &mut self.yield_pending;
1120            *first = value;
1121            return;
1122        }
1123        let core = self.core();
1124        if let Some(slot) = self.yield_pending.get_mut(core) {
1125            *slot = value;
1126        }
1127    }
1128
1129    /// `portGET_CORE_ID()`, as an index into the per-core state.
1130    ///
1131    /// On a one-core build this is the CONSTANT zero -- the branch is on a
1132    /// `const` and folds -- so every `self.cur()` is the same
1133    /// load the scalar field was, and the port is never asked.
1134    #[inline(always)]
1135    pub(crate) fn core(&self) -> usize {
1136        if C::NUMBER_OF_CORES <= 1 {
1137            0
1138        } else {
1139            // A MASK, not a clamp. `portGET_CORE_ID()` is below the core
1140            // count by definition, and for every such id the two agree; the
1141            // clamp compiled to a compare and a `setne` at every kernel entry
1142            // where the mask is one `and`. An id outside the contract lands
1143            // on SOME core's slot either way, never outside the arrays.
1144            //
1145            // The `.min` is for the ONE-core build, and is folded away here
1146            // (the masked id is already in range). Without it this body is
1147            // lighter MIR than the clamp was, and rustc's MIR inliner -- which
1148            // weighs it before `C::NUMBER_OF_CORES` folds -- inlined the
1149            // one-core kernel differently: +63,896 Ir on `bench/kernel-ir`'s
1150            // kernel rows (step, suspend, delay) for a branch that never runs.
1151            (usize::from(self.port.core_id()) & (MAX_CORES - 1)).min(MAX_CORES - 1)
1152        }
1153    }
1154
1155    /// Write a task's scheduling priority, keeping [`Self::current_priority`]
1156    /// with it when that task is the running one.
1157    ///
1158    /// The other of the two. Note it is the SCHEDULING priority only:
1159    /// `base_priority` does not decide which ready list a task sits on, so it
1160    /// is written directly at its call sites.
1161    ///
1162    /// # Errors
1163    /// [`Error::InvalidHandle`] or [`Error::Gone`] for a handle the arena
1164    /// refuses, exactly as `resolve_mut` does.
1165    fn set_task_priority(&mut self, task: TaskHandle, priority: u8) -> Result<()> {
1166        self.tcbs.resolve_mut(task)?.priority = priority;
1167        // Every core that is running `task` caches its priority; on one core
1168        // the loop is the single compare it always was.
1169        for core in 0..Self::cores() {
1170            if task == self.current_of(core) {
1171                if let Some(slot) = self.current_priority.get_mut(core) {
1172                    *slot = priority;
1173                }
1174            }
1175        }
1176        Ok(())
1177    }
1178
1179    /// `uxTaskPriorityGet`: the C API, which takes a critical section.
1180    ///
1181    /// The distinction matters: on the sim a critical-section exit is where
1182    /// time passes, so reading a priority through the API costs a tick
1183    /// sixteen calls later, exactly as it does in C
1184    /// (`portBASE_TYPE_ENTER_CRITICAL` is `taskENTER_CRITICAL` on the Posix
1185    /// port). The kernel's own reads use the field directly, as C's do.
1186    ///
1187    /// # Errors
1188    /// [`Error::Gone`] for a stale handle.
1189    pub fn task_priority_get(&mut self, task: Option<TaskHandle>) -> Result<u8> {
1190        self.enter_critical();
1191        let result = self.priority_of(task);
1192        self.exit_critical();
1193        result
1194    }
1195
1196    /// `eTaskGetState`: the C API, which takes a critical section for any
1197    /// task but the running one.
1198    ///
1199    /// # Errors
1200    /// A list error surfaces as itself.
1201    pub fn task_state_get(&mut self, task: TaskHandle) -> Result<TaskState> {
1202        // One core only: the SMP `eTaskGetState` has no shortcut for the
1203        // caller's own task -- it always reads the lists, in the section.
1204        if C::NUMBER_OF_CORES <= 1 && task == self.cur() {
1205            return Ok(TaskState::Running);
1206        }
1207        self.enter_critical();
1208        let result = self.state_of(task);
1209        self.exit_critical();
1210        result
1211    }
1212
1213    /// `eTaskGetState`, derived from the list the task is in.
1214    ///
1215    /// # Errors
1216    /// A list error surfaces as itself.
1217    pub fn state_of(&self, task: TaskHandle) -> Result<TaskState> {
1218        if !self.tcbs.contains(task) {
1219            return Ok(TaskState::Deleted);
1220        }
1221        // `pxTCB == pxCurrentTCB` is a ONE-core shortcut in the C. On two
1222        // cores there is none: a task that is current here but has just
1223        // placed itself on the delayed list reads Blocked, and "running" is
1224        // decided below, for a task on a ready list, on ANY core.
1225        if C::NUMBER_OF_CORES <= 1 && task == self.cur() {
1226            return Ok(TaskState::Running);
1227        }
1228        let Some(list) = self.lists.container(Self::state_item(task))? else {
1229            return Ok(TaskState::Deleted);
1230        };
1231        if list == Self::suspended_list() {
1232            if self.lists.container(Self::event_item(task))?.is_some() {
1233                return Ok(TaskState::Blocked);
1234            }
1235            // A task blocked indefinitely on a notification is in the
1236            // suspended list and on no event list at all, so the C scans
1237            // the notification array before it calls such a task suspended.
1238            // Miss this and a stream buffer's reader looks suspended to
1239            // `eTaskGetState` while it is plainly waiting.
1240            let waiting_notification = self
1241                .tcbs
1242                .resolve(task)
1243                .map(|t| {
1244                    t.notify_state
1245                        .iter()
1246                        .take(C::NOTIFICATION_ARRAY_ENTRIES)
1247                        .any(|s| *s == NotifyState::Waiting)
1248                })
1249                .unwrap_or(false);
1250            return Ok(if waiting_notification {
1251                TaskState::Blocked
1252            } else {
1253                TaskState::Suspended
1254            });
1255        }
1256        if list == self.delayed_list() || list == self.overflow_delayed_list() {
1257            return Ok(TaskState::Blocked);
1258        }
1259        // SMP: `taskTASK_IS_RUNNING( pxTCB )` -- current on SOME core and not
1260        // asked to yield it. Found by the API differential (P1): a task
1261        // running on the other core read Ready, on its very first step.
1262        if C::NUMBER_OF_CORES > 1 && self.running_core(task).is_some() {
1263            return Ok(TaskState::Running);
1264        }
1265        Ok(TaskState::Ready)
1266    }
1267
1268    /// How many tasks sit on a priority's ready list — what the idle task
1269    /// checks before yielding (`configIDLE_SHOULD_YIELD`).
1270    ///
1271    /// # Errors
1272    /// [`Error::InvalidPriority`] for a priority outside the configuration.
1273    pub fn ready_len(&self, priority: u8) -> Result<usize> {
1274        if priority >= C::MAX_PRIORITIES {
1275            return Err(Error::InvalidPriority);
1276        }
1277        self.lists.len(Self::ready_list(priority))
1278    }
1279
1280    /// Record that the scheduler could not choose.
1281    ///
1282    /// Counts every occurrence and keeps the FIRST reason: the first is the
1283    /// diagnosis and the rest are consequences.
1284    #[cold]
1285    #[inline(never)]
1286    fn note_stall(&mut self, why: Stall) {
1287        self.stalls = self.stalls.saturating_add(1);
1288        if self.first_stall == Stall::None {
1289            self.first_stall = why;
1290        }
1291    }
1292
1293    /// How many times the scheduler could not choose a task.
1294    ///
1295    /// **Non-zero means an invariant broke.** A healthy kernel never stalls:
1296    /// the idle task is always ready. A firmware that prints this alongside
1297    /// its own counters turns a hang into a sentence.
1298    #[must_use]
1299    pub const fn stalls(&self) -> u32 {
1300        self.stalls
1301    }
1302
1303    /// Why the scheduler first failed to choose. [`Stall::None`] if it never
1304    /// has.
1305    #[must_use]
1306    pub const fn first_stall(&self) -> Stall {
1307        self.first_stall
1308    }
1309
1310    // ------------------------------------------------------------ tracing --
1311
1312    pub(crate) fn trace_task<F>(&mut self, task: TaskHandle, make: F)
1313    where
1314        F: for<'a> FnOnce(TaskHandle, &'a str) -> Event<'a>,
1315    {
1316        // `tcbs` and `trace` are different fields, so the name can be read
1317        // where it lives rather than copied onto the stack to end a borrow
1318        // that never had to end.
1319        //
1320        // A sink that never reads the name pays for neither the lookup nor
1321        // the UTF-8 validation that turning one into a `&str` costs.
1322        let Self {
1323            tcbs,
1324            trace,
1325            port,
1326            tick,
1327            ..
1328        } = self;
1329        let name = if T::WANTS_NAMES {
1330            tcbs.get(task).map(|t| t.name.as_str()).unwrap_or("")
1331        } else {
1332            ""
1333        };
1334        let tick = *tick;
1335        // Gated as `Kernel::note_exits` is. A silent sink discards the count,
1336        // but on a port whose counter is an atomic the LOAD survives the
1337        // discarded value: `lw zero, ...` in every traced path on rv32
1338        // (`RiscvPort`), one per wake in `block_cycle` alone.
1339        if T::EMITS {
1340            trace.note_exits(port.exits());
1341        }
1342        trace.event(tick, make(task, name));
1343    }
1344
1345    pub(crate) fn priority_value(raw: u8) -> Priority {
1346        Priority::new(raw, C::MAX_PRIORITIES).unwrap_or(Priority::IDLE)
1347    }
1348
1349    // --------------------------------------------------- critical section --
1350
1351    /// What `pvPortMalloc` costs in sim time on a heap-backed C kernel:
1352    /// `vTaskSuspendAll()` around the allocation, then `xTaskResumeAll()`,
1353    /// whose critical section is one outermost exit.
1354    ///
1355    /// This kernel allocates nothing, so the call is the whole of it. On a
1356    /// configuration with [`Config::DYNAMIC_ALLOCATION`] off it compiles
1357    /// away to nothing at all.
1358    // Out of line: 12 cold create/delete sites, each otherwise inlining
1359    // suspend_all + resume_all. Worth -46 B, and the Ir cost is nil because
1360    // every caller is a create or a delete.
1361    #[inline(never)]
1362    pub(crate) fn account_for_allocation(&mut self) {
1363        if C::DYNAMIC_ALLOCATION {
1364            self.suspend_all();
1365            let _ = self.resume_all();
1366        }
1367    }
1368
1369    /// `taskENTER_CRITICAL()`.
1370    pub fn enter_critical(&mut self) {
1371        self.port.enter_critical();
1372        if C::NUMBER_OF_CORES > 1 {
1373            let core = self.core();
1374            if let Some(n) = self.nesting.get_mut(core) {
1375                *n = n.wrapping_add(1);
1376            }
1377        }
1378    }
1379
1380    /// `taskEXIT_CRITICAL()`.
1381    ///
1382    /// On the sim this is where time passes: the port raises a tick on
1383    /// every sixteenth outermost exit and the kernel takes it here, which
1384    /// is exactly where the C kernel's `SIGALRM` handler would have run.
1385    ///
1386    /// NOT in line, though it is three statements called 118,640 times in
1387    /// a 20,000-tick `StreamBufferDemo`. `#[inline(always)]` was measured:
1388    /// it saves that scenario 210,292 and costs `BlockQ` 200,912 and
1389    /// `GenQTest` 280,540, for a net loss of 271,962 over six scenarios.
1390    /// Whatever the hot sites gain, the switch-heavy ones lose more.
1391    ///
1392    /// # On two cores the outermost exit is a yield point
1393    ///
1394    /// SMP `vTaskExitCritical` reads `xYieldPendings[ xCoreID ]` as the
1395    /// nesting reaches zero and yields if it is set -- whoever set it. A yield
1396    /// left pending by an interrupt that had no woken pointer to report it
1397    /// through (`xEventGroupClearBitsFromISR`, or any `FromISR` call passed
1398    /// `NULL`) is taken at the next task-level exit on that core. Kairos took
1399    /// it only at the sites that had set it themselves, so the task the
1400    /// interrupt readied waited for some later switch: the API differential
1401    /// caught it (two cores, a `vTaskSuspend` four steps after the ISR).
1402    ///
1403    /// Sampled BEFORE the port exit, as the C samples before it re-enables
1404    /// interrupts, and taken after it. One core has no such rule -- its
1405    /// `vTaskExitCritical` never yields -- and compiles none of this.
1406    pub fn exit_critical(&mut self) {
1407        let owed = C::NUMBER_OF_CORES > 1 && self.leave_critical_smp();
1408        self.port.exit_critical();
1409        if self.port.take_pending_tick() {
1410            self.tick_on_exit();
1411        }
1412        if owed {
1413            self.port_yield();
1414        }
1415    }
1416
1417    /// Drop this core's nesting; `true` when that was the outermost exit and
1418    /// a yield is pending here. Not before the scheduler runs, and not in the
1419    /// tail of a frame a tick switched away from (`unwinding`): the C thread
1420    /// is stopped there. [`Kernel::port_yield`] never reaches this -- its
1421    /// section goes to the port directly.
1422    #[inline(always)]
1423    fn leave_critical_smp(&mut self) -> bool {
1424        let core = self.core();
1425        let Some(n) = self.nesting.get_mut(core) else {
1426            return false;
1427        };
1428        *n = n.saturating_sub(1);
1429        *n == 0 && self.running && self.unwinding.is_null() && self.pending_on(core)
1430    }
1431
1432    /// The tick as [`Kernel::exit_critical`] reaches it, deliberately out of
1433    /// line.
1434    ///
1435    /// `exit_critical` is inlined into every queue walker and every demo
1436    /// step, and the tick is its rare arm -- 8,000 of 48,000 sends in
1437    /// `khot-ir` raise one, and 38,013 ticks across 18 scenarios in
1438    /// `kernel-ir`. Left in line it put `increment_tick` and
1439    /// `switch_context` inside all of those frames.
1440    ///
1441    /// Taking it out is worth 233,207 on `kernel-ir` and costs the four
1442    /// kernel micro-instruments 9,650 between them, which is about what one
1443    /// extra call layer costs at their call counts.
1444    ///
1445    /// The other half was refuted: `#[inline(always)]` on
1446    /// [`Kernel::tick_from_isr`] itself costs `khot-ir` 178,245 while saving
1447    /// the three lighter instruments 82,642, and stacked on top of this
1448    /// outlining it still costs `khot-ir` 12,001, `ksched-ir` 7,555 and
1449    /// `kipc-ir` 2,000 for nothing `kernel-ir` did not already have. So the
1450    /// body keeps LLVM's own choice and only this call site is pinned.
1451    #[cold]
1452    #[inline(never)]
1453    fn tick_on_exit(&mut self) {
1454        self.tick_from_isr();
1455    }
1456
1457    /// `vPortSystemTickHandler`: the tick, from interrupt context.
1458    ///
1459    /// A silicon port calls this from its timer interrupt. The sim reaches
1460    /// it from [`Kernel::exit_critical`] -- through [`Kernel::tick_on_exit`],
1461    /// which is where the reason for this attribute is written down -- and
1462    /// from the idle hook.
1463    pub fn tick_from_isr(&mut self) {
1464        self.port.set_in_tick_entry(true);
1465        self.port.count_tick();
1466        // The C handler bumps `uxCriticalNesting` by hand and drops it the
1467        // same way — a raw `--`, never `vPortExitCritical` — so the bump is
1468        // invisible to the exit count and to anything that unwinds later.
1469        // Modelling it as a real critical section would invent an exit at
1470        // every tick that caused a switch.
1471        if self.increment_tick() {
1472            self.switch_context();
1473        }
1474        self.port.set_in_tick_entry(false);
1475    }
1476
1477    /// The idle hook's tick, `vPortKairosTick()` on the C side: a critical
1478    /// section around one unconditional tick. The enclosing section counts
1479    /// toward rule 3 exactly as the C one does.
1480    pub fn idle_hook_tick(&mut self) {
1481        self.enter_critical();
1482        self.tick_from_isr();
1483        self.exit_critical();
1484    }
1485
1486    // ---- tickless idle --------------------------------------------------
1487
1488    /// `tskIDLE_PRIORITY`.
1489    const IDLE_PRIORITY: u8 = 0;
1490
1491    /// `prvGetExpectedIdleTime`: how long nothing is due, in ticks.
1492    ///
1493    /// Zero whenever the answer would be unsafe to sleep on -- something
1494    /// above the idle priority is running, another idle-priority task is
1495    /// ready to share the slice, or a higher-priority task is ready. The
1496    /// last of those cannot happen with preemption on, and the C keeps the
1497    /// test anyway because a cooperative build reaches it.
1498    fn expected_idle_time(&self) -> u64 {
1499        // The C spells these as three arms so each carries its own reason;
1500        // they answer zero alike, so here they are one short-circuit chain
1501        // in the same order:
1502        //
1503        //  - something above the idle priority is running;
1504        //  - another idle-priority task is ready to share the slice, so the
1505        //    very next tick has to be processed;
1506        //  - a higher-priority task is ready, which only a cooperative
1507        //    build can reach and which the C tests for anyway.
1508        let due = self.current_priority() > Self::IDLE_PRIORITY
1509            || self.ready_len(Self::IDLE_PRIORITY).unwrap_or(0) > 1
1510            || self.top_ready_priority > Self::IDLE_PRIORITY;
1511
1512        if due {
1513            0
1514        } else {
1515            self.next_unblock_time.saturating_sub(self.tick)
1516        }
1517    }
1518
1519    /// `vTaskStepTick`: wind the tick count forward over a suppressed period.
1520    ///
1521    /// Note this does *not* call the tick hook for each stepped tick, which
1522    /// is the C's own note and the reason a tickless trace has no heartbeat
1523    /// in it.
1524    ///
1525    /// The C `configASSERT`s that the jump does not pass the next unblock
1526    /// time. This clamps instead: a port that overslept is a port bug, and
1527    /// winding the clock past a task's wake would lose the wake entirely,
1528    /// where losing the extra sleep is recoverable. `forbid(unsafe)` buys
1529    /// nothing if the kernel panics on a misbehaving port.
1530    ///
1531    /// Landing exactly on the next unblock time leaves the last tick pended
1532    /// rather than stepped, so `increment_tick` runs it when the scheduler
1533    /// resumes and the delayed task is woken by the same code that would
1534    /// have woken it.
1535    pub fn step_tick(&mut self, ticks_to_jump: u64) {
1536        let room = self.next_unblock_time.saturating_sub(self.tick);
1537        let mut jump = ticks_to_jump.min(room);
1538
1539        if self.tick.saturating_add(jump) == self.next_unblock_time && jump > 0 {
1540            // `taskENTER_CRITICAL(); xPendedTicks++; taskEXIT_CRITICAL();` --
1541            // a real section in the C, so a real exit here: on the sim the
1542            // clock can move at it, and on two cores it is a yield point.
1543            self.enter_critical();
1544            self.pended_ticks = self.pended_ticks.saturating_add(1);
1545            self.exit_critical();
1546            jump = jump.wrapping_sub(1);
1547        }
1548
1549        self.tick = self.tick.wrapping_add(jump) & Self::MAX_DELAY;
1550    }
1551
1552    /// The `configUSE_TICKLESS_IDLE` block of `prvIdleTask`.
1553    ///
1554    /// Compiled away entirely when the configuration leaves tickless idle
1555    /// off, which is the default and is why every existing scenario is
1556    /// byte-identical with this in the tree.
1557    ///
1558    /// The shape is the C's, including the part that looks redundant: the
1559    /// expected idle time is sampled once WITHOUT the scheduler suspended,
1560    /// and again with it. The first answer is not necessarily valid and the
1561    /// C takes it anyway, because suspending and resuming the scheduler on
1562    /// every pass of the idle task costs more than a sample that is
1563    /// sometimes wasted.
1564    pub fn idle_suppress_ticks(&mut self) {
1565        if !C::USE_TICKLESS_IDLE {
1566            return;
1567        }
1568
1569        if self.expected_idle_time() < C::EXPECTED_IDLE_TIME_BEFORE_SLEEP {
1570            return;
1571        }
1572
1573        self.suspend_all();
1574        let expected = self.expected_idle_time();
1575        if expected >= C::EXPECTED_IDLE_TIME_BEFORE_SLEEP {
1576            let tick = self.tick;
1577            self.note_exits();
1578            self.trace.event(tick, Event::LowPowerIdleBegin);
1579
1580            let slept = self.port.suppress_ticks_and_sleep(expected);
1581            self.step_tick(slept);
1582
1583            let tick = self.tick;
1584            self.note_exits();
1585            self.trace.event(tick, Event::LowPowerIdleEnd);
1586        }
1587        let _ = self.resume_all();
1588    }
1589
1590    /// `portYIELD()` as the Posix port spells it: a critical section around
1591    /// the context switch, so a tick raised during it lands on the way out.
1592    /// `#[cold]` because the call is GUARDED and reached from many sites on
1593    /// one hot path, which is the shape that pays: it lets LLVM keep the
1594    /// caller's frame setup out of the likely route. Measured on
1595    /// `riscv32-qemu-tick-work`, one attribute at a time.
1596    /// Worth queue -7, group -3 on its own.
1597    #[cold]
1598    pub(crate) fn port_yield(&mut self) {
1599        // Two cores, a committing port, inside a kernel critical section: the
1600        // C does not yield here, it PENDS -- `prvYieldCore` and
1601        // `vTaskYieldWithinAPI` both set `xYieldPendings[ core ]` while the
1602        // nesting is non-zero -- and the yield is the section's exit
1603        // (`exit_critical`). Yielding here as well counted a yield the C
1604        // never makes whenever the switch then declined, under a suspended
1605        // scheduler (two-core `StreamBufferDemo`).
1606        if C::NUMBER_OF_CORES > 1 && P::COMMITS_SWITCH {
1607            let core = self.core();
1608            if self.nesting.get(core).copied().unwrap_or(0) > 0 {
1609                self.set_pending_on(core, true);
1610                return;
1611            }
1612        }
1613        // The port's section, not the kernel's: on two cores the kernel's
1614        // outermost exit is a yield point, and a yield is not one of its
1615        // own -- on a port that commits its own switches the flag is still
1616        // set at this exit, so it would yield again for ever. On one core the
1617        // two are the same code.
1618        self.port.enter_critical();
1619        self.port.count_yield();
1620        if P::COMMITS_SWITCH {
1621            // A STACKED port. Raise its switching exception and leave
1622            // `current` where it is: the exception will call
1623            // `switch_context` itself, so the decision and the register
1624            // swap happen together and nothing runs in between as a task
1625            // the kernel has already moved on from.
1626            //
1627            // The exception cannot fire yet -- `exit_critical` below is
1628            // what unmasks -- so the kernel is consistent at the moment it
1629            // is taken.
1630            //
1631            // Two cores with the scheduler suspended: the C's yield is
1632            // counted and its switch DECLINES (`vTaskSwitchContext` re-pends
1633            // under `uxSchedulerSuspended`), so the yield that takes the
1634            // switch is the one the resume makes. Requesting it here would
1635            // hand a port that counts yields one request where the C made two
1636            // (two-core `StreamBufferDemo`: a send's completion notifying a
1637            // higher-priority reader inside the send's suspension).
1638            if !(C::NUMBER_OF_CORES > 1 && self.suspended_depth != 0) {
1639                self.port.yield_now();
1640            }
1641        } else {
1642            // A STACKLESS kernel: moving `current` IS the switch, because
1643            // no task owns a stack. This is the path every corpus
1644            // architecture takes, and it is unchanged.
1645            self.switch_context();
1646        }
1647        self.port.exit_critical();
1648        if self.port.take_pending_tick() {
1649            self.tick_on_exit();
1650        }
1651    }
1652
1653    /// `taskYIELD()` from a task body (the idle task's yield).
1654    pub fn task_yield(&mut self) {
1655        self.port_yield();
1656    }
1657
1658    // -------------------------------------------------------- task create --
1659
1660    /// `xTaskCreate`.
1661    ///
1662    /// Like the C, **the new task becomes current if its priority is at or
1663    /// above the running one's**. On a hosted firmware that is the sharp
1664    /// edge: the CPU is still on the caller's stack, so creating a
1665    /// higher-priority task hands `current` to something that is not
1666    /// running. See [`Kernel::start_scheduler`], which creates the timer
1667    /// daemon at [`Config::TIMER_TASK_PRIORITY`] for exactly this reason and
1668    /// documents the failure it produces — a clean run reporting zero laps.
1669    ///
1670    /// # Errors
1671    /// [`Error::Full`] when the task arena is full (the C
1672    /// `errCOULD_NOT_ALLOCATE_REQUIRED_MEMORY`).
1673    pub fn create_task(&mut self, name: &str, priority: u8) -> Result<TaskHandle> {
1674        let caller = self.cur();
1675        // configASSERT( uxPriority < configMAX_PRIORITIES ), then C clamps.
1676        let priority = priority.min(C::MAX_PRIORITIES.saturating_sub(1));
1677        // `prvCreateTask` takes the stack and then the TCB from the heap
1678        // before anything is initialised — `portSTACK_GROWTH` is negative on
1679        // the oracle's port, which fixes that order — and `pvPortMallocStack`
1680        // *is* `pvPortMalloc` with the MPU wrappers off. On heap_3 each one
1681        // suspends the scheduler, so a create costs two outermost exits.
1682        //
1683        // Every scenario before `death` created its tasks BEFORE
1684        // `vTaskStartScheduler`, and the sim port counts no exits there (the
1685        // C patch counts only on a FreeRTOS thread), so this cost was
1686        // invisible to the whole corpus. `death` is the first scenario that
1687        // creates a task with the scheduler running, and the first that can
1688        // tell the difference.
1689        self.account_for_allocation();
1690        self.account_for_allocation();
1691        let tcb = Tcb {
1692            name: Name::new(name, C::MAX_TASK_NAME_LEN),
1693            priority,
1694            base_priority: priority,
1695            mutexes_held: 0,
1696            wait: WaitFrame::default(),
1697            notified: [0; MAX_NOTIFICATION_ENTRIES],
1698            notify_state: [NotifyState::NotWaiting; MAX_NOTIFICATION_ENTRIES],
1699            notify_blocked: false,
1700            event_blocked: false,
1701            stream_resume: false,
1702            queue_resume: QueueResume::No,
1703            stream_timed: false,
1704            stream_waited: false,
1705            stream_local: 0,
1706            held: crate::timer::Split64::new(0),
1707            reap_seq: 0,
1708            _stride_pad: [0; STRIDE_PAD_WORDS],
1709        };
1710        let handle = match self.tcbs.try_insert(tcb) {
1711            Ok(h) => h,
1712            Err(_) => {
1713                let tick = self.tick;
1714                self.note_exits();
1715                self.trace.event(tick, Event::TaskCreateFailed);
1716                return Err(Error::Full);
1717            }
1718        };
1719        // A slot an idle task once held must not carry its mark into the
1720        // task that reuses it. SMP only: nothing else reads the bit.
1721        if C::NUMBER_OF_CORES > 1 {
1722            self.set_flag(handle.index() as usize, Self::F_IDLE, false);
1723        }
1724        // The event item sorts by `configMAX_PRIORITIES - uxPriority`, so a
1725        // higher-priority task waits nearer the head of an event list.
1726        let event_value = u64::from(C::MAX_PRIORITIES.saturating_sub(priority));
1727        self.lists
1728            .set_value(Self::event_item(handle), event_value)?;
1729        // `prvInitialiseNewTask` ends by calling `pxPortInitialiseStack`,
1730        // which on the oracle's Posix port wraps `pthread_create` in a
1731        // critical section — one more exit, and a PORT cost rather than a
1732        // kernel one, which is why it has a flag of its own.
1733        if C::PORT_STACK_INIT_CRITICAL {
1734            self.enter_critical();
1735            self.exit_critical();
1736        }
1737        // Any of the three exits above can release a tick that switches the
1738        // creator out. In the C the creator is a thread and simply STOPS
1739        // there: `prvAddNewTaskToReadyList` has not run, so the new task is
1740        // on no ready list and `traceTASK_CREATE` has not fired. Running it
1741        // now would put both events on the wrong side of the switch — which
1742        // is precisely how `death` first diverged.
1743        if self.cur() != caller {
1744            if let Some(slot) = self.owed_trace.get_mut(caller.index() as usize) {
1745                *slot = OwedTrace::AddNewTaskToReadyList {
1746                    task: handle,
1747                    priority,
1748                };
1749            }
1750            self.owe(caller);
1751            return Ok(handle);
1752        }
1753        self.add_new_task_to_ready_list(handle, priority)?;
1754        Ok(handle)
1755    }
1756
1757    /// `prvAddNewTaskToReadyList`.
1758    fn add_new_task_to_ready_list(&mut self, task: TaskHandle, priority: u8) -> Result<()> {
1759        self.enter_critical();
1760        {
1761            self.task_count = self.task_count.wrapping_add(1);
1762            if C::NUMBER_OF_CORES > 1 {
1763                // SMP never assigns `current` at creation: the idle tasks
1764                // hold every core from `start_scheduler` on.
1765            } else if self.cur().is_null() {
1766                self.set_current_at(task, priority);
1767            } else if !self.running {
1768                // `<=`, so the last-created task of the highest priority is
1769                // the one that runs first.
1770                if self.current_priority() <= priority {
1771                    self.set_current_at(task, priority);
1772                }
1773            }
1774            self.trace_task(task, |task, name| Event::TaskCreate {
1775                task,
1776                name,
1777                priority: Self::priority_value(priority),
1778            });
1779            self.add_task_to_ready_list(task)?;
1780        }
1781        self.exit_critical();
1782        if C::NUMBER_OF_CORES > 1 {
1783            // taskYIELD_ANY_CORE_IF_USING_PREEMPTION( pxNewTCB ): the new task
1784            // may displace this core's task or another core's.
1785            if self.running && C::USE_PREEMPTION {
1786                self.yield_for_task(task, priority);
1787                if self.pending_here() {
1788                    self.port_yield();
1789                }
1790            }
1791            return Ok(());
1792        }
1793        // taskYIELD_IF_USING_PREEMPTION(), outside the section.
1794        if self.running && C::USE_PREEMPTION && self.current_priority() < priority {
1795            self.port_yield();
1796        }
1797        Ok(())
1798    }
1799
1800    /// `prvAddTaskToReadyList`.
1801    /// Returns `Result<()>`, and that is a measured decision, not an
1802    /// oversight.
1803    ///
1804    /// Its two hot callers -- the tick's wake loop and the pending-ready
1805    /// drain -- resolve the SAME TCB again on the next line for the SAME
1806    /// field, to ask whether the woken task outranks the running one. Having
1807    /// this hand the priority back instead costs **+3.69% on kdelay-ir and
1808    /// +3.10% on ksched-ir**.
1809    ///
1810    /// Wrapping it -- narrow signature for the eleven callers that ignore the
1811    /// value, a wide inner for the two that want it -- does NOT help: that
1812    /// measured bit-for-bit identical to the unwrapped version on all four
1813    /// arms, because LLVM inlines through the wrapper and propagates the wide
1814    /// return anyway.
1815    ///
1816    /// The likely mechanism is the tail call. At `Result<()>` this function
1817    /// ends by handing back `insert_end`'s own result; at `Result<u8>` it has
1818    /// to capture that result, test it, and build a new one, at every site.
1819    pub(crate) fn add_task_to_ready_list(&mut self, task: TaskHandle) -> Result<()> {
1820        let priority = self.tcbs.resolve(task)?.priority;
1821        self.add_task_to_ready_list_at(task, priority)
1822    }
1823
1824    /// [`Kernel::add_task_to_ready_list`] for a caller that already holds
1825    /// `task`'s priority. `priority` MUST be the TCB's own.
1826    ///
1827    /// The priority goes IN, not out: handing it back measured +3.69% on
1828    /// kdelay-ir (the doc above). `remove_from_event_list` resolves the woken
1829    /// TCB right after `handle_at` read the same slot, where the two fold, and
1830    /// passes the field down instead of resolving twice more around the list
1831    /// edits, which LLVM cannot see past.
1832    ///
1833    /// In line on the speed profile only: at that site it is +30 B of flash
1834    /// for -6 on rv32 `block_cycle`, and `small` does not make that trade.
1835    #[cfg_attr(not(feature = "small"), inline(always))]
1836    #[cfg_attr(feature = "small", inline(never))]
1837    fn add_task_to_ready_list_at(&mut self, task: TaskHandle, priority: u8) -> Result<()> {
1838        self.trace_task(task, |task, name| Event::MovedTaskToReadyState {
1839            task,
1840            name,
1841        });
1842        // taskRECORD_READY_PRIORITY
1843        if priority > self.top_ready_priority {
1844            self.top_ready_priority = priority;
1845        }
1846        self.lists
1847            .insert_end(Self::ready_list(priority), Self::state_item(task))
1848    }
1849
1850    // ---------------------------------------------------- scheduler start --
1851
1852    /// `vTaskStartScheduler`, minus the part that never returns.
1853    ///
1854    /// Creates the idle task, the timer command queue and the timer service
1855    /// task, fires `TASK_SWITCHED_IN` for whichever task will run first and
1856    /// then `STARTING_SCHEDULER`, and returns the handles so the runner can
1857    /// attach their bodies.
1858    ///
1859    /// # ★ The timer daemon is created at [`Config::TIMER_TASK_PRIORITY`],
1860    /// and it will outrank a task you created below it
1861    ///
1862    /// This is the C's behaviour and it is deliberate, but on a hosted
1863    /// firmware it has a failure mode that looks like nothing at all.
1864    ///
1865    /// `Tmr Svc` is created here **unconditionally**, and
1866    /// [`Kernel::create_task`] makes the highest-priority task current. So a
1867    /// firmware whose own `main` runs below `TIMER_TASK_PRIORITY` leaves the
1868    /// kernel believing a **stackless** task is running: `current` names the
1869    /// daemon while the CPU is on `main`'s stack. Every
1870    /// [`Kernel::switch_context`] then declines as `from == to`, the workers
1871    /// never start, and the symptom is a clean run reporting **zero laps**
1872    /// rather than a fault.
1873    ///
1874    /// Nothing here can detect it — a stackless kernel cannot see whose stack
1875    /// the CPU is actually on — so it is a caller's invariant:
1876    ///
1877    /// > **Give the task that owns the CPU a priority at or above
1878    /// > [`Config::TIMER_TASK_PRIORITY`]**, or lower `TIMER_TASK_PRIORITY`
1879    /// > below it.
1880    ///
1881    /// Reported from the Janus side, where it cost a board run
1882    /// (`docs/plans/janus-rtos.md` §5b), and seen again in the radio cell.
1883    ///
1884    /// # Errors
1885    /// As [`Kernel::create_task`].
1886    pub fn start_scheduler(&mut self) -> Result<StartHandles> {
1887        if C::NUMBER_OF_CORES > 1 {
1888            return self.start_scheduler_smp();
1889        }
1890        let idle = self.create_task("IDLE", 0)?;
1891        // `xTimerCreateTimerTask` opens with `prvCheckForValidListAndQueue`,
1892        // which makes the command queue only if no `xTimerCreate` has made
1893        // it already.
1894        self.check_for_valid_list_and_queue()?;
1895        let timer_queue = self.timer_queue;
1896        let timer = self.create_task("Tmr Svc", C::TIMER_TASK_PRIORITY)?;
1897        self.next_unblock_time = Self::MAX_DELAY;
1898        self.running = true;
1899        self.tick = C::INITIAL_TICK_COUNT;
1900        self.port.scheduler_started();
1901        // `vPortStartFirstTask` runs the first task through the same
1902        // first-start path as every other.
1903        let current = self.cur();
1904        self.note_first_start(current);
1905        self.trace_task(current, |task, name| Event::TaskSwitchedIn { task, name });
1906        let tick = self.tick;
1907        self.note_exits();
1908        self.trace.event(tick, Event::StartingScheduler);
1909        Ok(StartHandles {
1910            idle,
1911            passive_idle: TaskHandle::NULL,
1912            timer,
1913            timer_queue,
1914        })
1915    }
1916
1917    /// `vTaskStartScheduler` with `configNUMBER_OF_CORES > 1`.
1918    ///
1919    /// `prvCreateIdleTasks` makes one idle task per core -- `IDLE0` runs
1920    /// `prvIdleTask`, the rest `prvPassiveIdleTask` -- and makes each the
1921    /// CURRENT task of its core, which is why SMP task creation never
1922    /// touches `current`: before the scheduler starts no core is assigned,
1923    /// and after it every core is running something. The first switch on
1924    /// each core then picks the highest ready task.
1925    #[cold]
1926    #[inline(never)]
1927    fn start_scheduler_smp(&mut self) -> Result<StartHandles> {
1928        let cores = Self::cores();
1929        for core in 0..cores {
1930            let name = if core == 0 { "IDLE0" } else { "IDLE1" };
1931            let idle = self.create_task(name, Self::IDLE_PRIORITY)?;
1932            if let Some(slot) = self.idle.get_mut(core) {
1933                *slot = idle;
1934            }
1935            self.set_flag(idle.index() as usize, Self::F_IDLE, true);
1936            self.set_current_of(core, idle, Self::IDLE_PRIORITY);
1937        }
1938        self.check_for_valid_list_and_queue()?;
1939        let timer_queue = self.timer_queue;
1940        let timer = self.create_task("Tmr Svc", C::TIMER_TASK_PRIORITY)?;
1941        self.next_unblock_time = Self::MAX_DELAY;
1942        self.running = true;
1943        self.tick = C::INITIAL_TICK_COUNT;
1944        self.port.scheduler_started();
1945        for core in 0..cores {
1946            let idle = self.current_of(core);
1947            self.note_first_start(idle);
1948        }
1949        let current = self.cur();
1950        self.trace_task(current, |task, name| Event::TaskSwitchedIn { task, name });
1951        let tick = self.tick;
1952        self.note_exits();
1953        self.trace.event(tick, Event::StartingScheduler);
1954        Ok(StartHandles {
1955            idle: self.idle_of(0),
1956            passive_idle: if cores > 1 {
1957                self.idle_of(1)
1958            } else {
1959                TaskHandle::NULL
1960            },
1961            timer,
1962            timer_queue,
1963        })
1964    }
1965
1966    // ------------------------------------------------------------------- SMP --
1967
1968    /// `configNUMBER_OF_CORES`, clamped to what this kernel can hold.
1969    #[inline(always)]
1970    const fn cores() -> usize {
1971        let n = C::NUMBER_OF_CORES as usize;
1972        if n < 1 {
1973            1
1974        } else if n > MAX_CORES {
1975            MAX_CORES
1976        } else {
1977            n
1978        }
1979    }
1980
1981    /// `pxCurrentTCBs[ core ]`, or null for a core out of range.
1982    #[inline(always)]
1983    fn current_of(&self, core: usize) -> TaskHandle {
1984        self.current.get(core).copied().unwrap_or(TaskHandle::NULL)
1985    }
1986
1987    /// The cached running priority of `core`.
1988    #[inline(always)]
1989    fn priority_on(&self, core: usize) -> u8 {
1990        self.current_priority.get(core).copied().unwrap_or(0)
1991    }
1992
1993    /// Make `task` (of `priority`) the running task of `core`.
1994    fn set_current_of(&mut self, core: usize, task: TaskHandle, priority: u8) {
1995        if let Some(slot) = self.current.get_mut(core) {
1996            *slot = task;
1997        }
1998        if let Some(slot) = self.current_priority.get_mut(core) {
1999            *slot = priority;
2000        }
2001    }
2002
2003    fn idle_of(&self, core: usize) -> TaskHandle {
2004        self.idle.get(core).copied().unwrap_or(TaskHandle::NULL)
2005    }
2006
2007    fn pending_on(&self, core: usize) -> bool {
2008        self.yield_pending.get(core).copied().unwrap_or(false)
2009    }
2010
2011    fn set_pending_on(&mut self, core: usize, value: bool) {
2012        if let Some(slot) = self.yield_pending.get_mut(core) {
2013            *slot = value;
2014        }
2015    }
2016
2017    fn requested_on(&self, core: usize) -> bool {
2018        self.yield_requested.get(core).copied().unwrap_or(false)
2019    }
2020
2021    fn set_requested_on(&mut self, core: usize, value: bool) {
2022        if let Some(slot) = self.yield_requested.get_mut(core) {
2023            *slot = value;
2024        }
2025    }
2026
2027    /// `taskTASK_IS_RUNNING( pxTCB )`: the core `task` is running on, if any,
2028    /// NOT counting a core that has been asked to yield it.
2029    fn running_core(&self, task: TaskHandle) -> Option<usize> {
2030        (0..Self::cores()).find(|&c| self.current_of(c) == task && !self.requested_on(c))
2031    }
2032
2033    /// `xTaskRunState != taskTASK_NOT_RUNNING`: whether ANY core holds `task`,
2034    /// including one asked to yield it -- which is what selection must skip.
2035    fn held_by_any_core(&self, task: TaskHandle) -> bool {
2036        (0..Self::cores()).any(|c| self.current_of(c) == task)
2037    }
2038
2039    /// `prvYieldCore( xCoreID )`.
2040    ///
2041    /// The calling core only records the yield -- it is inside a critical
2042    /// section and acts on it on the way out. Another core is asked once:
2043    /// its bit goes into [`Kernel::take_core_yields`] for the port to raise,
2044    /// and its task is marked `taskTASK_SCHEDULED_TO_YIELD` so a second
2045    /// request does not interrupt it twice.
2046    fn yield_core(&mut self, core: usize) {
2047        if core >= Self::cores() {
2048            return;
2049        }
2050        if core == self.core() {
2051            self.set_pending_on(core, true);
2052        } else if !self.requested_on(core) {
2053            self.core_yields |= 1u8.checked_shl(core as u32).unwrap_or(0);
2054            self.set_requested_on(core, true);
2055        }
2056    }
2057
2058    /// `prvYieldForTask( pxTCB )`, for `configRUN_MULTIPLE_PRIORITIES == 1`
2059    /// and no core affinity.
2060    ///
2061    /// `task` has just become ready. Find the core running the LOWEST
2062    /// priority task that `task` outranks -- a running idle task counts as
2063    /// one below priority 0, so a real priority-0 task displaces an idle
2064    /// one -- and yield it. Ties go to the HIGHER core number, because the
2065    /// C walks upwards with `<=`. A core already owing a yield, or already
2066    /// asked for one, is not a candidate.
2067    #[cold]
2068    #[inline(never)]
2069    fn yield_for_task(&mut self, task: TaskHandle, priority: u8) {
2070        // `taskTASK_IS_RUNNING`, by slot index. Before the scheduler starts
2071        // every core's current task is null (slot 0), so no core can be
2072        // running `task` and the test is skipped; after it, a core's current
2073        // task and the task just readied are both live, and a slot holds one
2074        // live task at a time -- so the index IS the handle.
2075        let slot = task.index();
2076        if self.running
2077            && (0..Self::cores())
2078                .any(|c| self.current_of(c).index() == slot && !self.requested_on(c))
2079        {
2080            return;
2081        }
2082        let mut lowest = i16::from(priority).saturating_sub(1);
2083        let mut lowest_core = None;
2084        for core in 0..Self::cores() {
2085            // The two flags first: a core that already owes or was asked for
2086            // a yield is no candidate.
2087            if self.requested_on(core) || self.pending_on(core) {
2088                continue;
2089            }
2090            let running = self.current_of(core).index();
2091            let mut p = i16::from(self.priority_on(core));
2092            // `taskATTRIBUTE_IS_IDLE`, as the C keeps it: one bit of the
2093            // running task's own flags, where comparing against the idle
2094            // tasks' handles rebuilt their slots on every call. Before the
2095            // scheduler starts no task carries the bit, so a core's null
2096            // current (slot 0) answers "not idle", as the handle test did.
2097            if self.flag(running as usize, Self::F_IDLE, false) {
2098                p = p.saturating_sub(1);
2099            }
2100            if p <= lowest {
2101                lowest = p;
2102                lowest_core = Some(core);
2103            }
2104        }
2105        if let Some(core) = lowest_core {
2106            self.yield_core(core);
2107        }
2108    }
2109
2110    /// SMP: `task` (now at `priority`) has just become ready, so run
2111    /// `prvYieldForTask` -- `taskYIELD_ANY_CORE_IF_USING_PREEMPTION` -- and
2112    /// answer whether THIS core now owes a yield.
2113    ///
2114    /// Every one-core "did the woken task outrank the running one?" test has
2115    /// this as its SMP arm.
2116    ///
2117    /// In line: the body is one call (`yield_for_task`, out of line itself)
2118    /// and a flag read, so out of line it was a call to make a call. The
2119    /// MIR-inliner risk that keeps the SMP tick bodies out of line was
2120    /// measured here and is absent: `bench/kernel-ir`'s one-core kernel rows
2121    /// are unchanged.
2122    #[inline(always)]
2123    fn smp_readied(&mut self, task: TaskHandle, priority: u8) -> bool {
2124        if C::USE_PREEMPTION {
2125            self.yield_for_task(task, priority);
2126        }
2127        self.pending_here()
2128    }
2129
2130    /// The core holding `task` -- running it, or asked to yield it.
2131    fn held_core(&self, task: TaskHandle) -> Option<usize> {
2132        (0..Self::cores()).find(|&c| self.current_of(c) == task)
2133    }
2134
2135    /// The OTHER cores this kernel has asked to yield since the last call,
2136    /// one bit per core, cleared by the read.
2137    ///
2138    /// A port calls this on its way out of the kernel lock and raises the
2139    /// inter-processor interrupt of each core named. Always zero on one core.
2140    pub fn take_core_yields(&mut self) -> u8 {
2141        core::mem::take(&mut self.core_yields)
2142    }
2143
2144    /// `prvSelectHighestPriorityTask( xCoreID )` for
2145    /// `configRUN_MULTIPLE_PRIORITIES == 1` and no core affinity.
2146    ///
2147    /// Answers whether a task was scheduled. Read from the pinned `tasks.c`
2148    /// (V11.3.1), not remembered:
2149    ///
2150    /// 1. The core's current task, if it is still in its ready list, goes to
2151    ///    the END of that list first -- so a task that yields is not picked
2152    ///    again ahead of tasks that have waited longer.
2153    /// 2. From `uxTopReadyPriority` down, each ready list is walked from its
2154    ///    HEAD (not the round-robin cursor): the first task no core holds is
2155    ///    swapped in; this core's own current task may be kept; a task held
2156    ///    by another core is skipped.
2157    /// 3. `uxTopReadyPriority` comes down only past levels that are EMPTY.
2158    ///
2159    /// In line: its one caller is [`Kernel::switch_context_smp`], itself the
2160    /// out-of-line SMP body, so this costs no flash and saves a call, a
2161    /// return and two sets of saved registers on every switch.
2162    #[inline(always)]
2163    fn select_for_core(&mut self, core: usize) -> bool {
2164        let current = self.current_of(core);
2165        let current_list = Self::ready_list(self.priority_on(core));
2166        // `if listIS_CONTAINED_WITHIN(...) { uxListRemove; vListInsertEnd }`,
2167        // as one list operation that does nothing when the task is not in its
2168        // ready list.
2169        let _ = self
2170            .lists
2171            .move_to_end(current_list, Self::state_item(current));
2172        // Which state items the cores hold, by item rather than by handle: a
2173        // ready list holds only live tasks and a slot holds one live task at a
2174        // time, so the item IS the task, and the walk can skip a held one
2175        // without resolving it through the arena. Only the task chosen is
2176        // resolved. A core holding nothing names an item no list can contain.
2177        //
2178        // Once the scheduler runs, no core's current task is null: the idle
2179        // tasks take every core before `running` is set, and only a pick ever
2180        // replaces one. So the null test -- a compare and a select per core,
2181        // on every switch -- is for a switch made before the start.
2182        let held: [ItemId; MAX_CORES] = if self.running {
2183            core::array::from_fn(|c| {
2184                if c < Self::cores() {
2185                    Self::state_item(self.current_of(c))
2186                } else {
2187                    ItemId::MAX
2188                }
2189            })
2190        } else {
2191            core::array::from_fn(|c| {
2192                let task = self.current_of(c);
2193                if c < Self::cores() && !task.is_null() {
2194                    Self::state_item(task)
2195                } else {
2196                    ItemId::MAX
2197                }
2198            })
2199        };
2200        let mine = held.get(core).copied().unwrap_or(ItemId::MAX);
2201        let mut priority = self.top_ready_priority;
2202        let mut decrement_top = true;
2203        loop {
2204            let list = Self::ready_list(priority);
2205            if !self.lists.is_empty_of(list) {
2206                decrement_top = false;
2207                // The list's own iterator follows the links with none of the
2208                // per-step container and bounds checks `head`/`next` make. The
2209                // first item no other core holds wins: a free task (swapped
2210                // in) or this core's own (kept).
2211                //
2212                // A `for` with a `break`, not `find_map`: handing the pick
2213                // back through the closure's nested `Option` cost one register
2214                // copy of the chosen slot on every switch that swaps a task
2215                // in (`mov %r11d,%r12d`, 53,176 times in semtest).
2216                let mut pick = None;
2217                for item in self.lists.iter(list) {
2218                    if !held.contains(&item) {
2219                        if let Some(task) = self.tcbs.handle_at(item) {
2220                            pick = Some(Some(task));
2221                            break;
2222                        }
2223                    } else if item == mine {
2224                        pick = Some(None);
2225                        break;
2226                    }
2227                }
2228                match pick {
2229                    Some(Some(task)) => {
2230                        self.set_requested_on(core, false);
2231                        self.set_current_of(core, task, priority);
2232                        return true;
2233                    }
2234                    Some(None) => {
2235                        self.set_requested_on(core, false);
2236                        return true;
2237                    }
2238                    None => {}
2239                }
2240            }
2241            if decrement_top {
2242                self.top_ready_priority = self.top_ready_priority.saturating_sub(1);
2243            }
2244            if priority == 0 {
2245                return false;
2246            }
2247            priority = priority.saturating_sub(1);
2248        }
2249    }
2250
2251    /// `xTaskIncrementTick`'s SMP time slicing: EVERY core whose running
2252    /// priority has a second ready task owes a yield (`xYieldPendings`).
2253    ///
2254    /// Out of line, like every SMP body: rustc's MIR inliner weighs a
2255    /// function before monomorphisation folds `C::NUMBER_OF_CORES`, so an
2256    /// SMP loop written inline in `increment_tick` changes how a ONE-core
2257    /// build inlines the tick even though it never runs there.
2258    #[cold]
2259    #[inline(never)]
2260    fn tick_time_slice_smp(&mut self) {
2261        if C::USE_PREEMPTION && C::USE_TIME_SLICING {
2262            for core in 0..Self::cores() {
2263                let list = Self::ready_list(self.priority_on(core));
2264                if self.lists.len(list).unwrap_or(0) > 1 {
2265                    self.set_pending_on(core, true);
2266                }
2267            }
2268        }
2269    }
2270
2271    /// `xTaskIncrementTick`'s SMP preemption: this core switches on the way
2272    /// out (answered `true`); every other core that owes a yield is
2273    /// interrupted (`prvYieldCore`).
2274    #[cold]
2275    #[inline(never)]
2276    fn tick_yields_smp(&mut self) -> bool {
2277        let mut here = false;
2278        if C::USE_PREEMPTION {
2279            let me = self.core();
2280            for core in 0..Self::cores() {
2281                if self.pending_on(core) {
2282                    if core == me {
2283                        here = true;
2284                    } else {
2285                        self.yield_core(core);
2286                    }
2287                }
2288            }
2289        }
2290        here
2291    }
2292
2293    /// `vTaskSwitchContext( xCoreID )`, for the calling core.
2294    #[cold]
2295    #[inline(never)]
2296    fn switch_context_smp(&mut self) {
2297        let core = self.core();
2298        if self.suspended_depth != 0 {
2299            self.set_pending_on(core, true);
2300            return;
2301        }
2302        self.set_pending_on(core, false);
2303        let outgoing = self.current_of(core);
2304        self.trace_task(outgoing, |task, name| Event::TaskSwitchedOut { task, name });
2305        if !self.select_for_core(core) {
2306            self.note_stall(Stall::NoReadyTask);
2307            return;
2308        }
2309        let incoming = self.current_of(core);
2310        if incoming.index() != outgoing.index() {
2311            self.hand_over(incoming);
2312        }
2313        self.trace_task(incoming, |task, name| Event::TaskSwitchedIn { task, name });
2314    }
2315
2316    // -------------------------------------------------------- the switch --
2317
2318    /// `vTaskSwitchContext`.
2319    pub fn switch_context(&mut self) {
2320        if C::NUMBER_OF_CORES > 1 {
2321            return self.switch_context_smp();
2322        }
2323        if self.suspended_depth != 0 {
2324            self.set_pending_here(true);
2325            return;
2326        }
2327        self.set_pending_here(false);
2328        let current = self.cur();
2329        self.trace_task(current, |task, name| Event::TaskSwitchedOut { task, name });
2330        // taskSELECT_HIGHEST_PRIORITY_TASK
2331        //
2332        // ONE call per priority level, not two. The search used to ask
2333        // `is_empty` — a bounded read of the list's metadata — and then hand
2334        // the winning level to `next_round_robin`, which reads that same
2335        // metadata again to find the cursor. `next_round_robin` already
2336        // answers `None` for an empty list, and does so BEFORE it touches the
2337        // cursor, so it is its own emptiness probe and the level that loses
2338        // is left exactly as it was found.
2339        let mut top = self.top_ready_priority;
2340        // ★ `Err` is folded into "that level is empty" below, and the fold IS
2341        // the win: with three arms the walk carried a packed
2342        // `Result<Option<ItemId>>` across the back edge -- a `setb`/`shl`/`or`
2343        // to build it and a `test`/`jne` to take it apart, on every one of the
2344        // 1.58 levels a call walks. Two arms took this function from 168.0 to
2345        // 157.7 Ir per call, -937,965 on `bench/kernel-ir`, all of it here.
2346        //
2347        // But an out-of-range list id is a different fault from an empty level,
2348        // and the fold alone would have reported it as `NoReadyTask`. So it is
2349        // proved HERE instead, once, where it can still name itself:
2350        // `ready_list` is the identity, so the only way the walk can index out
2351        // of range is a `top_ready_priority` above the configured ceiling.
2352        // That is the diagnostic bought back, and the cost of buying it is in
2353        // the commit message beside the fold's.
2354        if top >= C::MAX_PRIORITIES {
2355            self.note_stall(Stall::ListError);
2356            return;
2357        }
2358        let item = loop {
2359            match self.lists.next_round_robin(Self::ready_list(top)) {
2360                Ok(Some(item)) => break item,
2361                Ok(None) | Err(_) => {
2362                    if top == 0 {
2363                        // The C `configASSERT( uxTopPriority )`. The idle
2364                        // task keeps priority 0 non-empty, so reaching here
2365                        // means it is gone. The kernel keeps the current
2366                        // task rather than indexing out of range -- and now
2367                        // SAYS so, because carrying on quietly is how this
2368                        // costs an afternoon on a board.
2369                        self.note_stall(Stall::NoReadyTask);
2370                        return;
2371                    }
2372                    // At least 1: the arm above returns at zero.
2373                    top = top.wrapping_sub(1);
2374                }
2375            }
2376        };
2377        let Ok(next) = self.task_of_state_item(item) else {
2378            self.note_stall(Stall::UnknownTask);
2379            return;
2380        };
2381        self.top_ready_priority = top;
2382        // INDICES, and `hand_over` takes only the incoming handle: together
2383        // those two delete the load of `current`'s generation word entirely.
2384        // Either alone leaves it live -- the full-handle `!=` compares both
2385        // halves, and a `hand_over` taking the outgoing handle needs both -- so
2386        // the index compare measured 0 on its own. `rusty-compiler-leverage`
2387        // A2: price the SET, not the halves.
2388        //
2389        // Comparing indices is sound because both handles are LIVE: `next` came
2390        // out of a slot whose generation was just matched, and `current` names
2391        // the running task. The arena issues one live handle per slot at a time,
2392        // so two live handles sharing an index share a generation. That is the
2393        // same argument `Handle::is_null` makes in the other direction.
2394        if next.index() != current.index() {
2395            self.hand_over(next);
2396        }
2397        // `top` is `next`'s priority by construction: the loop above found
2398        // the highest non-empty ready list and `next` came out of it.
2399        self.set_current_at(next, top);
2400        self.trace_task(next, |task, name| Event::TaskSwitchedIn { task, name });
2401    }
2402
2403    // ----------------------------------------------------------- the tick --
2404
2405    /// Pay whatever the running task still owes from a call it was
2406    /// preempted inside. The runner calls this before stepping a body;
2407    /// `true` means work was done and the current task may have changed
2408    /// again, so nothing else should be assumed.
2409    #[inline(always)]
2410    pub fn resume_pending(&mut self) -> bool {
2411        // The fast path, in line at the runner's one call site.
2412        //
2413        // `Runner::step_once` asks this before every step -- 214,488 times
2414        // in a 20,000-tick `StreamBufferDemo` -- and the answer is "nothing
2415        // owed" almost every time. Two loads and two branches decide it,
2416        // and out of line they cost a call and a frame as well.
2417        //
2418        // The two tests are the entry conditions of the two things the
2419        // body does: `settle_unwind` returns at once unless `unwinding` is
2420        // set, and the rest returns at once unless this task owes
2421        // something. Neither is duplicated -- the cold half still runs
2422        // both in full.
2423        // On a `COMMITS_SWITCH` port `unwinding` is never set (see `hand_over`),
2424        // and saying so with the const lets LLVM drop the load and the branch.
2425        if P::COMMITS_SWITCH || self.unwinding.is_null() {
2426            // Nothing to settle, which is the shape of almost every call.
2427            // `settle_unwind` would test `unwinding` and return at once, and
2428            // the cold half would then re-read the very byte this line reads
2429            // — so on this shape both are skipped and the owed body is
2430            // entered directly.
2431            let index = self.cur().index() as usize;
2432            // B1, the shape win 7 established: `self.cur()` is read RAW here -- no
2433            // bound-checking accessor between the field and the use -- and `index` then
2434            // feeds `owes_anything.get(index)`. Contrast `remove_from_event_list`, where
2435            // the same proof measured EXACTLY +0 because its handle comes back from
2436            // `handle_at`, which bound-checks, so the fact was already local.
2437            if index >= TASKS {
2438                return false;
2439            }
2440            if !self.owes_anything.get(index).copied().unwrap_or(true) {
2441                return false;
2442            }
2443            return self.resume_pending_owed(index);
2444        }
2445        self.resume_pending_cold()
2446    }
2447
2448    /// Everything [`Kernel::resume_pending`] does when something IS owed.
2449    ///
2450    /// Out of line on purpose: it is reached on a small fraction of steps,
2451    /// and inlining it would put the whole `OwedTrace` match at the hot
2452    /// site. Same body/symbol split the trace sink's `num` measured.
2453    #[inline(never)]
2454    fn resume_pending_cold(&mut self) -> bool {
2455        self.settle_unwind();
2456        let index = self.cur().index() as usize;
2457        // B1, the shape win 7 established: `self.cur()` is read RAW here -- no
2458        // bound-checking accessor between the field and the use -- and `index` then
2459        // feeds `owes_anything.get(index)`. Contrast `remove_from_event_list`, where
2460        // the same proof measured EXACTLY +0 because its handle comes back from
2461        // `handle_at`, which bound-checks, so the fact was already local.
2462        if index >= TASKS {
2463            return false;
2464        }
2465
2466        // One load and one test for the overwhelmingly common case. The
2467        // combined check below is the fallback, and it is what clears the hint
2468        // -- so a hint left standing costs one slow path and then goes away.
2469        //
2470        // This read cannot be threaded in from `resume_pending`: settling an
2471        // unwind calls `owe` itself, so the answer here is not always the
2472        // answer there. That is exactly why the no-unwind shape gets its own
2473        // entry rather than this one being made cheaper.
2474        if !self.owes_anything.get(index).copied().unwrap_or(true) {
2475            return false;
2476        }
2477        self.resume_pending_owed(index)
2478    }
2479
2480    /// Everything owed to the current task, once it is known that something
2481    /// is.
2482    ///
2483    /// Its own symbol so that both shapes above can reach it without either
2484    /// paying for the other's entry conditions.
2485    #[inline(never)]
2486    fn resume_pending_owed(&mut self, index: usize) -> bool {
2487        // First the stack unwinds — the sections the task had open when it
2488        // was switched out, and whatever its abandoned frame opened after
2489        // that — and only then does the statement after the yield run.
2490        // Reversing the two moves every tick.
2491        //
2492        // HERE and not in the inner body, because this is the shape of
2493        // nearly every call (BlockQ: 43,104 of 43,104) and it needs three
2494        // saved registers, where the inner body's `OwedTrace` match needs
2495        // six and a 0x68-byte frame. In one symbol every call paid for the
2496        // match's frame.
2497        let owed = self.owed_exits.get(index).copied().unwrap_or(0);
2498        if owed > 0 {
2499            if let Some(slot) = self.owed_exits.get_mut(index) {
2500                *slot = 0;
2501            }
2502            // One counted exit each: the tally is already in outermost
2503            // exits, so replaying it as nesting would lose the sections the
2504            // tail opened and closed on its own.
2505            for _ in 0..owed {
2506                self.enter_critical();
2507                self.exit_critical();
2508                // A replayed exit can release a tick that switches this
2509                // task straight back out. The rest of the loop is then the
2510                // tail of *this* frame, which the port tallies and the next
2511                // resume pays — so there is nothing to unwind by hand.
2512            }
2513            self.clear_owe_if_settled(index);
2514            return true;
2515        }
2516        let did = self.resume_pending_owed_inner(index);
2517        if did {
2518            // Every branch below that did work returned with the hint still
2519            // standing, so the runner came straight back and spent a second
2520            // entry re-deriving "nothing owed". Clearing it here covers all
2521            // of them at one site.
2522            self.clear_owe_if_settled(index);
2523        }
2524        did
2525    }
2526
2527    /// The body of [`Kernel::resume_pending_owed`] once no exits are owed.
2528    #[inline(never)]
2529    fn resume_pending_owed_inner(&mut self, index: usize) -> bool {
2530        // The caller has replayed any owed exits, so `owed_exits` is zero
2531        // here. Of what is left, almost every call answers "nothing owed":
2532        // a match over `OwedTrace`, then a compare, the same two loads with
2533        // one branch. No `.copied()` on `owed_trace`: that materialises an
2534        // `Option<OwedTrace>`, and `OwedTrace` is forty bytes wide because of
2535        // `TimerCommandSend`'s `Name`. Matching through the reference tests
2536        // the discriminant where the copy moved the whole variant to test it.
2537        if matches!(self.owed_trace.get(index), Some(OwedTrace::None) | None)
2538            && !self.flag(index, Self::F_YIELD, false)
2539        {
2540            if let Some(f) = self.owes_anything.get_mut(index) {
2541                *f = false;
2542            }
2543            return false;
2544        }
2545        // Then the line the abandoned frame had not reached yet.
2546        match self.owed_trace.get(index).copied() {
2547            Some(OwedTrace::SendFailed(queue)) => {
2548                if let Some(slot) = self.owed_trace.get_mut(index) {
2549                    *slot = crate::kernel::OwedTrace::None;
2550                }
2551                let tick = self.tick;
2552                self.note_exits();
2553                self.trace
2554                    .event(tick, Event::QueueSendFailed { queue, name: "" });
2555                return true;
2556            }
2557            Some(OwedTrace::ReceiveFailed(queue)) => {
2558                if let Some(slot) = self.owed_trace.get_mut(index) {
2559                    *slot = crate::kernel::OwedTrace::None;
2560                }
2561                let tick = self.tick;
2562                self.note_exits();
2563                self.trace
2564                    .event(tick, Event::QueueReceiveFailed { queue, name: "" });
2565                return true;
2566            }
2567            Some(OwedTrace::EventGroupWaitBitsEnd {
2568                group,
2569                bits,
2570                timed_out,
2571            }) => {
2572                if let Some(slot) = self.owed_trace.get_mut(index) {
2573                    *slot = crate::kernel::OwedTrace::None;
2574                }
2575                let tick = self.tick;
2576                self.note_exits();
2577                self.trace.event(
2578                    tick,
2579                    Event::EventGroupWaitBitsEnd {
2580                        group,
2581                        bits,
2582                        timed_out,
2583                    },
2584                );
2585                return true;
2586            }
2587            Some(OwedTrace::StreamBufferCreate {
2588                buffer,
2589                is_message_buffer,
2590            }) => {
2591                if let Some(slot) = self.owed_trace.get_mut(index) {
2592                    *slot = crate::kernel::OwedTrace::None;
2593                }
2594                let tick = self.tick;
2595                self.note_exits();
2596                self.trace.event(
2597                    tick,
2598                    Event::StreamBufferCreate {
2599                        buffer,
2600                        is_message_buffer,
2601                    },
2602                );
2603                return true;
2604            }
2605            Some(OwedTrace::TimerCommandSend {
2606                timer,
2607                name,
2608                command,
2609                value,
2610            }) => {
2611                if let Some(slot) = self.owed_trace.get_mut(index) {
2612                    *slot = crate::kernel::OwedTrace::None;
2613                }
2614                let tick = self.tick;
2615                self.note_exits();
2616                self.trace.event(
2617                    tick,
2618                    Event::TimerCommandSend {
2619                        timer,
2620                        name: name.as_str(),
2621                        command,
2622                        value,
2623                    },
2624                );
2625                return true;
2626            }
2627            Some(OwedTrace::AddNewTaskToReadyList { task, priority }) => {
2628                if let Some(slot) = self.owed_trace.get_mut(index) {
2629                    *slot = crate::kernel::OwedTrace::None;
2630                }
2631                let _ = self.add_new_task_to_ready_list(task, priority);
2632                return true;
2633            }
2634            Some(OwedTrace::None) | None => {}
2635        }
2636        if !self.flag(index, Self::F_YIELD, false) {
2637            return false;
2638        }
2639        self.set_flag(index, Self::F_YIELD, false);
2640        self.port_yield();
2641        true
2642    }
2643
2644    /// Collect the tally of an abandoned frame that has finished running.
2645    ///
2646    /// The frame runs to its end before control comes back to the runner,
2647    /// so this is called once, at the top of [`Kernel::resume_pending`],
2648    /// which is the first thing the runner does.
2649    fn settle_unwind(&mut self) {
2650        // Never anything to settle on a port that commits its own switches.
2651        if P::COMMITS_SWITCH {
2652            return;
2653        }
2654        // Nothing to settle is the common case by a wide margin, and `take()`
2655        // writes `None` back even then -- a store, on every call, to clear a
2656        // slot that was already clear. Peeking first leaves the store for the
2657        // calls that actually have something to clear.
2658        if self.unwinding.is_null() {
2659            return;
2660        }
2661        let task = core::mem::replace(&mut self.unwinding, TaskHandle::NULL);
2662        if task.is_null() {
2663            return;
2664        };
2665        let owed = self.port.end_unwind();
2666        if let Some(slot) = self.owed_exits.get_mut(task.index() as usize) {
2667            // Accumulate: a replay that is itself interrupted leaves the
2668            // rest of its own loop as a tail, and that tail is more of the
2669            // same debt.
2670            //
2671            // `wrapping_add`, for the reason `SimPort` gives for its `unwound`
2672            // tally: the debt is the exits of ONE abandoned frame plus a
2673            // replay's tail -- a handful, never four billion -- and the
2674            // saturation cost a `mov $-1` and a `cmov` on every switch.
2675            debug_assert!(
2676                slot.checked_add(owed).is_some(),
2677                "an owed-exit debt is a handful of exits"
2678            );
2679            *slot = slot.wrapping_add(owed);
2680        }
2681        if owed > 0 {
2682            self.owe(task);
2683        }
2684    }
2685
2686    /// Emit a queue failure line now, or owe it if a tick switched the
2687    /// caller out of the call that was about to emit it.
2688    /// In line on purpose, at all seven call sites. It is reached 46,671
2689    /// times in kernel-ir and every one of them paid a call and a frame to
2690    /// choose between two events the caller had already decided between.
2691    #[inline(always)]
2692    pub(crate) fn trace_failure_or_owe(&mut self, caller: TaskHandle, owed: OwedTrace) {
2693        #[cfg(feature = "census")]
2694        CENSUS_OWE.fetch_add(1, core::sync::atomic::Ordering::Relaxed);
2695        // Unlike every other trace site, this one has a side effect a no-op
2696        // sink cannot delete: the miss path STORES an `OwedTrace` and raises
2697        // `owes_anything`, which sends the task down the owed body on its next
2698        // run to reproduce a line the sink will drop.
2699        if !T::EMITS {
2700            return;
2701        }
2702        // "Still the running task" has two halves on a port that commits its
2703        // own switches. A stackless port moves `current` at the yield, so the
2704        // first test says it all; a committing port takes the switch AFTER
2705        // the call (PendSV, the two-core sim's runner), so `current` still
2706        // names the caller while a yield this core owes is about to take it
2707        // away -- and the C, switched at the yield, prints this line only
2708        // when the task runs again (two-core TaskNotify, tick 574: an
2709        // `xTimerStart` that readied the daemon).
2710        if self.cur() == caller && !(P::COMMITS_SWITCH && self.pending_here()) {
2711            let tick = self.tick;
2712            self.note_exits();
2713            match owed {
2714                OwedTrace::SendFailed(queue) => self
2715                    .trace
2716                    .event(tick, Event::QueueSendFailed { queue, name: "" }),
2717                OwedTrace::ReceiveFailed(queue) => self
2718                    .trace
2719                    .event(tick, Event::QueueReceiveFailed { queue, name: "" }),
2720                OwedTrace::StreamBufferCreate {
2721                    buffer,
2722                    is_message_buffer,
2723                } => self.trace.event(
2724                    tick,
2725                    Event::StreamBufferCreate {
2726                        buffer,
2727                        is_message_buffer,
2728                    },
2729                ),
2730                OwedTrace::TimerCommandSend {
2731                    timer,
2732                    name,
2733                    command,
2734                    value,
2735                } => self.trace.event(
2736                    tick,
2737                    Event::TimerCommandSend {
2738                        timer,
2739                        name: name.as_str(),
2740                        command,
2741                        value,
2742                    },
2743                ),
2744                OwedTrace::EventGroupWaitBitsEnd {
2745                    group,
2746                    bits,
2747                    timed_out,
2748                } => self.trace.event(
2749                    tick,
2750                    Event::EventGroupWaitBitsEnd {
2751                        group,
2752                        bits,
2753                        timed_out,
2754                    },
2755                ),
2756                // Not reachable through this function: `create_task` owes
2757                // its second half directly, because that half is not a
2758                // trace line — it inserts into a ready list — and it must
2759                // run INSIDE the section this one has already left.
2760                OwedTrace::AddNewTaskToReadyList { .. } | OwedTrace::None => {}
2761            }
2762            return;
2763        }
2764        if let Some(slot) = self.owed_trace.get_mut(caller.index() as usize) {
2765            *slot = owed;
2766        }
2767        self.owe(caller);
2768    }
2769
2770    /// Raise the hint that `task` owes something.
2771    ///
2772    /// Called by every writer of `owed_trace`, `owes_yield` and `owed_exits`.
2773    /// Missing one would let `resume_pending` skip work that was really owed,
2774    /// which the conformance differential would show as a missing trace line.
2775    /// `owes_yield`: a `portYIELD()` the task was preempted before reaching.
2776    const F_YIELD: u8 = 1 << 1;
2777    /// `wait_set`: a blocking call's `WaitFrame` is live in the TCB.
2778    const F_WAIT: u8 = 1 << 2;
2779    /// `taskATTRIBUTE_IS_IDLE`, SMP only: set on each core's idle task by
2780    /// `start_scheduler_smp`, cleared whenever a slot gets a new task.
2781    ///
2782    /// The SIGN bit on purpose. `yield_for_task` subtracts it as 0 or 1,
2783    /// which LLVM builds as a sign mask: from bit 3 that took `shl $4`,
2784    /// `sar $7` and a sign-extension per core; from bit 7 the `shl` is gone.
2785    const F_IDLE: u8 = 1 << 7;
2786
2787    /// Read one per-task flag.
2788    ///
2789    /// `missing` is what an index past the end answers. Each caller chooses it
2790    /// so an impossible index takes the SLOW path rather than the fast one,
2791    /// which is what the five separate arrays did with `unwrap_or`.
2792    #[inline(always)]
2793    fn flag(&self, index: usize, bit: u8, missing: bool) -> bool {
2794        match self.flags.get(index) {
2795            Some(f) => f & bit != 0,
2796            None => missing,
2797        }
2798    }
2799
2800    /// Write one per-task flag. An index past the end writes nothing, as
2801    /// `get_mut` did.
2802    #[inline(always)]
2803    fn set_flag(&mut self, index: usize, bit: u8, on: bool) {
2804        if let Some(f) = self.flags.get_mut(index) {
2805            if on {
2806                *f |= bit;
2807            } else {
2808                *f &= !bit;
2809            }
2810        }
2811    }
2812
2813    /// Drop the hint if nothing is owed any more.
2814    ///
2815    /// Worth one whole entry to [`Kernel::resume_pending_owed`] per owed item.
2816    /// A census over three scenarios found every raise producing exactly TWO
2817    /// entries — one that did the work and returned `true`, and a second that
2818    /// re-derived "nothing owed" and cleared the hint on its way out (BlockQ:
2819    /// 43,110 raises, 86,207 entries, 43,103 of them finding nothing). Doing
2820    /// it at the end of the branch that did the work turns that second entry
2821    /// into a `resume_pending` fast path, which is a few instructions instead
2822    /// of forty-odd.
2823    ///
2824    /// This cannot clear a live hint. Everything that opens new debt raises
2825    /// the hint itself — a replay that gets switched out leaves its tail to
2826    /// `settle_unwind`, which raises it again when it collects the tally.
2827    fn clear_owe_if_settled(&mut self, index: usize) {
2828        if self.owed_exits.get(index).copied().unwrap_or(0) == 0
2829            && matches!(self.owed_trace.get(index), Some(OwedTrace::None) | None)
2830            && !self.flag(index, Self::F_YIELD, false)
2831        {
2832            if let Some(f) = self.owes_anything.get_mut(index) {
2833                *f = false;
2834            }
2835        }
2836    }
2837
2838    fn owe(&mut self, task: TaskHandle) {
2839        if let Some(f) = self.owes_anything.get_mut(task.index() as usize) {
2840            *f = true;
2841        }
2842    }
2843
2844    /// Record that `task` was preempted before the `portYIELD()` at the end
2845    /// of the call it is inside.
2846    pub(crate) fn owe_yield(&mut self, task: TaskHandle) {
2847        self.set_flag(task.index() as usize, Self::F_YIELD, true);
2848        self.owe(task);
2849    }
2850
2851    /// Hand the CPU from `outgoing` to `incoming`: what `prvSwitchThread`
2852    /// does to `uxCriticalNesting`.
2853    ///
2854    /// Everything the outgoing task's call still does from here belongs to
2855    /// that task at the time it runs again, so the port stops counting the
2856    /// frame's exits as sim time and starts tallying them;
2857    /// [`Kernel::settle_unwind`] collects the tally once the frame has
2858    /// finished. A task being switched in for the first time has a fresh
2859    /// stack and owes nothing.
2860    /// Takes only the INCOMING handle. The outgoing one is `self.cur()`, which
2861    /// this reads for itself -- `set_current_at` has not run yet, so the field
2862    /// still names the task being left. Passing it in made `switch_context` load
2863    /// BOTH words of the handle at the top of the function, and the generation
2864    /// half is read only on the unwind arm below, which is skipped on every
2865    /// switch after the first.
2866    fn hand_over(&mut self, incoming: TaskHandle) {
2867        // A tail that switches again is still the first frame's tail: the
2868        // code after the second switch is on the same abandoned stack.
2869        // The unwinding marker exists for a port that does NOT commit the switch:
2870        // there `switch_context` returns into the OUTGOING task's Rust frame,
2871        // whose tail then runs on an abandoned stack and must be tallied rather
2872        // than counted. A `COMMITS_SWITCH` port saves the registers itself and
2873        // resumes the incoming task in ITS frame; nothing is abandoned, its
2874        // `begin_unwind` is a no-op and its `end_unwind` answers 0 -- so on
2875        // those ports this only forced `resume_pending` down its cold path
2876        // once per switch to settle a tally of zero. The const folds the whole
2877        // thing away on silicon and leaves the sim untouched.
2878        if !P::COMMITS_SWITCH && self.unwinding.is_null() {
2879            self.unwinding = self.cur();
2880            self.port.begin_unwind();
2881        }
2882        let index = incoming.index() as usize;
2883        // ONE access to the byte, not two: the test and the two bit writes all
2884        // work on the same `&mut`. STARTED goes on and YIELD goes off together,
2885        // which is the whole point of their sharing a byte.
2886        let first = match self.started.get_mut(index) {
2887            Some(f) if !*f => {
2888                *f = true;
2889                true
2890            }
2891            _ => false,
2892        };
2893        if first {
2894            self.set_flag(index, Self::F_YIELD, false);
2895            if let Some(slot) = self.owed_exits.get_mut(index) {
2896                *slot = 0;
2897            }
2898            if let Some(slot) = self.owed_trace.get_mut(index) {
2899                *slot = crate::kernel::OwedTrace::None;
2900            }
2901        }
2902    }
2903
2904    /// Mark a task as having run, without a hand-over: the first task the
2905    /// scheduler starts has no predecessor.
2906    fn note_first_start(&mut self, task: TaskHandle) {
2907        if let Some(f) = self.started.get_mut(task.index() as usize) {
2908            *f = true;
2909        }
2910    }
2911
2912    /// `xTaskIncrementTick`; `true` when a context switch is required.
2913    pub fn increment_tick(&mut self) -> bool {
2914        let tick = self.tick;
2915        self.note_exits();
2916        self.trace.event(tick, Event::TaskIncrementTick { tick });
2917        let mut switch_required = false;
2918        if self.suspended_depth == 0 {
2919            let next = self.tick.wrapping_add(1) & Self::MAX_DELAY;
2920            self.tick = next;
2921            if next == 0 {
2922                self.switch_delayed_lists();
2923            }
2924            // The mask is not arithmetic, it is INFORMATION. Both sides are
2925            // tick values and every writer of either has already masked with
2926            // `MAX_DELAY` -- `next` two lines above, `next_unblock_time` from a
2927            // `wake_at` that `add_current_task_to_delayed_list` masked, or from
2928            // `MAX_DELAY` itself. So at a 32-bit tick width the high half of
2929            // both is invariantly zero.
2930            //
2931            // LLVM proves that for `next`, which it just computed, and cannot
2932            // for `next_unblock_time`, which it loaded -- so the comparison was
2933            // emitted as a full 64-bit one: a second `lw` for the high word,
2934            // then `sltu`/`snez`/`or` to combine. Masking says the high half is
2935            // zero and the compare collapses to the single `bltu` the values
2936            // deserve. At a 64-bit width `MAX_DELAY` is `u64::MAX` and the mask
2937            // folds to nothing, so this costs those configurations no
2938            // instruction at all.
2939            // The mask is only sound while every writer of the field has
2940            // already applied it. All six do -- three store `MAX_DELAY`, two
2941            // store a delayed list's head value, and the one place that MINTS
2942            // such a value masks it (`add_current_task_to_delayed_list`'s
2943            // `wake_at`). This is the proof, and it costs nothing in release:
2944            // a writer that forgets fails the kernel's own tests, the Kani
2945            // proofs and the conformance corpus instead of silently changing a
2946            // comparison. Same guard shape as `current_priority`'s.
2947            debug_assert_eq!(
2948                self.next_unblock_time,
2949                self.next_unblock_time & Self::MAX_DELAY,
2950                "next_unblock_time holds a value wider than the tick, so masking                  it in the comparison below would change the answer"
2951            );
2952            if next >= self.next_unblock_time & Self::MAX_DELAY {
2953                switch_required = self.wake_due_tasks(next);
2954            }
2955            // `!switch_required` first: when the tick already woke a
2956            // task, this can only set what is already set, and asking
2957            // costs a TCB resolve for the running priority plus a list
2958            // read. Setting a `true` to `true` is not worth either.
2959            if C::NUMBER_OF_CORES > 1 {
2960                self.tick_time_slice_smp();
2961            } else if C::USE_PREEMPTION
2962                && C::USE_TIME_SLICING
2963                && !switch_required
2964                && self
2965                    .lists
2966                    .len(Self::ready_list(self.current_priority()))
2967                    .unwrap_or(0)
2968                    > 1
2969            {
2970                switch_required = true;
2971            }
2972            // The C guards this site with `xPendedTicks == 0`, so a hook
2973            // does not fire again for each tick being unwound by
2974            // `xTaskResumeAll`. It sits after the time-slicing test and
2975            // before the yield-pending one, and a `FromISR` call inside it
2976            // can set `yield_pending`, so the order is load-bearing.
2977            if self.pended_ticks == 0 {
2978                self.run_tick_hook();
2979            }
2980            if C::NUMBER_OF_CORES > 1 {
2981                switch_required |= self.tick_yields_smp();
2982            } else if C::USE_PREEMPTION && self.pending_here() {
2983                switch_required = true;
2984            }
2985        } else {
2986            self.pended_ticks = self.pended_ticks.wrapping_add(1);
2987            // "The tick hook gets called at regular intervals, even if the
2988            // scheduler is locked" — and here with no guard, because this
2989            // tick is being pended rather than unwound.
2990            self.run_tick_hook();
2991        }
2992        switch_required
2993    }
2994
2995    // ------------------------------------------- task notifications --
2996    //
2997    // A notification is the lightest thing a task can block on: no object,
2998    // no event list, just a slot in the TCB and the delayed list. That is
2999    // why the stream buffers use one rather than a queue, and why waking a
3000    // task this way is a list move rather than an event-list walk.
3001
3002    /// `xTaskGenericNotifyWait`.
3003    ///
3004    /// `true` is `pdTRUE`: a notification was received. `Blocked` means the
3005    /// caller must call again — the C blocks inside this function, and a
3006    /// kernel with no stacks returns instead.
3007    ///
3008    /// # Errors
3009    /// [`Error::InvalidArgument`] for an index past the configured array.
3010    pub fn notify_wait(
3011        &mut self,
3012        index: usize,
3013        clear_on_entry: u32,
3014        clear_on_exit: u32,
3015        ticks: u64,
3016    ) -> Result<Wait<(bool, u32)>> {
3017        if index >= C::NOTIFICATION_ARRAY_ENTRIES {
3018            return Err(Error::InvalidArgument);
3019        }
3020        let caller = self.cur();
3021        let state = self.notify_state_of(caller, index);
3022        if state != NotifyState::Received && ticks > 0 && !self.notify_blocked(caller) {
3023            self.suspend_all();
3024            self.enter_critical();
3025            let mut should_block = false;
3026            if self.notify_state_of(caller, index) != NotifyState::Received {
3027                if let Ok(tcb) = self.tcbs.resolve_mut(caller) {
3028                    if let Some(slot) = tcb.notified.get_mut(index) {
3029                        *slot &= !clear_on_entry;
3030                    }
3031                    if let Some(slot) = tcb.notify_state.get_mut(index) {
3032                        *slot = NotifyState::Waiting;
3033                    }
3034                }
3035                should_block = true;
3036            }
3037            self.exit_critical();
3038            if should_block {
3039                let tick = self.tick;
3040                self.note_exits();
3041                self.trace_task(caller, |task, name| Event::TaskNotifyWaitBlock {
3042                    task,
3043                    name,
3044                    index,
3045                });
3046                let _ = tick;
3047                self.add_current_task_to_delayed_list(ticks, true)?;
3048            }
3049            let already_yielded = self.resume_all();
3050            if should_block && !already_yielded {
3051                if self.cur() == caller {
3052                    self.port_yield();
3053                } else {
3054                    self.owe_yield(caller);
3055                }
3056            }
3057            // The C blocks here; this kernel comes back and is called again.
3058            self.set_notify_blocked(caller, true);
3059            return Ok(Wait::Blocked);
3060        }
3061        self.set_notify_blocked(caller, false);
3062        self.enter_critical();
3063        self.note_exits();
3064        self.trace_task(caller, |task, name| Event::TaskNotifyWait {
3065            task,
3066            name,
3067            index,
3068        });
3069        let value = self.notified_value_of(caller, index);
3070        let received = self.notify_state_of(caller, index) == NotifyState::Received;
3071        if let Ok(tcb) = self.tcbs.resolve_mut(caller) {
3072            if received {
3073                if let Some(slot) = tcb.notified.get_mut(index) {
3074                    *slot &= !clear_on_exit;
3075                }
3076            }
3077            if let Some(slot) = tcb.notify_state.get_mut(index) {
3078                *slot = NotifyState::NotWaiting;
3079            }
3080        }
3081        self.exit_critical();
3082        Ok(Wait::Ready((received, value)))
3083    }
3084
3085    /// `ulTaskGenericNotifyTake`: the notification used as a lightweight
3086    /// counting semaphore — block until the value is non-zero, then either
3087    /// zero it or decrement it by one.
3088    ///
3089    /// This is the half of the notification API the kernel was missing.
3090    /// `notify_wait` waits on *bits*; this waits on a *count*, and the two
3091    /// are separate calls in the C with their own trace events. Its
3092    /// partner, `xTaskNotifyGive`, needs no method of its own: the C
3093    /// defines it as `xTaskGenericNotify( .., 0, eIncrement, NULL )` and
3094    /// it traces as an ordinary notify, so
3095    /// `notify(task, index, 0, NotifyAction::Increment)` already is it.
3096    ///
3097    /// # The shape, and why it is not `notify_wait`'s
3098    ///
3099    /// The C blocks on `ulNotifiedValue == 0`, **not** on the notify
3100    /// state, and it does no clear-on-entry. Both halves run on the way
3101    /// out: `traceTASK_NOTIFY_TAKE` fires whether or not the call ever
3102    /// blocked, which is what makes a take that finds a count already
3103    /// waiting still emit one line.
3104    ///
3105    /// This kernel has no stack, so the block is a return: the first call
3106    /// answers [`Wait::Blocked`] and the caller is entered again when the
3107    /// scheduler runs it, taking the second half. `notify_wait_pending`
3108    /// answers for this call too — a task is only ever inside one of the
3109    /// two.
3110    ///
3111    /// # Errors
3112    /// [`Error::InvalidArgument`] for an index past the configured array.
3113    pub fn notify_take(
3114        &mut self,
3115        index: usize,
3116        clear_on_exit: bool,
3117        ticks: u64,
3118    ) -> Result<Wait<u32>> {
3119        if index >= C::NOTIFICATION_ARRAY_ENTRIES {
3120            return Err(Error::InvalidArgument);
3121        }
3122        let caller = self.cur();
3123        if self.notified_value_of(caller, index) == 0 && ticks > 0 && !self.notify_blocked(caller) {
3124            self.suspend_all();
3125            self.enter_critical();
3126            let mut should_block = false;
3127            // Re-checked inside the section, as the C does: a notify from
3128            // an ISR between the two reads would otherwise be lost.
3129            if self.notified_value_of(caller, index) == 0 {
3130                if let Ok(tcb) = self.tcbs.resolve_mut(caller) {
3131                    if let Some(slot) = tcb.notify_state.get_mut(index) {
3132                        *slot = NotifyState::Waiting;
3133                    }
3134                }
3135                should_block = true;
3136            }
3137            self.exit_critical();
3138            if should_block {
3139                // The C traces BEFORE it adds itself to the delayed list.
3140                self.note_exits();
3141                self.trace_task(caller, |task, name| Event::TaskNotifyTakeBlock {
3142                    task,
3143                    name,
3144                    index,
3145                });
3146                self.add_current_task_to_delayed_list(ticks, true)?;
3147            }
3148            let already_yielded = self.resume_all();
3149            if should_block && !already_yielded {
3150                if self.cur() == caller {
3151                    self.port_yield();
3152                } else {
3153                    self.owe_yield(caller);
3154                }
3155            }
3156            self.set_notify_blocked(caller, true);
3157            return Ok(Wait::Blocked);
3158        }
3159        self.enter_critical();
3160        self.note_exits();
3161        self.trace_task(caller, |task, name| Event::TaskNotifyTake {
3162            task,
3163            name,
3164            index,
3165        });
3166        // One resolve for the read and both writes. `notified_value_of`
3167        // resolved the TCB to read the value and this line resolved it again
3168        // to write it back -- the same trade `unlock_queue` makes, and a
3169        // `&mut self` call sits between this and the read at the top of the
3170        // function, so the two cannot be folded by the compiler.
3171        //
3172        // `notify_blocked` is cleared on this resolve too. It was its own
3173        // resolve before the section -- `set_notify_blocked` -- and on rv32
3174        // the section's entry is a compiler barrier, so the two never folded.
3175        // Nothing between the old place and this one reads the flag.
3176        let mut value = 0;
3177        if let Ok(tcb) = self.tcbs.resolve_mut(caller) {
3178            tcb.notify_blocked = false;
3179            value = tcb.notified.get(index).copied().unwrap_or(0);
3180            if value != 0 {
3181                if let Some(slot) = tcb.notified.get_mut(index) {
3182                    // Guarded by `value != 0` above, so the saturation
3183                    // never fires; it is here because the workspace forbids
3184                    // bare arithmetic, and a decrement is exactly the place
3185                    // an unguarded one would wrap.
3186                    *slot = if clear_on_exit {
3187                        0
3188                    } else {
3189                        value.saturating_sub(1)
3190                    };
3191                }
3192            }
3193            if let Some(slot) = tcb.notify_state.get_mut(index) {
3194                *slot = NotifyState::NotWaiting;
3195            }
3196        }
3197        self.exit_critical();
3198        Ok(Wait::Ready(value))
3199    }
3200
3201    /// `xTaskGetHandle`: the task registered under `name`.
3202    ///
3203    /// # It is not free, and that is the point
3204    ///
3205    /// The C walks every ready list, then the delayed lists, then the
3206    /// suspended and waiting-termination lists, all inside
3207    /// `vTaskSuspendAll` / `xTaskResumeAll` — and `xTaskResumeAll` takes a
3208    /// critical section. So a lookup that emits **no trace event** still
3209    /// costs **one critical-section exit**, which under the sim contract is
3210    /// one unit of time.
3211    ///
3212    /// That is why this exists rather than a scenario keeping the handle it
3213    /// was given at creation. `AbortDelay`'s remake did exactly that, on the
3214    /// reasoning that a lookup emitting no event could not change the trace,
3215    /// and the corpus caught it: every event still agreed and the
3216    /// exit column was one short at the first block.
3217    ///
3218    /// The arena is searched instead of five lists, which cannot change the
3219    /// answer — a live task is in exactly one of them — and does not change
3220    /// the accounting either, because the cost is the suspend/resume pair.
3221    ///
3222    /// # Errors
3223    /// [`Error::Gone`] when no live task carries that name.
3224    pub fn task_get_handle(&mut self, name: &str) -> Result<TaskHandle> {
3225        self.suspend_all();
3226        let mut found = None;
3227        for index in 0..u16::try_from(self.tcbs.capacity()).unwrap_or(u16::MAX) {
3228            let Some(handle) = self.tcbs.handle_at(index) else {
3229                continue;
3230            };
3231            if self
3232                .tcbs
3233                .resolve(handle)
3234                .is_ok_and(|tcb| tcb.name.matches(name))
3235            {
3236                found = Some(handle);
3237                break;
3238            }
3239        }
3240        let _ = self.resume_all();
3241        found.ok_or(Error::Gone)
3242    }
3243
3244    /// `xTaskGenericNotify`.
3245    ///
3246    /// # Errors
3247    /// [`Error::InvalidArgument`] for a bad index; [`Error::Gone`] for a
3248    /// stale handle.
3249    pub fn notify(
3250        &mut self,
3251        task: TaskHandle,
3252        index: usize,
3253        value: u32,
3254        action: NotifyAction,
3255    ) -> Result<bool> {
3256        if index >= C::NOTIFICATION_ARRAY_ENTRIES {
3257            return Err(Error::InvalidArgument);
3258        }
3259        self.enter_critical();
3260        let result = self.notify_locked(task, index, value, action, false);
3261        self.exit_critical();
3262        result.map(|(ok, _)| ok)
3263    }
3264
3265    /// `xTaskGenericNotifyFromISR`: the same, without a critical section
3266    /// that costs anything, and putting a woken task on the pending-ready
3267    /// list when the scheduler is suspended.
3268    ///
3269    /// # Errors
3270    /// As [`Kernel::notify`].
3271    pub fn notify_from_isr(
3272        &mut self,
3273        task: TaskHandle,
3274        index: usize,
3275        value: u32,
3276        action: NotifyAction,
3277    ) -> Result<(bool, Woken)> {
3278        if index >= C::NOTIFICATION_ARRAY_ENTRIES {
3279            return Err(Error::InvalidArgument);
3280        }
3281        let mask = self.port.enter_critical_from_isr();
3282        let result = self.notify_locked(task, index, value, action, true);
3283        self.port.exit_critical_from_isr(mask);
3284        result.map(|(ok, woken)| (ok, if woken { Woken::YES } else { Woken::NO }))
3285    }
3286
3287    /// `xTaskGenericNotify` with its `pulPreviousNotificationValue`
3288    /// out-parameter filled: `xTaskNotifyAndQuery`.
3289    ///
3290    /// It exists because the answer has to come out of the SAME critical
3291    /// section as the notification. Reading [`Kernel::notify_value`] first
3292    /// and then notifying yields the same two numbers and costs a second
3293    /// critical section -- and on the sim a critical-section exit is a
3294    /// tick opportunity, so the extra one moves the tick and the trace
3295    /// stops matching the C. `TaskNotify.c` calls this eleven times.
3296    ///
3297    /// # Errors
3298    /// As [`Kernel::notify`].
3299    pub fn notify_and_query(
3300        &mut self,
3301        task: TaskHandle,
3302        index: usize,
3303        value: u32,
3304        action: NotifyAction,
3305    ) -> Result<(bool, u32)> {
3306        if index >= C::NOTIFICATION_ARRAY_ENTRIES {
3307            return Err(Error::InvalidArgument);
3308        }
3309        self.enter_critical();
3310        let previous = self
3311            .tcbs
3312            .resolve(task)
3313            .map(|tcb| tcb.notified.get(index).copied().unwrap_or(0));
3314        let result = self.notify_locked(task, index, value, action, false);
3315        self.exit_critical();
3316        Ok((result?.0, previous?))
3317    }
3318
3319    /// `xTaskGenericNotifyAndQueryFromISR`.
3320    ///
3321    /// # Errors
3322    /// As [`Kernel::notify`].
3323    pub fn notify_and_query_from_isr(
3324        &mut self,
3325        task: TaskHandle,
3326        index: usize,
3327        value: u32,
3328        action: NotifyAction,
3329    ) -> Result<(bool, u32, Woken)> {
3330        if index >= C::NOTIFICATION_ARRAY_ENTRIES {
3331            return Err(Error::InvalidArgument);
3332        }
3333        let mask = self.port.enter_critical_from_isr();
3334        let previous = self
3335            .tcbs
3336            .resolve(task)
3337            .map(|tcb| tcb.notified.get(index).copied().unwrap_or(0));
3338        let result = self.notify_locked(task, index, value, action, true);
3339        self.port.exit_critical_from_isr(mask);
3340        let (ok, woken) = result?;
3341        Ok((ok, previous?, if woken { Woken::YES } else { Woken::NO }))
3342    }
3343
3344    /// The body both notify paths share.
3345    fn notify_locked(
3346        &mut self,
3347        task: TaskHandle,
3348        index: usize,
3349        value: u32,
3350        action: NotifyAction,
3351        from_isr: bool,
3352    ) -> Result<(bool, bool)> {
3353        let original = self.notify_state_of(task, index);
3354        let mut ok = true;
3355        {
3356            let tcb = self.tcbs.resolve_mut(task)?;
3357            if let Some(slot) = tcb.notify_state.get_mut(index) {
3358                *slot = NotifyState::Received;
3359            }
3360            if let Some(slot) = tcb.notified.get_mut(index) {
3361                match action {
3362                    NotifyAction::SetBits => *slot |= value,
3363                    NotifyAction::Increment => *slot = slot.wrapping_add(1),
3364                    NotifyAction::Overwrite => *slot = value,
3365                    NotifyAction::NoOverwrite => {
3366                        if original == NotifyState::Received {
3367                            ok = false;
3368                        } else {
3369                            *slot = value;
3370                        }
3371                    }
3372                    NotifyAction::None => {}
3373                }
3374            }
3375        }
3376        // `traceTASK_NOTIFY` is hooked; `traceTASK_NOTIFY_FROM_ISR` is not,
3377        // so an interrupt's notification says nothing on either side.
3378        if !from_isr {
3379            self.note_exits();
3380            self.trace_task(task, |t, name| Event::TaskNotify {
3381                task: t,
3382                name,
3383                index,
3384            });
3385        }
3386        let mut woke_higher = false;
3387        if original == NotifyState::Waiting {
3388            let item = Self::state_item(task);
3389            if from_isr && self.suspended_depth != 0 {
3390                let _ = self.lists.remove(Self::event_item(task));
3391                let value = self.lists.value(Self::event_item(task))?;
3392                self.lists
3393                    .insert(Self::pending_ready_list(), Self::event_item(task), value)?;
3394            } else {
3395                if self.lists.container(item)?.is_some() {
3396                    let _ = self.lists.remove(item);
3397                }
3398                self.add_task_to_ready_list(task)?;
3399                // `configUSE_TICKLESS_IDLE`: a task that leaves the delayed
3400                // list early must not leave its wake time behind as the next
3401                // unblock time, or a tickless sleep is cut short and a step
3402                // (`step_tick`) stops at a wake that is no longer there.
3403                if C::USE_TICKLESS_IDLE {
3404                    self.reset_next_task_unblock_time();
3405                }
3406            }
3407            let woken = self.tcbs.resolve(task).map(|t| t.priority).unwrap_or(0);
3408            if C::NUMBER_OF_CORES > 1 {
3409                // `prvYieldForTask`; the caller's flag is whether THIS core
3410                // now owes a yield, and a task-side notify takes it here.
3411                woke_higher = self.smp_readied(task, woken);
3412                if woke_higher && !from_isr {
3413                    self.port_yield();
3414                }
3415            } else if woken > self.current_priority() {
3416                woke_higher = true;
3417                if from_isr {
3418                    // The C sets `xYieldPendings[ 0 ]` here as well as
3419                    // reporting the wake through the caller's flag — so an
3420                    // interrupt that discards the flag still gets the
3421                    // switch, on the way out of `xTaskIncrementTick`. That
3422                    // is why the tick hook runs before the yield-pending
3423                    // test and not after it.
3424                    self.set_pending_here(true);
3425                } else {
3426                    // taskYIELD_ANY_CORE_IF_USING_PREEMPTION, inside the
3427                    // section.
3428                    self.port_yield();
3429                }
3430            }
3431        }
3432        Ok((ok, woke_higher))
3433    }
3434
3435    /// `xTaskGenericNotifyStateClear`. `None` is the calling task.
3436    ///
3437    /// # Errors
3438    /// As [`Kernel::notify`].
3439    pub fn notify_state_clear(&mut self, task: Option<TaskHandle>, index: usize) -> Result<bool> {
3440        if index >= C::NOTIFICATION_ARRAY_ENTRIES {
3441            return Err(Error::InvalidArgument);
3442        }
3443        let target = task.unwrap_or(self.cur());
3444        self.enter_critical();
3445        let cleared = self.notify_state_of(target, index) == NotifyState::Received;
3446        if cleared {
3447            if let Ok(tcb) = self.tcbs.resolve_mut(target) {
3448                if let Some(slot) = tcb.notify_state.get_mut(index) {
3449                    *slot = NotifyState::NotWaiting;
3450                }
3451            }
3452        }
3453        self.exit_critical();
3454        Ok(cleared)
3455    }
3456
3457    /// `ulNotifiedValue[ index ]` for a task. `None` is the calling task.
3458    ///
3459    /// The C reaches this through `xTaskNotifyAndQuery`, which hands back
3460    /// the value a notification is about to REPLACE, and through
3461    /// `ulTaskNotifyValueClear`. Neither could be implemented without it:
3462    /// `xTaskNotifyAndQuery` is `xTaskGenericNotify` with a
3463    /// `pulPreviousNotificationValue` out-parameter, and `TaskNotify.c:373`
3464    /// asserts on what comes back. The field existed; only the way to read
3465    /// it was missing, which is the kind of gap a C ABI finds and a Rust
3466    /// face never does.
3467    ///
3468    /// # Errors
3469    /// `InvalidArgument` if `index` is outside the notification array, or
3470    /// the resolve error if the handle is stale.
3471    pub fn notify_value(&mut self, task: Option<TaskHandle>, index: usize) -> Result<u32> {
3472        if index >= C::NOTIFICATION_ARRAY_ENTRIES {
3473            return Err(Error::InvalidArgument);
3474        }
3475        let target = task.unwrap_or(self.cur());
3476        self.enter_critical();
3477        let value = self
3478            .tcbs
3479            .resolve(target)
3480            .map(|tcb| tcb.notified.get(index).copied().unwrap_or(0));
3481        self.exit_critical();
3482        value
3483    }
3484
3485    /// `ulTaskGenericNotifyValueClear`: clear `bits` in the task's
3486    /// notification value and answer what it was BEFORE the clear.
3487    ///
3488    /// # Errors
3489    /// As [`Kernel::notify_value`].
3490    pub fn notify_value_clear(
3491        &mut self,
3492        task: Option<TaskHandle>,
3493        index: usize,
3494        bits: u32,
3495    ) -> Result<u32> {
3496        if index >= C::NOTIFICATION_ARRAY_ENTRIES {
3497            return Err(Error::InvalidArgument);
3498        }
3499        let target = task.unwrap_or(self.cur());
3500        self.enter_critical();
3501        let before = match self.tcbs.resolve_mut(target) {
3502            Ok(tcb) => match tcb.notified.get_mut(index) {
3503                Some(slot) => {
3504                    let was = *slot;
3505                    *slot &= !bits;
3506                    Ok(was)
3507                }
3508                None => Err(Error::InvalidArgument),
3509            },
3510            Err(e) => Err(e),
3511        };
3512        self.exit_critical();
3513        before
3514    }
3515
3516    /// Whether `task` is part-way through a notification wait — it blocked
3517    /// once and has not yet run the half of `xTaskGenericNotifyWait` that
3518    /// happens when a task wakes.
3519    ///
3520    /// A caller that blocks on a notification needs this: the C's wait
3521    /// returns *after* the wake, so its second half always runs, and a
3522    /// stackless caller that decided not to block again would otherwise
3523    /// skip it and leave the slot marked as received for ever.
3524    #[must_use]
3525    pub fn notify_wait_pending(&self, task: TaskHandle) -> bool {
3526        self.notify_blocked(task)
3527    }
3528
3529    /// Whether `task` has a stream-buffer call to resume, and the local it
3530    /// left behind. Taking it clears the marker.
3531    pub(crate) fn take_stream_resume(&mut self, task: TaskHandle) -> Option<usize> {
3532        let tcb = self.tcbs.resolve_mut(task).ok()?;
3533        if !tcb.stream_resume {
3534            return None;
3535        }
3536        tcb.stream_resume = false;
3537        Some(tcb.stream_local)
3538    }
3539
3540    /// Remember where a preempted stream-buffer call has to start again.
3541    pub(crate) fn set_stream_resume(&mut self, task: TaskHandle, local: usize) {
3542        if let Ok(tcb) = self.tcbs.resolve_mut(task) {
3543            tcb.stream_resume = true;
3544            tcb.stream_local = local;
3545        }
3546    }
3547
3548    /// Keep a blocking send's last `xSpace` across the wait it parks in.
3549    ///
3550    /// The C's local survives the block, and a send that then TIMES OUT
3551    /// writes with it -- stale -- re-reading only if it was zero. This
3552    /// kernel returns at the block, so the TCB holds it instead. No marker:
3553    /// the notify wait itself says the call is pending.
3554    pub(crate) fn park_stream_sample(&mut self, task: TaskHandle, local: usize) {
3555        if let Ok(tcb) = self.tcbs.resolve_mut(task) {
3556            tcb.stream_local = local;
3557        }
3558    }
3559
3560    /// Whether `task` is part-way through a blocking call on `queue` -- its
3561    /// wait frame is that queue's -- so the call it is making now is the
3562    /// retry of one that blocked.
3563    pub(crate) fn waiting_on(&self, task: TaskHandle, queue: QueueHandle) -> bool {
3564        if !self.flag(task.index() as usize, Self::F_WAIT, false) {
3565            return false;
3566        }
3567        self.tcbs
3568            .resolve(task)
3569            .is_ok_and(|t| t.wait.entry_set && t.wait.queue == queue)
3570    }
3571
3572    /// The value [`Kernel::hold`] kept for `task`.
3573    pub(crate) fn held(&self, task: TaskHandle) -> u64 {
3574        self.tcbs.resolve(task).map_or(0, |t| t.held.get())
3575    }
3576
3577    /// Keep what a blocked timer command is sending, for its retry.
3578    pub(crate) fn hold(&mut self, task: TaskHandle, value: u64) {
3579        if let Ok(tcb) = self.tcbs.resolve_mut(task) {
3580            tcb.held = crate::timer::Split64::new(value);
3581        }
3582    }
3583
3584    /// The `xSpace` [`Kernel::park_stream_sample`] kept.
3585    pub(crate) fn stream_sample(&self, task: TaskHandle) -> usize {
3586        self.tcbs.resolve(task).map_or(0, |t| t.stream_local)
3587    }
3588
3589    /// Whether `task` has a queue call to resume below its sampling exit.
3590    /// Taking it clears the marker.
3591    pub(crate) fn take_queue_resume(&mut self, task: TaskHandle) -> QueueResume {
3592        // Nobody is parked at a sampling exit, so there is nothing to read.
3593        // This is the answer on all but a handful of queue calls, and it costs
3594        // one field load where resolving the TCB costs an arena lookup.
3595        if self.queue_resumes == 0 {
3596            return QueueResume::No;
3597        }
3598        let Ok(tcb) = self.tcbs.resolve_mut(task) else {
3599            return QueueResume::No;
3600        };
3601        let resume = tcb.queue_resume;
3602        // Peek before clearing, the same trade `settle_unwind` makes: every
3603        // queue send and every queue receive passes through here, and on all
3604        // but the handful that were preempted at a sampling exit the slot is
3605        // already `No` — so the unconditional write was a store, on every
3606        // queue call, to clear something already clear.
3607        if resume != QueueResume::No {
3608            tcb.queue_resume = QueueResume::No;
3609            self.queue_resumes = self.queue_resumes.saturating_sub(1);
3610        }
3611        resume
3612    }
3613
3614    /// Remember that a queue call was preempted at its sampling exit.
3615    ///
3616    /// No local travels with it, unlike [`Kernel::set_stream_resume`]: the C
3617    /// re-reads the queue under `prvLockQueue` on the other side of the
3618    /// exit, so there is nothing sampled to carry.
3619    pub(crate) fn set_queue_resume(&mut self, task: TaskHandle, at: QueueResume) {
3620        debug_assert_ne!(
3621            at,
3622            QueueResume::No,
3623            "set_queue_resume only ever parks a task; clearing is take's job, and the summary count depends on that"
3624        );
3625        if let Ok(tcb) = self.tcbs.resolve_mut(task) {
3626            // Count the slot, not the call: re-parking a task already parked
3627            // replaces its slot and must not be counted twice.
3628            if tcb.queue_resume == QueueResume::No {
3629                self.queue_resumes = self.queue_resumes.wrapping_add(1);
3630            }
3631            tcb.queue_resume = at;
3632        }
3633    }
3634
3635    /// Has this task's blocking send already paid for `vTaskSetTimeOutState`?
3636    ///
3637    /// A one-shot, like [`Kernel::take_stream_resume`], and for the same
3638    /// reason: `vTaskSetTimeOutState`'s own exit can release a tick and
3639    /// switch the caller away, and the C's thread then resumes **below** it,
3640    /// in the sampling loop. Running it again on re-entry would spend a
3641    /// second exit the C never spends -- which is a sixteenth of a tick, and
3642    /// `StreamBufferDemo`'s echo client hit it at tick 419.
3643    pub(crate) fn take_stream_timed(&mut self, task: TaskHandle) -> bool {
3644        let Ok(tcb) = self.tcbs.resolve_mut(task) else {
3645            return false;
3646        };
3647        let was = tcb.stream_timed;
3648        tcb.stream_timed = false;
3649        was
3650    }
3651
3652    /// Remember that the timeout state is set and must not be set again.
3653    pub(crate) fn set_stream_timed(&mut self, task: TaskHandle) {
3654        if let Ok(tcb) = self.tcbs.resolve_mut(task) {
3655            tcb.stream_timed = true;
3656        }
3657    }
3658
3659    /// Has this task's stream-buffer call already finished its wait?
3660    ///
3661    /// The other half of [`Kernel::take_stream_resume`]. A call can be
3662    /// preempted at two different exits, and the two resume in different
3663    /// places: at the SAMPLING exit the C goes on to block, and at the
3664    /// WAIT's own exit it goes on to move the bytes and trace them. Without
3665    /// this the second case is indistinguishable from the first, and a call
3666    /// that had already waited would wait again.
3667    pub(crate) fn take_stream_waited(&mut self, task: TaskHandle) -> bool {
3668        let Ok(tcb) = self.tcbs.resolve_mut(task) else {
3669            return false;
3670        };
3671        let was = tcb.stream_waited;
3672        tcb.stream_waited = false;
3673        was
3674    }
3675
3676    /// Remember that the wait is done and only the transfer is left.
3677    pub(crate) fn set_stream_waited(&mut self, task: TaskHandle) {
3678        if let Ok(tcb) = self.tcbs.resolve_mut(task) {
3679            tcb.stream_waited = true;
3680        }
3681    }
3682
3683    fn notify_blocked(&self, task: TaskHandle) -> bool {
3684        self.tcbs
3685            .resolve(task)
3686            .map(|t| t.notify_blocked)
3687            .unwrap_or(false)
3688    }
3689
3690    fn set_notify_blocked(&mut self, task: TaskHandle, blocked: bool) {
3691        if let Ok(tcb) = self.tcbs.resolve_mut(task) {
3692            tcb.notify_blocked = blocked;
3693        }
3694    }
3695
3696    fn notify_state_of(&self, task: TaskHandle, index: usize) -> NotifyState {
3697        self.tcbs
3698            .resolve(task)
3699            .ok()
3700            .and_then(|t| t.notify_state.get(index).copied())
3701            .unwrap_or(NotifyState::NotWaiting)
3702    }
3703
3704    fn notified_value_of(&self, task: TaskHandle, index: usize) -> u32 {
3705        self.tcbs
3706            .resolve(task)
3707            .ok()
3708            .and_then(|t| t.notified.get(index).copied())
3709            .unwrap_or(0)
3710    }
3711
3712    /// `xTaskGetTickCountFromISR`.
3713    ///
3714    /// No critical section, exactly as `xTaskGetTickCount` takes none: the
3715    /// Posix port sets `portTICK_TYPE_IS_ATOMIC`, so reading the count is
3716    /// a single load on both sides and costs no sim time.
3717    #[must_use]
3718    pub const fn tick_count_from_isr(&self) -> u64 {
3719        self.tick
3720    }
3721
3722    /// `vTaskMissedYield`: a yield that could not be taken where it was
3723    /// asked for — because a task is walking an event list, or because the
3724    /// scheduler is suspended — and must happen on the way out instead.
3725    pub(crate) fn missed_yield(&mut self) {
3726        self.set_pending_here(true);
3727    }
3728
3729    /// `vApplicationTickHook()`, when `configUSE_TICK_HOOK` is 1.
3730    ///
3731    /// The hook is copied out, run, and the result stored: that is what
3732    /// lets it borrow the kernel mutably while the kernel owns it. A hook
3733    /// that reads the kernel's own copy of itself therefore sees the value
3734    /// from before this call.
3735    ///
3736    /// On two cores the test is in line and the run is not. There the run had
3737    /// a symbol of its own with the hook's whole body inlined into it, and the
3738    /// test that skips it paid that body's six saved registers and its frame
3739    /// on every tick -- 18 instructions to return. One core keeps the run in
3740    /// line, inside `increment_tick`: outlined there it cost a scenario with
3741    /// an installed hook +11 a tick (TimerDemo +231,916).
3742    #[inline(always)]
3743    fn run_tick_hook(&mut self) {
3744        // A hook that says it has nothing to do is not copied at all: the
3745        // copy out and back is the hook's full size every tick, 448 bytes
3746        // each way for the demo's `TickIsr`, which most scenarios install as
3747        // `None`.
3748        if !C::USE_TICK_HOOK || !self.tick_hook.wants_tick() {
3749            return;
3750        }
3751        if C::NUMBER_OF_CORES > 1 {
3752            self.run_tick_hook_body();
3753            return;
3754        }
3755        let hook = self.tick_hook;
3756        self.tick_hook = hook.tick(self);
3757    }
3758
3759    /// The copy out, the run and the store back of [`Kernel::run_tick_hook`].
3760    #[inline(never)]
3761    fn run_tick_hook_body(&mut self) {
3762        let hook = self.tick_hook;
3763        self.tick_hook = hook.tick(self);
3764    }
3765
3766    /// The tick hook, as it now stands.
3767    ///
3768    /// A scenario's `xAre...StillRunning()` reads the status its interrupt
3769    /// half latched, so the owner of the kernel needs to see it.
3770    pub const fn tick_hook(&self) -> &H {
3771        &self.tick_hook
3772    }
3773
3774    /// The tick hook, to install or reset one after construction.
3775    pub const fn tick_hook_mut(&mut self) -> &mut H {
3776        &mut self.tick_hook
3777    }
3778
3779    /// The wake loop inside `xTaskIncrementTick`.
3780    // Out of line: this walks the delayed list and runs only when a tick
3781    // reaches the next unblock time, but `increment_tick` runs on EVERY tick
3782    // and was carrying a frame sized for the walk regardless.
3783    #[inline(never)]
3784    fn wake_due_tasks(&mut self, now: u64) -> bool {
3785        let mut switch_required = false;
3786        // Hoisted: see the note at the bottom of the loop.
3787        let running = if C::USE_PREEMPTION {
3788            self.current_priority()
3789        } else {
3790            0
3791        };
3792        loop {
3793            let delayed = self.delayed_list();
3794            if self.lists.is_empty_of(delayed) {
3795                self.next_unblock_time = Self::MAX_DELAY;
3796                return switch_required;
3797            }
3798            let Ok(Some(item)) = self.lists.head(delayed) else {
3799                self.next_unblock_time = Self::MAX_DELAY;
3800                return switch_required;
3801            };
3802            let Ok(wake_at) = self.lists.value(item) else {
3803                return switch_required;
3804            };
3805            if now < wake_at {
3806                self.next_unblock_time = wake_at;
3807                return switch_required;
3808            }
3809            let Ok(task) = self.task_of_state_item(item) else {
3810                return switch_required;
3811            };
3812            // Beside `handle_at`'s read of the same slot, and carried, as
3813            // `remove_from_event_list` does: after the list edits LLVM could
3814            // not prove the TCB unchanged and resolved it twice more. A
3815            // failure here is the one `add_task_to_ready_list` would have hit.
3816            let Ok(woken) = self.tcbs.resolve(task).map(|t| t.priority) else {
3817                return switch_required;
3818            };
3819            let _ = self.lists.remove(item);
3820            if self
3821                .lists
3822                .container(Self::event_item(task))
3823                .unwrap_or(None)
3824                .is_some()
3825            {
3826                let _ = self.lists.remove(Self::event_item(task));
3827            }
3828            if self.add_task_to_ready_list_at(task, woken).is_err() {
3829                return switch_required;
3830            }
3831            // `!switch_required` FIRST, and the running priority read once
3832            // for the whole drain.
3833            //
3834            // Both of these cost a TCB resolve -- a generation check, a
3835            // bounds check and an `Option` -- and this loop runs once per
3836            // task the tick wakes. `self.cur()` cannot move while the
3837            // drain is running (nothing here switches), so the running
3838            // priority is a loop invariant that was being re-derived on
3839            // every lap. And once a switch is already required, asking again
3840            // can only set a `true` to `true`, which is not worth the
3841            // resolve it costs -- the same reasoning `increment_tick`
3842            // already applies one level up.
3843            if C::NUMBER_OF_CORES > 1 {
3844                // SMP: the tick's yield decision is made per core at the end
3845                // of `increment_tick`, from `xYieldPendings`.
3846                if C::USE_PREEMPTION {
3847                    self.yield_for_task(task, woken);
3848                }
3849            } else if C::USE_PREEMPTION && !switch_required && woken > running {
3850                switch_required = true;
3851            }
3852        }
3853    }
3854
3855    /// `taskSWITCH_DELAYED_LISTS()`.
3856    // Out of line for the same reason: the overflow swap happens when the
3857    // tick count wraps, which is once in a very long while.
3858    // In line at its ONE caller (A3): win 26. It carried #[inline(never)] AND a
3859    // later #[inline] together, so the A3 change was dead from the day it was
3860    // written -- and rustc said so on every build.
3861    #[inline(always)]
3862    fn switch_delayed_lists(&mut self) {
3863        self.delayed_swapped = !self.delayed_swapped;
3864        self.overflows = self.overflows.wrapping_add(1);
3865        self.reset_next_task_unblock_time();
3866    }
3867
3868    /// `prvResetNextTaskUnblockTime`.
3869    ///
3870    /// ★ The EMPTY case is `MAX_DELAY`, not the list's own maximum.
3871    ///
3872    /// `head_value` cannot answer "empty": on an empty list the head IS the end
3873    /// marker, whose value is `u64::MAX` so that sorted inserts always find a
3874    /// larger value to stop at. So `unwrap_or` never fired and a reset over an
3875    /// empty delayed list stored `u64::MAX`, where the C stores `portMAX_DELAY`:
3876    ///
3877    /// ```c
3878    /// if( listLIST_IS_EMPTY( pxDelayedTaskList ) != pdFALSE ) {
3879    ///     xNextTaskUnblockTime = portMAX_DELAY;
3880    /// ```
3881    ///
3882    /// Unobservable in the corpus -- the difference only shows at a tick equal
3883    /// to `MAX_DELAY`, and `switch_delayed_lists` resets at the wrap anyway --
3884    /// which is why a test had PINNED the divergence rather than a gate
3885    /// catching it. Asking the emptiness question directly matches the C and
3886    /// makes `next_unblock_time <= MAX_DELAY` an invariant of the field.
3887    ///
3888    /// That invariant is worth five instructions per TICK: `increment_tick`
3889    /// compares `next` against this field, and while the field could hold a
3890    /// value wider than the tick that comparison had to be a full 64-bit one on
3891    /// a 32-bit target -- a second load, a `snez` and an `or` on top of the
3892    /// ordinary compare. With the invariant true, the comparison masks to the
3893    /// tick width and collapses. `bench/tick-work`: tick_idle and tick_delayed
3894    /// 14 -> 9 against the C's 15, 0.93x -> 0.60x.
3895    /// # Out of line, and that is load-bearing
3896    ///
3897    /// `switch_delayed_lists` is `#[inline(always)]` (win 26) and it calls this,
3898    /// so this body lands inside `increment_tick` -- on the arm taken when the
3899    /// tick WRAPS, i.e. once in `MAX_DELAY` ticks. Asking the emptiness question
3900    /// above made the body bigger, and in line that cost the tick row
3901    /// **14 -> 34** on `bench/tick-work`: twenty instructions of register
3902    /// pressure on every tick, to serve an arm that had not run once in the
3903    /// measurement. `rusty-compiler-leverage` A2, and the largest single
3904    /// instance of it this kernel has produced.
3905    #[cold]
3906    #[inline(never)]
3907    fn reset_next_task_unblock_time(&mut self) {
3908        let delayed = self.delayed_list();
3909        self.next_unblock_time = if self.lists.is_empty_of(delayed) {
3910            Self::MAX_DELAY
3911        } else {
3912            self.lists.head_value(delayed).unwrap_or(Self::MAX_DELAY)
3913        };
3914    }
3915
3916    // ------------------------------------------------------------- delay --
3917
3918    /// `vTaskDelay`.
3919    ///
3920    /// # Errors
3921    /// A list error surfaces as itself.
3922    pub fn delay(&mut self, ticks: u64) -> Result<()> {
3923        let caller = self.cur();
3924        let mut already_yielded = false;
3925        if ticks > 0 {
3926            self.suspend_all();
3927            {
3928                let current = self.cur();
3929                self.trace_task(current, |task, name| Event::TaskDelay { task, name, ticks });
3930                self.add_current_task_to_delayed_list(ticks, false)?;
3931            }
3932            already_yielded = self.resume_all();
3933        }
3934        // taskYIELD_WITHIN_API(), outside any critical section — unless a
3935        // tick switched us out somewhere above, in which case this line is
3936        // on a stack that is not running, and runs when the task does.
3937        if !already_yielded {
3938            if self.cur() == caller {
3939                self.port_yield();
3940            } else {
3941                self.owe_yield(caller);
3942            }
3943        }
3944        Ok(())
3945    }
3946
3947    /// `xTaskAbortDelay`: pull a task out of the Blocked state early.
3948    ///
3949    /// `true` is `pdPASS` — the task really was blocked and is now ready.
3950    /// A task that was not blocked is left alone and `false` comes back,
3951    /// which is the C's `pdFAIL`.
3952    ///
3953    /// The shape matters as much as the effect: the whole thing runs with
3954    /// the scheduler suspended, the event list is touched inside a critical
3955    /// section of its own because an interrupt can reach it, and a yield
3956    /// that the higher priority of the woken task calls for is *pended*
3957    /// rather than taken, so it happens on the way out of `xTaskResumeAll`.
3958    ///
3959    /// # Errors
3960    /// [`Error::Gone`] for a stale handle.
3961    pub fn abort_delay(&mut self, task: TaskHandle) -> Result<bool> {
3962        if !self.tcbs.contains(task) {
3963            return Err(Error::Gone);
3964        }
3965        self.suspend_all();
3966        // eTaskGetState: a critical section of its own on any task but the
3967        // running one, and the running one is never Blocked.
3968        let blocked = self.task_state_get(task)? == TaskState::Blocked;
3969        if !blocked {
3970            let _ = self.resume_all();
3971            return Ok(false);
3972        }
3973        let item = Self::state_item(task);
3974        // `remove` is its own guard, as above.
3975        let _ = self.lists.remove(item);
3976        self.enter_critical();
3977        {
3978            let event = Self::event_item(task);
3979            // `remove` answers `NotActive` when the item is in no list, so
3980            // it is its own guard -- and `is_ok` is exactly "something was
3981            // removed", which is what the flag below depends on.
3982            if self.lists.remove(event).is_ok() {
3983                if let Some(f) = self.delay_aborted.get_mut(task.index() as usize) {
3984                    *f = true;
3985                }
3986            }
3987        }
3988        self.exit_critical();
3989        self.add_task_to_ready_list(task)?;
3990        // configUSE_PREEMPTION, one core: pend the yield rather than take
3991        // it, so it lands where `xTaskResumeAll` puts it.
3992        let woken = self.tcbs.resolve(task).map(|t| t.priority).unwrap_or(0);
3993        if C::NUMBER_OF_CORES > 1 {
3994            let _ = self.smp_readied(task, woken);
3995        } else if woken > self.current_priority() {
3996            self.set_pending_here(true);
3997        }
3998        let _ = self.resume_all();
3999        Ok(true)
4000    }
4001
4002    /// `prvAddCurrentTaskToDelayedList`.
4003    pub(crate) fn add_current_task_to_delayed_list(
4004        &mut self,
4005        ticks: u64,
4006        can_block_indefinitely: bool,
4007    ) -> Result<()> {
4008        // Both masked, and for the reason `increment_tick`'s comparison is: at a
4009        // 32-bit tick width the high half of every tick value is invariantly
4010        // zero, LLVM cannot know that of a field it loaded or a parameter it was
4011        // handed, and without being told it emits full 64-bit compares on a
4012        // 32-bit target. Three of them live below -- `ticks == MAX_DELAY`,
4013        // `wake_at < now`, and `wake_at < next_unblock_time`.
4014        //
4015        // `self.tick` is masked by every writer (`increment_tick` and
4016        // `step_tick` both `& MAX_DELAY`). `ticks` reaches here from
4017        // `check_for_timeout`, whose answers are all at most `MAX_DELAY`, and
4018        // from `begin_wait`, which truncates what it stores. At a 64-bit width
4019        // `MAX_DELAY` is `u64::MAX` and both masks fold to nothing.
4020        debug_assert_eq!(
4021            self.tick,
4022            self.tick & Self::MAX_DELAY,
4023            "the tick is wider than its own width"
4024        );
4025        let now = self.tick & Self::MAX_DELAY;
4026        let ticks = ticks & Self::MAX_DELAY;
4027        let current = self.cur();
4028        // B1, and this function is where it pays most: by its own note below it
4029        // is the single largest consumer of the blocking workload, and it
4030        // derives FOUR bound-checked accesses from an index it read out of
4031        // memory -- `delay_aborted`, then `state_item` feeding `lists.remove`
4032        // and one of three inserts. `self.cur()` is a live task on every path
4033        // that reaches here, but it is a FIELD, so nothing carries that across
4034        // the calls above.
4035        if current.index() as usize >= TASKS {
4036            return Err(Error::Gone);
4037        }
4038        // About to enter a delayed list, so the abort flag is cleared here
4039        // and can only be seen set by a task that really was aborted.
4040        if let Some(f) = self.delay_aborted.get_mut(current.index() as usize) {
4041            *f = false;
4042        }
4043        let item = Self::state_item(current);
4044        // `uxListRemove` already answers `NotActive` for an item that is
4045        // in no list, and both it and the container test are one read of
4046        // the same node — so asking first was reading it twice. This is
4047        // the running task's state item, so it is in the ready list and
4048        // the test passed anyway.
4049        let _ = self.lists.remove(item);
4050        if ticks == Self::MAX_DELAY && can_block_indefinitely {
4051            self.lists.insert_end(Self::suspended_list(), item)?;
4052            return Ok(());
4053        }
4054        let wake_at = now.wrapping_add(ticks) & Self::MAX_DELAY;
4055        // `vListInsert` sets the item's value from the one it sorts by, so
4056        // both arms below write `wake_at` themselves. Setting it here as
4057        // well was a second lookup of the same item to store the same
4058        // number into it.
4059        if wake_at < now {
4060            self.trace_task(current, |task, name| {
4061                Event::MovedTaskToOverflowDelayedList { task, name }
4062            });
4063            let list = self.overflow_delayed_list();
4064            self.lists.insert(list, item, wake_at)?;
4065        } else {
4066            self.trace_task(current, |task, name| Event::MovedTaskToDelayedList {
4067                task,
4068                name,
4069            });
4070            let list = self.delayed_list();
4071            // ---- append, when the wake time says we may ------------------
4072            //
4073            // `vListInsert` walks from the head until it finds a larger
4074            // value, so a delayed list `n` deep costs `n` steps -- and this
4075            // function is the single largest consumer of the blocking
4076            // workload, most of it that walk.
4077            //
4078            // The walk is almost always finding the same answer: the END.
4079            // `wake_at` is `now + ticks` and `now` only increases, so tasks
4080            // blocking with comparable timeouts produce ASCENDING wake times
4081            // and belong after everything already there.
4082            //
4083            // `insert_sorted` takes that shortcut, and it asks this caller
4084            // for exactly ONE thing: that the list is already sorted. The
4085            // delayed list is, because it is only ever built by sorted
4086            // inserts and by this append -- so its last item carries its
4087            // largest value.
4088            //
4089            // The other two promises this call site used to make are gone
4090            // into the list, where they are written once instead of at
4091            // every future call site: the `>=` tie rule, and the fact that
4092            // `insert_end` only appends while the cursor is at the marker.
4093            // Both were silent when wrong. See `ListsOf::insert_sorted`,
4094            // and `ListsOf::is_sorted` for checking the one that is left.
4095            self.lists.insert_sorted(list, item, wake_at)?;
4096            // Masked for the same reason `increment_tick`'s comparison is, and
4097            // it is the second site of that win: `next_unblock_time <=
4098            // MAX_DELAY` is an invariant of the field (see
4099            // `reset_next_task_unblock_time`, which is what made it one), and
4100            // `wake_at` was masked where it was computed above. So at a 32-bit
4101            // tick width the high half of both is invariantly zero, and without
4102            // saying so LLVM emits a full 64-bit compare on a 32-bit target --
4103            // a second load, a `snez` and an `or` on top of the ordinary one.
4104            //
4105            // At a 64-bit width `MAX_DELAY` is `u64::MAX` and the mask folds to
4106            // nothing, so that configuration pays no instruction for this.
4107            debug_assert_eq!(
4108                self.next_unblock_time,
4109                self.next_unblock_time & Self::MAX_DELAY,
4110                "next_unblock_time holds a value wider than the tick"
4111            );
4112            if wake_at < self.next_unblock_time & Self::MAX_DELAY {
4113                self.next_unblock_time = wake_at;
4114            }
4115        }
4116        Ok(())
4117    }
4118
4119    // --------------------------------------------------- suspend / resume --
4120
4121    /// `vTaskSuspend`. `None` suspends the calling task.
4122    ///
4123    /// # Errors
4124    /// [`Error::Gone`] for a stale handle.
4125    pub fn suspend(&mut self, task: Option<TaskHandle>) -> Result<()> {
4126        let caller = self.cur();
4127        let target = task.unwrap_or(caller);
4128        self.enter_critical();
4129        {
4130            if !self.tcbs.contains(target) {
4131                self.exit_critical();
4132                return Err(Error::Gone);
4133            }
4134            self.trace_task(target, |task, name| Event::TaskSuspend { task, name });
4135            let item = Self::state_item(target);
4136            // `remove` is its own guard: it answers `NotActive` for an item
4137            // in no list, and the result was discarded either way.
4138            let _ = self.lists.remove(item);
4139            let event = Self::event_item(target);
4140            if self.lists.container(event)?.is_some() {
4141                let _ = self.lists.remove(event);
4142            }
4143            self.lists.insert_end(Self::suspended_list(), item)?;
4144            // `vTaskSuspend`, `tasks.c`:
4145            //
4146            // ```c
4147            // if( pxTCB->ucNotifyState[ x ] == taskWAITING_NOTIFICATION )
4148            // {
4149            //     /* The task was blocked to wait for a notification, but
4150            //      * is now suspended, so no notification was received. */
4151            //     pxTCB->ucNotifyState[ x ] = taskNOT_WAITING_NOTIFICATION;
4152            // }
4153            // ```
4154            //
4155            // Not tidying: `eTaskGetState` calls a task on the suspended
4156            // list BLOCKED rather than SUSPENDED if any of its notification
4157            // slots is still waiting, so leaving the flag set makes a
4158            // suspended task report as blocked for ever.
4159            // `TaskNotify.c:498` asserts exactly that, in a timer callback
4160            // that suspends a task which is waiting for a notification --
4161            // and a task that reports the wrong state there is a task the
4162            // rest of that test never resumes correctly.
4163            if let Ok(tcb) = self.tcbs.resolve_mut(target) {
4164                for slot in &mut tcb.notify_state {
4165                    if *slot == NotifyState::Waiting {
4166                        *slot = NotifyState::NotWaiting;
4167                    }
4168                }
4169            }
4170        }
4171        self.exit_critical();
4172        // A second, separate critical section — two outermost exits, which
4173        // is what the C does and therefore what the tick count depends on.
4174        if self.running {
4175            self.enter_critical();
4176            self.reset_next_task_unblock_time();
4177            self.exit_critical();
4178        }
4179        if C::NUMBER_OF_CORES > 1 {
4180            // `vTaskSuspend`: the core running `target` switches away -- this
4181            // one now, another by `prvYieldCore`.
4182            if self.running {
4183                // By slot index, as `yield_for_task`: `target` was found
4184                // live above, and so is every core's current task once the
4185                // scheduler runs.
4186                let slot = target.index();
4187                let core = (0..Self::cores())
4188                    .find(|&c| self.current_of(c).index() == slot && !self.requested_on(c));
4189                match core {
4190                    Some(core) if core == self.core() => self.port_yield(),
4191                    Some(core) => self.yield_core(core),
4192                    None => {}
4193                }
4194            }
4195            return Ok(());
4196        }
4197        if target == caller {
4198            if self.running {
4199                // portYIELD_WITHIN_API(), outside the section.
4200                if self.cur() == caller {
4201                    self.port_yield();
4202                } else {
4203                    self.owe_yield(caller);
4204                }
4205            } else if self.lists.len(Self::suspended_list()).unwrap_or(0) != self.task_count {
4206                self.switch_context();
4207            }
4208        }
4209        Ok(())
4210    }
4211
4212    /// `vTaskResume`.
4213    ///
4214    /// # Errors
4215    /// [`Error::Gone`] for a stale handle.
4216    pub fn resume(&mut self, task: TaskHandle) -> Result<()> {
4217        // One core: not the running task. Two cores: only not NULL -- the C
4218        // enters the section for any task, the running one included, because
4219        // SMP asks inside it whether the task is suspended. That section's
4220        // exit is a yield point, so resuming a running task can take a yield
4221        // an ISR left pending (the API differential, fresh seed 0x3c6ef74a,
4222        // two cores, step 22597).
4223        if task.is_null() || (C::NUMBER_OF_CORES == 1 && task == self.cur()) {
4224            return Ok(());
4225        }
4226        if !self.tcbs.contains(task) {
4227            return Err(Error::Gone);
4228        }
4229        self.enter_critical();
4230        {
4231            if self.task_is_suspended(task)? {
4232                // Read once and handed to `add_task_to_ready_list_at`, as
4233                // `remove_from_event_list` does: after the list edits LLVM
4234                // cannot prove the TCB unchanged, and resolved it twice more.
4235                let resumed = self.tcbs.resolve(task)?.priority;
4236                self.trace_task(task, |task, name| Event::TaskResume { task, name });
4237                let _ = self.lists.remove(Self::state_item(task));
4238                self.add_task_to_ready_list_at(task, resumed)?;
4239                // taskYIELD_ANY_CORE_IF_USING_PREEMPTION: inside the
4240                // section, and only when the resumed task outranks the
4241                // running one.
4242                if C::NUMBER_OF_CORES > 1 {
4243                    if self.smp_readied(task, resumed) {
4244                        self.port_yield();
4245                    }
4246                } else if C::USE_PREEMPTION && self.current_priority() < resumed {
4247                    self.port_yield();
4248                }
4249            }
4250        }
4251        self.exit_critical();
4252        Ok(())
4253    }
4254
4255    /// `prvTaskIsTaskSuspended`: on the suspended list, not on the pending
4256    /// ready list, and not waiting on an event.
4257    fn task_is_suspended(&self, task: TaskHandle) -> Result<bool> {
4258        if self.lists.container(Self::state_item(task))? != Some(Self::suspended_list()) {
4259            return Ok(false);
4260        }
4261        let event_list = self.lists.container(Self::event_item(task))?;
4262        if event_list == Some(Self::pending_ready_list()) {
4263            return Ok(false);
4264        }
4265        Ok(event_list.is_none())
4266    }
4267
4268    // -------------------------------------------------------- priorities --
4269
4270    /// `vTaskDelete`. `None` deletes the calling task.
4271    ///
4272    /// # Two paths, and the C picks between them for a reason
4273    ///
4274    /// Deleting *another* task frees its TCB on the spot. Deleting the
4275    /// **running** task cannot: the caller is still executing out of it. The
4276    /// C parks such a task on `xTasksWaitingTermination` and lets the idle
4277    /// task reclaim it, which is what `prvCheckTasksWaitingTermination` is
4278    /// for — so [`Kernel::check_tasks_waiting_termination`] is where the
4279    /// deferred half lands here, called from the idle body at the same point
4280    /// in the same loop.
4281    ///
4282    /// # The frees are outside the critical section, and they cost exits
4283    ///
4284    /// `prvDeleteTCB` runs **after** `taskEXIT_CRITICAL()` and frees twice —
4285    /// `vPortFreeStack( pxStack )` then `vPortFree( pxTCB )`. On the oracle's
4286    /// heap every free suspends the scheduler, so a delete is one exit for
4287    /// its own section plus two for the frees, in that order. Folding the
4288    /// frees into the section, or forgetting them, moves every later tick.
4289    ///
4290    /// # Errors
4291    /// [`Error::Gone`] for a stale handle.
4292    pub fn task_delete(&mut self, task: Option<TaskHandle>) -> Result<()> {
4293        // `prvGetTCBFromHandle`: NULL is the calling task.
4294        let target = task.unwrap_or(self.cur());
4295        self.tcbs.resolve(target)?;
4296
4297        self.enter_critical();
4298        // `uxListRemove( &( pxTCB->xStateListItem ) )`. The C follows this
4299        // with `taskRESET_READY_PRIORITY`, which is an EMPTY MACRO under the
4300        // oracle's `configUSE_PORT_OPTIMISED_TASK_SELECTION 0` — there is no
4301        // priority bitmap to clear, and `vTaskSwitchContext` rediscovers the
4302        // top priority by walking down. Nothing to model.
4303        let state = Self::state_item(target);
4304        if matches!(self.lists.container(state), Ok(Some(_))) {
4305            let _ = self.lists.remove(state);
4306        }
4307        // `if( listLIST_ITEM_CONTAINER( &( pxTCB->xEventListItem ) ) != NULL )`
4308        // — a task blocked on a queue is on two lists, and both must let go.
4309        let event = Self::event_item(target);
4310        if matches!(self.lists.container(event), Ok(Some(_))) {
4311            let _ = self.lists.remove(event);
4312        }
4313        // `uxTaskNumber++` is not modelled: it exists so kernel-aware
4314        // debuggers can tell the task lists changed, and nothing the trace
4315        // records reads it.
4316
4317        // `xSchedulerRunning != pdFALSE && taskTASK_IS_RUNNING_OR_SCHEDULED_
4318        // _TO_YIELD( pxTCB )`, which at `configNUMBER_OF_CORES 1` is exactly
4319        // `pxTCB == pxCurrentTCB`.
4320        // SMP: a task any core still holds -- running, or asked to yield --
4321        // cannot have its TCB freed under that core (`vTaskDelete`'s
4322        // `taskTASK_IS_RUNNING_OR_SCHEDULED_TO_YIELD`).
4323        let holder = if C::NUMBER_OF_CORES > 1 {
4324            self.held_core(target)
4325        } else {
4326            None
4327        };
4328        let defer = self.running
4329            && if C::NUMBER_OF_CORES > 1 {
4330                holder.is_some()
4331            } else {
4332                target == self.cur()
4333            };
4334        if defer {
4335            // `vListInsertEnd( &xTasksWaitingTermination, ... )` and
4336            // `++uxDeletedTasksWaitingCleanUp`. The count is NOT decremented
4337            // here — the idle task does that when it reaps.
4338            // In deletion order, as `vListInsertEnd` puts it at the tail.
4339            self.reap_next = self.reap_next.wrapping_add(1).max(1);
4340            let seq = self.reap_next;
4341            if let Ok(tcb) = self.tcbs.resolve_mut(target) {
4342                tcb.reap_seq = seq;
4343            }
4344            self.reap_waiting = self.reap_waiting.wrapping_add(1);
4345            self.trace_task(target, |task, name| Event::TaskDelete { task, name });
4346        } else {
4347            self.task_count = self.task_count.saturating_sub(1);
4348            self.trace_task(target, |task, name| Event::TaskDelete { task, name });
4349            self.reset_next_task_unblock_time();
4350        }
4351        self.exit_critical();
4352
4353        // `if( xDeleteTCBInIdleTask != pdTRUE ) { prvDeleteTCB( pxTCB ); }`,
4354        // outside the section.
4355        if !defer {
4356            self.delete_tcb(target);
4357        }
4358
4359        // `taskYIELD_WITHIN_API()`. The C guards this with
4360        // `xSchedulerRunning && pxTCB == pxCurrentTCB`, which is the same
4361        // condition that chose the deferred path.
4362        if defer {
4363            if C::NUMBER_OF_CORES > 1 {
4364                // The holder switches away: this core now, another core by
4365                // `prvYieldCore` -- unless it was already asked.
4366                match holder {
4367                    Some(core) if core == self.core() => self.port_yield(),
4368                    Some(core) if self.running_core(target) == Some(core) => {
4369                        self.yield_core(core);
4370                    }
4371                    _ => {}
4372                }
4373            } else {
4374                self.port_yield();
4375            }
4376        }
4377        Ok(())
4378    }
4379
4380    /// `prvDeleteTCB`: two frees, and on the oracle's heap each one suspends
4381    /// the scheduler, so each is one outermost exit.
4382    fn delete_tcb(&mut self, task: TaskHandle) {
4383        // `vPortFreeStack( pxTCB->pxStack )`.
4384        self.account_for_allocation();
4385        // `vPortFree( pxTCB )`.
4386        self.account_for_allocation();
4387        let _ = self.tcbs.remove(task);
4388
4389        // The index is now free for reuse, so mark the task UNSTARTED.
4390        //
4391        // The C frees the TCB and a later `xTaskCreate` gets fresh memory,
4392        // so a new task cannot inherit anything. Our arena hands back the
4393        // same index, and the per-index bookkeeping — owed exits, an owed
4394        // yield, an owed trace line — is keyed on that index, not on the
4395        // TCB. Without this a task deleted while it still owed the tail of
4396        // a call bequeaths that debt to its successor, and the successor
4397        // pays an exit it never incurred: exactly how `death`'s second
4398        // cycle first diverged, one exit early, 1,100 lines after the first
4399        // cycle had matched perfectly.
4400        //
4401        // Clearing this ONE flag is the whole fix, and deliberately so.
4402        // `hand_over` already empties all three of those on a task's first
4403        // switch-in, and a reused index is a first switch-in again. Zeroing
4404        // them here as well would work and would be worse: two places that
4405        // must agree about what a fresh task owes, and a gate that cannot
4406        // catch either one being dropped because the other still covers it.
4407        if let Some(f) = self.started.get_mut(task.index() as usize) {
4408            *f = false;
4409        }
4410        // `ucDelayAborted`, which is NOT owed-call state and which nothing
4411        // clears on a switch-in: in the C it is a TCB field, freed with the
4412        // TCB. A task aborted out of an event list and deleted before its call
4413        // consumed the flag left it set for the next task on this index, whose
4414        // first blocking call then timed out at once -- `xTaskCheckForTimeOut`
4415        // reads the flag BEFORE the task is placed, and placing is what
4416        // clears it (the API differential, fresh seed 0x3c6ef74a, two cores,
4417        // step 25739: a new task's peek returned empty instead of blocking).
4418        if let Some(f) = self.delay_aborted.get_mut(task.index() as usize) {
4419            *f = false;
4420        }
4421    }
4422
4423    /// `prvCheckTasksWaitingTermination`, called from the idle task.
4424    ///
4425    /// # This must cost NOTHING when nothing is pending
4426    ///
4427    /// The C's body is `while( uxDeletedTasksWaitingCleanUp > 0 )`, so an
4428    /// idle loop with no deleted task takes no critical section and pays no
4429    /// exit. That is what lets this be wired into the idle body without
4430    /// disturbing a single scenario in the corpus — and it is the property
4431    /// the corpus is checking when it stays byte-identical.
4432    ///
4433    /// MORE than one task can be pending, which this used to deny: a task
4434    /// that deletes itself yields to the next ready task, not to idle, and
4435    /// that task can delete itself too before idle ever runs. Each waits, in
4436    /// order, as on the C's list.
4437    pub fn check_tasks_waiting_termination(&mut self) {
4438        // Nothing deleted is the answer on almost every idle pass: say so here
4439        // rather than in a call to the out-of-line body.
4440        if self.reap_waiting == 0 {
4441            return;
4442        }
4443        self.reap_waiting_tasks();
4444    }
4445
4446    /// `prvCheckTasksWaitingTermination`'s loop, one core and two: one
4447    /// critical section per task reclaimed, the oldest first. On two cores a
4448    /// head that a core still holds -- running it, or asked to yield it --
4449    /// ends the pass after its section, "and try again next time".
4450    #[cold]
4451    #[inline(never)]
4452    fn reap_waiting_tasks(&mut self) {
4453        while self.reap_waiting > 0 {
4454            self.enter_critical();
4455            let Some(task) = self.reap_head() else {
4456                // The count says a task waits and no TCB says so: nothing to
4457                // free. Unreachable while the two are kept together.
4458                self.reap_waiting = 0;
4459                self.exit_critical();
4460                return;
4461            };
4462            if C::NUMBER_OF_CORES > 1 && self.held_by_any_core(task) {
4463                self.exit_critical();
4464                return;
4465            }
4466            if let Ok(tcb) = self.tcbs.resolve_mut(task) {
4467                tcb.reap_seq = 0;
4468            }
4469            self.task_count = self.task_count.saturating_sub(1);
4470            self.reap_waiting = self.reap_waiting.saturating_sub(1);
4471            self.exit_critical();
4472            self.delete_tcb(task);
4473        }
4474    }
4475
4476    /// `listGET_OWNER_OF_HEAD_ENTRY( &xTasksWaitingTermination )`: the task
4477    /// that has waited longest. A walk over the arena, on an idle pass that
4478    /// found a task waiting -- which is rare, and the arena is small.
4479    fn reap_head(&self) -> Option<TaskHandle> {
4480        let mut head: Option<(u32, TaskHandle)> = None;
4481        for (task, tcb) in self.tcbs.iter() {
4482            if tcb.reap_seq != 0 && head.is_none_or(|(seq, _)| tcb.reap_seq < seq) {
4483                head = Some((tcb.reap_seq, task));
4484            }
4485        }
4486        head.map(|(_, task)| task)
4487    }
4488
4489    /// `vTaskPrioritySet`. `None` is the calling task.    /// `vTaskPrioritySet`. `None` is the calling task.
4490    ///
4491    /// # Errors
4492    /// [`Error::Gone`] for a stale handle.
4493    pub fn set_priority(&mut self, task: Option<TaskHandle>, new_priority: u8) -> Result<()> {
4494        let new_priority = new_priority.min(C::MAX_PRIORITIES.saturating_sub(1));
4495        let target = task.unwrap_or(self.cur());
4496        let mut yield_required = false;
4497        self.enter_critical();
4498        {
4499            if !self.tcbs.contains(target) {
4500                self.exit_critical();
4501                return Err(Error::Gone);
4502            }
4503            self.trace_task(target, |task, name| Event::TaskPrioritySet {
4504                task,
4505                name,
4506                priority: Self::priority_value(new_priority),
4507            });
4508            let base = self.tcbs.resolve(target)?.base_priority;
4509            // SMP (`vTaskPrioritySet`): raising readies a contender anywhere
4510            // (`taskYIELD_ANY_CORE`); lowering a RUNNING task yields the core
4511            // running it (`taskYIELD_TASK_CORE`), which may not be this one.
4512            let mut smp_yield_for = C::NUMBER_OF_CORES > 1 && new_priority > base;
4513            let smp_yield_core = if C::NUMBER_OF_CORES > 1 && new_priority < base {
4514                self.running_core(target)
4515            } else {
4516                None
4517            };
4518            if base != new_priority {
4519                if new_priority > base {
4520                    if target != self.cur() && new_priority > self.current_priority() {
4521                        yield_required = true;
4522                    }
4523                } else if target == self.cur() {
4524                    yield_required = true;
4525                }
4526                let used_on_entry = self.tcbs.resolve(target)?.priority;
4527                {
4528                    let tcb = self.tcbs.resolve(target)?;
4529                    let raise = tcb.base_priority == tcb.priority || new_priority > tcb.priority;
4530                    if raise {
4531                        self.set_task_priority(target, new_priority)?;
4532                    }
4533                    self.tcbs.resolve_mut(target)?.base_priority = new_priority;
4534                }
4535                // "Only reset the event list item value if the value is not
4536                // being used for anything else" (taskEVENT_LIST_ITEM_VALUE_IN_USE).
4537                self.set_event_priority_value(target, new_priority)?;
4538                let item = Self::state_item(target);
4539                if self.lists.container(item)? == Some(Self::ready_list(used_on_entry)) {
4540                    let _ = self.lists.remove(item);
4541                    self.add_task_to_ready_list(target)?;
4542                } else {
4543                    // Not ready (blocked or suspended): nothing to yield for.
4544                    smp_yield_for = false;
4545                }
4546                // taskYIELD_TASK_CORE_IF_USING_PREEMPTION: inside the
4547                // section.
4548                if C::NUMBER_OF_CORES > 1 {
4549                    if C::USE_PREEMPTION {
4550                        if let Some(core) = smp_yield_core {
4551                            self.yield_core(core);
4552                        } else if smp_yield_for {
4553                            let now = self.tcbs.resolve(target)?.priority;
4554                            self.yield_for_task(target, now);
4555                        }
4556                        if self.pending_here() {
4557                            self.port_yield();
4558                        }
4559                    }
4560                } else if yield_required && C::USE_PREEMPTION {
4561                    self.port_yield();
4562                }
4563            }
4564        }
4565        self.exit_critical();
4566        Ok(())
4567    }
4568
4569    // ------------------------------------------------- scheduler suspend --
4570
4571    /// `vTaskSuspendAll`.
4572    pub fn suspend_all(&mut self) {
4573        self.suspended_depth = self.suspended_depth.wrapping_add(1);
4574    }
4575
4576    /// Move everything an ISR readied onto the ready lists.
4577    ///
4578    /// Out of line because it is the rare half of [`Kernel::resume_all`]: the
4579    /// pending list is empty on essentially every call, and inlining this made
4580    /// every one of those calls carry a frame sized for the walk.
4581    ///
4582    /// **The GUARD is inlined and only the WALK is out of line.** Outlining
4583    /// the whole thing — which is what this used to do — meant every call paid
4584    /// a call and a frame to discover there was nothing to do, on a path taken
4585    /// three times inside `queue_send_blocking` and three more inside
4586    /// `queue_take_blocking`. Asking "is the list empty" is one list read; the
4587    /// walk behind it is what wanted outlining.
4588    #[inline(always)]
4589    fn drain_pending_ready(&mut self) {
4590        if self.lists.is_empty_of(Self::pending_ready_list()) {
4591            // Empty, or a list error: either way the walk below moves nothing,
4592            // so `moved_any` would stay false and the reset would not happen.
4593            return;
4594        }
4595        self.drain_pending_ready_walk();
4596    }
4597
4598    /// The walk itself. See [`Kernel::drain_pending_ready`].
4599    #[cold]
4600    #[inline(never)]
4601    fn drain_pending_ready_walk(&mut self) {
4602        let mut moved_any = false;
4603        while !self.lists.is_empty_of(Self::pending_ready_list()) {
4604            let Ok(Some(item)) = self.lists.head(Self::pending_ready_list()) else {
4605                break;
4606            };
4607            let Ok(task) = self.task_of_event_item(item) else {
4608                break;
4609            };
4610            let _ = self.lists.remove(Self::event_item(task));
4611            let _ = self.lists.remove(Self::state_item(task));
4612            if self.add_task_to_ready_list(task).is_err() {
4613                break;
4614            }
4615            moved_any = true;
4616            // One core: a task readied above the running one owes a yield.
4617            // Two cores: nothing. The C's SMP `xTaskResumeAll` makes no yield
4618            // decision here -- "all appropriate tasks yield at the moment a
4619            // task is added to xPendingReadyList" -- and every insertion
4620            // into that list runs `prvYieldForTask` already. Asking again
4621            // found the core the first ask had picked already requested and
4622            // picked ANOTHER: two cores both running idle, an ISR give under
4623            // suspension woke a task onto core 1, and the resume yielded
4624            // core 0 as well (the API differential, step 14312).
4625            if C::NUMBER_OF_CORES == 1 {
4626                let woken = self.tcbs.resolve(task).map(|t| t.priority).unwrap_or(0);
4627                if woken > self.current_priority() {
4628                    self.set_pending_here(true);
4629                }
4630            }
4631        }
4632        if moved_any {
4633            self.reset_next_task_unblock_time();
4634        }
4635    }
4636
4637    /// Run the ticks that landed while the scheduler was suspended.
4638    ///
4639    /// Out of line for the same reason as its sibling: `pended_ticks` is zero
4640    /// on essentially every call — and, as there, **only the loop is out of
4641    /// line now**. The guard is a field read, and it also skips the
4642    /// `pended_ticks = 0` store that the zero case was writing over a zero.
4643    #[inline(always)]
4644    fn unwind_pended_ticks(&mut self) {
4645        if self.pended_ticks == 0 {
4646            return;
4647        }
4648        self.unwind_pended_ticks_loop();
4649    }
4650
4651    /// The loop itself. See [`Kernel::unwind_pended_ticks`].
4652    #[cold]
4653    #[inline(never)]
4654    fn unwind_pended_ticks_loop(&mut self) {
4655        let mut pended = self.pended_ticks;
4656        while pended > 0 {
4657            if self.increment_tick() {
4658                self.set_pending_here(true);
4659            }
4660            pended = pended.saturating_sub(1);
4661        }
4662        self.pended_ticks = 0;
4663    }
4664
4665    /// Hand the port's critical-section exit tally to the trace sink.
4666    ///
4667    /// Gated on `T::EMITS` because `Port::exits` reads an ATOMIC counter, and
4668    /// LLVM may not delete an atomic load even when the value it produces is
4669    /// unused. With `NoTrace` the body folds to nothing while the `lw` behind it
4670    /// would stay, so every site was paying a load of a counter no sink was
4671    /// going to read. The gate is a compile-time constant, so an emitting sink
4672    /// is unchanged and a silent one loses the load entirely.
4673    #[inline(always)]
4674    pub(crate) fn note_exits(&mut self) {
4675        if T::EMITS {
4676            self.trace.note_exits(self.port.exits());
4677        }
4678    }
4679
4680    /// `xTaskResumeAll`; `true` when it yielded on the caller's behalf.
4681    ///
4682    /// Out of line for the cold callers; the body is [`Kernel::resume_all_inline`].
4683    #[inline(never)]
4684    pub fn resume_all(&mut self) -> bool {
4685        self.resume_all_inline()
4686    }
4687
4688    /// The body of [`Kernel::resume_all`], in line for the hot callers
4689    /// (one shared body under the `small` feature).
4690    #[cfg_attr(not(feature = "small"), inline(always))]
4691    #[cfg_attr(feature = "small", inline(never))]
4692    pub(crate) fn resume_all_inline(&mut self) -> bool {
4693        let mut already_yielded = false;
4694        self.enter_critical();
4695        {
4696            self.suspended_depth = self.suspended_depth.saturating_sub(1);
4697            if self.suspended_depth == 0 && self.task_count > 0 {
4698                if !self.lists.is_empty_of(Self::pending_ready_list()) {
4699                    self.drain_pending_ready();
4700                }
4701                if self.pended_ticks > 0 {
4702                    self.unwind_pended_ticks();
4703                }
4704                if self.pending_here() {
4705                    if C::USE_PREEMPTION {
4706                        already_yielded = true;
4707                    }
4708                    // taskYIELD_TASK_CORE_IF_USING_PREEMPTION: inside the
4709                    // section.
4710                    self.port_yield();
4711                }
4712            }
4713        }
4714        self.exit_critical();
4715        already_yielded
4716    }
4717
4718    // ------------------------------------------- the blocking-call frame --
4719
4720    /// Start (or continue) a blocking call's `for(;;)`.
4721    ///
4722    /// The first pass records the block time and the entry timestamp, the
4723    /// way `xEntryTimeSet` / `vTaskInternalSetTimeOutState` do; later
4724    /// passes leave the running total alone, because the `ticks` the caller
4725    /// passes is the original block time and the kernel is holding what is
4726    /// left of it.
4727    ///
4728    /// Answers the block time that is LEFT. Both call sites wanted it right
4729    /// afterwards and used to resolve the same TCB a second time to read a
4730    /// field this function is already holding a `&mut` to. The answer is exact
4731    /// rather than merely equal to `ticks`, because the frame may be an earlier
4732    /// pass's — in which case the remaining time is what the kernel has been
4733    /// counting down, not what the caller passed.
4734    ///
4735    /// `None` means no frame was written at all, so the caller can skip the
4736    /// `end_wait` that would tear one down.
4737    pub(crate) fn begin_wait(
4738        &mut self,
4739        task: TaskHandle,
4740        queue: QueueHandle,
4741        ticks: u64,
4742    ) -> Option<u64> {
4743        // ★ TRUNCATE TO THE TICK WIDTH, at the boundary, because that is what
4744        // the C does and it is what makes every later comparison 32-bit.
4745        //
4746        // C's `xTicksToWait` is a `TickType_t`. At a 32-bit configuration a
4747        // caller CANNOT express a wait longer than `MAX_DELAY` -- the value is
4748        // truncated by the assignment, so `2^32` ticks becomes 0 and
4749        // `u64::MAX` becomes `portMAX_DELAY`, i.e. forever. This API takes a
4750        // `u64` so the kernel is width-independent, which means it must do that
4751        // truncation itself rather than let a value the C could not represent
4752        // reach a wait frame.
4753        //
4754        // The masking is BEFORE the zero test on purpose: C's truncation
4755        // happens at the assignment, so a caller asking for exactly `2^32`
4756        // ticks gets C's answer -- do not block -- and not a very long wait.
4757        //
4758        // What it buys: every field of a `WaitFrame` is then provably no wider
4759        // than the tick, so `check_for_timeout`'s comparisons against them stop
4760        // being 64-bit ones on a 32-bit target. That is the same lever that took
4761        // `tick_idle` 14 -> 9; see `reset_next_task_unblock_time`.
4762        let ticks = ticks & Self::MAX_DELAY;
4763        // The zero-block-time exit, taken before anything is read. `wait_set`
4764        // mirrors the TCB's `entry_set` in one byte, which is the same trade
4765        // `end_wait` documents and makes — so the frame can be ruled out
4766        // without an arena lookup, and without reading the clock either.
4767        // Out of range reads as SET, so an impossible index still goes the
4768        // long way round.
4769        if ticks == 0 && !self.flag(task.index() as usize, Self::F_WAIT, true) {
4770            // `None`, not `Some(0)`: no frame was written, so the caller can
4771            // skip the `end_wait` that would tear one down. It reads the same
4772            // `wait_set` byte to find nothing to do.
4773            return None;
4774        }
4775        let tick = self.tick;
4776        let overflows = self.overflows;
4777        let Ok(tcb) = self.tcbs.resolve_mut(task) else {
4778            // A queue call made before the scheduler started, from what the
4779            // C would call `main()`: there is no task to keep a frame for,
4780            // and such a call never blocks — `xSemaphoreGive` on a fresh
4781            // semaphore is the usual one. The old read answered 0 here
4782            // (`unwrap_or(0)` on the same failed resolve), so 0 is what this
4783            // must answer to keep the call sites reading the same.
4784            return Some(0);
4785        };
4786        if tcb.wait.queue != queue || !tcb.wait.entry_set {
4787            tcb.wait = WaitFrame {
4788                queue,
4789                ticks: crate::timer::Split64::new(ticks),
4790                entering: crate::timer::Split64::new(tick),
4791                overflows,
4792                entry_set: true,
4793                inherited: false,
4794            };
4795        }
4796        let remaining = tcb.wait.ticks.get();
4797        // The frame is set either way on this path -- either it was written
4798        // just now, or it was already this queue's.
4799        self.set_flag(task.index() as usize, Self::F_WAIT, true);
4800        Some(remaining)
4801    }
4802
4803    /// The call finished, one way or the other.
4804    pub(crate) fn end_wait(&mut self, task: TaskHandle) {
4805        // Almost every call arrives with no frame to clear, and the mirror
4806        // says so in one byte where resolving the TCB to read `entry_set`
4807        // cost an arena lookup. Out of range reads as SET, so an impossible
4808        // index still takes the slow path below.
4809        let at = task.index() as usize;
4810        if !self.flag(at, Self::F_WAIT, true) {
4811            return;
4812        }
4813        self.set_flag(at, Self::F_WAIT, false);
4814        if let Ok(tcb) = self.tcbs.resolve_mut(task) {
4815            // Now that the frame is only set on the path that blocks, most
4816            // calls reach here with nothing to clear, and this was writing
4817            // a default frame over a default frame. `entry_set` is the
4818            // sentinel the C calls `xEntryTimeSet`, and it is the first
4819            // thing set — `inherited` is only reached far inside the
4820            // blocking path, so it cannot be true while this is false.
4821            if tcb.wait.entry_set {
4822                tcb.wait = WaitFrame::default();
4823            }
4824        }
4825    }
4826
4827    /// `xInheritanceOccurred`.
4828    pub(crate) fn wait_inherited(&self, task: TaskHandle) -> bool {
4829        self.tcbs
4830            .resolve(task)
4831            .map(|t| t.wait.inherited)
4832            .unwrap_or(false)
4833    }
4834
4835    /// `xInheritanceOccurred = xTaskPriorityInherit( ... )`: an ASSIGNMENT,
4836    /// made every time the take blocks. A waiter that inherited on its first
4837    /// block and not on a later one (its own priority lowered meanwhile) ends
4838    /// with `false`, and its timeout then disinherits nothing.
4839    pub(crate) fn set_wait_inherited(&mut self, task: TaskHandle, inherited: bool) {
4840        if let Ok(tcb) = self.tcbs.resolve_mut(task) {
4841            tcb.wait.inherited = inherited;
4842        }
4843    }
4844
4845    /// `xTaskCheckForTimeOut`: `true` when the block time has run out.
4846    /// Takes a critical section, as the C does, and charges the elapsed
4847    /// time against what is left.
4848    /// `None` is `pdTRUE` — the block time has run out. `Some(left)` is
4849    /// `pdFALSE`, and carries what is left of it.
4850    ///
4851    /// The block time is threaded out because both call sites asked for it
4852    /// immediately afterwards by resolving the TCB again, which
4853    /// the same TCB out of the arena again to read the very field the match
4854    /// below has just written.
4855    /// ★ `Option<NonZeroU64>`, not `Option<u64>`, and the reason is the
4856    /// CALLER's register budget.
4857    ///
4858    /// `Option<u64>` has no niche, so it is sixteen bytes — two registers —
4859    /// and `queue_take_blocking` holds it live across the branch that tests
4860    /// it. That function sits at a register cliff: six callee-saved pushes, a
4861    /// 72-byte frame, `caller`'s index spilled and re-read five times, and
4862    /// four separate attempts to make it cheaper all LOST because each added a
4863    /// live value or a branch (+200,668, +11,619, +254,429, +624,386; see
4864    /// `docs/LEDGER.md`). The one shape left is to make an existing value
4865    /// smaller, which is `rusty-compiler-leverage` B5 — the representation
4866    /// change that won −987,806 elsewhere by turning a 16-byte `Option` into
4867    /// one word.
4868    ///
4869    /// `Some(0)` is unreachable, which is what makes this free: the indefinite
4870    /// arm answers `MAX_DELAY` (non-zero at either tick width), and the
4871    /// still-waiting arm answers `held - elapsed` under `elapsed < held`, so at
4872    /// least one. `NonZeroU64::new` therefore maps every reachable `Some` to
4873    /// itself; a zero would become `None`, which is the same verdict a caller
4874    /// draws from zero ticks left anyway.
4875    pub(crate) fn check_for_timeout(&mut self, task: TaskHandle) -> Option<NonZeroU64> {
4876        self.enter_critical();
4877        // Prove the index in range ONCE. Three bounded reads below key on the
4878        // same task -- the abort flag, its clear, and the TCB resolve -- and an
4879        // out-of-range task already answers `None` through every one of them, so
4880        // saying so up front is free of behaviour and lets their checks fold.
4881        // `rusty-compiler-leverage` B1, the lever that took `switch_context`
4882        // -969,669 and the tick 14 -> 9 today.
4883        //
4884        // AFTER `enter_critical`, and the exit below is not optional: on the sim
4885        // an outermost critical-section exit IS the clock, so a return that
4886        // skipped it would move every later event in the trace and `conform`
4887        // would fail on all twenty-six scenarios.
4888        let index = task.index() as usize;
4889        if index >= TASKS {
4890            self.exit_critical();
4891            return None;
4892        }
4893        // An aborted delay is not the same as a time out, but it has the
4894        // same result: stop waiting.
4895        if self.delay_aborted.get(index).copied().unwrap_or(false) {
4896            if let Some(f) = self.delay_aborted.get_mut(index) {
4897                *f = false;
4898            }
4899            self.exit_critical();
4900            return None;
4901        }
4902        let now = self.tick;
4903        let overflows = self.overflows;
4904        let result = match self.tcbs.resolve_mut(task) {
4905            Ok(tcb) => {
4906                // Masked: `begin_wait` truncates what it stores to the tick
4907                // width, so these are no-ops on the VALUE and information to
4908                // LLVM, which cannot otherwise know a field it loaded is
4909                // narrow. Without them every comparison below is a 64-bit one.
4910                debug_assert_eq!(
4911                    tcb.wait.entering.get(),
4912                    tcb.wait.entering.get() & Self::MAX_DELAY,
4913                    "a wait frame's `entering` is wider than the tick"
4914                );
4915                debug_assert_eq!(
4916                    tcb.wait.ticks.get(),
4917                    tcb.wait.ticks.get() & Self::MAX_DELAY,
4918                    "a wait frame's `ticks` is wider than the tick"
4919                );
4920                let entering = tcb.wait.entering.get() & Self::MAX_DELAY;
4921                let held = tcb.wait.ticks.get() & Self::MAX_DELAY;
4922                let elapsed = now.wrapping_sub(entering) & Self::MAX_DELAY;
4923                if held == Self::MAX_DELAY {
4924                    // An indefinite block never times out. `MAX_DELAY` is
4925                    // non-zero at either tick width, so this is always `Some`.
4926                    debug_assert_ne!(Self::MAX_DELAY, 0, "MAX_DELAY must be non-zero");
4927                    NonZeroU64::new(Self::MAX_DELAY)
4928                } else if overflows != tcb.wait.overflows && now >= entering {
4929                    tcb.wait.ticks = crate::timer::Split64::new(0);
4930                    None
4931                } else if elapsed < held {
4932                    let left = held.wrapping_sub(elapsed);
4933                    tcb.wait.ticks = crate::timer::Split64::new(left);
4934                    tcb.wait.entering = crate::timer::Split64::new(now);
4935                    tcb.wait.overflows = overflows;
4936                    // `elapsed < held` on this arm, so `left` is at least one
4937                    // and this is always `Some`. The assert is what keeps the
4938                    // niche honest if that ever stops being true.
4939                    debug_assert_ne!(left, 0, "left must be non-zero on the still-waiting arm");
4940                    NonZeroU64::new(left)
4941                } else {
4942                    tcb.wait.ticks = crate::timer::Split64::new(0);
4943                    None
4944                }
4945            }
4946            Err(_) => None,
4947        };
4948        self.exit_critical();
4949        result
4950    }
4951
4952    /// `vTaskPlaceOnUnorderedEventList`: the item value carries the
4953    /// condition rather than the priority, so the item goes on the end and
4954    /// the list is never sorted.
4955    ///
4956    /// `taskEVENT_LIST_ITEM_VALUE_IN_USE` is set, as the C sets it: the same
4957    /// item is a priority-ordered event item the rest of the time, and the
4958    /// flag is how the priority setters know to leave a waiter's condition
4959    /// alone (`set_event_priority_value`). This used to be omitted on the
4960    /// argument that only the event-group code reads the value back; the
4961    /// priority setters read it too, and the API differential caught one
4962    /// rewriting a waiting task's mask (P1.5, two cores, step 4266).
4963    pub(crate) fn place_on_unordered_event_list(
4964        &mut self,
4965        list: ListId,
4966        value: u64,
4967        ticks: u64,
4968    ) -> Result<()> {
4969        let item = Self::event_item(self.cur());
4970        self.lists
4971            .set_value(item, value | EVENT_LIST_ITEM_VALUE_IN_USE)?;
4972        self.lists.insert_end(list, item)?;
4973        self.add_current_task_to_delayed_list(ticks, true)
4974    }
4975
4976    /// `vTaskRemoveFromUnorderedEventList`: write the answer into the item
4977    /// value, then wake the task.
4978    ///
4979    /// Unlike `xTaskRemoveFromEventList` this always goes straight to the
4980    /// ready list. It is only ever called with the scheduler suspended —
4981    /// `xEventGroupSetBits` holds it across the whole walk — and the C
4982    /// still bypasses the pending-ready list, because the value it just
4983    /// wrote is the task's return value and a second pass would overwrite
4984    /// it.
4985    pub(crate) fn remove_from_unordered_event_list(
4986        &mut self,
4987        item: ItemId,
4988        value: u64,
4989    ) -> Result<()> {
4990        self.lists
4991            .set_value(item, value | EVENT_LIST_ITEM_VALUE_IN_USE)?;
4992        let task = self.task_of_event_item(item)?;
4993        let _ = self.lists.remove(item);
4994        // `configUSE_TICKLESS_IDLE`: here the C resets BEFORE the task leaves
4995        // its delayed list, so the answer can still be the task's own wake.
4996        if C::USE_TICKLESS_IDLE {
4997            self.reset_next_task_unblock_time();
4998        }
4999        let _ = self.lists.remove(Self::state_item(task));
5000        self.add_task_to_ready_list(task)?;
5001        let woken = self.tcbs.resolve(task)?.priority;
5002        if C::NUMBER_OF_CORES > 1 {
5003            let _ = self.smp_readied(task, woken);
5004        } else if woken > self.current_priority() {
5005            self.set_pending_here(true);
5006        }
5007        Ok(())
5008    }
5009
5010    /// Rewrite `task`'s event item with `priority`'s ordering value -- unless
5011    /// the item holds an event-group wait's condition
5012    /// (`taskEVENT_LIST_ITEM_VALUE_IN_USE`). `vTaskPrioritySet`,
5013    /// `xTaskPriorityInherit` and `vTaskPriorityDisinheritAfterTimeout` all
5014    /// leave such an item alone; overwriting it turned a waiter's "bit 3,
5015    /// clear on exit" into "bits 0 and 1" (API differential P1.5).
5016    pub(crate) fn set_event_priority_value(
5017        &mut self,
5018        task: TaskHandle,
5019        priority: u8,
5020    ) -> Result<()> {
5021        let item = Self::event_item(task);
5022        if self.lists.value(item)? & EVENT_LIST_ITEM_VALUE_IN_USE == 0 {
5023            self.lists
5024                .set_value(item, u64::from(C::MAX_PRIORITIES.saturating_sub(priority)))?;
5025        }
5026        Ok(())
5027    }
5028
5029    /// `uxTaskResetEventItemValue`: read the answer the unblocker left, and
5030    /// put the item back to the priority order an ordinary event list wants.
5031    pub(crate) fn reset_event_item_value(&mut self, task: TaskHandle) -> Result<u64> {
5032        let item = Self::event_item(task);
5033        let value = self.lists.value(item)?;
5034        let priority = self.tcbs.resolve(task)?.priority;
5035        self.lists
5036            .set_value(item, u64::from(C::MAX_PRIORITIES.saturating_sub(priority)))?;
5037        Ok(value)
5038    }
5039
5040    /// Whether `task` is resuming an event-group wait rather than starting
5041    /// one. Taking it clears the marker.
5042    pub(crate) fn take_event_resume(&mut self, task: TaskHandle) -> bool {
5043        // Nobody is parked in a wait, so there is nothing to read — one field
5044        // load in place of an arena lookup.
5045        if self.event_resumes == 0 {
5046            return false;
5047        }
5048        match self.tcbs.resolve_mut(task) {
5049            Ok(tcb) if tcb.event_blocked => {
5050                tcb.event_blocked = false;
5051                self.event_resumes = self.event_resumes.wrapping_sub(1);
5052                true
5053            }
5054            _ => false,
5055        }
5056    }
5057
5058    /// Mark that `task` blocked inside an event-group wait, so the call it
5059    /// is inside resumes rather than restarts.
5060    pub(crate) fn set_event_resume(&mut self, task: TaskHandle) {
5061        if let Ok(tcb) = self.tcbs.resolve_mut(task) {
5062            // Count the slot, not the call.
5063            if !tcb.event_blocked {
5064                self.event_resumes = self.event_resumes.wrapping_add(1);
5065            }
5066            tcb.event_blocked = true;
5067        }
5068    }
5069
5070    /// `vTaskPlaceOnEventList`: sorted by priority, then blocked.
5071    // The attribute is load-bearing and the measurement is the reason. The
5072    // bound proof below grew this function past LLVM's own inlining threshold,
5073    // and outlining it turned a -668,718 row win in `queue_take_blocking` into
5074    // a **+154,439 LOSS on the program** -- A3: the frame and the argument
5075    // marshalling cost more than the proof saves. Asking for the inline puts it
5076    // back to -96,574.
5077    //
5078    // `#[inline]` and `#[inline(always)]` measured IDENTICALLY here, on both
5079    // axes, so the weaker word is the honest one: the 22 bytes of rv32 flash
5080    // this costs are the bound check duplicated at each site, not the
5081    // attribute's doing.
5082    #[inline]
5083    pub(crate) fn place_on_event_list(&mut self, list: ListId, ticks: u64) -> Result<()> {
5084        let current = self.cur();
5085        // B1, and the bound is the whole point: `event_item` is
5086        // `TASKS.saturating_add(index)`, which tells LLVM only that the result
5087        // fits a `u16` -- so `insert_keeping_value` still emits its own check
5088        // that the item names a real node, and the saturate costs a `mov` and a
5089        // `cmov` besides. Proving the index here instead makes the sum provably
5090        // below `2 * TASKS`, and both of those fold.
5091        //
5092        // It has to be proved HERE rather than relied on: `self.cur()` is a
5093        // live task on every path that reaches this function, but the calls in
5094        // between take `&mut self`, so LLVM cannot carry that across them.
5095        if current.index() as usize >= TASKS {
5096            self.note_stall(Stall::ListError);
5097            return Err(Error::Gone);
5098        }
5099        let item = Self::event_item(current);
5100        // The item keeps the value its call set; reading it out only to
5101        // hand it back made the list read the item twice and write it once
5102        // for no change.
5103        self.lists.insert_keeping_value(list, item)?;
5104        self.add_current_task_to_delayed_list(ticks, true)
5105    }
5106
5107    /// Take the trailing `portYIELD()` now, or owe it if a tick already
5108    /// switched us out of this call.
5109    pub(crate) fn yield_or_owe(&mut self, caller: TaskHandle) {
5110        if self.cur() == caller {
5111            self.port_yield();
5112        } else {
5113            self.owe_yield(caller);
5114        }
5115    }
5116
5117    /// `pvTaskIncrementMutexHeldCount`.
5118    pub(crate) fn increment_mutexes_held(&mut self, task: TaskHandle) {
5119        if let Ok(tcb) = self.tcbs.resolve_mut(task) {
5120            tcb.mutexes_held = tcb.mutexes_held.wrapping_add(1);
5121        }
5122    }
5123
5124    // ------------------------------------------------ priority inheritance --
5125
5126    /// `xTaskPriorityInherit`: lift the mutex holder to the waiter's
5127    /// priority. `true` when the holder is (or already was) lifted, which
5128    /// is what the waiter remembers so it can undo it on a timeout.
5129    #[cold]
5130    pub(crate) fn priority_inherit(&mut self, holder: TaskHandle) -> Result<bool> {
5131        if holder.is_null() || !self.tcbs.contains(holder) {
5132            return Ok(false);
5133        }
5134        let waiter_priority = self.current_priority();
5135        let (holder_priority, holder_base) = {
5136            let tcb = self.tcbs.resolve(holder)?;
5137            (tcb.priority, tcb.base_priority)
5138        };
5139        if holder_priority >= waiter_priority {
5140            // Already at least as urgent; the waiter still records that the
5141            // holder is running above its base, if it is.
5142            return Ok(holder_base < waiter_priority);
5143        }
5144        self.set_event_priority_value(holder, waiter_priority)?;
5145        let item = Self::state_item(holder);
5146        if self.lists.container(item)? == Some(Self::ready_list(holder_priority)) {
5147            let _ = self.lists.remove(item);
5148            self.set_task_priority(holder, waiter_priority)?;
5149            // The priority just set, handed down rather than read back
5150            // through a second resolve (`add_task_to_ready_list_at`).
5151            self.add_task_to_ready_list_at(holder, waiter_priority)?;
5152            // SMP: "The priority of the task is raised. Yield for this task
5153            // if it is not running." (`xTaskPriorityInherit`) -- whether or
5154            // not preemption is on: the C calls `prvYieldForTask` here
5155            // DIRECTLY (tasks.c:6798), the one call of it not made through
5156            // `taskYIELD_ANY_CORE_IF_USING_PREEMPTION` or under
5157            // `configUSE_PREEMPTION == 1`. This gated it on preemption too,
5158            // and no cooperative two-core run existed to show it.
5159            if C::NUMBER_OF_CORES > 1 && self.running_core(holder).is_none() {
5160                self.yield_for_task(holder, waiter_priority);
5161            }
5162        } else {
5163            self.set_task_priority(holder, waiter_priority)?;
5164        }
5165        self.trace_task(holder, |task, name| Event::TaskPriorityInherit {
5166            task,
5167            name,
5168            priority: Self::priority_value(waiter_priority),
5169        });
5170        Ok(true)
5171    }
5172
5173    /// `xTaskPriorityDisinherit`: giving the mutex back drops the
5174    /// inherited priority. `true` when a yield is wanted.
5175    #[cold]
5176    pub(crate) fn priority_disinherit(&mut self, holder: TaskHandle) -> Result<bool> {
5177        if holder.is_null() {
5178            return Ok(false);
5179        }
5180        // One lookup where there were three: `contains` asked the arena
5181        // whether the holder is there, `resolve` asked again to read its
5182        // three fields, and `resolve_mut` asked a third time to put one back
5183        // -- with nothing but a subtraction in between. A holder the arena
5184        // does not have still answers `Ok(false)`, which is what `contains`
5185        // was for.
5186        let (priority, base, remaining) = match self.tcbs.resolve_mut(holder) {
5187            Ok(tcb) => {
5188                let remaining = tcb.mutexes_held.saturating_sub(1);
5189                tcb.mutexes_held = remaining;
5190                (tcb.priority, tcb.base_priority, remaining)
5191            }
5192            Err(_) => return Ok(false),
5193        };
5194        if priority == base || remaining != 0 {
5195            return Ok(false);
5196        }
5197        let item = Self::state_item(holder);
5198        if self.lists.remove(item).is_err() {
5199            return Ok(false);
5200        }
5201        self.trace_task(holder, |task, name| Event::TaskPriorityDisinherit {
5202            task,
5203            name,
5204            priority: Self::priority_value(base),
5205        });
5206        self.set_task_priority(holder, base)?;
5207        let event_value = u64::from(C::MAX_PRIORITIES.saturating_sub(base));
5208        self.lists
5209            .set_value(Self::event_item(holder), event_value)?;
5210        // The priority just set, as in `priority_inherit`.
5211        self.add_task_to_ready_list_at(holder, base)?;
5212        // SMP: "The priority of the task is dropped. Yield the core on which
5213        // the task is running." (`xTaskPriorityDisinherit`) Found by the
5214        // two-core corpus: without it a give that disinherits AND wakes a
5215        // waiter sends the waiter to the other core, where the C runs it on
5216        // the giver's.
5217        if C::NUMBER_OF_CORES > 1 {
5218            if let Some(core) = self.running_core(holder) {
5219                self.yield_core(core);
5220            }
5221        }
5222        Ok(true)
5223    }
5224
5225    /// `vTaskPriorityDisinheritAfterTimeout`: a waiter gave up, so the
5226    /// holder keeps only what the tasks still waiting justify.
5227    #[cold]
5228    pub(crate) fn priority_disinherit_after_timeout(
5229        &mut self,
5230        holder: TaskHandle,
5231        highest_waiting: u8,
5232    ) -> Result<()> {
5233        if holder.is_null() || !self.tcbs.contains(holder) {
5234            return Ok(());
5235        }
5236        let (priority, base, held) = {
5237            let tcb = self.tcbs.resolve(holder)?;
5238            (tcb.priority, tcb.base_priority, tcb.mutexes_held)
5239        };
5240        // NOT `priority == base` as well, which this also tested: a holder
5241        // still at its base priority can be RAISED here. A waiter whose
5242        // priority was set above the holder's AFTER it blocked passed nothing
5243        // on (`vTaskPrioritySet` does not re-inherit), and when another
5244        // waiter times out the C sets the holder to the highest waiter left
5245        // -- 0 to 3 in the API differential (fresh seed 0x9e377d99, one core,
5246        // step 14559). The C tests `uxPriority != uxPriorityToUse` and the
5247        // one-mutex rule, nothing else.
5248        if held != 1 {
5249            return Ok(());
5250        }
5251        let target = if highest_waiting > base {
5252            highest_waiting
5253        } else {
5254            base
5255        };
5256        if priority == target {
5257            return Ok(());
5258        }
5259        self.trace_task(holder, |task, name| Event::TaskPriorityDisinherit {
5260            task,
5261            name,
5262            priority: Self::priority_value(target),
5263        });
5264        self.set_task_priority(holder, target)?;
5265        self.set_event_priority_value(holder, target)?;
5266        let item = Self::state_item(holder);
5267        if self.lists.container(item)? == Some(Self::ready_list(priority)) {
5268            let _ = self.lists.remove(item);
5269            // The priority just set, as in `priority_inherit`.
5270            self.add_task_to_ready_list_at(holder, target)?;
5271            // SMP: as `priority_disinherit` (`vTaskPriorityDisinheritAfterTimeout`).
5272            if C::NUMBER_OF_CORES > 1 {
5273                if let Some(core) = self.running_core(holder) {
5274                    self.yield_core(core);
5275                }
5276            }
5277        }
5278        Ok(())
5279    }
5280
5281    // ------------------------------------------------------- delay until --
5282
5283    /// `xTaskDelayUntil`: wake at `*previous_wake + period`, whatever the
5284    /// task did in between. `false` when the deadline has already passed,
5285    /// in which case nothing blocks — exactly as the C returns `pdFALSE`.
5286    ///
5287    /// # Errors
5288    /// A list error surfaces as itself.
5289    pub fn delay_until(&mut self, previous_wake: &mut u64, period: u64) -> Result<bool> {
5290        let caller = self.cur();
5291        let mut should_delay = false;
5292        self.suspend_all();
5293        {
5294            let now = self.tick;
5295            let wake_at = previous_wake.wrapping_add(period) & Self::MAX_DELAY;
5296            if now < *previous_wake {
5297                // The tick count overflowed since the last wake.
5298                if wake_at < *previous_wake && wake_at > now {
5299                    should_delay = true;
5300                }
5301            } else if wake_at < *previous_wake || wake_at > now {
5302                should_delay = true;
5303            }
5304            *previous_wake = wake_at;
5305            if should_delay {
5306                self.trace_task(caller, |task, name| Event::TaskDelayUntil {
5307                    task,
5308                    name,
5309                    wake_at,
5310                });
5311                let ticks = wake_at.wrapping_sub(now) & Self::MAX_DELAY;
5312                self.add_current_task_to_delayed_list(ticks, false)?;
5313            }
5314        }
5315        if !self.resume_all() {
5316            self.yield_or_owe(caller);
5317        }
5318        Ok(should_delay)
5319    }
5320
5321    /// Which error `task_of_event_item` would have produced, re-derived out of
5322    /// line so its two discriminants do not sit in the caller's entry block.
5323    // `#[inline(never)]` is load-bearing, not decoration: with `#[cold]` alone
5324    // LLVM inlines this back and re-materialises the constants in the caller's
5325    // entry block -- measured EXACTLY +0 Ir for -2 B. The hint only reweights
5326    // the branch; MOVING the code is what sinks the constants.
5327    #[cold]
5328    #[inline(never)]
5329    fn event_item_error(&self, item: ItemId) -> Error {
5330        match item.checked_sub(Self::task_item_base()) {
5331            None => Error::InvalidArgument,
5332            Some(_) => Error::Gone,
5333        }
5334    }
5335
5336    /// `xTaskRemoveFromEventList`; `true` when the woken task outranks the
5337    /// running one and a yield is therefore required.
5338    /// `#[cold]` because the call is GUARDED and reached from many sites on
5339    /// one hot path, which is the shape that pays: it lets LLVM keep the
5340    /// caller's frame setup out of the likely route. Measured on
5341    /// `riscv32-qemu-tick-work`, one attribute at a time.
5342    /// Worth queue -7 on its own.
5343    #[cold]
5344    pub(crate) fn remove_from_event_list(&mut self, list: ListId) -> Result<bool> {
5345        let Some(item) = self.lists.head(list)? else {
5346            return Ok(false);
5347        };
5348        // The `?` put FIVE constants in the entry block -- `xor %eax,%eax`,
5349        // `mov $0x0,%edx`, `mov $0x1,%al`, `mov $0x8,%dl`, `mov $0x5,%dl` --
5350        // the Ok/Err tags and both error discriminants, materialised before the
5351        // tests that would use them. The census says ALL 21,487 BlockQ calls
5352        // pass every one of those tests, so that is five instructions of dead
5353        // setup on the only path anybody takes. Producing the error inside a
5354        // `#[cold]` callee sinks them out of the entry block.
5355        let Ok(task) = self.task_of_event_item(item) else {
5356            return Err(self.event_item_error(item));
5357        };
5358        // Read here, beside `handle_at`'s read of the same slot, and carried:
5359        // nothing below writes a priority, and after the list edits LLVM can
5360        // no longer prove that, so it resolved the TCB twice more. `task`
5361        // came live out of `handle_at`, so this cannot fail -- and its error
5362        // is built in the cold helper for the reason the one above is: a `?`
5363        // here put its constants back in the entry block (+30 B of flash).
5364        let Ok(woken) = self.tcbs.resolve(task).map(|t| t.priority) else {
5365            return Err(self.event_item_error(item));
5366        };
5367        let _ = self.lists.remove(item);
5368        if self.suspended_depth == 0 {
5369            let _ = self.lists.remove(Self::state_item(task));
5370            self.add_task_to_ready_list_at(task, woken)?;
5371            // `configUSE_TICKLESS_IDLE`, as in `notify`.
5372            if C::USE_TICKLESS_IDLE {
5373                self.reset_next_task_unblock_time();
5374            }
5375        } else {
5376            self.lists.insert_end(Self::pending_ready_list(), item)?;
5377        }
5378        if C::NUMBER_OF_CORES > 1 {
5379            // `prvYieldForTask`, and the answer is whether THIS core now owes
5380            // a yield -- the C's `xYieldPendings[ portGET_CORE_ID() ]`.
5381            if C::USE_PREEMPTION {
5382                self.yield_for_task(task, woken);
5383            }
5384            return Ok(self.pending_here());
5385        }
5386        if woken > self.current_priority() {
5387            self.set_pending_here(true);
5388            return Ok(true);
5389        }
5390        Ok(false)
5391    }
5392}
5393
5394// ================================================================ tests ==
5395
5396/// `kernel.rs` is corpus-oracled BY DESIGN (`docs/HOLES.md` H3), and these
5397/// do not change that. They exist for the mutants that survived BOTH
5398/// oracles — measured, 80 of 415 viable — because the corpus and the unit
5399/// suite each cannot reach them:
5400///
5401/// * the corpus runs `PosixDemoConfig` for 2,000 ticks, so the TICK
5402///   OVERFLOW arms of `delay_until` never execute. Nine survivors sat
5403///   there, and the only way to reach them is to choose a
5404///   `previous_wake` near the maximum rather than to simulate 2^32 ticks;
5405/// * the diagnostic accessors (`task_at`, `ready_items`, `ready_cursor`)
5406///   and the geometry guard in `with_tick_hook` are called by no scenario
5407///   at all — they are also on H2's list of APIs with no differential, so
5408///   two independent methods agree on the same surface.
5409#[cfg(test)]
5410#[allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
5411mod tests {
5412    use rusty_rtos_core::config::Config;
5413    use rusty_rtos_core::hooks::NoTickHook;
5414
5415    use crate::system::tests::{NoTrace, TestConfig, TestPort};
5416
5417    type K = crate::Kernel<
5418        TestConfig,
5419        TestPort,
5420        NoTrace,
5421        NoTickHook,
5422        4,
5423        { crate::list_slots_for(4, 0, crate::lists_for(TestConfig::MAX_PRIORITIES, 1, 0)) },
5424        { crate::lists_for(TestConfig::MAX_PRIORITIES, 1, 0) },
5425        1,
5426        1,
5427        0,
5428        0,
5429        0,
5430        0,
5431        { <TestConfig as ::rusty_rtos_core::config::Config>::TIMER_QUEUE_LENGTH },
5432    >;
5433
5434    fn kernel() -> K {
5435        K::new(TestPort::default(), NoTrace).expect("the declared geometry adds up")
5436    }
5437
5438    /// A kernel with one task of our own running, so the calls that act on
5439    /// "the current task" have one.
5440    fn running() -> K {
5441        let mut k = kernel();
5442        k.create_task("t", 1).expect("a task");
5443        let h = k.start_scheduler().expect("start");
5444        k.suspend(Some(h.timer)).expect("park the daemon");
5445        for _ in 0..50_u32 {
5446            if k.name_of(k.current()).expect("a name").as_str() == "t" {
5447                return k;
5448            }
5449            k.switch_context();
5450        }
5451        panic!("the task never reached the CPU");
5452    }
5453
5454    /// A tick hook that counts its calls and wants every tick.
5455    #[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
5456    struct Counting(u32);
5457
5458    impl<K> rusty_rtos_core::hooks::TickHook<K> for Counting {
5459        fn tick(self, _kernel: &mut K) -> Self {
5460            Self(self.0.wrapping_add(1))
5461        }
5462    }
5463
5464    /// `configUSE_TICK_HOOK 0` means the hook is never called -- even one
5465    /// that says it wants the tick. `run_tick_hook`'s `||` as `&&` survived
5466    /// every oracle (kernel a87a716 added `wants_tick` beside the config
5467    /// test): nothing ran a hook that wants ticks under a config that has
5468    /// them off.
5469    #[test]
5470    fn a_config_without_the_tick_hook_never_calls_one() {
5471        type KH = crate::Kernel<
5472            TestConfig,
5473            TestPort,
5474            NoTrace,
5475            Counting,
5476            4,
5477            { crate::list_slots_for(4, 0, crate::lists_for(TestConfig::MAX_PRIORITIES, 1, 0)) },
5478            { crate::lists_for(TestConfig::MAX_PRIORITIES, 1, 0) },
5479            1,
5480            1,
5481            0,
5482            0,
5483            0,
5484            0,
5485            { <TestConfig as ::rusty_rtos_core::config::Config>::TIMER_QUEUE_LENGTH },
5486        >;
5487        // The premise, checked when the test compiles: this config has the hook off.
5488        const { assert!(!TestConfig::USE_TICK_HOOK) };
5489        let mut k = KH::with_tick_hook(TestPort::default(), NoTrace, Counting(0))
5490            .expect("the declared geometry adds up");
5491        k.create_task("t", 1).expect("a task");
5492        k.start_scheduler().expect("start");
5493        for _ in 0..8 {
5494            let _ = k.increment_tick();
5495        }
5496        assert_eq!(*k.tick_hook(), Counting(0), "the hook never ran");
5497    }
5498
5499    /// Two tasks delete themselves before the idle task runs; the idle task
5500    /// reclaims BOTH, oldest first, as `prvCheckTasksWaitingTermination` walks
5501    /// `xTasksWaitingTermination`.
5502    ///
5503    /// The kernel kept one slot for this and the second delete overwrote the
5504    /// first: A's slot was never freed and the task count stayed one high.
5505    /// Found 2026-10-04 while proving the two-core gates of the mutant survey.
5506    #[test]
5507    fn two_self_deletes_before_idle_are_both_reaped() {
5508        let mut k = kernel();
5509        let a = k.create_task("A", 2).expect("A");
5510        let b = k.create_task("B", 2).expect("B");
5511        let h = k.start_scheduler().expect("start");
5512        k.suspend(Some(h.timer)).expect("park the daemon");
5513        let before = k.task_count();
5514        while k.current() != a {
5515            k.switch_context();
5516        }
5517        k.task_delete(None).expect("A deletes itself");
5518        // A yields to B -- the next ready task, NOT idle.
5519        while k.current() != b {
5520            k.switch_context();
5521        }
5522        k.task_delete(None).expect("B deletes itself");
5523        assert_eq!(k.task_count(), before, "both still counted until reaped");
5524        k.check_tasks_waiting_termination();
5525        assert_eq!(k.task_count(), before - 2, "one idle pass reclaims both");
5526        assert!(k.name_of(a).is_err(), "A's slot is free");
5527        assert!(k.name_of(b).is_err(), "B's slot is free");
5528        k.check_tasks_waiting_termination();
5529        assert_eq!(k.task_count(), before - 2, "and nothing is reaped twice");
5530    }
5531
5532    /// `vTaskDelayUntil` with no overflow: a deadline still ahead delays,
5533    /// and one already passed does NOT.
5534    ///
5535    /// ```c
5536    /// else if( ( xTimeIncrement > ( xConstTickCount - *pxPreviousWakeTime ) ) ... )
5537    /// ```
5538    ///
5539    /// The second half is the one that matters and the one a naive test
5540    /// misses: a task that overran its period must NOT sleep a whole extra
5541    /// one, it must return immediately and say it did not delay.
5542    #[test]
5543    fn delay_until_skips_a_deadline_that_has_already_passed() {
5544        let mut k = running();
5545
5546        // now is 0; the deadline is 20 ticks away, so it delays.
5547        let mut previous = 0_u64;
5548        assert_eq!(k.delay_until(&mut previous, 20), Ok(true));
5549        assert_eq!(previous, 20, "the wake time advances by the period");
5550
5551        // Run the clock well past the next deadline.
5552        for _ in 0..60_u32 {
5553            k.tick_from_isr();
5554        }
5555
5556        // now is 60 and the deadline was 40: already gone.
5557        let mut previous = 20_u64;
5558        assert_eq!(
5559            k.delay_until(&mut previous, 20),
5560            Ok(false),
5561            "the period was overrun, so it must not sleep another one"
5562        );
5563        assert_eq!(
5564            previous, 40,
5565            "and the wake time STILL advances, or the task never catches up"
5566        );
5567    }
5568
5569    /// The tick-overflow arm, which the corpus cannot reach in 2,000 ticks.
5570    ///
5571    /// ```c
5572    /// if( xConstTickCount < *pxPreviousWakeTime ) {
5573    ///     if( ( xTimeToWake < *pxPreviousWakeTime ) && ( xTimeToWake > xConstTickCount ) )
5574    /// ```
5575    ///
5576    /// Both halves are needed: the wake time must have wrapped TOO, and it
5577    /// must still be ahead of the wrapped-around now. Reaching it needs a
5578    /// `previous_wake` near the maximum, not 2^32 ticks of simulation.
5579    #[test]
5580    fn delay_until_handles_a_wake_time_that_wrapped_with_the_tick() {
5581        let mut k = running();
5582        let max = K::MAX_DELAY;
5583
5584        // A few ticks, so `now` is small and positive.
5585        for _ in 0..5_u32 {
5586            k.tick_from_isr();
5587        }
5588
5589        // previous_wake sits just below the maximum; the period carries the
5590        // wake time over the top, to 9. now is 5, so 9 is still ahead.
5591        let mut previous = max.wrapping_sub(10);
5592        assert_eq!(
5593            k.delay_until(&mut previous, 20),
5594            Ok(true),
5595            "wrapped, and still in the future: it must delay"
5596        );
5597        assert_eq!(previous, 9, "the wake time wrapped with the tick");
5598
5599        // Same wrap, but the wake time is already behind a larger now.
5600        let mut k = running();
5601        for _ in 0..40_u32 {
5602            k.tick_from_isr();
5603        }
5604        let mut previous = max.wrapping_sub(10);
5605        assert_eq!(
5606            k.delay_until(&mut previous, 20),
5607            Ok(false),
5608            "wrapped, and now is already past it: it must NOT delay"
5609        );
5610    }
5611
5612    /// `task_at` walks the arena by SLOT, which is how a debugger or a
5613    /// health check enumerates tasks without holding handles.
5614    #[test]
5615    fn task_at_enumerates_live_slots_and_stops_at_the_end() {
5616        let mut k = kernel();
5617        let a = k.create_task("a", 1).expect("a task");
5618
5619        assert_eq!(k.task_at(0), Some(a), "slot zero is the first task");
5620        assert_eq!(
5621            k.task_at(usize::from(u16::MAX)),
5622            None,
5623            "past the arena is None, not a handle"
5624        );
5625        assert_eq!(
5626            k.task_at(usize::from(u16::MAX) + 1),
5627            None,
5628            "and an index that does not even fit a u16 is None, not a panic"
5629        );
5630    }
5631
5632    /// `ready_items` fills the caller's buffer and answers how many it
5633    /// wrote, stopping at the buffer's end rather than running past it.
5634    #[test]
5635    fn ready_items_fills_what_it_can_and_counts_what_it_filled() {
5636        let mut k = kernel();
5637        k.create_task("a", 1).expect("a");
5638        k.create_task("b", 1).expect("b");
5639
5640        let mut out = [0_u16; 8];
5641        let n = k.ready_items(1, &mut out);
5642        assert_eq!(n, 2, "two tasks are ready at priority 1");
5643
5644        // A buffer too small must be filled to its end and no further.
5645        let mut small = [0_u16; 1];
5646        assert_eq!(
5647            k.ready_items(1, &mut small),
5648            1,
5649            "it wrote one, because one is all there was room for"
5650        );
5651
5652        // An empty priority writes nothing.
5653        let mut out = [0_u16; 8];
5654        assert_eq!(k.ready_items(0, &mut out), 0);
5655    }
5656
5657    /// `ready_cursor` is the diagnostic that answers "did the round robin
5658    /// move", so it must report the list it was asked about.
5659    #[test]
5660    fn ready_cursor_reports_the_list_it_was_asked_about() {
5661        let mut k = kernel();
5662        k.create_task("a", 1).expect("a");
5663        k.create_task("b", 1).expect("b");
5664
5665        let before = k.ready_cursor(1);
5666        k.switch_context();
5667        let after = k.ready_cursor(1);
5668        assert_ne!(after, before, "a switch moved the cursor at priority 1");
5669        assert_eq!(
5670            k.ready_cursor(0),
5671            k.ready_cursor(0),
5672            "and an untouched priority answers the same thing twice"
5673        );
5674    }
5675
5676    /// `with_tick_hook`'s geometry guard, which is the only thing standing
5677    /// between a mis-declared kernel and a silently wrong one.
5678    ///
5679    /// `ITEMS` and `LISTS` are derived constants; if a caller writes them
5680    /// by hand and gets them wrong, every list index is off. The guard
5681    /// refuses, and these are the arms of it.
5682    #[test]
5683    fn the_geometry_guard_refuses_a_kernel_that_does_not_add_up() {
5684        // ITEMS too small for the declared tasks and timers.
5685        type WrongItems = crate::Kernel<
5686            TestConfig,
5687            TestPort,
5688            NoTrace,
5689            NoTickHook,
5690            4,
5691            // Legal as a geometry — a power of two above `LISTS` — but not
5692            // the derived value, which is 32. A grosser error (2 slots for
5693            // 12 lists) is now a COMPILE error in `SIZES_FIT`, so it cannot
5694            // be used to exercise the runtime guard any more. That the
5695            // check moved earlier is the improvement; this keeps the later
5696            // one honest.
5697            16,
5698            { crate::lists_for(TestConfig::MAX_PRIORITIES, 1, 0) },
5699            1,
5700            1,
5701            0,
5702            0,
5703            0,
5704            0,
5705            { <TestConfig as ::rusty_rtos_core::config::Config>::TIMER_QUEUE_LENGTH },
5706        >;
5707        assert!(
5708            WrongItems::new(TestPort::default(), NoTrace).is_err(),
5709            "ITEMS must be list_slots_for(TASKS, TIMERS, LISTS)"
5710        );
5711
5712        // LISTS not derived from the priorities, queues and groups.
5713        type WrongLists = crate::Kernel<
5714            TestConfig,
5715            TestPort,
5716            NoTrace,
5717            NoTickHook,
5718            4,
5719            // Correct for the two lists declared below, so the refusal
5720            // below is attributable to `LISTS` and nothing else.
5721            { crate::list_slots_for(4, 0, 2) },
5722            2,
5723            1,
5724            1,
5725            0,
5726            0,
5727            0,
5728            0,
5729            { <TestConfig as ::rusty_rtos_core::config::Config>::TIMER_QUEUE_LENGTH },
5730        >;
5731        assert!(
5732            WrongLists::new(TestPort::default(), NoTrace).is_err(),
5733            "LISTS must be lists_for(MAX_PRIORITIES, QUEUES, GROUPS)"
5734        );
5735
5736        // A kernel with no room for a single task.
5737        type NoTasks = crate::Kernel<
5738            TestConfig,
5739            TestPort,
5740            NoTrace,
5741            NoTickHook,
5742            0,
5743            { crate::list_slots_for(0, 0, crate::lists_for(TestConfig::MAX_PRIORITIES, 1, 0)) },
5744            { crate::lists_for(TestConfig::MAX_PRIORITIES, 1, 0) },
5745            1,
5746            1,
5747            0,
5748            0,
5749            0,
5750            0,
5751            { <TestConfig as ::rusty_rtos_core::config::Config>::TIMER_QUEUE_LENGTH },
5752        >;
5753        assert!(
5754            NoTasks::new(TestPort::default(), NoTrace).is_err(),
5755            "TASKS == 0 cannot even hold the idle task"
5756        );
5757
5758        // And the geometry that DOES add up is accepted, or the three
5759        // assertions above would pass for the wrong reason.
5760        assert!(K::new(TestPort::default(), NoTrace).is_ok());
5761    }
5762    /// `prvGetExpectedIdleTime`, arm one: something above the idle
5763    /// priority is running, so there is no window to sleep through.
5764    ///
5765    /// The three arms are a short-circuit chain and each is separately
5766    /// constructible, which is how they get pinned one at a time. A `>`
5767    /// turned `>=` in any of them would let the kernel sleep through work
5768    /// that is ready NOW.
5769    #[test]
5770    fn expected_idle_time_is_zero_while_anything_outranks_the_idle_task() {
5771        let mut k = kernel();
5772        k.create_task("hi", 1).expect("a task above idle");
5773        k.start_scheduler().expect("start");
5774
5775        assert!(
5776            k.current_priority() > 0,
5777            "the task above idle is the one running"
5778        );
5779        assert_eq!(
5780            k.expected_idle_time(),
5781            0,
5782            "something outranks idle, so the window is zero"
5783        );
5784    }
5785
5786    /// Arm two: another task shares the idle priority, so the very next
5787    /// tick has to be processed to give it its slice.
5788    #[test]
5789    fn expected_idle_time_is_zero_when_another_idle_task_wants_the_slice() {
5790        let mut k = kernel();
5791        k.create_task("a", 0).expect("a task AT the idle priority");
5792        k.start_scheduler().expect("start");
5793
5794        assert!(
5795            k.ready_len(0).unwrap_or(0) > 1,
5796            "the idle task and ours are both ready at priority 0"
5797        );
5798        assert_eq!(
5799            k.expected_idle_time(),
5800            0,
5801            "a shared slice cannot be slept through"
5802        );
5803    }
5804
5805    /// And with none of the three arms true, the window is the time to the
5806    /// next unblock -- which is the number the whole tickless feature
5807    /// rests on.
5808    #[test]
5809    fn expected_idle_time_is_the_window_to_the_next_unblock() {
5810        let mut k = running();
5811        k.delay(30).expect("park our task for thirty ticks");
5812
5813        // Only the idle task is ready now, and nothing outranks it.
5814        assert_eq!(k.current_priority(), 0, "the idle task is running");
5815        assert_eq!(k.ready_len(0).unwrap_or(0), 1, "and it is alone at zero");
5816        assert_eq!(
5817            k.expected_idle_time(),
5818            30,
5819            "so the port may sleep right up to the next unblock"
5820        );
5821    }
5822
5823    /// `ulTaskNotifyValueClear`'s reader half, and `xTaskNotifyAndQuery`'s:
5824    /// it answers the slot it was ASKED for, for the task it was asked
5825    /// about.
5826    ///
5827    /// Pinned with a value that is neither 0 nor 1, because those are what
5828    /// the surviving mutants replaced it with.
5829    #[test]
5830    fn notify_value_answers_the_slot_it_was_asked_for() {
5831        use crate::kernel::NotifyAction;
5832
5833        let mut k = running();
5834        let me = k.current();
5835        k.notify(me, 0, 0xabcd_1234, NotifyAction::Overwrite)
5836            .expect("notify");
5837
5838        assert_eq!(
5839            k.notify_value(Some(me), 0),
5840            Ok(0xabcd_1234),
5841            "the value that was set, not zero and not one"
5842        );
5843        assert_eq!(
5844            k.notify_value(None, 0),
5845            Ok(0xabcd_1234),
5846            "None means the CURRENT task"
5847        );
5848        assert!(
5849            k.notify_value(Some(me), TestConfig::NOTIFICATION_ARRAY_ENTRIES)
5850                .is_err(),
5851            "an index past the configured array is an error, not a zero"
5852        );
5853    }
5854    /// `delay_until`'s BOUNDARIES, which is where the four mutants that
5855    /// survived both oracles live.
5856    ///
5857    /// Each comparison in it has an edge the ordinary tests never touch,
5858    /// and each edge is reachable once you notice that `previous_wake` is
5859    /// a CALLER variable and the period may be zero:
5860    ///
5861    /// | line | comparison | the edge |
5862    /// |---|---|---|
5863    /// | 3228 | `wake_at < *previous_wake` | equal, via a period of 0 |
5864    /// | 3228 | `wake_at > now` | equal, by ticking to the wake time |
5865    /// | 3231 | `wake_at < *previous_wake` | equal, via a period of 0 |
5866    ///
5867    /// All three answers must be "do not delay". A `<=` or a `>=` in any of
5868    /// them turns a zero-length wait into a full period of sleep.
5869    #[test]
5870    fn delay_until_does_not_delay_on_any_of_its_boundaries() {
5871        let max = K::MAX_DELAY;
5872
5873        // (1) OVERFLOW arm, period zero: the wake time equals the previous
5874        // one, so there is nothing to wait for.
5875        let mut k = running();
5876        for _ in 0..5_u32 {
5877            k.tick_from_isr();
5878        }
5879        let mut previous = max.wrapping_sub(10);
5880        assert_eq!(
5881            k.delay_until(&mut previous, 0),
5882            Ok(false),
5883            "a period of zero is not a period of MAX_DELAY"
5884        );
5885
5886        // (2) OVERFLOW arm, wake time exactly equal to now: already due.
5887        // (max - 10) + 20 wraps to 9, so tick to 9.
5888        let mut k = running();
5889        for _ in 0..9_u32 {
5890            k.tick_from_isr();
5891        }
5892        let mut previous = max.wrapping_sub(10);
5893        assert_eq!(
5894            previous.wrapping_add(20) & max,
5895            9,
5896            "the wake time wraps to 9"
5897        );
5898        assert_eq!(
5899            k.delay_until(&mut previous, 20),
5900            Ok(false),
5901            "the wrapped wake time is NOW, so it is not in the future"
5902        );
5903
5904        // (3) NON-overflow arm, period zero with now already at the wake
5905        // time: neither disjunct holds.
5906        let mut k = running();
5907        for _ in 0..10_u32 {
5908            k.tick_from_isr();
5909        }
5910        let mut previous = 10_u64;
5911        assert_eq!(
5912            k.delay_until(&mut previous, 0),
5913            Ok(false),
5914            "nothing to wait for, and no period to sleep"
5915        );
5916        assert_eq!(previous, 10, "and the wake time does not move");
5917    }
5918    /// The tick-wrap bookkeeping, which `xTaskIncrementTick` runs once
5919    /// every `MAX_DELAY + 1` ticks.
5920    ///
5921    /// Neither oracle can reach it: the corpus runs 2,000 ticks and the
5922    /// unit suite does not wrap either, so every mutation of it survived
5923    /// both. It is a private `fn` and this module is inside `kernel.rs`,
5924    /// so the test calls it directly rather than trying to simulate 2^32
5925    /// ticks — which is the right answer when the trigger is unreachable
5926    /// but the behaviour is not.
5927    #[test]
5928    fn switching_the_delayed_lists_flips_the_pair_and_counts_the_wrap() {
5929        let mut k = running();
5930        let swapped = k.delayed_swapped;
5931        let overflows = k.overflows;
5932
5933        k.switch_delayed_lists();
5934        assert_ne!(
5935            k.delayed_swapped, swapped,
5936            "the delayed and overflow lists change places"
5937        );
5938        assert_eq!(
5939            k.overflows,
5940            overflows.wrapping_add(1),
5941            "and the wrap is counted, because `overflows` is how a trace \
5942             tells one epoch from the next"
5943        );
5944
5945        k.switch_delayed_lists();
5946        assert_eq!(k.delayed_swapped, swapped, "flipping twice is the identity");
5947        assert_eq!(k.overflows, overflows.wrapping_add(2));
5948    }
5949
5950    /// `prvResetNextTaskUnblockTime`, which the wrap calls: after the swap
5951    /// the next unblock time has to be recomputed from the list that is
5952    /// NOW the delayed one.
5953    ///
5954    /// That list is empty immediately after a wrap, so the answer must be
5955    /// `MAX_DELAY` — a stale value from the old list would make the
5956    /// kernel think a task is due and spin, or sleep through one that is.
5957    #[test]
5958    fn switching_the_delayed_lists_recomputes_the_next_unblock_time() {
5959        let mut k = running();
5960        let due = k.tick_count().wrapping_add(30);
5961        k.delay(30).expect("park our task on the delayed list");
5962        assert_eq!(k.next_unblock_time, due, "the parked task is what is next");
5963
5964        k.switch_delayed_lists();
5965        assert_eq!(
5966            k.next_unblock_time,
5967            K::MAX_DELAY,
5968            "the list that is now 'delayed' is empty, so nothing is due"
5969        );
5970    }
5971
5972    /// The sentinel for "nothing is due" is the SAME value in both places this
5973    /// kernel writes it, and this pins that.
5974    ///
5975    /// ★ It used to differ, and this test used to pin the difference. A reset
5976    /// over an empty delayed list stored `u64::MAX` because `head_value` cannot
5977    /// answer "empty" -- on an empty list the head IS the end marker, whose
5978    /// value is `u64::MAX` so sorted inserts always find something larger to
5979    /// stop at -- so the `unwrap_or(MAX_DELAY)` never fired. The C stores
5980    /// `portMAX_DELAY` there.
5981    ///
5982    /// The divergence was unobservable (it shows only at a tick equal to
5983    /// `MAX_DELAY`, and the wrap resets anyway), which is why a test pinned it
5984    /// instead of a gate catching it -- and the test one screen up had said in
5985    /// prose that `MAX_DELAY` "must" be the answer while asserting `u64::MAX`.
5986    /// The prose was right.
5987    ///
5988    /// Worth five instructions per tick: with the field provably no wider than
5989    /// the tick, `increment_tick`'s comparison against it stops being a 64-bit
5990    /// one on a 32-bit target. See `reset_next_task_unblock_time`.
5991    ///
5992    /// `new()` starts it at `MAX_DELAY` -- the tick mask, and what the C's
5993    /// `portMAX_DELAY` is. `reset_next_task_unblock_time` instead takes
5994    /// `head_value` of an empty list, which answers `ListsOf::MAX_VALUE`
5995    /// = `u64::MAX`, so the `unwrap_or(MAX_DELAY)` beside it never fires
5996    /// for emptiness.
5997    ///
5998    /// Neither is reachable as a tick, because the tick is masked to
5999    /// `MAX_DELAY`, so no scheduling decision can tell them apart and the
6000    /// corpus is green either way. It IS visible to the tickless path:
6001    /// `expected_idle_time` answers `next_unblock_time - tick`, which is
6002    /// the window the port is handed, and the two sentinels differ there
6003    /// by four billion.
6004    ///
6005    /// Recorded rather than changed: making them agree is an owner
6006    /// decision, because `expected_idle_time` feeds a port and any change
6007    /// to it moves what a tickless build asks for.
6008    #[test]
6009    fn the_nothing_is_due_sentinel_differs_between_init_and_reset() {
6010        let k = kernel();
6011        assert_eq!(
6012            k.next_unblock_time,
6013            K::MAX_DELAY,
6014            "a fresh kernel uses the tick mask, which is the C's portMAX_DELAY"
6015        );
6016
6017        let mut k = running();
6018        k.reset_next_task_unblock_time();
6019        assert_eq!(
6020            k.next_unblock_time,
6021            K::MAX_DELAY,
6022            "and a reset over an empty list uses the SAME sentinel, as the C does"
6023        );
6024        assert_ne!(
6025            K::MAX_DELAY,
6026            u64::MAX,
6027            "MAX_DELAY must be narrower than u64::MAX at this width, or the              invariant this test defends is vacuous"
6028        );
6029        assert_eq!(
6030            k.next_unblock_time & K::MAX_DELAY,
6031            k.next_unblock_time,
6032            "the field must never hold a value wider than the tick: that is what              lets increment_tick's comparison stay 32-bit"
6033        );
6034    }
6035
6036    /// The owe hint must not outlive the debt it stands for.
6037    ///
6038    /// A census over three scenarios found every raise producing exactly TWO
6039    /// entries to `resume_pending_owed` — one that did the work and returned
6040    /// `true`, and a second that re-derived "nothing owed" and cleared the
6041    /// hint on its way out. BlockQ: 43,110 raises, 86,207 entries. Clearing
6042    /// the hint at the end of the work made entries equal raises.
6043    ///
6044    /// Both directions are asserted. A `clear_owe_if_settled` that cleared
6045    /// unconditionally would pass the first half and lose a real owed item,
6046    /// which is a missing trace line in the differential — so the second half
6047    /// is the half that matters.
6048    #[test]
6049    fn the_owe_hint_does_not_outlive_the_debt() {
6050        let mut k = running();
6051        let me = k.current();
6052        let i = me.index() as usize;
6053        fn hint(k: &K, i: usize) -> bool {
6054            k.owes_anything.get(i).copied().unwrap_or(false)
6055        }
6056
6057        k.owe(me);
6058        assert!(hint(&k, i), "the hint was raised");
6059        k.clear_owe_if_settled(i);
6060        assert!(
6061            !hint(&k, i),
6062            "nothing is owed, so the hint is dead and must go -- leaving it standing costs one whole entry to the owed body per owed item"
6063        );
6064
6065        // The negative case, once per kind of debt. A `clear_owe_if_settled`
6066        // that cleared unconditionally would pass the half above and lose a
6067        // real owed item, which is a missing line in the differential.
6068        for kind in 0..3 {
6069            match kind {
6070                0 => *k.owed_exits.get_mut(i).expect("in range") = 1,
6071                1 => k.set_flag(i, K::F_YIELD, true),
6072                _ => {
6073                    *k.owed_trace.get_mut(i).expect("in range") =
6074                        crate::kernel::OwedTrace::AddNewTaskToReadyList {
6075                            task: me,
6076                            priority: 1,
6077                        }
6078                }
6079            }
6080            k.owe(me);
6081            k.clear_owe_if_settled(i);
6082            assert!(hint(&k, i), "kind {kind} is live debt; the hint must stay");
6083            *k.owed_exits.get_mut(i).expect("in range") = 0;
6084            k.set_flag(i, K::F_YIELD, false);
6085            *k.owed_trace.get_mut(i).expect("in range") = crate::kernel::OwedTrace::None;
6086        }
6087    }
6088
6089    /// The stream-buffer resume flag is a ONE-SHOT: the take clears it, so
6090    /// a second take answers None. A `take` that did not clear would let a
6091    /// blocked reader wake twice on one send.
6092    #[test]
6093    fn a_stream_resume_is_taken_once_and_then_gone() {
6094        let mut k = running();
6095        let me = k.current();
6096        assert_eq!(k.take_stream_resume(me), None, "nothing set yet");
6097
6098        k.set_stream_resume(me, 7);
6099        assert_eq!(k.take_stream_resume(me), Some(7), "the value that was set");
6100        assert_eq!(k.take_stream_resume(me), None, "and it is consumed");
6101    }
6102
6103    /// `vTaskMissedYield`: a yield asked for while the scheduler is
6104    /// suspended has to be remembered, or it is simply lost.
6105    #[test]
6106    fn a_missed_yield_is_remembered_rather_than_dropped() {
6107        let mut k = running();
6108        k.yield_pending = [false; super::MAX_CORES];
6109        k.missed_yield();
6110        assert!(
6111            k.yield_pending[0],
6112            "the yield survives until something can act on it"
6113        );
6114    }
6115
6116    // ---- plan P5: what the mutation survey found no oracle judging --------
6117
6118    /// One core is one core: `cores()` must not round a configured 1 up.
6119    #[test]
6120    fn a_one_core_configuration_has_one_core() {
6121        assert_eq!(K::cores(), 1);
6122    }
6123
6124    /// More cores than this kernel holds are clamped to `MAX_CORES`, not
6125    /// taken at their word: every per-core array is `MAX_CORES` long.
6126    #[test]
6127    fn a_configuration_past_the_kernel_s_cores_is_clamped() {
6128        struct ThreeCores;
6129        impl Config for ThreeCores {
6130            type Tick = rusty_rtos_core::tick::Bits32;
6131            const TICK_RATE_HZ: u32 = 100;
6132            const MAX_PRIORITIES: u8 = 4;
6133            const MINIMAL_STACK_SIZE: usize = 1;
6134            const MAX_TASK_NAME_LEN: usize = 8;
6135            const TIMER_TASK_PRIORITY: u8 = 3;
6136            const TIMER_TASK_STACK_DEPTH: usize = 1;
6137            const TIMER_QUEUE_LENGTH: usize = 1;
6138            const NOTIFICATION_ARRAY_ENTRIES: usize = 1;
6139            const NUMBER_OF_CORES: u8 = 3;
6140        }
6141        type K3 = crate::Kernel<
6142            ThreeCores,
6143            TestPort,
6144            NoTrace,
6145            NoTickHook,
6146            4,
6147            { crate::list_slots_for(4, 0, crate::lists_for(4, 1, 0)) },
6148            { crate::lists_for(4, 1, 0) },
6149            1,
6150            1,
6151            0,
6152            0,
6153            0,
6154            0,
6155            1,
6156        >;
6157        assert_eq!(K3::cores(), super::MAX_CORES);
6158    }
6159
6160    /// Two cores on a port that does not commit its own switches are
6161    /// refused at construction -- the same geometry on one core is not.
6162    #[test]
6163    fn two_cores_need_a_port_that_commits_its_switches() {
6164        struct TwoCores;
6165        impl Config for TwoCores {
6166            type Tick = rusty_rtos_core::tick::Bits32;
6167            const TICK_RATE_HZ: u32 = 100;
6168            const MAX_PRIORITIES: u8 = 4;
6169            const MINIMAL_STACK_SIZE: usize = 1;
6170            const MAX_TASK_NAME_LEN: usize = 8;
6171            const TIMER_TASK_PRIORITY: u8 = 3;
6172            const TIMER_TASK_STACK_DEPTH: usize = 1;
6173            const TIMER_QUEUE_LENGTH: usize = 1;
6174            const NOTIFICATION_ARRAY_ENTRIES: usize = 1;
6175            const USE_TIME_SLICING: bool = false;
6176            const NUMBER_OF_CORES: u8 = 2;
6177        }
6178        type K2 = crate::Kernel<
6179            TwoCores,
6180            TestPort,
6181            NoTrace,
6182            NoTickHook,
6183            6,
6184            { crate::list_slots_for(6, 1, crate::lists_for(4, 2, 1)) },
6185            { crate::lists_for(4, 2, 1) },
6186            2,
6187            8,
6188            1,
6189            8,
6190            1,
6191            1,
6192            1,
6193        >;
6194        const { assert!(!<TestPort as rusty_rtos_core::port::Port>::COMMITS_SWITCH) };
6195        assert_eq!(
6196            K2::new(TestPort::default(), NoTrace).err(),
6197            Some(rusty_rtos_core::error::Error::InvalidArgument)
6198        );
6199        assert!(
6200            KTie::new(TestPort::default(), NoTrace).is_ok(),
6201            "one core, same geometry"
6202        );
6203    }
6204
6205    /// `state_of` answers Running for the running task on one core, called
6206    /// directly -- `task_state_get` answers it first, so no oracle reached
6207    /// this spelling's own shortcut.
6208    #[test]
6209    fn state_of_the_running_task_is_running_on_one_core() {
6210        let k = running();
6211        assert_eq!(k.state_of(k.current()), Ok(super::TaskState::Running));
6212    }
6213
6214    /// `vTaskResume( NULL )` is a no-op, not a stale-handle error.
6215    #[test]
6216    fn resuming_null_is_a_no_op() {
6217        let mut k = running();
6218        assert_eq!(k.resume(rusty_rtos_core::handle::TaskHandle::NULL), Ok(()));
6219    }
6220
6221    /// `xSchedulerRunning`, both ways round.
6222    #[test]
6223    fn the_scheduler_is_running_only_once_started() {
6224        let k = kernel();
6225        assert!(!k.is_running());
6226        assert!(running().is_running());
6227    }
6228
6229    /// A stall is counted every time and its FIRST reason kept.
6230    #[test]
6231    fn a_stall_counts_and_keeps_its_first_reason() {
6232        let mut k = kernel();
6233        assert_eq!((k.stalls(), k.first_stall()), (0, super::Stall::None));
6234        k.switch_context();
6235        assert_eq!(
6236            (k.stalls(), k.first_stall()),
6237            (1, super::Stall::NoReadyTask)
6238        );
6239        k.switch_context();
6240        assert_eq!(
6241            (k.stalls(), k.first_stall()),
6242            (2, super::Stall::NoReadyTask)
6243        );
6244    }
6245
6246    /// `xPendedTicks`: ticks taken while the scheduler is suspended.
6247    #[test]
6248    fn ticks_under_suspension_are_pended_and_counted() {
6249        let mut k = running();
6250        k.suspend_all();
6251        k.tick_from_isr();
6252        k.tick_from_isr();
6253        assert_eq!(k.pended_ticks(), 2);
6254        let _ = k.resume_all();
6255        assert_eq!(k.pended_ticks(), 0);
6256    }
6257
6258    /// [`TestConfig`], started at the last tick before the wrap.
6259    struct LastTickConfig;
6260    impl Config for LastTickConfig {
6261        type Tick = rusty_rtos_core::tick::Bits32;
6262        const TICK_RATE_HZ: u32 = 100;
6263        const INITIAL_TICK_COUNT: u64 = 0xFFFF_FFFF;
6264        const MAX_PRIORITIES: u8 = 4;
6265        const MINIMAL_STACK_SIZE: usize = 1;
6266        const MAX_TASK_NAME_LEN: usize = 8;
6267        const TIMER_TASK_PRIORITY: u8 = 3;
6268        const TIMER_TASK_STACK_DEPTH: usize = 1;
6269        const TIMER_QUEUE_LENGTH: usize = 1;
6270        const NOTIFICATION_ARRAY_ENTRIES: usize = 1;
6271        const USE_TIME_SLICING: bool = false;
6272    }
6273
6274    type KLast = crate::Kernel<
6275        LastTickConfig,
6276        TestPort,
6277        NoTrace,
6278        NoTickHook,
6279        4,
6280        { crate::list_slots_for(4, 0, crate::lists_for(LastTickConfig::MAX_PRIORITIES, 1, 0)) },
6281        { crate::lists_for(LastTickConfig::MAX_PRIORITIES, 1, 0) },
6282        1,
6283        1,
6284        0,
6285        0,
6286        0,
6287        0,
6288        { <LastTickConfig as ::rusty_rtos_core::config::Config>::TIMER_QUEUE_LENGTH },
6289    >;
6290
6291    fn at_the_last_tick() -> KLast {
6292        let mut k =
6293            KLast::new(TestPort::default(), NoTrace).expect("the declared geometry adds up");
6294        k.create_task("t", 1).expect("a task");
6295        let h = k.start_scheduler().expect("start");
6296        k.suspend(Some(h.timer)).expect("park the daemon");
6297        assert_eq!(k.tick_count(), 0xFFFF_FFFF);
6298        k
6299    }
6300
6301    /// `xNumOfOverflows` counts the wrap, and only the wrap.
6302    #[test]
6303    fn the_wrap_is_counted_once() {
6304        let mut k = at_the_last_tick();
6305        assert_eq!(k.overflows(), 0);
6306        k.tick_from_isr();
6307        assert_eq!((k.tick_count(), k.overflows()), (0, 1));
6308    }
6309
6310    /// With nothing delayed the next unblock time is the maximum, so at the
6311    /// last tick there is no room to step: `step_tick` must neither pend a
6312    /// tick nor move the clock (the C would `configASSERT`; Kairos clamps
6313    /// the jump to zero, and a zero jump is nothing).
6314    #[test]
6315    fn a_step_with_no_room_does_nothing() {
6316        let mut k = at_the_last_tick();
6317        k.step_tick(1);
6318        assert_eq!((k.tick_count(), k.pended_ticks()), (0xFFFF_FFFF, 0));
6319    }
6320
6321    /// A port that counts its outermost critical-section exits, and a sink
6322    /// that keeps the count the kernel noted before each event.
6323    #[derive(Default)]
6324    struct ExitsPort {
6325        nesting: core::cell::Cell<u32>,
6326        exits: std::rc::Rc<core::cell::Cell<u64>>,
6327    }
6328    impl rusty_rtos_core::port::Port for ExitsPort {
6329        fn yield_now(&self) {}
6330        fn yield_from_isr(&self, _woken: rusty_rtos_core::isr::Woken) {}
6331        fn enter_critical(&self) {
6332            self.nesting.set(self.nesting.get().saturating_add(1));
6333        }
6334        fn exit_critical(&self) {
6335            self.nesting.set(self.nesting.get().saturating_sub(1));
6336            if self.nesting.get() == 0 {
6337                self.exits.set(self.exits.get().saturating_add(1));
6338            }
6339        }
6340        fn set_interrupt_mask_from_isr(&self) -> u32 {
6341            0
6342        }
6343        fn clear_interrupt_mask_from_isr(&self, _saved: u32) {}
6344        fn in_isr(&self) -> bool {
6345            false
6346        }
6347        fn set_in_tick_entry(&self, _yes: bool) {}
6348        fn exits(&self) -> u64 {
6349            self.exits.get()
6350        }
6351    }
6352    #[derive(Default)]
6353    struct ExitsTrace {
6354        noted: u64,
6355        /// The port's own counter, read live at each event.
6356        live: std::rc::Rc<core::cell::Cell<u64>>,
6357        seen: std::vec::Vec<(u64, u64)>,
6358    }
6359    impl rusty_rtos_core::trace::Trace for ExitsTrace {
6360        fn note_exits(&mut self, exits: u64) {
6361            self.noted = exits;
6362        }
6363        fn event(&mut self, _tick: u64, _event: rusty_rtos_core::trace::Event<'_>) {
6364            self.seen.push((self.noted, self.live.get()));
6365        }
6366    }
6367
6368    /// The kernel hands an emitting sink the port's exit count before each
6369    /// event (the sim's clock column) -- and the count moves.
6370    #[test]
6371    fn an_emitting_sink_is_told_the_exit_count() {
6372        type KE = crate::Kernel<
6373            TestConfig,
6374            ExitsPort,
6375            ExitsTrace,
6376            NoTickHook,
6377            4,
6378            { crate::list_slots_for(4, 0, crate::lists_for(TestConfig::MAX_PRIORITIES, 1, 0)) },
6379            { crate::lists_for(TestConfig::MAX_PRIORITIES, 1, 0) },
6380            1,
6381            1,
6382            0,
6383            0,
6384            0,
6385            0,
6386            { <TestConfig as ::rusty_rtos_core::config::Config>::TIMER_QUEUE_LENGTH },
6387        >;
6388        let port = ExitsPort::default();
6389        let trace = ExitsTrace {
6390            live: std::rc::Rc::clone(&port.exits),
6391            ..ExitsTrace::default()
6392        };
6393        let mut k = KE::new(port, trace).expect("geometry");
6394        k.create_task("a", 1).expect("a");
6395        let h = k.start_scheduler().expect("start");
6396        k.suspend(Some(h.timer)).expect("park");
6397        k.delay(3).expect("delay");
6398        for _ in 0..4_u32 {
6399            k.tick_from_isr();
6400        }
6401        let seen = &k.trace().seen;
6402        assert!(
6403            seen.iter().any(|&(noted, _)| noted > 0),
6404            "no event carried an exit count: {seen:?}"
6405        );
6406        assert!(
6407            seen.iter().all(|&(noted, live)| noted == live),
6408            "an event carried a stale exit count (noted, live): {seen:?}"
6409        );
6410    }
6411
6412    /// The footprint lines add up to what they claim, and to nearly all of
6413    /// the kernel: the remainder is the scalar tail and padding.
6414    #[test]
6415    fn the_footprint_lines_account_for_the_kernel() {
6416        use core::mem::size_of;
6417        assert_eq!(
6418            K::FOOTPRINT_FREE_LISTS,
6419            2 * size_of::<usize>(),
6420            "BUFFERS = 0 and QUEUES = 1: one (base, length) pair"
6421        );
6422        assert_eq!(
6423            K::FOOTPRINT_PER_TASK_SIDE,
6424            4 * (1 + 3 + size_of::<super::OwedTrace>() + 4),
6425            "four tasks: a byte, three flags, an owed line and a word each"
6426        );
6427        let parts = [
6428            K::FOOTPRINT_TCBS,
6429            K::FOOTPRINT_QUEUES,
6430            K::FOOTPRINT_LISTS,
6431            K::FOOTPRINT_SLOTS,
6432            K::FOOTPRINT_BUFFERS,
6433            K::FOOTPRINT_TIMERS,
6434            K::FOOTPRINT_GROUPS,
6435            K::FOOTPRINT_TIMER_MESSAGES,
6436            K::FOOTPRINT_BYTES,
6437            K::FOOTPRINT_FREE_LISTS,
6438            K::FOOTPRINT_PER_TASK_SIDE,
6439        ];
6440        assert_eq!(K::FOOTPRINT_ACCOUNTED, parts.iter().sum::<usize>());
6441        // Both at compile time: the remainder is the scalar tail and padding,
6442        // and a line that stopped adding up would leave far more than that.
6443        const {
6444            assert!(K::FOOTPRINT_ACCOUNTED <= K::FOOTPRINT);
6445            assert!(K::FOOTPRINT - K::FOOTPRINT_ACCOUNTED < 512);
6446        };
6447    }
6448
6449    /// The configuration a geometry test varies: [`TestConfig`] with the
6450    /// task-name limit as a parameter.
6451    struct NameConfig<const N: usize>;
6452    impl<const N: usize> Config for NameConfig<N> {
6453        type Tick = rusty_rtos_core::tick::Bits32;
6454        const TICK_RATE_HZ: u32 = 100;
6455        const MAX_PRIORITIES: u8 = 4;
6456        const MINIMAL_STACK_SIZE: usize = 1;
6457        const MAX_TASK_NAME_LEN: usize = N;
6458        const TIMER_TASK_PRIORITY: u8 = 3;
6459        const TIMER_TASK_STACK_DEPTH: usize = 1;
6460        const TIMER_QUEUE_LENGTH: usize = 1;
6461        const NOTIFICATION_ARRAY_ENTRIES: usize = 1;
6462    }
6463
6464    type KN<const N: usize> = crate::Kernel<
6465        NameConfig<N>,
6466        TestPort,
6467        NoTrace,
6468        NoTickHook,
6469        4,
6470        { crate::list_slots_for(4, 0, crate::lists_for(4, 1, 0)) },
6471        { crate::lists_for(4, 1, 0) },
6472        1,
6473        1,
6474        0,
6475        0,
6476        0,
6477        0,
6478        1,
6479    >;
6480
6481    /// A task name may use the whole name buffer, and not one byte more.
6482    #[test]
6483    fn a_task_name_limit_is_refused_only_past_the_buffer() {
6484        assert!(KN::<{ crate::NAME_CAPACITY }>::new(TestPort::default(), NoTrace).is_ok());
6485        assert_eq!(
6486            KN::<{ crate::NAME_CAPACITY + 1 }>::new(TestPort::default(), NoTrace).err(),
6487            Some(rusty_rtos_core::error::Error::InvalidArgument)
6488        );
6489    }
6490
6491    /// The timer mailbox's declared length must be the configuration's.
6492    #[test]
6493    fn a_mailbox_that_disagrees_with_its_config_is_refused() {
6494        type KM = crate::Kernel<
6495            TestConfig,
6496            TestPort,
6497            NoTrace,
6498            NoTickHook,
6499            4,
6500            { crate::list_slots_for(4, 0, crate::lists_for(TestConfig::MAX_PRIORITIES, 1, 0)) },
6501            { crate::lists_for(TestConfig::MAX_PRIORITIES, 1, 0) },
6502            1,
6503            1,
6504            0,
6505            0,
6506            0,
6507            0,
6508            { <TestConfig as ::rusty_rtos_core::config::Config>::TIMER_QUEUE_LENGTH + 1 },
6509        >;
6510        assert_eq!(
6511            KM::new(TestPort::default(), NoTrace).err(),
6512            Some(rusty_rtos_core::error::Error::InvalidArgument)
6513        );
6514    }
6515
6516    /// [`TestConfig`] with the message length prefix as a parameter.
6517    struct PrefixConfig<const N: usize>;
6518    impl<const N: usize> Config for PrefixConfig<N> {
6519        type Tick = rusty_rtos_core::tick::Bits32;
6520        const TICK_RATE_HZ: u32 = 100;
6521        const MAX_PRIORITIES: u8 = 4;
6522        const MINIMAL_STACK_SIZE: usize = 1;
6523        const MAX_TASK_NAME_LEN: usize = 8;
6524        const TIMER_TASK_PRIORITY: u8 = 3;
6525        const TIMER_TASK_STACK_DEPTH: usize = 1;
6526        const TIMER_QUEUE_LENGTH: usize = 1;
6527        const NOTIFICATION_ARRAY_ENTRIES: usize = 1;
6528        const MESSAGE_LENGTH_BYTES: usize = N;
6529    }
6530
6531    type KP<const N: usize> = crate::Kernel<
6532        PrefixConfig<N>,
6533        TestPort,
6534        NoTrace,
6535        NoTickHook,
6536        4,
6537        { crate::list_slots_for(4, 0, crate::lists_for(4, 1, 0)) },
6538        { crate::lists_for(4, 1, 0) },
6539        1,
6540        1,
6541        0,
6542        0,
6543        0,
6544        0,
6545        1,
6546    >;
6547
6548    /// `configMESSAGE_BUFFER_LENGTH_TYPE` is one to eight bytes: each
6549    /// bound refused on its own, the widest accepted.
6550    #[test]
6551    fn a_message_length_prefix_is_one_to_eight_bytes() {
6552        use rusty_rtos_core::error::Error;
6553        assert_eq!(
6554            KP::<0>::new(TestPort::default(), NoTrace).err(),
6555            Some(Error::InvalidArgument)
6556        );
6557        assert_eq!(
6558            KP::<9>::new(TestPort::default(), NoTrace).err(),
6559            Some(Error::InvalidArgument)
6560        );
6561        assert!(KP::<1>::new(TestPort::default(), NoTrace).is_ok());
6562        assert!(KP::<8>::new(TestPort::default(), NoTrace).is_ok());
6563    }
6564
6565    /// The footprint sum again, on a kernel that HAS a byte arena -- with
6566    /// none, its line is zero and adding it or taking it away is the same.
6567    #[test]
6568    fn the_footprint_lines_account_for_a_kernel_with_buffers() {
6569        type KB = crate::Kernel<
6570            TestConfig,
6571            TestPort,
6572            NoTrace,
6573            NoTickHook,
6574            4,
6575            { crate::list_slots_for(4, 0, crate::lists_for(TestConfig::MAX_PRIORITIES, 1, 0)) },
6576            { crate::lists_for(TestConfig::MAX_PRIORITIES, 1, 0) },
6577            1,
6578            1,
6579            2,
6580            64,
6581            0,
6582            0,
6583            { <TestConfig as ::rusty_rtos_core::config::Config>::TIMER_QUEUE_LENGTH },
6584        >;
6585        assert_eq!(KB::FOOTPRINT_BYTES, 64);
6586        let parts = [
6587            KB::FOOTPRINT_TCBS,
6588            KB::FOOTPRINT_QUEUES,
6589            KB::FOOTPRINT_LISTS,
6590            KB::FOOTPRINT_SLOTS,
6591            KB::FOOTPRINT_BUFFERS,
6592            KB::FOOTPRINT_TIMERS,
6593            KB::FOOTPRINT_GROUPS,
6594            KB::FOOTPRINT_TIMER_MESSAGES,
6595            KB::FOOTPRINT_BYTES,
6596            KB::FOOTPRINT_FREE_LISTS,
6597            KB::FOOTPRINT_PER_TASK_SIDE,
6598        ];
6599        assert_eq!(KB::FOOTPRINT_ACCOUNTED, parts.iter().sum::<usize>());
6600    }
6601
6602    use crate::Wait;
6603    use rusty_rtos_core::isr::Woken;
6604
6605    // ------------------------------------------------ a yield already owed --
6606    //
6607    // Every one-core "did the readied task outrank the running one?" test has
6608    // a two-core arm behind `C::NUMBER_OF_CORES > 1`: `yield_for_task`, then
6609    // "does THIS core owe a yield?". On one core the two differ in one place
6610    // only -- idle is not marked idle there (`F_IDLE` is set by
6611    // `start_scheduler_smp`), so there is no idle tie -- and that place is a
6612    // yield ALREADY owed: the two-core arm answers yes for any task readied
6613    // while one is, the one-core test answers whether THIS task outranks the
6614    // running one. Relaxing the gate to `>= 1` changed nothing any oracle
6615    // could see, and the mutant survey could list it only as evidence
6616    // (`core-gate`).
6617    //
6618    // A yield is owed and not yet taken when an ISR woke a higher task and
6619    // did not ask for the switch (`pxHigherPriorityTaskWoken` ignored, which
6620    // the C allows: the switch waits for the next tick). Readying a LOWER
6621    // task then must not take it early. `TestPort::yield_from_isr` does
6622    // nothing, so the owed yield stays owed for as long as the test needs.
6623
6624    /// [`TestConfig`] with room for a mutex beside the timer daemon's queue.
6625    type KTie = crate::Kernel<
6626        TestConfig,
6627        TestPort,
6628        NoTrace,
6629        NoTickHook,
6630        6,
6631        { crate::list_slots_for(6, 1, crate::lists_for(TestConfig::MAX_PRIORITIES, 2, 1)) },
6632        { crate::lists_for(TestConfig::MAX_PRIORITIES, 2, 1) },
6633        2,
6634        8,
6635        1,
6636        8,
6637        1,
6638        1,
6639        { <TestConfig as ::rusty_rtos_core::config::Config>::TIMER_QUEUE_LENGTH },
6640    >;
6641
6642    /// `a` (priority 2) running, owing a yield to `h` (3), which an ISR
6643    /// notified; `l` (1) blocked in a notify wait of its own.
6644    fn a_yield_owed() -> (KTie, [crate::kernel::TaskHandle; 3]) {
6645        use crate::kernel::NotifyAction;
6646        let mut k = KTie::new(TestPort::default(), NoTrace).expect("the declared geometry adds up");
6647        let l = k.create_task("l", 1).expect("l");
6648        let a = k.create_task("a", 2).expect("a");
6649        let h = k.create_task("h", 3).expect("h");
6650        let started = k.start_scheduler().expect("start");
6651        k.suspend(Some(started.timer)).expect("park the daemon");
6652        for task in [h, a, l] {
6653            for _ in 0..8_u32 {
6654                if k.current() == task {
6655                    break;
6656                }
6657                k.switch_context();
6658            }
6659            assert_eq!(k.current(), task);
6660            if task != a {
6661                assert_eq!(k.notify_wait(0, 0, 0, 100), Ok(Wait::Blocked));
6662            } else {
6663                k.delay(1).expect("a steps aside for l");
6664            }
6665        }
6666        if k.increment_tick() {
6667            k.switch_context();
6668        }
6669        assert_eq!(k.current(), a, "h and l wait; a runs");
6670        let (_, woken) = k
6671            .notify_from_isr(h, 0, 1, NotifyAction::SetBits)
6672            .expect("h notified");
6673        assert_eq!(woken, Woken::YES);
6674        assert!(
6675            k.pending_here(),
6676            "the ISR did not take its switch: a yield is owed"
6677        );
6678        assert_eq!(k.current(), a);
6679        (k, [l, a, h])
6680    }
6681
6682    /// `taskYIELD_IF_USING_PREEMPTION` on create: only a task created ABOVE
6683    /// the running one yields, whatever was owed before.
6684    #[test]
6685    fn creating_a_lower_task_does_not_take_a_yield_already_owed() {
6686        let (mut k, [_, a, _]) = a_yield_owed();
6687        let _u = k.create_task("u", 1).expect("u");
6688        assert_eq!(k.current(), a, "xTaskCreate of a lower task: no yield");
6689        assert!(k.pending_here(), "the owed yield is still owed");
6690    }
6691
6692    /// `xTaskNotify` and `xTaskNotifyFromISR` readying a task BELOW the
6693    /// running one: no yield, and `pxHigherPriorityTaskWoken` stays false --
6694    /// it answers for THIS call, not for one an earlier interrupt made.
6695    #[test]
6696    fn notifying_a_lower_task_does_not_take_or_report_a_yield_already_owed() {
6697        use crate::kernel::NotifyAction;
6698        let (mut k, [l, a, _]) = a_yield_owed();
6699        assert_eq!(k.notify(l, 0, 1, NotifyAction::SetBits), Ok(true));
6700        assert_eq!(k.current(), a, "xTaskNotify of a lower task: no yield");
6701        assert!(k.pending_here(), "the owed yield is still owed");
6702
6703        let (mut k, [l, a, _]) = a_yield_owed();
6704        let (_, woken) = k
6705            .notify_from_isr(l, 0, 1, NotifyAction::SetBits)
6706            .expect("l notified");
6707        assert_eq!(woken, Woken::NO, "xTaskNotifyFromISR: l does not outrank a");
6708        assert_eq!(k.current(), a);
6709    }
6710
6711    /// `xTaskPriorityDisinherit` answering "yield" while the give wakes a
6712    /// waiter that no longer outranks the giver: the C consults that answer
6713    /// only when there was no waiter (P1.3, step 4105), so it does not
6714    /// yield -- and the disinherit itself must not have left a yield owed
6715    /// for the next tick or resume to take. On two cores it does: it yields
6716    /// the core running the holder, which is the gate's other side.
6717    #[test]
6718    fn a_give_that_disinherits_and_wakes_a_lower_waiter_owes_no_yield() {
6719        let mut k = KTie::new(TestPort::default(), NoTrace).expect("the declared geometry adds up");
6720        let holder = k.create_task("h", 1).expect("h");
6721        let waiter = k.create_task("w", 2).expect("w");
6722        let m = k.mutex_create().expect("a mutex");
6723        let h = k.start_scheduler().expect("start");
6724        k.suspend(Some(h.timer)).expect("park the daemon");
6725        for _ in 0..8_u32 {
6726            if k.current() == waiter {
6727                break;
6728            }
6729            k.switch_context();
6730        }
6731        assert_eq!(k.current(), waiter);
6732        k.delay(1).expect("w steps aside");
6733        assert_eq!(k.current(), holder);
6734        assert_eq!(k.semaphore_take(m, 0), Ok(Wait::Ready(())));
6735        if k.increment_tick() {
6736            k.switch_context();
6737        }
6738        assert_eq!(k.current(), waiter, "w wakes and preempts");
6739        assert_eq!(k.semaphore_take(m, 10), Ok(Wait::Blocked));
6740        assert_eq!(k.current(), holder, "h runs, at w's priority");
6741        k.set_priority(Some(waiter), 0)
6742            .expect("w lowered below h's base");
6743        assert!(!k.pending_here());
6744        assert_eq!(k.semaphore_give(m), Ok(Wait::Ready(())));
6745        assert_eq!(k.current(), holder, "w, woken, does not outrank h");
6746        assert!(
6747            !k.pending_here(),
6748            "the disinherit's yield request is not consulted when the give woke a waiter,              and must not be left owed"
6749        );
6750    }
6751}