rusty_rtos_kernel_core/queue.rs
1//! `queue.c`: the queue, and the semaphore and mutex built on it.
2//!
3//! In FreeRTOS a semaphore *is* a queue with a zero-size item, and a mutex
4//! is that plus a holder and priority inheritance. Kairos keeps the same
5//! shape, because the trace does: a semaphore give fires `QUEUE_SEND` and a
6//! mutex take fires `QUEUE_RECEIVE`, and a conformance diff would notice
7//! any tidier arrangement.
8//!
9//! # A blocking call with no stack to block on
10//!
11//! `xQueueReceive( q, &v, 100 )` blocks: the C task stops inside the call,
12//! and when it is woken it loops back to the top, re-checks the queue, and
13//! either returns the item or blocks again. This kernel has no stack to
14//! suspend, so the loop is turned inside out — a blocking call returns
15//! [`Blocked`] and the caller invokes it again, unchanged, when it next
16//! runs:
17//!
18//! ```ignore
19//! // in a task body, at some `pc`:
20//! match k.queue_receive(q, 100)? {
21//! Blocked => {} // stay at this pc; the kernel parked us
22//! Ready(value) => { ...; self.pc = next; }
23//! }
24//! ```
25//!
26//! The task is off the ready list when `Blocked` comes back, so the runner
27//! will not step it again until the kernel wakes it — which is exactly when
28//! the C thread would have resumed. What the C keeps on its stack across
29//! the block (the timeout, the remaining ticks, whether inheritance
30//! happened) this kernel keeps in the TCB, in [`Wait`], because that *is*
31//! the stack frame.
32
33use rusty_rtos_core::config::Config;
34use rusty_rtos_core::error::{Error, Result};
35use rusty_rtos_core::handle::{QueueHandle, TaskHandle};
36use rusty_rtos_core::hooks::TickHook;
37use rusty_rtos_core::isr::Woken;
38use rusty_rtos_core::port::Port;
39use rusty_rtos_core::trace::{Event, Trace};
40
41use crate::kernel::{Kernel, OwedTrace, QueueResume};
42
43/// What a blocking call answers.
44#[derive(Debug, Clone, Copy, PartialEq, Eq)]
45pub enum Wait<T> {
46 /// The kernel parked the calling task. Leave the program counter where
47 /// it is and make the same call again when the task next runs.
48 Blocked,
49 /// The call completed.
50 Ready(T),
51}
52
53pub use Wait::{Blocked, Ready};
54
55/// Where an item goes (`xCopyPosition`).
56#[derive(Debug, Clone, Copy, PartialEq, Eq)]
57pub enum Position {
58 /// `queueSEND_TO_BACK`.
59 Back,
60 /// `queueSEND_TO_FRONT`.
61 Front,
62 /// `queueOVERWRITE`: write to the front of a length-one queue whether
63 /// or not it already holds something, and leave the message count
64 /// where it was rather than raising it.
65 Overwrite,
66}
67
68/// What a queue is underneath (`ucQueueType`).
69#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
70pub enum Kind {
71 // ORDER IS LOAD-BEARING, and only for the two predicates below. The kinds
72 // that carry data are first and the two mutex kinds are last, so
73 // `carries_data` and `is_mutex` are each a range the compiler can test
74 // with one comparison instead of two. Nothing reads a discriminant, so
75 // the order carries no other meaning.
76 /// A queue of values. `#[default]` so a free arena slot holds something
77 /// real without unsafe; the ORDER above is untouched, and nothing reads
78 /// a free slot.
79 #[default]
80 Queue,
81 /// A queue set: a queue whose items are the handles of the queues in
82 /// it. `xQueueSelectFromSet` is a receive from this one.
83 Set,
84 /// A binary or counting semaphore: a queue with a zero-size item, so
85 /// only the count matters.
86 Semaphore,
87 /// A mutex: a one-deep semaphore with a holder and priority
88 /// inheritance.
89 Mutex,
90 /// A recursive mutex: as above, and the holder may take it again.
91 RecursiveMutex,
92}
93
94impl Kind {
95 /// `uxItemSize == 0`: semaphores and mutexes carry a count, not data.
96 pub(crate) const fn carries_data(self) -> bool {
97 matches!(self, Self::Queue | Self::Set)
98 }
99
100 /// The two mutex kinds, which inherit priority.
101 pub(crate) const fn is_mutex(self) -> bool {
102 matches!(self, Self::Mutex | Self::RecursiveMutex)
103 }
104}
105
106/// One queue: the C `Queue_t` minus the byte pointers. Storage is a range
107/// of the kernel's shared slot pool, and the read and write cursors follow
108/// C's `pcReadFrom` / `pcWriteTo` exactly — `pcReadFrom` points at the item
109/// last read, which is what makes `queueSEND_TO_FRONT` a single step back.
110#[derive(Debug, Clone, Copy, Default)]
111pub(crate) struct Queue {
112 pub(crate) base: usize,
113 /// `uxLength`.
114 pub(crate) length: usize,
115 /// `uxMessagesWaiting`.
116 pub(crate) waiting: usize,
117 /// `pcReadFrom`, as an index: the item last read.
118 pub(crate) read_from: usize,
119 /// `pcWriteTo`, as an index: where the next back-send writes.
120 pub(crate) write_to: usize,
121 pub(crate) kind: Kind,
122 /// `u.xSemaphore.xMutexHolder`.
123 pub(crate) holder: TaskHandle,
124 /// `u.xSemaphore.uxRecursiveCallCount`.
125 pub(crate) recursions: u32,
126 /// `cTxLock`: [`UNLOCKED`] when no task holds the queue locked;
127 /// otherwise how many sends an interrupt made while it was locked, and
128 /// therefore how many receivers `prvUnlockQueue` still owes a wake-up.
129 pub(crate) tx_lock: i8,
130 /// `cRxLock`, the same for receives and the tasks waiting to send.
131 pub(crate) rx_lock: i8,
132 /// `pxQueueSetContainer`: the set this queue belongs to, if any. An
133 /// item arriving here is announced there.
134 pub(crate) set_container: QueueHandle,
135}
136
137/// `queueUNLOCKED`.
138pub(crate) const UNLOCKED: i8 = -1;
139/// `queueLOCKED_UNMODIFIED`.
140pub(crate) const LOCKED_UNMODIFIED: i8 = 0;
141
142impl Queue {
143 // A3: ONE caller, so the out-of-line body pays a prologue and epilogue for
144 // a single call. Inlining moves the body rather than duplicating it.
145 #[inline]
146 pub(crate) const fn new(base: usize, length: usize, kind: Kind) -> Self {
147 Self {
148 base,
149 length,
150 waiting: 0,
151 // `xQueueGenericReset`: pcReadFrom = pcHead + (length - 1) * itemSize.
152 read_from: length.saturating_sub(1),
153 write_to: 0,
154 kind,
155 holder: TaskHandle::NULL,
156 recursions: 0,
157 tx_lock: UNLOCKED,
158 rx_lock: UNLOCKED,
159 set_container: QueueHandle::NULL,
160 }
161 }
162}
163
164impl<
165 C: Config,
166 P: Port,
167 T: Trace,
168 H,
169 const TASKS: usize,
170 const ITEMS: usize,
171 const LISTS: usize,
172 const QUEUES: usize,
173 const SLOTS: usize,
174 const BUFFERS: usize,
175 const BYTES: usize,
176 const TIMERS: usize,
177 const GROUPS: usize,
178 const TIMER_CMDS: usize,
179> Kernel<C, P, T, H, TASKS, ITEMS, LISTS, QUEUES, SLOTS, BUFFERS, BYTES, TIMERS, GROUPS, TIMER_CMDS>
180where
181 H: TickHook<Self>,
182{
183 // ------------------------------------------------------------ create --
184
185 /// `xQueueCreate`.
186 ///
187 /// # Errors
188 /// [`Error::Full`] when the queue arena or the shared slot pool is
189 /// exhausted; [`Error::InvalidArgument`] for a zero length.
190 pub fn queue_create(&mut self, length: usize) -> Result<QueueHandle> {
191 self.new_queue(length, Kind::Queue)
192 }
193
194 /// `vQueueDelete`, and `vSemaphoreDelete`, which the C defines as the
195 /// same function.
196 ///
197 /// # It traces nothing and still costs time
198 ///
199 /// `vQueueDelete` takes no critical section of its own — but it ends in
200 /// `vPortFree( pxQueue )`, and every `heap_N.c` wraps free in
201 /// `vTaskSuspendAll` / `xTaskResumeAll` exactly as it wraps malloc. So
202 /// a delete costs **one outermost critical-section exit**, which under
203 /// the sim contract is one unit of time, while emitting no trace line
204 /// at all (this harness defines no `traceQUEUE_DELETE`).
205 ///
206 /// `AbortDelay`'s remake left the deletes out on the reasoning that an
207 /// event-less call cannot move the trace. Every event still agreed and
208 /// the exit column was one short from the first delete onwards. The
209 /// sibling `event_group_delete` already carried this note; the queue
210 /// had no delete at all.
211 ///
212 /// # Errors
213 /// [`Error::Gone`] for a stale handle.
214 pub fn queue_delete(&mut self, queue: QueueHandle) -> Result<()> {
215 let q = *self.queues.resolve(queue)?;
216 // Give the item storage back when it is the last thing allocated.
217 //
218 // `new_queue` bump-allocates from `slots_used` and, until
219 // 2026-09-21, nothing ever lowered it: the DESCRIPTOR came back and
220 // the storage did not, so a create/delete loop exhausted `SLOTS`
221 // permanently. `AbortDelay` hits it after ~108 passes, which is how
222 // it was found, and any application with a queue per connection or
223 // per job hits it too.
224 //
225 // This is the same move `give_bytes` already makes for the byte
226 // arena — that allocator exists because "a bump allocator would run
227 // out in a few hundred ticks", and the identical argument applies
228 // here and was simply never made.
229 //
230 // Deliberately the END-OF-ARENA case only, and not a free list.
231 // The pattern that needs this is create-then-delete of the same
232 // object, where the deleted extent IS the last one, and it is what
233 // the corpus exercises. An out-of-order delete still strands its
234 // slots; that wants the full `free_blocks` treatment and is a
235 // larger change with more to prove.
236 if q.kind.carries_data() {
237 self.give_slots(q.base, q.length);
238 }
239 let _ = self.queues.discard(queue);
240 // `vPortFree( pxQueue )`.
241 self.account_for_allocation();
242 Ok(())
243 }
244
245 /// `xSemaphoreCreateBinary`: a one-deep semaphore, created empty.
246 ///
247 /// # Errors
248 /// As [`Kernel::queue_create`].
249 pub fn semaphore_create_binary(&mut self) -> Result<QueueHandle> {
250 self.new_queue(1, Kind::Semaphore)
251 }
252
253 /// `xSemaphoreCreateCounting`.
254 ///
255 /// # Errors
256 /// As [`Kernel::queue_create`]; [`Error::InvalidArgument`] if the
257 /// initial count exceeds the maximum.
258 pub fn semaphore_create_counting(&mut self, max: usize, initial: usize) -> Result<QueueHandle> {
259 if initial > max {
260 return Err(Error::InvalidArgument);
261 }
262 let handle = self.new_queue(max, Kind::Semaphore)?;
263 if let Ok(q) = self.queues.resolve_mut(handle) {
264 q.waiting = initial;
265 }
266 Ok(handle)
267 }
268
269 /// `xSemaphoreCreateMutex`: created **available**, unlike a semaphore.
270 ///
271 /// # Errors
272 /// As [`Kernel::queue_create`].
273 pub fn mutex_create(&mut self) -> Result<QueueHandle> {
274 self.new_mutex(Kind::Mutex)
275 }
276
277 /// `xSemaphoreCreateRecursiveMutex`.
278 ///
279 /// # Errors
280 /// As [`Kernel::queue_create`].
281 pub fn mutex_create_recursive(&mut self) -> Result<QueueHandle> {
282 self.new_mutex(Kind::RecursiveMutex)
283 }
284
285 /// `prvInitialiseMutex`: give the mutex once so it starts available.
286 ///
287 /// This goes through the general send for the same reason the C does — and
288 /// the byte count is the argument. Until 2026-09-25 a private `prime_mutex`
289 /// carried a hand-written copy of the send, specialised on the four
290 /// constants supplied below, because inlining the general body here cost
291 /// **534 bytes**. That defence was aimed at the wrong thing: the choice is
292 /// not "specialise or inline", it is *which symbol to call*, and
293 /// [`Self::send_generic_outlined`] — the A4 handle that already existed for
294 /// the cold callers — costs a five-argument frame, not a body.
295 ///
296 /// Calling it instead and deleting the specialisation measured
297 /// **−316 bytes** of flash (19,302 → 18,986), `mv` − 8, `slli` − 5,
298 /// `andi` − 2 and 99 fewer rv32 instructions — every pin down, none up.
299 /// `conform --all` stays at 26/26, which is what proves the trace is
300 /// unchanged: one critical section, `QueueSend` at the same point, the same
301 /// `exits`. A mutex therefore still shows a `QUEUE_SEND` line immediately
302 /// after its `QUEUE_CREATE`, before any task exists.
303 ///
304 /// The law: a specialisation written to escape an unwanted INLINE is only
305 /// worth its bytes if no out-of-line handle on the general body exists.
306 /// Check for the handle first.
307 fn new_mutex(&mut self, kind: Kind) -> Result<QueueHandle> {
308 let handle = self.new_queue(1, kind)?;
309 let _ = self.send_generic_outlined(handle, 0, 0, Position::Back)?;
310 Ok(handle)
311 }
312
313 /// Take `length` slots: first fit from what deleted queues gave back,
314 /// and only then from the untouched end.
315 ///
316 /// First fit rather than best fit for the same reason `take_bytes` uses
317 /// it — the pattern that needs an allocator at all is create-then-delete
318 /// of the *same size*, which first fit serves exactly and without
319 /// fragmenting.
320 fn take_slots(&mut self, length: usize) -> Option<usize> {
321 if length == 0 {
322 return Some(self.slots_used);
323 }
324 for i in 0..self.free_slot_count {
325 let (base, free) = *self.free_slots.get(i)?;
326 if free < length {
327 continue;
328 }
329 if free == length {
330 self.drop_free_slot(i);
331 } else if let Some(slot) = self.free_slots.get_mut(i) {
332 *slot = (base.saturating_add(length), free.wrapping_sub(length));
333 }
334 return Some(base);
335 }
336 let end = self.slots_used.checked_add(length)?;
337 if end > SLOTS {
338 return None;
339 }
340 let base = self.slots_used;
341 self.slots_used = end;
342 Some(base)
343 }
344
345 /// Give an extent back, coalescing with whichever neighbours touch it so
346 /// the list cannot grow past one entry per hole.
347 pub(crate) fn give_slots(&mut self, base: usize, length: usize) {
348 if length == 0 {
349 return;
350 }
351 let mut base = base;
352 let mut length = length;
353 let mut i = 0;
354 while i < self.free_slot_count {
355 let Some(&(other_base, other_len)) = self.free_slots.get(i) else {
356 break;
357 };
358 if other_base.saturating_add(other_len) == base {
359 base = other_base;
360 length = length.saturating_add(other_len);
361 self.drop_free_slot(i);
362 continue;
363 }
364 if base.saturating_add(length) == other_base {
365 length = length.saturating_add(other_len);
366 self.drop_free_slot(i);
367 continue;
368 }
369 i = i.wrapping_add(1);
370 }
371 // An extent at the very end goes back to the bump pointer instead of
372 // the list, which is what keeps a create/delete loop free.
373 if base.saturating_add(length) == self.slots_used {
374 self.slots_used = base;
375 return;
376 }
377 if let Some(slot) = self.free_slots.get_mut(self.free_slot_count) {
378 *slot = (base, length);
379 self.free_slot_count = self.free_slot_count.wrapping_add(1);
380 }
381 }
382
383 fn drop_free_slot(&mut self, index: usize) {
384 let last = self.free_slot_count.saturating_sub(1);
385 if index < last {
386 if let Some(&moved) = self.free_slots.get(last) {
387 if let Some(slot) = self.free_slots.get_mut(index) {
388 *slot = moved;
389 }
390 }
391 }
392 self.free_slot_count = last;
393 }
394
395 fn new_queue(&mut self, length: usize, kind: Kind) -> Result<QueueHandle> {
396 if length == 0 {
397 return Err(Error::InvalidArgument);
398 }
399 // A zero-item-size queue needs no storage; only real queues do.
400 let slots = if kind.carries_data() { length } else { 0 };
401 let base = self.take_slots(slots).ok_or(Error::Full)?;
402 let handle = match self.queues.try_insert(Queue::new(base, length, kind)) {
403 Ok(handle) => handle,
404 Err(_) => {
405 // The descriptor arena refused after the slots were taken,
406 // so give them straight back rather than stranding them.
407 self.give_slots(base, slots);
408 return Err(Error::Full);
409 }
410 };
411 // `xQueueGenericCreate` takes the queue and its storage from the
412 // heap before it initialises anything.
413 self.account_for_allocation();
414 // `xQueueGenericReset` runs in a critical section before
415 // `traceQUEUE_CREATE` fires at the end of `prvInitialiseNewQueue`.
416 self.enter_critical();
417 self.exit_critical();
418 let tick = self.tick;
419 self.note_exits();
420 self.trace.event(
421 tick,
422 Event::QueueCreate {
423 queue: handle,
424 name: "",
425 length,
426 },
427 );
428 Ok(handle)
429 }
430
431 // ----------------------------------------------------------- getters --
432
433 /// `uxQueueMessagesWaiting`, which takes a critical section.
434 ///
435 /// # Errors
436 /// [`Error::Gone`] for a stale handle.
437 pub fn queue_messages_waiting(&mut self, queue: QueueHandle) -> Result<usize> {
438 self.enter_critical();
439 let n = self.queues.resolve(queue).map(|q| q.waiting);
440 self.exit_critical();
441 n
442 }
443
444 /// `xQueueIsQueueFullFromISR`, which takes NO critical section.
445 ///
446 /// The C compares `uxMessagesWaiting` against `uxLength` and returns,
447 /// with nothing around it — it is meant to be called from an interrupt,
448 /// where a critical section would be the wrong thing. That is why this
449 /// takes `&self` and [`Kernel::queue_messages_waiting`] takes `&mut
450 /// self`: the borrow is the proof that this one cannot charge the
451 /// clock, and on the sim a charged section is a sixteenth of a tick.
452 ///
453 /// # Errors
454 /// [`Error::Gone`] for a stale handle.
455 pub fn queue_is_full_from_isr(&self, queue: QueueHandle) -> Result<bool> {
456 self.queues.resolve(queue).map(|q| q.waiting == q.length)
457 }
458
459 /// `uxSemaphoreGetCount`, which is the same thing.
460 ///
461 /// # Errors
462 /// [`Error::Gone`] for a stale handle.
463 pub fn semaphore_count(&mut self, semaphore: QueueHandle) -> Result<usize> {
464 self.queue_messages_waiting(semaphore)
465 }
466
467 /// `xSemaphoreGetMutexHolderFromISR`, which is
468 /// `xQueueGetMutexHolderFromISR`: the same answer, read without a
469 /// critical section because an ISR is already inside one.
470 ///
471 /// # Errors
472 /// [`Error::Gone`] for a stale handle.
473 pub fn mutex_holder_from_isr(&mut self, mutex: QueueHandle) -> Result<TaskHandle> {
474 self.queues.resolve(mutex).map(|q| q.holder)
475 }
476
477 /// `xSemaphoreGetMutexHolder`, which takes a critical section.
478 ///
479 /// # Errors
480 /// [`Error::Gone`] for a stale handle.
481 pub fn mutex_holder(&mut self, mutex: QueueHandle) -> Result<TaskHandle> {
482 self.enter_critical();
483 let holder = self.queues.resolve(mutex).map(|q| q.holder);
484 self.exit_critical();
485 holder
486 }
487
488 // -------------------------------------------------------------- send --
489
490 /// `xQueueSend` / `xQueueSendToBack`.
491 ///
492 /// # Errors
493 /// [`Error::Full`] when the block time expires with the queue still
494 /// full; [`Error::Gone`] for a stale handle.
495 pub fn queue_send(&mut self, queue: QueueHandle, value: u64, ticks: u64) -> Result<Wait<()>> {
496 self.queue_send_generic(queue, value, ticks, Position::Back)
497 }
498
499 /// `xQueueReset`, which is `xQueueGenericReset( q, pdFALSE )`: empty
500 /// the queue and unlock it, waking one waiting sender if there is one.
501 ///
502 /// The event lists survive — only a brand-new queue re-initialises
503 /// those — so a task blocked on this queue stays blocked, and the one
504 /// woken here is woken because the reset made room.
505 ///
506 /// # Errors
507 /// [`Error::Gone`] for a stale handle.
508 pub fn queue_reset(&mut self, queue: QueueHandle) -> Result<()> {
509 self.enter_critical();
510 let result = (|| {
511 let length = self.queues.resolve(queue)?.length;
512 {
513 let q = self.queues.resolve_mut(queue)?;
514 q.waiting = 0;
515 q.write_to = 0;
516 q.read_from = length.saturating_sub(1);
517 q.rx_lock = UNLOCKED;
518 q.tx_lock = UNLOCKED;
519 }
520 let senders = Self::queue_send_list(queue);
521 if !self.lists.is_empty_of(senders) && self.remove_from_event_list(senders)? {
522 // queueYIELD_IF_USING_PREEMPTION(), inside the section.
523 self.port_yield();
524 }
525 Ok(())
526 })();
527 self.exit_critical();
528 result
529 }
530
531 /// `uxQueueSpacesAvailable`: how many more items would fit.
532 ///
533 /// # Errors
534 /// [`Error::Gone`] for a stale handle.
535 pub fn queue_spaces_available(&mut self, queue: QueueHandle) -> Result<usize> {
536 self.enter_critical();
537 let spaces = self
538 .queues
539 .resolve(queue)
540 .map(|q| q.length.wrapping_sub(q.waiting));
541 self.exit_critical();
542 spaces
543 }
544
545 // --------------------------------------------------- queue sets --
546 //
547 // A queue set is a queue whose items are the handles of the queues in
548 // it. Nothing is *moved* into the set: an item arriving on a member
549 // queue puts that queue's handle on the set, and the task that wakes on
550 // the set reads the handle and then reads the member queue itself. So a
551 // set holds an announcement, not the data — which is why adding a queue
552 // that already has items in it is refused, and why the two counts would
553 // otherwise drift apart forever.
554
555 /// `xQueueCreateSet`: a set that can hold `length` announcements.
556 ///
557 /// # Errors
558 /// As [`Kernel::queue_create`].
559 pub fn queue_create_set(&mut self, length: usize) -> Result<QueueHandle> {
560 self.new_queue(length, Kind::Set)
561 }
562
563 /// `xQueueAddToSet`.
564 ///
565 /// `false` is the C's `pdFAIL`: the target is not a set, the queue is
566 /// already in one, or it has items waiting that the set never saw.
567 ///
568 /// # Errors
569 /// [`Error::Gone`] for a stale handle.
570 pub fn queue_add_to_set(&mut self, queue: QueueHandle, set: QueueHandle) -> Result<bool> {
571 self.enter_critical();
572 let result = (|| {
573 if self.queues.resolve(set)?.kind != Kind::Set {
574 return Ok(false);
575 }
576 let member = self.queues.resolve(queue)?;
577 if !member.set_container.is_null() || member.waiting != 0 {
578 return Ok(false);
579 }
580 self.queues.resolve_mut(queue)?.set_container = set;
581 Ok(true)
582 })();
583 self.exit_critical();
584 result
585 }
586
587 /// `xQueueRemoveFromSet`.
588 ///
589 /// # Errors
590 /// [`Error::Gone`] for a stale handle.
591 pub fn queue_remove_from_set(&mut self, queue: QueueHandle, set: QueueHandle) -> Result<bool> {
592 // Both refusals are tested OUTSIDE the section and only the removal
593 // itself is inside it, which is what the C does -- and it is NOT
594 // what `xQueueAddToSet` does, three functions up, where the whole
595 // body including both refusals is inside one section. The asymmetry
596 // is upstream's and it is load-bearing here: a failing remove costs
597 // the clock nothing, and `QueueSet`'s setup makes one deliberately,
598 // to prove a queue cannot be removed from a set it is not in.
599 // Charging for it put a tick one event early, 30 lines in.
600 let member = *self.queues.resolve(queue)?;
601 if member.set_container != set || member.waiting != 0 {
602 return Ok(false);
603 }
604 self.enter_critical();
605 let result = self
606 .queues
607 .resolve_mut(queue)
608 .map(|q| q.set_container = QueueHandle::NULL);
609 self.exit_critical();
610 result?;
611 Ok(true)
612 }
613
614 /// `xQueueSelectFromSet`: which member queue has something on it.
615 ///
616 /// The C is one line — `xQueueReceive( xQueueSet, &xReturn, xTicksToWait )`
617 /// — so this blocks, traces and costs exactly what a receive does, and
618 /// `None` is the C's `NULL`.
619 ///
620 /// # Errors
621 /// As [`Kernel::queue_receive`].
622 pub fn queue_select_from_set(
623 &mut self,
624 set: QueueHandle,
625 ticks: u64,
626 ) -> Result<Wait<Option<QueueHandle>>> {
627 match self.queue_receive(set, ticks) {
628 Ok(Blocked) => Ok(Blocked),
629 Ok(Ready(raw)) => {
630 let handle = QueueHandle::from_raw(u32::try_from(raw).unwrap_or(0));
631 Ok(Ready((!handle.is_null()).then_some(handle)))
632 }
633 Err(Error::Empty) => Ok(Ready(None)),
634 Err(e) => Err(e),
635 }
636 }
637
638 /// `xQueueSelectFromSetFromISR`: which queue in `set` has data, from
639 /// an interrupt.
640 ///
641 /// The C is three lines -- `xQueueReceiveFromISR` on the set itself,
642 /// with a NULL handle meaning "none" -- because a set IS a queue of
643 /// member handles. This is that, and it takes no critical section for
644 /// the same reason the rest of the FromISR surface does not.
645 ///
646 /// # Errors
647 /// [`Error::Gone`] for a stale handle.
648 pub fn queue_select_from_set_from_isr(
649 &mut self,
650 set: QueueHandle,
651 ) -> Result<Option<QueueHandle>> {
652 match self.queue_receive_from_isr(set) {
653 Ok((raw, _woken)) => {
654 let handle = QueueHandle::from_raw(u32::try_from(raw).unwrap_or(0));
655 Ok((!handle.is_null()).then_some(handle))
656 }
657 // An empty set is "no queue has data", which is what the C's
658 // NULL means; it is not a failure.
659 Err(Error::Empty) => Ok(None),
660 Err(e) => Err(e),
661 }
662 }
663
664 /// `prvNotifyQueueSetContainer`: put `queue`'s handle on the set it
665 /// belongs to. `true` when that woke a task that outranks the current
666 /// one.
667 /// `#[cold]` because the call is GUARDED and reached from many sites on
668 /// one hot path, which is the shape that pays: it lets LLVM keep the
669 /// caller's frame setup out of the likely route. Measured on
670 /// `riscv32-qemu-tick-work`, one attribute at a time.
671 /// Worth queue -1 on its own.
672 #[cold]
673 fn notify_queue_set_container(&mut self, queue: QueueHandle) -> Result<bool> {
674 let set = self.queues.resolve(queue)?.set_container;
675 let container = *self.queues.resolve(set)?;
676 if container.waiting >= container.length {
677 // The C asserts this cannot happen and does nothing if it does.
678 return Ok(false);
679 }
680 let tx_lock = container.tx_lock;
681 let tick = self.tick;
682 self.note_exits();
683 // `traceQUEUE_SET_SEND` is `traceQUEUE_SEND` unless a port says
684 // otherwise, and this port does not.
685 self.trace.event(
686 tick,
687 Event::QueueSend {
688 queue: set,
689 name: "",
690 },
691 );
692 let mut woke =
693 self.copy_data_to_queue(set, &container, u64::from(queue.to_raw()), Position::Back)?;
694 if tx_lock == UNLOCKED {
695 let receivers = Self::queue_receive_list(set);
696 if !self.lists.is_empty_of(receivers) && self.remove_from_event_list(receivers)? {
697 woke = true;
698 }
699 } else {
700 self.increment_tx_lock(set, tx_lock);
701 }
702 Ok(woke)
703 }
704
705 /// `xQueueOverwrite`: write to a length-one queue whether or not it
706 /// already holds something.
707 ///
708 /// The C is `xQueueGenericSend( q, item, 0, queueOVERWRITE )` — a send
709 /// with no block time, because a queue that is always writable can
710 /// never make the caller wait.
711 ///
712 /// # Errors
713 /// As [`Kernel::queue_send`].
714 pub fn queue_overwrite(&mut self, queue: QueueHandle, value: u64) -> Result<Wait<()>> {
715 self.send_generic_outlined(queue, value, 0, Position::Overwrite)
716 }
717
718 // ------------------------------------------------------- from an ISR --
719 //
720 // The `FromISR` half of the API. Three things make it a different
721 // animal from the task half, and all three are visible in a trace:
722 //
723 // * It never blocks and never yields. It reports that a higher
724 // priority task woke, by returning a [`Woken`], and the interrupt
725 // decides what to do about it on the way out.
726 // * Its critical section is `portSET_INTERRUPT_MASK_FROM_ISR`, not
727 // `portENTER_CRITICAL`. On the Posix port both are empty, so a call
728 // here costs no sim time at all — which is exactly what the C does
729 // and therefore what a matching trace requires.
730 // * If a task has the queue locked it may not touch the event lists.
731 // It counts what it did in `cTxLock` / `cRxLock` instead, and
732 // `prvUnlockQueue` pays the wake-ups back when the task is done.
733
734 /// `xQueueGenericSendFromISR`.
735 ///
736 /// # Errors
737 /// [`Error::Full`] when the queue is full and this is not an overwrite;
738 /// [`Error::Gone`] for a stale handle.
739 pub fn queue_send_generic_from_isr(
740 &mut self,
741 queue: QueueHandle,
742 value: u64,
743 position: Position,
744 ) -> Result<Woken> {
745 let mask = self.port.enter_critical_from_isr();
746 let result = self.send_from_isr_locked(queue, value, position);
747 self.port.exit_critical_from_isr(mask);
748 result
749 }
750
751 fn send_from_isr_locked(
752 &mut self,
753 queue: QueueHandle,
754 value: u64,
755 position: Position,
756 ) -> Result<Woken> {
757 let snapshot = *self.queues.resolve(queue)?;
758 if snapshot.waiting >= snapshot.length && position != Position::Overwrite {
759 // `traceQUEUE_SEND_FROM_ISR_FAILED` is not one of the harness's
760 // hooks, so a full queue says nothing on either side.
761 return Err(Error::Full);
762 }
763 let tx_lock = snapshot.tx_lock;
764 let tick = self.tick;
765 self.note_exits();
766 self.trace
767 .event(tick, Event::QueueSendFromIsr { queue, name: "" });
768 let previously_waiting = snapshot.waiting;
769 let _ = self.copy_data_to_queue(queue, &snapshot, value, position)?;
770 let mut woken = Woken::NO;
771 if tx_lock == UNLOCKED {
772 if C::USE_QUEUE_SETS && !snapshot.set_container.is_null() {
773 let overwrote = position == Position::Overwrite && previously_waiting > 0;
774 if !overwrote && self.notify_queue_set_container(queue)? {
775 woken = Woken::YES;
776 }
777 return Ok(woken);
778 }
779 let receivers = Self::queue_receive_list(queue);
780 if !self.lists.is_empty_of(receivers) && self.remove_from_event_list(receivers)? {
781 woken = Woken::YES;
782 }
783 } else {
784 self.increment_tx_lock(queue, tx_lock);
785 }
786 Ok(woken)
787 }
788
789 /// `xQueueSendToBackFromISR`.
790 ///
791 /// # Errors
792 /// As [`Kernel::queue_send_generic_from_isr`].
793 pub fn queue_send_from_isr(&mut self, queue: QueueHandle, value: u64) -> Result<Woken> {
794 self.queue_send_generic_from_isr(queue, value, Position::Back)
795 }
796
797 /// `xQueueSendToFrontFromISR`.
798 ///
799 /// # Errors
800 /// As [`Kernel::queue_send_generic_from_isr`].
801 pub fn queue_send_to_front_from_isr(
802 &mut self,
803 queue: QueueHandle,
804 value: u64,
805 ) -> Result<Woken> {
806 self.queue_send_generic_from_isr(queue, value, Position::Front)
807 }
808
809 /// `xQueueOverwriteFromISR`. Only for a queue of length one.
810 ///
811 /// # Errors
812 /// As [`Kernel::queue_send_generic_from_isr`].
813 pub fn queue_overwrite_from_isr(&mut self, queue: QueueHandle, value: u64) -> Result<Woken> {
814 self.queue_send_generic_from_isr(queue, value, Position::Overwrite)
815 }
816
817 /// `xSemaphoreGiveFromISR`.
818 ///
819 /// # Errors
820 /// As [`Kernel::queue_send_generic_from_isr`].
821 pub fn semaphore_give_from_isr(&mut self, semaphore: QueueHandle) -> Result<Woken> {
822 self.queue_send_generic_from_isr(semaphore, 0, Position::Back)
823 }
824
825 /// `xQueueReceiveFromISR`: the value, and whether a waiting sender that
826 /// this made room for outranks the interrupted task.
827 ///
828 /// # Errors
829 /// [`Error::Empty`] when the queue is empty; [`Error::Gone`] for a
830 /// stale handle.
831 pub fn queue_receive_from_isr(&mut self, queue: QueueHandle) -> Result<(u64, Woken)> {
832 let mask = self.port.enter_critical_from_isr();
833 let result = self.receive_from_isr_locked(queue);
834 self.port.exit_critical_from_isr(mask);
835 result
836 }
837
838 fn receive_from_isr_locked(&mut self, queue: QueueHandle) -> Result<(u64, Woken)> {
839 let snapshot = *self.queues.resolve(queue)?;
840 if snapshot.waiting == 0 {
841 return Err(Error::Empty);
842 }
843 let rx_lock = snapshot.rx_lock;
844 let tick = self.tick;
845 self.note_exits();
846 self.trace
847 .event(tick, Event::QueueReceiveFromIsr { queue, name: "" });
848 let value = self.copy_data_from_queue(
849 queue,
850 snapshot.kind,
851 snapshot.base,
852 snapshot.length,
853 snapshot.read_from,
854 false,
855 )?;
856 let mut woken = Woken::NO;
857 if rx_lock == UNLOCKED {
858 let senders = Self::queue_send_list(queue);
859 if !self.lists.is_empty_of(senders) && self.remove_from_event_list(senders)? {
860 woken = Woken::YES;
861 }
862 } else {
863 self.increment_rx_lock(queue, rx_lock);
864 }
865 Ok((value, woken))
866 }
867
868 /// `xQueuePeekFromISR`: the head without removing it.
869 ///
870 /// It wakes nobody and traces nothing — the C has no hook for it —
871 /// which is why it returns a bare value rather than a [`Woken`].
872 ///
873 /// # Errors
874 /// [`Error::Empty`] when the queue is empty; [`Error::Gone`] for a
875 /// stale handle.
876 pub fn queue_peek_from_isr(&mut self, queue: QueueHandle) -> Result<u64> {
877 let mask = self.port.enter_critical_from_isr();
878 let result = match self.queues.resolve(queue) {
879 Ok(q) if q.waiting > 0 => {
880 let snapshot = *q;
881 self.copy_data_from_queue(
882 queue,
883 snapshot.kind,
884 snapshot.base,
885 snapshot.length,
886 snapshot.read_from,
887 true,
888 )
889 }
890 Ok(_) => Err(Error::Empty),
891 Err(e) => Err(e),
892 };
893 self.port.exit_critical_from_isr(mask);
894 result
895 }
896
897 /// `prvIncrementQueueTxLock`, capped at the task count as the C caps it.
898 fn increment_tx_lock(&mut self, queue: QueueHandle, tx_lock: i8) {
899 let tasks = self.task_count();
900 if let Ok(q) = self.queues.resolve_mut(queue) {
901 // 32-bit throughout. This was `i64::from(lock) < tasks as i64`,
902 // which is a SIXTY-FOUR-bit signed compare on a target whose
903 // registers are 32 -- to order an `i8` against a task count. The
904 // `< 0` arm is what makes the unsigned cast safe and is the
905 // UNLOCKED sentinel's case, which cannot reach the increment.
906 if tx_lock < 0 || (tx_lock as usize) < tasks {
907 q.tx_lock = tx_lock.wrapping_add(1);
908 }
909 }
910 }
911
912 /// `prvIncrementQueueRxLock`.
913 fn increment_rx_lock(&mut self, queue: QueueHandle, rx_lock: i8) {
914 let tasks = self.task_count();
915 if let Ok(q) = self.queues.resolve_mut(queue) {
916 // 32-bit throughout. This was `i64::from(lock) < tasks as i64`,
917 // which is a SIXTY-FOUR-bit signed compare on a target whose
918 // registers are 32 -- to order an `i8` against a task count. The
919 // `< 0` arm is what makes the unsigned cast safe and is the
920 // UNLOCKED sentinel's case, which cannot reach the increment.
921 if rx_lock < 0 || (rx_lock as usize) < tasks {
922 q.rx_lock = rx_lock.wrapping_add(1);
923 }
924 }
925 }
926
927 /// `xQueueSendToFront`.
928 ///
929 /// # Errors
930 /// As [`Kernel::queue_send`].
931 pub fn queue_send_to_front(
932 &mut self,
933 queue: QueueHandle,
934 value: u64,
935 ticks: u64,
936 ) -> Result<Wait<()>> {
937 self.send_generic_outlined(queue, value, ticks, Position::Front)
938 }
939
940 /// `xSemaphoreGive`: a send with no data.
941 ///
942 /// # Errors
943 /// [`Error::Full`] when the semaphore is already at its maximum count.
944 pub fn semaphore_give(&mut self, semaphore: QueueHandle) -> Result<Wait<()>> {
945 self.send_generic_outlined(semaphore, 0, 0, Position::Back)
946 }
947
948 /// The out-of-line handle on [`Kernel::queue_send_generic`]'s body.
949 ///
950 /// One shared copy for the wrappers that are not on a measured hot row --
951 /// `queue_send_to_front`, `queue_overwrite`, `semaphore_give` and the
952 /// timer daemon's post. `queue_send` keeps the inlined body because
953 /// `send_full`, `queue_roundtrip` and `block_cycle` all go through it and
954 /// all three pay for the call.
955 #[inline(never)]
956 pub(crate) fn send_generic_outlined(
957 &mut self,
958 queue: QueueHandle,
959 value: u64,
960 ticks: u64,
961 position: Position,
962 ) -> Result<Wait<()>> {
963 self.queue_send_generic(queue, value, ticks, position)
964 }
965
966 /// `xQueueGenericSend`, one pass of its `for(;;)`.
967 ///
968 /// # Errors
969 /// [`Error::Full`] on timeout; [`Error::Gone`] for a stale handle.
970 /// In line on purpose, for the same reason as
971 /// [`Kernel::queue_take`]: seven instructions of prologue and ten of
972 /// epilogue on every one of khot-ir's 48,000 sends, so that `xQueueSend`
973 /// and its siblings could pass four arguments through unchanged. This
974 /// one pays on every instrument, `kernel-ir` included.
975 ///
976 /// **But only the HOT site wants it.** With the flash probe fixed so that
977 /// real bodies link at all (2026-09-24), this was inlined at five places
978 /// and cost about 1,666 bytes of duplicate.
979 ///
980 /// RE-PRICED 2026-09-25, because that figure was taken at FIVE inline sites
981 /// and there is now one. Dropping the hint today measures:
982 ///
983 /// * flash **19,302 -> 18,704, i.e. -598 B**, with `mv` -31, `andi` -7,
984 /// `slli` -8 and 221 fewer instructions;
985 /// * rv32 work rows **+118 instructions** over seven of them —
986 /// `queue_roundtrip` 122 -> 169, `block_cycle` 983 -> 1022,
987 /// `send_full` 36 -> 62, and +1 or +2 on four more.
988 ///
989 /// That is **5.07 bytes of flash per instruction**, a worse rate than the
990 /// `queue_peek` trade declined at 7.3 and than the 10.7 this hint was
991 /// originally kept at. So it stays — but the K3 flash row is the one
992 /// FAILING its target while the work rows are absolutes with no C
993 /// comparison, so if that priority ever inverts this is the single largest
994 /// item in the send path and the numbers are here. `rusty-compiler-leverage` A4: a function has one
995 /// body, so one inlining decision serves callers who want different
996 /// things -- give it two names instead of averaging them.
997 /// [`Kernel::send_generic_outlined`] is the out-of-line handle, and the
998 /// four colder wrappers call that.
999 #[cfg_attr(not(feature = "small"), inline(always))]
1000 #[cfg_attr(feature = "small", inline(never))]
1001 pub fn queue_send_generic(
1002 &mut self,
1003 queue: QueueHandle,
1004 value: u64,
1005 ticks: u64,
1006 position: Position,
1007 ) -> Result<Wait<()>> {
1008 let caller = self.cur();
1009 // Resuming below the sampling exit, as the C's thread does. See
1010 // `Kernel::take_queue_resume`: the prologue has already run, and
1011 // repeating it would spend a second critical-section exit that the
1012 // C never spends.
1013 if self.take_queue_resume(caller) != QueueResume::No {
1014 // The send's only resume point is below its sampling exit: its
1015 // timed-out branch has nothing after `resume_all` that costs an
1016 // exit, so being switched away there changes no count.
1017 return self.queue_send_blocking(caller, queue);
1018 }
1019 self.enter_critical();
1020 let snapshot = match self.queues.resolve(queue) {
1021 Ok(q) => *q,
1022 Err(e) => {
1023 self.exit_critical();
1024 return Err(e);
1025 }
1026 };
1027 // The wait frame is set where the C sets it: `xQueueReceive` and
1028 // `xQueueGenericSend` call `vTaskInternalSetTimeOutState` only
1029 // after finding the queue unusable AND the block time non-zero,
1030 // guarded by `xEntryTimeSet`. Setting it up here instead meant
1031 // every *successful* call wrote a six-field frame and then had
1032 // `end_wait` wipe it on the way out.
1033 //
1034 // It also has to be after the resolve, so that a handle naming
1035 // nothing leaves the caller exactly as it found it -- which is
1036 // what `configASSERT( pxQueue )` means in the C.
1037 // `( uxMessagesWaiting < uxLength ) || ( xCopyPosition == queueOVERWRITE )`
1038 if snapshot.waiting < snapshot.length || position == Position::Overwrite {
1039 self.note_exits();
1040 let tick = self.tick;
1041 self.trace.event(tick, Event::QueueSend { queue, name: "" });
1042 let previously_waiting = snapshot.waiting;
1043 let yield_required = self.copy_data_to_queue(queue, &snapshot, value, position)?;
1044 if C::USE_QUEUE_SETS && !snapshot.set_container.is_null() {
1045 // A queue in a set announces the arrival there, not here.
1046 // An overwrite of an item that was already present is not an
1047 // arrival: the count did not change, so the set is not told.
1048 let overwrote = position == Position::Overwrite && previously_waiting > 0;
1049 if !overwrote && self.notify_queue_set_container(queue)? {
1050 self.port_yield();
1051 }
1052 self.exit_critical();
1053 self.end_wait(caller);
1054 return Ok(Ready(()));
1055 }
1056 let receivers = Self::queue_receive_list(queue);
1057 let woke_higher = if !self.lists.is_empty_of(receivers) {
1058 self.remove_from_event_list(receivers)?
1059 } else {
1060 false
1061 };
1062 // queueYIELD_IF_USING_PREEMPTION(), inside the section.
1063 // `yield_required` first: it is false on every send that is
1064 // not a mutex give, and it is a local, so on the common path
1065 // the list is not read a second time at all. Both operands are
1066 // pure, so the order is free to choose. (The read cannot be
1067 // hoisted above the removal above it -- removing the last
1068 // waiter is exactly what changes the answer.)
1069 if woke_higher || (yield_required && self.lists.is_empty_of(receivers)) {
1070 self.port_yield();
1071 }
1072 self.exit_critical();
1073 self.end_wait(caller);
1074 return Ok(Ready(()));
1075 }
1076 let remaining = self.begin_wait(caller, queue, ticks);
1077 if remaining.unwrap_or(0) == 0 {
1078 self.exit_critical();
1079 self.trace_failure_or_owe(caller, OwedTrace::SendFailed(queue));
1080 if remaining.is_some() {
1081 // A frame was written, so it has to be torn down. `None` means
1082 // `begin_wait` took its zero-block-time exit and wrote
1083 // nothing, and then `end_wait` is a provable no-op.
1084 self.end_wait(caller);
1085 }
1086 return Err(Error::Full);
1087 }
1088 self.exit_critical();
1089 // That exit can release a tick, and a tick can switch this task
1090 // away. The C's thread stops INSIDE the exit, so everything below
1091 // runs when the task is scheduled again -- and names that task.
1092 if self.cur() != caller {
1093 self.set_queue_resume(caller, QueueResume::BelowSample);
1094 return Ok(Blocked);
1095 }
1096 self.queue_send_blocking(caller, queue)
1097 }
1098
1099 /// Everything `xQueueGenericSend` does below its sampling exit.
1100 ///
1101 /// Split out so a call preempted at that exit re-enters here rather
1102 /// than at the top. See [`Kernel::take_queue_resume`].
1103 #[cold]
1104 fn queue_send_blocking(&mut self, caller: TaskHandle, queue: QueueHandle) -> Result<Wait<()>> {
1105 self.suspend_all();
1106 self.lock_queue(queue);
1107 let left = self.check_for_timeout(caller);
1108 if left.is_none() {
1109 self.unlock_queue(queue)?;
1110 let _ = self.resume_all();
1111 self.trace_failure_or_owe(caller, OwedTrace::SendFailed(queue));
1112 self.end_wait(caller);
1113 return Err(Error::Full);
1114 }
1115 if self.is_queue_full(queue) {
1116 self.note_exits();
1117 let tick = self.tick;
1118 self.trace
1119 .event(tick, Event::BlockingOnQueueSend { queue, name: "" });
1120 // `check_for_timeout` above already resolved the TCB and wrote
1121 // this field; `remaining_ticks` resolved it a second time to read
1122 // it back.
1123 let ticks_left = left.map_or(0, core::num::NonZeroU64::get);
1124 self.place_on_event_list(Self::queue_send_list(queue), ticks_left)?;
1125 self.unlock_queue(queue)?;
1126 if !self.resume_all() {
1127 self.yield_or_owe(caller);
1128 }
1129 } else {
1130 self.unlock_queue(queue)?;
1131 let _ = self.resume_all();
1132 }
1133 Ok(Blocked)
1134 }
1135
1136 /// `prvCopyDataToQueue`; `true` when giving a mutex back lowered the
1137 /// giver's priority and a yield is therefore wanted.
1138 ///
1139 /// This resolves rather than taking the caller's snapshot, and
1140 /// [`Kernel::copy_data_from_queue`] does the opposite. That is not an
1141 /// inconsistency: both forms were measured on both helpers and they
1142 /// disagree, because the writer's caller does not keep its snapshot
1143 /// live across the call and the reader's does.
1144 ///
1145 /// In line on purpose, at ALL THREE call sites.
1146 ///
1147 /// The rationale here used to say `queue_send_generic` was its only caller.
1148 /// That is no longer true — `queue_add_to_set` and
1149 /// `queue_send_generic_from_isr` call it too — so an A4 split was tried on
1150 /// 2026-09-25, keeping the hint for the hot caller and routing the cold ones
1151 /// through an out-of-line handle. It measured **+72 B of flash**, `mv` +12
1152 /// and 49 more instructions, and was reverted. `prime_mutex` was a fourth
1153 /// caller when that was measured; it was deleted later the same day (see
1154 /// [`Self::new_mutex`]) and the split was **re-tested at three callers**:
1155 /// **+6 B**, still a loss but only a twelfth of the old one. So ~66 of the
1156 /// original 72 bytes were `prime_mutex`'s own site — an A4 verdict can be
1157 /// carried almost entirely by ONE caller, which is why §9's re-test law
1158 /// exists. At three callers this is now a near-tie held by 6 bytes.
1159 ///
1160 /// The reason is worth keeping: the cold callers pass CONSTANT arguments
1161 /// (`Position::Back` from `queue_add_to_set`), so each inlined copy is
1162 /// SPECIALISED and folds well below the 138 bytes C's `prvCopyDataToQueue`
1163 /// occupies. Out of line they pay a five-argument frame and force
1164 /// `&snapshot` into memory, which costs more than the copies they remove.
1165 ///
1166 /// The hot reason also still stands: out of line it paid a call and a frame
1167 /// on all 48,000 sends of `bench/kernel-ir` to move one value into a slot
1168 /// the caller had already resolved.
1169 #[inline(always)]
1170 fn copy_data_to_queue(
1171 &mut self,
1172 queue: QueueHandle,
1173 snapshot: &Queue,
1174 value: u64,
1175 position: Position,
1176 ) -> Result<bool> {
1177 // The snapshot comes from the caller, which has already resolved this
1178 // handle -- and nothing between that resolve and this call can reach
1179 // the arena. Resolving again here made every send pay for the same
1180 // four checks twice. `copy_data_from_queue` has always taken its
1181 // snapshot this way.
1182 let mut yield_required = false;
1183 // `prvCopyDataToQueue` counts the item in — except on an overwrite
1184 // of a queue that already held one, where it decrements first so
1185 // the increment cancels and the count stays where it was.
1186 //
1187 // Decided once, up here, so that each arm below can fold the count
1188 // into the resolve it is already making. Deciding it after the arms
1189 // is what forced a further resolve for one field.
1190 let counted = !(position == Position::Overwrite && snapshot.waiting > 0);
1191 if snapshot.kind.carries_data() {
1192 let (index, next_read, next_write) = match position {
1193 Position::Back => (
1194 snapshot.write_to,
1195 snapshot.read_from,
1196 wrap_next(snapshot.write_to, snapshot.length),
1197 ),
1198 // Write at the item last read, then step back, so the next
1199 // read's pre-increment lands on it. An overwrite takes the
1200 // same path; only the message count differs.
1201 Position::Front | Position::Overwrite => (
1202 snapshot.read_from,
1203 wrap_prev(snapshot.read_from, snapshot.length),
1204 snapshot.write_to,
1205 ),
1206 };
1207 // `base + index` is below `base + length`, which the geometry
1208 // put inside `SLOTS`; and the `get_mut` refuses anything it is
1209 // not, so nothing rests on the arithmetic either way.
1210 if let Some(slot) = self.slots.get_mut(snapshot.base.wrapping_add(index)) {
1211 *slot = value;
1212 }
1213 let q = self.queues.resolve_mut(queue)?;
1214 q.read_from = next_read;
1215 q.write_to = next_write;
1216 if counted {
1217 q.waiting = q.waiting.wrapping_add(1);
1218 }
1219 } else if snapshot.kind.is_mutex() {
1220 // Giving a mutex back: the holder drops any inherited
1221 // priority. The disinherit touches the holder's TCB, not this
1222 // queue, so the resolve after it is still the first one.
1223 yield_required = self.priority_disinherit(snapshot.holder)?;
1224 let q = self.queues.resolve_mut(queue)?;
1225 q.holder = TaskHandle::NULL;
1226 if counted {
1227 q.waiting = q.waiting.wrapping_add(1);
1228 }
1229 } else if counted {
1230 // A counting or binary semaphore: only the count moves.
1231 let q = self.queues.resolve_mut(queue)?;
1232 q.waiting = q.waiting.wrapping_add(1);
1233 }
1234 Ok(yield_required)
1235 }
1236
1237 // ----------------------------------------------------------- receive --
1238
1239 /// The out-of-line handle on [`Kernel::queue_take`]s body.
1240 ///
1241 /// `peek` was a `const PEEK: bool`, which gave the whole take chain TWO
1242 /// monomorphisations and three inlined copies of a ~600-byte body that no
1243 /// A4 split could share, because `<true>` and `<false>` are different
1244 /// functions. As a runtime flag there is one body to share.
1245 ///
1246 /// Only `semaphore_take` uses this handle. `queue_receive` and `queue_peek`
1247 /// keep the inlined body because all four of `recv_empty`, `peek_ok`,
1248 /// `queue_roundtrip` and `block_cycle` go through one or the other --
1249 /// routing `queue_peek` here cost `peek_ok` **45 -> 108**, which is what a
1250 /// call and a frame are worth on a row that short.
1251 #[inline(never)]
1252 fn take_outlined(&mut self, queue: QueueHandle, ticks: u64, peek: bool) -> Result<Wait<u64>> {
1253 self.queue_take(queue, ticks, peek)
1254 }
1255
1256 /// `xQueueReceive`.
1257 ///
1258 /// # Errors
1259 /// [`Error::Empty`] when the block time expires with the queue still
1260 /// empty; [`Error::Gone`] for a stale handle.
1261 pub fn queue_receive(&mut self, queue: QueueHandle, ticks: u64) -> Result<Wait<u64>> {
1262 self.queue_take(queue, ticks, false)
1263 }
1264
1265 /// `xQueuePeek`: read the head without removing it.
1266 ///
1267 /// # Errors
1268 /// As [`Kernel::queue_receive`].
1269 pub fn queue_peek(&mut self, queue: QueueHandle, ticks: u64) -> Result<Wait<u64>> {
1270 self.queue_take(queue, ticks, true)
1271 }
1272
1273 /// `xSemaphoreTake`, which is `xQueueSemaphoreTake`: a receive with no
1274 /// data, and — on a mutex — with priority inheritance while it waits.
1275 ///
1276 /// # Errors
1277 /// [`Error::Empty`] on timeout; [`Error::Gone`] for a stale handle.
1278 // A3: ONE caller, so the out-of-line body pays a prologue and epilogue for
1279 // a single call. Inlining moves the body rather than duplicating it.
1280 #[inline]
1281 pub fn semaphore_take(&mut self, semaphore: QueueHandle, ticks: u64) -> Result<Wait<()>> {
1282 match self.take_outlined(semaphore, ticks, false)? {
1283 Blocked => Ok(Blocked),
1284 Ready(_) => Ok(Ready(())),
1285 }
1286 }
1287
1288 /// The shared body of `xQueueReceive`, `xQueuePeek` and
1289 /// `xQueueSemaphoreTake` — one pass of the C `for(;;)`.
1290 /// `peek` is a const because it is one at every call site: a receive
1291 /// passes `false`, a peek `true`, and a semaphore take `false`. As a
1292 /// parameter it cost an argument to set up and a branch at each of the
1293 /// ten places this function and its copy helper consult it.
1294 /// In line on purpose, into all three of `xQueueReceive`, `xQueuePeek`
1295 /// and `xSemaphoreTake`.
1296 ///
1297 /// Out of line it charged eight instructions of prologue and ten of
1298 /// epilogue to every take -- 48,000 of them in `khot-ir`, which is 7.6%
1299 /// of that instrument -- so that three wrappers could each hand it two
1300 /// arguments and return its result unchanged. LLVM declined on size; the
1301 /// frame is the reason to overrule that.
1302 ///
1303 /// The cost is stated rather than hidden: `kernel-ir` pays 214,804 for
1304 /// the larger wrappers. Splitting the body from the symbol does not
1305 /// recover it -- outlining at the semaphore site alone leaves `kernel-ir`
1306 /// 262,828 up and gives back 427,983 of `khot-ir`'s win, and outlining at
1307 /// the receive site leaves it 190,238 up and gives back 1,104,183 -- so
1308 /// the regression is the wrappers growing, not any one call site.
1309 #[cfg_attr(not(feature = "small"), inline(always))]
1310 #[cfg_attr(feature = "small", inline(never))]
1311 fn queue_take(&mut self, queue: QueueHandle, ticks: u64, peek: bool) -> Result<Wait<u64>> {
1312 let caller = self.cur();
1313 // Resuming below the sampling exit, as `queue_send_generic` does.
1314 // The snapshot is re-read rather than carried: nothing below uses
1315 // anything of it that a concurrent access could change, and the
1316 // read takes no critical section, so it costs the clock nothing.
1317 match self.take_queue_resume(caller) {
1318 QueueResume::No => {}
1319 QueueResume::BelowSample => {
1320 return self.queue_take_blocking(caller, queue, peek);
1321 }
1322 QueueResume::BelowTimedOutResume => {
1323 return self.queue_take_timed_out(queue, peek);
1324 }
1325 }
1326 self.enter_critical();
1327 // ONE resolve, mutable, for the whole fast path, reading only the
1328 // fields it uses. Two earlier shapes, both measured:
1329 // - `*q`, the whole eleven-field descriptor copied to reach five of
1330 // them: nine words of `usize` on rv32 with four dead on every
1331 // successful receive, and more live fields than LLVM keeps in
1332 // hand, so the function claimed callee-saved registers it had no
1333 // use for;
1334 // - the five fields read through `resolve`, then the item taken
1335 // through `copy_data_from_queue`, which resolved the handle AGAIN
1336 // to write `read_from` and `waiting` back. On rv32 LLVM merged the
1337 // two at run time, but every inlined copy carried the second body
1338 // (-242 B of flash, -395,353 kernel-ir), and on the Xtensa S3 the
1339 // copy was an outlined call on every receive.
1340 // The blocking tail below still wants the whole descriptor and
1341 // re-resolves for it, outside the section and at no cost to the
1342 // clock, exactly as the resume arms above do.
1343 let q = match self.queues.resolve_mut(queue) {
1344 Ok(q) => q,
1345 Err(e) => {
1346 self.exit_critical();
1347 return Err(e);
1348 }
1349 };
1350 let (waiting, kind) = (q.waiting, q.kind);
1351 // The wait frame is set where the C sets it: `xQueueReceive` and
1352 // `xQueueGenericSend` call `vTaskInternalSetTimeOutState` only
1353 // after finding the queue unusable AND the block time non-zero,
1354 // guarded by `xEntryTimeSet`. Setting it up here instead meant
1355 // every *successful* call wrote a six-field frame and then had
1356 // `end_wait` wipe it on the way out.
1357 //
1358 // It also has to be after the resolve, so that a handle naming
1359 // nothing leaves the caller exactly as it found it -- which is
1360 // what `configASSERT( pxQueue )` means in the C.
1361 if waiting > 0 {
1362 // `copy_data_from_queue`, in place, on the reference above.
1363 let value = if kind.carries_data() {
1364 let next = wrap_next(q.read_from, q.length);
1365 let value = self
1366 .slots
1367 .get(q.base.wrapping_add(next))
1368 .copied()
1369 .unwrap_or(0);
1370 if !peek {
1371 q.read_from = next;
1372 // Wrapping: `waiting > 0` was read under this section.
1373 q.waiting = q.waiting.wrapping_sub(1);
1374 }
1375 value
1376 } else {
1377 if !peek {
1378 q.waiting = q.waiting.wrapping_sub(1);
1379 }
1380 0
1381 };
1382 self.note_exits();
1383 let tick = self.tick;
1384 // `xQueuePeek` is its own function in the C with its own trace
1385 // macro; `xQueueSemaphoreTake` shares `xQueueReceive`'s.
1386 let event = if peek {
1387 Event::QueuePeek { queue, name: "" }
1388 } else {
1389 Event::QueueReceive { queue, name: "" }
1390 };
1391 self.trace.event(tick, event);
1392 if peek {
1393 // A peek wakes another *receiver*, not a sender: the item
1394 // is still there.
1395 let receivers = Self::queue_receive_list(queue);
1396 if !self.lists.is_empty_of(receivers) && self.remove_from_event_list(receivers)? {
1397 self.port_yield();
1398 }
1399 } else {
1400 if kind.is_mutex() {
1401 self.queues.resolve_mut(queue)?.holder = caller;
1402 self.increment_mutexes_held(caller);
1403 }
1404 let senders = Self::queue_send_list(queue);
1405 if !self.lists.is_empty_of(senders) && self.remove_from_event_list(senders)? {
1406 self.port_yield();
1407 }
1408 }
1409 self.exit_critical();
1410 self.end_wait(caller);
1411 return Ok(Ready(value));
1412 }
1413 let remaining = self.begin_wait(caller, queue, ticks);
1414 if remaining.unwrap_or(0) == 0 {
1415 self.exit_critical();
1416 // `traceQUEUE_peek_FAILED` is not one of the harness's hooks,
1417 // so a failed peek says nothing on either side.
1418 if !peek {
1419 self.trace_failure_or_owe(caller, OwedTrace::ReceiveFailed(queue));
1420 }
1421 if remaining.is_some() {
1422 self.end_wait(caller);
1423 }
1424 return Err(Error::Empty);
1425 }
1426 self.exit_critical();
1427 // That exit can release a tick, and a tick can switch this task
1428 // away. The C's thread stops INSIDE the exit, so everything below
1429 // runs when the task is scheduled again -- and names that task.
1430 if self.cur() != caller {
1431 self.set_queue_resume(caller, QueueResume::BelowSample);
1432 return Ok(Blocked);
1433 }
1434 self.queue_take_blocking(caller, queue, peek)
1435 }
1436
1437 /// Everything `xQueueReceive` and `xQueuePeek` do below their sampling
1438 /// exit. See [`Kernel::take_queue_resume`].
1439 #[cold]
1440 fn queue_take_blocking(
1441 &mut self,
1442 caller: TaskHandle,
1443 queue: QueueHandle,
1444 peek: bool,
1445 ) -> Result<Wait<u64>> {
1446 // `kind` is read HERE rather than handed down from the caller, because
1447 // it is the one field of the descriptor that cannot change after
1448 // `new_queue` sets it -- so resolving it late reads the same byte, and
1449 // resolving it early only lengthened a live range across the hot
1450 // function's whole body.
1451 //
1452 // SINKING it below the timed-out branch -- which never reads it, and
1453 // whose callee resolves `kind` for itself -- removes a resolve from
1454 // that path and measured **+2 B** on 2026-09-25. Refuted: the resolve
1455 // is still emitted once at the later site and the branch around it grew
1456 // by more. Both paths are `#[cold]`, so there was no runtime case to
1457 // weigh against the bytes.
1458 let kind = self.queues.resolve(queue)?.kind;
1459 self.suspend_all();
1460 self.lock_queue(queue);
1461 let left = self.check_for_timeout(caller);
1462 if left.is_none() {
1463 self.unlock_queue(queue)?;
1464 let _ = self.resume_all();
1465 // `xTaskResumeAll` replays any tick that pended while the
1466 // scheduler was suspended, and replaying it can switch this task
1467 // away with `prvIsQueueEmpty` still to run. The C's thread stops
1468 // inside it; everything below runs when the task is next
1469 // scheduled. Without this, `prvIsQueueEmpty`'s section is spent
1470 // in the wrong window -- one exit, and IntQueue's 100,000-tick
1471 // run diverges at tick 16,260.
1472 if self.cur() != caller {
1473 self.set_queue_resume(caller, QueueResume::BelowTimedOutResume);
1474 return Ok(Blocked);
1475 }
1476 return self.queue_take_timed_out(queue, peek);
1477 }
1478 // `left` is `Some` here, and it carries the block time
1479 // `check_for_timeout` has just written into the TCB.
1480 self.queue_take_locked(
1481 caller,
1482 queue,
1483 peek,
1484 kind,
1485 left.map_or(0, core::num::NonZeroU64::get),
1486 )
1487 }
1488
1489 /// `xQueueReceive` below the `xTaskResumeAll` of its timed-out branch.
1490 /// `caller` is read here rather than passed, for the reason given on
1491 /// [`Kernel::queue_take_blocking`]: both call sites have already
1492 /// established that it is `self.cur()`.
1493 #[cold]
1494 fn queue_take_timed_out(&mut self, queue: QueueHandle, peek: bool) -> Result<Wait<u64>> {
1495 let caller = self.cur();
1496 // `kind` is read HERE rather than handed down from the caller,
1497 // because it is the one field of the descriptor that cannot change
1498 // after `new_queue` sets it -- so resolving it late reads the same
1499 // byte, and resolving it early only lengthened a live range across
1500 // the hot function's whole body.
1501 let kind = self.queues.resolve(queue)?.kind;
1502 {
1503 if self.is_queue_empty(queue) {
1504 if kind.is_mutex() && self.wait_inherited(caller) {
1505 // `vTaskPriorityDisinheritAfterTimeout`: the holder
1506 // keeps only what the still-waiting tasks justify.
1507 self.enter_critical();
1508 let highest = self.highest_waiting_priority(queue)?;
1509 let holder = self.queues.resolve(queue).map(|q| q.holder)?;
1510 self.priority_disinherit_after_timeout(holder, highest)?;
1511 self.exit_critical();
1512 }
1513 if !peek {
1514 self.trace_failure_or_owe(caller, OwedTrace::ReceiveFailed(queue));
1515 }
1516 self.end_wait(caller);
1517 return Err(Error::Empty);
1518 }
1519 Ok(Blocked)
1520 }
1521 }
1522
1523 /// `xQueueReceive` below its timeout test, with the queue still locked.
1524 fn queue_take_locked(
1525 &mut self,
1526 caller: TaskHandle,
1527 queue: QueueHandle,
1528 peek: bool,
1529 // The descriptor's `kind`, and ONLY that. All three of these
1530 // functions took the whole `Queue` by value -- thirty-six bytes on
1531 // rv32, copied at every hop of a three-deep chain -- to read
1532 // `kind.is_mutex()` twice. Nothing else of it was ever touched.
1533 kind: Kind,
1534 ticks_left: u64,
1535 ) -> Result<Wait<u64>> {
1536 // Both arms end the same way -- unlock, then resume -- and
1537 // `resume_all_inline` is exactly that: INLINE. Written as two arms it
1538 // was two copies of its body in one function. The sequence below is
1539 // the sequence both arms had, in the same order, because on the sim a
1540 // critical-section exit is the clock and reordering one is a
1541 // divergence.
1542 let was_empty = self.is_queue_empty(queue);
1543 if was_empty {
1544 let event = if peek {
1545 Event::BlockingOnQueuePeek { queue, name: "" }
1546 } else {
1547 Event::BlockingOnQueueReceive { queue, name: "" }
1548 };
1549 self.note_exits();
1550 let tick = self.tick;
1551 self.trace.event(tick, event);
1552 if kind.is_mutex() {
1553 self.enter_critical();
1554 let holder = self.queues.resolve(queue).map(|q| q.holder)?;
1555 let inherited = self.priority_inherit(holder)?;
1556 self.exit_critical();
1557 if inherited {
1558 self.set_wait_inherited(caller);
1559 }
1560 }
1561 self.place_on_event_list(Self::queue_receive_list(queue), ticks_left)?;
1562 }
1563 self.unlock_queue(queue)?;
1564 let yielded = self.resume_all_inline();
1565 if was_empty && !yielded {
1566 self.yield_or_owe(caller);
1567 }
1568 Ok(Blocked)
1569 }
1570
1571 /// `prvCopyDataFromQueue`, with the peek variant putting the cursor
1572 /// back as `xQueuePeek` does -- over fields its caller already read.
1573 ///
1574 /// Every caller resolves this queue to decide whether there is
1575 /// anything to read; this used to resolve it again to do the reading.
1576 /// The snapshot cannot be stale — nothing between the two touches this
1577 /// queue, and both are inside the same critical section.
1578 ///
1579 /// The same change to [`Kernel::copy_data_to_queue`] is **refused**,
1580 /// and the pair is why this bench exists: threading the snapshot into
1581 /// the writer is worth -245,389 Ir inside it and +211,717 in
1582 /// `queue_send_generic`, because a `&Queue` argument forces the
1583 /// caller's local to be addressable and it stops living in registers.
1584 /// Here the caller is `queue_take`, which keeps its snapshot alive
1585 /// across the call anyway, so there is no spill to pay for.
1586 fn copy_data_from_queue(
1587 &mut self,
1588 queue: QueueHandle,
1589 kind: Kind,
1590 base: usize,
1591 length: usize,
1592 read_from: usize,
1593 peek: bool,
1594 ) -> Result<u64> {
1595 if !kind.carries_data() {
1596 if !peek {
1597 let q = self.queues.resolve_mut(queue)?;
1598 // Wrapping: all three callers reach here under
1599 // `waiting > 0`, and the doc above says the snapshot they
1600 // tested cannot be stale, so there is an item to take.
1601 q.waiting = q.waiting.wrapping_sub(1);
1602 }
1603 return Ok(0);
1604 }
1605 let next = wrap_next(read_from, length);
1606 let value = self
1607 .slots
1608 .get(base.wrapping_add(next))
1609 .copied()
1610 .unwrap_or(0);
1611 // `xQueuePeek` saves and restores `pcReadFrom` — which is to
1612 // say it leaves it exactly as it found it. Writing it back stored
1613 // the value already there, and the resolve that reached it was a
1614 // whole handle validation for a no-op.
1615 if !peek {
1616 let q = self.queues.resolve_mut(queue)?;
1617 q.read_from = next;
1618 // Wrapping, for the same reason as the branch above.
1619 q.waiting = q.waiting.wrapping_sub(1);
1620 }
1621 Ok(value)
1622 }
1623
1624 // ------------------------------------------------------ recursive mutex --
1625
1626 /// `xSemaphoreTakeRecursive`.
1627 ///
1628 /// # Errors
1629 /// [`Error::Empty`] on timeout; [`Error::Gone`] for a stale handle.
1630 pub fn mutex_take_recursive(&mut self, mutex: QueueHandle, ticks: u64) -> Result<Wait<()>> {
1631 let caller = self.cur();
1632 // One resolve reads the holder and counts the re-entry.
1633 // `resolve_mut` runs the same four checks and answers the same
1634 // errors, so asking twice only asked twice.
1635 {
1636 let q = self.queues.resolve_mut(mutex)?;
1637 if q.holder == caller {
1638 q.recursions = q.recursions.wrapping_add(1);
1639 return Ok(Ready(()));
1640 }
1641 }
1642 match self.semaphore_take(mutex, ticks)? {
1643 Blocked => Ok(Blocked),
1644 Ready(()) => {
1645 let q = self.queues.resolve_mut(mutex)?;
1646 q.recursions = q.recursions.wrapping_add(1);
1647 Ok(Ready(()))
1648 }
1649 }
1650 }
1651
1652 /// `xSemaphoreGiveRecursive`.
1653 ///
1654 /// # Errors
1655 /// [`Error::NotActive`] when the caller does not hold the mutex.
1656 pub fn mutex_give_recursive(&mut self, mutex: QueueHandle) -> Result<()> {
1657 let caller = self.cur();
1658 // One resolve, as in the take above.
1659 let remaining = {
1660 let q = self.queues.resolve_mut(mutex)?;
1661 if q.holder != caller {
1662 return Err(Error::NotActive);
1663 }
1664 q.recursions = q.recursions.wrapping_sub(1);
1665 q.recursions
1666 };
1667 if remaining == 0 {
1668 let _ = self.semaphore_give(mutex)?;
1669 }
1670 Ok(())
1671 }
1672
1673 // -------------------------------------------------------- queue locks --
1674
1675 /// `prvLockQueue`: one critical section. The lock counters themselves
1676 /// only matter to an ISR, and the sim has none — but the section is
1677 /// where sim time passes, so it is here.
1678 /// `prvLockQueue`: stop interrupts taking tasks off this queue's event
1679 /// lists while a task walks them. An interrupt that finds the queue
1680 /// locked counts what it did instead, and [`Kernel::unlock_queue`] pays
1681 /// it back.
1682 // In line at both callers (`rusty-compiler-leverage` A3): win 23, which
1683 // took this and `is_queue_empty`/`lock_queue`'s siblings together for
1684 // -48 B of flash. A two-caller body this small pays two frames for a
1685 // resolve and a critical section; `#[inline]` alone was declined.
1686 #[inline(always)]
1687 fn lock_queue(&mut self, queue: QueueHandle) {
1688 self.enter_critical();
1689 if let Ok(q) = self.queues.resolve_mut(queue) {
1690 if q.rx_lock == UNLOCKED {
1691 q.rx_lock = LOCKED_UNMODIFIED;
1692 }
1693 if q.tx_lock == UNLOCKED {
1694 q.tx_lock = LOCKED_UNMODIFIED;
1695 }
1696 }
1697 self.exit_critical();
1698 }
1699
1700 /// `prvUnlockQueue`: two critical sections, one per lock counter. The
1701 /// loops inside them wake tasks that an ISR queued while the lock was
1702 /// held, and on the sim there are none.
1703 /// `prvUnlockQueue`: two critical sections, one per lock count, each
1704 /// waking one task per send or receive an interrupt made while the
1705 /// queue was locked. The yields are *missed* yields — pended, not
1706 /// taken — because a task is walking the list.
1707 fn unlock_queue(&mut self, queue: QueueHandle) -> Result<()> {
1708 self.enter_critical();
1709 {
1710 // One resolve for both fields. They were two, and a resolve
1711 // is a null test, a bounds check, a generation compare and an
1712 // `Option` unwrap -- all of it repeated to reach the same slot.
1713 // The unlock is written on the SAME lookup that read the lock,
1714 // whenever the loop below is not going to run -- which is every
1715 // call where nothing was sent while the queue was held, i.e. the
1716 // common one. It used to take a second `resolve_mut` after the
1717 // loop to store a value already known here.
1718 let (mut tx_lock, container) = match self.queues.resolve_mut(queue) {
1719 Ok(q) => {
1720 // The container is read only when queue sets exist. With
1721 // `USE_QUEUE_SETS` false nothing ever writes `set_container`,
1722 // so the const makes it a known NULL and the whole branch
1723 // below folds -- the other two queue-set sites already put
1724 // the const FIRST and this one tested the field first.
1725 let read = (
1726 q.tx_lock,
1727 if C::USE_QUEUE_SETS {
1728 q.set_container
1729 } else {
1730 QueueHandle::NULL
1731 },
1732 );
1733 if read.0 <= LOCKED_UNMODIFIED {
1734 q.tx_lock = UNLOCKED;
1735 }
1736 read
1737 }
1738 Err(_) => (UNLOCKED, QueueHandle::NULL),
1739 };
1740 let tx_looped = tx_lock > LOCKED_UNMODIFIED;
1741 while tx_lock > LOCKED_UNMODIFIED {
1742 if C::USE_QUEUE_SETS && !container.is_null() {
1743 if C::USE_QUEUE_SETS && self.notify_queue_set_container(queue)? {
1744 self.missed_yield();
1745 }
1746 // Wrapping: the loop runs only while this is above
1747 // `LOCKED_UNMODIFIED`, so one off it stays in range.
1748 tx_lock = tx_lock.wrapping_sub(1);
1749 continue;
1750 }
1751 let receivers = Self::queue_receive_list(queue);
1752 if self.lists.is_empty_of(receivers) {
1753 break;
1754 }
1755 if self.remove_from_event_list(receivers)? {
1756 self.missed_yield();
1757 }
1758 // Wrapping: bounded by the loop, as above.
1759 tx_lock = tx_lock.wrapping_sub(1);
1760 }
1761 if tx_looped {
1762 if let Ok(q) = self.queues.resolve_mut(queue) {
1763 q.tx_lock = UNLOCKED;
1764 }
1765 }
1766 }
1767 self.exit_critical();
1768 self.enter_critical();
1769 {
1770 // Same trade as the tx half above.
1771 let mut rx_lock = match self.queues.resolve_mut(queue) {
1772 Ok(q) => {
1773 let read = q.rx_lock;
1774 if read <= LOCKED_UNMODIFIED {
1775 q.rx_lock = UNLOCKED;
1776 }
1777 read
1778 }
1779 Err(_) => UNLOCKED,
1780 };
1781 let rx_looped = rx_lock > LOCKED_UNMODIFIED;
1782 while rx_lock > LOCKED_UNMODIFIED {
1783 let senders = Self::queue_send_list(queue);
1784 if self.lists.is_empty_of(senders) {
1785 break;
1786 }
1787 if self.remove_from_event_list(senders)? {
1788 self.missed_yield();
1789 }
1790 // Wrapping: bounded by the loop, as above.
1791 rx_lock = rx_lock.wrapping_sub(1);
1792 }
1793 if rx_looped {
1794 if let Ok(q) = self.queues.resolve_mut(queue) {
1795 q.rx_lock = UNLOCKED;
1796 }
1797 }
1798 }
1799 self.exit_critical();
1800 Ok(())
1801 }
1802
1803 /// `vQueueWaitForMessageRestricted`: what the timer service task does
1804 /// when it has no timer to run — block on the command queue without
1805 /// suspending the scheduler.
1806 ///
1807 /// `wait_indefinitely` is the C third argument: when set the wait
1808 /// becomes `portMAX_DELAY` and the task goes to the suspended list
1809 /// rather than a delayed one. The queue locks around it are C's
1810 /// `prvLockQueue` and `prvUnlockQueue` — three critical sections, and
1811 /// therefore three sixteenths of a tick's worth of sim time.
1812 ///
1813 /// # Errors
1814 /// A list error surfaces as itself.
1815 pub fn wait_for_message_restricted(
1816 &mut self,
1817 queue: QueueHandle,
1818 ticks: u64,
1819 wait_indefinitely: bool,
1820 ) -> Result<()> {
1821 self.lock_queue(queue);
1822 let empty = self
1823 .queues
1824 .resolve(queue)
1825 .map(|q| q.waiting == 0)
1826 .unwrap_or(false);
1827 if empty {
1828 // vTaskPlaceOnEventListRestricted
1829 let current = self.cur();
1830 let event = Self::event_item(current);
1831 self.lists
1832 .insert_end(Self::queue_receive_list(queue), event)?;
1833 let ticks = if wait_indefinitely {
1834 Self::MAX_DELAY
1835 } else {
1836 ticks
1837 };
1838 let wake_at = self.tick.wrapping_add(ticks) & Self::MAX_DELAY;
1839 self.trace_task(current, |task, name| Event::TaskDelayUntil {
1840 task,
1841 name,
1842 wake_at,
1843 });
1844 self.add_current_task_to_delayed_list(ticks, wait_indefinitely)?;
1845 }
1846 self.unlock_queue(queue)?;
1847 Ok(())
1848 }
1849
1850 /// `prvIsQueueEmpty`, critical section and all — it is one of the
1851 /// four the blocking path spends, and therefore a sixteenth of a tick
1852 /// every four calls.
1853 // In line at both callers (`rusty-compiler-leverage` A3): win 23, which
1854 // took this and `is_queue_empty`/`lock_queue`'s siblings together for
1855 // -48 B of flash. A two-caller body this small pays two frames for a
1856 // resolve and a critical section; `#[inline]` alone was declined.
1857 #[inline(always)]
1858 fn is_queue_empty(&mut self, queue: QueueHandle) -> bool {
1859 self.enter_critical();
1860 let empty = self
1861 .queues
1862 .resolve(queue)
1863 .map(|q| q.waiting == 0)
1864 .unwrap_or(true);
1865 self.exit_critical();
1866 empty
1867 }
1868
1869 /// `prvIsQueueFull`, likewise.
1870 fn is_queue_full(&mut self, queue: QueueHandle) -> bool {
1871 self.enter_critical();
1872 let full = self
1873 .queues
1874 .resolve(queue)
1875 .map(|q| q.waiting >= q.length)
1876 .unwrap_or(false);
1877 self.exit_critical();
1878 full
1879 }
1880
1881 /// `prvGetHighestPriorityOfWaitToReceiveList`.
1882 fn highest_waiting_priority(&self, queue: QueueHandle) -> Result<u8> {
1883 let list = Self::queue_receive_list(queue);
1884 match self.lists.head(list)? {
1885 // The event item's value is `configMAX_PRIORITIES - priority`.
1886 Some(item) => {
1887 let value = self.lists.value(item)?;
1888 let max = u64::from(C::MAX_PRIORITIES);
1889 Ok(u8::try_from(max.saturating_sub(value)).unwrap_or(0))
1890 }
1891 None => Ok(0),
1892 }
1893 }
1894}
1895
1896/// The next index in a ring of `length`.
1897///
1898/// Five instructions behind a call at six sites. `#[inline]` lets the ring
1899/// bound fold into the caller, which already knows it.
1900#[inline]
1901const fn wrap_next(index: usize, length: usize) -> usize {
1902 // `wrapping_add`, and it is the SAME function, not an approximation
1903 // of it: at `usize::MAX` saturating yields `usize::MAX`, which is
1904 // `>= length`, so the caller gets 0 -- and wrapping yields 0, which is
1905 // `< length`, so the caller gets 0 as well. Every other index agrees
1906 // without argument.
1907 let next = index.wrapping_add(1);
1908 if next >= length { 0 } else { next }
1909}
1910
1911/// The previous index in a ring of `length`.
1912#[inline]
1913const fn wrap_prev(index: usize, length: usize) -> usize {
1914 match index.checked_sub(1) {
1915 Some(prev) => prev,
1916 None => length.saturating_sub(1),
1917 }
1918}
1919
1920/// The Rust face's raw surface: see [`crate::typed`].
1921///
1922/// Every method here is one of the kernel's own calls under another name.
1923/// The one that is not — [`Raw::raw_queue_has_room`] — is the kernel
1924/// reading its own queue rather than a task asking, so it takes no
1925/// critical section and costs no sim time, which is what lets the typed
1926/// face be free.
1927impl<
1928 C: Config,
1929 P: Port,
1930 T: Trace,
1931 H,
1932 const TASKS: usize,
1933 const ITEMS: usize,
1934 const LISTS: usize,
1935 const QUEUES: usize,
1936 const SLOTS: usize,
1937 const BUFFERS: usize,
1938 const BYTES: usize,
1939 const TIMERS: usize,
1940 const GROUPS: usize,
1941 const TIMER_CMDS: usize,
1942> crate::typed::Raw
1943 for Kernel<
1944 C,
1945 P,
1946 T,
1947 H,
1948 TASKS,
1949 ITEMS,
1950 LISTS,
1951 QUEUES,
1952 SLOTS,
1953 BUFFERS,
1954 BYTES,
1955 TIMERS,
1956 GROUPS,
1957 TIMER_CMDS,
1958 >
1959where
1960 H: TickHook<Self>,
1961{
1962 fn raw_queue_create(&mut self, length: usize) -> Result<QueueHandle> {
1963 self.queue_create(length)
1964 }
1965
1966 fn raw_queue_send(&mut self, queue: QueueHandle, value: u64, ticks: u64) -> Result<Wait<()>> {
1967 self.queue_send(queue, value, ticks)
1968 }
1969
1970 fn raw_queue_receive(&mut self, queue: QueueHandle, ticks: u64) -> Result<Wait<u64>> {
1971 self.queue_receive(queue, ticks)
1972 }
1973
1974 fn raw_queue_messages_waiting(&mut self, queue: QueueHandle) -> Result<usize> {
1975 self.queue_messages_waiting(queue)
1976 }
1977
1978 fn raw_queue_has_room(&self, queue: QueueHandle) -> bool {
1979 self.queues
1980 .resolve(queue)
1981 .is_ok_and(|q| q.waiting < q.length)
1982 }
1983
1984 fn raw_in_isr(&self) -> bool {
1985 self.port.in_isr()
1986 }
1987
1988 fn raw_queue_send_from_isr(&mut self, queue: QueueHandle, value: u64) -> Result<Woken> {
1989 self.queue_send_from_isr(queue, value)
1990 }
1991
1992 fn raw_mutex_create(&mut self) -> Result<QueueHandle> {
1993 self.mutex_create()
1994 }
1995
1996 fn raw_mutex_take(&mut self, mutex: QueueHandle, ticks: u64) -> Result<Wait<()>> {
1997 self.semaphore_take(mutex, ticks)
1998 }
1999
2000 fn raw_mutex_give(&mut self, mutex: QueueHandle) -> Result<()> {
2001 self.semaphore_give(mutex).map(|_| ())
2002 }
2003}