rusty_rtos_kernel_core/stream.rs
1//! `stream_buffer.c`: a byte ring one task writes and another reads.
2//!
3//! A stream buffer is the only kernel object here that is not built on a
4//! queue and not built on an event list. It is a ring of bytes plus two
5//! task handles, and a task waiting on one blocks on a **task
6//! notification** rather than on an event list — which is why the
7//! notification API had to exist before this file could.
8//!
9//! Two shapes share the code:
10//!
11//! * a **stream buffer** passes bytes, and a reader wakes when at least
12//! the trigger level of them have arrived;
13//! * a **message buffer** passes whole messages, by writing the length in
14//! front of each one. A read returns one message or nothing.
15//!
16//! The length prefix is `sizeof( configMESSAGE_BUFFER_LENGTH_TYPE )` wide,
17//! which is `size_t` unless a configuration says otherwise — eight bytes on
18//! the machine the oracle runs on. That width is in the arithmetic of every
19//! message send, so it is a [`Config`] const here rather than a guess.
20
21use rusty_rtos_core::config::Config;
22use rusty_rtos_core::error::{Error, Result};
23use rusty_rtos_core::handle::{StreamBufferHandle, TaskHandle};
24use rusty_rtos_core::hooks::TickHook;
25use rusty_rtos_core::isr::Woken;
26use rusty_rtos_core::port::Port;
27use rusty_rtos_core::trace::{Event, Trace};
28
29use crate::kernel::{Kernel, NotifyAction, OwedTrace};
30use crate::queue::{Blocked, Ready, Wait};
31
32/// One stream or message buffer.
33#[derive(Debug, Clone, Copy, Default)]
34pub(crate) struct StreamBuffer {
35 /// Where this buffer's bytes start in the kernel's byte arena.
36 pub(crate) base: usize,
37 /// `xLength`: the ring is one byte longer than the buffer asked for,
38 /// because a ring that is full and a ring that is empty would otherwise
39 /// look the same.
40 pub(crate) length: usize,
41 /// `xTriggerLevelBytes`.
42 pub(crate) trigger: usize,
43 /// `xHead`: where the next byte is written.
44 pub(crate) head: usize,
45 /// `xTail`: where the next byte is read.
46 pub(crate) tail: usize,
47 /// `xTaskWaitingToReceive`.
48 pub(crate) waiting_to_receive: TaskHandle,
49 /// `xTaskWaitingToSend`.
50 pub(crate) waiting_to_send: TaskHandle,
51 /// `sbFLAGS_IS_MESSAGE_BUFFER`.
52 pub(crate) is_message: bool,
53 /// `uxNotificationIndex`.
54 pub(crate) notify_index: usize,
55}
56
57impl StreamBuffer {
58 /// `prvBytesInBuffer`.
59 ///
60 /// The C adds the length before subtracting the tail so the unsigned
61 /// arithmetic cannot go below zero, then folds the extra length back
62 /// out. These are the C's own operations, and they need no checks for
63 /// the same reason the C's do not.
64 ///
65 /// `head` and `tail` are kept below `length` by the wrap arithmetic that
66 /// writes them, and `length` is at least 1. So `length + head` is at
67 /// most `2 * length - 1` and cannot overflow; `length + head - tail` is
68 /// at least `head + 1` and cannot underflow; and the fold only runs when
69 /// `count >= length`, where `count` is at most `2 * length - 1`, so it
70 /// lands back inside `0..length`.
71 #[inline]
72 pub(crate) const fn bytes_in_buffer(&self) -> usize {
73 let mut count = self.length.wrapping_add(self.head).wrapping_sub(self.tail);
74 if count >= self.length {
75 count = count.wrapping_sub(self.length);
76 }
77 count
78 }
79
80 /// `xStreamBufferSpacesAvailable`: one less than the gap, because the
81 /// ring keeps a spare byte so full and empty do not look alike.
82 ///
83 /// Bounded as [`StreamBuffer::bytes_in_buffer`] is, with one more step:
84 /// `length + tail - head` is at least 1, because `head` is at most
85 /// `length - 1`, so taking the spare byte off it cannot underflow.
86 #[inline]
87 pub(crate) const fn spaces_available(&self) -> usize {
88 let mut space = self
89 .length
90 .wrapping_add(self.tail)
91 .wrapping_sub(self.head)
92 .wrapping_sub(1);
93 if space >= self.length {
94 space = space.wrapping_sub(self.length);
95 }
96 space
97 }
98
99 /// `prvBytesInBufferMeetTriggerLevel`, for the two shapes that are not
100 /// a batching buffer.
101 pub(crate) const fn meets_trigger(&self, bytes: usize) -> bool {
102 bytes >= self.trigger
103 }
104}
105
106impl<
107 C: Config,
108 P: Port,
109 T: Trace,
110 H,
111 const TASKS: usize,
112 const ITEMS: usize,
113 const LISTS: usize,
114 const QUEUES: usize,
115 const SLOTS: usize,
116 const BUFFERS: usize,
117 const BYTES: usize,
118 const TIMERS: usize,
119 const GROUPS: usize,
120 const TIMER_CMDS: usize,
121> Kernel<C, P, T, H, TASKS, ITEMS, LISTS, QUEUES, SLOTS, BUFFERS, BYTES, TIMERS, GROUPS, TIMER_CMDS>
122where
123 H: TickHook<Self>,
124{
125 /// How many bytes a message buffer spends on each message's length.
126 pub const MESSAGE_LENGTH_BYTES: usize = C::MESSAGE_LENGTH_BYTES;
127
128 /// `xStreamBufferCreate`.
129 ///
130 /// A trigger level of zero becomes one, as the C does: a reader that
131 /// woke on nothing would spin.
132 ///
133 /// # Errors
134 /// [`Error::Full`] when the buffer arena or the byte arena is full;
135 /// [`Error::InvalidArgument`] for a zero-length buffer.
136 pub fn stream_buffer_create(
137 &mut self,
138 size: usize,
139 trigger: usize,
140 ) -> Result<StreamBufferHandle> {
141 self.new_stream_buffer(size, trigger, false)
142 }
143
144 /// `xMessageBufferCreate`.
145 ///
146 /// # Errors
147 /// As [`Kernel::stream_buffer_create`]; a message buffer must also be
148 /// longer than one length prefix.
149 pub fn message_buffer_create(&mut self, size: usize) -> Result<StreamBufferHandle> {
150 if size <= Self::MESSAGE_LENGTH_BYTES {
151 return Err(Error::InvalidArgument);
152 }
153 self.new_stream_buffer(size, 1, true)
154 }
155
156 fn new_stream_buffer(
157 &mut self,
158 size: usize,
159 trigger: usize,
160 is_message: bool,
161 ) -> Result<StreamBufferHandle> {
162 let caller = self.current;
163 if size == 0 || trigger > size {
164 return Err(Error::InvalidArgument);
165 }
166 let trigger = if trigger == 0 { 1 } else { trigger };
167 // `xBufferSizeBytes++` before the allocation: the spare byte.
168 let length = size.saturating_add(1);
169 let base = self.take_bytes(length).ok_or(Error::Full)?;
170 let handle = self
171 .buffers
172 .try_insert(StreamBuffer {
173 base,
174 length,
175 trigger,
176 head: 0,
177 tail: 0,
178 waiting_to_receive: TaskHandle::NULL,
179 waiting_to_send: TaskHandle::NULL,
180 is_message,
181 notify_index: 0,
182 })
183 .map_err(|_| Error::Full)?;
184 self.account_for_allocation();
185 // The malloc's `xTaskResumeAll` can release a tick and switch us
186 // away, and the C's thread then stops there with
187 // `traceSTREAM_BUFFER_CREATE` still ahead of it -- so the line
188 // belongs to this task's NEXT run, not to whoever took the CPU.
189 self.trace_failure_or_owe(
190 caller,
191 OwedTrace::StreamBufferCreate {
192 buffer: handle,
193 is_message_buffer: is_message,
194 },
195 );
196 Ok(handle)
197 }
198
199 /// Take `length` bytes of the arena: first fit from what deleted
200 /// buffers gave back, and only then from the untouched end.
201 ///
202 /// First fit rather than best fit because the pattern that needs an
203 /// allocator at all is create-then-delete of the *same size*, which
204 /// first fit serves exactly and without fragmenting.
205 fn take_bytes(&mut self, length: usize) -> Option<usize> {
206 for i in 0..self.free_count {
207 let (base, free) = *self.free_blocks.get(i)?;
208 if free < length {
209 continue;
210 }
211 if free == length {
212 self.drop_free_block(i);
213 } else if let Some(slot) = self.free_blocks.get_mut(i) {
214 *slot = (base.saturating_add(length), free.wrapping_sub(length));
215 }
216 return Some(base);
217 }
218 let end = self.bytes_used.checked_add(length)?;
219 if end > BYTES {
220 return None;
221 }
222 let base = self.bytes_used;
223 self.bytes_used = end;
224 Some(base)
225 }
226
227 /// Give a block back, coalescing with whichever neighbours touch it so
228 /// the list cannot grow past one entry per hole.
229 fn give_bytes(&mut self, base: usize, length: usize) {
230 let mut base = base;
231 let mut length = length;
232 let mut i = 0;
233 while i < self.free_count {
234 let Some(&(other_base, other_len)) = self.free_blocks.get(i) else {
235 break;
236 };
237 if other_base.saturating_add(other_len) == base {
238 base = other_base;
239 length = length.saturating_add(other_len);
240 self.drop_free_block(i);
241 continue;
242 }
243 if base.saturating_add(length) == other_base {
244 length = length.saturating_add(other_len);
245 self.drop_free_block(i);
246 continue;
247 }
248 i = i.wrapping_add(1);
249 }
250 // A block at the very end goes back to the bump pointer instead of
251 // the list, which is what keeps a create/delete loop free.
252 if base.saturating_add(length) == self.bytes_used {
253 self.bytes_used = base;
254 return;
255 }
256 if let Some(slot) = self.free_blocks.get_mut(self.free_count) {
257 *slot = (base, length);
258 self.free_count = self.free_count.wrapping_add(1);
259 }
260 }
261
262 fn drop_free_block(&mut self, index: usize) {
263 let last = self.free_count.saturating_sub(1);
264 if index < last {
265 if let Some(&moved) = self.free_blocks.get(last) {
266 if let Some(slot) = self.free_blocks.get_mut(index) {
267 *slot = moved;
268 }
269 }
270 }
271 self.free_count = last;
272 }
273
274 /// `vStreamBufferDelete`: the handle goes stale and the ring's bytes go
275 /// back to the arena.
276 ///
277 /// # Errors
278 /// [`Error::Gone`] for a stale handle.
279 pub fn stream_buffer_delete(&mut self, buffer: StreamBufferHandle) -> Result<()> {
280 let b = *self.buffers.resolve(buffer)?;
281 let _ = self.buffers.discard(buffer);
282 self.give_bytes(b.base, b.length);
283 // `vStreamBufferDelete` ends in `vPortFree` (stream_buffer.c:582),
284 // and `heap_4` brackets that with `vTaskSuspendAll` /
285 // `xTaskResumeAll` -- which takes a critical section, and on the sim
286 // a critical-section exit is the clock. The create paid for its
287 // `pvPortMalloc` and the delete did not, so every deleted buffer
288 // lost one exit. `StreamBufferDemo`'s trigger-level test deletes one
289 // per pass and drifted by exactly 1 at its second `vTaskDelay`.
290 //
291 // The timer service's `Delete` arm had the identical defect.
292 self.account_for_allocation();
293 Ok(())
294 }
295
296 /// `xStreamBufferBytesAvailable`. No critical section, as in the C.
297 ///
298 /// # Errors
299 /// [`Error::Gone`] for a stale handle.
300 pub fn stream_buffer_bytes_available(&self, buffer: StreamBufferHandle) -> Result<usize> {
301 Ok(self.buffers.resolve(buffer)?.bytes_in_buffer())
302 }
303
304 /// `xStreamBufferSpacesAvailable`. No critical section, as in the C.
305 ///
306 /// # Errors
307 /// [`Error::Gone`] for a stale handle.
308 pub fn stream_buffer_spaces_available(&self, buffer: StreamBufferHandle) -> Result<usize> {
309 Ok(self.buffers.resolve(buffer)?.spaces_available())
310 }
311
312 /// `xStreamBufferIsEmpty`: head and tail in the same place.
313 ///
314 /// # Errors
315 /// [`Error::Gone`] for a stale handle.
316 pub fn stream_buffer_is_empty(&self, buffer: StreamBufferHandle) -> Result<bool> {
317 let b = self.buffers.resolve(buffer)?;
318 Ok(b.head == b.tail)
319 }
320
321 /// `xStreamBufferIsFull`: no room for another message.
322 ///
323 /// For a message buffer "another message" means at least one more byte
324 /// than its length prefix, so a buffer with exactly a prefix's worth of
325 /// room left is already full. A stream buffer compares against zero.
326 ///
327 /// # Errors
328 /// [`Error::Gone`] for a stale handle.
329 pub fn stream_buffer_is_full(&self, buffer: StreamBufferHandle) -> Result<bool> {
330 let b = self.buffers.resolve(buffer)?;
331 let prefix = if b.is_message {
332 Self::MESSAGE_LENGTH_BYTES
333 } else {
334 0
335 };
336 Ok(b.spaces_available() <= prefix)
337 }
338
339 /// `xStreamBufferSetTriggerLevel`.
340 ///
341 /// # Errors
342 /// [`Error::Gone`] for a stale handle.
343 pub fn stream_buffer_set_trigger_level(
344 &mut self,
345 buffer: StreamBufferHandle,
346 trigger: usize,
347 ) -> Result<bool> {
348 let length = self.buffers.resolve(buffer)?.length;
349 // Coerce first, then bounds-check, which is the order the C uses:
350 //
351 // if( xTriggerLevel == ( size_t ) 0 ) { xTriggerLevel = 1; }
352 // if( xTriggerLevel < pxStreamBuffer->xLength ) { ...pdPASS }
353 // else { xReturn = pdFALSE; }
354 //
355 // The two orders disagree on exactly one input: a zero trigger
356 // against a length of ONE. Coercing first makes it 1, finds `1 < 1`
357 // false, and refuses without storing; bounds-checking first lets 0
358 // through and stores the coerced 1.
359 //
360 // THAT INPUT IS UNREACHABLE HERE, and the ordering is matched anyway
361 // rather than left to depend on it. `length` is `size + 1` -- the
362 // spare byte a ring buffer needs to tell full from empty, the C's
363 // own `xBufferSizeBytes++` -- and `size == 0` is refused at
364 // creation, so no buffer has a length of 1 and the orders agree on
365 // every input a caller can construct.
366 //
367 // Written down because the first version of this comment claimed a
368 // live divergence. It is not one; it is an inconsistency with the C
369 // that one arithmetic detail in a different function is keeping
370 // harmless.
371 let trigger = if trigger == 0 { 1 } else { trigger };
372 if trigger >= length {
373 return Ok(false);
374 }
375 self.buffers.resolve_mut(buffer)?.trigger = trigger;
376 Ok(true)
377 }
378
379 /// `xStreamBufferReset`: empty it, unless a task is waiting on it.
380 ///
381 /// # Errors
382 /// [`Error::Gone`] for a stale handle.
383 pub fn stream_buffer_reset(&mut self, buffer: StreamBufferHandle) -> Result<bool> {
384 self.enter_critical();
385 let done = match self.buffers.resolve_mut(buffer) {
386 Ok(b) if b.waiting_to_receive.is_null() && b.waiting_to_send.is_null() => {
387 b.head = 0;
388 b.tail = 0;
389 true
390 }
391 _ => false,
392 };
393 self.exit_critical();
394 Ok(done)
395 }
396
397 /// `xStreamBufferNextMessageLengthBytes`.
398 ///
399 /// # Errors
400 /// [`Error::Gone`] for a stale handle.
401 pub fn stream_buffer_next_message_length(&self, buffer: StreamBufferHandle) -> Result<usize> {
402 let b = *self.buffers.resolve(buffer)?;
403 if !b.is_message {
404 return Ok(0);
405 }
406 let available = b.bytes_in_buffer();
407 if available <= Self::MESSAGE_LENGTH_BYTES {
408 return Ok(0);
409 }
410 Ok(Self::read_length_prefix_from(&self.bytes, b.base, b.length, b.tail).0)
411 }
412
413 // ---------------------------------------------------------- sending --
414
415 /// `xTaskCheckForTimeOut`, the tail of `xStreamBufferSend`'s do-while.
416 ///
417 /// The C wraps the sample-and-wait in
418 /// `do { ... } while( xTaskCheckForTimeOut( &xTimeOut, &xTicksToWait )
419 /// == pdFALSE )`, and that call takes a critical section of its own
420 /// (tasks.c:5703) -- so on the sim it is one more exit, and one more
421 /// sixteenth of a tick.
422 ///
423 /// It runs **only when the wait actually blocked**: a send that finds
424 /// the space it needs on the first pass leaves the loop with a `break`
425 /// from inside the sampling section and never reaches the condition.
426 /// `xStreamBufferReceive` has no loop and no such call, which is why
427 /// this is on the send alone.
428 fn check_for_time_out(&mut self) {
429 self.enter_critical();
430 self.exit_critical();
431 }
432
433 /// `xStreamBufferSend`.
434 ///
435 /// Returns how many bytes went in. `Blocked` means the caller must
436 /// make the same call again: the C loops here waiting for space, and a
437 /// kernel with no stacks returns between passes.
438 ///
439 /// # Errors
440 /// [`Error::Gone`] for a stale handle.
441 /// Sim contract v2: what a kernel call costs when it took no critical
442 /// section.
443 ///
444 /// v1 had two tick sources, the idle hook and every 16th outermost
445 /// critical-section exit. Both are kernel-visible points, and between
446 /// them they left one hole: a task that is always ready and whose
447 /// no-progress path takes no critical section takes NO TIME, never
448 /// yields, and stops the clock for every other task.
449 ///
450 /// `xStreamBufferSend` and `xStreamBufferReceive` are the only calls in
451 /// this corpus that can return that way -- the zero-wait path reads the
452 /// ring and leaves. Every other object closes the hole by accident:
453 /// `uxQueueMessagesWaiting`, `eTaskGetState` and `uxTaskPriorityGet`
454 /// all take a section. `StreamBufferDemo`'s `prvNonBlockingReceiverTask`
455 /// is the task that found it, and under v1 the whole run froze at
456 /// `ticks=1 exits=16` (`docs/HOLES.md`, H9).
457 ///
458 /// So a blind call costs exactly one empty critical section. Not a new
459 /// clock -- the SAME one, so the count, the every-16th rule, the
460 /// running-task test and the switch all reuse paths v1 already proved,
461 /// and `exits` keeps its meaning: the number of kernel-visible points.
462 /// The C port does the identical thing from
463 /// `traceRETURN_xStreamBufferSend` / `..._Receive`; see
464 /// `vPortKairosApiReturn`.
465 ///
466 /// A call that BLOCKED is not blind: the C's thread stops inside it,
467 /// other tasks run, and the counter has moved by the time the return
468 /// hook is reached. Comparing the count is what says so on both sides.
469 ///
470 /// Gated on `T::EMITS`, for the same reason [`Kernel::note_exits`] is, and
471 /// to MATCH the C rather than diverge from it. In the C this bracket is
472 /// `traceRETURN_xStreamBufferSend`, which `FreeRTOS.h` defines as **empty**
473 /// unless a config wires it. Only `oracle/harness/FreeRTOSConfig.h` does --
474 /// the sim. `bench/kernel-ram/c/FreeRTOSConfig.h`, which the rv32 flash arm
475 /// compiles against, defines no `traceRETURN_*` at all, so **C pays zero
476 /// bytes for it in the arm we measure** while we were paying for it
477 /// unconditionally. Same defect class as relaxation being off on our arm
478 /// only (scorecard 14).
479 ///
480 /// Nothing observes the bracket under a silent sink: the exits counter it
481 /// samples is read by no one, and the critical section it takes is a closed
482 /// window that restores the mask it found. Under an emitting sink -- which
483 /// is every `conform` scenario -- the gate is true and the behaviour is
484 /// exactly as before, which is why 26/26 is unchanged.
485 fn blind_call(&mut self, exits_at_entry: u64) {
486 if !T::EMITS {
487 return;
488 }
489 if self.port.exits() != exits_at_entry {
490 return;
491 }
492 self.enter_critical();
493 self.exit_critical();
494 }
495
496 /// `xStreamBufferSend`, with the contract-v2 bracket around it.
497 ///
498 /// # Errors
499 /// As [`Kernel::stream_buffer_send_inner`].
500 pub fn stream_buffer_send(
501 &mut self,
502 buffer: StreamBufferHandle,
503 data: &[u8],
504 ticks: u64,
505 ) -> Result<Wait<usize>> {
506 let entry = if T::EMITS { self.port.exits() } else { 0 };
507 let out = self.stream_buffer_send_inner(buffer, data, ticks);
508 self.blind_call(entry);
509 out
510 }
511
512 fn stream_buffer_send_inner(
513 &mut self,
514 buffer: StreamBufferHandle,
515 data: &[u8],
516 ticks: u64,
517 ) -> Result<Wait<usize>> {
518 let caller = self.current;
519 // ONLY the fields this body reads. It was `*self.buffers.resolve(..)`
520 // -- the whole nine-field descriptor, about thirty-two bytes on rv32 --
521 // to reach three of them. `notify_index` is read at four
522 // points spread across the body, each behind an `&mut self` call, so
523 // the descriptor had to stay LIVE across all of them and was spilled
524 // for it. Two scalars live in registers. Same finding as `queue_take`:
525 // the cost was the live range, not the copy.
526 let (is_message, length, notify_index) = {
527 let b = self.buffers.resolve(buffer)?;
528 (b.is_message, b.length, b.notify_index)
529 };
530 let max_reported = length.saturating_sub(1);
531 let mut required = data.len();
532 let mut ticks = ticks;
533 if is_message {
534 required = required.saturating_add(Self::MESSAGE_LENGTH_BYTES);
535 if required > max_reported {
536 // It will never fit, so do not wait for it to.
537 ticks = 0;
538 }
539 } else if required > max_reported {
540 required = max_reported;
541 }
542
543 let mut sampled = self.take_stream_resume(caller);
544 if let Some(space) = sampled {
545 // Resuming after the exit that sampled the space. The same
546 // point as in `stream_buffer_receive`: the C tests `xSpace <
547 // xRequiredSpace` below the exit, against the sample taken
548 // inside it, so a task preempted there still blocks.
549 if space < required {
550 match self.notify_wait(notify_index, 0, 0, ticks)? {
551 Blocked => return Ok(Blocked),
552 Ready(_) => {
553 self.buffers.resolve_mut(buffer)?.waiting_to_send = TaskHandle::NULL;
554 self.check_for_time_out();
555 // The sample it went to sleep on is stale now, and
556 // the C re-reads whenever that sample was zero.
557 sampled = None;
558 }
559 }
560 }
561 } else if self.notify_wait_pending(caller) {
562 match self.notify_wait(notify_index, 0, 0, ticks)? {
563 Blocked => return Ok(Blocked),
564 Ready(_) => {}
565 }
566 // Spelled out at all three resume arms rather than factored into a
567 // helper the way the receive side's `after_stream_wait` is. Factoring
568 // it measured **+28 B** on 2026-09-25, identical with and without
569 // `#[inline(never)]` -- so LLVM outlines it either way, and one shared
570 // body plus three calls costs more than three copies that each fold
571 // against their own arm.
572 self.buffers.resolve_mut(buffer)?.waiting_to_send = TaskHandle::NULL;
573 self.check_for_time_out();
574 } else if ticks > 0 {
575 // `vTaskSetTimeOutState( &xTimeOut )` (stream_buffer.c:878),
576 // which the C runs before it looks at the buffer at all.
577 //
578 // It is the PUBLIC entry point, so it takes a critical section
579 // of its own -- unlike the queue calls, which reach the same
580 // state through `vTaskInternalSetTimeOutState` from inside one.
581 // One exit per blocking send, and `StreamBufferDemo` makes the
582 // first of them 350 ticks in.
583 //
584 // ... and only once per call: a task preempted at this very
585 // exit resumes BELOW it, so re-entry must not buy it again.
586 if !self.take_stream_timed(caller) {
587 self.enter_critical();
588 self.exit_critical();
589 if self.current != caller {
590 self.set_stream_timed(caller);
591 return Ok(Blocked);
592 }
593 }
594 self.enter_critical();
595 let space = self.buffers.resolve(buffer)?.spaces_available();
596 let must_block = space < required;
597 if must_block {
598 let _ = self.notify_state_clear(None, notify_index);
599 self.buffers.resolve_mut(buffer)?.waiting_to_send = caller;
600 }
601 self.exit_critical();
602 // That exit can release a tick, and a tick can switch this task
603 // away. The C's thread stops inside the exit and everything
604 // below it runs when the task is resumed — with the sample it
605 // already took, and without paying for the section twice.
606 if self.current != caller {
607 self.set_stream_resume(caller, space);
608 return Ok(Blocked);
609 }
610 if must_block {
611 // `traceBLOCKING_ON_STREAM_BUFFER_SEND` is not one of the
612 // harness's hooks, so blocking says nothing on either side.
613 match self.notify_wait(notify_index, 0, 0, ticks)? {
614 Blocked => return Ok(Blocked),
615 Ready(_) => {
616 self.buffers.resolve_mut(buffer)?.waiting_to_send = TaskHandle::NULL;
617 self.check_for_time_out();
618 }
619 }
620 }
621 }
622
623 let space = match sampled.take() {
624 Some(space) => space,
625 None => self.buffers.resolve(buffer)?.spaces_available(),
626 };
627 let written = self.write_message(buffer, data, space, required)?;
628 if written > 0 {
629 let tick = self.tick;
630 self.note_exits();
631 self.trace.event(
632 tick,
633 Event::StreamBufferSend {
634 buffer,
635 bytes: written,
636 },
637 );
638 // One resolve, not two: nothing moves between the byte count and
639 // the trigger test, and both read the same buffer. The block ends
640 // the borrow before the completion call needs `self` mutably.
641 let trigger = {
642 let b = self.buffers.resolve(buffer)?;
643 b.meets_trigger(b.bytes_in_buffer())
644 };
645 if trigger {
646 self.send_completed(buffer)?;
647 }
648 }
649 Ok(Ready(written))
650 }
651
652 /// `xStreamBufferSendFromISR`.
653 ///
654 /// # Errors
655 /// [`Error::Gone`] for a stale handle.
656 pub fn stream_buffer_send_from_isr(
657 &mut self,
658 buffer: StreamBufferHandle,
659 data: &[u8],
660 ) -> Result<(usize, Woken)> {
661 let snapshot = *self.buffers.resolve(buffer)?;
662 let max_reported = snapshot.length.saturating_sub(1);
663 let mut required = data.len();
664 if snapshot.is_message {
665 required = required.saturating_add(Self::MESSAGE_LENGTH_BYTES);
666 } else if required > max_reported {
667 required = max_reported;
668 }
669 let space = snapshot.spaces_available();
670 let written = self.write_message(buffer, data, space, required)?;
671 let mut woken = Woken::NO;
672 if written > 0 {
673 // `traceSTREAM_BUFFER_SEND_FROM_ISR` is a different macro from
674 // `traceSTREAM_BUFFER_SEND`, and the harness hooks only the
675 // latter — so an interrupt's send says nothing on either side.
676 // One resolve, not two: nothing moves between the byte count and
677 // the trigger test, and both read the same buffer. The block ends
678 // the borrow before the completion call needs `self` mutably.
679 let trigger = {
680 let b = self.buffers.resolve(buffer)?;
681 b.meets_trigger(b.bytes_in_buffer())
682 };
683 if trigger {
684 woken = self.send_completed_from_isr(buffer)?;
685 }
686 }
687 Ok((written, woken))
688 }
689
690 // -------------------------------------------------------- receiving --
691
692 /// What `xStreamBufferReceive` runs once its notify wait returns.
693 ///
694 /// `xTaskNotifyWaitIndexed`'s own trailing critical section can release
695 /// a tick and switch the caller away, and the C's thread then stops
696 /// THERE -- with `prvReadMessageFromBuffer` and
697 /// `traceSTREAM_BUFFER_RECEIVE` still ahead of it. So the bytes move,
698 /// and the line appears, on this task's next run and not inside
699 /// whoever took the CPU. `StreamBufferDemo`'s echo clients showed it
700 /// four times in a 2,000-tick run, each a receive traced six exits
701 /// early.
702 ///
703 /// `None` means "stop here"; the caller returns `Blocked` and the body
704 /// makes the same call again, which then takes the resume path.
705 // In line at all THREE callers, which are all inside
706 // `stream_buffer_receive_inner`.
707 //
708 // This said "A3: ONE caller in the linked kernel, so this pays a prologue
709 // and epilogue for a single call" until 2026-09-25. The caller count was
710 // simply wrong -- there are three -- so the A3 argument did not apply, and
711 // with three callers `#[inline]` DUPLICATES the body rather than moving it,
712 // which is the opposite of what the comment claimed.
713 //
714 // The decision survives the correction, measured: `#[inline(never)]` costs
715 // **+100 B** (receive 732 -> 686, but a 146 B symbol appears). Each inlined
716 // copy folds to roughly fifteen bytes against the surrounding control flow,
717 // while a standalone body pays a frame and two real handle resolves. Right
718 // answer, wrong reason, now with a number.
719 #[inline(always)]
720 fn after_stream_wait(
721 &mut self,
722 caller: TaskHandle,
723 buffer: StreamBufferHandle,
724 ) -> Result<Option<usize>> {
725 self.buffers.resolve_mut(buffer)?.waiting_to_receive = TaskHandle::NULL;
726 let available = self.buffers.resolve(buffer)?.bytes_in_buffer();
727 if self.current != caller {
728 // Merging these two into one accessor that resolves the TCB once
729 // measured **0 B** on 2026-09-25: `&mut self` is `noalias`, so LLVM
730 // had already shared the resolve across both setters. `list.rs`'s
731 // fourth pass says exactly this, and it applies to the TCB flag
732 // accessors too. Merging adjacent accessors only pays when an
733 // `&mut self` call stands BETWEEN them.
734 self.set_stream_waited(caller);
735 self.set_stream_resume(caller, available);
736 return Ok(None);
737 }
738 Ok(Some(available))
739 }
740
741 /// `xStreamBufferReceive`: bytes into `out`, and how many arrived.
742 ///
743 /// # Errors
744 /// [`Error::Gone`] for a stale handle.
745 /// `xStreamBufferReceive`, with the contract-v2 bracket around it.
746 /// See [`Kernel::blind_call`].
747 ///
748 /// # Errors
749 /// As [`Kernel::stream_buffer_receive_inner`].
750 pub fn stream_buffer_receive(
751 &mut self,
752 buffer: StreamBufferHandle,
753 out: &mut [u8],
754 ticks: u64,
755 ) -> Result<Wait<usize>> {
756 let entry = if T::EMITS { self.port.exits() } else { 0 };
757 let received = self.stream_buffer_receive_inner(buffer, out, ticks);
758 self.blind_call(entry);
759 received
760 }
761
762 fn stream_buffer_receive_inner(
763 &mut self,
764 buffer: StreamBufferHandle,
765 out: &mut [u8],
766 ticks: u64,
767 ) -> Result<Wait<usize>> {
768 let caller = self.current;
769 // ONLY the fields this body reads. It was `*self.buffers.resolve(..)`
770 // -- the whole nine-field descriptor, about thirty-two bytes on rv32 --
771 // to reach two of them. `notify_index` is read at four
772 // points spread across the body, each behind an `&mut self` call, so
773 // the descriptor had to stay LIVE across all of them and was spilled
774 // for it. Two scalars live in registers. Same finding as `queue_take`:
775 // the cost was the live range, not the copy.
776 let (is_message, notify_index) = {
777 let b = self.buffers.resolve(buffer)?;
778 (b.is_message, b.notify_index)
779 };
780 let prefix = if is_message {
781 Self::MESSAGE_LENGTH_BYTES
782 } else {
783 0
784 };
785 let mut available;
786 if let Some(local) = self.take_stream_resume(caller) {
787 // Resuming after the exit that sampled the buffer.
788 //
789 // The C's thread stops inside `taskEXIT_CRITICAL`, and when it
790 // runs again it still evaluates the `if( xBytesAvailable <=
791 // xBytesToStoreMessageLength )` BELOW that exit -- against the
792 // sample it took inside the section. So a task preempted there
793 // goes on to block exactly like one that was not.
794 //
795 // Falling straight through to the read instead returns 0 from a
796 // call that was told to wait 350 ticks, which is what
797 // `StreamBufferDemo`'s lower-priority echo server caught: the C
798 // blocked at tick 1 and the sim created its client instead.
799 available = local;
800 if !self.take_stream_waited(caller) && available <= prefix {
801 match self.notify_wait(notify_index, 0, 0, ticks)? {
802 Blocked => return Ok(Blocked),
803 Ready(_) => match self.after_stream_wait(caller, buffer)? {
804 Some(now) => available = now,
805 None => return Ok(Blocked),
806 },
807 }
808 }
809 } else if self.notify_wait_pending(caller) {
810 // Resuming: the C's wait returns here, so its second half runs
811 // whatever the buffer now holds.
812 match self.notify_wait(notify_index, 0, 0, ticks)? {
813 Blocked => return Ok(Blocked),
814 Ready(_) => {}
815 }
816 match self.after_stream_wait(caller, buffer)? {
817 Some(now) => available = now,
818 None => return Ok(Blocked),
819 }
820 } else if ticks > 0 {
821 self.enter_critical();
822 available = self.buffers.resolve(buffer)?.bytes_in_buffer();
823 let must_block = available <= prefix;
824 if must_block {
825 let _ = self.notify_state_clear(None, notify_index);
826 self.buffers.resolve_mut(buffer)?.waiting_to_receive = caller;
827 }
828 self.exit_critical();
829 // That exit can release a tick, and a tick can switch this task
830 // away. The C's thread stops inside the exit and everything
831 // below it runs when the task is resumed — with the sample it
832 // already took, and without paying for the section twice.
833 if self.current != caller {
834 self.set_stream_resume(caller, available);
835 return Ok(Blocked);
836 }
837 if must_block {
838 match self.notify_wait(notify_index, 0, 0, ticks)? {
839 Blocked => return Ok(Blocked),
840 Ready(_) => match self.after_stream_wait(caller, buffer)? {
841 Some(now) => available = now,
842 None => return Ok(Blocked),
843 },
844 }
845 }
846 } else {
847 available = self.buffers.resolve(buffer)?.bytes_in_buffer();
848 }
849
850 let mut received = 0;
851 if available > prefix {
852 received = self.read_message(buffer, out, available)?;
853 if received != 0 {
854 let tick = self.tick;
855 self.note_exits();
856 self.trace.event(
857 tick,
858 Event::StreamBufferReceive {
859 buffer,
860 bytes: received,
861 },
862 );
863 self.receive_completed(buffer)?;
864 }
865 }
866 Ok(Ready(received))
867 }
868
869 /// `xStreamBufferReceiveFromISR`.
870 ///
871 /// # Errors
872 /// [`Error::Gone`] for a stale handle.
873 pub fn stream_buffer_receive_from_isr(
874 &mut self,
875 buffer: StreamBufferHandle,
876 out: &mut [u8],
877 ) -> Result<(usize, Woken)> {
878 let snapshot = *self.buffers.resolve(buffer)?;
879 let prefix = if snapshot.is_message {
880 Self::MESSAGE_LENGTH_BYTES
881 } else {
882 0
883 };
884 let available = snapshot.bytes_in_buffer();
885 let mut received = 0;
886 let mut woken = Woken::NO;
887 if available > prefix {
888 received = self.read_message(buffer, out, available)?;
889 if received != 0 {
890 // `traceSTREAM_BUFFER_RECEIVE_FROM_ISR`, likewise unhooked.
891 woken = self.receive_completed_from_isr(buffer)?;
892 }
893 }
894 Ok((received, woken))
895 }
896
897 // --------------------------------------------------- the completions --
898
899 /// `sbSEND_COMPLETED`: tell a waiting reader, with the scheduler
900 /// suspended so the notify cannot switch away mid-update.
901 ///
902 /// The application may replace it — that is what the macro is for, and
903 /// `MessageBufferAMP` is the demo that does — so the hook gets first
904 /// refusal and `true` means it handled the whole thing.
905 // No `inline`: one was tried and cost 323,801 instructions on
906 // `StreamBufferDemo`. The body is small but it is reached from inside
907 // `stream_buffer_send`, which is already large.
908 // A3: ONE caller in the linked kernel, so this pays a prologue and epilogue
909 // for a single call. Inlining moves the body rather than duplicating it.
910 #[inline(always)]
911 fn send_completed(&mut self, buffer: StreamBufferHandle) -> Result<()> {
912 if H::send_completed(self, buffer) {
913 return Ok(());
914 }
915 self.suspend_all();
916 let waiting = self.buffers.resolve(buffer)?.waiting_to_receive;
917 if !waiting.is_null() {
918 let index = self.buffers.resolve(buffer)?.notify_index;
919 let _ = self.notify(waiting, index, 0, NotifyAction::None)?;
920 self.buffers.resolve_mut(buffer)?.waiting_to_receive = TaskHandle::NULL;
921 }
922 let _ = self.resume_all();
923 Ok(())
924 }
925
926 /// `xStreamBufferSendCompletedFromISR` / `sbSEND_COMPLETED_FROM_ISR`.
927 ///
928 /// Public because an application that replaced [`TickHook::send_completed`]
929 /// has to be able to do the notify itself, later and from interrupt
930 /// context — which is exactly what `MessageBufferAMP` does once its
931 /// stand-in interrupt has read the handle back off the control buffer.
932 ///
933 /// # Errors
934 /// [`Error::Gone`] for a stale handle.
935 pub fn send_completed_from_isr(&mut self, buffer: StreamBufferHandle) -> Result<Woken> {
936 let waiting = self.buffers.resolve(buffer)?.waiting_to_receive;
937 if waiting.is_null() {
938 return Ok(Woken::NO);
939 }
940 let index = self.buffers.resolve(buffer)?.notify_index;
941 let (_, woken) = self.notify_from_isr(waiting, index, 0, NotifyAction::None)?;
942 self.buffers.resolve_mut(buffer)?.waiting_to_receive = TaskHandle::NULL;
943 Ok(woken)
944 }
945
946 /// `sbRECEIVE_COMPLETED`.
947 fn receive_completed(&mut self, buffer: StreamBufferHandle) -> Result<()> {
948 self.suspend_all();
949 let waiting = self.buffers.resolve(buffer)?.waiting_to_send;
950 if !waiting.is_null() {
951 let index = self.buffers.resolve(buffer)?.notify_index;
952 let _ = self.notify(waiting, index, 0, NotifyAction::None)?;
953 self.buffers.resolve_mut(buffer)?.waiting_to_send = TaskHandle::NULL;
954 }
955 let _ = self.resume_all();
956 Ok(())
957 }
958
959 /// `sbRECEIVE_COMPLETED_FROM_ISR`.
960 fn receive_completed_from_isr(&mut self, buffer: StreamBufferHandle) -> Result<Woken> {
961 let waiting = self.buffers.resolve(buffer)?.waiting_to_send;
962 if waiting.is_null() {
963 return Ok(Woken::NO);
964 }
965 let index = self.buffers.resolve(buffer)?.notify_index;
966 let (_, woken) = self.notify_from_isr(waiting, index, 0, NotifyAction::None)?;
967 self.buffers.resolve_mut(buffer)?.waiting_to_send = TaskHandle::NULL;
968 Ok(woken)
969 }
970
971 // ------------------------------------------------ the ring itself --
972
973 /// `prvWriteMessageToBuffer`.
974 fn write_message(
975 &mut self,
976 buffer: StreamBufferHandle,
977 data: &[u8],
978 space: usize,
979 required: usize,
980 ) -> Result<usize> {
981 // ONE resolve, and the descriptor is written through in place.
982 //
983 // This used to copy the whole nine-field descriptor out
984 // (`*self.buffers.resolve(..)`) and then resolve a SECOND time to store
985 // the new head, because `write_bytes` wanted `&mut self` and so nothing
986 // borrowed from `self` could be alive across it. Destructuring `self`
987 // into disjoint field borrows removes both the copy and the second
988 // resolve: `buffers` and `bytes` are different fields, so the borrow
989 // checker allows what C does with one pointer.
990 let Self { buffers, bytes, .. } = self;
991 let b = buffers.resolve_mut(buffer)?;
992 let (base, ring, is_message) = (b.base, b.length, b.is_message);
993 let mut next_head = b.head;
994 let mut length = data.len();
995 let mut space = space;
996 if is_message {
997 if space >= required {
998 // `write_length_prefix`, folded in: it was a one-caller wrapper
999 // whose whole body was this.
1000 // `length.to_le_bytes()`, not `(length as u64).to_le_bytes()`:
1001 // the widening wrote eight bytes on a target whose `size_t` is
1002 // four, and the top half was never sent.
1003 let raw = length.to_le_bytes();
1004 let prefix = raw.get(..Self::MESSAGE_LENGTH_BYTES).unwrap_or(&raw);
1005 next_head = Self::write_bytes_into(bytes, base, ring, prefix, next_head);
1006 } else {
1007 length = 0;
1008 }
1009 space = space.saturating_sub(Self::MESSAGE_LENGTH_BYTES);
1010 }
1011 let length = length.min(space);
1012 if length != 0 {
1013 let chunk = data.get(..length).unwrap_or(data);
1014 b.head = Self::write_bytes_into(bytes, base, ring, chunk, next_head);
1015 }
1016 Ok(length)
1017 }
1018
1019 /// `prvReadMessageFromBuffer`.
1020 fn read_message(
1021 &mut self,
1022 buffer: StreamBufferHandle,
1023 out: &mut [u8],
1024 available: usize,
1025 ) -> Result<usize> {
1026 // ONE resolve, and the descriptor is written through in place -- the
1027 // mirror of `write_message`. Two resolves and a nine-field copy were
1028 // needed only while the helpers took `&self`/`&mut self`; giving them
1029 // the arena as a SLICE lets `buffers` and `bytes` be borrowed disjointly.
1030 let Self { buffers, bytes, .. } = self;
1031 let b = buffers.resolve_mut(buffer)?;
1032 let (base, ring, is_message) = (b.base, b.length, b.is_message);
1033 let mut next_tail = b.tail;
1034 let mut available = available;
1035 let next_length;
1036 if is_message {
1037 let (message_length, tail) =
1038 Self::read_length_prefix_from(bytes, base, ring, next_tail);
1039 next_tail = tail;
1040 available = available.saturating_sub(Self::MESSAGE_LENGTH_BYTES);
1041 next_length = if message_length > out.len() {
1042 // The C returns nothing rather than half a message.
1043 0
1044 } else {
1045 message_length
1046 };
1047 } else {
1048 next_length = out.len();
1049 }
1050 let count = next_length.min(available);
1051 if count != 0 {
1052 b.tail = match out.get_mut(..count) {
1053 Some(slot) => Self::read_bytes_from(bytes, base, ring, slot, next_tail),
1054 None => next_tail,
1055 };
1056 }
1057 Ok(count)
1058 }
1059
1060 /// `prvWriteBytesToBuffer`: the ring wrap, in two `memcpy`s.
1061 ///
1062 /// The index arithmetic here is `wrapping_*`, because the geometry has
1063 /// already bounded every term: `head` is below `length`, `base + length`
1064 /// is at most `BYTES`, `first` is `min(len, upto)` so `len - first`
1065 /// cannot go below zero, and `head + first` is at most `length` -- which
1066 /// the fold on the line after turns back into 0, exactly as `wrap_next`
1067 /// does. Each index then passes through a `get_mut` that refuses
1068 /// anything the ring does not own.
1069 /// Takes the byte arena as a SLICE rather than `&mut self`, so a caller can
1070 /// hold `&mut` to a descriptor in `self.buffers` at the same time. It asked
1071 /// for `&mut self` until 2026-09-25, and that is why `write_message` had to
1072 /// copy the whole descriptor out before calling it: under `forbid(unsafe)`
1073 /// a `&StreamBuffer` borrowed from `self` cannot coexist with `&mut self`.
1074 /// Destructuring `self` into disjoint field borrows is the safe form of what
1075 /// C gets for free by holding one pointer.
1076 fn write_bytes_into(
1077 bytes: &mut [u8],
1078 base: usize,
1079 ring: usize,
1080 data: &[u8],
1081 head: usize,
1082 ) -> usize {
1083 // The two copies, and the two the C makes. A payload longer than the
1084 // ring is refused before this, so that case cannot arrive.
1085 if data.len() <= ring {
1086 let upto = ring.wrapping_sub(head);
1087 let first = data.len().min(upto);
1088 let from = base.wrapping_add(head);
1089 if let (Some(dst), Some(src)) = (
1090 bytes.get_mut(from..from.wrapping_add(first)),
1091 data.get(..first),
1092 ) {
1093 dst.copy_from_slice(src);
1094 }
1095 let rest = data.len().wrapping_sub(first);
1096 if rest > 0 {
1097 if let (Some(dst), Some(src)) = (
1098 bytes.get_mut(base..base.wrapping_add(rest)),
1099 data.get(first..),
1100 ) {
1101 dst.copy_from_slice(src);
1102 }
1103 return rest;
1104 }
1105 let next = head.wrapping_add(first);
1106 return if next >= ring { 0 } else { next };
1107 }
1108 head
1109 }
1110
1111 /// `prvReadBytesFromBuffer`.
1112 ///
1113 /// Bounded as [`Kernel::write_bytes`] is, and for the same reasons.
1114 /// The mirror of [`Self::write_bytes_into`], and takes the arena the same
1115 /// way -- as a slice, so a caller can hold `&mut` to a descriptor in
1116 /// `self.buffers` across it. A request longer than the ring cannot arrive:
1117 /// `read_message` clamps `count` to `available`, which the ring's spare byte
1118 /// holds below `ring`.
1119 ///
1120 /// It asked for `&mut self` until 2026-09-25 while only ever READING the
1121 /// arena, and that single word is why `read_length_prefix` carried its own
1122 /// copy of this whole ring walk: with `&self` it could not call this.
1123 ///
1124 /// In line at both callers: out of line it became a shared symbol and cost
1125 /// +12 B, because each caller freezes a different length.
1126 #[inline(always)]
1127 fn read_bytes_from(
1128 bytes: &[u8],
1129 base: usize,
1130 ring: usize,
1131 out: &mut [u8],
1132 tail: usize,
1133 ) -> usize {
1134 if out.len() <= ring {
1135 let upto = ring.wrapping_sub(tail);
1136 let first = out.len().min(upto);
1137 let from = base.wrapping_add(tail);
1138 if let (Some(dst), Some(src)) = (
1139 out.get_mut(..first),
1140 bytes.get(from..from.wrapping_add(first)),
1141 ) {
1142 dst.copy_from_slice(src);
1143 }
1144 let rest = out.len().wrapping_sub(first);
1145 if rest > 0 {
1146 if let (Some(dst), Some(src)) = (
1147 out.get_mut(first..),
1148 bytes.get(base..base.wrapping_add(rest)),
1149 ) {
1150 dst.copy_from_slice(src);
1151 }
1152 return rest;
1153 }
1154 let next = tail.wrapping_add(first);
1155 return if next >= ring { 0 } else { next };
1156 }
1157 tail
1158 }
1159
1160 /// The message length back out, and where the message starts.
1161 /// Bounded as [`Kernel::read_bytes`] is, and converted for the same
1162 /// reasons: `tail` is below `length`, `base + length` is at most
1163 /// `BYTES`, `first` is `min(want, upto)`, and every index passes through
1164 /// a `get` that refuses anything the ring does not own.
1165 /// The prefix is converted at the TARGET's width, not always through `u64`.
1166 ///
1167 /// This used `[0_u8; 8]` and `u64::from_le_bytes(..) as usize`. On rv32 that
1168 /// reads eight bytes as two words and throws the top one away, because
1169 /// `configMESSAGE_BUFFER_LENGTH_TYPE` is `size_t` and `size_t` is FOUR bytes
1170 /// there. `usize` is the width the C's type actually has on whichever target
1171 /// this is, so it is the right one to build.
1172 fn read_length_prefix_from(
1173 bytes: &[u8],
1174 base: usize,
1175 ring: usize,
1176 tail: usize,
1177 ) -> (usize, usize) {
1178 let mut raw = [0_u8; core::mem::size_of::<usize>()];
1179 // Delegated. This held its OWN two-copy ring walk plus a byte-at-a-time
1180 // fallback -- a third hand-rolled version of the same walk -- and the
1181 // only reason was that `read_bytes` asked for `&mut self` when it never
1182 // needed it.
1183 // A prefix wider than the target's `usize` cannot be represented, and
1184 // on the oracle host the two are equal, so this clamp is the identity
1185 // wherever the arms are matched.
1186 let want = Self::MESSAGE_LENGTH_BYTES.min(raw.len());
1187 let next = match raw.get_mut(..want) {
1188 Some(slot) => Self::read_bytes_from(bytes, base, ring, slot, tail),
1189 None => tail,
1190 };
1191 (usize::from_le_bytes(raw), next)
1192 }
1193}
1194
1195// ================================================================ tests ==
1196
1197/// `stream.rs` had no unit test at all, and six of its APIs are also the
1198/// ones no conformance scenario reaches (`docs/HOLES.md`, H2 and H3). For
1199/// those, the only evidence available is a semantic audit against the C
1200/// source plus tests that pin what it says.
1201///
1202/// So every test here quotes the `stream_buffer.c` line it is pinning, and
1203/// pins **the C's contract** rather than this kernel's behaviour. A test
1204/// written the other way round proves only that the code still does what it
1205/// did, which is worth having and is not evidence of conformance.
1206#[cfg(test)]
1207#[allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
1208mod tests {
1209 use rusty_rtos_core::config::Config;
1210 use rusty_rtos_core::hooks::NoTickHook;
1211
1212 use crate::queue::Wait;
1213 use crate::system::tests::{NoTrace, TestConfig, TestPort};
1214
1215 /// A kernel with buffers, which `system!` never declares: the macro
1216 /// sizes `BUFFERS` and `BYTES` at zero. Four tasks is the idle task,
1217 /// the timer daemon and one of our own, with room to spare.
1218 type K = crate::Kernel<
1219 TestConfig,
1220 TestPort,
1221 NoTrace,
1222 NoTickHook,
1223 4,
1224 { crate::list_slots_for(4, 0, crate::lists_for(TestConfig::MAX_PRIORITIES, 1, 0)) },
1225 { crate::lists_for(TestConfig::MAX_PRIORITIES, 1, 0) },
1226 1,
1227 1,
1228 2,
1229 64,
1230 0,
1231 0,
1232 { <TestConfig as ::rusty_rtos_core::config::Config>::TIMER_QUEUE_LENGTH },
1233 >;
1234
1235 fn kernel() -> K {
1236 K::new(TestPort::default(), NoTrace).expect("the declared geometry adds up")
1237 }
1238
1239 /// The header a message buffer writes in front of every message.
1240 const HDR: usize = K::MESSAGE_LENGTH_BYTES;
1241
1242 /// `xStreamBufferIsEmpty`:
1243 ///
1244 /// ```c
1245 /// if( pxStreamBuffer->xHead == xTail ) { xReturn = pdTRUE; }
1246 /// ```
1247 ///
1248 /// Head-equals-tail, which is emptiness and NOT "nothing readable" --
1249 /// the two differ on a message buffer holding only a header, which the
1250 /// zero-length-message test below pins.
1251 #[test]
1252 fn is_empty_is_head_equals_tail() {
1253 let mut k = kernel();
1254 let b = k.stream_buffer_create(8, 1).expect("a buffer");
1255 assert_eq!(k.stream_buffer_is_empty(b), Ok(true));
1256
1257 k.stream_buffer_send(b, b"abc", 0).expect("send");
1258 assert_eq!(k.stream_buffer_is_empty(b), Ok(false));
1259
1260 let mut out = [0_u8; 8];
1261 k.stream_buffer_receive(b, &mut out, 0).expect("receive");
1262 assert_eq!(
1263 k.stream_buffer_is_empty(b),
1264 Ok(true),
1265 "drained is empty again, wherever head and tail have walked to"
1266 );
1267 }
1268
1269 /// `xStreamBufferIsFull` on a STREAM buffer:
1270 ///
1271 /// ```c
1272 /// xBytesToStoreMessageLength = 0; /* not a message buffer */
1273 /// if( xStreamBufferSpacesAvailable( xStreamBuffer ) <= xBytesToStoreMessageLength )
1274 /// ```
1275 ///
1276 /// so full is exactly "no spaces left".
1277 #[test]
1278 fn is_full_on_a_stream_buffer_means_no_spaces_at_all() {
1279 let mut k = kernel();
1280 let b = k.stream_buffer_create(8, 1).expect("a buffer");
1281 assert_eq!(k.stream_buffer_spaces_available(b), Ok(8));
1282 assert_eq!(k.stream_buffer_is_full(b), Ok(false));
1283
1284 k.stream_buffer_send(b, b"1234567", 0).expect("send seven");
1285 assert_eq!(k.stream_buffer_spaces_available(b), Ok(1));
1286 assert_eq!(
1287 k.stream_buffer_is_full(b),
1288 Ok(false),
1289 "one free byte is not full on a STREAM buffer"
1290 );
1291
1292 k.stream_buffer_send(b, b"8", 0).expect("send the last");
1293 assert_eq!(k.stream_buffer_spaces_available(b), Ok(0));
1294 assert_eq!(k.stream_buffer_is_full(b), Ok(true));
1295 }
1296
1297 /// `xStreamBufferIsFull` on a MESSAGE buffer, which is the one that
1298 /// surprises:
1299 ///
1300 /// ```c
1301 /// if( ( pxStreamBuffer->ucFlags & sbFLAGS_IS_MESSAGE_BUFFER ) != 0 )
1302 /// xBytesToStoreMessageLength = sbBYTES_TO_STORE_MESSAGE_LENGTH;
1303 /// if( xStreamBufferSpacesAvailable( xStreamBuffer ) <= xBytesToStoreMessageLength )
1304 /// ```
1305 ///
1306 /// `<=`, against the size of the LENGTH HEADER rather than against
1307 /// zero. So a message buffer reports FULL while it still has free
1308 /// bytes, because those bytes could not hold even a zero-length
1309 /// message's header.
1310 ///
1311 /// A caller that polls `is_full` and expects `spaces_available() == 0`
1312 /// is wrong by up to a header, and only on message buffers.
1313 #[test]
1314 fn is_full_on_a_message_buffer_leaves_a_header_of_bytes_free() {
1315 let mut k = kernel();
1316 let b = k.message_buffer_create(8).expect("a message buffer");
1317 assert_eq!(k.stream_buffer_spaces_available(b), Ok(8));
1318 assert_eq!(k.stream_buffer_is_full(b), Ok(false));
1319
1320 // One three-byte message costs the header plus the payload.
1321 k.stream_buffer_send(b, b"abc", 0).expect("send");
1322 let spaces = k.stream_buffer_spaces_available(b).expect("spaces");
1323 assert_eq!(spaces, 8 - (HDR + 3));
1324 assert!(spaces > 0, "there ARE free bytes");
1325 assert!(spaces <= HDR, "but not enough for another header");
1326 assert_eq!(
1327 k.stream_buffer_is_full(b),
1328 Ok(true),
1329 "full with {spaces} byte(s) free: the C compares against the header, not zero"
1330 );
1331 }
1332
1333 /// `xStreamBufferNextMessageLengthBytes` on a stream buffer:
1334 ///
1335 /// ```c
1336 /// if( ( pxStreamBuffer->ucFlags & sbFLAGS_IS_MESSAGE_BUFFER ) != 0 ) { ... }
1337 /// else { xReturn = 0; }
1338 /// ```
1339 ///
1340 /// Zero for a stream buffer NO MATTER WHAT IT HOLDS. That is not "no
1341 /// message waiting", it is "the question does not apply" -- and the
1342 /// return type cannot tell a caller which of the two it got.
1343 #[test]
1344 fn next_message_length_is_zero_on_a_stream_buffer_however_full() {
1345 let mut k = kernel();
1346 let b = k.stream_buffer_create(8, 1).expect("a buffer");
1347 assert_eq!(k.stream_buffer_next_message_length(b), Ok(0));
1348
1349 k.stream_buffer_send(b, b"abcde", 0).expect("send");
1350 assert_eq!(k.stream_buffer_bytes_available(b), Ok(5));
1351 assert_eq!(
1352 k.stream_buffer_next_message_length(b),
1353 Ok(0),
1354 "five bytes waiting and still zero, because it is not a message buffer"
1355 );
1356 }
1357
1358 /// The same call on a message buffer PEEKS:
1359 ///
1360 /// ```c
1361 /// ( void ) prvReadBytesFromBuffer( pxStreamBuffer, ..., pxStreamBuffer->xTail );
1362 /// ```
1363 ///
1364 /// It reads at the tail without advancing it, so asking twice answers
1365 /// twice and the message is still there to be received.
1366 #[test]
1367 fn next_message_length_peeks_and_does_not_consume() {
1368 let mut k = kernel();
1369 let b = k.message_buffer_create(16).expect("a message buffer");
1370 k.stream_buffer_send(b, b"hello", 0).expect("send");
1371
1372 assert_eq!(k.stream_buffer_next_message_length(b), Ok(5));
1373 assert_eq!(
1374 k.stream_buffer_next_message_length(b),
1375 Ok(5),
1376 "asking did not consume it"
1377 );
1378
1379 let mut out = [0_u8; 16];
1380 assert_eq!(k.stream_buffer_receive(b, &mut out, 0), Ok(Wait::Ready(5)));
1381 assert_eq!(&out[..5], b"hello");
1382 assert_eq!(k.stream_buffer_next_message_length(b), Ok(0), "drained");
1383 }
1384
1385 /// A zero-length message is a NO-OP that is indistinguishable from a
1386 /// send that failed -- which is not visible from the call site, and is
1387 /// the sort of thing only a source audit finds.
1388 ///
1389 /// `prvWriteMessageToBuffer`:
1390 ///
1391 /// ```c
1392 /// if( message buffer ) {
1393 /// xMessageLength = ( configMESSAGE_BUFFER_LENGTH_TYPE ) xDataLengthBytes; /* 0 */
1394 /// if( xSpace >= xRequiredSpace ) {
1395 /// xNextHead = prvWriteBytesToBuffer( pxStreamBuffer, &xMessageLength,
1396 /// sbBYTES_TO_STORE_MESSAGE_LENGTH, xNextHead );
1397 /// }
1398 /// }
1399 /// if( xDataLengthBytes != ( size_t ) 0 ) {
1400 /// pxStreamBuffer->xHead = prvWriteBytesToBuffer( ..., xNextHead );
1401 /// }
1402 /// return xDataLengthBytes;
1403 /// ```
1404 ///
1405 /// The header IS written into the storage array -- but the new position
1406 /// goes into `xNextHead`, a LOCAL. `pxStreamBuffer->xHead` is only
1407 /// committed inside the `xDataLengthBytes != 0` branch, which a
1408 /// zero-length message does not take.
1409 ///
1410 /// So the bytes are in the array and the buffer does not know it: the
1411 /// head never moves, the buffer stays empty, the call returns 0, and
1412 /// the next send writes over them. A caller cannot tell this apart from
1413 /// a send that was refused for want of space.
1414 ///
1415 /// Pinned because the earlier version of this test asserted the
1416 /// opposite -- that the header would show up in `bytes_available` --
1417 /// and the kernel was right.
1418 #[test]
1419 fn a_zero_length_message_is_a_no_op_that_looks_like_a_failed_send() {
1420 let mut k = kernel();
1421 let b = k.message_buffer_create(16).expect("a message buffer");
1422 assert_eq!(k.stream_buffer_is_empty(b), Ok(true));
1423
1424 assert_eq!(
1425 k.stream_buffer_send(b, b"", 0),
1426 Ok(Wait::Ready(0)),
1427 "zero bytes written, which is also what a refused send answers"
1428 );
1429 assert_eq!(
1430 k.stream_buffer_bytes_available(b),
1431 Ok(0),
1432 "the header went into the array and the head was never committed"
1433 );
1434 assert_eq!(k.stream_buffer_is_empty(b), Ok(true));
1435 assert_eq!(k.stream_buffer_next_message_length(b), Ok(0));
1436
1437 // And a real message lands in the same place, over the top of it.
1438 k.stream_buffer_send(b, b"ok", 0).expect("a real message");
1439 assert_eq!(k.stream_buffer_next_message_length(b), Ok(2));
1440 let mut out = [0_u8; 16];
1441 assert_eq!(k.stream_buffer_receive(b, &mut out, 0), Ok(Wait::Ready(2)));
1442 assert_eq!(&out[..2], b"ok");
1443 assert_eq!(k.stream_buffer_is_empty(b), Ok(true));
1444 }
1445
1446 /// `xStreamBufferReset`:
1447 ///
1448 /// ```c
1449 /// if( ( pxStreamBuffer->xTaskWaitingToReceive == NULL ) &&
1450 /// ( pxStreamBuffer->xTaskWaitingToSend == NULL ) )
1451 /// {
1452 /// prvInitialiseNewStreamBuffer( ..., pxStreamBuffer->xTriggerLevelBytes, ... );
1453 /// xReturn = pdPASS;
1454 /// }
1455 /// ```
1456 ///
1457 /// The buffer's own trigger level is passed straight back in, so a
1458 /// reset empties the buffer and KEEPS the trigger. It is not a return
1459 /// to the creation default, which is what the name suggests.
1460 #[test]
1461 fn reset_empties_the_buffer_and_keeps_the_trigger_level() {
1462 let mut k = kernel();
1463 let b = k.stream_buffer_create(8, 1).expect("a buffer");
1464 assert_eq!(k.stream_buffer_set_trigger_level(b, 5), Ok(true));
1465 k.stream_buffer_send(b, b"abcd", 0).expect("send");
1466 assert_eq!(k.stream_buffer_bytes_available(b), Ok(4));
1467
1468 assert_eq!(k.stream_buffer_reset(b), Ok(true));
1469 assert_eq!(k.stream_buffer_is_empty(b), Ok(true));
1470 assert_eq!(k.stream_buffer_bytes_available(b), Ok(0));
1471 assert_eq!(
1472 k.stream_buffer_spaces_available(b),
1473 Ok(8),
1474 "the whole usable length is back"
1475 );
1476 }
1477
1478 /// The other half of that guard: a reset must REFUSE while a task is
1479 /// blocked on the buffer, because reinitialising would drop the handle
1480 /// that task is waiting on.
1481 #[test]
1482 fn reset_refuses_while_a_task_is_waiting_to_receive() {
1483 let mut k = kernel();
1484 let b = k.stream_buffer_create(8, 1).expect("a buffer");
1485 k.create_task("rx", 1).expect("a task");
1486 let started = k.start_scheduler().expect("start");
1487 k.suspend(Some(started.timer)).expect("park the daemon");
1488
1489 // Run until our task is the one on the CPU, then block it.
1490 let mut out = [0_u8; 8];
1491 let mut blocked = false;
1492 for _ in 0..50_u32 {
1493 let running = k.name_of(k.current()).expect("a running task has a name");
1494 if running.as_str() == "rx" {
1495 assert_eq!(
1496 k.stream_buffer_receive(b, &mut out, 20),
1497 Ok(Wait::Blocked),
1498 "an empty buffer with a timeout blocks"
1499 );
1500 blocked = true;
1501 break;
1502 }
1503 k.switch_context();
1504 }
1505 assert!(blocked, "never got the task onto the CPU");
1506
1507 assert_eq!(
1508 k.stream_buffer_reset(b),
1509 Ok(false),
1510 "a waiting receiver refuses the reset"
1511 );
1512 }
1513
1514 /// `xStreamBufferReceiveFromISR` never blocks: it takes a whole
1515 /// message, or nothing.
1516 #[test]
1517 fn receive_from_isr_takes_a_whole_message_or_nothing() {
1518 let mut k = kernel();
1519 let b = k.message_buffer_create(16).expect("a message buffer");
1520 let mut out = [0_u8; 16];
1521
1522 let (n, _) = k.stream_buffer_receive_from_isr(b, &mut out).expect("isr");
1523 assert_eq!(n, 0, "nothing waiting, and it did not block");
1524
1525 k.stream_buffer_send(b, b"xy", 0).expect("send");
1526 let (n, _) = k.stream_buffer_receive_from_isr(b, &mut out).expect("isr");
1527 assert_eq!(n, 2);
1528 assert_eq!(&out[..2], b"xy");
1529 assert_eq!(k.stream_buffer_is_empty(b), Ok(true));
1530 }
1531
1532 /// `bytes_available` and `spaces_available` partition the USABLE
1533 /// length, which is one less than the ring's own -- the spare byte that
1534 /// tells full from empty, the C's `xBufferSizeBytes++` at creation.
1535 #[test]
1536 fn bytes_and_spaces_partition_the_usable_length() {
1537 let mut k = kernel();
1538 let b = k.stream_buffer_create(8, 1).expect("a buffer");
1539 for n in 0..=8_usize {
1540 let bytes = k.stream_buffer_bytes_available(b).expect("bytes");
1541 let spaces = k.stream_buffer_spaces_available(b).expect("spaces");
1542 assert_eq!(
1543 bytes.saturating_add(spaces),
1544 8,
1545 "at {n} byte(s) in: {bytes} + {spaces} should be the usable 8"
1546 );
1547 if n < 8 {
1548 k.stream_buffer_send(b, b"z", 0).expect("send one");
1549 }
1550 }
1551 }
1552 /// A STREAM buffer takes a partial write; a MESSAGE buffer is
1553 /// all-or-nothing. `prvWriteMessageToBuffer`:
1554 ///
1555 /// ```c
1556 /// if( message buffer ) {
1557 /// if( xSpace >= xRequiredSpace ) { ...write the header... }
1558 /// else { xDataLengthBytes = 0; } /* nothing at all */
1559 /// } else {
1560 /// xDataLengthBytes = configMIN( xDataLengthBytes, xSpace ); /* as much as fits */
1561 /// }
1562 /// ```
1563 ///
1564 /// So the same call with the same arguments loses the tail on one kind
1565 /// and loses everything on the other, and the return value is the only
1566 /// way to tell which happened.
1567 #[test]
1568 fn a_stream_takes_a_partial_write_and_a_message_is_all_or_nothing() {
1569 let mut k = kernel();
1570 let s = k.stream_buffer_create(8, 1).expect("a stream buffer");
1571 k.stream_buffer_send(s, b"123456", 0).expect("six");
1572 assert_eq!(
1573 k.stream_buffer_send(s, b"789", 0),
1574 Ok(Wait::Ready(2)),
1575 "two of the three fit, and two is what it took"
1576 );
1577 assert_eq!(k.stream_buffer_spaces_available(s), Ok(0));
1578
1579 let mut k = kernel();
1580 let m = k.message_buffer_create(8).expect("a message buffer");
1581 // One message of two costs the header plus two: six of the eight.
1582 k.stream_buffer_send(m, b"xy", 0)
1583 .expect("the first message");
1584 assert_eq!(
1585 k.stream_buffer_send(m, b"zzz", 0),
1586 Ok(Wait::Ready(0)),
1587 "the header plus three does not fit, so NOTHING was written"
1588 );
1589 assert_eq!(
1590 k.stream_buffer_next_message_length(m),
1591 Ok(2),
1592 "and the message already there is untouched"
1593 );
1594 }
1595
1596 /// `xStreamBufferSendFromISR` writes what fits and never blocks -- the
1597 /// same body as the task-side send with the block time forced to zero.
1598 #[test]
1599 fn send_from_isr_writes_what_fits_and_never_blocks() {
1600 let mut k = kernel();
1601 let b = k.stream_buffer_create(8, 1).expect("a buffer");
1602
1603 let (n, _) = k.stream_buffer_send_from_isr(b, b"abc").expect("isr");
1604 assert_eq!(n, 3);
1605 assert_eq!(k.stream_buffer_bytes_available(b), Ok(3));
1606
1607 // Past the end: it takes the five that fit rather than waiting.
1608 let (n, _) = k.stream_buffer_send_from_isr(b, b"defghijkl").expect("isr");
1609 assert_eq!(n, 5, "five of the nine fit");
1610 assert_eq!(k.stream_buffer_is_full(b), Ok(true));
1611
1612 let (n, _) = k.stream_buffer_send_from_isr(b, b"m").expect("isr");
1613 assert_eq!(n, 0, "full, and it still did not block");
1614 }
1615
1616 /// A round trip through the two ISR halves, which is what an
1617 /// interrupt-driven driver actually does.
1618 #[test]
1619 fn a_message_survives_a_round_trip_through_both_isr_halves() {
1620 let mut k = kernel();
1621 let b = k.message_buffer_create(16).expect("a message buffer");
1622 let (n, _) = k.stream_buffer_send_from_isr(b, b"ping").expect("isr");
1623 assert_eq!(n, 4);
1624
1625 let mut out = [0_u8; 16];
1626 let (n, _) = k.stream_buffer_receive_from_isr(b, &mut out).expect("isr");
1627 assert_eq!(n, 4);
1628 assert_eq!(&out[..4], b"ping");
1629 assert_eq!(k.stream_buffer_is_empty(b), Ok(true));
1630 }
1631}