gpu_async_reactor/core/
line.rs

1// SPDX-License-Identifier: MIT OR Apache-2.0
2
3//! The completion line and the memory descriptor.
4//!
5//! A [`CompletionLine`] is one 64-byte line of eight 64-bit completion words. A device writes a
6//! sequence number into a word behind its work; a host waits on the line. On cores whose
7//! exclusive-reservation granule is the line (aarch64 with a 64-byte ERG, x86 monitor ranges),
8//! any store to the line wakes one waiter, so one waiter multiplexes up to [`LINE_WORDS`]
9//! streams.
10//!
11//! A [`MemoryDescriptor`] names one word and the sequence number that completes it: the GPU
12//! analogue of a file descriptor's readiness, and the unit the executor waits on.
13
14use core::sync::atomic::{AtomicU64, Ordering};
15
16/// Completion words per line.
17pub const LINE_WORDS: usize = 8;
18
19/// A 64-byte aligned line of completion words.
20#[repr(C, align(64))]
21#[derive(Debug, Default)]
22pub struct CompletionLine {
23    /// One word per slot; a device writes a sequence number into its slot's word.
24    pub words: [AtomicU64; LINE_WORDS],
25}
26
27impl AsRef<CompletionLine> for CompletionLine {
28    fn as_ref(&self) -> &CompletionLine {
29        self
30    }
31}
32
33const _: () = assert!(core::mem::size_of::<CompletionLine>() == 64);
34const _: () = assert!(core::mem::align_of::<CompletionLine>() == 64);
35
36/// The slot of a word in a [`CompletionLine`] (`0..LINE_WORDS`, checked once at construction).
37#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
38pub struct Slot(u8);
39
40impl Slot {
41    /// Slot `i`, or `None` when `i >= LINE_WORDS`.
42    #[must_use]
43    pub const fn new(i: usize) -> Option<Self> {
44        if i < LINE_WORDS {
45            // The bound above makes the narrowing exact.
46            #[expect(clippy::cast_possible_truncation, reason = "i < LINE_WORDS = 8")]
47            Some(Self(i as u8))
48        } else {
49            None
50        }
51    }
52
53    /// The index into [`CompletionLine::words`].
54    #[must_use]
55    pub const fn index(self) -> usize {
56        self.0 as usize
57    }
58}
59
60/// One awaitable completion: word `slot` of `line` reaching `target`. Sequence numbers increase
61/// by one per store, so "reached" is `word >= target` in wrapping order (a later store that lands
62/// before the waiter looks still completes it).
63#[derive(Clone, Copy, Debug)]
64pub struct MemoryDescriptor<'l> {
65    line: &'l CompletionLine,
66    slot: Slot,
67    target: u64,
68}
69
70impl<'l> MemoryDescriptor<'l> {
71    /// Word `slot` of `line` reaching `target`.
72    #[must_use]
73    pub const fn new(line: &'l CompletionLine, slot: Slot, target: u64) -> Self {
74        Self { line, slot, target }
75    }
76
77    /// The line the word lives in.
78    #[must_use]
79    pub const fn line(&self) -> &'l CompletionLine {
80        self.line
81    }
82
83    /// The word's slot.
84    #[must_use]
85    pub const fn slot(&self) -> Slot {
86        self.slot
87    }
88
89    /// The sequence number that completes the descriptor.
90    #[must_use]
91    pub const fn target(&self) -> u64 {
92        self.target
93    }
94
95    /// Whether the word has reached the target (an `Acquire` load: the device's writes before
96    /// its store are visible after this returns `true`).
97    #[must_use]
98    pub fn is_complete(&self) -> bool {
99        reached(self.line, self.slot, self.target)
100    }
101}
102
103/// Whether word `slot` of `line` has reached `target` in wrapping sequence order.
104#[must_use]
105pub fn reached(line: &CompletionLine, slot: Slot, target: u64) -> bool {
106    let w = line.words[slot.index()].load(Ordering::Acquire);
107    w.wrapping_sub(target) < (1 << 63)
108}
109
110#[cfg(test)]
111mod tests {
112    use super::*;
113
114    #[test]
115    fn a_word_below_its_target_is_pending_and_at_or_past_it_is_complete() {
116        let line = CompletionLine::default();
117        let s = Slot::new(3).expect("slot 3");
118        let d = MemoryDescriptor::new(&line, s, 5);
119        assert!(!d.is_complete(), "0 < 5");
120        line.words[3].store(4, Ordering::Release);
121        assert!(!d.is_complete(), "4 < 5");
122        line.words[3].store(5, Ordering::Release);
123        assert!(d.is_complete(), "5 == 5");
124        line.words[3].store(6, Ordering::Release);
125        assert!(d.is_complete(), "a later store still completes it");
126        line.words[2].store(99, Ordering::Release);
127        assert!(MemoryDescriptor::new(&line, Slot::new(2).expect("2"), 99).is_complete());
128    }
129
130    #[test]
131    fn reached_is_wrapping_order() {
132        let line = CompletionLine::default();
133        let s = Slot::new(0).expect("slot 0");
134        line.words[0].store(1, Ordering::Release);
135        assert!(reached(&line, s, u64::MAX), "1 follows u64::MAX after a wrap");
136        line.words[0].store(u64::MAX, Ordering::Release);
137        assert!(!reached(&line, s, 1), "u64::MAX precedes 1");
138    }
139
140    #[test]
141    fn a_slot_past_the_line_is_refused() {
142        assert!(Slot::new(LINE_WORDS).is_none());
143        assert_eq!(Slot::new(LINE_WORDS - 1).map(Slot::index), Some(LINE_WORDS - 1));
144    }
145}
gpu_async_reactor/core/line.rs

gpu_async_reactor/core/
line.rs
