Skip to main content

cubecl_runtime/memory_management/drop_queue/
queue.rs

1use alloc::vec::Vec;
2use cubecl_common::bytes::Bytes;
3
4use crate::memory_management::{
5    drop_queue::FlushingPolicy, drop_queue::policy::FlushingPolicyState,
6};
7
8/// A synchronization primitive that blocks until the device has finished
9/// processing all commands submitted before the fence was created.
10pub trait Fence {
11    /// Block the current thread until the signals this fence.
12    fn sync(self);
13}
14
15/// Defers the drop of CPU-side [`Bytes`] allocations until the device has
16/// finished reading them.
17///
18/// # How it works
19///
20/// The device uploads are asynchronous: after you copy bytes into a staging buffer
21/// and enqueue an upload command, the CPU memory must remain valid until the
22/// device is done. `PendingDropQueue` manages this lifetime with a two-phase
23/// approach:
24///
25/// 1. **Stage** – call [`push`](Self::push) to hand over bytes that are
26///    in-flight. They land in the `staged` list.
27/// 2. **Flush** – call [`flush`](Self::flush) to rotate the lists. The
28///    previously staged bytes move to `pending`, a new [`Fence`] is created
29///    to mark the end of the current upload batch, and any bytes that were
30///    *already* pending (i.e. the batch before that) are freed after syncing
31///    the previous fence.
32///
33/// This double-buffer scheme means CPU memory is held for at most two flush
34/// cycles, while avoiding any unnecessary stalls on the hot path.
35///
36/// # Flushing policy
37///
38/// Call [`should_flush`](Self::should_flush) to check whether enough bytes
39/// have accumulated to warrant a flush. You may also flush unconditionally
40/// (e.g. at the end of a frame).
41pub struct PendingDropQueue<E: Fence> {
42    /// Fence signalling that the device has consumed everything in `pending`.
43    fence: Option<E>,
44    /// Bytes from the *previous* flush cycle, kept alive until `event` fires.
45    pending: Vec<Bytes>,
46    /// Bytes queued in the *current* cycle, not yet associated with a fence.
47    staged: Vec<Bytes>,
48    /// The configuration of the queue.
49    policy: FlushingPolicy,
50    /// The current state of the policy.
51    policy_state: FlushingPolicyState,
52}
53
54impl<E: Fence> core::fmt::Debug for PendingDropQueue<E> {
55    fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
56        f.debug_struct("PendingDropQueue")
57            .field("pending", &self.pending)
58            .field("staged", &self.staged)
59            .field("policy", &self.policy)
60            .field("policy_state", &self.policy_state)
61            .finish()
62    }
63}
64
65impl<E: Fence> Default for PendingDropQueue<E> {
66    fn default() -> Self {
67        Self::new(Default::default())
68    }
69}
70
71impl<F: Fence> PendingDropQueue<F> {
72    /// Creates a new `PendingDropQueue`.
73    pub fn new(policy: FlushingPolicy) -> Self {
74        Self {
75            fence: None,
76            pending: Vec::new(),
77            staged: Vec::new(),
78            policy,
79            policy_state: Default::default(),
80        }
81    }
82    /// Enqueue `bytes` to be dropped once the device has finished reading them.
83    ///
84    /// The bytes are added to the current staged batch and will be freed on
85    /// the flush cycle *after* the next call to [`flush`](Self::flush).
86    pub fn push(&mut self, bytes: Bytes) {
87        self.policy_state.register(&bytes);
88        self.staged.push(bytes);
89    }
90
91    /// Returns `true` when the staged batch is large enough to justify a
92    /// flush.
93    pub fn should_flush(&self) -> bool {
94        self.policy_state.should_flush(&self.policy)
95    }
96
97    /// Rotate the double-buffer and free any memory the device is done with.
98    ///
99    /// THE POOL IS ALREADY REUSING THESE BUFFERS; THE BLOCKING IS THE PROOF,
100    /// NOT THE REUSE. `reserve_pinned` hands out POOLED slices, so nothing here
101    /// is allocating fresh host memory per upload -- the only thing this
102    /// function adds is the evidence that the device has finished reading a
103    /// slice before the pool may hand the same one out again. That is why the
104    /// stall it causes is queue-depth serialisation rather than a data
105    /// dependency, and why raising the threshold is not merely "flush less
106    /// often": past the depth where the fence being waited on was recorded long
107    /// enough ago that the device has already passed it, the wait goes to
108    /// approximately zero rather than to a fraction of itself.
109    ///
110    /// The cure that removes the reason for the flush rather than the frequency
111    /// of it is to stop staging per launch at all -- see
112    /// `docs/METADATA_CACHE.md` in mary, which is where the 483 uploads a
113    /// decode step come from.
114    ///
115    /// `factory` is called to produce a [`Fence`]. It should submit (or
116    /// record) a device signal command so that syncing the fence guarantees all
117    /// preceding device work is complete.
118    pub fn flush<Factory: Fn() -> F>(&mut self, factory: Factory) {
119        // Sync the fence from the previous flush and free the bytes it was
120        // protecting.
121        if let Some(event) = self.fence.take() {
122            event.sync();
123            self.pending.clear();
124        }
125
126        // Safety net: if pending is somehow still populated (no prior fence),
127        // stall immediately rather than freeing memory the GPU might still
128        // be reading.
129        if !self.pending.is_empty() {
130            let event = factory();
131            event.sync();
132            self.pending.clear();
133        }
134
135        // The current staged batch becomes the new pending batch.
136        core::mem::swap(&mut self.pending, &mut self.staged);
137
138        // Record a fence so the *next* flush knows when this batch is safe to
139        // free.
140        self.fence = Some(factory());
141        self.policy_state.reset();
142    }
143}
144
145#[cfg(test)]
146mod tests {
147    use super::*;
148    use alloc::vec;
149    use core::cell::Cell;
150
151    // ---------------------------------------------------------------------------
152    // Test helpers
153    // ---------------------------------------------------------------------------
154
155    #[derive(Clone)]
156    struct MockFence<'a> {
157        sync_count: &'a Cell<u32>,
158    }
159
160    impl Fence for MockFence<'_> {
161        fn sync(self) {
162            self.sync_count.set(self.sync_count.get() + 1);
163        }
164    }
165
166    fn make_queue<'a>(
167        sync_count: &'a Cell<u32>,
168    ) -> (
169        PendingDropQueue<MockFence<'a>>,
170        impl Fn() -> MockFence<'a> + 'a,
171    ) {
172        let queue = PendingDropQueue::new(test_policy());
173        let factory = move || MockFence { sync_count };
174        (queue, factory)
175    }
176
177    fn sample_bytes() -> Bytes {
178        Bytes::from_elems(vec![1u8, 2, 3])
179    }
180
181    fn test_policy() -> FlushingPolicy {
182        FlushingPolicy {
183            max_bytes_count: 2048,
184            max_bytes_size: 8,
185        }
186    }
187
188    // ---------------------------------------------------------------------------
189    // push / should_flush
190    // ---------------------------------------------------------------------------
191
192    #[test]
193    fn push_at_count_threshold_triggers_flush_hint() {
194        let sync_count = Cell::new(0u32);
195        let (mut queue, _factory) = make_queue(&sync_count);
196
197        for _ in 0..test_policy().max_bytes_count {
198            queue.push(sample_bytes());
199        }
200
201        assert!(queue.should_flush());
202    }
203
204    #[test]
205    fn push_large_allocation_triggers_flush_via_size_threshold() {
206        let sync_count = Cell::new(0u32);
207        let (mut queue, _factory) = make_queue(&sync_count);
208        let big = Bytes::from_elems(vec![0u8; test_policy().max_bytes_size as usize + 1]);
209
210        queue.push(big);
211
212        assert!(queue.should_flush());
213    }
214
215    // ---------------------------------------------------------------------------
216    // flush – fence / sync behaviour
217    // ---------------------------------------------------------------------------
218
219    #[test]
220    fn first_flush_creates_fence_without_syncing() {
221        let sync_count = Cell::new(0u32);
222        let (mut queue, factory) = make_queue(&sync_count);
223
224        queue.push(sample_bytes());
225        queue.flush(&factory);
226
227        // The fence is created but must not be synced yet — that happens on
228        // the next flush.
229        assert_eq!(
230            sync_count.get(),
231            0,
232            "fence should not be synced on first flush"
233        );
234    }
235
236    #[test]
237    fn second_flush_syncs_fence_from_first_flush() {
238        let sync_count = Cell::new(0u32);
239        let (mut queue, factory) = make_queue(&sync_count);
240
241        queue.push(sample_bytes());
242        queue.flush(&factory); // flush 1 – creates fence A
243
244        queue.push(sample_bytes());
245        queue.flush(&factory); // flush 2 – syncs fence A, creates fence B
246
247        assert_eq!(sync_count.get(), 1, "exactly one sync after two flushes");
248    }
249
250    #[test]
251    fn each_subsequent_flush_syncs_the_previous_fence() {
252        let sync_count = Cell::new(0u32);
253        let (mut queue, factory) = make_queue(&sync_count);
254
255        for _ in 0..10 {
256            queue.push(sample_bytes());
257            queue.flush(&factory);
258        }
259
260        // Each flush except the first syncs the fence from the previous one.
261        assert_eq!(sync_count.get(), 9);
262    }
263
264    // ---------------------------------------------------------------------------
265    // flush – buffer rotation
266    // ---------------------------------------------------------------------------
267
268    #[test]
269    fn staged_is_empty_after_flush() {
270        let sync_count = Cell::new(0u32);
271        let (mut queue, factory) = make_queue(&sync_count);
272
273        for _ in 0..5 {
274            queue.push(sample_bytes());
275        }
276        queue.flush(&factory);
277
278        assert!(queue.staged.is_empty());
279    }
280
281    #[test]
282    fn pending_holds_previously_staged_bytes_after_flush() {
283        let sync_count = Cell::new(0u32);
284        let (mut queue, factory) = make_queue(&sync_count);
285
286        for _ in 0..5 {
287            queue.push(sample_bytes());
288        }
289        queue.flush(&factory);
290
291        assert_eq!(queue.pending.len(), 5);
292    }
293
294    #[test]
295    fn pending_is_replaced_on_second_flush() {
296        let sync_count = Cell::new(0u32);
297        let (mut queue, factory) = make_queue(&sync_count);
298
299        for _ in 0..5 {
300            queue.push(sample_bytes());
301        }
302        queue.flush(&factory); // pending = 5 items
303
304        queue.push(sample_bytes());
305        queue.flush(&factory); // syncs fence → pending cleared, rotated
306
307        // Only the one item staged between the two flushes should be pending.
308        assert_eq!(queue.pending.len(), 1);
309    }
310
311    // ---------------------------------------------------------------------------
312    // flush – policy state reset
313    // ---------------------------------------------------------------------------
314
315    #[test]
316    fn should_flush_resets_after_flush() {
317        let sync_count = Cell::new(0u32);
318        let (mut queue, factory) = make_queue(&sync_count);
319
320        for _ in 0..test_policy().max_bytes_count {
321            queue.push(sample_bytes());
322        }
323        assert!(queue.should_flush());
324
325        queue.flush(&factory);
326
327        assert!(
328            !queue.should_flush(),
329            "policy state should be reset after flush"
330        );
331    }
332
333    // ---------------------------------------------------------------------------
334    // Edge cases
335    // ---------------------------------------------------------------------------
336
337    #[test]
338    fn flush_on_empty_queue_is_safe() {
339        let sync_count = Cell::new(0u32);
340        let (mut queue, factory) = make_queue(&sync_count);
341
342        // Should not panic regardless of how many times it is called.
343        queue.flush(&factory);
344        queue.flush(&factory);
345        queue.flush(&factory);
346    }
347}