cubecl_runtime/memory_management/drop_queue/queue.rs
1use alloc::vec::Vec;
2use cubecl_common::bytes::Bytes;
3
4use crate::memory_management::{
5 drop_queue::FlushingPolicy, drop_queue::policy::FlushingPolicyState,
6};
7
8/// A synchronization primitive that blocks until the device has finished
9/// processing all commands submitted before the fence was created.
10pub trait Fence {
11 /// Block the current thread until the signals this fence.
12 fn sync(self);
13}
14
15/// Defers the drop of CPU-side [`Bytes`] allocations until the device has
16/// finished reading them.
17///
18/// # How it works
19///
20/// The device uploads are asynchronous: after you copy bytes into a staging buffer
21/// and enqueue an upload command, the CPU memory must remain valid until the
22/// device is done. `PendingDropQueue` manages this lifetime with a two-phase
23/// approach:
24///
25/// 1. **Stage** – call [`push`](Self::push) to hand over bytes that are
26/// in-flight. They land in the `staged` list.
27/// 2. **Flush** – call [`flush`](Self::flush) to rotate the lists. The
28/// previously staged bytes move to `pending`, a new [`Fence`] is created
29/// to mark the end of the current upload batch, and any bytes that were
30/// *already* pending (i.e. the batch before that) are freed after syncing
31/// the previous fence.
32///
33/// This double-buffer scheme means CPU memory is held for at most two flush
34/// cycles, while avoiding any unnecessary stalls on the hot path.
35///
36/// # Flushing policy
37///
38/// Call [`should_flush`](Self::should_flush) to check whether enough bytes
39/// have accumulated to warrant a flush. You may also flush unconditionally
40/// (e.g. at the end of a frame).
41pub struct PendingDropQueue<E: Fence> {
42 /// Fence signalling that the device has consumed everything in `pending`.
43 fence: Option<E>,
44 /// Bytes from the *previous* flush cycle, kept alive until `event` fires.
45 pending: Vec<Bytes>,
46 /// Bytes queued in the *current* cycle, not yet associated with a fence.
47 staged: Vec<Bytes>,
48 /// The configuration of the queue.
49 policy: FlushingPolicy,
50 /// The current state of the policy.
51 policy_state: FlushingPolicyState,
52}
53
54impl<E: Fence> core::fmt::Debug for PendingDropQueue<E> {
55 fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
56 f.debug_struct("PendingDropQueue")
57 .field("pending", &self.pending)
58 .field("staged", &self.staged)
59 .field("policy", &self.policy)
60 .field("policy_state", &self.policy_state)
61 .finish()
62 }
63}
64
65impl<E: Fence> Default for PendingDropQueue<E> {
66 fn default() -> Self {
67 Self::new(Default::default())
68 }
69}
70
71impl<F: Fence> PendingDropQueue<F> {
72 /// Creates a new `PendingDropQueue`.
73 pub fn new(policy: FlushingPolicy) -> Self {
74 Self {
75 fence: None,
76 pending: Vec::new(),
77 staged: Vec::new(),
78 policy,
79 policy_state: Default::default(),
80 }
81 }
82 /// Enqueue `bytes` to be dropped once the device has finished reading them.
83 ///
84 /// The bytes are added to the current staged batch and will be freed on
85 /// the flush cycle *after* the next call to [`flush`](Self::flush).
86 pub fn push(&mut self, bytes: Bytes) {
87 self.policy_state.register(&bytes);
88 self.staged.push(bytes);
89 }
90
91 /// Returns `true` when the staged batch is large enough to justify a
92 /// flush.
93 pub fn should_flush(&self) -> bool {
94 self.policy_state.should_flush(&self.policy)
95 }
96
97 /// Rotate the double-buffer and free any memory the device is done with.
98 ///
99 /// THE POOL IS ALREADY REUSING THESE BUFFERS; THE BLOCKING IS THE PROOF,
100 /// NOT THE REUSE. `reserve_pinned` hands out POOLED slices, so nothing here
101 /// is allocating fresh host memory per upload -- the only thing this
102 /// function adds is the evidence that the device has finished reading a
103 /// slice before the pool may hand the same one out again. That is why the
104 /// stall it causes is queue-depth serialisation rather than a data
105 /// dependency, and why raising the threshold is not merely "flush less
106 /// often": past the depth where the fence being waited on was recorded long
107 /// enough ago that the device has already passed it, the wait goes to
108 /// approximately zero rather than to a fraction of itself.
109 ///
110 /// The cure that removes the reason for the flush rather than the frequency
111 /// of it is to stop staging per launch at all -- see
112 /// `docs/METADATA_CACHE.md` in mary, which is where the 483 uploads a
113 /// decode step come from.
114 ///
115 /// `factory` is called to produce a [`Fence`]. It should submit (or
116 /// record) a device signal command so that syncing the fence guarantees all
117 /// preceding device work is complete.
118 pub fn flush<Factory: Fn() -> F>(&mut self, factory: Factory) {
119 // Sync the fence from the previous flush and free the bytes it was
120 // protecting.
121 if let Some(event) = self.fence.take() {
122 event.sync();
123 self.pending.clear();
124 }
125
126 // Safety net: if pending is somehow still populated (no prior fence),
127 // stall immediately rather than freeing memory the GPU might still
128 // be reading.
129 if !self.pending.is_empty() {
130 let event = factory();
131 event.sync();
132 self.pending.clear();
133 }
134
135 // The current staged batch becomes the new pending batch.
136 core::mem::swap(&mut self.pending, &mut self.staged);
137
138 // Record a fence so the *next* flush knows when this batch is safe to
139 // free.
140 self.fence = Some(factory());
141 self.policy_state.reset();
142 }
143}
144
145#[cfg(test)]
146mod tests {
147 use super::*;
148 use alloc::vec;
149 use core::cell::Cell;
150
151 // ---------------------------------------------------------------------------
152 // Test helpers
153 // ---------------------------------------------------------------------------
154
155 #[derive(Clone)]
156 struct MockFence<'a> {
157 sync_count: &'a Cell<u32>,
158 }
159
160 impl Fence for MockFence<'_> {
161 fn sync(self) {
162 self.sync_count.set(self.sync_count.get() + 1);
163 }
164 }
165
166 fn make_queue<'a>(
167 sync_count: &'a Cell<u32>,
168 ) -> (
169 PendingDropQueue<MockFence<'a>>,
170 impl Fn() -> MockFence<'a> + 'a,
171 ) {
172 let queue = PendingDropQueue::new(test_policy());
173 let factory = move || MockFence { sync_count };
174 (queue, factory)
175 }
176
177 fn sample_bytes() -> Bytes {
178 Bytes::from_elems(vec![1u8, 2, 3])
179 }
180
181 fn test_policy() -> FlushingPolicy {
182 FlushingPolicy {
183 max_bytes_count: 2048,
184 max_bytes_size: 8,
185 }
186 }
187
188 // ---------------------------------------------------------------------------
189 // push / should_flush
190 // ---------------------------------------------------------------------------
191
192 #[test]
193 fn push_at_count_threshold_triggers_flush_hint() {
194 let sync_count = Cell::new(0u32);
195 let (mut queue, _factory) = make_queue(&sync_count);
196
197 for _ in 0..test_policy().max_bytes_count {
198 queue.push(sample_bytes());
199 }
200
201 assert!(queue.should_flush());
202 }
203
204 #[test]
205 fn push_large_allocation_triggers_flush_via_size_threshold() {
206 let sync_count = Cell::new(0u32);
207 let (mut queue, _factory) = make_queue(&sync_count);
208 let big = Bytes::from_elems(vec![0u8; test_policy().max_bytes_size as usize + 1]);
209
210 queue.push(big);
211
212 assert!(queue.should_flush());
213 }
214
215 // ---------------------------------------------------------------------------
216 // flush – fence / sync behaviour
217 // ---------------------------------------------------------------------------
218
219 #[test]
220 fn first_flush_creates_fence_without_syncing() {
221 let sync_count = Cell::new(0u32);
222 let (mut queue, factory) = make_queue(&sync_count);
223
224 queue.push(sample_bytes());
225 queue.flush(&factory);
226
227 // The fence is created but must not be synced yet — that happens on
228 // the next flush.
229 assert_eq!(
230 sync_count.get(),
231 0,
232 "fence should not be synced on first flush"
233 );
234 }
235
236 #[test]
237 fn second_flush_syncs_fence_from_first_flush() {
238 let sync_count = Cell::new(0u32);
239 let (mut queue, factory) = make_queue(&sync_count);
240
241 queue.push(sample_bytes());
242 queue.flush(&factory); // flush 1 – creates fence A
243
244 queue.push(sample_bytes());
245 queue.flush(&factory); // flush 2 – syncs fence A, creates fence B
246
247 assert_eq!(sync_count.get(), 1, "exactly one sync after two flushes");
248 }
249
250 #[test]
251 fn each_subsequent_flush_syncs_the_previous_fence() {
252 let sync_count = Cell::new(0u32);
253 let (mut queue, factory) = make_queue(&sync_count);
254
255 for _ in 0..10 {
256 queue.push(sample_bytes());
257 queue.flush(&factory);
258 }
259
260 // Each flush except the first syncs the fence from the previous one.
261 assert_eq!(sync_count.get(), 9);
262 }
263
264 // ---------------------------------------------------------------------------
265 // flush – buffer rotation
266 // ---------------------------------------------------------------------------
267
268 #[test]
269 fn staged_is_empty_after_flush() {
270 let sync_count = Cell::new(0u32);
271 let (mut queue, factory) = make_queue(&sync_count);
272
273 for _ in 0..5 {
274 queue.push(sample_bytes());
275 }
276 queue.flush(&factory);
277
278 assert!(queue.staged.is_empty());
279 }
280
281 #[test]
282 fn pending_holds_previously_staged_bytes_after_flush() {
283 let sync_count = Cell::new(0u32);
284 let (mut queue, factory) = make_queue(&sync_count);
285
286 for _ in 0..5 {
287 queue.push(sample_bytes());
288 }
289 queue.flush(&factory);
290
291 assert_eq!(queue.pending.len(), 5);
292 }
293
294 #[test]
295 fn pending_is_replaced_on_second_flush() {
296 let sync_count = Cell::new(0u32);
297 let (mut queue, factory) = make_queue(&sync_count);
298
299 for _ in 0..5 {
300 queue.push(sample_bytes());
301 }
302 queue.flush(&factory); // pending = 5 items
303
304 queue.push(sample_bytes());
305 queue.flush(&factory); // syncs fence → pending cleared, rotated
306
307 // Only the one item staged between the two flushes should be pending.
308 assert_eq!(queue.pending.len(), 1);
309 }
310
311 // ---------------------------------------------------------------------------
312 // flush – policy state reset
313 // ---------------------------------------------------------------------------
314
315 #[test]
316 fn should_flush_resets_after_flush() {
317 let sync_count = Cell::new(0u32);
318 let (mut queue, factory) = make_queue(&sync_count);
319
320 for _ in 0..test_policy().max_bytes_count {
321 queue.push(sample_bytes());
322 }
323 assert!(queue.should_flush());
324
325 queue.flush(&factory);
326
327 assert!(
328 !queue.should_flush(),
329 "policy state should be reset after flush"
330 );
331 }
332
333 // ---------------------------------------------------------------------------
334 // Edge cases
335 // ---------------------------------------------------------------------------
336
337 #[test]
338 fn flush_on_empty_queue_is_safe() {
339 let sync_count = Cell::new(0u32);
340 let (mut queue, factory) = make_queue(&sync_count);
341
342 // Should not panic regardless of how many times it is called.
343 queue.flush(&factory);
344 queue.flush(&factory);
345 queue.flush(&factory);
346 }
347}