Skip to main content

cubecl_runtime/memory_management/drop_queue/
policy.rs

1use cubecl_common::bytes::Bytes;
2
3/// Defines the thresholds that determine when a [`PendingDropQueue`] should be
4/// flushed.
5///
6/// A flush is triggered when **either** limit is exceeded — whichever comes
7/// first. Set a field to `u32::MAX` / `usize::MAX` to effectively disable it.
8#[derive(Debug)]
9pub struct FlushingPolicy {
10    /// Flush when this many allocations have been staged.
11    pub max_bytes_count: u32,
12    /// Flush when the total staged size reaches this many bytes.
13    pub max_bytes_size: u32,
14}
15
16/// The staged-allocation count a flush triggers at, overridable at runtime.
17///
18/// MEASURED AND FOUND NOT TO HELP -- IT HURTS. Recorded here rather than in a
19/// commit message, because a knob that was tried and rejected is exactly the
20/// artifact that evaporates and gets re-proposed six weeks later.
21///
22/// FRAMING: ms per decode step, two-node pipeline (head zgx-16ec layers 0:21,
23/// tail zgx-0d6e layers 21:42), ctx3732, INK_KV=1, INK_GEN=64, 7 reps, arms
24/// interleaved within each rep, byte-identical binaries on both boxes, both
25/// gated idle. Median over 62 warm passes per rep.
26///
27///   64      88.6  88.6  88.2  88.7  88.7  88.2  88.1   median 88.6, spread 0.7%
28///   4096    89.3  92.3  94.2  90.2  97.6  96.7  92.2   median 92.3, spread 9.3%
29///
30/// -4.00% on median tok/s, against ~1.1% resolution at 2 sem for 7 reps. The
31/// bands do not overlap: 4096's BEST rep (89.3) is worse than 64's WORST
32/// (88.7). It is also nine times noisier, which is the more interesting half --
33/// a change that widens the spread that much is doing something structural to
34/// the step, not adding a constant.
35///
36/// What this does NOT establish is that the stalls were free. It establishes
37/// that this particular cure costs more than they are worth. The queue holds
38/// staged buffers for two cycles, so at 4096 it can hold 8192 pinned host
39/// buffers where 64 holds 128, and pinned pages cannot be reclaimed on a box
40/// whose weights are read through the page cache. Separating "the stalls were
41/// free" from "the retention costs more than the stalls" needs the cure that
42/// removes the REASON rather than the frequency: a metadata cache, so that a
43/// launch whose shapes have not changed uploads nothing and stages nothing.
44/// See mary's docs/METADATA_CACHE.md.
45///
46/// The knob stays because it is the instrument that produced this row, and
47/// because 512 is a different question from 4096. It is not a lever to ship.
48///
49/// WHY THIS IS A KNOB AND WHAT IT MEASURES. A flush waits on the fence from the
50/// PREVIOUS flush cycle, so it blocks the launching thread until the device has
51/// drained to a point one batch back. That wait needs no VALUE from the device
52/// -- it exists only so staging buffers two batches old can be freed -- which
53/// makes it queue-depth SERIALISATION rather than a data dependency, and
54/// therefore recoverable. The two are indistinguishable in a profile: both are
55/// `cuEventSynchronize` on the launching thread.
56///
57/// The default of 64 was chosen against uploads of ordinary size. A decode step
58/// of this model stages roughly 483 of them -- one per launch that binds a
59/// ranked tensor, each carrying that tensor's shape and stride list, 19280
60/// bytes for the whole step -- so the count threshold fires about seven times a
61/// step while the 64 MiB size threshold never fires at all. Raising the count
62/// is the A/B that says how much of the blocked time is serialisation, in one
63/// binary and one flag, and it costs only that many small pinned buffers held
64/// one cycle longer.
65fn flush_count_default() -> u32 {
66    static N: std::sync::OnceLock<u32> = std::sync::OnceLock::new();
67    *N.get_or_init(|| {
68        std::env::var("CUBECL_DROP_FLUSH_COUNT")
69            .ok()
70            .and_then(|v| v.parse().ok())
71            .unwrap_or(64)
72    })
73}
74
75impl Default for FlushingPolicy {
76    fn default() -> Self {
77        Self {
78            max_bytes_count: flush_count_default(),
79            max_bytes_size: 64 * 1024 * 1024, // 64 MiB
80        }
81    }
82}
83
84/// Tracks staged allocations and evaluates them against a [`FlushingPolicy`].
85#[derive(Default, Debug)]
86pub(crate) struct FlushingPolicyState {
87    bytes_count: u32,
88    bytes_size: u32,
89}
90
91impl FlushingPolicyState {
92    /// Record a newly staged [`Bytes`] allocation.
93    pub(crate) fn register(&mut self, bytes: &Bytes) {
94        self.bytes_count += 1;
95        self.bytes_size += bytes.len() as u32;
96    }
97
98    /// Reset all counters, typically called after a flush.
99    pub(crate) fn reset(&mut self) {
100        self.bytes_count = 0;
101        self.bytes_size = 0;
102    }
103
104    /// Returns `true` if either threshold in `policy` has been reached.
105    pub(crate) fn should_flush(&self, policy: &FlushingPolicy) -> bool {
106        self.bytes_count >= policy.max_bytes_count || self.bytes_size >= policy.max_bytes_size
107    }
108}
109
110#[cfg(test)]
111mod policy_tests {
112    use std::vec;
113
114    use super::*;
115
116    fn policy() -> FlushingPolicy {
117        FlushingPolicy {
118            max_bytes_count: 4,
119            max_bytes_size: 100,
120        }
121    }
122
123    fn state() -> FlushingPolicyState {
124        FlushingPolicyState {
125            bytes_count: 0,
126            bytes_size: 0,
127        }
128    }
129
130    #[test]
131    fn no_flush_when_below_both_thresholds() {
132        let s = state();
133        assert!(!s.should_flush(&policy()));
134    }
135
136    #[test]
137    fn flush_when_count_threshold_reached() {
138        let mut s = state();
139        for _ in 0..4 {
140            s.register(&Bytes::from_elems(vec![0u8]));
141        }
142        assert!(s.should_flush(&policy()));
143    }
144
145    #[test]
146    fn flush_when_size_threshold_reached() {
147        let mut s = state();
148        s.register(&Bytes::from_elems(vec![0u8; 101]));
149        assert!(s.should_flush(&policy()));
150    }
151
152    #[test]
153    fn flush_triggered_by_whichever_limit_comes_first() {
154        let mut s = state();
155        // Only 2 allocations but already over the size limit.
156        s.register(&Bytes::from_elems(vec![0u8; 60]));
157        s.register(&Bytes::from_elems(vec![0u8; 60]));
158        assert!(s.should_flush(&policy()));
159    }
160
161    #[test]
162    fn reset_clears_state() {
163        let mut s = state();
164        for _ in 0..4 {
165            s.register(&Bytes::from_elems(vec![0u8]));
166        }
167        assert!(s.should_flush(&policy()));
168        s.reset();
169        assert!(!s.should_flush(&policy()));
170    }
171}