cubecl_runtime/memory_management/drop_queue/policy.rs
1use cubecl_common::bytes::Bytes;
2
3/// Defines the thresholds that determine when a [`PendingDropQueue`] should be
4/// flushed.
5///
6/// A flush is triggered when **either** limit is exceeded — whichever comes
7/// first. Set a field to `u32::MAX` / `usize::MAX` to effectively disable it.
8#[derive(Debug)]
9pub struct FlushingPolicy {
10 /// Flush when this many allocations have been staged.
11 pub max_bytes_count: u32,
12 /// Flush when the total staged size reaches this many bytes.
13 pub max_bytes_size: u32,
14}
15
16/// The staged-allocation count a flush triggers at, overridable at runtime.
17///
18/// MEASURED AND FOUND NOT TO HELP -- IT HURTS. Recorded here rather than in a
19/// commit message, because a knob that was tried and rejected is exactly the
20/// artifact that evaporates and gets re-proposed six weeks later.
21///
22/// FRAMING: ms per decode step, two-node pipeline (head zgx-16ec layers 0:21,
23/// tail zgx-0d6e layers 21:42), ctx3732, INK_KV=1, INK_GEN=64, 7 reps, arms
24/// interleaved within each rep, byte-identical binaries on both boxes, both
25/// gated idle. Median over 62 warm passes per rep.
26///
27/// 64 88.6 88.6 88.2 88.7 88.7 88.2 88.1 median 88.6, spread 0.7%
28/// 4096 89.3 92.3 94.2 90.2 97.6 96.7 92.2 median 92.3, spread 9.3%
29///
30/// -4.00% on median tok/s, against ~1.1% resolution at 2 sem for 7 reps. The
31/// bands do not overlap: 4096's BEST rep (89.3) is worse than 64's WORST
32/// (88.7). It is also nine times noisier, which is the more interesting half --
33/// a change that widens the spread that much is doing something structural to
34/// the step, not adding a constant.
35///
36/// What this does NOT establish is that the stalls were free. It establishes
37/// that this particular cure costs more than they are worth. The queue holds
38/// staged buffers for two cycles, so at 4096 it can hold 8192 pinned host
39/// buffers where 64 holds 128, and pinned pages cannot be reclaimed on a box
40/// whose weights are read through the page cache. Separating "the stalls were
41/// free" from "the retention costs more than the stalls" needs the cure that
42/// removes the REASON rather than the frequency: a metadata cache, so that a
43/// launch whose shapes have not changed uploads nothing and stages nothing.
44/// See mary's docs/METADATA_CACHE.md.
45///
46/// The knob stays because it is the instrument that produced this row, and
47/// because 512 is a different question from 4096. It is not a lever to ship.
48///
49/// WHY THIS IS A KNOB AND WHAT IT MEASURES. A flush waits on the fence from the
50/// PREVIOUS flush cycle, so it blocks the launching thread until the device has
51/// drained to a point one batch back. That wait needs no VALUE from the device
52/// -- it exists only so staging buffers two batches old can be freed -- which
53/// makes it queue-depth SERIALISATION rather than a data dependency, and
54/// therefore recoverable. The two are indistinguishable in a profile: both are
55/// `cuEventSynchronize` on the launching thread.
56///
57/// The default of 64 was chosen against uploads of ordinary size. A decode step
58/// of this model stages roughly 483 of them -- one per launch that binds a
59/// ranked tensor, each carrying that tensor's shape and stride list, 19280
60/// bytes for the whole step -- so the count threshold fires about seven times a
61/// step while the 64 MiB size threshold never fires at all. Raising the count
62/// is the A/B that says how much of the blocked time is serialisation, in one
63/// binary and one flag, and it costs only that many small pinned buffers held
64/// one cycle longer.
65fn flush_count_default() -> u32 {
66 static N: std::sync::OnceLock<u32> = std::sync::OnceLock::new();
67 *N.get_or_init(|| {
68 std::env::var("CUBECL_DROP_FLUSH_COUNT")
69 .ok()
70 .and_then(|v| v.parse().ok())
71 .unwrap_or(64)
72 })
73}
74
75impl Default for FlushingPolicy {
76 fn default() -> Self {
77 Self {
78 max_bytes_count: flush_count_default(),
79 max_bytes_size: 64 * 1024 * 1024, // 64 MiB
80 }
81 }
82}
83
84/// Tracks staged allocations and evaluates them against a [`FlushingPolicy`].
85#[derive(Default, Debug)]
86pub(crate) struct FlushingPolicyState {
87 bytes_count: u32,
88 bytes_size: u32,
89}
90
91impl FlushingPolicyState {
92 /// Record a newly staged [`Bytes`] allocation.
93 pub(crate) fn register(&mut self, bytes: &Bytes) {
94 self.bytes_count += 1;
95 self.bytes_size += bytes.len() as u32;
96 }
97
98 /// Reset all counters, typically called after a flush.
99 pub(crate) fn reset(&mut self) {
100 self.bytes_count = 0;
101 self.bytes_size = 0;
102 }
103
104 /// Returns `true` if either threshold in `policy` has been reached.
105 pub(crate) fn should_flush(&self, policy: &FlushingPolicy) -> bool {
106 self.bytes_count >= policy.max_bytes_count || self.bytes_size >= policy.max_bytes_size
107 }
108}
109
110#[cfg(test)]
111mod policy_tests {
112 use std::vec;
113
114 use super::*;
115
116 fn policy() -> FlushingPolicy {
117 FlushingPolicy {
118 max_bytes_count: 4,
119 max_bytes_size: 100,
120 }
121 }
122
123 fn state() -> FlushingPolicyState {
124 FlushingPolicyState {
125 bytes_count: 0,
126 bytes_size: 0,
127 }
128 }
129
130 #[test]
131 fn no_flush_when_below_both_thresholds() {
132 let s = state();
133 assert!(!s.should_flush(&policy()));
134 }
135
136 #[test]
137 fn flush_when_count_threshold_reached() {
138 let mut s = state();
139 for _ in 0..4 {
140 s.register(&Bytes::from_elems(vec![0u8]));
141 }
142 assert!(s.should_flush(&policy()));
143 }
144
145 #[test]
146 fn flush_when_size_threshold_reached() {
147 let mut s = state();
148 s.register(&Bytes::from_elems(vec![0u8; 101]));
149 assert!(s.should_flush(&policy()));
150 }
151
152 #[test]
153 fn flush_triggered_by_whichever_limit_comes_first() {
154 let mut s = state();
155 // Only 2 allocations but already over the size limit.
156 s.register(&Bytes::from_elems(vec![0u8; 60]));
157 s.register(&Bytes::from_elems(vec![0u8; 60]));
158 assert!(s.should_flush(&policy()));
159 }
160
161 #[test]
162 fn reset_clears_state() {
163 let mut s = state();
164 for _ in 0..4 {
165 s.register(&Bytes::from_elems(vec![0u8]));
166 }
167 assert!(s.should_flush(&policy()));
168 s.reset();
169 assert!(!s.should_flush(&policy()));
170 }
171}