cubecl_runtime/memory_management/base.rs
1use alloc::string::{String, ToString};
2use alloc::vec::Vec;
3
4/// Amount of memory in use by this allocator
5/// and statistics on how much memory is reserved and
6/// wasted in total.
7#[derive(Debug, Clone, Default, PartialEq, Eq)]
8pub struct MemoryUsage {
9 /// The number of allocations currently active.
10 ///
11 /// This is not the number of times an actual allocation happens to create a new memory page,
12 /// but really the number of active slices.
13 pub number_allocs: u64,
14 /// The number of bytes that are currently actually in use.
15 ///
16 /// This doesn't include any padding or other memory that needs to be
17 /// reserved, and is the minimum amount of memory that could possible
18 /// be allocated.
19 pub bytes_in_use: u64,
20 /// The amount of bytes used for padding memory in currently active allocations.
21 pub bytes_padding: u64,
22 /// The total amount of memory reserved on the device.
23 ///
24 /// This will be at least as much as `bytes_in_use` but in practice will
25 /// be higher, as allocations reserve memory for future allocations
26 /// and for padding.
27 pub bytes_reserved: u64,
28}
29
30impl MemoryUsage {
31 /// Calculate the combined memory usage of two reports (summing them).
32 pub fn combine(&self, other: MemoryUsage) -> MemoryUsage {
33 MemoryUsage {
34 number_allocs: self.number_allocs + other.number_allocs,
35 bytes_in_use: self.bytes_in_use + other.bytes_in_use,
36 bytes_padding: self.bytes_padding + other.bytes_padding,
37 bytes_reserved: self.bytes_reserved + other.bytes_reserved,
38 }
39 }
40}
41
42#[derive(new)]
43pub(crate) struct BytesFormat {
44 bytes: u64,
45}
46
47impl core::fmt::Display for BytesFormat {
48 fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
49 let unit = 1000;
50
51 if self.bytes < unit {
52 f.write_fmt(format_args!("{} B", self.bytes))
53 } else {
54 let size = self.bytes as f64;
55 let exp = match size.log(1000.0).floor() as usize {
56 0 => 1,
57 e => e,
58 };
59 let unit_prefix = "KMGTPEZY".as_bytes();
60 f.write_fmt(format_args!(
61 "{:.2} {}B",
62 (size / unit.pow(exp as u32) as f64),
63 unit_prefix[exp - 1] as char,
64 ))
65 }
66 }
67}
68
69fn bytes_format(bytes: u64) -> String {
70 BytesFormat::new(bytes).to_string()
71}
72
73impl core::fmt::Display for MemoryUsage {
74 fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
75 // In the future it'd be nice if MemoryUsage also held some stats about say,
76 // the 5 biggest allocations, to show when you an OOM.
77 let usage_percentage = (self.bytes_in_use as f32 / self.bytes_reserved as f32) * 100.0;
78 let padding_percentage = (self.bytes_padding as f32 / self.bytes_in_use as f32) * 100.0;
79 writeln!(f, "Memory Usage Report:")?;
80 writeln!(f, " Number of allocations: {}", self.number_allocs)?;
81 writeln!(f, " Bytes in use: {}", bytes_format(self.bytes_in_use))?;
82 writeln!(
83 f,
84 " Bytes used for padding: {}",
85 bytes_format(self.bytes_padding)
86 )?;
87 writeln!(
88 f,
89 " Total bytes reserved: {}",
90 bytes_format(self.bytes_reserved)
91 )?;
92 writeln!(f, " Usage efficiency: {usage_percentage:.2}%")?;
93 writeln!(f, " Padding overhead: {padding_percentage:.2}%")
94 }
95}
96
97/// The pool shape a [`MemoryPoolReport`] describes, carrying the pool's
98/// effective configuration (after alignment rounding and page-size shrinking).
99#[derive(Debug, Clone, Copy, PartialEq, Eq)]
100pub enum MemoryPoolKind {
101 /// Allocations are slices carved from shared pages.
102 Sliced {
103 /// The size of each device page.
104 page_size: u64,
105 /// The largest allocation the pool accepts.
106 max_slice_size: u64,
107 /// The pool's byte cap (`None` grows unbounded).
108 max_pool_size: Option<u64>,
109 },
110 /// Every allocation is its own device page.
111 Exclusive {
112 /// The largest allocation the pool accepts.
113 max_alloc_size: u64,
114 },
115 /// One device allocation per reservation, sized to the request, reused by
116 /// exact size and returned to the driver only under memory pressure.
117 /// Wastes only alignment padding, and pays a driver allocation per
118 /// distinct size rather than per page.
119 Direct,
120 /// Exact-fit slices that are reused only by identical size.
121 Persistent,
122}
123
124/// A structured snapshot of one memory pool: its shape, its current usage, and
125/// the high-water marks a memory plan is derived from.
126#[derive(Debug, Clone, PartialEq, Eq)]
127pub struct MemoryPoolReport {
128 /// The pool's shape and effective configuration.
129 pub kind: MemoryPoolKind,
130 /// The pool's current usage.
131 pub usage: MemoryUsage,
132 /// Device allocations (pages) currently held.
133 pub pages: u64,
134 /// The most device allocations ever held at once.
135 ///
136 /// For a sliced pool this is the number a capped layout needs:
137 /// pages are carved by a deterministic first-fit policy, so replaying the
138 /// same allocation stream against `pages_peak * page_size` fits by
139 /// construction.
140 pub pages_peak: u64,
141 /// How many of the current pages have no device backing yet — carved
142 /// under a dry run and never resolved into anything that executes. They
143 /// count toward `pages`/`pages_peak` (the plan is the *reserved* stream)
144 /// while costing no device memory; `pages - pages_unmapped` is the dry
145 /// run's actual footprint in this pool.
146 pub pages_unmapped: u64,
147 /// The largest single allocation this pool ever served, in requested
148 /// (pre-padding) bytes.
149 pub largest_alloc: u64,
150}
151
152/// A per-pool report of one [`MemoryManagement`](super::MemoryManagement)
153/// instance — the read side of a measured memory plan.
154///
155/// The intended cycle: install a growable layout, run the workload once under
156/// a [`DryRun`](crate::dry_run::DryRun) (same allocation stream, no compute),
157/// read this report, and re-install the same layout capped at the observed
158/// `pages_peak`. Padding then comes only from alignment and the first-fit
159/// remainders the dry run already measured.
160///
161/// A tuning pass inside the measured run allocates too, and its scratch counts
162/// toward these marks like anything else. Warming the tune caches in an
163/// earlier pass and rebuilding the pools
164/// ([`install_pools`](super::MemoryManagement::install_pools), which resets the
165/// marks)
166/// before the measured one leaves the peaks to the workload alone.
167#[derive(Debug, Clone, PartialEq, Eq)]
168pub struct MemoryReport {
169 /// One entry per dynamic pool, in allocation-routing order — the same
170 /// order the layout was configured with.
171 pub dynamic: Vec<MemoryPoolReport>,
172 /// The persistent pool (weights, caches; explicit persistent windows).
173 pub persistent: MemoryPoolReport,
174}
175
176/// The managed tensor buffer handle that points to some memory segment.
177/// It should not contain actual data.
178pub trait MemoryHandle<Binding>: Clone + core::fmt::Debug {
179 /// Checks if the underlying memory can be safely mutated.
180 fn can_mut(&self) -> bool;
181 /// Get the binding associated to the current handle.
182 fn binding(self) -> Binding;
183}