Skip to main content

cubecl_runtime/memory_management/
memory_manage.rs

1use super::{
2    MemoryConfiguration, MemoryPoolOptions, MemoryUsage, PoolType,
3    memory_pool::{ExclusiveMemoryPool, MemoryPool, PersistentPool, SlicedPool},
4};
5use crate::{
6    config::{
7        CubeClRuntimeConfig, RuntimeConfig,
8        memory::{MemoryLogLevel, PersistentMemory},
9    },
10    logging::ServerLogger,
11    memory_management::{BytesFormat, memory_pool::Slice},
12    server::IoError,
13    storage::{ComputeStorage, StorageHandle},
14};
15
16use alloc::format;
17use alloc::string::{String, ToString};
18#[cfg(not(exclusive_memory_only))]
19use alloc::vec;
20use alloc::vec::Vec;
21use cubecl_common::{backtrace::BackTrace, stub::Arc};
22use cubecl_ir::MemoryDeviceProperties;
23
24pub use super::memory_pool::{ManagedMemoryBinding, handle::*};
25
26// These are 288 bytes vs 64 bytes. Adding boxing isn't really worth
27// saving the 200 bytes.
28#[allow(clippy::large_enum_variant)]
29enum DynamicPool {
30    Sliced(SlicedPool),
31    Exclusive(ExclusiveMemoryPool),
32}
33
34impl MemoryPool for DynamicPool {
35    fn accept(&self, size: u64) -> bool {
36        match self {
37            DynamicPool::Sliced(pool) => pool.accept(size),
38            DynamicPool::Exclusive(pool) => pool.accept(size),
39        }
40    }
41
42    fn find(&self, binding: &ManagedMemoryBinding) -> Result<&Slice, IoError> {
43        match self {
44            DynamicPool::Sliced(m) => m.find(binding),
45            DynamicPool::Exclusive(m) => m.find(binding),
46        }
47    }
48
49    #[cfg_attr(feature = "tracing", tracing::instrument(level = "trace", skip(self)))]
50    fn try_reserve(&mut self, size: u64) -> Option<ManagedMemoryHandle> {
51        match self {
52            DynamicPool::Sliced(m) => m.try_reserve(size),
53            DynamicPool::Exclusive(m) => m.try_reserve(size),
54        }
55    }
56
57    #[cfg_attr(
58        feature = "tracing",
59        tracing::instrument(level = "trace", skip(self, storage))
60    )]
61    fn alloc<Storage: ComputeStorage>(
62        &mut self,
63        storage: &mut Storage,
64        size: u64,
65    ) -> Result<ManagedMemoryHandle, IoError> {
66        match self {
67            DynamicPool::Sliced(m) => m.alloc(storage, size),
68            DynamicPool::Exclusive(m) => m.alloc(storage, size),
69        }
70    }
71
72    fn get_memory_usage(&self) -> MemoryUsage {
73        match self {
74            DynamicPool::Sliced(m) => m.get_memory_usage(),
75            DynamicPool::Exclusive(m) => m.get_memory_usage(),
76        }
77    }
78
79    fn cleanup<Storage: ComputeStorage>(
80        &mut self,
81        storage: &mut Storage,
82        alloc_nr: u64,
83        explicit: bool,
84    ) {
85        match self {
86            DynamicPool::Sliced(m) => m.cleanup(storage, alloc_nr, explicit),
87            DynamicPool::Exclusive(m) => m.cleanup(storage, alloc_nr, explicit),
88        };
89        storage.flush();
90    }
91
92    fn bind(
93        &mut self,
94        reserved: ManagedMemoryHandle,
95        assigned: ManagedMemoryHandle,
96        cursor: u64,
97    ) -> Result<(), IoError> {
98        match self {
99            DynamicPool::Sliced(m) => m.bind(reserved, assigned, cursor),
100            DynamicPool::Exclusive(m) => m.bind(reserved, assigned, cursor),
101        }
102    }
103}
104
105#[derive(Default, Clone, Copy, Debug)]
106/// The mode of allocation used.
107pub enum MemoryAllocationMode {
108    /// Use the automatic memory management strategy for allocation.
109    #[default]
110    Auto,
111    /// Use a persistent memory management strategy, meaning that all allocations are for data that is
112    /// likely never going to be freed.
113    Persistent,
114}
115
116/// What the capture arena is holding, and how it got there.
117///
118/// `misses` is the load-bearing one: a miss taken while a stream capture is
119/// open is a driver allocation recorded as a graph MEMORY node, which is both a
120/// per-replay cost and an address the next capture will not reproduce.
121#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
122pub struct ArenaStats {
123    /// How many times the arena has been opened.
124    pub generation: u64,
125    /// Reservations served from the arena since the last counter reset.
126    pub served: u64,
127    /// Of those, the ones that needed a fresh allocation from the storage.
128    pub misses: u64,
129    /// Slices the arena owns.
130    pub slices: u64,
131    /// Bytes the arena keeps reserved, whether in use or free.
132    pub bytes_reserved: u64,
133    /// Bytes of that currently held by a live handle.
134    pub bytes_in_use: u64,
135}
136
137/// Whether [`MemoryManagement::reserve`] times itself. Read once: a `getenv`
138/// per reservation would be a cost on the path this exists to measure.
139fn time_reserve() -> bool {
140    static ON: std::sync::OnceLock<bool> = std::sync::OnceLock::new();
141    *ON.get_or_init(|| std::env::var("CUBECL_TIME_RESERVE").as_deref() == Ok("1"))
142}
143
144/// Reserves and keeps track of chunks of memory in the storage, and slices upon these chunks.
145pub struct MemoryManagement<Storage> {
146    name: String,
147    persistent: PersistentPool,
148    /// The capture-scoped arena. See [`MemoryManagement::arena_begin`].
149    arena: PersistentPool,
150    /// The `pool` byte an arena slice's location carries, so `find` and `bind`
151    /// can route back here. One past the persistent pool's.
152    arena_pool_pos: u8,
153    /// While true, EVERY `reserve` is served from `arena` and none from the
154    /// ordinary pools.
155    arena_active: bool,
156    /// Bumped on every [`MemoryManagement::arena_begin`]. Reporting only: what
157    /// decides whether two graphs may share this arena is `arena_sizes`, not
158    /// how many times it was opened.
159    arena_generation: u64,
160    /// Requests the arena served, and the subset it could not satisfy from an
161    /// already-owned slice. A miss inside a capture is exactly a graph memory
162    /// node, so this is the number the warm pass exists to drive to zero.
163    arena_served: u64,
164    arena_misses: u64,
165    /// Every size the arena has served since it was last opened, in order.
166    ///
167    /// This is the region's IDENTITY as far as the arena is concerned. Two
168    /// captures that share an arena share its slices -- that is the point, it
169    /// is what makes their addresses comparable -- and sharing scratch is safe
170    /// exactly when both are captures of the same region, replayed serially.
171    /// The same region issues the same request sequence; a different one does
172    /// not, and a hash over the window is what lets a replay say so.
173    arena_sizes: Vec<u64>,
174    pools: Vec<DynamicPool>,
175    storage: Storage,
176    alloc_reserve_count: u64,
177    /// Wall time spent inside [`MemoryManagement::reserve`], in nanoseconds,
178    /// and how many calls it covers.
179    ///
180    /// Off unless `CUBECL_TIME_RESERVE=1`, because it costs two clock reads a
181    /// reservation and a decode step makes about one per kernel launch. It
182    /// exists because "1869 allocation requests a step" is a COUNT, and a
183    /// count is not a time: at 0.5 us a request that is 0.9 ms of a 37 ms
184    /// enqueue and marginal, at 5 us it is 9.3 ms and it is the largest item
185    /// after the launches themselves. Nobody had any business asserting which
186    /// until it was timed.
187    ///
188    /// FRAMING: this is the RESERVE half only. Returning a slice to the pool
189    /// happens on handle drop, elsewhere, and is not in this number.
190    reserve_nanos: u64,
191    reserve_calls: u64,
192    mode: MemoryAllocationMode,
193    config: PersistentMemory,
194    logger: Arc<ServerLogger>,
195    /// Externally-registered storage handles (e.g. GPU buffers aliasing mmap'd
196    /// host memory), keyed by memory-handle id. Resolved in `get_storage`,
197    /// bypassing the allocation pools entirely — never reserved, reused, or
198    /// reclaimed by the pool machinery. See [`Self::register_external`].
199    external: hashbrown::HashMap<usize, StorageHandle>,
200}
201
202fn generate_bucket_sizes(
203    start_size: u64,
204    end_size: u64,
205    max_buckets: usize,
206    alignment: u64,
207) -> Vec<u64> {
208    let mut buckets = Vec::with_capacity(max_buckets);
209    let log_min = (start_size as f64).ln();
210    let log_max = (end_size as f64).ln();
211    let log_range = log_max - log_min;
212
213    // Pure exponential performed best, but let's try slightly denser in lower-mid range
214    for i in 0..max_buckets {
215        let p = i as f64 / (max_buckets - 1) as f64;
216        // Slight bias toward lower-mid range with less aggressive curve than sigmoid
217        let log_size = log_min + log_range * p;
218        let size = log_size.exp() as u64;
219        let aligned_size = size.next_multiple_of(alignment);
220        buckets.push(aligned_size);
221    }
222
223    buckets.dedup();
224    buckets
225}
226
227const DEALLOC_SCALE_MB: u64 = 1024 * 1024 * 1024;
228const BASE_DEALLOC_PERIOD: u64 = 5000;
229
230/// The options for creating a new [`MemoryManagement`] instance.
231#[derive(Debug)]
232pub struct MemoryManagementOptions {
233    /// The name of the memory management.
234    name: String,
235    /// The [`MemoryAllocationOption`] used by this instance.
236    memory: MemoryAllocationOption,
237}
238
239impl MemoryManagementOptions {
240    /// Creates a new [`MemoryManagementOptions`].
241    pub fn new<S: Into<String>>(name: S) -> Self {
242        Self {
243            name: name.into(),
244            memory: MemoryAllocationOption::FromConfig,
245        }
246    }
247
248    /// Forces the [`MemoryAllocationMode`] during execution to always be the provided one.
249    pub fn mode(mut self, mode: MemoryAllocationMode) -> Self {
250        self.memory = MemoryAllocationOption::Provided(mode);
251        self
252    }
253}
254
255#[derive(Default, Debug)]
256/// Determines which [`MemoryAllocationMode`] is used during allocations.
257enum MemoryAllocationOption {
258    #[default]
259    /// Uses the [`GlobalConfig`] to determine the mode of allocation.
260    FromConfig,
261    /// Use the provided [`MemoryAllocationMode`].
262    Provided(MemoryAllocationMode),
263}
264
265impl<Storage: ComputeStorage> MemoryManagement<Storage> {
266    /// Creates the options from device limits.
267    pub fn from_configuration(
268        storage: Storage,
269        properties: &MemoryDeviceProperties,
270        config: MemoryConfiguration,
271        logger: Arc<ServerLogger>,
272        options: MemoryManagementOptions,
273    ) -> Self {
274        let pool_options = match config {
275            #[cfg(not(exclusive_memory_only))]
276            MemoryConfiguration::SubSlices => {
277                // Round chunk size to be aligned.
278                let memory_alignment = properties.alignment;
279                let max_page = properties.max_page_size;
280                let mut pools = Vec::new();
281
282                const MB: u64 = 1024 * 1024;
283
284                // Add in a pool for allocations that are smaller than the min alignment,
285                // as they can't use offsets at all (on wgpu at least).
286                pools.push(MemoryPoolOptions {
287                    pool_type: PoolType::ExclusivePages { max_alloc_size: 0 },
288                    dealloc_period: None,
289                });
290
291                let mut current = max_page;
292                let mut max_sizes = vec![];
293                let mut page_sizes = vec![];
294                let mut base = pools.len() as u32;
295
296                while current >= 32 * MB {
297                    current /= 4;
298
299                    // Make sure every pool has an aligned size.
300                    current = current.next_multiple_of(memory_alignment);
301
302                    max_sizes.push(current / 2u64.pow(base));
303                    page_sizes.push(current);
304                    base += 1;
305                }
306
307                max_sizes.reverse();
308                page_sizes.reverse();
309
310                for i in 0..max_sizes.len() {
311                    let max = max_sizes[i];
312                    let page_size = page_sizes[i];
313
314                    pools.push(MemoryPoolOptions {
315                        // Creating max slices lower than the chunk size reduces fragmentation.
316                        pool_type: PoolType::SlicedPages {
317                            page_size,
318                            max_slice_size: max,
319                        },
320                        dealloc_period: None,
321                    });
322                }
323
324                // Add pools from big to small.
325                pools.push(MemoryPoolOptions {
326                    pool_type: PoolType::SlicedPages {
327                        page_size: max_page / memory_alignment * memory_alignment,
328                        max_slice_size: max_page / memory_alignment * memory_alignment,
329                    },
330                    dealloc_period: None,
331                });
332                pools
333            }
334            MemoryConfiguration::ExclusivePages => {
335                // Add all bin sizes. Nb: because of alignment some buckets
336                // end up as the same size, so only want unique ones,
337                // but also keep the order, so a BTree will do.
338                const MIN_BUCKET_SIZE: u64 = 1024 * 32;
339                const NUM_POOLS: usize = 24;
340
341                let sizes = generate_bucket_sizes(
342                    MIN_BUCKET_SIZE,
343                    properties.max_page_size,
344                    NUM_POOLS,
345                    properties.alignment,
346                );
347
348                sizes
349                    .iter()
350                    .map(|&size| {
351                        let dealloc_period = (BASE_DEALLOC_PERIOD as f64
352                            * (1.0 + size as f64 / (DEALLOC_SCALE_MB as f64)).round())
353                            as u64;
354
355                        MemoryPoolOptions {
356                            pool_type: PoolType::ExclusivePages {
357                                max_alloc_size: size,
358                            },
359                            dealloc_period: Some(dealloc_period),
360                        }
361                    })
362                    .collect()
363            }
364            MemoryConfiguration::Custom { pool_options } => pool_options,
365        };
366
367        logger.log_memory(
368            |level| !matches!(level, MemoryLogLevel::Disabled),
369            || {
370                let mut msg = String::new();
371                for pool in pool_options.iter() {
372                    msg += &format!("[{}] Using memory pool: \n {pool:?}\n", options.name);
373                }
374                msg
375            },
376        );
377
378        let pools: Vec<_> = pool_options
379            .iter()
380            .enumerate()
381            .map(|(pool_pos, options)| {
382                let pool_pos = pool_pos as u8;
383
384                match options.pool_type {
385                    PoolType::SlicedPages {
386                        page_size,
387                        max_slice_size,
388                    } => DynamicPool::Sliced(SlicedPool::new(
389                        page_size,
390                        max_slice_size,
391                        properties.alignment,
392                        pool_pos,
393                    )),
394                    PoolType::ExclusivePages { max_alloc_size } => {
395                        DynamicPool::Exclusive(ExclusiveMemoryPool::new(
396                            max_alloc_size,
397                            properties.alignment,
398                            options.dealloc_period.unwrap_or(u64::MAX),
399                            pool_pos,
400                        ))
401                    }
402                }
403            })
404            .collect();
405
406        let config = CubeClRuntimeConfig::get().memory.persistent_memory.clone();
407
408        let mode = match options.memory {
409            MemoryAllocationOption::Provided(mode) => mode,
410            MemoryAllocationOption::FromConfig => match config {
411                PersistentMemory::Enabled => MemoryAllocationMode::Auto,
412                PersistentMemory::Disabled => MemoryAllocationMode::Auto,
413                PersistentMemory::Enforced => MemoryAllocationMode::Persistent,
414            },
415        };
416
417        Self {
418            name: options.name,
419            persistent: PersistentPool::new(
420                properties.max_page_size,
421                properties.alignment,
422                pools.len() as u8,
423            ),
424            // The arena accepts any size the storage will accept: a captured
425            // region does not get to choose which of its buffers are small.
426            arena: PersistentPool::new(u64::MAX, properties.alignment, pools.len() as u8 + 1),
427            arena_pool_pos: pools.len() as u8 + 1,
428            arena_active: false,
429            arena_generation: 0,
430            arena_served: 0,
431            arena_misses: 0,
432            arena_sizes: Vec::new(),
433            pools,
434            storage,
435            alloc_reserve_count: 0,
436            reserve_nanos: 0,
437            reserve_calls: 0,
438            mode,
439            config,
440            logger,
441            external: hashbrown::HashMap::new(),
442        }
443    }
444
445    /// Change the mode of allocation.
446    pub fn mode(&mut self, mode: MemoryAllocationMode) {
447        // We override the mode based on the cubecl config.
448        let mode = match self.config {
449            PersistentMemory::Enabled => mode,
450            PersistentMemory::Disabled | PersistentMemory::Enforced => return,
451        };
452
453        self.logger.log_memory(
454            |level| !matches!(level, MemoryLogLevel::Disabled),
455            || {
456                format!(
457                    "[{}] Setting memory allocation mode: from {:?} => {mode:?}",
458                    self.name, self.mode
459                )
460            },
461        );
462        self.mode = mode;
463    }
464
465    /// Route every subsequent `reserve` into the capture arena, and return the
466    /// generation this opening is.
467    ///
468    /// The arena exists because a captured graph records ADDRESSES. Three
469    /// things have to be true at once, and no ordinary pool gives all three:
470    ///
471    /// 1. **Reuse inside the region.** A buffer born inside the captured region
472    ///    and dead inside it must hand its slice back, so the region's peak
473    ///    footprint is its live set and not its allocation count. Without this
474    ///    every intra-region allocation becomes a driver allocation made while
475    ///    the stream is capturing, i.e. a graph MEMORY node.
476    /// 2. **Whole-arena reservation.** Once the region is captured, every slice
477    ///    the arena ever handed out is baked into a node, so NOTHING outside a
478    ///    capture may be given one for as long as the graph can be replayed.
479    ///    The arena is only ever consulted while it is open, and it never
480    ///    returns pages to the storage, so this holds for the whole arena and
481    ///    not just for the slices that happened to be live at capture end.
482    /// 3. **A deterministic base.** Reopening the arena resets no slice and
483    ///    frees nothing; slices are matched by exact effective size in a fixed
484    ///    order. A periodic region therefore issues an identical request
485    ///    sequence against an identical free set and gets identical addresses,
486    ///    which is what makes two captures of the same region comparable and a
487    ///    single graph legitimate for every step.
488    ///
489    /// Note what this is NOT: it is not "hold the buffers a capture touched".
490    /// Holding gets reservation right and reuse wrong -- it is precisely what
491    /// stops (1) -- and holding only the buffers that were live when the region
492    /// opened gets reuse right and reservation wrong for everything born inside
493    /// it. The two properties have to be separated onto different memory, which
494    /// is what the arena is.
495    ///
496    /// Buffers that were live when the arena opened are NOT in it. They belong
497    /// to the ordinary pools, and if one dies inside the region its slice goes
498    /// back to a pool the region is not allocating from -- so it cannot be
499    /// handed to a later node of the same region. That is the whole of the
500    /// intra-region aliasing bug, closed structurally rather than by holding.
501    /// The caller still owes such a buffer a hold for the graph's lifetime, to
502    /// stop the pool handing it out AFTER the capture closes.
503    ///
504    /// TWO GRAPHS MAY SHARE ONE ARENA, and under (3) they positively should:
505    /// that is what makes their recorded addresses comparable, and it is what
506    /// lets a graph captured on one step stand in for the next. They then share
507    /// scratch, which is safe exactly when they are captures of the SAME region
508    /// and are replayed serially. Serial is a property of the one stream they
509    /// replay on; same-region is what [`Self::arena_signature`] is for.
510    pub fn arena_begin(&mut self) -> u64 {
511        self.arena_generation += 1;
512        self.arena_active = true;
513        self.arena_sizes.clear();
514        self.logger.log_memory(
515            |level| !matches!(level, MemoryLogLevel::Disabled),
516            || {
517                format!(
518                    "[{}] Capture arena open, generation {}",
519                    self.name, self.arena_generation
520                )
521            },
522        );
523        self.arena_generation
524    }
525
526    /// Stop routing `reserve` into the arena. The arena keeps everything it
527    /// owns: closing it is the end of allocation from it, never the end of its
528    /// reservation.
529    pub fn arena_end(&mut self) {
530        self.arena_active = false;
531    }
532
533    /// Whether `reserve` is currently being served from the arena.
534    pub fn arena_active(&self) -> bool {
535        self.arena_active
536    }
537
538    /// What the arena is holding, and how it got there.
539    pub fn arena_stats(&self) -> ArenaStats {
540        let usage = self.arena.get_memory_usage();
541        ArenaStats {
542            generation: self.arena_generation,
543            served: self.arena_served,
544            misses: self.arena_misses,
545            slices: self.arena.slice_count() as u64,
546            bytes_reserved: usage.bytes_reserved,
547            bytes_in_use: usage.bytes_in_use,
548        }
549    }
550
551    /// Zero the served/miss counters, so a capture pass can be counted apart
552    /// from the warm pass that preceded it. Does not free or reset anything.
553    pub fn arena_reset_counters(&mut self) {
554        self.arena_served = 0;
555        self.arena_misses = 0;
556    }
557
558    /// Give the arena's free pages back to the storage.
559    ///
560    /// Only legal once no captured graph built against it can still be
561    /// replayed -- every such graph holds pointers into these pages. Slices
562    /// still held by a live handle are kept.
563    pub fn arena_release(&mut self) {
564        assert!(
565            !self.arena_active,
566            "[{}] arena_release while the arena is open",
567            self.name
568        );
569        self.arena
570            .cleanup(&mut self.storage, self.alloc_reserve_count, true);
571    }
572
573    /// Whether this binding's memory came from the arena.
574    pub fn arena_owns(&self, binding: &ManagedMemoryBinding) -> bool {
575        binding.descriptor().location().pool == self.arena_pool_pos
576    }
577
578    /// How many requests the open arena window has served so far. A caller
579    /// brackets a region with two of these to name the window it owns.
580    pub fn arena_mark(&self) -> usize {
581        self.arena_sizes.len()
582    }
583
584    /// A hash of every size served since `from`, in order: the signature of the
585    /// region that allocated them. See [`Self::arena_sizes`].
586    pub fn arena_signature(&self, from: usize) -> u64 {
587        let mut h: u64 = 0xcbf2_9ce4_8422_2325;
588        for size in &self.arena_sizes[from.min(self.arena_sizes.len())..] {
589            h ^= *size;
590            h = h.wrapping_mul(0x1000_0000_01b3);
591        }
592        h
593    }
594
595    fn arena_reserve(&mut self, size: u64) -> Result<ManagedMemoryHandle, IoError> {
596        self.arena_served += 1;
597        self.arena_sizes.push(size);
598        if let Some(handle) = self.arena.try_reserve(size) {
599            return Ok(handle);
600        }
601        // A miss is a real driver allocation. Inside a capture that is a graph
602        // memory node; the warm pass exists so this branch is not taken then.
603        self.arena_misses += 1;
604        self.arena.alloc(&mut self.storage, size)
605    }
606
607    /// Cleanup allocations in pools that are deemed unnecessary.
608    pub fn cleanup(&mut self, explicit: bool) {
609        self.logger.log_memory(
610            |level| !matches!(level, MemoryLogLevel::Disabled) && explicit,
611            || "Manual memory cleanup ...".to_string(),
612        );
613
614        self.persistent
615            .cleanup(&mut self.storage, self.alloc_reserve_count, explicit);
616
617        for pool in self.pools.iter_mut() {
618            pool.cleanup(&mut self.storage, self.alloc_reserve_count, explicit);
619        }
620    }
621
622    /// Returns the storage from the specified binding
623    pub fn get_cursor(&self, binding: ManagedMemoryBinding) -> Result<u64, IoError> {
624        // External storage is registered rather than reserved: it has no
625        // `Slice`, so it has no cursor to read. Returning 0 is not a fallback,
626        // it is the right answer -- the cursor exists to order a read after the
627        // stream operation that last WROTE the memory, and external storage is
628        // pinned `can_mut() == false` precisely so that nothing ever writes it.
629        // There is no producing operation to wait for.
630        //
631        // Only the CUDA backend reaches this: its multi-stream scheduler calls
632        // `handle_cursor` on every binding, while the wgpu backend the seam was
633        // first written for has no cursors at all. That is why an external
634        // handle used to die here with "Memory page 0 doesn't exist".
635        if self.external.contains_key(&binding.descriptor().id.value) {
636            return Ok(0);
637        }
638        let slice = self.find(binding)?;
639        Ok(slice.cursor)
640    }
641
642    /// Returns the storage from the specified binding
643    fn find(&self, binding: ManagedMemoryBinding) -> Result<&Slice, IoError> {
644        let id = binding.descriptor();
645
646        if id.location().pool == self.arena_pool_pos {
647            return self.arena.find(&binding);
648        }
649
650        if id.location().pool >= self.pools.len() as u8 {
651            return self.persistent.find(&binding);
652        }
653
654        let pool =
655            self.pools
656                .get(id.location().pool as usize)
657                .ok_or_else(|| IoError::NotFound {
658                    backtrace: BackTrace::capture(),
659                    reason: format!("Pool {} doesn't exist", id.location().pool).into(),
660                })?;
661
662        let slice = pool.find(&binding)?;
663
664        assert_eq!(slice.handle.descriptor(), binding.descriptor());
665
666        Ok(slice)
667    }
668
669    /// Returns the storage from the specified binding
670    pub fn get_storage(&mut self, binding: ManagedMemoryBinding) -> Result<StorageHandle, IoError> {
671        if let Some(storage) = self.external.get(&binding.descriptor().id.value) {
672            return Ok(storage.clone());
673        }
674        let slice = self.find(binding)?;
675        Ok(slice.storage.clone())
676    }
677
678    /// Register an externally-created [`StorageHandle`] (e.g. a GPU buffer that
679    /// aliases mmap'd host memory) as a tensor-addressable memory handle,
680    /// BYPASSING the allocation pools. The returned handle resolves to `storage`
681    /// via [`Self::get_storage`]/[`Self::get_resource`] but is never reserved,
682    /// reused, sub-sliced, or reclaimed — the caller owns the underlying storage
683    /// lifetime. This is the zero-copy seam for weights-as-tribles: an mmap'd f16
684    /// pile page becomes a GPU tensor with no copy and no allocation.
685    ///
686    /// The returned handle (and every clone of it) permanently reports
687    /// `can_mut() == false`: external storage aliases READ-ONLY host memory,
688    /// so it must never be picked as an in-place kernel destination. Burn's
689    /// elementwise kernels reuse an "owned" input handle as the output buffer
690    /// (`can_mut()` = a handle-count heuristic, `strong_count <= 2`); on a
691    /// read-only mmap the GPU's in-place write is silently dropped by the
692    /// pager and the op returns its INPUT bytes — e.g. voxtral's folded-ear
693    /// load saw `q.mul_scalar(1/sqrt(d))` come back unscaled (2026-07-12,
694    /// caught by the aliased-vs-materialized A/B being token-divergent).
695    /// Pinning two extra handle clones for the registration's life (external
696    /// entries are never removed anyway) pushes every user-facing clone over
697    /// the heuristic's threshold, forcing all consumers to allocate outputs.
698    pub fn register_external(&mut self, storage: StorageHandle) -> ManagedMemoryHandle {
699        let handle = ManagedMemoryHandle::new();
700        core::mem::forget(handle.clone());
701        core::mem::forget(handle.clone());
702        self.external.insert(handle.descriptor().id.value, storage);
703        handle
704    }
705
706    /// Returns the resource from the storage at the specified handle
707    pub fn get_resource(
708        &mut self,
709        binding: ManagedMemoryBinding,
710        offset_start: Option<u64>,
711        offset_end: Option<u64>,
712    ) -> Result<Storage::Resource, IoError> {
713        let handle = self.get_storage(binding)?;
714
715        let handle = match offset_start {
716            Some(offset) => handle.offset_start(offset),
717            None => handle,
718        };
719        let handle = match offset_end {
720            Some(offset) => handle.offset_end(offset),
721            None => handle,
722        };
723        Ok(self.storage().get(&handle))
724    }
725
726    /// Finds a spot in memory for a resource with the given size in bytes, and returns a handle to it
727    #[cfg_attr(feature = "tracing", tracing::instrument(level = "trace", skip(self)))]
728    pub fn reserve(&mut self, size: u64) -> Result<ManagedMemoryHandle, IoError> {
729        if time_reserve() {
730            let t = web_time::Instant::now();
731            let out = self.reserve_inner(size);
732            self.reserve_nanos += t.elapsed().as_nanos() as u64;
733            self.reserve_calls += 1;
734            return out;
735        }
736        self.reserve_inner(size)
737    }
738
739    /// What a reservation costs, as `(calls, nanoseconds)` since the last
740    /// reset. Zero unless `CUBECL_TIME_RESERVE=1`.
741    pub fn reserve_timing(&self) -> (u64, u64) {
742        (self.reserve_calls, self.reserve_nanos)
743    }
744
745    /// Zero the reservation timer, so one pass can be counted apart from the
746    /// startup that preceded it.
747    pub fn reserve_timing_reset(&mut self) {
748        self.reserve_calls = 0;
749        self.reserve_nanos = 0;
750    }
751
752    fn reserve_inner(&mut self, size: u64) -> Result<ManagedMemoryHandle, IoError> {
753        // If this happens every nanosecond, counts overflows after 585 years, so not worth thinking too
754        // hard about overflow here.
755        self.alloc_reserve_count += 1;
756
757        // The arena is not one pool among the others: while it is open it is
758        // the ONLY one, because an address a captured node records has to come
759        // from memory nothing outside the capture can be given.
760        if self.arena_active {
761            return self.arena_reserve(size);
762        }
763
764        if let Some(val) = self.persistent.try_reserve(size) {
765            self.logger.log_memory(
766                |level| matches!(level, MemoryLogLevel::Full),
767                || {
768                    format!(
769                        "[{}] Reserved memory {size} using persistent memory",
770                        self.name
771                    )
772                },
773            );
774            return Ok(val);
775        }
776
777        if matches!(self.mode, MemoryAllocationMode::Persistent) || self.persistent.has_size(size) {
778            let allocated = self.persistent.alloc(&mut self.storage, size);
779
780            self.logger.log_memory(
781                |level| !matches!(level, MemoryLogLevel::Disabled),
782                || {
783                    format!(
784                        "[{}] Allocated a new memory page using persistent memory, \n{}",
785                        self.name, self,
786                    )
787                },
788            );
789            return allocated;
790        }
791
792        self.logger.log_memory(
793            |level| matches!(level, MemoryLogLevel::Full),
794            || {
795                format!(
796                    "[{}] Reserved memory {} using dynamic pool",
797                    self.name,
798                    BytesFormat::new(size)
799                )
800            },
801        );
802
803        // Find first pool that fits this allocation
804        let pool = self
805            .pools
806            .iter_mut()
807            .find(|p| p.accept(size))
808            .ok_or(IoError::BufferTooBig {
809                size,
810                backtrace: BackTrace::capture(),
811            })?;
812
813        if let Some(slice) = pool.try_reserve(size) {
814            return Ok(slice);
815        }
816
817        let allocated = pool.alloc(&mut self.storage, size);
818
819        self.logger.log_memory(
820            |level| matches!(level, MemoryLogLevel::Full),
821            || {
822                format!(
823                    "[{}], Allocated a new memory page, current usage: \n{}",
824                    self.name, self
825                )
826            },
827        );
828
829        allocated
830    }
831
832    /// Fetch the storage used by the memory manager.
833    ///
834    /// # Notes
835    ///
836    /// The storage should probably not be used for allocations since the handles won't be
837    /// compatible with the ones provided by the current trait. Prefer using the
838    /// [alloc](ComputeStorage::alloc) and [dealloc](ComputeStorage::dealloc) functions.
839    ///
840    /// This is useful if you need to time the deallocations based on async computation, or to
841    /// change the mode of storage for different reasons.
842    pub fn storage(&mut self) -> &mut Storage {
843        &mut self.storage
844    }
845
846    /// Get the current memory usage.
847    pub fn memory_usage(&self) -> MemoryUsage {
848        let memory_usage = self.pools.iter().map(|x| x.get_memory_usage()).fold(
849            MemoryUsage {
850                number_allocs: 0,
851                bytes_in_use: 0,
852                bytes_padding: 0,
853                bytes_reserved: 0,
854            },
855            |m1, m2| m1.combine(m2),
856        );
857        memory_usage
858            .combine(self.persistent.get_memory_usage())
859            .combine(self.arena.get_memory_usage())
860    }
861
862    /// Print out a report of the current memory usage.
863    pub fn print_memory_usage(&self) {
864        #[cfg(feature = "std")]
865        log::info!("{}", self.memory_usage());
866    }
867
868    /// Binds the given [handle](HandleId) to a [`MemorySlot`].
869    pub fn bind(
870        &mut self,
871        reserved: ManagedMemoryHandle,
872        assigned: ManagedMemoryHandle,
873        cursor: u64,
874    ) -> Result<(), IoError> {
875        let descriptor = reserved.descriptor();
876
877        if descriptor.location().init == 0 {
878            return Err(IoError::NotFound {
879                backtrace: BackTrace::capture(),
880                reason: "Reserved memory isn't initialized".into(),
881            });
882        }
883
884        let pool_index = descriptor.location().pool as usize;
885        if pool_index == self.arena_pool_pos as usize {
886            return self.arena.bind(reserved, assigned, cursor);
887        }
888        if pool_index >= self.pools.len() {
889            return self.persistent.bind(reserved, assigned, cursor);
890        }
891
892        self.pools
893            .get_mut(pool_index)
894            .map(|p| p.bind(reserved, assigned, cursor))
895            .ok_or_else(|| IoError::NotFound {
896                backtrace: BackTrace::capture(),
897                reason: format!("Memory pool {} doesn't exist", pool_index).into(),
898            })?
899    }
900}
901
902impl<Storage: ComputeStorage> core::fmt::Display for MemoryManagement<Storage> {
903    fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
904        f.write_str("\n# MemoryManagement\n\n")?;
905        f.write_fmt(format_args!(" - name: {:?}\n", self.name))?;
906        f.write_fmt(format_args!("\n## Persistent\n\n{}", self.persistent))?;
907        f.write_fmt(format_args!(
908            "\n## Capture arena (generation {}, {})\n\n{}",
909            self.arena_generation,
910            match self.arena_active {
911                true => "open",
912                false => "closed",
913            },
914            self.arena
915        ))?;
916        f.write_str("\n## Dynamic\n\n")?;
917
918        for pool in self.pools.iter() {
919            match pool {
920                DynamicPool::Sliced(pool) => f.write_fmt(format_args!("{pool}\n"))?,
921                DynamicPool::Exclusive(pool) => f.write_fmt(format_args!("{pool}\n"))?,
922            }
923        }
924        let memory_usage = self.memory_usage();
925        f.write_fmt(format_args!("\n## Summary\n\n{memory_usage}"))?;
926
927        Ok(())
928    }
929}
930
931impl<Storage> core::fmt::Debug for MemoryManagement<Storage> {
932    fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
933        f.write_str(
934            alloc::format!(
935                "DynamicMemoryManagement {:?}",
936                core::any::type_name::<Storage>(),
937            )
938            .as_str(),
939        )
940    }
941}
942
943#[cfg(test)]
944mod tests {
945    use super::*;
946    use crate::{memory_management::MemoryManagement, storage::BytesStorage};
947    use alloc::vec;
948
949    const DUMMY_MEM_PROPS: MemoryDeviceProperties = MemoryDeviceProperties {
950        max_page_size: 128 * 1024 * 1024,
951        alignment: 32,
952    };
953
954    fn options() -> MemoryManagementOptions {
955        MemoryManagementOptions {
956            name: "test".into(),
957            memory: MemoryAllocationOption::FromConfig,
958        }
959    }
960
961    // Test pools with slices.
962    #[test_log::test]
963    #[cfg(not(exclusive_memory_only))]
964    fn test_handle_mutability() {
965        let mut memory_management = MemoryManagement::from_configuration(
966            BytesStorage::default(),
967            &DUMMY_MEM_PROPS,
968            MemoryConfiguration::SubSlices,
969            Arc::new(ServerLogger::default()),
970            options(),
971        );
972        let handle = memory_management.reserve(10).unwrap();
973        let other_ref = handle.clone();
974        assert!(!handle.can_mut(), "Handle can't be mut when multiple ref.");
975        drop(other_ref);
976        assert!(handle.can_mut(), "Handle should be mut when only one ref.");
977    }
978
979    // Test pools with slices.
980    #[test_log::test]
981    #[cfg(not(exclusive_memory_only))]
982    fn test_memory_usage() {
983        let max_page_size = 512;
984
985        let mut memory_management = MemoryManagement::from_configuration(
986            BytesStorage::default(),
987            &DUMMY_MEM_PROPS,
988            MemoryConfiguration::Custom {
989                pool_options: vec![MemoryPoolOptions {
990                    pool_type: PoolType::ExclusivePages {
991                        max_alloc_size: max_page_size,
992                    },
993                    dealloc_period: None,
994                }],
995            },
996            Arc::new(ServerLogger::default()),
997            options(),
998        );
999        let handle = memory_management.reserve(100);
1000        let usage = memory_management.memory_usage();
1001
1002        assert_eq!(usage.bytes_in_use, 100);
1003        assert!(usage.bytes_reserved >= 100 && usage.bytes_reserved <= max_page_size);
1004
1005        // Drop and re-alloc.
1006        drop(handle);
1007        let _handle = memory_management.reserve(100);
1008        let usage_new = memory_management.memory_usage();
1009        assert_eq!(usage, usage_new);
1010    }
1011
1012    #[test_log::test]
1013    fn alloc_two_chunks_on_one_page() {
1014        let page_size = 2048;
1015
1016        let mut memory_management = MemoryManagement::from_configuration(
1017            BytesStorage::default(),
1018            &DUMMY_MEM_PROPS,
1019            MemoryConfiguration::Custom {
1020                pool_options: vec![MemoryPoolOptions {
1021                    pool_type: PoolType::SlicedPages {
1022                        page_size,
1023                        max_slice_size: page_size,
1024                    },
1025                    dealloc_period: None,
1026                }],
1027            },
1028            Arc::new(ServerLogger::default()),
1029            options(),
1030        );
1031
1032        let alloc_size = 512;
1033        let _handle = memory_management.reserve(alloc_size);
1034        let _new_handle = memory_management.reserve(alloc_size);
1035
1036        let usage = memory_management.memory_usage();
1037        assert_eq!(usage.number_allocs, 2);
1038        assert_eq!(usage.bytes_in_use, alloc_size * 2);
1039        assert_eq!(usage.bytes_reserved, page_size);
1040    }
1041
1042    #[test_log::test]
1043    fn alloc_reuses_storage() {
1044        // If no storage is re-used, this will allocate two pages.
1045        let page_size = 512;
1046
1047        let mut memory_management = MemoryManagement::from_configuration(
1048            BytesStorage::default(),
1049            &DUMMY_MEM_PROPS,
1050            MemoryConfiguration::Custom {
1051                pool_options: vec![MemoryPoolOptions {
1052                    pool_type: PoolType::SlicedPages {
1053                        page_size,
1054                        max_slice_size: page_size,
1055                    },
1056                    dealloc_period: None,
1057                }],
1058            },
1059            Arc::new(ServerLogger::default()),
1060            options(),
1061        );
1062
1063        let alloc_size = 512;
1064        let _handle = memory_management.reserve(alloc_size);
1065        drop(_handle);
1066        let _new_handle = memory_management.reserve(alloc_size);
1067
1068        let usage = memory_management.memory_usage();
1069        assert_eq!(usage.number_allocs, 1);
1070        assert_eq!(usage.bytes_in_use, alloc_size);
1071        assert_eq!(usage.bytes_reserved, page_size);
1072    }
1073
1074    #[test_log::test]
1075    fn alloc_allocs_new_storage() {
1076        let page_size = 1024;
1077
1078        let mut memory_management = MemoryManagement::from_configuration(
1079            BytesStorage::default(),
1080            &DUMMY_MEM_PROPS,
1081            MemoryConfiguration::Custom {
1082                pool_options: vec![MemoryPoolOptions {
1083                    pool_type: PoolType::SlicedPages {
1084                        page_size,
1085                        max_slice_size: page_size,
1086                    },
1087                    dealloc_period: None,
1088                }],
1089            },
1090            Arc::new(ServerLogger::default()),
1091            options(),
1092        );
1093
1094        let alloc_size = 768;
1095        let _handle = memory_management.reserve(alloc_size);
1096        let _new_handle = memory_management.reserve(alloc_size);
1097
1098        let usage = memory_management.memory_usage();
1099        assert_eq!(usage.number_allocs, 2);
1100        assert_eq!(usage.bytes_in_use, alloc_size * 2);
1101        assert_eq!(usage.bytes_reserved, page_size * 2);
1102    }
1103
1104    #[test_log::test]
1105    fn alloc_respects_alignment_size() {
1106        let page_size = 500;
1107        let mut memory_management = MemoryManagement::from_configuration(
1108            BytesStorage::default(),
1109            &MemoryDeviceProperties {
1110                max_page_size: page_size,
1111                alignment: 50,
1112            },
1113            MemoryConfiguration::Custom {
1114                pool_options: vec![MemoryPoolOptions {
1115                    pool_type: PoolType::SlicedPages {
1116                        page_size,
1117                        max_slice_size: page_size,
1118                    },
1119                    dealloc_period: None,
1120                }],
1121            },
1122            Arc::new(ServerLogger::default()),
1123            options(),
1124        );
1125        let alloc_size = 40;
1126        let _handle = memory_management.reserve(alloc_size);
1127        let _new_handle = memory_management.reserve(alloc_size);
1128        let usage = memory_management.memory_usage();
1129        // Each slice should be aligned to 50 bytes, so 20 padding bytes.
1130        assert_eq!(usage.bytes_padding, 10 * 2);
1131    }
1132
1133    #[test_log::test]
1134    fn allocs_on_correct_page() {
1135        let sizes = [100, 200, 300, 400];
1136
1137        let pools = sizes
1138            .iter()
1139            .map(|size| MemoryPoolOptions {
1140                pool_type: PoolType::SlicedPages {
1141                    page_size: *size,
1142                    max_slice_size: *size,
1143                },
1144                dealloc_period: None,
1145            })
1146            .collect();
1147        let mut memory_management = MemoryManagement::from_configuration(
1148            BytesStorage::default(),
1149            &MemoryDeviceProperties {
1150                max_page_size: 128 * 1024 * 1024,
1151                alignment: 10,
1152            },
1153            MemoryConfiguration::Custom {
1154                pool_options: pools,
1155            },
1156            Arc::new(ServerLogger::default()),
1157            options(),
1158        );
1159        // Allocate one thing on each page.
1160        let alloc_sizes = [50, 150, 250, 350];
1161        let _handles = alloc_sizes.map(|s| memory_management.reserve(s));
1162
1163        let usage = memory_management.memory_usage();
1164
1165        // Total memory should be size of all pages, and no more.
1166        assert_eq!(usage.bytes_in_use, alloc_sizes.iter().sum::<u64>());
1167        assert!(usage.bytes_reserved >= sizes.iter().sum::<u64>());
1168    }
1169
1170    #[test_log::test]
1171    #[cfg(not(exclusive_memory_only))]
1172    fn allocate_deallocate_reallocate() {
1173        let mut memory_management = MemoryManagement::from_configuration(
1174            BytesStorage::default(),
1175            &MemoryDeviceProperties {
1176                max_page_size: 128 * 1024 * 1024,
1177                alignment: 32,
1178            },
1179            MemoryConfiguration::SubSlices,
1180            Arc::new(ServerLogger::default()),
1181            options(),
1182        );
1183        // Allocate a bunch
1184        let handles: Vec<_> = (0..5)
1185            .map(|i| memory_management.reserve(1000 * (i + 1)))
1186            .collect();
1187        let usage_before = memory_management.memory_usage();
1188        // Deallocate
1189        drop(handles);
1190        // Reallocate
1191        let _new_handles: Vec<_> = (0..5)
1192            .map(|i| memory_management.reserve(1000 * (i + 1)))
1193            .collect();
1194        let usage_after = memory_management.memory_usage();
1195        assert_eq!(usage_before.number_allocs, usage_after.number_allocs);
1196        assert_eq!(usage_before.bytes_in_use, usage_after.bytes_in_use);
1197        // Usage after can actually be _less_ because of defragging.
1198        assert!(usage_before.bytes_reserved >= usage_after.bytes_reserved);
1199    }
1200
1201    #[test_log::test]
1202    #[cfg(not(exclusive_memory_only))]
1203    fn test_fragmentation_resistance() {
1204        let mut memory_management = MemoryManagement::from_configuration(
1205            BytesStorage::default(),
1206            &MemoryDeviceProperties {
1207                max_page_size: 128 * 1024 * 1024,
1208                alignment: 32,
1209            },
1210            MemoryConfiguration::SubSlices,
1211            Arc::new(ServerLogger::default()),
1212            options(),
1213        );
1214        // Allocate a mix of small and large chunks
1215        let sizes = [50, 1000, 100, 5000, 200, 10000, 300];
1216        let handles: Vec<_> = sizes
1217            .iter()
1218            .map(|&size| memory_management.reserve(size).unwrap())
1219            .collect();
1220        let usage_before = memory_management.memory_usage();
1221        // Deallocate every other allocation
1222        for i in (0..handles.len()).step_by(2) {
1223            drop(handles[i].clone());
1224        }
1225        // Reallocate similar sizes
1226        for &size in &sizes[0..sizes.len() / 2] {
1227            memory_management.reserve(size).unwrap();
1228        }
1229        let usage_after = memory_management.memory_usage();
1230        // Check that we haven't increased our memory usage significantly
1231        assert!(usage_after.bytes_reserved <= (usage_before.bytes_reserved as f64 * 1.1) as u64);
1232    }
1233
1234    // Test pools without slices. More or less same as tests above.
1235    #[test_log::test]
1236    fn noslice_test_handle_mutability() {
1237        let mut memory_management = MemoryManagement::from_configuration(
1238            BytesStorage::default(),
1239            &(MemoryDeviceProperties {
1240                max_page_size: 128 * 1024 * 1024,
1241                alignment: 32,
1242            }),
1243            MemoryConfiguration::ExclusivePages,
1244            Arc::new(ServerLogger::default()),
1245            options(),
1246        );
1247        let handle = memory_management.reserve(10).unwrap();
1248        let other_ref = handle.clone();
1249        assert!(!handle.can_mut(), "Handle can't be mut when multiple ref.");
1250        drop(other_ref);
1251        assert!(handle.can_mut(), "Handle should be mut when only one ref.");
1252    }
1253
1254    #[test_log::test]
1255    fn noslice_alloc_two_chunk() {
1256        let mut memory_management = MemoryManagement::from_configuration(
1257            BytesStorage::default(),
1258            &DUMMY_MEM_PROPS,
1259            MemoryConfiguration::Custom {
1260                pool_options: vec![MemoryPoolOptions {
1261                    pool_type: PoolType::ExclusivePages {
1262                        max_alloc_size: 1024,
1263                    },
1264                    dealloc_period: None,
1265                }],
1266            },
1267            Arc::new(ServerLogger::default()),
1268            options(),
1269        );
1270
1271        let alloc_size = 512;
1272        let _handle = memory_management.reserve(alloc_size);
1273        let _new_handle = memory_management.reserve(alloc_size);
1274
1275        let usage = memory_management.memory_usage();
1276        assert_eq!(usage.number_allocs, 2);
1277        assert_eq!(usage.bytes_in_use, alloc_size * 2);
1278        assert!(usage.bytes_reserved >= alloc_size * 2);
1279    }
1280
1281    #[test_log::test]
1282    fn noslice_alloc_reuses_storage() {
1283        // If no storage is re-used, this will allocate two pages.
1284        let mut memory_management = MemoryManagement::from_configuration(
1285            BytesStorage::default(),
1286            &DUMMY_MEM_PROPS,
1287            MemoryConfiguration::Custom {
1288                pool_options: vec![MemoryPoolOptions {
1289                    pool_type: PoolType::ExclusivePages {
1290                        max_alloc_size: 1024,
1291                    },
1292                    dealloc_period: None,
1293                }],
1294            },
1295            Arc::new(ServerLogger::default()),
1296            options(),
1297        );
1298
1299        let alloc_size = 512;
1300        let _handle = memory_management.reserve(alloc_size);
1301        drop(_handle);
1302        let _new_handle = memory_management.reserve(alloc_size);
1303
1304        let usage = memory_management.memory_usage();
1305        assert_eq!(usage.number_allocs, 1);
1306        assert_eq!(usage.bytes_in_use, alloc_size);
1307        assert!(usage.bytes_reserved >= alloc_size);
1308    }
1309
1310    #[test_log::test]
1311    fn noslice_alloc_allocs_new_storage() {
1312        let mut memory_management = MemoryManagement::from_configuration(
1313            BytesStorage::default(),
1314            &DUMMY_MEM_PROPS,
1315            MemoryConfiguration::Custom {
1316                pool_options: vec![MemoryPoolOptions {
1317                    pool_type: PoolType::ExclusivePages {
1318                        max_alloc_size: 1024,
1319                    },
1320                    dealloc_period: None,
1321                }],
1322            },
1323            Arc::new(ServerLogger::default()),
1324            options(),
1325        );
1326
1327        let alloc_size = 768;
1328        let _handle = memory_management.reserve(alloc_size);
1329        let _new_handle = memory_management.reserve(alloc_size);
1330        let usage = memory_management.memory_usage();
1331        assert_eq!(usage.number_allocs, 2);
1332        assert_eq!(usage.bytes_in_use, alloc_size * 2);
1333        assert!(usage.bytes_reserved >= alloc_size * 2);
1334    }
1335
1336    #[test_log::test]
1337    fn noslice_alloc_respects_alignment_size() {
1338        let mut memory_management = MemoryManagement::from_configuration(
1339            BytesStorage::default(),
1340            &MemoryDeviceProperties {
1341                max_page_size: DUMMY_MEM_PROPS.max_page_size,
1342                alignment: 50,
1343            },
1344            MemoryConfiguration::Custom {
1345                pool_options: vec![MemoryPoolOptions {
1346                    pool_type: PoolType::ExclusivePages {
1347                        max_alloc_size: 50 * 20,
1348                    },
1349                    dealloc_period: None,
1350                }],
1351            },
1352            Arc::new(ServerLogger::default()),
1353            options(),
1354        );
1355        let alloc_size = 40;
1356        let _handle = memory_management.reserve(alloc_size);
1357        let _new_handle = memory_management.reserve(alloc_size);
1358        let usage = memory_management.memory_usage();
1359        // Each slice should be aligned to 60 bytes, so 20 padding bytes.
1360        assert_eq!(usage.bytes_padding, 10 * 2);
1361    }
1362
1363    #[test_log::test]
1364    fn noslice_allocs_on_correct_page() {
1365        let pools = [100, 200, 300, 400]
1366            .iter()
1367            .map(|&size| MemoryPoolOptions {
1368                pool_type: PoolType::SlicedPages {
1369                    page_size: size,
1370                    max_slice_size: size,
1371                },
1372                dealloc_period: None,
1373            })
1374            .collect();
1375        let mut memory_management = MemoryManagement::from_configuration(
1376            BytesStorage::default(),
1377            &MemoryDeviceProperties {
1378                max_page_size: DUMMY_MEM_PROPS.max_page_size,
1379                alignment: 10,
1380            },
1381            MemoryConfiguration::Custom {
1382                pool_options: pools,
1383            },
1384            Arc::new(ServerLogger::default()),
1385            options(),
1386        );
1387        // Allocate one thing on each page.
1388        let alloc_sizes = [50, 150, 250, 350];
1389        let _handles = alloc_sizes.map(|s| memory_management.reserve(s));
1390        let usage = memory_management.memory_usage();
1391        // Total memory should be size of all pages, and no more.
1392        assert_eq!(usage.bytes_in_use, alloc_sizes.iter().sum::<u64>());
1393    }
1394
1395    #[test_log::test]
1396    fn noslice_allocate_deallocate_reallocate() {
1397        let mut memory_management = MemoryManagement::from_configuration(
1398            BytesStorage::default(),
1399            &MemoryDeviceProperties {
1400                max_page_size: 128 * 1024 * 1024,
1401                alignment: 32,
1402            },
1403            MemoryConfiguration::ExclusivePages,
1404            Arc::new(ServerLogger::default()),
1405            options(),
1406        );
1407        // Allocate a bunch
1408        let handles: Vec<_> = (0..5)
1409            .map(|i| memory_management.reserve(1000 * (i + 1)))
1410            .collect();
1411        let usage_before = memory_management.memory_usage();
1412        // Deallocate
1413        drop(handles);
1414        // Reallocate
1415        let _new_handles: Vec<_> = (0..5)
1416            .map(|i| memory_management.reserve(1000 * (i + 1)))
1417            .collect();
1418        let usage_after = memory_management.memory_usage();
1419        assert_eq!(usage_before.number_allocs, usage_after.number_allocs);
1420        assert_eq!(usage_before.bytes_in_use, usage_after.bytes_in_use);
1421        assert_eq!(usage_before.bytes_reserved, usage_after.bytes_reserved);
1422    }
1423}