1use super::{
2 MemoryConfiguration, MemoryPoolOptions, MemoryUsage, PoolType,
3 memory_pool::{ExclusiveMemoryPool, MemoryPool, PersistentPool, SlicedPool},
4};
5use crate::{
6 config::{
7 CubeClRuntimeConfig, RuntimeConfig,
8 memory::{MemoryLogLevel, PersistentMemory},
9 },
10 logging::ServerLogger,
11 memory_management::{BytesFormat, memory_pool::Slice},
12 server::IoError,
13 storage::{ComputeStorage, StorageHandle},
14};
15
16use alloc::format;
17use alloc::string::{String, ToString};
18#[cfg(not(exclusive_memory_only))]
19use alloc::vec;
20use alloc::vec::Vec;
21use cubecl_common::{backtrace::BackTrace, stub::Arc};
22use cubecl_ir::MemoryDeviceProperties;
23
24pub use super::memory_pool::{ManagedMemoryBinding, handle::*};
25
26#[allow(clippy::large_enum_variant)]
29enum DynamicPool {
30 Sliced(SlicedPool),
31 Exclusive(ExclusiveMemoryPool),
32}
33
34impl MemoryPool for DynamicPool {
35 fn accept(&self, size: u64) -> bool {
36 match self {
37 DynamicPool::Sliced(pool) => pool.accept(size),
38 DynamicPool::Exclusive(pool) => pool.accept(size),
39 }
40 }
41
42 fn find(&self, binding: &ManagedMemoryBinding) -> Result<&Slice, IoError> {
43 match self {
44 DynamicPool::Sliced(m) => m.find(binding),
45 DynamicPool::Exclusive(m) => m.find(binding),
46 }
47 }
48
49 #[cfg_attr(feature = "tracing", tracing::instrument(level = "trace", skip(self)))]
50 fn try_reserve(&mut self, size: u64) -> Option<ManagedMemoryHandle> {
51 match self {
52 DynamicPool::Sliced(m) => m.try_reserve(size),
53 DynamicPool::Exclusive(m) => m.try_reserve(size),
54 }
55 }
56
57 #[cfg_attr(
58 feature = "tracing",
59 tracing::instrument(level = "trace", skip(self, storage))
60 )]
61 fn alloc<Storage: ComputeStorage>(
62 &mut self,
63 storage: &mut Storage,
64 size: u64,
65 ) -> Result<ManagedMemoryHandle, IoError> {
66 match self {
67 DynamicPool::Sliced(m) => m.alloc(storage, size),
68 DynamicPool::Exclusive(m) => m.alloc(storage, size),
69 }
70 }
71
72 fn get_memory_usage(&self) -> MemoryUsage {
73 match self {
74 DynamicPool::Sliced(m) => m.get_memory_usage(),
75 DynamicPool::Exclusive(m) => m.get_memory_usage(),
76 }
77 }
78
79 fn cleanup<Storage: ComputeStorage>(
80 &mut self,
81 storage: &mut Storage,
82 alloc_nr: u64,
83 explicit: bool,
84 ) {
85 match self {
86 DynamicPool::Sliced(m) => m.cleanup(storage, alloc_nr, explicit),
87 DynamicPool::Exclusive(m) => m.cleanup(storage, alloc_nr, explicit),
88 };
89 storage.flush();
90 }
91
92 fn bind(
93 &mut self,
94 reserved: ManagedMemoryHandle,
95 assigned: ManagedMemoryHandle,
96 cursor: u64,
97 ) -> Result<(), IoError> {
98 match self {
99 DynamicPool::Sliced(m) => m.bind(reserved, assigned, cursor),
100 DynamicPool::Exclusive(m) => m.bind(reserved, assigned, cursor),
101 }
102 }
103}
104
105#[derive(Default, Clone, Copy, Debug)]
106pub enum MemoryAllocationMode {
108 #[default]
110 Auto,
111 Persistent,
114}
115
116#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
122pub struct ArenaStats {
123 pub generation: u64,
125 pub served: u64,
127 pub misses: u64,
129 pub slices: u64,
131 pub bytes_reserved: u64,
133 pub bytes_in_use: u64,
135}
136
137fn time_reserve() -> bool {
140 static ON: std::sync::OnceLock<bool> = std::sync::OnceLock::new();
141 *ON.get_or_init(|| std::env::var("CUBECL_TIME_RESERVE").as_deref() == Ok("1"))
142}
143
144pub struct MemoryManagement<Storage> {
146 name: String,
147 persistent: PersistentPool,
148 arena: PersistentPool,
150 arena_pool_pos: u8,
153 arena_active: bool,
156 arena_generation: u64,
160 arena_served: u64,
164 arena_misses: u64,
165 arena_sizes: Vec<u64>,
174 pools: Vec<DynamicPool>,
175 storage: Storage,
176 alloc_reserve_count: u64,
177 reserve_nanos: u64,
191 reserve_calls: u64,
192 mode: MemoryAllocationMode,
193 config: PersistentMemory,
194 logger: Arc<ServerLogger>,
195 external: hashbrown::HashMap<usize, StorageHandle>,
200}
201
202fn generate_bucket_sizes(
203 start_size: u64,
204 end_size: u64,
205 max_buckets: usize,
206 alignment: u64,
207) -> Vec<u64> {
208 let mut buckets = Vec::with_capacity(max_buckets);
209 let log_min = (start_size as f64).ln();
210 let log_max = (end_size as f64).ln();
211 let log_range = log_max - log_min;
212
213 for i in 0..max_buckets {
215 let p = i as f64 / (max_buckets - 1) as f64;
216 let log_size = log_min + log_range * p;
218 let size = log_size.exp() as u64;
219 let aligned_size = size.next_multiple_of(alignment);
220 buckets.push(aligned_size);
221 }
222
223 buckets.dedup();
224 buckets
225}
226
227const DEALLOC_SCALE_MB: u64 = 1024 * 1024 * 1024;
228const BASE_DEALLOC_PERIOD: u64 = 5000;
229
230#[derive(Debug)]
232pub struct MemoryManagementOptions {
233 name: String,
235 memory: MemoryAllocationOption,
237}
238
239impl MemoryManagementOptions {
240 pub fn new<S: Into<String>>(name: S) -> Self {
242 Self {
243 name: name.into(),
244 memory: MemoryAllocationOption::FromConfig,
245 }
246 }
247
248 pub fn mode(mut self, mode: MemoryAllocationMode) -> Self {
250 self.memory = MemoryAllocationOption::Provided(mode);
251 self
252 }
253}
254
255#[derive(Default, Debug)]
256enum MemoryAllocationOption {
258 #[default]
259 FromConfig,
261 Provided(MemoryAllocationMode),
263}
264
265impl<Storage: ComputeStorage> MemoryManagement<Storage> {
266 pub fn from_configuration(
268 storage: Storage,
269 properties: &MemoryDeviceProperties,
270 config: MemoryConfiguration,
271 logger: Arc<ServerLogger>,
272 options: MemoryManagementOptions,
273 ) -> Self {
274 let pool_options = match config {
275 #[cfg(not(exclusive_memory_only))]
276 MemoryConfiguration::SubSlices => {
277 let memory_alignment = properties.alignment;
279 let max_page = properties.max_page_size;
280 let mut pools = Vec::new();
281
282 const MB: u64 = 1024 * 1024;
283
284 pools.push(MemoryPoolOptions {
287 pool_type: PoolType::ExclusivePages { max_alloc_size: 0 },
288 dealloc_period: None,
289 });
290
291 let mut current = max_page;
292 let mut max_sizes = vec![];
293 let mut page_sizes = vec![];
294 let mut base = pools.len() as u32;
295
296 while current >= 32 * MB {
297 current /= 4;
298
299 current = current.next_multiple_of(memory_alignment);
301
302 max_sizes.push(current / 2u64.pow(base));
303 page_sizes.push(current);
304 base += 1;
305 }
306
307 max_sizes.reverse();
308 page_sizes.reverse();
309
310 for i in 0..max_sizes.len() {
311 let max = max_sizes[i];
312 let page_size = page_sizes[i];
313
314 pools.push(MemoryPoolOptions {
315 pool_type: PoolType::SlicedPages {
317 page_size,
318 max_slice_size: max,
319 },
320 dealloc_period: None,
321 });
322 }
323
324 pools.push(MemoryPoolOptions {
326 pool_type: PoolType::SlicedPages {
327 page_size: max_page / memory_alignment * memory_alignment,
328 max_slice_size: max_page / memory_alignment * memory_alignment,
329 },
330 dealloc_period: None,
331 });
332 pools
333 }
334 MemoryConfiguration::ExclusivePages => {
335 const MIN_BUCKET_SIZE: u64 = 1024 * 32;
339 const NUM_POOLS: usize = 24;
340
341 let sizes = generate_bucket_sizes(
342 MIN_BUCKET_SIZE,
343 properties.max_page_size,
344 NUM_POOLS,
345 properties.alignment,
346 );
347
348 sizes
349 .iter()
350 .map(|&size| {
351 let dealloc_period = (BASE_DEALLOC_PERIOD as f64
352 * (1.0 + size as f64 / (DEALLOC_SCALE_MB as f64)).round())
353 as u64;
354
355 MemoryPoolOptions {
356 pool_type: PoolType::ExclusivePages {
357 max_alloc_size: size,
358 },
359 dealloc_period: Some(dealloc_period),
360 }
361 })
362 .collect()
363 }
364 MemoryConfiguration::Custom { pool_options } => pool_options,
365 };
366
367 logger.log_memory(
368 |level| !matches!(level, MemoryLogLevel::Disabled),
369 || {
370 let mut msg = String::new();
371 for pool in pool_options.iter() {
372 msg += &format!("[{}] Using memory pool: \n {pool:?}\n", options.name);
373 }
374 msg
375 },
376 );
377
378 let pools: Vec<_> = pool_options
379 .iter()
380 .enumerate()
381 .map(|(pool_pos, options)| {
382 let pool_pos = pool_pos as u8;
383
384 match options.pool_type {
385 PoolType::SlicedPages {
386 page_size,
387 max_slice_size,
388 } => DynamicPool::Sliced(SlicedPool::new(
389 page_size,
390 max_slice_size,
391 properties.alignment,
392 pool_pos,
393 )),
394 PoolType::ExclusivePages { max_alloc_size } => {
395 DynamicPool::Exclusive(ExclusiveMemoryPool::new(
396 max_alloc_size,
397 properties.alignment,
398 options.dealloc_period.unwrap_or(u64::MAX),
399 pool_pos,
400 ))
401 }
402 }
403 })
404 .collect();
405
406 let config = CubeClRuntimeConfig::get().memory.persistent_memory.clone();
407
408 let mode = match options.memory {
409 MemoryAllocationOption::Provided(mode) => mode,
410 MemoryAllocationOption::FromConfig => match config {
411 PersistentMemory::Enabled => MemoryAllocationMode::Auto,
412 PersistentMemory::Disabled => MemoryAllocationMode::Auto,
413 PersistentMemory::Enforced => MemoryAllocationMode::Persistent,
414 },
415 };
416
417 Self {
418 name: options.name,
419 persistent: PersistentPool::new(
420 properties.max_page_size,
421 properties.alignment,
422 pools.len() as u8,
423 ),
424 arena: PersistentPool::new(u64::MAX, properties.alignment, pools.len() as u8 + 1),
427 arena_pool_pos: pools.len() as u8 + 1,
428 arena_active: false,
429 arena_generation: 0,
430 arena_served: 0,
431 arena_misses: 0,
432 arena_sizes: Vec::new(),
433 pools,
434 storage,
435 alloc_reserve_count: 0,
436 reserve_nanos: 0,
437 reserve_calls: 0,
438 mode,
439 config,
440 logger,
441 external: hashbrown::HashMap::new(),
442 }
443 }
444
445 pub fn mode(&mut self, mode: MemoryAllocationMode) {
447 let mode = match self.config {
449 PersistentMemory::Enabled => mode,
450 PersistentMemory::Disabled | PersistentMemory::Enforced => return,
451 };
452
453 self.logger.log_memory(
454 |level| !matches!(level, MemoryLogLevel::Disabled),
455 || {
456 format!(
457 "[{}] Setting memory allocation mode: from {:?} => {mode:?}",
458 self.name, self.mode
459 )
460 },
461 );
462 self.mode = mode;
463 }
464
465 pub fn arena_begin(&mut self) -> u64 {
511 self.arena_generation += 1;
512 self.arena_active = true;
513 self.arena_sizes.clear();
514 self.logger.log_memory(
515 |level| !matches!(level, MemoryLogLevel::Disabled),
516 || {
517 format!(
518 "[{}] Capture arena open, generation {}",
519 self.name, self.arena_generation
520 )
521 },
522 );
523 self.arena_generation
524 }
525
526 pub fn arena_end(&mut self) {
530 self.arena_active = false;
531 }
532
533 pub fn arena_active(&self) -> bool {
535 self.arena_active
536 }
537
538 pub fn arena_stats(&self) -> ArenaStats {
540 let usage = self.arena.get_memory_usage();
541 ArenaStats {
542 generation: self.arena_generation,
543 served: self.arena_served,
544 misses: self.arena_misses,
545 slices: self.arena.slice_count() as u64,
546 bytes_reserved: usage.bytes_reserved,
547 bytes_in_use: usage.bytes_in_use,
548 }
549 }
550
551 pub fn arena_reset_counters(&mut self) {
554 self.arena_served = 0;
555 self.arena_misses = 0;
556 }
557
558 pub fn arena_release(&mut self) {
564 assert!(
565 !self.arena_active,
566 "[{}] arena_release while the arena is open",
567 self.name
568 );
569 self.arena
570 .cleanup(&mut self.storage, self.alloc_reserve_count, true);
571 }
572
573 pub fn arena_owns(&self, binding: &ManagedMemoryBinding) -> bool {
575 binding.descriptor().location().pool == self.arena_pool_pos
576 }
577
578 pub fn arena_mark(&self) -> usize {
581 self.arena_sizes.len()
582 }
583
584 pub fn arena_signature(&self, from: usize) -> u64 {
587 let mut h: u64 = 0xcbf2_9ce4_8422_2325;
588 for size in &self.arena_sizes[from.min(self.arena_sizes.len())..] {
589 h ^= *size;
590 h = h.wrapping_mul(0x1000_0000_01b3);
591 }
592 h
593 }
594
595 fn arena_reserve(&mut self, size: u64) -> Result<ManagedMemoryHandle, IoError> {
596 self.arena_served += 1;
597 self.arena_sizes.push(size);
598 if let Some(handle) = self.arena.try_reserve(size) {
599 return Ok(handle);
600 }
601 self.arena_misses += 1;
604 self.arena.alloc(&mut self.storage, size)
605 }
606
607 pub fn cleanup(&mut self, explicit: bool) {
609 self.logger.log_memory(
610 |level| !matches!(level, MemoryLogLevel::Disabled) && explicit,
611 || "Manual memory cleanup ...".to_string(),
612 );
613
614 self.persistent
615 .cleanup(&mut self.storage, self.alloc_reserve_count, explicit);
616
617 for pool in self.pools.iter_mut() {
618 pool.cleanup(&mut self.storage, self.alloc_reserve_count, explicit);
619 }
620 }
621
622 pub fn get_cursor(&self, binding: ManagedMemoryBinding) -> Result<u64, IoError> {
624 if self.external.contains_key(&binding.descriptor().id.value) {
636 return Ok(0);
637 }
638 let slice = self.find(binding)?;
639 Ok(slice.cursor)
640 }
641
642 fn find(&self, binding: ManagedMemoryBinding) -> Result<&Slice, IoError> {
644 let id = binding.descriptor();
645
646 if id.location().pool == self.arena_pool_pos {
647 return self.arena.find(&binding);
648 }
649
650 if id.location().pool >= self.pools.len() as u8 {
651 return self.persistent.find(&binding);
652 }
653
654 let pool =
655 self.pools
656 .get(id.location().pool as usize)
657 .ok_or_else(|| IoError::NotFound {
658 backtrace: BackTrace::capture(),
659 reason: format!("Pool {} doesn't exist", id.location().pool).into(),
660 })?;
661
662 let slice = pool.find(&binding)?;
663
664 assert_eq!(slice.handle.descriptor(), binding.descriptor());
665
666 Ok(slice)
667 }
668
669 pub fn get_storage(&mut self, binding: ManagedMemoryBinding) -> Result<StorageHandle, IoError> {
671 if let Some(storage) = self.external.get(&binding.descriptor().id.value) {
672 return Ok(storage.clone());
673 }
674 let slice = self.find(binding)?;
675 Ok(slice.storage.clone())
676 }
677
678 pub fn register_external(&mut self, storage: StorageHandle) -> ManagedMemoryHandle {
699 let handle = ManagedMemoryHandle::new();
700 core::mem::forget(handle.clone());
701 core::mem::forget(handle.clone());
702 self.external.insert(handle.descriptor().id.value, storage);
703 handle
704 }
705
706 pub fn get_resource(
708 &mut self,
709 binding: ManagedMemoryBinding,
710 offset_start: Option<u64>,
711 offset_end: Option<u64>,
712 ) -> Result<Storage::Resource, IoError> {
713 let handle = self.get_storage(binding)?;
714
715 let handle = match offset_start {
716 Some(offset) => handle.offset_start(offset),
717 None => handle,
718 };
719 let handle = match offset_end {
720 Some(offset) => handle.offset_end(offset),
721 None => handle,
722 };
723 Ok(self.storage().get(&handle))
724 }
725
726 #[cfg_attr(feature = "tracing", tracing::instrument(level = "trace", skip(self)))]
728 pub fn reserve(&mut self, size: u64) -> Result<ManagedMemoryHandle, IoError> {
729 if time_reserve() {
730 let t = web_time::Instant::now();
731 let out = self.reserve_inner(size);
732 self.reserve_nanos += t.elapsed().as_nanos() as u64;
733 self.reserve_calls += 1;
734 return out;
735 }
736 self.reserve_inner(size)
737 }
738
739 pub fn reserve_timing(&self) -> (u64, u64) {
742 (self.reserve_calls, self.reserve_nanos)
743 }
744
745 pub fn reserve_timing_reset(&mut self) {
748 self.reserve_calls = 0;
749 self.reserve_nanos = 0;
750 }
751
752 fn reserve_inner(&mut self, size: u64) -> Result<ManagedMemoryHandle, IoError> {
753 self.alloc_reserve_count += 1;
756
757 if self.arena_active {
761 return self.arena_reserve(size);
762 }
763
764 if let Some(val) = self.persistent.try_reserve(size) {
765 self.logger.log_memory(
766 |level| matches!(level, MemoryLogLevel::Full),
767 || {
768 format!(
769 "[{}] Reserved memory {size} using persistent memory",
770 self.name
771 )
772 },
773 );
774 return Ok(val);
775 }
776
777 if matches!(self.mode, MemoryAllocationMode::Persistent) || self.persistent.has_size(size) {
778 let allocated = self.persistent.alloc(&mut self.storage, size);
779
780 self.logger.log_memory(
781 |level| !matches!(level, MemoryLogLevel::Disabled),
782 || {
783 format!(
784 "[{}] Allocated a new memory page using persistent memory, \n{}",
785 self.name, self,
786 )
787 },
788 );
789 return allocated;
790 }
791
792 self.logger.log_memory(
793 |level| matches!(level, MemoryLogLevel::Full),
794 || {
795 format!(
796 "[{}] Reserved memory {} using dynamic pool",
797 self.name,
798 BytesFormat::new(size)
799 )
800 },
801 );
802
803 let pool = self
805 .pools
806 .iter_mut()
807 .find(|p| p.accept(size))
808 .ok_or(IoError::BufferTooBig {
809 size,
810 backtrace: BackTrace::capture(),
811 })?;
812
813 if let Some(slice) = pool.try_reserve(size) {
814 return Ok(slice);
815 }
816
817 let allocated = pool.alloc(&mut self.storage, size);
818
819 self.logger.log_memory(
820 |level| matches!(level, MemoryLogLevel::Full),
821 || {
822 format!(
823 "[{}], Allocated a new memory page, current usage: \n{}",
824 self.name, self
825 )
826 },
827 );
828
829 allocated
830 }
831
832 pub fn storage(&mut self) -> &mut Storage {
843 &mut self.storage
844 }
845
846 pub fn memory_usage(&self) -> MemoryUsage {
848 let memory_usage = self.pools.iter().map(|x| x.get_memory_usage()).fold(
849 MemoryUsage {
850 number_allocs: 0,
851 bytes_in_use: 0,
852 bytes_padding: 0,
853 bytes_reserved: 0,
854 },
855 |m1, m2| m1.combine(m2),
856 );
857 memory_usage
858 .combine(self.persistent.get_memory_usage())
859 .combine(self.arena.get_memory_usage())
860 }
861
862 pub fn print_memory_usage(&self) {
864 #[cfg(feature = "std")]
865 log::info!("{}", self.memory_usage());
866 }
867
868 pub fn bind(
870 &mut self,
871 reserved: ManagedMemoryHandle,
872 assigned: ManagedMemoryHandle,
873 cursor: u64,
874 ) -> Result<(), IoError> {
875 let descriptor = reserved.descriptor();
876
877 if descriptor.location().init == 0 {
878 return Err(IoError::NotFound {
879 backtrace: BackTrace::capture(),
880 reason: "Reserved memory isn't initialized".into(),
881 });
882 }
883
884 let pool_index = descriptor.location().pool as usize;
885 if pool_index == self.arena_pool_pos as usize {
886 return self.arena.bind(reserved, assigned, cursor);
887 }
888 if pool_index >= self.pools.len() {
889 return self.persistent.bind(reserved, assigned, cursor);
890 }
891
892 self.pools
893 .get_mut(pool_index)
894 .map(|p| p.bind(reserved, assigned, cursor))
895 .ok_or_else(|| IoError::NotFound {
896 backtrace: BackTrace::capture(),
897 reason: format!("Memory pool {} doesn't exist", pool_index).into(),
898 })?
899 }
900}
901
902impl<Storage: ComputeStorage> core::fmt::Display for MemoryManagement<Storage> {
903 fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
904 f.write_str("\n# MemoryManagement\n\n")?;
905 f.write_fmt(format_args!(" - name: {:?}\n", self.name))?;
906 f.write_fmt(format_args!("\n## Persistent\n\n{}", self.persistent))?;
907 f.write_fmt(format_args!(
908 "\n## Capture arena (generation {}, {})\n\n{}",
909 self.arena_generation,
910 match self.arena_active {
911 true => "open",
912 false => "closed",
913 },
914 self.arena
915 ))?;
916 f.write_str("\n## Dynamic\n\n")?;
917
918 for pool in self.pools.iter() {
919 match pool {
920 DynamicPool::Sliced(pool) => f.write_fmt(format_args!("{pool}\n"))?,
921 DynamicPool::Exclusive(pool) => f.write_fmt(format_args!("{pool}\n"))?,
922 }
923 }
924 let memory_usage = self.memory_usage();
925 f.write_fmt(format_args!("\n## Summary\n\n{memory_usage}"))?;
926
927 Ok(())
928 }
929}
930
931impl<Storage> core::fmt::Debug for MemoryManagement<Storage> {
932 fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
933 f.write_str(
934 alloc::format!(
935 "DynamicMemoryManagement {:?}",
936 core::any::type_name::<Storage>(),
937 )
938 .as_str(),
939 )
940 }
941}
942
943#[cfg(test)]
944mod tests {
945 use super::*;
946 use crate::{memory_management::MemoryManagement, storage::BytesStorage};
947 use alloc::vec;
948
949 const DUMMY_MEM_PROPS: MemoryDeviceProperties = MemoryDeviceProperties {
950 max_page_size: 128 * 1024 * 1024,
951 alignment: 32,
952 };
953
954 fn options() -> MemoryManagementOptions {
955 MemoryManagementOptions {
956 name: "test".into(),
957 memory: MemoryAllocationOption::FromConfig,
958 }
959 }
960
961 #[test_log::test]
963 #[cfg(not(exclusive_memory_only))]
964 fn test_handle_mutability() {
965 let mut memory_management = MemoryManagement::from_configuration(
966 BytesStorage::default(),
967 &DUMMY_MEM_PROPS,
968 MemoryConfiguration::SubSlices,
969 Arc::new(ServerLogger::default()),
970 options(),
971 );
972 let handle = memory_management.reserve(10).unwrap();
973 let other_ref = handle.clone();
974 assert!(!handle.can_mut(), "Handle can't be mut when multiple ref.");
975 drop(other_ref);
976 assert!(handle.can_mut(), "Handle should be mut when only one ref.");
977 }
978
979 #[test_log::test]
981 #[cfg(not(exclusive_memory_only))]
982 fn test_memory_usage() {
983 let max_page_size = 512;
984
985 let mut memory_management = MemoryManagement::from_configuration(
986 BytesStorage::default(),
987 &DUMMY_MEM_PROPS,
988 MemoryConfiguration::Custom {
989 pool_options: vec![MemoryPoolOptions {
990 pool_type: PoolType::ExclusivePages {
991 max_alloc_size: max_page_size,
992 },
993 dealloc_period: None,
994 }],
995 },
996 Arc::new(ServerLogger::default()),
997 options(),
998 );
999 let handle = memory_management.reserve(100);
1000 let usage = memory_management.memory_usage();
1001
1002 assert_eq!(usage.bytes_in_use, 100);
1003 assert!(usage.bytes_reserved >= 100 && usage.bytes_reserved <= max_page_size);
1004
1005 drop(handle);
1007 let _handle = memory_management.reserve(100);
1008 let usage_new = memory_management.memory_usage();
1009 assert_eq!(usage, usage_new);
1010 }
1011
1012 #[test_log::test]
1013 fn alloc_two_chunks_on_one_page() {
1014 let page_size = 2048;
1015
1016 let mut memory_management = MemoryManagement::from_configuration(
1017 BytesStorage::default(),
1018 &DUMMY_MEM_PROPS,
1019 MemoryConfiguration::Custom {
1020 pool_options: vec![MemoryPoolOptions {
1021 pool_type: PoolType::SlicedPages {
1022 page_size,
1023 max_slice_size: page_size,
1024 },
1025 dealloc_period: None,
1026 }],
1027 },
1028 Arc::new(ServerLogger::default()),
1029 options(),
1030 );
1031
1032 let alloc_size = 512;
1033 let _handle = memory_management.reserve(alloc_size);
1034 let _new_handle = memory_management.reserve(alloc_size);
1035
1036 let usage = memory_management.memory_usage();
1037 assert_eq!(usage.number_allocs, 2);
1038 assert_eq!(usage.bytes_in_use, alloc_size * 2);
1039 assert_eq!(usage.bytes_reserved, page_size);
1040 }
1041
1042 #[test_log::test]
1043 fn alloc_reuses_storage() {
1044 let page_size = 512;
1046
1047 let mut memory_management = MemoryManagement::from_configuration(
1048 BytesStorage::default(),
1049 &DUMMY_MEM_PROPS,
1050 MemoryConfiguration::Custom {
1051 pool_options: vec![MemoryPoolOptions {
1052 pool_type: PoolType::SlicedPages {
1053 page_size,
1054 max_slice_size: page_size,
1055 },
1056 dealloc_period: None,
1057 }],
1058 },
1059 Arc::new(ServerLogger::default()),
1060 options(),
1061 );
1062
1063 let alloc_size = 512;
1064 let _handle = memory_management.reserve(alloc_size);
1065 drop(_handle);
1066 let _new_handle = memory_management.reserve(alloc_size);
1067
1068 let usage = memory_management.memory_usage();
1069 assert_eq!(usage.number_allocs, 1);
1070 assert_eq!(usage.bytes_in_use, alloc_size);
1071 assert_eq!(usage.bytes_reserved, page_size);
1072 }
1073
1074 #[test_log::test]
1075 fn alloc_allocs_new_storage() {
1076 let page_size = 1024;
1077
1078 let mut memory_management = MemoryManagement::from_configuration(
1079 BytesStorage::default(),
1080 &DUMMY_MEM_PROPS,
1081 MemoryConfiguration::Custom {
1082 pool_options: vec![MemoryPoolOptions {
1083 pool_type: PoolType::SlicedPages {
1084 page_size,
1085 max_slice_size: page_size,
1086 },
1087 dealloc_period: None,
1088 }],
1089 },
1090 Arc::new(ServerLogger::default()),
1091 options(),
1092 );
1093
1094 let alloc_size = 768;
1095 let _handle = memory_management.reserve(alloc_size);
1096 let _new_handle = memory_management.reserve(alloc_size);
1097
1098 let usage = memory_management.memory_usage();
1099 assert_eq!(usage.number_allocs, 2);
1100 assert_eq!(usage.bytes_in_use, alloc_size * 2);
1101 assert_eq!(usage.bytes_reserved, page_size * 2);
1102 }
1103
1104 #[test_log::test]
1105 fn alloc_respects_alignment_size() {
1106 let page_size = 500;
1107 let mut memory_management = MemoryManagement::from_configuration(
1108 BytesStorage::default(),
1109 &MemoryDeviceProperties {
1110 max_page_size: page_size,
1111 alignment: 50,
1112 },
1113 MemoryConfiguration::Custom {
1114 pool_options: vec![MemoryPoolOptions {
1115 pool_type: PoolType::SlicedPages {
1116 page_size,
1117 max_slice_size: page_size,
1118 },
1119 dealloc_period: None,
1120 }],
1121 },
1122 Arc::new(ServerLogger::default()),
1123 options(),
1124 );
1125 let alloc_size = 40;
1126 let _handle = memory_management.reserve(alloc_size);
1127 let _new_handle = memory_management.reserve(alloc_size);
1128 let usage = memory_management.memory_usage();
1129 assert_eq!(usage.bytes_padding, 10 * 2);
1131 }
1132
1133 #[test_log::test]
1134 fn allocs_on_correct_page() {
1135 let sizes = [100, 200, 300, 400];
1136
1137 let pools = sizes
1138 .iter()
1139 .map(|size| MemoryPoolOptions {
1140 pool_type: PoolType::SlicedPages {
1141 page_size: *size,
1142 max_slice_size: *size,
1143 },
1144 dealloc_period: None,
1145 })
1146 .collect();
1147 let mut memory_management = MemoryManagement::from_configuration(
1148 BytesStorage::default(),
1149 &MemoryDeviceProperties {
1150 max_page_size: 128 * 1024 * 1024,
1151 alignment: 10,
1152 },
1153 MemoryConfiguration::Custom {
1154 pool_options: pools,
1155 },
1156 Arc::new(ServerLogger::default()),
1157 options(),
1158 );
1159 let alloc_sizes = [50, 150, 250, 350];
1161 let _handles = alloc_sizes.map(|s| memory_management.reserve(s));
1162
1163 let usage = memory_management.memory_usage();
1164
1165 assert_eq!(usage.bytes_in_use, alloc_sizes.iter().sum::<u64>());
1167 assert!(usage.bytes_reserved >= sizes.iter().sum::<u64>());
1168 }
1169
1170 #[test_log::test]
1171 #[cfg(not(exclusive_memory_only))]
1172 fn allocate_deallocate_reallocate() {
1173 let mut memory_management = MemoryManagement::from_configuration(
1174 BytesStorage::default(),
1175 &MemoryDeviceProperties {
1176 max_page_size: 128 * 1024 * 1024,
1177 alignment: 32,
1178 },
1179 MemoryConfiguration::SubSlices,
1180 Arc::new(ServerLogger::default()),
1181 options(),
1182 );
1183 let handles: Vec<_> = (0..5)
1185 .map(|i| memory_management.reserve(1000 * (i + 1)))
1186 .collect();
1187 let usage_before = memory_management.memory_usage();
1188 drop(handles);
1190 let _new_handles: Vec<_> = (0..5)
1192 .map(|i| memory_management.reserve(1000 * (i + 1)))
1193 .collect();
1194 let usage_after = memory_management.memory_usage();
1195 assert_eq!(usage_before.number_allocs, usage_after.number_allocs);
1196 assert_eq!(usage_before.bytes_in_use, usage_after.bytes_in_use);
1197 assert!(usage_before.bytes_reserved >= usage_after.bytes_reserved);
1199 }
1200
1201 #[test_log::test]
1202 #[cfg(not(exclusive_memory_only))]
1203 fn test_fragmentation_resistance() {
1204 let mut memory_management = MemoryManagement::from_configuration(
1205 BytesStorage::default(),
1206 &MemoryDeviceProperties {
1207 max_page_size: 128 * 1024 * 1024,
1208 alignment: 32,
1209 },
1210 MemoryConfiguration::SubSlices,
1211 Arc::new(ServerLogger::default()),
1212 options(),
1213 );
1214 let sizes = [50, 1000, 100, 5000, 200, 10000, 300];
1216 let handles: Vec<_> = sizes
1217 .iter()
1218 .map(|&size| memory_management.reserve(size).unwrap())
1219 .collect();
1220 let usage_before = memory_management.memory_usage();
1221 for i in (0..handles.len()).step_by(2) {
1223 drop(handles[i].clone());
1224 }
1225 for &size in &sizes[0..sizes.len() / 2] {
1227 memory_management.reserve(size).unwrap();
1228 }
1229 let usage_after = memory_management.memory_usage();
1230 assert!(usage_after.bytes_reserved <= (usage_before.bytes_reserved as f64 * 1.1) as u64);
1232 }
1233
1234 #[test_log::test]
1236 fn noslice_test_handle_mutability() {
1237 let mut memory_management = MemoryManagement::from_configuration(
1238 BytesStorage::default(),
1239 &(MemoryDeviceProperties {
1240 max_page_size: 128 * 1024 * 1024,
1241 alignment: 32,
1242 }),
1243 MemoryConfiguration::ExclusivePages,
1244 Arc::new(ServerLogger::default()),
1245 options(),
1246 );
1247 let handle = memory_management.reserve(10).unwrap();
1248 let other_ref = handle.clone();
1249 assert!(!handle.can_mut(), "Handle can't be mut when multiple ref.");
1250 drop(other_ref);
1251 assert!(handle.can_mut(), "Handle should be mut when only one ref.");
1252 }
1253
1254 #[test_log::test]
1255 fn noslice_alloc_two_chunk() {
1256 let mut memory_management = MemoryManagement::from_configuration(
1257 BytesStorage::default(),
1258 &DUMMY_MEM_PROPS,
1259 MemoryConfiguration::Custom {
1260 pool_options: vec![MemoryPoolOptions {
1261 pool_type: PoolType::ExclusivePages {
1262 max_alloc_size: 1024,
1263 },
1264 dealloc_period: None,
1265 }],
1266 },
1267 Arc::new(ServerLogger::default()),
1268 options(),
1269 );
1270
1271 let alloc_size = 512;
1272 let _handle = memory_management.reserve(alloc_size);
1273 let _new_handle = memory_management.reserve(alloc_size);
1274
1275 let usage = memory_management.memory_usage();
1276 assert_eq!(usage.number_allocs, 2);
1277 assert_eq!(usage.bytes_in_use, alloc_size * 2);
1278 assert!(usage.bytes_reserved >= alloc_size * 2);
1279 }
1280
1281 #[test_log::test]
1282 fn noslice_alloc_reuses_storage() {
1283 let mut memory_management = MemoryManagement::from_configuration(
1285 BytesStorage::default(),
1286 &DUMMY_MEM_PROPS,
1287 MemoryConfiguration::Custom {
1288 pool_options: vec![MemoryPoolOptions {
1289 pool_type: PoolType::ExclusivePages {
1290 max_alloc_size: 1024,
1291 },
1292 dealloc_period: None,
1293 }],
1294 },
1295 Arc::new(ServerLogger::default()),
1296 options(),
1297 );
1298
1299 let alloc_size = 512;
1300 let _handle = memory_management.reserve(alloc_size);
1301 drop(_handle);
1302 let _new_handle = memory_management.reserve(alloc_size);
1303
1304 let usage = memory_management.memory_usage();
1305 assert_eq!(usage.number_allocs, 1);
1306 assert_eq!(usage.bytes_in_use, alloc_size);
1307 assert!(usage.bytes_reserved >= alloc_size);
1308 }
1309
1310 #[test_log::test]
1311 fn noslice_alloc_allocs_new_storage() {
1312 let mut memory_management = MemoryManagement::from_configuration(
1313 BytesStorage::default(),
1314 &DUMMY_MEM_PROPS,
1315 MemoryConfiguration::Custom {
1316 pool_options: vec![MemoryPoolOptions {
1317 pool_type: PoolType::ExclusivePages {
1318 max_alloc_size: 1024,
1319 },
1320 dealloc_period: None,
1321 }],
1322 },
1323 Arc::new(ServerLogger::default()),
1324 options(),
1325 );
1326
1327 let alloc_size = 768;
1328 let _handle = memory_management.reserve(alloc_size);
1329 let _new_handle = memory_management.reserve(alloc_size);
1330 let usage = memory_management.memory_usage();
1331 assert_eq!(usage.number_allocs, 2);
1332 assert_eq!(usage.bytes_in_use, alloc_size * 2);
1333 assert!(usage.bytes_reserved >= alloc_size * 2);
1334 }
1335
1336 #[test_log::test]
1337 fn noslice_alloc_respects_alignment_size() {
1338 let mut memory_management = MemoryManagement::from_configuration(
1339 BytesStorage::default(),
1340 &MemoryDeviceProperties {
1341 max_page_size: DUMMY_MEM_PROPS.max_page_size,
1342 alignment: 50,
1343 },
1344 MemoryConfiguration::Custom {
1345 pool_options: vec![MemoryPoolOptions {
1346 pool_type: PoolType::ExclusivePages {
1347 max_alloc_size: 50 * 20,
1348 },
1349 dealloc_period: None,
1350 }],
1351 },
1352 Arc::new(ServerLogger::default()),
1353 options(),
1354 );
1355 let alloc_size = 40;
1356 let _handle = memory_management.reserve(alloc_size);
1357 let _new_handle = memory_management.reserve(alloc_size);
1358 let usage = memory_management.memory_usage();
1359 assert_eq!(usage.bytes_padding, 10 * 2);
1361 }
1362
1363 #[test_log::test]
1364 fn noslice_allocs_on_correct_page() {
1365 let pools = [100, 200, 300, 400]
1366 .iter()
1367 .map(|&size| MemoryPoolOptions {
1368 pool_type: PoolType::SlicedPages {
1369 page_size: size,
1370 max_slice_size: size,
1371 },
1372 dealloc_period: None,
1373 })
1374 .collect();
1375 let mut memory_management = MemoryManagement::from_configuration(
1376 BytesStorage::default(),
1377 &MemoryDeviceProperties {
1378 max_page_size: DUMMY_MEM_PROPS.max_page_size,
1379 alignment: 10,
1380 },
1381 MemoryConfiguration::Custom {
1382 pool_options: pools,
1383 },
1384 Arc::new(ServerLogger::default()),
1385 options(),
1386 );
1387 let alloc_sizes = [50, 150, 250, 350];
1389 let _handles = alloc_sizes.map(|s| memory_management.reserve(s));
1390 let usage = memory_management.memory_usage();
1391 assert_eq!(usage.bytes_in_use, alloc_sizes.iter().sum::<u64>());
1393 }
1394
1395 #[test_log::test]
1396 fn noslice_allocate_deallocate_reallocate() {
1397 let mut memory_management = MemoryManagement::from_configuration(
1398 BytesStorage::default(),
1399 &MemoryDeviceProperties {
1400 max_page_size: 128 * 1024 * 1024,
1401 alignment: 32,
1402 },
1403 MemoryConfiguration::ExclusivePages,
1404 Arc::new(ServerLogger::default()),
1405 options(),
1406 );
1407 let handles: Vec<_> = (0..5)
1409 .map(|i| memory_management.reserve(1000 * (i + 1)))
1410 .collect();
1411 let usage_before = memory_management.memory_usage();
1412 drop(handles);
1414 let _new_handles: Vec<_> = (0..5)
1416 .map(|i| memory_management.reserve(1000 * (i + 1)))
1417 .collect();
1418 let usage_after = memory_management.memory_usage();
1419 assert_eq!(usage_before.number_allocs, usage_after.number_allocs);
1420 assert_eq!(usage_before.bytes_in_use, usage_after.bytes_in_use);
1421 assert_eq!(usage_before.bytes_reserved, usage_after.bytes_reserved);
1422 }
1423}