use super::{
d_free, d_malloc_on,
vm_pool::{VirtualMemoryPool, VpmmConfig},
};
use crate::{
d_buffer::DeviceBuffer,
error::MemoryError,
stream::{GpuDeviceCtx, StreamGuard},
};
#[link(name = "cudart")]
extern "C" {
fn cudaMemGetInfo(free_bytes: *mut usize, total_bytes: *mut usize) -> i32;
}
fn get_gpu_free_memory() -> usize {
let mut free = 0usize;
let mut total = 0usize;
let err = unsafe { cudaMemGetInfo(&mut free, &mut total) };
assert_eq!(err, 0, "cudaMemGetInfo failed: {}", err);
free
}
fn test_ctx() -> GpuDeviceCtx {
GpuDeviceCtx::for_current_device().unwrap()
}
fn test_stream() -> StreamGuard {
test_ctx().stream
}
#[test]
fn test_coalescing_via_combined_alloc() {
let ctx = test_ctx();
let len = 2 << 30; let buf_a = DeviceBuffer::<u8>::with_capacity_on(len, &ctx);
let buf_b = DeviceBuffer::<u8>::with_capacity_on(len, &ctx);
let buf_c = DeviceBuffer::<u8>::with_capacity_on(len, &ctx);
let addr_a = buf_a.as_raw_ptr();
drop(buf_b);
drop(buf_a);
drop(buf_c);
let combined_len = 3 * len;
let buf_combined = DeviceBuffer::<u8>::with_capacity_on(combined_len, &ctx);
assert_eq!(
addr_a,
buf_combined.as_raw_ptr(),
"Should reuse coalesced region starting at A"
);
}
#[test]
fn test_va_exhaustion_reserves_more() {
let config = VpmmConfig {
page_size: None,
va_size: 4 << 20, initial_pages: 0,
};
let mut pool = VirtualMemoryPool::new(config);
if pool.page_size == usize::MAX {
println!("VPMM not supported, skipping test");
return;
}
let page_size = pool.page_size;
let stream = test_stream();
assert_eq!(pool.roots.len(), 1);
let mut ptrs = Vec::new();
for _ in 0..4 {
match pool.malloc_internal(page_size, &stream) {
Ok(ptr) => ptrs.push(ptr),
Err(e) => panic!("Allocation failed: {:?}", e),
}
}
assert!(
pool.roots.len() >= 2,
"Should have reserved additional VA chunks. Got {} roots",
pool.roots.len()
);
for ptr in ptrs {
pool.free_internal(ptr).unwrap();
}
}
fn run_doc_scenario(
initial_pages: usize,
) -> (
VirtualMemoryPool,
usize, // page_size
*mut std::ffi::c_void, // ptr_1 (kept)
*mut std::ffi::c_void, // ptr_4
*mut std::ffi::c_void, // ptr_11
) {
let config = VpmmConfig {
page_size: None, va_size: 1 << 30, initial_pages,
};
let mut pool = VirtualMemoryPool::new(config);
if pool.page_size == usize::MAX {
panic!("VPMM not supported");
}
let page_size = pool.page_size;
let stream = test_stream();
let ptr_10 = pool.malloc_internal(10 * page_size, &stream).unwrap();
assert!(!ptr_10.is_null());
let ptr_1 = pool.malloc_internal(page_size, &stream).unwrap();
assert!(!ptr_1.is_null());
assert_eq!(ptr_1 as usize, ptr_10 as usize + 10 * page_size);
pool.free_internal(ptr_10).unwrap();
let ptr_4 = pool.malloc_internal(4 * page_size, &stream).unwrap();
assert!(!ptr_4.is_null());
let ptr_11 = pool.malloc_internal(11 * page_size, &stream).unwrap();
assert!(!ptr_11.is_null());
(pool, page_size, ptr_1, ptr_4, ptr_11)
}
#[test]
fn test_defrag_case_a_enough_free_pages() {
let initial_pages = 22;
let (pool, page_size, ptr_1, ptr_4, ptr_11) = run_doc_scenario(initial_pages);
assert_eq!(
pool.memory_usage(),
22 * page_size,
"Case A: no new pages allocated"
);
assert_eq!(ptr_4 as usize, pool.roots[0] as usize, "4 at VA start");
assert!(
ptr_11 as usize > ptr_1 as usize,
"Case A: 11 should be after 1 (no defrag)"
);
let mut pool = pool;
pool.free_internal(ptr_1).unwrap();
pool.free_internal(ptr_4).unwrap();
pool.free_internal(ptr_11).unwrap();
}
#[test]
fn test_defrag_case_b_defrag_no_new_pages() {
let initial_pages = 18;
let (pool, page_size, ptr_1, ptr_4, ptr_11) = run_doc_scenario(initial_pages);
assert_eq!(
pool.memory_usage(),
18 * page_size,
"Case B: no new pages allocated"
);
assert_eq!(
ptr_4 as usize,
ptr_1 as usize + page_size,
"Case B: 4 right after 1"
);
let mut pool = pool;
pool.free_internal(ptr_1).unwrap();
pool.free_internal(ptr_4).unwrap();
pool.free_internal(ptr_11).unwrap();
}
#[test]
fn test_defrag_case_c_defrag_plus_new_page() {
let initial_pages = 15;
let (pool, page_size, ptr_1, ptr_4, ptr_11) = run_doc_scenario(initial_pages);
assert_eq!(
pool.memory_usage(),
16 * page_size,
"Case C: 1 new page allocated"
);
assert_eq!(
ptr_4 as usize,
ptr_1 as usize + page_size,
"Case C: 4 right after 1"
);
let mut pool = pool;
pool.free_internal(ptr_1).unwrap();
pool.free_internal(ptr_4).unwrap();
pool.free_internal(ptr_11).unwrap();
}
#[test]
fn test_defrag_case_d_not_enough_for_4() {
let initial_pages = 12;
let (pool, page_size, ptr_1, ptr_4, ptr_11) = run_doc_scenario(initial_pages);
assert_eq!(
pool.memory_usage(),
16 * page_size,
"Case D: 4 new pages allocated"
);
assert_eq!(
ptr_4 as usize, pool.roots[0] as usize,
"Case D: 4 at VA start"
);
let mut pool = pool;
pool.free_internal(ptr_1).unwrap();
pool.free_internal(ptr_4).unwrap();
pool.free_internal(ptr_11).unwrap();
}
#[test]
fn test_mixed_allocations() {
let runtime = tokio::runtime::Builder::new_multi_thread()
.worker_threads(4)
.max_blocking_threads(4)
.enable_all()
.build()
.unwrap();
runtime.block_on(async {
let mut handles = Vec::new();
for thread_idx in 0..4 {
let handle = tokio::task::spawn_blocking(move || {
let ctx = test_ctx();
let mut buffers: Vec<DeviceBuffer<u8>> = Vec::new();
for op in 0..15 {
let len = if op % 3 == 0 {
((thread_idx + 1) * (op + 1) * 1024) % (100 << 10) + 1024
} else {
((thread_idx + 1) * (op + 1) % 4 + 1) * (100 << 20)
};
let buf = DeviceBuffer::<u8>::with_capacity_on(len, &ctx);
buffers.push(buf);
if op % 2 == 0 && !buffers.is_empty() {
buffers.remove(0);
}
}
});
handles.push(handle);
}
for handle in handles {
handle.await.expect("thread failed");
}
});
let ctx = test_ctx();
ctx.stream.synchronize().expect("stream sync failed");
let large = DeviceBuffer::<u8>::with_capacity_on(1 << 30, &ctx);
assert!(
!large.as_ptr().is_null(),
"Large allocation should work after mixed operations"
);
}
#[test]
#[ignore] fn test_oom_recovery_after_error() {
let ctx = test_ctx();
let chunk_size = 2 << 30; let mut buffers: Vec<*mut std::ffi::c_void> = Vec::new();
loop {
match d_malloc_on(chunk_size, &ctx.stream) {
Ok(ptr) => buffers.push(ptr),
Err(MemoryError::OutOfMemory { .. }) => break,
Err(e) => panic!("Expected OOM, got {:?}", e),
}
}
let small = d_malloc_on(1 << 20, &ctx.stream).expect("Small allocation after OOM failed");
let free_after_oom = get_gpu_free_memory();
let safety = 64 << 20; assert!(
free_after_oom > safety + (2 << 20),
"Not enough free memory to attempt medium alloc after OOM"
);
let medium_req = free_after_oom - safety;
let medium = d_malloc_on(medium_req, &ctx.stream).expect("Pool allocation after OOM failed");
unsafe { d_free(small).unwrap() };
unsafe { d_free(medium).unwrap() };
for ptr in buffers {
unsafe { d_free(ptr).unwrap() };
}
ctx.stream.synchronize().expect("stream sync after cleanup");
}
fn create_test_pool(initial_pages: usize) -> VirtualMemoryPool {
let config = VpmmConfig {
page_size: None, va_size: 1 << 30, initial_pages,
};
let pool = VirtualMemoryPool::new(config);
if pool.page_size == usize::MAX {
panic!("VPMM not supported, cannot run this test");
}
pool
}
#[test]
fn test_multiple_defrag_cycles() {
let mut pool = create_test_pool(8);
let page_size = pool.page_size;
let stream = test_stream();
for cycle in 0..3 {
let ptrs: Vec<_> = (0..4)
.map(|_| pool.malloc_internal(2 * page_size, &stream).unwrap())
.collect();
pool.free_internal(ptrs[0]).unwrap();
pool.free_internal(ptrs[2]).unwrap();
let ptr_big = pool.malloc_internal(4 * page_size, &stream).unwrap();
assert!(!ptr_big.is_null(), "Cycle {} failed to allocate", cycle);
pool.free_internal(ptrs[1]).unwrap();
pool.free_internal(ptrs[3]).unwrap();
pool.free_internal(ptr_big).unwrap();
}
}
#[test]
fn test_coalesce_all_neighbor_cases() {
let mut pool = create_test_pool(6);
let page_size = pool.page_size;
let stream = test_stream();
let ptr_a = pool.malloc_internal(2 * page_size, &stream).unwrap();
let ptr_b = pool.malloc_internal(2 * page_size, &stream).unwrap();
let ptr_c = pool.malloc_internal(2 * page_size, &stream).unwrap();
pool.free_internal(ptr_b).unwrap();
pool.free_internal(ptr_a).unwrap();
let ptr_4 = pool.malloc_internal(4 * page_size, &stream).unwrap();
assert_eq!(ptr_4 as usize, ptr_a as usize, "Prev: should start at A");
assert_eq!(pool.memory_usage(), 6 * page_size, "Prev: no new alloc");
pool.free_internal(ptr_c).unwrap();
pool.free_internal(ptr_4).unwrap();
let ptr_a = pool.malloc_internal(2 * page_size, &stream).unwrap();
let ptr_b = pool.malloc_internal(2 * page_size, &stream).unwrap();
let ptr_c = pool.malloc_internal(2 * page_size, &stream).unwrap();
pool.free_internal(ptr_a).unwrap();
pool.free_internal(ptr_b).unwrap();
let ptr_4 = pool.malloc_internal(4 * page_size, &stream).unwrap();
assert_eq!(ptr_4 as usize, ptr_a as usize, "Next: should start at A");
pool.free_internal(ptr_c).unwrap();
pool.free_internal(ptr_4).unwrap();
let ptr_a = pool.malloc_internal(2 * page_size, &stream).unwrap();
let ptr_b = pool.malloc_internal(2 * page_size, &stream).unwrap();
let ptr_c = pool.malloc_internal(2 * page_size, &stream).unwrap();
pool.free_internal(ptr_a).unwrap();
pool.free_internal(ptr_c).unwrap();
pool.free_internal(ptr_b).unwrap();
let ptr_6 = pool.malloc_internal(6 * page_size, &stream).unwrap();
assert_eq!(ptr_6 as usize, ptr_a as usize, "Both: should start at A");
assert_eq!(pool.memory_usage(), 6 * page_size, "Both: no new alloc");
pool.free_internal(ptr_6).unwrap();
}
#[test]
fn test_no_coalesce_across_streams() {
let mut pool = create_test_pool(6);
let page_size = pool.page_size;
let stream_1 = test_stream();
let stream_2 = test_stream();
let ptr_a = pool.malloc_internal(2 * page_size, &stream_1).unwrap();
let ptr_b = pool.malloc_internal(2 * page_size, &stream_1).unwrap();
let ptr_c = pool.malloc_internal(2 * page_size, &stream_1).unwrap();
pool.free_internal(ptr_a).unwrap();
pool.free_internal(ptr_b).unwrap();
pool.free_internal(ptr_c).unwrap();
let memory_before = pool.memory_usage();
let ptr_4 = pool.malloc_internal(4 * page_size, &stream_2).unwrap();
assert!(!ptr_4.is_null());
assert_eq!(pool.memory_usage(), memory_before, "Defrag reused existing");
pool.free_internal(ptr_4).unwrap();
}
#[test]
fn test_defrag_tail_regions_returned() {
let mut pool = create_test_pool(10);
let page_size = pool.page_size;
let stream = test_stream();
let ptrs: Vec<_> = (0..5)
.map(|_| pool.malloc_internal(2 * page_size, &stream).unwrap())
.collect();
pool.free_internal(ptrs[0]).unwrap();
pool.free_internal(ptrs[2]).unwrap();
pool.free_internal(ptrs[4]).unwrap();
let memory_before = pool.memory_usage();
let ptr_5 = pool.malloc_internal(5 * page_size, &stream).unwrap();
assert_eq!(pool.memory_usage(), memory_before, "No new pages needed");
let ptr_1 = pool.malloc_internal(page_size, &stream).unwrap();
assert_eq!(pool.memory_usage(), memory_before, "Tail page reused");
let ptr_extra = pool.malloc_internal(page_size, &stream).unwrap();
assert_eq!(
pool.memory_usage(),
memory_before + page_size,
"New page allocated"
);
pool.free_internal(ptrs[1]).unwrap();
pool.free_internal(ptrs[3]).unwrap();
pool.free_internal(ptr_5).unwrap();
pool.free_internal(ptr_1).unwrap();
pool.free_internal(ptr_extra).unwrap();
}
#[test]
fn test_unmapped_region_coalescing_comprehensive() {
let mut pool = create_test_pool(6);
let page_size = pool.page_size;
let stream = test_stream();
let ptr_a = pool.malloc_internal(2 * page_size, &stream).unwrap();
let ptr_b = pool.malloc_internal(2 * page_size, &stream).unwrap();
let ptr_c = pool.malloc_internal(2 * page_size, &stream).unwrap();
pool.free_internal(ptr_a).unwrap();
pool.free_internal(ptr_b).unwrap();
let ptr_4 = pool.malloc_internal(4 * page_size, &stream).unwrap();
assert_eq!(ptr_4 as usize, ptr_a as usize);
pool.free_internal(ptr_4).unwrap();
pool.free_internal(ptr_c).unwrap();
let memory_before = pool.memory_usage();
let ptr_7 = pool.malloc_internal(7 * page_size, &stream).unwrap();
assert_eq!(pool.memory_usage(), memory_before + page_size);
pool.free_internal(ptr_7).unwrap();
}
#[test]
fn test_defrag_new_pages_merge_with_existing() {
let config = VpmmConfig {
page_size: None,
va_size: 1 << 30,
initial_pages: 0,
};
let mut pool = VirtualMemoryPool::new(config);
if pool.page_size == usize::MAX {
println!("VPMM not supported, skipping test");
return;
}
let page_size = pool.page_size;
let stream = test_stream();
let ptr_2 = pool.malloc_internal(2 * page_size, &stream).unwrap();
assert_eq!(pool.memory_usage(), 2 * page_size);
pool.free_internal(ptr_2).unwrap();
let ptr_4 = pool.malloc_internal(4 * page_size, &stream).unwrap();
assert_eq!(pool.memory_usage(), 4 * page_size);
assert!(!ptr_4.is_null());
pool.free_internal(ptr_4).unwrap();
let ptr_a = pool.malloc_internal(2 * page_size, &stream).unwrap();
let ptr_b = pool.malloc_internal(2 * page_size, &stream).unwrap();
pool.free_internal(ptr_a).unwrap();
let ptr_req = pool.malloc_internal(4 * page_size, &stream).unwrap();
assert!(!ptr_req.is_null());
assert_eq!(pool.memory_usage(), 6 * page_size);
pool.free_internal(ptr_b).unwrap();
pool.free_internal(ptr_req).unwrap();
}
#[test]
fn test_defrag_various_scenarios() {
let mut pool = create_test_pool(10);
let page_size = pool.page_size;
let stream = test_stream();
let ptrs: Vec<_> = (0..5)
.map(|_| pool.malloc_internal(2 * page_size, &stream).unwrap())
.collect();
pool.free_internal(ptrs[1]).unwrap();
pool.free_internal(ptrs[3]).unwrap();
pool.free_internal(ptrs[0]).unwrap();
pool.free_internal(ptrs[4]).unwrap();
pool.free_internal(ptrs[2]).unwrap();
let ptr_10 = pool.malloc_internal(10 * page_size, &stream).unwrap();
assert!(!ptr_10.is_null());
assert_eq!(pool.memory_usage(), 10 * page_size);
pool.free_internal(ptr_10).unwrap();
let ptr_a = pool.malloc_internal(6 * page_size, &stream).unwrap();
let ptr_b = pool.malloc_internal(4 * page_size, &stream).unwrap();
pool.free_internal(ptr_a).unwrap();
let memory_before = pool.memory_usage();
let ptr_7 = pool.malloc_internal(7 * page_size, &stream).unwrap();
assert_eq!(pool.memory_usage(), memory_before + page_size);
pool.free_internal(ptr_b).unwrap();
pool.free_internal(ptr_7).unwrap();
}