hpt-allocator 0.1.2

An internal library for memory allocator for hpt
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162

//! This crate provides a memory allocator for tensor

#![deny(missing_docs)]

mod allocators;
mod backend;
mod ptr;
mod storage;
pub(crate) mod utils {
    pub(crate) mod allocate;
    pub(crate) mod cache_resize;
    pub(crate) mod deallocate;
}
/// traits for the allocator
pub mod traits;

use std::marker::PhantomData;

use crate::allocators::cpu::CACHE;
#[cfg(feature = "cuda")]
use crate::allocators::cuda::CUDA_CACHE;
pub use crate::storage::clone_storage;
pub use allocators::cpu::resize_cpu_lru_cache;
#[cfg(feature = "cuda")]
pub use allocators::cuda::resize_cuda_lru_cache;
pub use backend::*;
pub use storage::cpu::CPU_STORAGE;
#[cfg(feature = "cuda")]
pub use storage::cuda::CUDA_STORAGE;
use traits::Allocator;
/// program will free all the memory before exit
#[allow(non_snake_case)]
#[ctor::dtor]
fn free_pools() {
    CACHE.lock().unwrap().clear();
    #[cfg(feature = "cuda")]
    CUDA_CACHE.lock().unwrap().clear();
}

/// Built-in allocator for Hpt
pub struct HptAllocator<B: BackendTy> {
    phantom: PhantomData<B>,
}

impl<B: BackendTy> Clone for HptAllocator<B> {
    fn clone(&self) -> Self {
        HptAllocator {
            phantom: PhantomData,
        }
    }
}

impl Allocator for HptAllocator<Cpu> {
    type Output = *mut u8;
    type CpuAllocator = HptAllocator<Cpu>;
    #[cfg(feature = "cuda")]
    type CudaAllocator = HptAllocator<Cuda>;
    fn allocate(
        &mut self,
        layout: std::alloc::Layout,
        device_id: usize,
    ) -> Result<Self::Output, hpt_common::error::base::TensorError> {
        CACHE.lock().unwrap().allocate(layout, device_id)
    }
    fn allocate_zeroed(
        &mut self,
        layout: std::alloc::Layout,
        device_id: usize,
    ) -> Result<Self::Output, hpt_common::error::base::TensorError> {
        CACHE.lock().unwrap().allocate_zeroed(layout, device_id)
    }
    fn deallocate(
        &mut self,
        ptr: *mut u8,
        layout: &std::alloc::Layout,
        should_drop: bool,
        device_id: usize,
    ) {
        CACHE
            .lock()
            .unwrap()
            .deallocate(ptr, layout, should_drop, device_id);
    }

    fn insert_ptr(&mut self, ptr: *mut u8, device_id: usize) {
        CACHE.lock().unwrap().insert_ptr(ptr, device_id);
    }

    fn clear(&mut self) {
        CACHE.lock().unwrap().clear();
    }

    fn new() -> Self {
        HptAllocator {
            phantom: PhantomData,
        }
    }

    fn forget(&mut self, ptr: *mut u8, device_id: usize) {
        CACHE.lock().unwrap().forget(ptr, device_id);
    }
}

#[cfg(feature = "cuda")]
impl Allocator for HptAllocator<Cuda> {
    type Output = (*mut u8, std::sync::Arc<cudarc::driver::CudaDevice>);
    type CpuAllocator = HptAllocator<Cpu>;
    type CudaAllocator = HptAllocator<Cuda>;

    fn allocate(
        &mut self,
        layout: std::alloc::Layout,
        device_id: usize,
    ) -> Result<Self::Output, hpt_common::error::base::TensorError> {
        CUDA_CACHE.lock().unwrap().allocate(layout, device_id)
    }

    fn allocate_zeroed(
        &mut self,
        layout: std::alloc::Layout,
        device_id: usize,
    ) -> Result<Self::Output, hpt_common::error::base::TensorError> {
        CUDA_CACHE
            .lock()
            .unwrap()
            .allocate_zeroed(layout, device_id)
    }

    fn deallocate(
        &mut self,
        ptr: *mut u8,
        layout: &std::alloc::Layout,
        should_drop: bool,
        device_id: usize,
    ) {
        CUDA_CACHE
            .lock()
            .unwrap()
            .deallocate(ptr, layout, should_drop, device_id);
    }

    fn insert_ptr(&mut self, ptr: *mut u8, device_id: usize) {
        CUDA_CACHE.lock().unwrap().insert_ptr(ptr, device_id);
    }

    fn clear(&mut self) {
        CUDA_CACHE.lock().unwrap().clear();
    }

    fn new() -> Self {
        HptAllocator {
            phantom: PhantomData,
        }
    }

    fn forget(&mut self, ptr: *mut u8, device_id: usize) {
        CUDA_CACHE.lock().unwrap().forget(ptr, device_id);
    }
}

unsafe impl<B: BackendTy> Send for HptAllocator<B> {}
unsafe impl<B: BackendTy> Sync for HptAllocator<B> {}