1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
use crate::raw::{AllocError, AllocResult, Storage, StorageWithCapacity};
use core::{alloc::Layout, ptr::NonNull};
use std::{alloc::handle_alloc_error, mem::MaybeUninit};
use std::alloc::Allocator;
enum OnFailure {
Abort,
Error,
}
type Heap<T, A> = Box<[MaybeUninit<T>], A>;
pub(crate) unsafe fn box_from_raw_parts_in<T, A: Allocator>(
ptr: NonNull<T>,
capacity: usize,
allocator: A,
) -> Heap<T, A> {
unsafe {
let ptr = std::ptr::slice_from_raw_parts_mut(ptr.as_ptr().cast(), capacity);
Box::from_raw_in(ptr, allocator)
}
}
pub(crate) fn box_into_raw_parts_with_alloc<T, A: Allocator>(b: Heap<T, A>) -> (NonNull<T>, usize, A) {
let (ptr, alloc) = Box::into_raw_with_allocator(b);
unsafe {
let (ptr, capacity) = ptr.to_raw_parts();
(NonNull::new_unchecked(ptr.cast()), capacity, alloc)
}
}
unsafe impl<T, A: Allocator> Storage for Heap<T, A> {
type Item = T;
fn reserve(&mut self, new_capacity: usize) {
if self.len() < new_capacity {
let _ = reserve_slow(self, new_capacity, OnFailure::Abort);
}
}
fn try_reserve(&mut self, new_capacity: usize) -> AllocResult {
if self.len() < new_capacity {
reserve_slow(self, new_capacity, OnFailure::Error)
} else {
Ok(())
}
}
}
unsafe impl<T, A: Default + Allocator> StorageWithCapacity for Heap<T, A> {
fn with_capacity(cap: usize) -> Self { Box::new_uninit_slice_in(cap, A::default()) }
}
#[cold]
#[inline(never)]
fn reserve_slow<T, A: Allocator>(b: &mut Heap<T, A>, new_capacity: usize, on_failure: OnFailure) -> AllocResult {
assert!(new_capacity > b.len());
let (ptr, cap, alloc) = unsafe { box_into_raw_parts_with_alloc(std::ptr::read(b)) };
let new_capacity = new_capacity
.max(cap.checked_mul(2).expect("Could not grow further"))
.max(super::INIT_ALLOC_CAPACITY);
let layout = Layout::new::<T>().repeat(new_capacity).expect("Invalid layout").0;
let ptr = if cap == 0 {
unsafe { alloc.allocate(layout) }
} else {
let new_layout = layout;
let old_layout = Layout::new::<T>().repeat(cap).expect("Invalid layout").0;
unsafe { alloc.grow(ptr.cast(), old_layout, new_layout) }
};
let ptr = match (ptr, on_failure) {
(Ok(ptr), _) => ptr,
(Err(_), OnFailure::Abort) => handle_alloc_error(layout),
(Err(_), OnFailure::Error) => return Err(AllocError),
};
unsafe {
let new = box_from_raw_parts_in(ptr.cast(), new_capacity, alloc);
let old = std::mem::replace(b, new);
std::mem::forget(old);
}
Ok(())
}