1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
use crate::allocator::{AllocatorProvider, Global};
use crate::dynamic::types::InlineVec;
use super::limit::{LEN_INLINE_TAG, LEN_VALUE_MASK, LIMIT};
impl InlineVec<LIMIT, Global> {
#[inline]
pub const fn new() -> Self {
Self::new_in(Global)
}
/// Construct an `InlineVec` from a byte slice.
///
/// Returns `Err(())` if the slice exceeds the inline capacity (`LIMIT`).
///
/// This is a `const fn` so it can be called from const contexts such as
/// [`EcoString::inline`](crate::EcoString::inline).
#[inline]
pub const fn from_slice(bytes: &[u8]) -> Result<Self, ()> {
let len = bytes.len();
if len > LIMIT {
return Err(());
}
let mut buf = [0u8; LIMIT];
let mut i = 0;
while i < len {
buf[i] = bytes[i];
i += 1;
}
// Safety: len <= LIMIT was checked above.
unsafe { Ok(Self::from_buf(buf, len)) }
}
/// The given length may not exceed LIMIT.
#[inline]
#[allow(dead_code)]
pub const unsafe fn from_buf(buf: [u8; LIMIT], len: usize) -> Self {
unsafe { Self::from_buf_in(buf, len, Global) }
}
}
impl<const N: usize, A> InlineVec<N, A>
where
A: AllocatorProvider,
{
#[inline]
pub const fn new_in(allocator: A) -> Self {
// Safety: Trivially, 0 <= N
unsafe { Self::from_buf_in([0; N], 0, allocator) }
}
// Allocator-generic counterpart of the `Global` const `from_slice`.
// Currently unused (the const `from_slice` covers the default path);
// kept for the in-progress allocator-api feature work.
#[inline]
#[allow(dead_code)]
pub fn from_slice_in(bytes: &[u8], allocator: A) -> Result<Self, ()> {
let len = bytes.len();
if len > N {
return Err(());
}
let mut buf = [0; N];
buf[..len].copy_from_slice(bytes);
// Safety: If len > N, Err was returned earlier.
unsafe { Ok(Self::from_buf_in(buf, len, allocator)) }
}
/// The given length may not exceed N.
#[inline]
pub const unsafe fn from_buf_in(buf: [u8; N], len: usize, allocator: A) -> Self {
// Safe body; `unsafe fn` because the `len <= N` precondition is a caller contract.
debug_assert!(len <= N);
Self {
buf,
tagged_len: len as u8 | LEN_INLINE_TAG,
allocator,
}
}
#[inline]
pub fn len(&self) -> usize {
usize::from(self.tagged_len & LEN_VALUE_MASK)
}
/// The given length may not exceed N.
#[inline]
unsafe fn set_len(&mut self, len: usize) {
// Safe body; `unsafe fn` because the `len <= N` precondition is a caller contract.
debug_assert!(len <= N);
self.tagged_len = len as u8 | LEN_INLINE_TAG;
}
#[inline]
pub fn as_slice(&self) -> &[u8] {
// Safety: We have the invariant `len <= N`.
unsafe { self.buf.get_unchecked(..self.len()) }
}
#[inline]
pub fn as_mut_slice(&mut self) -> &mut [u8] {
// Safety: We have the invariant `len <= N`.
let len = self.len();
unsafe { self.buf.get_unchecked_mut(..len) }
}
#[inline]
pub fn clear(&mut self) {
unsafe {
// Safety: Trivially, `0 <= N`.
self.set_len(0);
}
}
#[inline]
pub fn push(&mut self, byte: u8) -> Result<(), ()> {
let len = self.len();
if let Some(slot) = self.buf.get_mut(len) {
*slot = byte;
unsafe {
// Safety: The `get_mut` call guarantees that `len < N`.
self.set_len(len + 1);
}
Ok(())
} else {
Err(())
}
}
#[inline]
pub fn extend_from_slice(&mut self, bytes: &[u8]) -> Result<(), ()> {
let len = self.len();
let grown = len + bytes.len();
if let Some(segment) = self.buf.get_mut(len..grown) {
segment.copy_from_slice(bytes);
unsafe {
// Safety: The `get_mut` call guarantees that `grown <= N`.
self.set_len(grown);
}
Ok(())
} else {
Err(())
}
}
#[inline]
pub fn truncate(&mut self, target: usize) {
if target < self.len() {
unsafe {
// Safety: Checked that it's smaller than the current length,
// which cannot exceed N itself.
self.set_len(target);
}
}
}
#[inline(always)]
pub(crate) fn get_allocator(&self) -> &A {
&self.allocator
}
/// Construct an inline `DynamicVec` directly into `place`.
///
/// This is the inline writer for the allocator-generic
/// [`from_slice_in_owned`](crate::dynamic::types::DynamicVec::from_slice_in_owned)
/// path. The intent is to write straight into the final location, but note
/// it only pays off when `place` genuinely *is* that location. For a
/// standalone `Global` construct (the `EcoString::from(&str)` hot path) the
/// `MaybeUninit` slot does **not** promote to registers, so emplacing here
/// adds a stack round-trip; that path therefore uses the by-value
/// `DynamicVec::from_slice` (`InlineVec::from_slice` → `from_inline`)
/// instead. Emplacing still earns its keep for non-ZST allocators and the
/// non-`TAG_OVERLAPS_LEN` layouts, where the tag byte must be written into
/// the slot in place (by-value union construction can't express that).
///
/// Uses word-sized writes to the (aligned) destination buffer and
/// unaligned reads from the source for optimal codegen on small copies
/// (the inline limit is typically 15 bytes = at most 1–2 words).
///
/// Returns `true` if the data fit inline (i.e. `bytes.len() + size_hint <= N`),
/// `false` otherwise (caller must fall back to a heap allocation).
/// The allocator is borrowed and only cloned when the data actually fits
/// inline, so the caller can reuse it for the spill fallback without a
/// redundant clone.
#[inline(always)]
pub fn emplace_from_slice_in<'a>(
place: &mut core::mem::MaybeUninit<crate::dynamic::types::DynamicVec<'a, N, A>>,
bytes: &[u8],
allocator: &A,
size_hint: usize,
) -> bool
where
A: Clone,
{
const WORD: usize = core::mem::size_of::<usize>();
let len = bytes.len();
if len + size_hint > N {
return false;
}
unsafe {
let place_ptr: *mut InlineVec<N, A> = place.as_mut_ptr().cast();
let buf_ptr: *mut u8 = place_ptr.cast();
let src_ptr: *const u8 = bytes.as_ptr();
// Zero the entire inline area (buf[0..N] + tagged_len) so that
// tail bytes [len..N) are deterministically 0. This makes the
// full-buf PartialEq comparison correct regardless of whether
// the DynamicVec was built via from_slice (zeroed) or emplace.
// On 64-bit (N=15) this is 2 word writes; on 32-bit (N=7) 1.
let dst = buf_ptr as *mut usize;
dst.write(0);
if N > WORD {
dst.add(1).write(0);
}
// N < 2*WORD on both 32-bit (N=7, WORD=4) and 64-bit (N=15,
// WORD=8), so there is at most one full word to copy. Using
// a conditional instead of a loop lets the compiler emit a
// single cmov + mov pair rather than loop scaffolding.
let offset = if len >= WORD {
let src = src_ptr as *const usize;
dst.write(src.read_unaligned());
WORD
} else {
0
};
// Copy remaining bytes (at most WORD-1 = 7 on 64-bit).
let mut j = offset;
while j < len {
*buf_ptr.add(j) = *src_ptr.add(j);
j += 1;
}
// Write tagged_len at offset N.
buf_ptr.add(N).write(len as u8 | LEN_INLINE_TAG);
// Write the allocator field — clone only on success.
core::ptr::addr_of_mut!((*place_ptr).allocator).write(allocator.clone());
}
true
}
}