1#![expect(
2 clippy::panic,
3 clippy::multiple_unsafe_ops_per_block,
4 clippy::allow_attributes,
5 reason = "vendored from upstream `append-only-vec`; matches stdlib panicking conventions and preserves upstream idioms"
6)]
7
8use crate::std::alloc::handle_alloc_error;
9
10use core::{mem::ManuallyDrop, ptr};
11
12#[cfg(not(all(loom, test)))]
13use crate::std::alloc::{Layout, alloc, dealloc};
14#[cfg(not(all(loom, test)))]
15use core::sync::atomic::{AtomicPtr, AtomicUsize, Ordering};
16
17#[cfg(all(loom, test))]
18use loom::{
19 alloc::{Layout, alloc, dealloc},
20 sync::atomic::{AtomicPtr, AtomicUsize, Ordering},
21};
22
23#[derive(Debug)]
24pub struct AppendOnlyVec<T, const AMOUNT_OF_BINS: usize = 32, const BIN_OFFSET: u32 = 3> {
42 count: AtomicUsize,
44 reserved: AtomicUsize,
46
47 data: [AtomicPtr<T>; AMOUNT_OF_BINS],
48}
49
50impl<T, const AMOUNT_OF_BINS: usize, const BIN_OFFSET: u32>
51 AppendOnlyVec<T, AMOUNT_OF_BINS, BIN_OFFSET>
52{
53 const INITIAL_BIN_SIZE: usize = (2_usize).pow(BIN_OFFSET);
54
55 pub fn new() -> Self {
69 const {
71 if Self::capacity() == 0 {
72 panic!("append only vec does not support 0 capacity")
73 }
74 };
75 const { Self::assert_layout() }
77
78 Self {
79 count: AtomicUsize::new(0),
80 reserved: AtomicUsize::new(0),
81 data: core::array::from_fn(|_| AtomicPtr::new(core::ptr::null_mut())),
82 }
83 }
84
85 pub fn push(&self, element: T) -> usize {
92 let idx = self.reserved.fetch_add(1, Ordering::Relaxed);
93 if idx >= Self::capacity() {
94 panic!("append only vec has exceeded max capacity")
95 }
96
97 let (bin_idx, offset) = Self::indices(idx);
98
99 let bucket_ptr = if offset == 0 {
115 self.create_bin_if_needed(bin_idx)
116 } else {
117 let mut failures = 0;
118 let mut ptr = self.data[bin_idx].load(Ordering::Acquire);
119 while ptr.is_null() {
120 spin_wait(&mut failures);
121 ptr = self.data[bin_idx].load(Ordering::Acquire);
122 }
123 ptr
124 };
125
126 unsafe {
130 bucket_ptr.add(offset).write(element);
131 }
132
133 let mut failures = 0;
134 while self
135 .count
136 .compare_exchange(idx, idx + 1, Ordering::Release, Ordering::Relaxed)
137 .is_err()
138 {
139 spin_wait(&mut failures);
140 }
141
142 idx
143 }
144
145 pub fn get(&self, idx: usize) -> Option<&T> {
161 if idx >= self.len() {
162 return None;
163 }
164 unsafe { Some(self.get_unchecked(idx)) }
166 }
167
168 pub fn is_empty(&self) -> bool {
169 self.count.load(Ordering::Acquire) == 0
170 }
171
172 pub fn len(&self) -> usize {
173 self.count.load(Ordering::Acquire)
174 }
175
176 pub const fn capacity() -> usize {
179 Self::INITIAL_BIN_SIZE * ((1 << AMOUNT_OF_BINS) - 1)
180 }
181
182 pub fn iter(&self) -> Iter<'_, T, AMOUNT_OF_BINS, BIN_OFFSET> {
185 Iter {
186 vec: self,
187 start: 0,
188 end: self.len(),
189 }
190 }
191
192 fn create_bin_if_needed(&self, bin_idx: usize) -> *mut T {
198 let mut ptr = self.data[bin_idx].load(Ordering::Acquire);
199 if ptr.is_null() {
200 let (layout, new_ptr) = if core::mem::size_of::<T>() == 0 {
202 (None, core::ptr::NonNull::<T>::dangling().as_ptr())
203 } else {
204 #[allow(
205 clippy::expect_used,
206 reason = "constructor has checked this on creation"
207 )]
208 let layout = Layout::array::<T>(Self::bin_size(bin_idx))
209 .expect("layout of array T with size");
210
211 let ptr = unsafe { alloc(layout) as *mut T };
215 if ptr.is_null() {
216 handle_alloc_error(layout);
217 }
218 (Some(layout), ptr)
219 };
220
221 match self.data[bin_idx].compare_exchange(
222 ptr::null_mut(),
223 new_ptr,
224 Ordering::Release,
225 Ordering::Acquire,
226 ) {
227 Ok(_) => ptr = new_ptr,
228 Err(found) => {
231 if let Some(layout) = layout {
232 unsafe { dealloc(new_ptr as *mut u8, layout) };
236 }
237 ptr = found;
238 }
239 }
240 }
241 ptr
242 }
243
244 const fn indices(i: usize) -> (usize, usize) {
248 let i = i + Self::INITIAL_BIN_SIZE;
250
251 let bin = (i.ilog2() - BIN_OFFSET) as usize;
253
254 let offset = i - Self::bin_size(bin);
256 (bin, offset)
257 }
258
259 const fn bin_size(idx: usize) -> usize {
264 Self::INITIAL_BIN_SIZE << idx
265 }
266
267 pub unsafe fn get_unchecked(&self, idx: usize) -> &T {
272 let (bin_idx, offset) = Self::indices(idx);
273 let bucket = self.data[bin_idx].load(Ordering::Acquire);
274
275 unsafe { &*bucket.add(offset) }
277 }
278
279 const fn assert_layout() {
282 if BIN_OFFSET >= usize::BITS {
283 panic!("BIN_OFFSET is too large for the system's pointer width");
284 }
285
286 if BIN_OFFSET as usize + AMOUNT_OF_BINS >= usize::BITS as usize {
287 panic!("The combination of BIN_OFFSET and AMOUNT_OF_BINS exceeds usize capacity");
288 }
289
290 let max_elements = Self::bin_size(AMOUNT_OF_BINS - 1);
291
292 let size_of_t = core::mem::size_of::<T>();
293 if size_of_t > 0 && max_elements > (isize::MAX as usize / size_of_t) {
294 panic!("The largest bin exceeds isize::MAX bytes; Layout creation would fail");
295 }
296 }
297
298 fn drop_manual(&mut self, mut skip_items: usize) {
302 #[cfg(not(all(loom, test)))]
303 let mut remaining = *self.count.get_mut();
304
305 #[cfg(all(test, loom))]
306 let mut remaining = self.count.with_mut(|v| *v);
307
308 let is_zst = core::mem::size_of::<T>() == 0;
309
310 for (i, atomic_ptr) in self.data.iter_mut().enumerate() {
311 #[cfg(not(all(loom, test)))]
312 let bucket_ptr = *atomic_ptr.get_mut();
313
314 #[cfg(all(test, loom))]
315 let bucket_ptr = atomic_ptr.with_mut(|ptr| *ptr);
316
317 if bucket_ptr.is_null() {
321 break;
322 }
323
324 let bin_cap = Self::bin_size(i);
325 let to_drop = core::cmp::min(remaining, bin_cap);
326
327 for offset in 0..to_drop {
330 if skip_items > 0 {
331 skip_items -= 1;
332 } else {
333 unsafe {
338 ptr::drop_in_place(bucket_ptr.add(offset));
339 }
340 }
341 }
342
343 if !is_zst {
345 #[allow(
346 clippy::expect_used,
347 reason = "constructor has checked this on creation"
348 )]
349 let layout = Layout::array::<T>(bin_cap).expect("Layout of array of T with cap");
350
351 unsafe { dealloc(bucket_ptr as *mut u8, layout) };
355 }
356
357 remaining -= to_drop;
358 }
359 }
360}
361
362fn spin_wait(failures: &mut usize) {
363 #[cfg(not(all(test, loom)))]
364 {
365 *failures += 1;
366 if *failures <= 10 {
367 core::hint::spin_loop();
368 } else {
369 #[cfg(feature = "std")]
370 std::thread::yield_now();
371
372 #[cfg(not(feature = "std"))]
373 core::hint::spin_loop();
374 }
375 }
376
377 #[cfg(all(test, loom))]
378 {
379 _ = failures;
380 loom::thread::yield_now();
381 }
382}
383
384unsafe impl<T: Send, const AMOUNT_OF_BINS: usize, const BIN_OFFSET: u32> Send
387 for AppendOnlyVec<T, AMOUNT_OF_BINS, BIN_OFFSET>
388{
389}
390
391unsafe impl<T: Send + Sync, const AMOUNT_OF_BINS: usize, const BIN_OFFSET: u32> Sync
395 for AppendOnlyVec<T, AMOUNT_OF_BINS, BIN_OFFSET>
396{
397}
398
399impl<T, const AMOUNT_OF_BINS: usize, const BIN_OFFSET: u32> Drop
400 for AppendOnlyVec<T, AMOUNT_OF_BINS, BIN_OFFSET>
401{
402 fn drop(&mut self) {
403 self.drop_manual(0);
404 }
405}
406
407impl<T, const AMOUNT_OF_BINS: usize, const BIN_OFFSET: u32> Default
408 for AppendOnlyVec<T, AMOUNT_OF_BINS, BIN_OFFSET>
409{
410 fn default() -> Self {
411 Self::new()
412 }
413}
414
415use core::ops::Index;
416
417impl<T, const AMOUNT_OF_BINS: usize, const BIN_OFFSET: u32> Index<usize>
418 for AppendOnlyVec<T, AMOUNT_OF_BINS, BIN_OFFSET>
419{
420 type Output = T;
421
422 fn index(&self, idx: usize) -> &Self::Output {
423 assert!(idx < self.len(), "Index out of bounds");
425
426 unsafe { self.get_unchecked(idx) }
428 }
429}
430
431pub struct Iter<'a, T, const AMOUNT_OF_BINS: usize, const BIN_OFFSET: u32> {
433 vec: &'a AppendOnlyVec<T, AMOUNT_OF_BINS, BIN_OFFSET>,
434 start: usize,
435 end: usize,
436}
437
438impl<'a, T, const AMOUNT_OF_BINS: usize, const BIN_OFFSET: u32> Iterator
439 for Iter<'a, T, AMOUNT_OF_BINS, BIN_OFFSET>
440{
441 type Item = &'a T;
442
443 fn next(&mut self) -> Option<Self::Item> {
444 if self.start < self.end {
445 let pos = self.start;
446 self.start += 1;
447 Some(unsafe { self.vec.get_unchecked(pos) })
449 } else {
450 None
451 }
452 }
453
454 fn size_hint(&self) -> (usize, Option<usize>) {
455 let len = self.end - self.start;
456 (len, Some(len))
457 }
458}
459
460impl<'a, T, const AMOUNT_OF_BINS: usize, const BIN_OFFSET: u32> DoubleEndedIterator
461 for Iter<'a, T, AMOUNT_OF_BINS, BIN_OFFSET>
462{
463 fn next_back(&mut self) -> Option<Self::Item> {
464 if self.start < self.end {
465 self.end -= 1;
466 let pos = self.end;
467 Some(unsafe { self.vec.get_unchecked(pos) })
469 } else {
470 None
471 }
472 }
473}
474
475impl<'a, T, const AMOUNT_OF_BINS: usize, const BIN_OFFSET: u32> ExactSizeIterator
476 for Iter<'a, T, AMOUNT_OF_BINS, BIN_OFFSET>
477{
478}
479
480impl<T, const AMOUNT_OF_BINS: usize, const BIN_OFFSET: u32> FromIterator<T>
481 for AppendOnlyVec<T, AMOUNT_OF_BINS, BIN_OFFSET>
482{
483 fn from_iter<I: IntoIterator<Item = T>>(iter: I) -> Self {
484 let this = Self::new();
485 for item in iter {
486 this.push(item);
487 }
488 this
489 }
490}
491
492impl<'a, T, const BINS: usize, const OFFSET: u32> IntoIterator
493 for &'a AppendOnlyVec<T, BINS, OFFSET>
494{
495 type Item = &'a T;
496 type IntoIter = Iter<'a, T, BINS, OFFSET>;
497
498 fn into_iter(self) -> Self::IntoIter {
499 self.iter()
500 }
501}
502
503pub struct IntoIterOwned<T, const BINS: usize, const OFFSET: u32> {
504 vec: ManuallyDrop<AppendOnlyVec<T, BINS, OFFSET>>,
506 consumed: usize,
507}
508
509impl<T, const BINS: usize, const OFFSET: u32> IntoIterator for AppendOnlyVec<T, BINS, OFFSET> {
510 type Item = T;
511 type IntoIter = IntoIterOwned<T, BINS, OFFSET>;
512
513 fn into_iter(self) -> Self::IntoIter {
514 IntoIterOwned {
515 vec: ManuallyDrop::new(self),
516 consumed: 0,
517 }
518 }
519}
520
521impl<T, const BINS: usize, const OFFSET: u32> Iterator for IntoIterOwned<T, BINS, OFFSET> {
522 type Item = T;
523
524 fn next(&mut self) -> Option<Self::Item> {
525 if self.consumed < self.vec.len() {
526 let idx = self.consumed;
527 self.consumed += 1;
528
529 let (bin_idx, offset) = AppendOnlyVec::<T, BINS, OFFSET>::indices(idx);
530 let bucket = self.vec.data[bin_idx].load(Ordering::Acquire);
531
532 unsafe { Some(core::ptr::read(bucket.add(offset))) }
535 } else {
536 None
537 }
538 }
539
540 fn size_hint(&self) -> (usize, Option<usize>) {
541 let remaining = self.vec.len() - self.consumed;
542 (remaining, Some(remaining))
543 }
544}
545
546impl<T, const BINS: usize, const OFFSET: u32> Drop for IntoIterOwned<T, BINS, OFFSET> {
547 fn drop(&mut self) {
548 self.vec.drop_manual(self.consumed);
549 }
550}
551
552impl<T, const BINS: usize, const OFFSET: u32> Extend<T> for AppendOnlyVec<T, BINS, OFFSET> {
553 fn extend<I: IntoIterator<Item = T>>(&mut self, iter: I) {
554 for item in iter {
555 self.push(item);
556 }
557 }
558}
559
560impl<T, const BINS: usize, const OFFSET: u32> Extend<T> for &AppendOnlyVec<T, BINS, OFFSET> {
563 fn extend<I: IntoIterator<Item = T>>(&mut self, iter: I) {
564 for item in iter {
565 self.push(item);
566 }
567 }
568}
569
570#[cfg(all(test, not(loom)))]
571mod tests {
572 use super::*;
573
574 #[test]
575 fn we_can_add_items_and_iter_them() {
576 let vec: AppendOnlyVec<usize> = AppendOnlyVec::new();
577 vec.push(1);
578 vec.push(3);
579
580 let mut iter = vec.iter();
581 assert_eq!(iter.size_hint().0, 2);
582 assert_eq!(*iter.next().unwrap(), 1);
583 assert_eq!(*iter.next().unwrap(), 3);
584 }
585
586 #[derive(Clone, Debug)]
587 struct NoSize;
588
589 #[test]
590 fn support_zero_sized_types() {
591 let vec: AppendOnlyVec<NoSize> = AppendOnlyVec::new();
592 vec.push(NoSize);
593 vec.push(NoSize);
594 }
595
596 #[test]
597 fn push_crosses_bin_boundaries_with_stable_order() {
598 let vec: AppendOnlyVec<usize, 4, 3> = AppendOnlyVec::new();
599
600 for i in 0..26 {
601 assert_eq!(vec.push(i), i);
602 }
603
604 assert_eq!(vec.len(), 26);
605 assert_eq!(vec[7], 7);
606 assert_eq!(vec[8], 8);
607 assert_eq!(vec[23], 23);
608 assert_eq!(vec[24], 24);
609
610 let items: Vec<usize> = vec.iter().copied().collect();
611 assert_eq!(items, (0..26).collect::<Vec<_>>());
612 }
613
614 #[test]
615 fn iter_is_a_snapshot_of_length_at_creation() {
616 let vec: AppendOnlyVec<usize, 3, 1> = AppendOnlyVec::new();
617 vec.push(1);
618
619 let iter = vec.iter();
620 vec.push(2);
621
622 let items: Vec<usize> = iter.copied().collect();
623 assert_eq!(items, vec![1]);
624 assert_eq!(vec.len(), 2);
625 }
626
627 #[test]
628 #[should_panic(expected = "append only vec has exceeded max capacity")]
629 fn push_panics_when_capacity_is_exceeded() {
630 let vec: AppendOnlyVec<u8, 1, 1> = AppendOnlyVec::new();
631 assert_eq!(AppendOnlyVec::<u8, 1, 1>::capacity(), 2);
632
633 vec.push(1);
634 vec.push(2);
635 vec.push(3);
636 }
637}
638
639#[cfg(all(test, loom))]
640mod loom_tests {
641 use std::sync::Arc;
642
643 use loom::thread;
644
645 use super::*;
646
647 fn create_builder() -> loom::model::Builder {
648 let mut builder = loom::model::Builder::new();
649 builder.max_branches = 100000;
650 builder
651 }
652
653 #[test]
654 fn basic() {
655 create_builder().check(|| {
656 let vec: Arc<AppendOnlyVec<usize>> = Arc::new(AppendOnlyVec::new());
657 vec.push(8);
658
659 let vec_cl = vec.clone();
660
661 let x = thread::spawn(move || {
662 vec_cl.push(16);
663 });
664
665 x.join().unwrap();
666 assert_eq!(vec.len(), 2);
667 });
668 }
669
670 #[test]
671 fn concurrent_push() {
672 create_builder().check(|| {
673 let vec = Arc::new(AppendOnlyVec::<usize, 2, 1>::new());
674
675 let vec_cl = vec.clone();
676 let t1 = loom::thread::spawn(move || vec_cl.push(1));
677 let vec_cl = vec.clone();
678 let t2 = loom::thread::spawn(move || vec_cl.clone().push(2));
679
680 t1.join().unwrap();
681 t2.join().unwrap();
682 assert_eq!(vec.len(), 2);
683
684 let sum: usize = vec.iter().sum();
686 assert_eq!(sum, 3);
687 });
688 }
689
690 #[test]
691 fn read_while_push() {
692 create_builder().check(|| {
693 let vec = Arc::new(AppendOnlyVec::<usize, 2, 1>::new());
694 let v1 = vec.clone();
695
696 let t1 = loom::thread::spawn(move || {
697 v1.push(42);
698 });
699
700 if vec.len() == 1 {
703 assert_eq!(*vec.get(0).unwrap(), 42);
704 }
705
706 t1.join().unwrap();
709 });
710 }
711
712 #[derive(Clone, Debug)]
732 struct NoSize;
733
734 #[test]
735 fn zero_sized_types() {
736 create_builder().check(|| {
737 let vec = AppendOnlyVec::<NoSize, 5, 1>::new();
738
739 vec.push(NoSize);
741 vec.push(NoSize);
742 });
743 }
744
745 #[test]
746 fn drop_of_partial_consumed_into_iter() {
747 create_builder().check(|| {
748 let vec = AppendOnlyVec::<String, 2, 1>::new();
749 vec.push("a".to_owned());
750 vec.push("b".to_owned());
751 vec.push("c".to_owned());
752 vec.push("d".to_owned());
753
754 let mut iter = vec.into_iter();
755
756 let item = iter.next();
757 assert_eq!(item.unwrap(), "a");
758
759 drop(iter);
761 });
762 }
763}