#![expect(
clippy::panic,
clippy::multiple_unsafe_ops_per_block,
clippy::allow_attributes,
reason = "vendored from upstream `append-only-vec`; matches stdlib panicking conventions and preserves upstream idioms"
)]
use crate::std::alloc::handle_alloc_error;
use core::{mem::ManuallyDrop, ptr};
#[cfg(not(all(loom, test)))]
use crate::std::alloc::{Layout, alloc, dealloc};
#[cfg(not(all(loom, test)))]
use core::sync::atomic::{AtomicPtr, AtomicUsize, Ordering};
#[cfg(all(loom, test))]
use loom::{
alloc::{Layout, alloc, dealloc},
sync::atomic::{AtomicPtr, AtomicUsize, Ordering},
};
#[derive(Debug)]
pub struct AppendOnlyVec<T, const AMOUNT_OF_BINS: usize = 32, const BIN_OFFSET: u32 = 3> {
count: AtomicUsize,
reserved: AtomicUsize,
data: [AtomicPtr<T>; AMOUNT_OF_BINS],
}
impl<T, const AMOUNT_OF_BINS: usize, const BIN_OFFSET: u32>
AppendOnlyVec<T, AMOUNT_OF_BINS, BIN_OFFSET>
{
const INITIAL_BIN_SIZE: usize = (2_usize).pow(BIN_OFFSET);
pub fn new() -> Self {
const {
if Self::capacity() == 0 {
panic!("append only vec does not support 0 capacity")
}
};
const { Self::assert_layout() }
Self {
count: AtomicUsize::new(0),
reserved: AtomicUsize::new(0),
data: core::array::from_fn(|_| AtomicPtr::new(core::ptr::null_mut())),
}
}
pub fn push(&self, element: T) -> usize {
let idx = self.reserved.fetch_add(1, Ordering::Relaxed);
if idx >= Self::capacity() {
panic!("append only vec has exceeded max capacity")
}
let (bin_idx, offset) = Self::indices(idx);
let bucket_ptr = if offset == 0 {
self.create_bin_if_needed(bin_idx)
} else {
let mut failures = 0;
let mut ptr = self.data[bin_idx].load(Ordering::Acquire);
while ptr.is_null() {
spin_wait(&mut failures);
ptr = self.data[bin_idx].load(Ordering::Acquire);
}
ptr
};
unsafe {
bucket_ptr.add(offset).write(element);
}
let mut failures = 0;
while self
.count
.compare_exchange(idx, idx + 1, Ordering::Release, Ordering::Relaxed)
.is_err()
{
spin_wait(&mut failures);
}
idx
}
pub fn get(&self, idx: usize) -> Option<&T> {
if idx >= self.len() {
return None;
}
unsafe { Some(self.get_unchecked(idx)) }
}
pub fn is_empty(&self) -> bool {
self.count.load(Ordering::Acquire) == 0
}
pub fn len(&self) -> usize {
self.count.load(Ordering::Acquire)
}
pub const fn capacity() -> usize {
Self::INITIAL_BIN_SIZE * ((1 << AMOUNT_OF_BINS) - 1)
}
pub fn iter(&self) -> Iter<'_, T, AMOUNT_OF_BINS, BIN_OFFSET> {
Iter {
vec: self,
start: 0,
end: self.len(),
}
}
fn create_bin_if_needed(&self, bin_idx: usize) -> *mut T {
let mut ptr = self.data[bin_idx].load(Ordering::Acquire);
if ptr.is_null() {
let (layout, new_ptr) = if core::mem::size_of::<T>() == 0 {
(None, core::ptr::NonNull::<T>::dangling().as_ptr())
} else {
#[allow(
clippy::expect_used,
reason = "constructor has checked this on creation"
)]
let layout = Layout::array::<T>(Self::bin_size(bin_idx))
.expect("layout of array T with size");
let ptr = unsafe { alloc(layout) as *mut T };
if ptr.is_null() {
handle_alloc_error(layout);
}
(Some(layout), ptr)
};
match self.data[bin_idx].compare_exchange(
ptr::null_mut(),
new_ptr,
Ordering::Release,
Ordering::Acquire,
) {
Ok(_) => ptr = new_ptr,
Err(found) => {
if let Some(layout) = layout {
unsafe { dealloc(new_ptr as *mut u8, layout) };
}
ptr = found;
}
}
}
ptr
}
const fn indices(i: usize) -> (usize, usize) {
let i = i + Self::INITIAL_BIN_SIZE;
let bin = (i.ilog2() - BIN_OFFSET) as usize;
let offset = i - Self::bin_size(bin);
(bin, offset)
}
const fn bin_size(idx: usize) -> usize {
Self::INITIAL_BIN_SIZE << idx
}
pub unsafe fn get_unchecked(&self, idx: usize) -> &T {
let (bin_idx, offset) = Self::indices(idx);
let bucket = self.data[bin_idx].load(Ordering::Acquire);
unsafe { &*bucket.add(offset) }
}
const fn assert_layout() {
if BIN_OFFSET >= usize::BITS {
panic!("BIN_OFFSET is too large for the system's pointer width");
}
if BIN_OFFSET as usize + AMOUNT_OF_BINS >= usize::BITS as usize {
panic!("The combination of BIN_OFFSET and AMOUNT_OF_BINS exceeds usize capacity");
}
let max_elements = Self::bin_size(AMOUNT_OF_BINS - 1);
let size_of_t = core::mem::size_of::<T>();
if size_of_t > 0 && max_elements > (isize::MAX as usize / size_of_t) {
panic!("The largest bin exceeds isize::MAX bytes; Layout creation would fail");
}
}
fn drop_manual(&mut self, mut skip_items: usize) {
#[cfg(not(all(loom, test)))]
let mut remaining = *self.count.get_mut();
#[cfg(all(test, loom))]
let mut remaining = self.count.with_mut(|v| *v);
let is_zst = core::mem::size_of::<T>() == 0;
for (i, atomic_ptr) in self.data.iter_mut().enumerate() {
#[cfg(not(all(loom, test)))]
let bucket_ptr = *atomic_ptr.get_mut();
#[cfg(all(test, loom))]
let bucket_ptr = atomic_ptr.with_mut(|ptr| *ptr);
if bucket_ptr.is_null() {
break;
}
let bin_cap = Self::bin_size(i);
let to_drop = core::cmp::min(remaining, bin_cap);
for offset in 0..to_drop {
if skip_items > 0 {
skip_items -= 1;
} else {
unsafe {
ptr::drop_in_place(bucket_ptr.add(offset));
}
}
}
if !is_zst {
#[allow(
clippy::expect_used,
reason = "constructor has checked this on creation"
)]
let layout = Layout::array::<T>(bin_cap).expect("Layout of array of T with cap");
unsafe { dealloc(bucket_ptr as *mut u8, layout) };
}
remaining -= to_drop;
}
}
}
fn spin_wait(failures: &mut usize) {
#[cfg(not(all(test, loom)))]
{
*failures += 1;
if *failures <= 10 {
core::hint::spin_loop();
} else {
#[cfg(feature = "std")]
std::thread::yield_now();
#[cfg(not(feature = "std"))]
core::hint::spin_loop();
}
}
#[cfg(all(test, loom))]
{
_ = failures;
loom::thread::yield_now();
}
}
unsafe impl<T: Send, const AMOUNT_OF_BINS: usize, const BIN_OFFSET: u32> Send
for AppendOnlyVec<T, AMOUNT_OF_BINS, BIN_OFFSET>
{
}
unsafe impl<T: Send + Sync, const AMOUNT_OF_BINS: usize, const BIN_OFFSET: u32> Sync
for AppendOnlyVec<T, AMOUNT_OF_BINS, BIN_OFFSET>
{
}
impl<T, const AMOUNT_OF_BINS: usize, const BIN_OFFSET: u32> Drop
for AppendOnlyVec<T, AMOUNT_OF_BINS, BIN_OFFSET>
{
fn drop(&mut self) {
self.drop_manual(0);
}
}
impl<T, const AMOUNT_OF_BINS: usize, const BIN_OFFSET: u32> Default
for AppendOnlyVec<T, AMOUNT_OF_BINS, BIN_OFFSET>
{
fn default() -> Self {
Self::new()
}
}
use core::ops::Index;
impl<T, const AMOUNT_OF_BINS: usize, const BIN_OFFSET: u32> Index<usize>
for AppendOnlyVec<T, AMOUNT_OF_BINS, BIN_OFFSET>
{
type Output = T;
fn index(&self, idx: usize) -> &Self::Output {
assert!(idx < self.len(), "Index out of bounds");
unsafe { self.get_unchecked(idx) }
}
}
pub struct Iter<'a, T, const AMOUNT_OF_BINS: usize, const BIN_OFFSET: u32> {
vec: &'a AppendOnlyVec<T, AMOUNT_OF_BINS, BIN_OFFSET>,
start: usize,
end: usize,
}
impl<'a, T, const AMOUNT_OF_BINS: usize, const BIN_OFFSET: u32> Iterator
for Iter<'a, T, AMOUNT_OF_BINS, BIN_OFFSET>
{
type Item = &'a T;
fn next(&mut self) -> Option<Self::Item> {
if self.start < self.end {
let pos = self.start;
self.start += 1;
Some(unsafe { self.vec.get_unchecked(pos) })
} else {
None
}
}
fn size_hint(&self) -> (usize, Option<usize>) {
let len = self.end - self.start;
(len, Some(len))
}
}
impl<'a, T, const AMOUNT_OF_BINS: usize, const BIN_OFFSET: u32> DoubleEndedIterator
for Iter<'a, T, AMOUNT_OF_BINS, BIN_OFFSET>
{
fn next_back(&mut self) -> Option<Self::Item> {
if self.start < self.end {
self.end -= 1;
let pos = self.end;
Some(unsafe { self.vec.get_unchecked(pos) })
} else {
None
}
}
}
impl<'a, T, const AMOUNT_OF_BINS: usize, const BIN_OFFSET: u32> ExactSizeIterator
for Iter<'a, T, AMOUNT_OF_BINS, BIN_OFFSET>
{
}
impl<T, const AMOUNT_OF_BINS: usize, const BIN_OFFSET: u32> FromIterator<T>
for AppendOnlyVec<T, AMOUNT_OF_BINS, BIN_OFFSET>
{
fn from_iter<I: IntoIterator<Item = T>>(iter: I) -> Self {
let this = Self::new();
for item in iter {
this.push(item);
}
this
}
}
impl<'a, T, const BINS: usize, const OFFSET: u32> IntoIterator
for &'a AppendOnlyVec<T, BINS, OFFSET>
{
type Item = &'a T;
type IntoIter = Iter<'a, T, BINS, OFFSET>;
fn into_iter(self) -> Self::IntoIter {
self.iter()
}
}
pub struct IntoIterOwned<T, const BINS: usize, const OFFSET: u32> {
vec: ManuallyDrop<AppendOnlyVec<T, BINS, OFFSET>>,
consumed: usize,
}
impl<T, const BINS: usize, const OFFSET: u32> IntoIterator for AppendOnlyVec<T, BINS, OFFSET> {
type Item = T;
type IntoIter = IntoIterOwned<T, BINS, OFFSET>;
fn into_iter(self) -> Self::IntoIter {
IntoIterOwned {
vec: ManuallyDrop::new(self),
consumed: 0,
}
}
}
impl<T, const BINS: usize, const OFFSET: u32> Iterator for IntoIterOwned<T, BINS, OFFSET> {
type Item = T;
fn next(&mut self) -> Option<Self::Item> {
if self.consumed < self.vec.len() {
let idx = self.consumed;
self.consumed += 1;
let (bin_idx, offset) = AppendOnlyVec::<T, BINS, OFFSET>::indices(idx);
let bucket = self.vec.data[bin_idx].load(Ordering::Acquire);
unsafe { Some(core::ptr::read(bucket.add(offset))) }
} else {
None
}
}
fn size_hint(&self) -> (usize, Option<usize>) {
let remaining = self.vec.len() - self.consumed;
(remaining, Some(remaining))
}
}
impl<T, const BINS: usize, const OFFSET: u32> Drop for IntoIterOwned<T, BINS, OFFSET> {
fn drop(&mut self) {
self.vec.drop_manual(self.consumed);
}
}
impl<T, const BINS: usize, const OFFSET: u32> Extend<T> for AppendOnlyVec<T, BINS, OFFSET> {
fn extend<I: IntoIterator<Item = T>>(&mut self, iter: I) {
for item in iter {
self.push(item);
}
}
}
impl<T, const BINS: usize, const OFFSET: u32> Extend<T> for &AppendOnlyVec<T, BINS, OFFSET> {
fn extend<I: IntoIterator<Item = T>>(&mut self, iter: I) {
for item in iter {
self.push(item);
}
}
}
#[cfg(all(test, not(loom)))]
mod tests {
use super::*;
#[test]
fn we_can_add_items_and_iter_them() {
let vec: AppendOnlyVec<usize> = AppendOnlyVec::new();
vec.push(1);
vec.push(3);
let mut iter = vec.iter();
assert_eq!(iter.size_hint().0, 2);
assert_eq!(*iter.next().unwrap(), 1);
assert_eq!(*iter.next().unwrap(), 3);
}
#[derive(Clone, Debug)]
struct NoSize;
#[test]
fn support_zero_sized_types() {
let vec: AppendOnlyVec<NoSize> = AppendOnlyVec::new();
vec.push(NoSize);
vec.push(NoSize);
}
#[test]
fn push_crosses_bin_boundaries_with_stable_order() {
let vec: AppendOnlyVec<usize, 4, 3> = AppendOnlyVec::new();
for i in 0..26 {
assert_eq!(vec.push(i), i);
}
assert_eq!(vec.len(), 26);
assert_eq!(vec[7], 7);
assert_eq!(vec[8], 8);
assert_eq!(vec[23], 23);
assert_eq!(vec[24], 24);
let items: Vec<usize> = vec.iter().copied().collect();
assert_eq!(items, (0..26).collect::<Vec<_>>());
}
#[test]
fn iter_is_a_snapshot_of_length_at_creation() {
let vec: AppendOnlyVec<usize, 3, 1> = AppendOnlyVec::new();
vec.push(1);
let iter = vec.iter();
vec.push(2);
let items: Vec<usize> = iter.copied().collect();
assert_eq!(items, vec![1]);
assert_eq!(vec.len(), 2);
}
#[test]
#[should_panic(expected = "append only vec has exceeded max capacity")]
fn push_panics_when_capacity_is_exceeded() {
let vec: AppendOnlyVec<u8, 1, 1> = AppendOnlyVec::new();
assert_eq!(AppendOnlyVec::<u8, 1, 1>::capacity(), 2);
vec.push(1);
vec.push(2);
vec.push(3);
}
}
#[cfg(all(test, loom))]
mod loom_tests {
use std::sync::Arc;
use loom::thread;
use super::*;
fn create_builder() -> loom::model::Builder {
let mut builder = loom::model::Builder::new();
builder.max_branches = 100000;
builder
}
#[test]
fn basic() {
create_builder().check(|| {
let vec: Arc<AppendOnlyVec<usize>> = Arc::new(AppendOnlyVec::new());
vec.push(8);
let vec_cl = vec.clone();
let x = thread::spawn(move || {
vec_cl.push(16);
});
x.join().unwrap();
assert_eq!(vec.len(), 2);
});
}
#[test]
fn concurrent_push() {
create_builder().check(|| {
let vec = Arc::new(AppendOnlyVec::<usize, 2, 1>::new());
let vec_cl = vec.clone();
let t1 = loom::thread::spawn(move || vec_cl.push(1));
let vec_cl = vec.clone();
let t2 = loom::thread::spawn(move || vec_cl.clone().push(2));
t1.join().unwrap();
t2.join().unwrap();
assert_eq!(vec.len(), 2);
let sum: usize = vec.iter().sum();
assert_eq!(sum, 3);
});
}
#[test]
fn read_while_push() {
create_builder().check(|| {
let vec = Arc::new(AppendOnlyVec::<usize, 2, 1>::new());
let v1 = vec.clone();
let t1 = loom::thread::spawn(move || {
v1.push(42);
});
if vec.len() == 1 {
assert_eq!(*vec.get(0).unwrap(), 42);
}
t1.join().unwrap();
});
}
#[derive(Clone, Debug)]
struct NoSize;
#[test]
fn zero_sized_types() {
create_builder().check(|| {
let vec = AppendOnlyVec::<NoSize, 5, 1>::new();
vec.push(NoSize);
vec.push(NoSize);
});
}
#[test]
fn drop_of_partial_consumed_into_iter() {
create_builder().check(|| {
let vec = AppendOnlyVec::<String, 2, 1>::new();
vec.push("a".to_owned());
vec.push("b".to_owned());
vec.push("c".to_owned());
vec.push("d".to_owned());
let mut iter = vec.into_iter();
let item = iter.next();
assert_eq!(item.unwrap(), "a");
drop(iter);
});
}
}