#[cfg(any(test, miri))]
pub(crate) const R: usize = 4;
#[cfg(not(any(test, miri)))]
const R: usize = 8;
use core::iter::FusedIterator;
use core::mem;
use hashbrown::{raw, TryReserveError};
pub struct Bucket<T> {
pub(crate) bucket: raw::Bucket<T>,
pub(crate) in_main: bool,
}
impl<T> Clone for Bucket<T> {
#[cfg_attr(feature = "inline-more", inline)]
fn clone(&self) -> Self {
Bucket {
bucket: self.bucket.clone(),
in_main: self.in_main,
}
}
}
impl<T> Bucket<T> {
pub fn will_move(&self) -> bool {
!self.in_main
}
}
impl<T> core::ops::Deref for Bucket<T> {
type Target = raw::Bucket<T>;
fn deref(&self) -> &Self::Target {
&self.bucket
}
}
#[derive(Clone)]
pub struct RawTable<T> {
table: raw::RawTable<T>,
leftovers: Option<OldTable<T>>,
}
impl<T> RawTable<T> {
#[cfg_attr(feature = "inline-more", inline)]
pub fn new() -> Self {
Self {
table: raw::RawTable::new(),
leftovers: None,
}
}
#[cfg_attr(feature = "inline-more", inline)]
pub fn with_capacity(capacity: usize) -> Self {
Self {
table: raw::RawTable::with_capacity(capacity),
leftovers: None,
}
}
#[cfg_attr(feature = "inline-more", inline)]
pub unsafe fn bucket(&self, index: usize) -> Bucket<T> {
Bucket {
bucket: self.table.bucket(index),
in_main: true,
}
}
#[cfg_attr(feature = "inline-more", inline)]
pub unsafe fn erase_no_drop(&mut self, item: &Bucket<T>) {
if item.in_main {
self.table.erase_no_drop(item);
} else if let Some(ref mut lo) = self.leftovers {
lo.table.erase_no_drop(item);
if lo.table.len() == 0 {
let _ = self.leftovers.take();
} else {
lo.items = lo.table.iter();
}
} else {
unreachable!("invalid bucket state");
}
}
#[cfg_attr(feature = "inline-more", inline)]
pub fn clear_no_drop(&mut self) {
self.table.clear_no_drop();
if let Some(mut lo) = self.leftovers.take() {
lo.table.clear_no_drop();
}
}
#[cfg_attr(feature = "inline-more", inline)]
pub fn clear(&mut self) {
let _ = self.leftovers.take();
self.table.clear();
}
#[cfg_attr(feature = "inline-more", inline)]
pub fn shrink_to(&mut self, min_size: usize, hasher: impl Fn(&T) -> u64) {
let mut need = self.table.len();
if let Some(ref lo) = self.leftovers {
need += lo.table.len();
need += (lo.table.len() + R - 1) / R;
}
let min_size = usize::max(need, min_size);
self.table.shrink_to(min_size, hasher);
}
#[cfg_attr(feature = "inline-more", inline)]
pub fn reserve(&mut self, additional: usize, hasher: impl Fn(&T) -> u64) {
let need = self.leftovers.as_ref().map_or(0, |t| t.table.len()) + additional;
if self.table.capacity() - self.table.len() > need {
if cfg!(debug_assertions) {
let buckets = self.table.buckets();
self.table.reserve(need, |_| unreachable!());
assert_eq!(
buckets,
self.table.buckets(),
"resize despite sufficient capacity"
);
} else {
self.table.reserve(need, |_| unreachable!());
}
} else if self.leftovers.is_some() {
self.carry_all(hasher);
self.grow(additional);
} else {
self.grow(additional);
}
}
#[cfg_attr(feature = "inline-more", inline)]
pub fn try_reserve(
&mut self,
additional: usize,
hasher: impl Fn(&T) -> u64,
) -> Result<(), TryReserveError> {
let need = self.leftovers.as_ref().map_or(0, |t| t.table.len()) + additional;
if self.table.capacity() - self.table.len() > need {
if cfg!(debug_assertions) {
let buckets = self.table.buckets();
self.table
.try_reserve(need, |_| unreachable!())
.expect("resize despite sufficient capacity");
assert_eq!(
buckets,
self.table.buckets(),
"resize despite sufficient capacity"
);
} else {
self.table
.try_reserve(need, |_| unreachable!())
.expect("resize despite sufficient capacity");
}
Ok(())
} else if self.leftovers.is_some() {
self.carry_all(hasher);
self.try_grow(additional, true)
} else {
self.try_grow(additional, true)
}
}
#[cfg_attr(feature = "inline-more", inline)]
pub fn insert(&mut self, hash: u64, value: T, hasher: impl Fn(&T) -> u64) -> Bucket<T> {
let bucket = if self.leftovers.is_some() {
let bucket = if cfg!(debug_assertions) {
let buckets = self.table.buckets();
let b = self.table.insert(hash, value, &hasher);
assert_eq!(
buckets,
self.table.buckets(),
"resize while elements are still left over"
);
b
} else {
self.table.insert(hash, value, &hasher)
};
self.carry(hasher);
bucket
} else if self.table.capacity() == self.table.len() {
self.grow(1);
return self.insert(hash, value, hasher);
} else {
self.table.insert(hash, value, hasher)
};
Bucket {
bucket,
in_main: true,
}
}
#[inline]
pub fn find(&self, hash: u64, mut eq: impl FnMut(&T) -> bool) -> Option<Bucket<T>> {
let e = self.table.find(hash, &mut eq);
if let Some(bucket) = e {
return Some(Bucket {
bucket,
in_main: true,
});
}
if let Some(OldTable { ref table, .. }) = self.leftovers {
table.find(hash, eq).map(|bucket| Bucket {
bucket,
in_main: false,
})
} else {
None
}
}
#[cfg_attr(feature = "inline-more", inline)]
pub fn capacity(&self) -> usize {
self.table.capacity()
}
#[cfg_attr(feature = "inline-more", inline)]
pub fn len(&self) -> usize {
self.table.len() + self.leftovers.as_ref().map_or(0, |t| t.table.len())
}
#[cfg_attr(feature = "inline-more", inline)]
pub fn buckets(&self) -> usize {
self.table.buckets()
}
#[cfg_attr(feature = "inline-more", inline)]
pub unsafe fn iter(&self) -> RawIter<T> {
RawIter {
table: self.table.iter(),
leftovers: self.leftovers.as_ref().map(|lo| lo.items.clone()),
}
}
}
impl<T: Clone> RawTable<T> {
#[cfg(feature = "raw")]
pub fn clone_from_with_hasher(&mut self, source: &Self, hasher: impl Fn(&T) -> u64) {
self.table.clone_from_with_hasher(&source.table, &hasher);
if let Some(ref lo_) = source.leftovers {
if let Some(ref mut lo) = self.leftovers {
lo.table.clone_from_with_hasher(&lo_.table, hasher);
lo.items = unsafe { lo.table.iter() };
} else {
self.leftovers = Some(lo_.clone());
}
}
}
}
impl<T> RawTable<T> {
#[cold]
#[inline(never)]
fn grow(&mut self, extra: usize) {
if let Err(_) = self.try_grow(extra, false) {
unsafe { core::hint::unreachable_unchecked() };
}
}
#[cold]
fn try_grow(&mut self, extra: usize, fallible: bool) -> Result<(), TryReserveError> {
debug_assert!(self.leftovers.is_none());
let need = self.table.len();
let inserts = (self.table.len() + R - 1) / R;
let add = usize::max(extra, inserts);
let new_table = if fallible {
let mut new_table = raw::RawTable::new();
new_table.try_reserve(need + inserts + add, |_| {
unreachable!("hasher should not be needed for empty resize")
})?;
new_table
} else {
raw::RawTable::with_capacity(need + inserts + add)
};
let old_table = mem::replace(&mut self.table, new_table);
if old_table.len() != 0 {
let old_table_items = unsafe { old_table.iter() };
self.leftovers = Some(OldTable {
table: old_table,
items: old_table_items,
});
}
Ok(())
}
#[cold]
#[inline(never)]
pub(crate) fn carry_all(&mut self, hasher: impl Fn(&T) -> u64) {
if let Some(ref mut lo) = self.leftovers {
while let Some(e) = lo.items.next() {
let value = unsafe { e.read() };
let hash = hasher(&value);
self.table.insert(hash, value, &hasher);
}
lo.table.clear_no_drop();
let _ = self.leftovers.take();
}
}
#[cold]
#[inline(never)]
pub(crate) fn carry(&mut self, hasher: impl Fn(&T) -> u64) {
if let Some(ref mut lo) = self.leftovers {
for _ in 0..R {
if let Some(e) = lo.items.next() {
let value = unsafe {
let v = e.read();
lo.table.erase_no_drop(&e);
v
};
let hash = hasher(&value);
self.table.insert(hash, value, &hasher);
} else {
let _ = self.leftovers.take();
return;
}
}
if lo.table.len() == 0 {
let _ = self.leftovers.take();
}
}
}
pub(crate) fn is_split(&self) -> bool {
self.leftovers.is_some()
}
#[cfg(any(test, feature = "rayon"))]
pub(crate) fn main(&self) -> &raw::RawTable<T> {
&self.table
}
#[cfg(any(test, feature = "rayon"))]
pub(crate) fn leftovers(&self) -> Option<&raw::RawTable<T>> {
self.leftovers.as_ref().map(|lo| &lo.table)
}
}
impl<T> IntoIterator for RawTable<T> {
type Item = T;
type IntoIter = RawIntoIter<T>;
#[cfg_attr(feature = "inline-more", inline)]
fn into_iter(self) -> RawIntoIter<T> {
RawIntoIter {
table: self.table.into_iter(),
leftovers: self.leftovers.map(|lo| {
lo.table.into_iter()
}),
}
}
}
struct OldTable<T> {
table: raw::RawTable<T>,
items: raw::RawIter<T>,
}
impl<T: Clone> Clone for OldTable<T> {
fn clone(&self) -> OldTable<T> {
let table = self.table.clone();
let items = unsafe { table.iter() };
OldTable { table, items }
}
fn clone_from(&mut self, source: &Self) {
self.table.clone_from(&source.table);
self.items = unsafe { self.table.iter() };
}
}
pub struct RawIter<T> {
table: raw::RawIter<T>,
leftovers: Option<raw::RawIter<T>>,
}
impl<T> Clone for RawIter<T> {
#[cfg_attr(feature = "inline-more", inline)]
fn clone(&self) -> Self {
Self {
table: self.table.clone(),
leftovers: self.leftovers.clone(),
}
}
}
impl<T> Iterator for RawIter<T> {
type Item = Bucket<T>;
#[cfg_attr(feature = "inline-more", inline)]
fn next(&mut self) -> Option<Self::Item> {
let leftovers = &mut self.leftovers;
self.table
.next()
.map(|bucket| Bucket {
bucket,
in_main: true,
})
.or_else(|| {
leftovers.as_mut()?.next().map(|bucket| Bucket {
bucket,
in_main: false,
})
})
}
#[cfg_attr(feature = "inline-more", inline)]
fn size_hint(&self) -> (usize, Option<usize>) {
let (mut lo, mut hi) = self.table.size_hint();
if let Some(ref left) = self.leftovers {
let (lo2, hi2) = left.size_hint();
lo += lo2;
if let (Some(ref mut hi), Some(hi2)) = (&mut hi, hi2) {
*hi += hi2;
}
}
(lo, hi)
}
}
impl<T> ExactSizeIterator for RawIter<T> {}
impl<T> FusedIterator for RawIter<T> {}
pub struct RawIntoIter<T> {
table: raw::RawIntoIter<T>,
leftovers: Option<raw::RawIntoIter<T>>,
}
impl<T> RawIntoIter<T> {
#[cfg_attr(feature = "inline-more", inline)]
pub fn iter(&self) -> RawIter<T> {
RawIter {
table: self.table.iter(),
leftovers: self.leftovers.as_ref().map(|lo| lo.iter()),
}
}
}
impl<T> Iterator for RawIntoIter<T> {
type Item = T;
#[cfg_attr(feature = "inline-more", inline)]
fn next(&mut self) -> Option<T> {
let leftovers = &mut self.leftovers;
self.table.next().or_else(|| leftovers.as_mut()?.next())
}
#[cfg_attr(feature = "inline-more", inline)]
fn size_hint(&self) -> (usize, Option<usize>) {
self.iter().size_hint()
}
}
impl<T> ExactSizeIterator for RawIntoIter<T> {}
impl<T> FusedIterator for RawIntoIter<T> {}