1use std::alloc::{Layout, alloc, dealloc, handle_alloc_error};
21use std::fmt;
22use std::marker::PhantomData;
23use std::mem::MaybeUninit;
24use std::ptr::{self, NonNull};
25
26use kevy_hash::KevyHash;
27
28use crate::iter::{Iter, Keys, Values};
29
30pub(crate) const GROUP_WIDTH: usize = 16;
32
33pub(crate) const EMPTY: u8 = 0xFF;
36pub(crate) const DELETED: u8 = 0x80;
38pub(crate) const MIN_CAP: usize = 16;
41
42#[inline]
45pub(crate) fn h2(hash: u64) -> u8 {
46 ((hash >> 57) & 0x7F) as u8
47}
48
49#[inline(always)]
55fn prefetch_t0(ptr: *const u8) {
56 #[cfg(all(target_arch = "x86_64", not(miri)))]
57 {
58 unsafe {
61 core::arch::x86_64::_mm_prefetch(
62 ptr as *const i8,
63 core::arch::x86_64::_MM_HINT_T0,
64 );
65 }
66 }
67 #[cfg(all(target_arch = "aarch64", not(miri)))]
68 {
69 unsafe {
71 core::arch::asm!(
72 "prfm pldl1keep, [{p}]",
73 p = in(reg) ptr,
74 options(nostack, preserves_flags, readonly),
75 );
76 }
77 }
78 #[cfg(any(miri, not(any(target_arch = "x86_64", target_arch = "aarch64"))))]
79 {
80 let _ = ptr;
81 }
82}
83
84#[inline]
98pub(crate) fn table_layout<KV>(cap: usize) -> (Layout, usize) {
99 let slots = Layout::array::<MaybeUninit<KV>>(cap).expect("slots layout overflow");
100 let meta = Layout::array::<u8>(cap + GROUP_WIDTH).expect("metadata layout overflow");
101 let (combined, meta_offset) = slots.extend(meta).expect("layout extend overflow");
102 (combined.pad_to_align(), meta_offset)
103}
104
105pub struct KevyMap<K, V> {
113 pub(crate) slots_ptr: NonNull<MaybeUninit<(K, V)>>,
116 pub(crate) metadata_ptr: NonNull<u8>,
120 pub(crate) cap: usize,
122 pub(crate) mask: usize,
124 pub(crate) occupied: usize,
126 pub(crate) deleted: usize,
128 _marker: PhantomData<(K, V)>,
131}
132
133unsafe impl<K: Send, V: Send> Send for KevyMap<K, V> {}
137unsafe impl<K: Sync, V: Sync> Sync for KevyMap<K, V> {}
138
139type SlotSlices<'a, K, V> = (&'a [u8], &'a [MaybeUninit<(K, V)>]);
143
144pub(crate) enum ProbeOutcome {
145 Found(usize),
146 NotFound {
147 insert_at: usize,
148 via_tombstone: bool,
149 },
150}
151
152impl<K, V> KevyMap<K, V> {
153 pub fn new() -> Self {
155 Self {
156 slots_ptr: NonNull::dangling(),
157 metadata_ptr: NonNull::dangling(),
158 cap: 0,
159 mask: 0,
160 occupied: 0,
161 deleted: 0,
162 _marker: PhantomData,
163 }
164 }
165
166 pub fn with_capacity(cap_hint: usize) -> Self {
169 if cap_hint == 0 {
170 return Self::new();
171 }
172 let needed = cap_hint.saturating_mul(8).div_ceil(7);
174 let cap = needed.next_power_of_two().max(MIN_CAP);
175 Self::alloc_table(cap)
176 }
177
178 pub(crate) fn alloc_table(cap: usize) -> Self {
179 debug_assert!(cap.is_power_of_two());
180 debug_assert!(cap >= MIN_CAP);
181
182 let (layout, meta_offset) = table_layout::<(K, V)>(cap);
183 let base = unsafe { alloc(layout) };
187 if base.is_null() {
188 handle_alloc_error(layout);
189 }
190 let meta_byte_ptr = unsafe { base.add(meta_offset) };
195 unsafe { ptr::write_bytes(meta_byte_ptr, EMPTY, cap + GROUP_WIDTH) };
196
197 let slots_ptr = base as *mut MaybeUninit<(K, V)>;
198 let metadata_ptr = meta_byte_ptr;
199
200 kevy_madvise::advise_hugepage(base as *const u8, layout.size());
207
208 Self {
209 slots_ptr: unsafe { NonNull::new_unchecked(slots_ptr) },
212 metadata_ptr: unsafe { NonNull::new_unchecked(metadata_ptr) },
213 cap,
214 mask: cap - 1,
215 occupied: 0,
216 deleted: 0,
217 _marker: PhantomData,
218 }
219 }
220
221 #[inline]
232 pub(crate) fn set_meta(&mut self, i: usize, v: u8) {
233 debug_assert!(i < self.cap);
234 let i2 = (i.wrapping_sub(GROUP_WIDTH) & self.mask) + GROUP_WIDTH;
237 unsafe {
238 *self.metadata_ptr.as_ptr().add(i) = v;
239 *self.metadata_ptr.as_ptr().add(i2) = v;
240 }
241 }
242
243 #[inline]
245 pub fn len(&self) -> usize {
246 self.occupied
247 }
248
249 #[inline]
251 pub fn is_empty(&self) -> bool {
252 self.occupied == 0
253 }
254
255 #[inline]
257 pub fn capacity(&self) -> usize {
258 self.cap
259 }
260
261 pub fn clear(&mut self) {
263 if self.cap == 0 {
264 return;
265 }
266 if std::mem::needs_drop::<(K, V)>() {
267 for i in 0..self.cap {
268 let meta = unsafe { *self.metadata_ptr.as_ptr().add(i) };
270 if meta & 0x80 == 0 {
271 unsafe {
273 ptr::drop_in_place(self.slots_ptr.as_ptr().add(i) as *mut (K, V));
274 }
275 }
276 }
277 }
278 unsafe {
281 ptr::write_bytes(self.metadata_ptr.as_ptr(), EMPTY, self.cap + GROUP_WIDTH);
282 }
283 self.occupied = 0;
284 self.deleted = 0;
285 }
286
287 pub fn iter(&self) -> Iter<'_, K, V> {
289 let (metadata, slots) = self.as_slices();
290 Iter::new(metadata, slots)
291 }
292
293 pub fn iter_from_bucket(&self, start: usize) -> Iter<'_, K, V> {
298 let (metadata, slots) = self.as_slices();
299 Iter::with_start(metadata, slots, start)
300 }
301
302 pub fn keys(&self) -> Keys<'_, K, V> {
304 Keys::new(self.iter())
305 }
306
307 pub fn values(&self) -> Values<'_, K, V> {
309 Values::new(self.iter())
310 }
311
312 #[inline]
317 fn as_slices(&self) -> SlotSlices<'_, K, V> {
318 if self.cap == 0 {
319 return (&[], &[]);
320 }
321 unsafe {
325 (
326 std::slice::from_raw_parts(self.metadata_ptr.as_ptr(), self.cap),
327 std::slice::from_raw_parts(self.slots_ptr.as_ptr(), self.cap),
328 )
329 }
330 }
331
332 #[inline(always)]
344 pub fn prefetch_for_hash(&self, hash: u64) {
345 if self.cap == 0 {
346 return;
347 }
348 let idx = (hash as usize) & self.mask;
349 let ptr = unsafe { self.metadata_ptr.as_ptr().add(idx) };
352 prefetch_t0(ptr);
353 }
354
355 #[inline]
357 pub(crate) fn threshold(&self) -> usize {
358 self.cap - (self.cap / 8)
359 }
360}
361
362
363impl<K, V> Default for KevyMap<K, V> {
364 fn default() -> Self {
365 Self::new()
366 }
367}
368
369impl<K, Q, V> std::ops::Index<&Q> for KevyMap<K, V>
371where
372 K: std::borrow::Borrow<Q>,
373 Q: KevyHash + Eq + ?Sized,
374{
375 type Output = V;
376 fn index(&self, key: &Q) -> &V {
377 self.get(key).expect("no entry found for key")
378 }
379}
380
381impl<K, V> Drop for KevyMap<K, V> {
382 fn drop(&mut self) {
383 if self.cap == 0 {
384 return;
385 }
386 if std::mem::needs_drop::<(K, V)>() {
387 for i in 0..self.cap {
388 let meta = unsafe { *self.metadata_ptr.as_ptr().add(i) };
390 if meta & 0x80 == 0 {
391 unsafe {
393 ptr::drop_in_place(self.slots_ptr.as_ptr().add(i) as *mut (K, V));
394 }
395 }
396 }
397 }
398 let (layout, _) = table_layout::<(K, V)>(self.cap);
402 unsafe {
405 dealloc(self.slots_ptr.as_ptr() as *mut u8, layout);
406 }
407 }
408}
409
410impl<K: fmt::Debug, V: fmt::Debug> fmt::Debug for KevyMap<K, V> {
411 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
412 f.debug_map().entries(self.iter()).finish()
413 }
414}
415
416impl<K: KevyHash + Eq, V> FromIterator<(K, V)> for KevyMap<K, V> {
417 fn from_iter<I: IntoIterator<Item = (K, V)>>(iter: I) -> Self {
418 let iter = iter.into_iter();
419 let mut m = match iter.size_hint() {
420 (lo, Some(hi)) if hi <= lo.saturating_mul(2) => Self::with_capacity(hi),
421 (lo, _) => Self::with_capacity(lo),
422 };
423 for (k, v) in iter {
424 m.insert(k, v);
425 }
426 m
427 }
428}
429
430impl<K: KevyHash + Eq, V> Extend<(K, V)> for KevyMap<K, V> {
431 fn extend<I: IntoIterator<Item = (K, V)>>(&mut self, iter: I) {
432 for (k, v) in iter {
433 self.insert(k, v);
434 }
435 }
436}
437
438#[cfg(test)]
439mod tests {
440 use super::*;
441 use crate::set::KevySet;
442 use std::cell::Cell;
443
444 #[test]
445 fn new_is_empty() {
446 let m: KevyMap<u64, u64> = KevyMap::new();
447 assert_eq!(m.len(), 0);
448 assert!(m.is_empty());
449 assert_eq!(m.capacity(), 0);
450 assert_eq!(m.get(&5u64), None);
451 assert!(!m.contains_key(&5u64));
452 }
453
454 #[test]
455 fn insert_get() {
456 let mut m = KevyMap::<u64, u64>::new();
457 assert!(m.insert(1, 10).is_none());
458 assert_eq!(m.len(), 1);
459 assert_eq!(m.get(&1u64), Some(&10));
460 assert_eq!(m.get(&2u64), None);
461 assert!(m.contains_key(&1u64));
462 }
463
464 #[test]
465 fn insert_duplicate_replaces_value_returns_old() {
466 let mut m = KevyMap::<u64, u64>::new();
467 assert_eq!(m.insert(1, 10), None);
468 assert_eq!(m.insert(1, 20), Some(10));
469 assert_eq!(m.len(), 1);
470 assert_eq!(m.get(&1u64), Some(&20));
471 }
472
473 #[test]
474 fn remove_returns_value_decreases_len() {
475 let mut m = KevyMap::<u64, u64>::new();
476 m.insert(1, 10);
477 m.insert(2, 20);
478 assert_eq!(m.remove(&1u64), Some(10));
479 assert_eq!(m.len(), 1);
480 assert_eq!(m.remove(&1u64), None);
481 assert_eq!(m.get(&1u64), None);
482 assert_eq!(m.get(&2u64), Some(&20));
483 }
484
485 #[test]
486 fn tombstone_reused_on_reinsert() {
487 let mut m = KevyMap::<u64, u64>::new();
488 m.insert(1, 10);
489 m.remove(&1u64);
490 assert_eq!(m.len(), 0);
491 m.insert(1, 30);
492 assert_eq!(m.len(), 1);
493 assert_eq!(m.get(&1u64), Some(&30));
494 }
495
496 #[test]
497 fn grow_preserves_all_entries_10k() {
498 let mut m = KevyMap::<u64, u64>::new();
499 for i in 0..10_000u64 {
500 m.insert(i, i.wrapping_mul(7));
501 }
502 assert_eq!(m.len(), 10_000);
503 for i in 0..10_000u64 {
504 assert_eq!(m.get(&i), Some(&i.wrapping_mul(7)));
505 }
506 }
507
508 #[test]
509 fn byte_string_keys_with_borrow_lookup() {
510 let mut m = KevyMap::<Vec<u8>, u64>::new();
511 m.insert(b"foo".to_vec(), 1);
512 m.insert(b"bar".to_vec(), 2);
513 assert_eq!(m.get(b"foo".as_slice()), Some(&1));
514 assert_eq!(m.get(b"missing".as_slice()), None);
515 assert_eq!(m.remove(b"bar".as_slice()), Some(2));
516 assert_eq!(m.len(), 1);
517 assert!(!m.contains_key(b"bar".as_slice()));
518 }
519
520 #[test]
521 fn iter_yields_all_entries() {
522 let mut m = KevyMap::<u64, u64>::new();
523 for i in 0..20u64 {
524 m.insert(i, i + 100);
525 }
526 let mut seen: Vec<(u64, u64)> = m.iter().map(|(&k, &v)| (k, v)).collect();
527 seen.sort();
528 let expected: Vec<(u64, u64)> = (0..20).map(|i| (i, i + 100)).collect();
529 assert_eq!(seen, expected);
530 }
531
532 struct DropCount<'a>(&'a Cell<usize>);
533 impl Drop for DropCount<'_> {
534 fn drop(&mut self) {
535 self.0.set(self.0.get() + 1);
536 }
537 }
538
539 #[test]
540 fn clear_drops_entries_and_resets_len() {
541 let counter = Cell::new(0);
542 let mut m: KevyMap<u64, DropCount<'_>> = KevyMap::new();
543 for i in 0..50u64 {
544 m.insert(i, DropCount(&counter));
545 }
546 assert_eq!(m.len(), 50);
547 m.clear();
548 assert_eq!(m.len(), 0);
549 assert_eq!(counter.get(), 50);
550 assert!(m.capacity() >= 50);
551 m.insert(0, DropCount(&counter));
552 assert_eq!(m.len(), 1);
553 drop(m);
554 assert_eq!(counter.get(), 51);
555 }
556
557 #[test]
558 fn drop_runs_for_remaining_entries() {
559 let counter = Cell::new(0);
560 {
561 let mut m: KevyMap<u64, DropCount<'_>> = KevyMap::new();
562 for i in 0..30u64 {
563 m.insert(i, DropCount(&counter));
564 }
565 m.remove(&5u64);
566 assert_eq!(counter.get(), 1);
567 }
568 assert_eq!(counter.get(), 30);
569 }
570
571 #[test]
572 fn grow_then_remove_then_grow_again_stays_consistent() {
573 let mut m = KevyMap::<u64, u64>::new();
574 for i in 0..2000u64 {
575 m.insert(i, i);
576 }
577 for i in 0..1000u64 {
578 assert_eq!(m.remove(&i), Some(i));
579 }
580 for i in 2000..4000u64 {
581 m.insert(i, i);
582 }
583 assert_eq!(m.len(), 3000);
584 for i in 1000..4000u64 {
585 assert_eq!(m.get(&i), Some(&i));
586 }
587 for i in 0..1000u64 {
588 assert_eq!(m.get(&i), None);
589 }
590 }
591
592 #[test]
593 fn with_capacity_preallocates() {
594 let m: KevyMap<u64, u64> = KevyMap::with_capacity(100);
595 assert_eq!(m.capacity(), 128);
597 let m: KevyMap<u64, u64> = KevyMap::with_capacity(0);
598 assert_eq!(m.capacity(), 0);
599 let m: KevyMap<u64, u64> = KevyMap::with_capacity(1);
600 assert_eq!(m.capacity(), MIN_CAP);
601 }
602
603 #[test]
604 fn get_mut_allows_mutation() {
605 let mut m = KevyMap::<u64, u64>::new();
606 m.insert(1, 10);
607 *m.get_mut(&1u64).unwrap() = 20;
608 assert_eq!(m.get(&1u64), Some(&20));
609 assert!(m.get_mut(&2u64).is_none());
610 }
611
612 #[test]
613 fn debug_format_matches_map_shape() {
614 let mut m = KevyMap::<u64, u64>::new();
615 m.insert(1, 10);
616 m.insert(2, 20);
617 let s = format!("{m:?}");
618 assert!(s.starts_with('{'));
620 assert!(s.ends_with('}'));
621 assert!(s.contains("1: 10") || s.contains("1:10"));
622 assert!(s.contains("2: 20") || s.contains("2:20"));
623 }
624
625 #[test]
626 fn into_iter_ref_works() {
627 let mut m = KevyMap::<u64, u64>::new();
628 m.insert(1, 10);
629 m.insert(2, 20);
630 let mut total = 0u64;
631 for (k, v) in &m {
632 total += *k + *v;
633 }
634 assert_eq!(total, 1 + 10 + 2 + 20);
635 }
636
637 #[test]
638 fn many_collisions_via_long_byte_keys() {
639 let mut m = KevyMap::<Vec<u8>, u64>::new();
644 let n = 5_000u64;
645 for i in 0..n {
646 let k = format!("session:{i:08}:user").into_bytes();
647 m.insert(k, i);
648 }
649 assert_eq!(m.len(), n as usize);
650 for i in 0..n {
651 let k = format!("session:{i:08}:user");
652 assert_eq!(m.get(k.as_bytes()), Some(&i));
653 }
654 }
655
656 #[test]
657 fn zst_value_type() {
658 let mut m = KevyMap::<u64, ()>::new();
659 assert!(m.insert(1, ()).is_none());
660 assert!(m.insert(1, ()).is_some());
661 assert!(m.contains_key(&1u64));
662 assert_eq!(m.remove(&1u64), Some(()));
663 }
664
665 #[test]
666 fn set_basic_ops() {
667 let mut s: KevySet<Vec<u8>> = KevySet::new();
668 assert!(s.insert(b"a".to_vec()));
669 assert!(!s.insert(b"a".to_vec())); assert_eq!(s.len(), 1);
671 assert!(s.contains(b"a".as_slice()));
672 assert!(!s.contains(b"b".as_slice()));
673 assert!(s.remove(b"a".as_slice()));
674 assert!(!s.remove(b"a".as_slice()));
675 assert!(s.is_empty());
676 }
677
678 #[test]
679 fn set_iter_yields_members() {
680 let mut s: KevySet<u64> = KevySet::new();
681 for i in 0..10u64 {
682 s.insert(i);
683 }
684 let mut got: Vec<u64> = s.iter().copied().collect();
685 got.sort();
686 assert_eq!(got, (0..10u64).collect::<Vec<_>>());
687 }
688
689 #[test]
690 fn prefetch_for_hash_is_safe_on_any_state() {
691 let m: KevyMap<u64, u64> = KevyMap::new();
695 m.prefetch_for_hash(0);
696 m.prefetch_for_hash(u64::MAX);
697 let mut m = KevyMap::<u64, u64>::new();
698 for i in 0..50u64 {
699 m.insert(i, i);
700 }
701 for i in 0..50u64 {
702 m.prefetch_for_hash(i.kevy_hash());
703 }
704 }
705
706 #[test]
707 fn capacity_grows_doubling_from_min_cap() {
708 let mut m = KevyMap::<u64, u64>::new();
709 m.insert(1, 1);
710 assert_eq!(m.capacity(), MIN_CAP);
711 for i in 2..=14u64 {
713 m.insert(i, i);
714 }
715 m.insert(15, 15);
717 assert_eq!(m.capacity(), MIN_CAP * 2);
718 }
719
720 #[test]
723 fn map_keys_iter() {
724 let mut m = KevyMap::<u64, u64>::new();
725 for i in 0..5u64 {
726 m.insert(i, i + 10);
727 }
728 let mut ks: Vec<u64> = m.keys().copied().collect();
729 ks.sort();
730 assert_eq!(ks, vec![0, 1, 2, 3, 4]);
731 }
732
733 #[test]
734 fn map_values_iter() {
735 let mut m = KevyMap::<u64, u64>::new();
736 for i in 0..5u64 {
737 m.insert(i, i + 10);
738 }
739 let mut vs: Vec<u64> = m.values().copied().collect();
740 vs.sort();
741 assert_eq!(vs, vec![10, 11, 12, 13, 14]);
742 }
743
744 #[test]
745 fn map_default_is_empty() {
746 let m: KevyMap<u64, u64> = KevyMap::default();
747 assert!(m.is_empty());
748 assert_eq!(m.capacity(), 0);
749 }
750
751 #[test]
752 fn map_from_iterator() {
753 let m: KevyMap<u64, u64> = (0..10u64).map(|i| (i, i * 2)).collect();
754 assert_eq!(m.len(), 10);
755 assert_eq!(m.get(&5u64), Some(&10));
756 }
757
758 #[test]
759 fn map_extend() {
760 let mut m = KevyMap::<u64, u64>::new();
761 m.extend((0..5u64).map(|i| (i, i)));
762 assert_eq!(m.len(), 5);
763 assert_eq!(m.get(&3u64), Some(&3));
764 }
765
766 #[test]
767 fn map_index_panics_on_missing() {
768 let mut m = KevyMap::<u64, u64>::new();
769 m.insert(1, 10);
770 assert_eq!(m[&1u64], 10);
771 let r = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
772 let _ = m[&99u64];
773 }));
774 assert!(r.is_err(), "Index on missing key should panic");
775 }
776
777 #[test]
778 fn set_with_capacity_capacity_clear() {
779 let mut s: KevySet<u64> = KevySet::with_capacity(50);
780 assert!(s.capacity() >= 50);
781 for i in 0..10u64 {
782 s.insert(i);
783 }
784 assert_eq!(s.len(), 10);
785 s.clear();
786 assert!(s.is_empty());
787 assert!(s.capacity() >= 50);
789 }
790
791 #[test]
792 fn set_as_map_smoke() {
793 let mut s: KevySet<u64> = KevySet::new();
794 s.insert(7);
795 assert_eq!(s.as_map().len(), 1);
796 assert!(s.as_map().contains_key(&7u64));
797 }
798
799 #[test]
800 fn set_default_debug() {
801 let s: KevySet<u64> = KevySet::default();
802 assert!(s.is_empty());
803 let dbg = format!("{s:?}");
804 assert_eq!(dbg, "{}");
805 }
806
807 #[test]
808 fn set_into_iter_ref() {
809 let mut s: KevySet<u64> = KevySet::new();
810 for i in 0..3u64 {
811 s.insert(i);
812 }
813 let mut sum = 0u64;
814 for k in &s {
815 sum += k;
816 }
817 assert_eq!(sum, 3);
818 }
819
820 #[test]
821 fn set_from_iterator() {
822 let s: KevySet<u64> = (0..5u64).collect();
823 assert_eq!(s.len(), 5);
824 assert!(s.contains(&3u64));
825 }
826
827 #[test]
828 fn set_extend() {
829 let mut s: KevySet<u64> = KevySet::new();
830 s.extend(0..5u64);
831 assert_eq!(s.len(), 5);
832 }
833}