1#![feature(option_result_unwrap_unchecked)]
74#![feature(ptr_metadata)]
75#![feature(maybe_uninit_array_assume_init)]
76#![feature(maybe_uninit_uninit_array)]
77#![feature(option_zip)]
78#![warn(missing_docs)]
79
80use std::panic::{RefUnwindSafe, UnwindSafe};
81use std::ptr::NonNull;
82
83use bucket::Bucket;
84
85use crate::block::Block;
86pub use crate::id::*;
87use crate::iter::{IntoIter, IterMut};
88use crate::lock::{ReadLock, WriteLock};
89use crate::rwstore_id::RwStoreId;
90pub use crate::timeout::*;
91use crate::timeout::{BlockResult, Timeout};
92use crate::util::sync::concurrent_queue::ConcurrentQueue;
93use crossbeam_utils::CachePadded;
94use std::cell::UnsafeCell;
95use std::mem::MaybeUninit;
96use util::sync::thread;
97
98mod block;
99mod bucket;
100mod header;
101pub mod id;
102pub mod iter;
103pub mod lock;
104#[cfg(all(test, loom))]
105mod loom;
106mod rwstore_id;
107pub mod timeout;
108mod util;
109
110pub use header::MAX_CONCURRENT_READS;
111
112const BUCKET_COUNT: usize = 16;
113
114pub struct RwStore<Element> {
117 buckets: [CachePadded<Bucket<Element>>; BUCKET_COUNT],
118 erasures: ConcurrentQueue<Erasure>,
120 store_id: RwStoreId,
123}
124
125impl<Element> RwStore<Element> {
126 pub fn new() -> Self {
128 let store_id = RwStoreId::generate();
129
130 let buckets = unsafe {
131 let mut buckets = MaybeUninit::uninit_array();
132
133 for bucket in &mut buckets {
134 *bucket = MaybeUninit::new(CachePadded::new(Bucket::new(store_id)));
135 }
136
137 MaybeUninit::array_assume_init(buckets)
138 };
139
140 Self {
141 store_id,
142 buckets,
143 erasures: ConcurrentQueue::new(),
144 }
145 }
146
147 pub fn insert(&self, element: Element) -> Id {
158 let slot_address = if let Some(erasure) = self.erasures.pop() {
159 erasure.slot_address
160 } else {
161 let bucket_id = self.arbitrary_bucket_id();
162 let bucket = &self.buckets[bucket_id as usize];
163 let slot_address = bucket.next_insert_location();
164 slot_address
165 };
166
167 unsafe { Block::insert(slot_address, element, self.store_id) }
168 }
169
170 pub fn remove(&self, id: Id) -> Option<Element> {
189 self.remove_with_timeout(id, Timeout::BlockIndefinitely)
190 .unwrap()
191 }
192
193 pub fn remove_with_timeout(&self, id: Id, timeout: Timeout) -> BlockResult<Option<Element>> {
215 self.assert_native_id(id);
216
217 unsafe {
218 if let Some(result) = Block::<Element>::remove(id, timeout)? {
219 if result.may_reuse {
220 self.push_erasure(id);
221 }
222
223 Ok(Some(result.element))
224 } else {
225 Ok(None)
226 }
227 }
228 }
229
230 pub fn remove_locked(&self, lock: WriteLock<Element>) -> Element {
250 let id = lock.forget();
251
252 self.assert_native_id(id);
253
254 let result = unsafe { Block::<Element>::remove_locked(id) };
255
256 if result.may_reuse {
257 self.push_erasure(id);
258 }
259
260 result.element
261 }
262
263 fn push_erasure(&self, id: Id) {
264 self.erasures.push(Erasure {
265 slot_address: id.slot(),
266 });
267 }
268
269 pub fn read(&self, id: Id) -> Option<ReadLock<Element>> {
291 self.read_with_timeout(id, Timeout::BlockIndefinitely)
292 .unwrap()
293 }
294
295 pub fn read_with_timeout(
319 &self,
320 id: Id,
321 timeout: Timeout,
322 ) -> BlockResult<Option<ReadLock<Element>>> {
323 self.assert_native_id(id);
324
325 unsafe {
326 if Block::<Element>::lock_read(id, timeout)? {
327 let lock = ReadLock::new(id);
328 Ok(Some(lock))
329 } else {
330 Ok(None)
331 }
332 }
333 }
334
335 pub fn write(&self, id: Id) -> Option<WriteLock<Element>> {
354 self.write_with_timeout(id, Timeout::BlockIndefinitely)
355 .unwrap()
356 }
357
358 pub fn write_with_timeout(
379 &self,
380 id: Id,
381 timeout: Timeout,
382 ) -> BlockResult<Option<WriteLock<Element>>> {
383 self.assert_native_id(id);
384
385 unsafe {
386 if Block::<Element>::lock_write(id, timeout)? {
387 let lock = WriteLock::new(id);
388 Ok(Some(lock))
389 } else {
390 Ok(None)
391 }
392 }
393 }
394
395 pub fn get_mut(&mut self, id: Id) -> Option<&mut Element> {
414 self.assert_native_id(id);
415
416 unsafe { Block::<Element>::get_exclusive(id).map(|mut ptr| ptr.as_mut()) }
417 }
418
419 pub unsafe fn get_mut_unchecked(&mut self, id: Id) -> &mut Element {
444 self.assert_native_id(id);
445 Block::<Element>::get_unchecked(id).as_mut()
446 }
447
448 pub fn iter_mut(&mut self) -> IterMut<Element> {
464 IterMut::new(self)
465 }
466
467 pub fn capacity(&self) -> (u32, u32) {
488 let (mut total_touched, mut total_allocated) = (0, 0);
489
490 for bucket in &self.buckets {
491 let (touched, allocated) = bucket.capacity();
492 total_touched += touched;
493 total_allocated += allocated;
494 }
495
496 (total_touched, total_allocated)
497 }
498
499 fn arbitrary_bucket_id(&self) -> u32 {
502 thread_local! {
505 static STATE: UnsafeCell<u32> = UnsafeCell::new(thread::current_thread_hash() as u32);
506 }
507
508 STATE.with(|state| unsafe {
509 const MULTIPLIER: u32 = 1103515245;
510 const CONSTANT: u32 = 12345;
511
512 let state = &mut *state.get();
513 *state = state.wrapping_mul(MULTIPLIER).wrapping_add(CONSTANT);
514 *state % BUCKET_COUNT as u32
515 })
516 }
517
518 fn assert_native_id(&self, id: Id) {
519 debug_assert!(
520 self.store_id == id.store_id(),
521 "attempted to use an ID created with a different store"
522 )
523 }
524}
525
526impl<Element> Default for RwStore<Element> {
527 fn default() -> Self {
528 Self::new()
529 }
530}
531
532impl<Element> IntoIterator for RwStore<Element> {
533 type Item = (Id, Element);
534 type IntoIter = IntoIter<Element>;
535
536 fn into_iter(self) -> IntoIter<Element> {
541 IntoIter::new(self)
542 }
543}
544
545unsafe impl<Element: Send + Sync> Sync for RwStore<Element> {}
546
547unsafe impl<Element: Send> Send for RwStore<Element> {}
548
549impl<Element: UnwindSafe> UnwindSafe for RwStore<Element> {}
550
551impl<Element: RefUnwindSafe> RefUnwindSafe for RwStore<Element> {}
552
553struct Erasure {
554 slot_address: NonNull<()>,
555}
556
557#[cfg(test)]
558mod test {
559 use std::ops::Deref;
560 use std::panic::{RefUnwindSafe, UnwindSafe};
561
562 use crate::Timeout::DontBlock;
563 use crate::{BlockResult, Id, RwStore};
564
565 #[test]
566 fn insert_creates_disparate_ids() {
567 let store = RwStore::new();
568 let id_a = store.insert(42);
569 let id_b = store.insert(42);
570
571 assert_ne!(id_a.slot::<()>(), id_b.slot());
572 }
573
574 #[test]
575 fn insert_reuses_space_after_removal() {
576 let store = RwStore::new();
577
578 let id_a = store.insert(42);
579 store.remove(id_a).unwrap();
580
581 let id_b = store.insert(42);
582
583 assert_eq!(id_a.slot::<()>(), id_b.slot());
584 }
585
586 #[test]
587 fn insert_reuses_space_after_locked_removal() {
588 let store = RwStore::new();
589
590 let id_a = store.insert(42);
591 store.remove_locked(store.write(id_a).unwrap());
592
593 let id_b = store.insert(42);
594
595 assert_eq!(id_a.slot::<()>(), id_b.slot());
596 }
597
598 #[test]
599 fn insert_doesnt_reuse_id_ordinals() {
600 let store = RwStore::new();
601
602 let id_a = store.insert(42);
603 store.remove(id_a).unwrap();
604
605 let id_b = store.insert(42);
606
607 assert_ne!(id_a.ordinal(), id_b.ordinal());
608 }
609
610 #[test]
611 fn remove_returns_the_element() {
612 gen_remove_returns_the_element(|store, id| store.remove(id));
613 }
614
615 #[test]
616 fn remove_returns_none_after_removal() {
617 gen_remove_returns_none_after_removal(|store, id| store.remove(id));
618 }
619
620 #[test]
621 fn remove_with_timeout_returns_the_element() {
622 gen_remove_returns_the_element(|store, id| {
623 store.remove_with_timeout(id, DontBlock).unwrap()
624 });
625 }
626
627 #[test]
628 fn remove_with_timeout_returns_none_after_removal() {
629 gen_remove_returns_none_after_removal(|store, id| {
630 store.remove_with_timeout(id, DontBlock).unwrap()
631 });
632 }
633
634 #[test]
635 fn remove_with_timeout_fails_when_read_locked() {
636 let store = RwStore::new();
637 let id = store.insert(42);
638
639 let _lock = store.read(id).unwrap();
640
641 assert!(store.remove_with_timeout(id, DontBlock).is_err());
642 }
643
644 #[test]
645 fn remove_with_timeout_fails_when_write_locked() {
646 let store = RwStore::new();
647 let id = store.insert(42);
648
649 let _lock = store.write(id).unwrap();
650
651 assert!(store.remove_with_timeout(id, DontBlock).is_err());
652 }
653
654 fn gen_remove_returns_the_element(remover: impl Fn(&RwStore<u32>, Id) -> Option<u32>) {
655 let store = RwStore::new();
656 let id = store.insert(42);
657
658 assert_eq!(remover(&store, id), Some(42));
659 }
660
661 fn gen_remove_returns_none_after_removal(remover: impl Fn(&RwStore<u32>, Id) -> Option<u32>) {
662 let store = RwStore::new();
663 let id = store.insert(42);
664 store.remove(id);
665
666 assert_eq!(remover(&store, id), None);
667 }
668
669 #[test]
670 fn remove_locked_returns_the_element() {
671 let store = RwStore::new();
672 let id = store.insert(42);
673
674 let lock = store.write(id).unwrap();
675
676 assert_eq!(store.remove_locked(lock), 42);
677 }
678
679 #[test]
680 fn remove_locked_removes_the_element() {
681 let store = RwStore::new();
682 let id = store.insert(42);
683
684 let lock = store.write(id).unwrap();
685 store.remove_locked(lock);
686
687 assert!(store.read(id).is_none());
688 }
689
690 #[test]
691 fn read_references_the_element() {
692 access_references_the_element::<ReadOperation>();
693 }
694
695 #[test]
696 fn write_references_the_element() {
697 access_references_the_element::<WriteOperation>();
698 }
699
700 #[test]
701 fn read_with_timeout_references_the_element() {
702 access_references_the_element::<ReadWithTimeoutOperation>();
703 }
704
705 #[test]
706 fn write_with_timeout_references_the_element() {
707 access_references_the_element::<WriteWithTimeoutOperation>();
708 }
709
710 fn access_references_the_element<A: AccessOperation>() {
711 let store = RwStore::new();
712 let id = store.insert(42);
713
714 let lock = A::access(&store, id).unwrap().unwrap();
715
716 assert_eq!(**lock, 42);
717 }
718
719 #[test]
720 fn read_fails_returns_none_removal() {
721 access_returns_none_after_removal::<ReadOperation>();
722 }
723
724 #[test]
725 fn write_fails_returns_none_removal() {
726 access_returns_none_after_removal::<WriteOperation>();
727 }
728
729 #[test]
730 fn read_with_timeout_returns_none_after_removal() {
731 access_returns_none_after_removal::<ReadWithTimeoutOperation>();
732 }
733
734 #[test]
735 fn write_with_timeout_returns_none_after_removal() {
736 access_returns_none_after_removal::<WriteWithTimeoutOperation>();
737 }
738
739 fn access_returns_none_after_removal<A: AccessOperation>() {
740 let store = RwStore::new();
741 let id = store.insert(42);
742
743 store.remove(id).unwrap();
744
745 assert!(A::access(&store, id).unwrap().is_none());
746 }
747
748 #[test]
749 fn write_with_timeout_fails_when_write_locked() {
750 access_fails_when_locked::<WriteOperation, WriteWithTimeoutOperation>();
751 }
752
753 #[test]
754 fn write_with_timeout_fails_when_read_locked() {
755 access_fails_when_locked::<ReadOperation, WriteWithTimeoutOperation>();
756 }
757
758 #[test]
759 fn read_with_timeout_fails_when_write_locked() {
760 access_fails_when_locked::<WriteOperation, ReadWithTimeoutOperation>();
761 }
762
763 fn access_fails_when_locked<Lock: AccessOperation, A: AccessOperation>() {
764 let store = RwStore::new();
765 let id = store.insert(42);
766
767 let _lock = Lock::access(&store, id);
768
769 assert!(A::access(&store, id).is_err());
770 }
771
772 #[test]
773 fn read_succeeds_when_read_locked() {
774 access_succeeds_when_locked::<ReadOperation, ReadOperation>();
775 }
776
777 #[test]
778 fn read_with_timeout_succeeds_when_read_locked() {
779 access_succeeds_when_locked::<ReadWithTimeoutOperation, ReadOperation>();
780 }
781
782 fn access_succeeds_when_locked<Lock: AccessOperation, A: AccessOperation>() {
783 let store = RwStore::new();
784 let id = store.insert(42);
785
786 let _lock = Lock::access(&store, id);
787
788 assert!(A::access(&store, id).is_ok());
789 }
790
791 #[test]
792 fn get_mut_references_the_element() {
793 let mut store = RwStore::new();
794 let id = store.insert(42);
795
796 assert_eq!(store.get_mut(id), Some(&mut 42));
797 }
798
799 #[test]
800 fn get_mut_returns_none_after_removal() {
801 let mut store = RwStore::new();
802 let id = store.insert(42);
803
804 store.remove(id).unwrap();
805
806 assert_eq!(store.get_mut(id), None);
807 }
808
809 #[test]
810 fn get_mut_unchecked_references_the_element() {
811 let mut store = RwStore::new();
812 let id = store.insert(42);
813
814 unsafe {
815 assert_eq!(store.get_mut_unchecked(id), &mut 42);
816 }
817 }
818
819 #[test]
820 fn capacity_returns_zeroes_initially() {
821 let store = RwStore::<u32>::new();
822 assert_eq!(store.capacity(), (0, 0));
823 }
824
825 #[test]
826 fn capacity_increases_on_first_insertion() {
827 let store = RwStore::<u32>::new();
828 store.insert(42);
829 assert_eq!(store.capacity().0, 1);
830 assert!(store.capacity().1 >= 1);
831 }
832
833 #[test]
834 fn capacity_increases_on_second_insertion() {
835 let store = RwStore::<u32>::new();
836 store.insert(42);
837 store.insert(42);
838 assert_eq!(store.capacity().0, 2);
839 assert!(store.capacity().1 >= 2);
840 }
841
842 #[test]
843 fn capacity_doesnt_decrease_after_removal() {
844 let store = RwStore::<u32>::new();
845 let id = store.insert(42);
846 store.remove(id).unwrap();
847 assert_eq!(store.capacity().0, 1);
848 assert!(store.capacity().1 >= 1);
849 }
850
851 #[test]
852 fn capacity_doesnt_increase_on_insertion_after_removal() {
853 let store = RwStore::<u32>::new();
854 let id = store.insert(42);
855 store.remove(id).unwrap();
856 store.insert(42);
857 assert_eq!(store.capacity().0, 1);
858 assert!(store.capacity().1 >= 1);
859 }
860
861 #[test]
862 fn capacity_doesnt_increase_on_insertion_after_locked_removal() {
863 let store = RwStore::<u32>::new();
864 let id = store.insert(42);
865 store.remove_locked(store.write(id).unwrap());
866 store.insert(42);
867 assert_eq!(store.capacity().0, 1);
868 assert!(store.capacity().1 >= 1);
869 }
870
871 #[test]
872 fn implements_sync() {
873 let store = RwStore::<u32>::new();
874 &store as &dyn Sync;
875 }
876
877 #[test]
878 fn implements_send() {
879 let store = RwStore::<u32>::new();
880 &store as &dyn Send;
881 }
882
883 #[test]
884 fn implements_unwind_safe() {
885 let store = RwStore::<u32>::new();
886 &store as &dyn UnwindSafe;
887 }
888
889 #[test]
890 fn implements_ref_unwind_safe() {
891 let store = RwStore::<u32>::new();
892 &store as &dyn RefUnwindSafe;
893 }
894
895 trait AccessOperation {
896 fn access<'a>(
897 store: &'a RwStore<u32>,
898 id: Id,
899 ) -> BlockResult<Option<Box<dyn Deref<Target = u32> + 'a>>>;
900 }
901
902 struct ReadOperation;
903
904 impl AccessOperation for ReadOperation {
905 fn access<'a>(
906 store: &'a RwStore<u32>,
907 id: Id,
908 ) -> BlockResult<Option<Box<dyn Deref<Target = u32> + 'a>>> {
909 let result = store
910 .read(id)
911 .map(|lock| Box::new(lock) as Box<dyn Deref<Target = u32>>);
912
913 Ok(result)
914 }
915 }
916
917 struct WriteOperation;
918
919 impl AccessOperation for WriteOperation {
920 fn access<'a>(
921 store: &'a RwStore<u32>,
922 id: Id,
923 ) -> BlockResult<Option<Box<dyn Deref<Target = u32> + 'a>>> {
924 let result = store
925 .write(id)
926 .map(|lock| Box::new(lock) as Box<dyn Deref<Target = u32>>);
927
928 Ok(result)
929 }
930 }
931
932 struct ReadWithTimeoutOperation;
933
934 impl AccessOperation for ReadWithTimeoutOperation {
935 fn access<'a>(
936 store: &'a RwStore<u32>,
937 id: Id,
938 ) -> BlockResult<Option<Box<dyn Deref<Target = u32> + 'a>>> {
939 store
940 .read_with_timeout(id, DontBlock)
941 .map(|result| result.map(|lock| Box::new(lock) as Box<dyn Deref<Target = u32>>))
942 }
943 }
944
945 struct WriteWithTimeoutOperation;
946
947 impl AccessOperation for WriteWithTimeoutOperation {
948 fn access<'a>(
949 store: &'a RwStore<u32>,
950 id: Id,
951 ) -> BlockResult<Option<Box<dyn Deref<Target = u32> + 'a>>> {
952 store
953 .write_with_timeout(id, DontBlock)
954 .map(|result| result.map(|lock| Box::new(lock) as Box<dyn Deref<Target = u32>>))
955 }
956 }
957}