catallaxy_runtime/
buf.rs

1//! Arrays of atomic words for containers whose readers do not lock (lists, and dicts read
2//! through their seqlock). Every slot is an `AtomicU64`, so a reader racing a writer sees some
3//! value that was stored there, never a torn one. A writer that outgrows a buffer copies it into
4//! a new one, publishes the new pointer with `Release`, and hands the old one to [`release`],
5//! which frees it only once no other thread can be reading it.
6
7use std::alloc::Layout;
8use std::sync::atomic::{AtomicU64, Ordering};
9
10use crate::ownership::{Access, PROMOTE_ON_ACCESS};
11
12/// A capacity word followed by `cap` atomic slots.
13#[repr(C)]
14pub struct WordBuf {
15    cap: usize,
16    slots: [AtomicU64; 0],
17}
18
19/// The buffer of every container with no capacity. Never written, never freed.
20static EMPTY: WordBuf = WordBuf { cap: 0, slots: [] };
21
22impl WordBuf {
23    fn layout(cap: usize) -> Layout {
24        Layout::from_size_align(8 * (cap + 1), 8).expect("container too large")
25    }
26
27    /// The shared empty buffer.
28    pub fn empty() -> *mut WordBuf {
29        &EMPTY as *const WordBuf as *mut WordBuf
30    }
31
32    /// A new buffer of `cap` slots, each holding `fill`.
33    pub fn alloc(cap: usize, fill: u64) -> *mut WordBuf {
34        if cap == 0 {
35            return Self::empty();
36        }
37        unsafe {
38            let p = std::alloc::alloc(Self::layout(cap)) as *mut WordBuf;
39            if p.is_null() {
40                std::alloc::handle_alloc_error(Self::layout(cap));
41            }
42            std::ptr::addr_of_mut!((*p).cap).write(cap);
43            let slots = Self::slots_ptr(p);
44            for i in 0..cap {
45                slots.add(i).write(AtomicU64::new(fill));
46            }
47            crate::note_external(cap * 8);
48            p
49        }
50    }
51
52    /// # Safety
53    /// `p` must come from [`WordBuf::alloc`] and be unreachable by any reader.
54    pub unsafe fn free(p: *mut u8) {
55        let b = p as *mut WordBuf;
56        let cap = (*b).cap;
57        if cap > 0 {
58            std::alloc::dealloc(p, Self::layout(cap));
59        }
60    }
61
62    #[inline(always)]
63    fn slots_ptr(p: *const WordBuf) -> *mut AtomicU64 {
64        unsafe { (p as *mut u8).add(8) as *mut AtomicU64 }
65    }
66
67    /// # Safety
68    /// `p` must be a live buffer, loaded with `Acquire` (or by its only writer); the slice must
69    /// not be held across a safepoint.
70    #[inline(always)]
71    pub unsafe fn slots<'a>(p: *const WordBuf) -> &'a [AtomicU64] {
72        std::slice::from_raw_parts(Self::slots_ptr(p), (*p).cap)
73    }
74
75    /// # Safety
76    /// As for [`WordBuf::slots`].
77    #[inline(always)]
78    pub unsafe fn cap(p: *const WordBuf) -> usize {
79        (*p).cap
80    }
81}
82
83/// Disposes of a buffer that was just replaced. With local access no other thread can be
84/// reading it, so it is freed now; otherwise the collector frees it at the next stop-the-world.
85///
86/// # Safety
87/// `old` must have been unpublished (replaced) by the caller, its only writer.
88pub unsafe fn release(old: *mut WordBuf, access: Access) {
89    if WordBuf::cap(old) == 0 {
90        return;
91    }
92    if access == Access::Local && !PROMOTE_ON_ACCESS.load(Ordering::Relaxed) {
93        WordBuf::free(old as *mut u8);
94    } else {
95        catallaxy_gc::retire(old as *mut u8, WordBuf::free);
96    }
97}
catallaxy_runtime/buf.rs

catallaxy_runtime/
buf.rs
