1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
use crate::{NullTag, Shared, Shield, Tag};
use core::{
    fmt,
    marker::PhantomData,
    mem,
    sync::atomic::{AtomicUsize, Ordering},
};

fn map_both<T, U, F>(result: Result<T, T>, f: F) -> Result<U, U>
where
    F: FnOnce(T) -> U + Copy,
{
    result.map(f).map_err(f)
}

/// An `Atomic` represents a tagged atomic pointer protected by the collection system.
///
/// This struct provides methods for manipulating the atomic pointer via
/// standard atomic operations using `Shared` as the corresponding non atomic version.
#[repr(transparent)]
pub struct Atomic<V, T1 = NullTag, T2 = NullTag>
where
    T1: Tag,
    T2: Tag,
{
    pub(crate) data: AtomicUsize,
    _m0: PhantomData<V>,
    _m1: PhantomData<T1>,
    _m2: PhantomData<T2>,
}

impl<V, T1, T2> Atomic<V, T1, T2>
where
    T1: Tag,
    T2: Tag,
{
    /// Constructs an `Atomic` from a raw tagged pointer represented as an integer.
    ///
    /// # Safety
    /// Marked unsafe because this is not usually what the user wants.
    /// `Atomic::null` should be preferred when possible.
    pub unsafe fn from_raw(raw: usize) -> Self {
        Self {
            data: AtomicUsize::new(raw),
            _m0: PhantomData,
            _m1: PhantomData,
            _m2: PhantomData,
        }
    }

    /// Constructs a new `Atomic` from a tagged pointer.
    ///
    /// # Safety
    /// The alignment of `V` must free up sufficient low bits so that `T` fits.
    pub fn new(shared: Shared<'_, V, T1, T2>) -> Self {
        unsafe { Self::from_raw(shared.into_raw()) }
    }

    /// Constructs a new `Atomic` with a null value.
    pub fn null() -> Self {
        unsafe { Self::from_raw(0) }
    }

    /// This constructs a `Vec<Atomic>` with null values in an optimized manner.
    pub fn null_vec(len: usize) -> Vec<Self> {
        unsafe { mem::transmute(vec![0_usize; len]) }
    }

    /// Load a the tagged pointer.
    pub fn load<'collector, 'shield, S>(
        &self,
        ordering: Ordering,
        _shield: &'shield S,
    ) -> Shared<'shield, V, T1, T2>
    where
        S: Shield<'collector>,
    {
        let raw = self.data.load(ordering);
        unsafe { Shared::from_raw(raw) }
    }

    /// Store a tagged pointer, replacing the previous value.
    pub fn store(&self, data: Shared<'_, V, T1, T2>, ordering: Ordering) {
        let raw = data.into_raw();
        self.data.store(raw, ordering);
    }

    /// Swap the stored tagged pointer, returning the old one.
    pub fn swap<'collector, 'shield, S>(
        &self,
        new: Shared<'_, V, T1, T2>,
        ordering: Ordering,
        _shield: &'shield S,
    ) -> Shared<'shield, V, T1, T2>
    where
        S: Shield<'collector>,
    {
        let new_raw = new.into_raw();
        let old_raw = self.data.swap(new_raw, ordering);
        unsafe { Shared::from_raw(old_raw) }
    }

    /// Conditionally swap the stored tagged pointer, always returns the previous value.
    pub fn compare_and_swap<'collector, 'shield, S>(
        &self,
        current: Shared<'_, V, T1, T2>,
        new: Shared<'_, V, T1, T2>,
        order: Ordering,
        _shield: &'shield S,
    ) -> Shared<'shield, V, T1, T2>
    where
        S: Shield<'collector>,
    {
        let current_raw = current.into_raw();
        let new_raw = new.into_raw();
        let old_raw = self.data.compare_and_swap(current_raw, new_raw, order);
        unsafe { Shared::from_raw(old_raw) }
    }

    /// Conditionally exchange the stored tagged pointer, always returns
    /// the previous value and a result indicating if it was written or not.
    /// On success this value is guaranteed to be equal to current.
    pub fn compare_exchange<'collector, 'shield, S>(
        &self,
        current: Shared<'_, V, T1, T2>,
        new: Shared<'_, V, T1, T2>,
        success: Ordering,
        failure: Ordering,
        _shield: &'shield S,
    ) -> Result<Shared<'shield, V, T1, T2>, Shared<'shield, V, T1, T2>>
    where
        S: Shield<'collector>,
    {
        let current_raw = current.into_raw();
        let new_raw = new.into_raw();
        let result = self
            .data
            .compare_exchange(current_raw, new_raw, success, failure);

        map_both(result, |raw| unsafe { Shared::from_raw(raw) })
    }

    /// Conditionally exchange the stored tagged pointer, always returns
    /// the previous value and a result indicating if it was written or not.
    /// On success this value is guaranteed to be equal to current.
    ///
    /// This variant may spuriously fail on platforms where LL/SC is used.
    /// This allows more efficient code generation on those platforms.
    pub fn compare_exchange_weak<'collector, 'shield, S>(
        &self,
        current: Shared<'_, V, T1, T2>,
        new: Shared<'_, V, T1, T2>,
        success: Ordering,
        failure: Ordering,
        _shield: &'shield S,
    ) -> Result<Shared<'shield, V, T1, T2>, Shared<'shield, V, T1, T2>>
    where
        S: Shield<'collector>,
    {
        let current_raw = current.into_raw();
        let new_raw = new.into_raw();
        let result = self
            .data
            .compare_exchange_weak(current_raw, new_raw, success, failure);

        map_both(result, |raw| unsafe { Shared::from_raw(raw) })
    }
}

unsafe impl<V, T1, T2> Send for Atomic<V, T1, T2>
where
    T1: Tag,
    T2: Tag,
{
}

unsafe impl<V, T1, T2> Sync for Atomic<V, T1, T2>
where
    T1: Tag,
    T2: Tag,
{
}

impl<V, T1, T2> Unpin for Atomic<V, T1, T2>
where
    T1: Tag,
    T2: Tag,
{
}

impl<V, T1, T2> fmt::Debug for Atomic<V, T1, T2>
where
    T1: Tag,
    T2: Tag,
{
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        use crate::tag;
        let data = self.data.load(Ordering::SeqCst);
        let lo = tag::read_tag::<T1>(data, tag::TagPosition::Lo);
        let hi = tag::read_tag::<T2>(data, tag::TagPosition::Hi);

        f.debug_struct("Atomic")
            .field("raw", &data)
            .field("low_tag", &lo)
            .field("high_tag", &hi)
            .finish()
    }
}