1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
//! This implements a bounded lock-free Queue
//!
//! # Example
//! ```
//! use nolock::queues::spsc::bounded;
//!
//! // Creates a new BoundedQueue with the Capacity for 5 Items
//! let (mut rx, mut tx) = bounded::queue(5);
//!
//! // Enqueues the Value 13 on the Queue
//! tx.try_enqueue(13);
//! // Dequeues 13 from the Queue again
//! assert_eq!(Ok(13), rx.try_dequeue());
//! ```
//!
//! # Reference:
//! * [FastForward for Efficient Pipeline Parallelism - A Cache-Optimized Concurrent Lock-Free Queue](https://www.researchgate.net/publication/213894711_FastForward_for_Efficient_Pipeline_Parallelism_A_Cache-Optimized_Concurrent_Lock-Free_Queue)

use std::{
    fmt::Debug,
    sync::{atomic, Arc},
};

use super::{DequeueError, EnqueueError};

#[cfg(feature = "async")]
mod async_queue;
#[cfg(feature = "async")]
pub use async_queue::*;

mod node;
use node::Node;

/// The Sending-Half for the queue
pub struct BoundedSender<T> {
    /// Indicates if the Queue has been closed or not
    closed: Arc<atomic::AtomicBool>,
    /// The Index of the next Node to read in the Buffer
    head: usize,
    /// The underlying Buffer of Nodes
    buffer: Arc<Vec<Node<T>>>,
}

/// The Receiving-Half for the Queue
pub struct BoundedReceiver<T> {
    /// Indicates if the Queue has been closed or not
    closed: Arc<atomic::AtomicBool>,
    /// The Index of the next Node to store Data into
    tail: usize,
    /// The underlying Buffer of Nodes
    buffer: Arc<Vec<Node<T>>>,
}

/// Calculates the Index of the next Element in the Buffer and wraps around
/// if the End of the Buffer has been reached
#[inline(always)]
const fn next_element(current: usize, length: usize) -> usize {
    // This code is logically speaking identical to simply returning
    // `(current + 1) % length`
    // However after performing some benchmarks, this code seems to be
    // significantly, in this context, faster than the simple naive variant
    let target = current + 1;
    if target >= length {
        0
    } else {
        target
    }
}

impl<T> BoundedSender<T> {
    /// Returns whether or not the Queue has been closed by the Consumer
    ///
    /// # Example
    /// ```
    /// # use nolock::queues::spsc::bounded;
    /// let (rx, tx) = bounded::queue::<usize>(3);
    ///
    /// // Drop the Consumer and therefore also close the Queue
    /// drop(rx);
    ///
    /// assert_eq!(true, tx.is_closed());
    /// ```
    pub fn is_closed(&self) -> bool {
        self.closed.load(atomic::Ordering::Acquire)
    }

    /// Attempts to Enqueue the given piece of Data
    ///
    /// # Example:
    /// Enqueue Data when there is still space
    /// ```
    /// # use nolock::queues::spsc::bounded;
    /// // Create a new Queue with the capacity for 16 Elements
    /// let (mut rx, mut tx) = bounded::queue::<usize>(16);
    ///
    /// // Enqueue some Data
    /// assert_eq!(Ok(()), tx.try_enqueue(13));
    ///
    /// # assert_eq!(Ok(13), rx.try_dequeue());
    /// ```
    ///
    /// Enqueue Data when there is no more space
    /// ```
    /// # use nolock::queues::spsc::bounded;
    /// # use nolock::queues::spsc::EnqueueError;
    /// // Create a new Queue with the capacity for 16 Elements
    /// let (mut rx, mut tx) = bounded::queue::<usize>(16);
    ///
    /// // Fill up the Queue
    /// for i in 0..16 {
    ///   assert_eq!(Ok(()), tx.try_enqueue(i));
    /// }
    ///
    /// // Attempt to enqueue some Data, but there is no more room
    /// assert_eq!(Err((13, EnqueueError::WouldBlock)), tx.try_enqueue(13));
    ///
    /// # drop(rx);
    /// ```
    pub fn try_enqueue(&mut self, data: T) -> Result<(), (T, EnqueueError)> {
        if self.is_closed() {
            return Err((data, EnqueueError::Closed));
        }

        // Get a reference to the current Entry where we would enqueue the next
        // Element
        let buffer_entry = unsafe { self.buffer.get_unchecked(self.head) };

        // If the Node is already set, that means we don't have anywhere to
        // store the new Element, meaning that the Buffer is full and we should
        // Error out indicating this
        if buffer_entry.is_set() {
            return Err((data, EnqueueError::WouldBlock));
        }

        // The Node is not already set meaning that we can simply store the
        // given Data into the Node
        buffer_entry.store(data);

        // Advance the current Head, where we insert the Elements, onto the
        // next Position
        self.head = next_element(self.head, self.buffer.len());

        // Return Ok to indicate a successful enqueue operation
        Ok(())
    }

    /// A blocking enqueue Operation. This is obviously not lock-free anymore
    /// and will simply spin while trying to enqueue the Data until it works
    pub fn enqueue(&mut self, mut data: T) -> Result<(), (T, EnqueueError)> {
        loop {
            match self.try_enqueue(data) {
                Ok(_) => return Ok(()),
                Err((d, e)) => match e {
                    EnqueueError::WouldBlock => {
                        data = d;
                    }
                    EnqueueError::Closed => return Err((d, EnqueueError::Closed)),
                },
            };
        }
    }

    /// Checks if the current Queue is full
    pub fn is_full(&self) -> bool {
        // If the Node where we would insert the next Element is already set
        // that means, that there is currently no room for new Elements in the
        // Queue, meaning that the Queue is currently full
        self.buffer[self.head].is_set()
    }
}

impl<T> Debug for BoundedSender<T> {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        write!(f, "BoundedSender ()")
    }
}

impl<T> Drop for BoundedSender<T> {
    fn drop(&mut self) {
        self.closed.store(true, atomic::Ordering::Release);
    }
}

unsafe impl<T> Send for BoundedSender<T> {}
unsafe impl<T> Sync for BoundedSender<T> {}

impl<T> BoundedReceiver<T> {
    /// Checks if the Queue has been closed by the Producer
    ///
    /// # Note
    /// Even when this indicates that the Queue has been closed, there might
    /// still be Items in the Queue left that should first be dequeued by the
    /// Consumer before discarding the entire Queue
    ///
    /// # Example
    /// ```
    /// # use nolock::queues::spsc::bounded;
    /// let (rx, tx) = bounded::queue::<usize>(3);
    ///
    /// // Drop the Producer and therefore also close the Queue
    /// drop(tx);
    ///
    /// assert_eq!(true, rx.is_closed());
    /// ```
    pub fn is_closed(&self) -> bool {
        self.closed.load(atomic::Ordering::Acquire)
    }

    /// Attempts to Dequeue a single Element from the Queue
    ///
    /// # Example
    /// There was something to dequeu
    /// ```
    /// # use nolock::queues::spsc::bounded;
    /// // Create a new Queue with the Capacity for 16-Elements
    /// let (mut rx, mut tx) = bounded::queue::<usize>(16);
    ///
    /// // Enqueue the Element
    /// tx.try_enqueue(13);
    ///
    /// // Dequeue the Element again
    /// assert_eq!(Ok(13), rx.try_dequeue());
    /// ```
    ///
    /// The Queue is empty and therefore nothing could be dequeued
    /// ```
    /// # use nolock::queues::spsc::bounded;
    /// # use nolock::queues::spsc::DequeueError;
    /// // Create a new Queue with the Capacity for 16-Elements
    /// let (mut rx, mut tx) = bounded::queue::<usize>(16);
    ///
    /// // Dequeue the Element again
    /// assert_eq!(Err(DequeueError::WouldBlock), rx.try_dequeue());
    ///
    /// # drop(tx);
    /// ```
    pub fn try_dequeue(&mut self) -> Result<T, DequeueError> {
        // Get the Node where would read the next Item from
        let buffer_entry = unsafe { self.buffer.get_unchecked(self.tail) };

        // If the Node is not set, we should return an Error as the Queue is
        // empty and there is nothing for us to return in this Operation
        if !buffer_entry.is_set() {
            // Check if the Queue has been marked as closed
            if self.is_closed() {
                // We need to recheck the current Node, because it may have
                // been set in the mean time and then the closed flag was
                // updated
                if !buffer_entry.is_set() {
                    return Err(DequeueError::Closed);
                }
            }

            return Err(DequeueError::WouldBlock);
        }

        // If the Node is set, we can load the Data out of the Node itself
        let data = buffer_entry.load();

        // Advance the current Tail, indicating where we should read the next
        // Element from, onto the next Node in the Buffer
        self.tail = next_element(self.tail, self.buffer.len());

        // Return `Ok` with the Data
        Ok(data)
    }

    /// A blocking dequeue operations. This is not lock-free anymore and simply
    /// spins while trying to dequeue until it works.
    pub fn dequeue(&mut self) -> Option<T> {
        loop {
            match self.try_dequeue() {
                Ok(d) => return Some(d),
                Err(e) => match e {
                    DequeueError::WouldBlock => {}
                    DequeueError::Closed => return None,
                },
            };
        }
    }

    /// Checks if the current queue is empty
    pub fn is_empty(&self) -> bool {
        // If the current Node where would dequeue the next Item from is not
        // marked as being set, the Node contains no `set` Nodes and therefore
        // the Queue is currently empty
        !self.buffer[self.tail].is_set()
    }
}

impl<T> Debug for BoundedReceiver<T> {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        write!(f, "BoundedReceiver ()")
    }
}

impl<T> Drop for BoundedReceiver<T> {
    fn drop(&mut self) {
        self.closed.store(true, atomic::Ordering::Release);
    }
}

unsafe impl<T> Send for BoundedReceiver<T> {}
unsafe impl<T> Sync for BoundedReceiver<T> {}

/// Creates a new Bounded-Queue with the given Capacity and returns the
/// corresponding Handles ([`BoundedReceiver`], [`BoundedSender`])
pub fn queue<T>(capacity: usize) -> (BoundedReceiver<T>, BoundedSender<T>) {
    // Create the underlying Buffer of Nodes and fill it up with empty Nodes
    // as the initial Configuration
    let mut raw_buffer = Vec::with_capacity(capacity);
    for _ in 0..capacity {
        raw_buffer.push(Node::new());
    }

    let closed = Arc::new(atomic::AtomicBool::new(false));
    let buffer = Arc::new(raw_buffer);

    (
        BoundedReceiver {
            closed: closed.clone(),
            buffer: buffer.clone(),
            tail: 0,
        },
        BoundedSender {
            closed,
            buffer,
            head: 0,
        },
    )
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn enqueue_dequeue() {
        let (mut rx, mut tx) = queue(10);

        assert_eq!(Ok(()), tx.try_enqueue(13));
        assert_eq!(Ok(13), rx.try_dequeue());
    }
    #[test]
    fn enqueue_will_block() {
        let (rx, mut tx) = queue(1);

        assert_eq!(Ok(()), tx.try_enqueue(13));
        assert_eq!(Err((14, EnqueueError::WouldBlock)), tx.try_enqueue(14));

        drop(rx);
    }
    #[test]
    fn dequeue_will_block() {
        let (mut rx, tx) = queue::<usize>(1);

        assert_eq!(Err(DequeueError::WouldBlock), rx.try_dequeue());

        drop(tx);
    }

    #[test]
    fn enqueue_dequeue_full_buffer() {
        let (mut rx, mut tx) = queue(3);

        for i in 0..4 {
            assert_eq!(Ok(()), tx.try_enqueue(i));
            assert_eq!(Ok(i), rx.try_dequeue());
        }
    }

    #[test]
    fn enqueue_is_closed() {
        let (rx, mut tx) = queue(3);

        drop(rx);
        assert_eq!(Err((13, EnqueueError::Closed)), tx.try_enqueue(13));
    }
    #[test]
    fn dequeue_is_closed() {
        let (mut rx, tx) = queue::<usize>(3);

        drop(tx);
        assert_eq!(Err(DequeueError::Closed), rx.try_dequeue());
    }
    #[test]
    fn enqueue_dequeue_is_closed() {
        let (mut rx, mut tx) = queue::<usize>(3);

        tx.try_enqueue(13).unwrap();
        drop(tx);

        assert_eq!(Ok(13), rx.try_dequeue());
        assert_eq!(Err(DequeueError::Closed), rx.try_dequeue());
    }

    #[test]
    fn blocking_enqueue_closed() {
        let (rx, mut tx) = queue::<usize>(3);
        drop(rx);

        assert_eq!(Err((13, EnqueueError::Closed)), tx.enqueue(13));
    }
    #[test]
    fn blocking_dequeue_closed() {
        let (mut rx, tx) = queue::<usize>(3);
        drop(tx);

        assert_eq!(None, rx.dequeue());
    }

    #[test]
    fn is_empty() {
        let (mut rx, mut tx) = queue::<usize>(3);

        assert_eq!(true, rx.is_empty());

        tx.try_enqueue(13).unwrap();
        assert_eq!(false, rx.is_empty());

        rx.try_dequeue().unwrap();
        assert_eq!(true, rx.is_empty());
    }

    #[test]
    fn is_full() {
        let (mut rx, mut tx) = queue::<usize>(1);

        assert_eq!(false, tx.is_full());

        tx.try_enqueue(13).unwrap();
        assert_eq!(true, tx.is_full());

        rx.try_dequeue().unwrap();
        assert_eq!(false, tx.is_full());
    }
}