takoude 0.2.0

Lock-free SPSC ring buffer with per-slot sequence flags, tuned for low latency.
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
//! A lock-free single-producer single-consumer (SPSC) ring buffer tuned for
//! single-message latency.
//!
//! # Design
//!
//! A slot-sequence queue in the FastForward/Vyukov family, with the
//! consumer's slot write-back replaced by lazy flow control.
//!
//! - **No shared head/tail atomics on the hot path.** Producer and consumer
//!   each keep a plain (non-atomic, register-resident) `u64` position
//!   counter. Publication happens through a per-slot flag in the same cache
//!   line as the payload, so a handoff costs one cache-line transfer.
//!   Cached-index Lamport queues need two serialized transfers per message,
//!   one for the index and one for the data.
//! - **The consumer never writes the slot line.** A pop publishes the
//!   consumer's position to a cache-padded `progress` counter on a
//!   consumer-owned line; the producer spends a cached credit
//!   (`limit = progress + N`) and reloads it at most once per lap. The hot
//!   slot line moves in one direction only, producer to consumer.
//! - **Publication protocol.** For position `p` and slot `p & (N - 1)`, the
//!   producer stores `seq = p + 1` after the payload; the consumer reads
//!   when `seq == p + 1`. Flags from earlier laps differ by a multiple of
//!   `N`, and only equality is used, so `u64` wraparound is safe while `N`
//!   is a power of two (enforced at compile time).
//! - **Memory ordering.** `Acquire` on availability checks, `Release` on
//!   publication; both compile to plain `mov` on x86-64. Position arithmetic
//!   is local and non-atomic.
//!
//! # Latency
//!
//! A cross-core handoff is bounded by the coherency protocol moving one
//! modified line between cores, roughly 20 to 100 ns one-way on current x86.
//! Sub-nanosecond figures describe the amortized cost of an inlined push and
//! pop pair on one core, not a cross-core message. This design minimizes
//! what software controls: one line transfer per handoff, no shared-index
//! traffic on top.
//!
//! # Example
//!
//! ```
//! let (mut tx, mut rx) = takoude::spsc_channel::<u64, 8>();
//! std::thread::spawn(move || {
//!     for i in 0..100 {
//!         tx.push(i);
//!     }
//! });
//! for i in 0..100 {
//!     assert_eq!(rx.pop(), i);
//! }
//! ```
//!
//! # Cargo features
//!
//! All features are off by default; the default paths are safe and portable.
//!
//! - `packed`: single-atomic publication for `AtomicValue` payloads up to
//!   4 bytes. Fully sound and portable. Use `spsc_packed_channel` to get
//!   endpoints whose encoding cannot be mixed with scalar operations.
//! - `readahead`: the consumer prefetches the slot 32 positions ahead on
//!   each pop. Raises sustained throughput on hosts with slow cross-core
//!   interconnects; benchmark on the target before enabling.

#![deny(missing_docs)]
#![cfg_attr(docsrs, feature(doc_cfg))]

mod ring;
mod slot;

use core::sync::atomic::Ordering::{Acquire, Release};
use ring::Ring;
use std::sync::Arc;

/// Prefetches a slot line for reading (consumer-side pipelining).
#[inline(always)]
fn prefetcht0_line<T>(_slot: &slot::Slot<T>) {
    #[cfg(all(feature = "readahead", target_arch = "x86_64"))]
    unsafe {
        core::arch::asm!(
            "prefetcht0 [{p}]",
            p = in(reg) core::ptr::from_ref(_slot),
            options(nostack, preserves_flags, readonly),
        );
    }
}

/// Error returned by [`Producer::try_push`] when the ring is full; gives the
/// value back to the caller.
#[derive(Debug, PartialEq, Eq)]
pub struct Full<T>(pub T);

impl<T> Full<T> {
    /// Returns the value that could not be pushed.
    pub fn into_inner(self) -> T {
        self.0
    }
}

impl<T> core::fmt::Display for Full<T> {
    fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
        f.write_str("ring buffer is full")
    }
}

impl<T: core::fmt::Debug> std::error::Error for Full<T> {}

/// A value whose initialized representation can be transported through an
/// integer atomic without losing validity or pointer provenance.
///
/// This is required by the `packed` endpoints because they copy the complete
/// object representation into an integer. Primitive values and arrays of
/// `AtomicValue`s are supported out of the box.
///
/// # Safety
///
/// Every byte in every valid value must be initialized (no padding), the
/// representation must not carry pointer provenance, and copying the bytes
/// out and back unchanged must produce the original valid value.
#[cfg(feature = "packed")]
#[cfg_attr(docsrs, doc(cfg(feature = "packed")))]
pub unsafe trait AtomicValue: Copy {}

#[cfg(feature = "packed")]
macro_rules! impl_atomic_value {
    ($($ty:ty),+ $(,)?) => {
        $(
            // SAFETY: these primitive types have no padding or provenance.
            unsafe impl AtomicValue for $ty {}
        )+
    };
}

#[cfg(feature = "packed")]
impl_atomic_value!(
    (),
    bool,
    char,
    u8,
    i8,
    u16,
    i16,
    u32,
    i32,
    u64,
    i64,
    usize,
    isize,
    f32,
    f64
);

// SAFETY: arrays contain their elements contiguously without inter-element
// padding; the element contract therefore extends to the complete array.
#[cfg(feature = "packed")]
unsafe impl<T: AtomicValue, const M: usize> AtomicValue for [T; M] {}

/// Creates a bounded SPSC channel with `N` slots (`N` must be a power of two
/// and at least 2; enforced at compile time).
///
/// All `N` slots are usable. The two handles may be sent to different
/// threads; neither is cloneable.
pub fn spsc_channel<T: Send, const N: usize>() -> (Producer<T, N>, Consumer<T, N>) {
    spsc_channel_at(0)
}

/// Creates a bounded SPSC channel using the `packed` single-atomic encoding.
///
/// `T` must implement [`AtomicValue`] and fit in 4 bytes. The dedicated
/// endpoint types prevent packed and scalar operations from being mixed.
#[cfg(feature = "packed")]
#[cfg_attr(docsrs, doc(cfg(feature = "packed")))]
pub fn spsc_packed_channel<T: Send + AtomicValue, const N: usize>()
-> (PackedProducer<T, N>, PackedConsumer<T, N>) {
    spsc_packed_channel_at(0)
}

/// Like [`spsc_packed_channel`], but starts at a multiple-of-capacity base
/// for wraparound testing.
#[cfg(feature = "packed")]
#[doc(hidden)]
pub fn spsc_packed_channel_at<T: Send + AtomicValue, const N: usize>(
    base: u64,
) -> (PackedProducer<T, N>, PackedConsumer<T, N>) {
    const {
        assert!(
            size_of::<T>() <= 4,
            "packed channel requires size_of::<T>() <= 4"
        );
        assert!(
            align_of::<T>() <= 4,
            "packed channel requires align_of::<T>() <= 4"
        );
        assert!((N as u128) < 1 << 32, "packed channel requires N < 2^32");
    }
    let (producer, consumer) = spsc_channel_at(base);
    (
        PackedProducer { inner: producer },
        PackedConsumer { inner: consumer },
    )
}

/// Like [`spsc_channel`], but starts both position counters at `base`
/// (must be a multiple of `N`). Exists so tests can start just below
/// `u64::MAX` and prove wraparound safety; not useful otherwise.
#[doc(hidden)]
pub fn spsc_channel_at<T: Send, const N: usize>(base: u64) -> (Producer<T, N>, Consumer<T, N>) {
    assert!(
        base & (N as u64 - 1) == 0,
        "base must be a multiple of the capacity"
    );
    let ring = Arc::new(Ring::<T, N>::new(base));
    let slots = ring.slots_ptr();
    (
        Producer {
            slots,
            ring: Arc::clone(&ring),
            pos: base,
            limit: base.wrapping_add(N as u64),
        },
        Consumer {
            slots,
            ring,
            pos: base,
        },
    )
}

/// Sending half of the channel. `!Clone`; owned by exactly one thread at a
/// time (methods take `&mut self`).
pub struct Producer<T: Send, const N: usize> {
    /// Cached base of the slot array (owned by `ring`): saves a
    /// serially-dependent load of the `Box` data pointer on every push.
    slots: core::ptr::NonNull<slot::Slot<T>>,
    /// Local position counter; never shared.
    pos: u64,
    /// Cached credit: positions below this are known free. Refreshed from
    /// the consumer's progress counter only when exhausted (once per lap).
    limit: u64,
    /// Keeps the ring (and thus `slots`) alive; also hosts the progress
    /// counter, whose address needs no extra load (fixed offset from the
    /// Arc pointer held right here).
    ring: Arc<Ring<T, N>>,
}

// NonNull suppresses the auto-impls the Arc field would otherwise provide.
// The pointer targets the Arc-owned ring, and all mutation is either via
// atomics or guarded by the sequence protocol.
unsafe impl<T: Send, const N: usize> Send for Producer<T, N> {}
unsafe impl<T: Send, const N: usize> Sync for Producer<T, N> {}

impl<T: Send, const N: usize> Producer<T, N> {
    #[inline(always)]
    fn slot(&self, pos: u64) -> &slot::Slot<T> {
        unsafe { &*self.slots.as_ptr().add((pos & Ring::<T, N>::MASK) as usize) }
    }
}

impl<T: Send, const N: usize> Drop for Producer<T, N> {
    fn drop(&mut self) {
        // Publish the final produced position so the ring can drop exactly
        // the unconsumed values in `[progress, producer_final)`.
        self.ring.producer_final.0.store(self.pos, Release);
    }
}

/// Receiving half of the channel.
pub struct Consumer<T: Send, const N: usize> {
    /// Cached base of the slot array (owned by `ring`); see [`Producer`].
    slots: core::ptr::NonNull<slot::Slot<T>>,
    pos: u64,
    ring: Arc<Ring<T, N>>,
}

/// Sending half of a channel created by [`spsc_packed_channel`].
#[cfg(feature = "packed")]
#[cfg_attr(docsrs, doc(cfg(feature = "packed")))]
pub struct PackedProducer<T: Send + AtomicValue, const N: usize> {
    inner: Producer<T, N>,
}

/// Receiving half of a channel created by [`spsc_packed_channel`].
#[cfg(feature = "packed")]
#[cfg_attr(docsrs, doc(cfg(feature = "packed")))]
pub struct PackedConsumer<T: Send + AtomicValue, const N: usize> {
    inner: Consumer<T, N>,
}

unsafe impl<T: Send, const N: usize> Send for Consumer<T, N> {}
unsafe impl<T: Send, const N: usize> Sync for Consumer<T, N> {}

impl<T: Send, const N: usize> Consumer<T, N> {
    #[inline(always)]
    fn slot(&self, pos: u64) -> &slot::Slot<T> {
        unsafe { &*self.slots.as_ptr().add((pos & Ring::<T, N>::MASK) as usize) }
    }
}

impl<T: Send, const N: usize> Producer<T, N> {
    /// Refreshes the cached credit from the consumer's progress counter.
    /// Returns `true` if position `self.pos` is free to write.
    #[cold]
    fn refresh_limit(&mut self) -> bool {
        self.limit = self.ring.progress.0.load(Acquire).wrapping_add(N as u64);
        self.pos != self.limit
    }

    /// Attempts to push without blocking. Returns the value back inside
    /// [`Full`] if the ring is full.
    #[inline(always)]
    pub fn try_push(&mut self, value: T) -> Result<(), Full<T>> {
        if self.pos == self.limit && !self.refresh_limit() {
            return Err(Full(value));
        }
        let slot = self.slot(self.pos);
        unsafe { slot.payload.get().cast::<T>().write(value) };
        slot.seq.store(self.pos.wrapping_add(1), Release);
        self.pos = self.pos.wrapping_add(1);
        Ok(())
    }

    /// Zero-copy variant of [`try_push`](Self::try_push): the value is
    /// constructed by `f` directly inside the slot (after inlining there is
    /// no intermediate copy). Returns `f` back if the ring is full.
    ///
    /// If `f` panics, nothing is published and the channel stays consistent.
    #[inline(always)]
    pub fn try_push_with<F: FnOnce() -> T>(&mut self, f: F) -> Result<(), F> {
        if self.pos == self.limit && !self.refresh_limit() {
            return Err(f);
        }
        let slot = self.slot(self.pos);
        unsafe { slot.payload.get().cast::<T>().write(f()) };
        slot.seq.store(self.pos.wrapping_add(1), Release);
        self.pos = self.pos.wrapping_add(1);
        Ok(())
    }

    /// Pushes, spinning with [`core::hint::spin_loop`] while the ring is
    /// full. Spins forever if the consumer is gone and the ring stays full,
    /// so prefer [`try_push`](Self::try_push) when that can happen.
    #[inline]
    pub fn push(&mut self, value: T) {
        while self.pos == self.limit && !self.refresh_limit() {
            core::hint::spin_loop();
        }
        let slot = self.slot(self.pos);
        unsafe { slot.payload.get().cast::<T>().write(value) };
        slot.seq.store(self.pos.wrapping_add(1), Release);
        self.pos = self.pos.wrapping_add(1);
    }

    /// Returns `true` if a `try_push` right now would fail.
    #[inline]
    pub fn is_full(&self) -> bool {
        self.pos == self.limit
            && self.pos == self.ring.progress.0.load(Acquire).wrapping_add(N as u64)
    }

    /// Total slot count (all `N` slots are usable).
    #[inline]
    pub const fn capacity(&self) -> usize {
        N
    }

    /// Returns `true` if the [`Consumer`] has been dropped.
    #[inline]
    pub fn is_disconnected(&self) -> bool {
        Arc::strong_count(&self.ring) == 1
    }
}

/// Packed single-atomic publication (feature `packed`).
///
/// For [`AtomicValue`] payloads up to 4 bytes, the flag and the payload bits
/// both fit in the slot's existing `AtomicU64`: the low 32 bits carry the
/// truncated flag `(pos + 1) as u32`, the high 32 bits carry the payload.
/// Push and pop are then a single ordinary atomic store or load, giving the
/// closed-publication-window benefit of a single store while staying inside
/// the Rust memory model: portable, Miri-checkable, no asm.
///
/// The 32-bit flag stays unambiguous for the same reason the 64-bit one
/// does: the producer never runs more than `N` ahead of the consumer, so the
/// only flag values observable at position `p` are from `p` itself or
/// `p - N`, which differ mod 2^32 for power-of-two `N`.
///
/// The packed encoding stores payload bits where the scalar path stores
/// flag bits. [`spsc_packed_channel`] exposes distinct endpoint types
/// so the encodings cannot be mixed accidentally. The slot's `payload` field
/// is unused here.
#[cfg(feature = "packed")]
impl<T: Send + AtomicValue, const N: usize> Producer<T, N> {
    #[inline(always)]
    fn try_push_packed(&mut self, value: T) -> Result<(), Full<T>> {
        const {
            assert!(
                size_of::<T>() <= 4,
                "packed push requires size_of::<T>() <= 4"
            );
            assert!(
                align_of::<T>() <= 4,
                "packed push requires align_of::<T>() <= 4"
            );
            // The truncated u32 flag disambiguates positions p and p - N
            // only while N does not wrap the 32-bit space.
            assert!((N as u128) < 1 << 32, "packed paths require N < 2^32");
        }
        if self.pos == self.limit && !self.refresh_limit() {
            return Err(Full(value));
        }
        let slot = self.slot(self.pos);
        let mut bits = 0u32;
        // Not transmute: T may be smaller than 4 bytes.
        unsafe {
            core::ptr::copy_nonoverlapping(
                (&raw const value).cast::<u8>(),
                (&raw mut bits).cast::<u8>(),
                size_of::<T>(),
            )
        };
        let flag = self.pos.wrapping_add(1) as u32 as u64;
        slot.seq.store((bits as u64) << 32 | flag, Release);
        self.pos = self.pos.wrapping_add(1);
        Ok(())
    }

    #[inline]
    fn push_packed(&mut self, value: T) {
        while self.pos == self.limit && !self.refresh_limit() {
            core::hint::spin_loop();
        }
        // The spin loop exits only when try_push_packed's full check passes.
        let res = self.try_push_packed(value);
        debug_assert!(res.is_ok(), "push_packed: slot vanished after spin");
        let _ = res;
    }
}

#[cfg(feature = "packed")]
impl<T: Send + AtomicValue, const N: usize> Consumer<T, N> {
    #[inline(always)]
    fn try_pop_packed(&mut self) -> Option<T> {
        const {
            assert!(
                size_of::<T>() <= 4,
                "packed pop requires size_of::<T>() <= 4"
            );
            assert!(
                align_of::<T>() <= 4,
                "packed pop requires align_of::<T>() <= 4"
            );
            assert!((N as u128) < 1 << 32, "packed paths require N < 2^32");
        }
        let slot = self.slot(self.pos);
        let word = slot.seq.load(Acquire);
        if word as u32 != self.pos.wrapping_add(1) as u32 {
            return None;
        }
        let bits = (word >> 32) as u32;
        let value = unsafe { core::ptr::read((&raw const bits).cast::<T>()) };
        self.pos = self.pos.wrapping_add(1);
        self.ring.progress.0.store(self.pos, Release);
        Some(value)
    }

    #[inline]
    fn pop_packed(&mut self) -> T {
        loop {
            if let Some(value) = self.try_pop_packed() {
                return value;
            }
            core::hint::spin_loop();
        }
    }
}

#[cfg(feature = "packed")]
#[cfg_attr(docsrs, doc(cfg(feature = "packed")))]
impl<T: Send + AtomicValue, const N: usize> PackedProducer<T, N> {
    /// Attempts to push without blocking. Returns the value inside [`Full`]
    /// if the ring is full.
    #[inline(always)]
    pub fn try_push(&mut self, value: T) -> Result<(), Full<T>> {
        self.inner.try_push_packed(value)
    }

    /// Pushes, spinning with [`core::hint::spin_loop`] while the ring is
    /// full. Spins forever if the consumer is gone and the ring stays full,
    /// so prefer [`try_push`](Self::try_push) when that can happen.
    #[inline]
    pub fn push(&mut self, value: T) {
        self.inner.push_packed(value);
    }

    /// Returns `true` if a [`try_push`](Self::try_push) right now would fail.
    #[inline]
    pub fn is_full(&self) -> bool {
        self.inner.is_full()
    }

    /// Total slot count (all `N` slots are usable).
    #[inline]
    pub const fn capacity(&self) -> usize {
        N
    }

    /// Returns `true` if the [`PackedConsumer`] has been dropped.
    #[inline]
    pub fn is_disconnected(&self) -> bool {
        self.inner.is_disconnected()
    }
}

#[cfg(feature = "packed")]
#[cfg_attr(docsrs, doc(cfg(feature = "packed")))]
impl<T: Send + AtomicValue, const N: usize> PackedConsumer<T, N> {
    /// Attempts to pop without blocking; `None` if the ring is empty.
    #[inline(always)]
    pub fn try_pop(&mut self) -> Option<T> {
        self.inner.try_pop_packed()
    }

    /// Pops, spinning with [`core::hint::spin_loop`] while the ring is empty.
    /// Spins forever if the producer is gone and the ring stays empty, so
    /// prefer [`try_pop`](Self::try_pop) when that can happen.
    #[inline]
    pub fn pop(&mut self) -> T {
        self.inner.pop_packed()
    }

    /// Returns `true` if a [`try_pop`](Self::try_pop) right now would return
    /// `None`.
    #[inline]
    pub fn is_empty(&self) -> bool {
        let slot = self.inner.slot(self.inner.pos);
        slot.seq.load(Acquire) as u32 != self.inner.pos.wrapping_add(1) as u32
    }

    /// Total slot count (all `N` slots are usable).
    #[inline]
    pub const fn capacity(&self) -> usize {
        N
    }

    /// Returns `true` if the [`PackedProducer`] has been dropped. The ring
    /// may still contain undelivered items.
    #[inline]
    pub fn is_disconnected(&self) -> bool {
        self.inner.is_disconnected()
    }
}

impl<T: Send, const N: usize> Consumer<T, N> {
    /// Attempts to pop without blocking; `None` if the ring is empty.
    #[inline(always)]
    pub fn try_pop(&mut self) -> Option<T> {
        let slot = self.slot(self.pos);
        if slot.seq.load(Acquire) != self.pos.wrapping_add(1) {
            return None;
        }
        let value = unsafe { slot.payload.get().cast::<T>().read() };
        self.pos = self.pos.wrapping_add(1);
        // Publish progress on a consumer-owned padded line; the slot line
        // itself is never written by the consumer.
        self.ring.progress.0.store(self.pos, Release);
        prefetcht0_line(self.slot(self.pos.wrapping_add(31)));
        Some(value)
    }

    /// Zero-copy variant of [`try_pop`](Self::try_pop): `f` observes the
    /// payload in place (no move out of the slot), the payload is then
    /// dropped in place and the slot released.
    ///
    /// If `f` panics, the element is left in the queue untouched (aside from
    /// any mutation `f` already performed) and the channel stays consistent.
    #[inline(always)]
    pub fn try_pop_with<R, F: FnOnce(&mut T) -> R>(&mut self, f: F) -> Option<R> {
        let slot = self.slot(self.pos);
        if slot.seq.load(Acquire) != self.pos.wrapping_add(1) {
            return None;
        }
        let payload = unsafe { &mut *slot.payload.get().cast::<T>() };
        // If `f` panics the element stays in the queue (nothing below runs).
        let result = f(payload);

        // Commit point: once drop_in_place is entered the value must count
        // as consumed even if T's Drop unwinds, or the slot would still
        // read as full and the value would be dropped a second time (by the
        // next pop or by Ring::drop). The guard advances pos and publishes
        // progress on both the normal and the unwinding path.
        struct Commit<'a> {
            pos: &'a mut u64,
            progress: &'a core::sync::atomic::AtomicU64,
        }
        impl Drop for Commit<'_> {
            #[inline]
            fn drop(&mut self) {
                *self.pos = self.pos.wrapping_add(1);
                self.progress.store(*self.pos, Release);
            }
        }
        let commit = Commit {
            pos: &mut self.pos,
            progress: &self.ring.progress.0,
        };
        unsafe { core::ptr::drop_in_place(payload) };
        drop(commit);
        prefetcht0_line(self.slot(self.pos.wrapping_add(31)));
        Some(result)
    }

    /// Pops, spinning with [`core::hint::spin_loop`] while the ring is
    /// empty. Spins forever if the producer is gone and the ring stays
    /// empty, so prefer [`try_pop`](Self::try_pop) when that can happen.
    #[inline]
    pub fn pop(&mut self) -> T {
        let slot = self.slot(self.pos);
        let full_seq = self.pos.wrapping_add(1);
        while slot.seq.load(Acquire) != full_seq {
            core::hint::spin_loop();
        }
        let value = unsafe { slot.payload.get().cast::<T>().read() };
        self.pos = self.pos.wrapping_add(1);
        self.ring.progress.0.store(self.pos, Release);
        prefetcht0_line(self.slot(self.pos.wrapping_add(31)));
        value
    }

    /// Returns `true` if a `try_pop` right now would return `None`.
    #[inline]
    pub fn is_empty(&self) -> bool {
        self.slot(self.pos).seq.load(Acquire) != self.pos.wrapping_add(1)
    }

    /// Total slot count (all `N` slots are usable).
    #[inline]
    pub const fn capacity(&self) -> usize {
        N
    }

    /// Returns `true` if the [`Producer`] has been dropped. The ring may
    /// still contain undelivered items.
    #[inline]
    pub fn is_disconnected(&self) -> bool {
        Arc::strong_count(&self.ring) == 1
    }
}