stackler 0.1.0

A lightweight, always-on stack telemetry library that requires no instrumentation.
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
//! The process-wide registry of tracked thread stacks.
//!
//! Every tracked thread owns one cache-aligned slot, and every measurement a
//! thread makes lands in its own slot — counters and depth histogram alike. No
//! counter is shared, so no observation ever bounces a cache line between cores;
//! measured on sixteen threads sampling in lockstep, keeping the histogram shared
//! instead cost all of the scaling. The aggregates a dashboard wants are computed
//! when a snapshot is taken, by walking the slots — cheap, because only occupied
//! ones are read.
//!
//! A slot is claimed the first time a thread is observed and released when that
//! thread exits, at which point its totals are folded into a set of process-wide
//! counters so that dead threads keep contributing to the lifetime picture.

use core::{cell::OnceCell, cmp, sync::atomic::Ordering};

#[cfg(feature = "watermark")]
use portable_atomic::AtomicBool;
use portable_atomic::{AtomicU8, AtomicU64, AtomicUsize};

#[cfg(feature = "watermark")]
use crate::{Paint, Peak};
use crate::{Stack, config, platform};

/// The number of threads that can be tracked at the same time.
///
/// Threads beyond this many are simply not tracked; [`crate::Stats::untracked`]
/// counts them so that the omission is visible rather than silent.
pub const MAX_THREADS: usize = 256;

/// The slot is unused.
const EMPTY: u8 = 0;
/// The slot has been reserved, but its contents are not yet valid.
const CLAIMING: u8 = 1;
/// The slot is owned by a live thread and its contents are valid.
const LIVE: u8 = 2;
/// Another thread is reading the owner's painted stack region right now.
const SCANNING: u8 = 3;
/// The owner is exiting; the slot is about to become [`EMPTY`].
const DYING: u8 = 4;

#[cfg(feature = "histograms")]
#[repr(align(64))]
struct CacheAligned<T>(T);

/// Whether the rest of a slot may be read, and whether it still describes the same
/// thread as it did a moment ago.
///
/// These two are written by threads *other* than the slot's owner — a sampler
/// takes the state to scan, a snapshot checks the generation — so they are given a
/// cache line to themselves. Being 64-byte aligned makes the type 64 bytes wide,
/// which means no other field can share a line with it, whatever order the compiler
/// chooses to lay `Slot` out in. Without that, a background sampler would bounce
/// the very counters it exists to observe without disturbing.
#[repr(align(64))]
struct Control {
    state: AtomicU8,
    generation: AtomicU64,
}

impl Control {
    const fn new() -> Self {
        Self {
            state: AtomicU8::new(EMPTY),
            generation: AtomicU64::new(0),
        }
    }
}

// The point of `Control` is that it fills a cache line by itself; if that ever
// stops being true, the isolation it exists for is gone.
const _: () = assert!(size_of::<Control>() == 64);

/// One tracked thread.
///
/// Only the owning thread writes the descriptive fields, and only between
/// [`CLAIMING`] and the release store that publishes [`LIVE`], so readers that
/// acquire the state see a consistent stack description.
///
/// A slot outlives the thread that occupies it, so a reader also has to know
/// whether it was recycled underneath the read: the generation changes — with
/// release ordering, before anything else does — every time the slot stops
/// describing the thread it used to, and [`Slot::snapshot`] rejects any view of
/// the slot that does not begin and end in the same generation.
#[repr(align(64))]
struct Slot {
    control: Control,
    id: AtomicU64,
    base: AtomicUsize,
    limit: AtomicUsize,
    #[cfg(feature = "watermark")]
    paint_lo: AtomicUsize,
    #[cfg(feature = "watermark")]
    paint_hi: AtomicUsize,
    #[cfg(feature = "watermark")]
    saturated: AtomicBool,
    depth_last: AtomicUsize,
    peak: AtomicUsize,
    sample_count: AtomicU64,
    sample_sum: AtomicU64,
    near_limit_count: AtomicU64,
    guard_trip_count: AtomicU64,
    /// The thread's own depth distribution. Per-slot rather than shared: threads
    /// running the same code sample the same depths, and a shared bucket array
    /// would put every one of those observations on one cache line. Only the
    /// buckets a thread actually uses are ever touched, so the cost in resident
    /// memory is a page or two per active thread rather than the size of the array.
    #[cfg(feature = "histograms")]
    buckets: [AtomicU64; 64],
}

impl Slot {
    const fn new() -> Self {
        Self {
            control: Control::new(),
            id: AtomicU64::new(0),
            base: AtomicUsize::new(0),
            limit: AtomicUsize::new(0),
            #[cfg(feature = "watermark")]
            paint_lo: AtomicUsize::new(0),
            #[cfg(feature = "watermark")]
            paint_hi: AtomicUsize::new(0),
            #[cfg(feature = "watermark")]
            saturated: AtomicBool::new(false),
            depth_last: AtomicUsize::new(0),
            peak: AtomicUsize::new(0),
            sample_count: AtomicU64::new(0),
            sample_sum: AtomicU64::new(0),
            near_limit_count: AtomicU64::new(0),
            guard_trip_count: AtomicU64::new(0),
            #[cfg(feature = "histograms")]
            buckets: [const { AtomicU64::new(0) }; 64],
        }
    }

    /// Returns the stack this slot describes.
    fn stack(&self) -> Option<Stack> {
        Stack::from_bounds(
            self.limit.load(Ordering::Relaxed),
            self.base.load(Ordering::Relaxed),
        )
    }

    /// Returns the region the owner painted, if any.
    #[cfg(feature = "watermark")]
    fn paint(&self) -> Option<Paint> {
        Paint::from_bounds(
            self.paint_lo.load(Ordering::Relaxed),
            self.paint_hi.load(Ordering::Relaxed),
        )
    }

    /// Records a depth observation. Every write here goes to this thread's own
    /// cache lines, which is what keeps concurrent sampling from serializing.
    #[inline]
    fn record(&self, depth: usize) {
        // The peak goes first: a reader that sees this observation's depth has
        // then already had the chance to see the peak it produced.
        self.peak.fetch_max(depth, Ordering::Relaxed);
        self.depth_last.store(depth, Ordering::Relaxed);
        self.sample_count.fetch_add(1, Ordering::Relaxed);
        self.sample_sum.fetch_add(depth as u64, Ordering::Relaxed);

        #[cfg(feature = "histograms")]
        if depth > 0 {
            self.buckets[crate::histogram::bucket_of(depth)].fetch_add(1, Ordering::Relaxed);
        }
    }

    /// Observes the depth a stack pointer corresponds to, returning `None` if it
    /// does not belong to this stack.
    #[inline]
    fn observe(&self, sp: usize) -> Option<usize> {
        let base = self.base.load(Ordering::Relaxed);
        let limit = self.limit.load(Ordering::Relaxed);
        if sp >= base || sp < limit {
            return None;
        }

        let depth = base - sp;
        self.record(depth);
        if sp - limit < config::reserve() {
            self.near_limit_count.fetch_add(1, Ordering::Relaxed);
        }

        Some(depth)
    }

    /// Returns a consistent view of the slot, or `None` if it is not occupied by
    /// a live thread or stopped describing the same one while being read.
    ///
    /// The fields are read with relaxed loads, so this is the only thing standing
    /// between a slot that changes hands mid-read and a snapshot that mixes two
    /// threads' stacks into one nonsensical pair of bounds. The generation is
    /// bumped with release ordering before any teardown or re-claim, and read
    /// here on both sides of the loads, with an acquire fence between them so
    /// that they cannot be reordered past the second read.
    fn snapshot(&self, idx: usize) -> Option<crate::ThreadStats> {
        let generation = self.control.generation.load(Ordering::Acquire);
        if !self.is_occupied() {
            return None;
        }

        let stack = self.stack()?;
        let depth_last = self.depth_last.load(Ordering::Relaxed);
        let thread = crate::ThreadStats {
            slot: idx,
            id: self.id.load(Ordering::Relaxed),
            base: stack.base(),
            limit: stack.limit(),
            size: stack.size(),
            depth_last,
            // Ordering the writes is not enough to make two relaxed loads agree,
            // and a peak shallower than a depth actually observed would be a
            // nonsense to report. The deepest of the two is what is known.
            peak: cmp::max(self.peak.load(Ordering::Relaxed), depth_last),
            #[cfg(feature = "watermark")]
            saturated: self.saturated.load(Ordering::Relaxed),
            sample_count: self.sample_count.load(Ordering::Relaxed),
            sample_sum: self.sample_sum.load(Ordering::Relaxed),
            near_limit_count: self.near_limit_count.load(Ordering::Relaxed),
            guard_trip_count: self.guard_trip_count.load(Ordering::Relaxed),
        };

        // Keeps the loads above from being reordered past the check below.
        core::sync::atomic::fence(Ordering::Acquire);

        if self.control.generation.load(Ordering::Relaxed) != generation || !self.is_occupied() {
            return None;
        }

        Some(thread)
    }

    /// Returns `true` if a live thread owns this slot. A slot being scanned still
    /// counts: its owner cannot exit while the scan is in progress.
    fn is_occupied(&self) -> bool {
        matches!(self.control.state.load(Ordering::Acquire), LIVE | SCANNING)
    }

    /// Zeroes everything, so that the next thread to claim the slot starts from
    /// a blank state.
    fn clear(&self) {
        self.id.store(0, Ordering::Relaxed);
        self.base.store(0, Ordering::Relaxed);
        self.limit.store(0, Ordering::Relaxed);
        #[cfg(feature = "watermark")]
        {
            self.paint_lo.store(0, Ordering::Relaxed);
            self.paint_hi.store(0, Ordering::Relaxed);
            self.saturated.store(false, Ordering::Relaxed);
        }
        self.depth_last.store(0, Ordering::Relaxed);
        self.peak.store(0, Ordering::Relaxed);
        self.sample_count.store(0, Ordering::Relaxed);
        self.sample_sum.store(0, Ordering::Relaxed);
        self.near_limit_count.store(0, Ordering::Relaxed);
        self.guard_trip_count.store(0, Ordering::Relaxed);
        #[cfg(feature = "histograms")]
        for bucket in &self.buckets {
            bucket.store(0, Ordering::Relaxed);
        }
    }
}

static SLOTS: [Slot; MAX_THREADS] = [const { Slot::new() }; MAX_THREADS];

/// Lifetime totals of threads that have already exited.
#[repr(align(64))]
struct Retired {
    peak_max: AtomicUsize,
    sample_count: AtomicU64,
    sample_sum: AtomicU64,
    near_limit_count: AtomicU64,
    guard_trip_count: AtomicU64,
    saturated: AtomicU64,
}

static RETIRED: Retired = Retired {
    peak_max: AtomicUsize::new(0),
    sample_count: AtomicU64::new(0),
    sample_sum: AtomicU64::new(0),
    near_limit_count: AtomicU64::new(0),
    guard_trip_count: AtomicU64::new(0),
    saturated: AtomicU64::new(0),
};

static THREADS_SEEN: AtomicU64 = AtomicU64::new(0);
static UNTRACKED: AtomicU64 = AtomicU64::new(0);

/// Where the depth distributions of exited threads are kept. Only written when a
/// thread retires, so it is never on an observation path.
#[cfg(feature = "histograms")]
static RETIRED_BUCKETS: CacheAligned<[AtomicU64; 64]> =
    CacheAligned([const { AtomicU64::new(0) }; 64]);

/// Ownership of a slot, tied to the lifetime of a thread by thread-local
/// storage: dropping it hands the slot back.
struct Handle(usize);

impl Drop for Handle {
    fn drop(&mut self) {
        retire(self.0);
    }
}

std::thread_local! {
    /// The calling thread's slot, resolved once. `None` means the thread cannot
    /// be tracked — its bounds are unknown, or the registry is full — and is
    /// cached just like a successful lookup, so the cost of finding out is paid
    /// once either way.
    static SLOT: OnceCell<Option<Handle>> = const { OnceCell::new() };
}

/// Runs `f` with the calling thread's slot, attaching the thread if this is the
/// first time it is seen.
///
/// Returns `None` if the thread cannot be tracked, or if its thread-local
/// storage is already gone — which happens while a thread is being torn down,
/// and is a reason to report nothing rather than to panic.
fn with_slot<R>(f: impl FnOnce(&Slot) -> R) -> Option<R> {
    SLOT.try_with(|cell| {
        cell.get_or_init(claim_slot)
            .as_ref()
            .map(|handle| f(&SLOTS[handle.0]))
    })
    .ok()
    .flatten()
}

/// Discovers the calling thread's stack, claims a slot for it and, when
/// watermarking is enabled, paints the unused part of the stack.
fn claim_slot() -> Option<Handle> {
    let Some(stack) = Stack::current() else {
        // A thread whose bounds the platform will not report cannot be tracked,
        // and saying so is better than quietly leaving it out.
        UNTRACKED.fetch_add(1, Ordering::Relaxed);
        return None;
    };
    let id = platform::thread_id();

    for (idx, slot) in SLOTS.iter().enumerate() {
        if slot
            .control
            .state
            .compare_exchange(EMPTY, CLAIMING, Ordering::Relaxed, Ordering::Relaxed)
            .is_err()
        {
            continue;
        }

        // Bumped before anything is written, and with release ordering, so that a
        // reader which acquires the new generation also sees the `CLAIMING` that
        // tells it the data is not ready.
        slot.control.generation.fetch_add(1, Ordering::Release);

        slot.id.store(id, Ordering::Relaxed);
        slot.base.store(stack.base(), Ordering::Relaxed);
        slot.limit.store(stack.limit(), Ordering::Relaxed);

        #[cfg(feature = "watermark")]
        if let Some(paint) = paint_current(&stack) {
            slot.paint_lo.store(paint.lo(), Ordering::Relaxed);
            slot.paint_hi.store(paint.hi(), Ordering::Relaxed);
        }

        // Publishes everything written above to anyone who acquires the state.
        slot.control.state.store(LIVE, Ordering::Release);
        THREADS_SEEN.fetch_add(1, Ordering::Relaxed);

        return Some(Handle(idx));
    }

    UNTRACKED.fetch_add(1, Ordering::Relaxed);
    None
}

/// Paints the calling thread's own stack to the configured depth.
#[cfg(feature = "watermark")]
fn paint_current(stack: &Stack) -> Option<Paint> {
    let depth = config::paint_depth();
    if depth == 0 {
        return None;
    }

    // SAFETY: the stack belongs to the calling thread — it was either just
    // discovered by it or read back from its own slot — and no live data exists
    // below the current stack pointer of ordinary Rust code.
    unsafe { stack.paint(depth) }
}

/// Hands a slot back after folding its totals into the lifetime counters.
///
/// Runs from a thread-local destructor, so it must not panic and must not touch
/// any other thread-local.
fn retire(idx: usize) {
    let slot = &SLOTS[idx];

    // Take the slot out of circulation. A cross-thread scan in flight is reading
    // a stack that is about to be unmapped, so wait for it to finish; scans are
    // bounded by the painted size, so the wait is too.
    loop {
        match slot
            .control
            .state
            .compare_exchange(LIVE, DYING, Ordering::AcqRel, Ordering::Acquire)
        {
            Ok(_) => break,
            Err(SCANNING) => std::thread::yield_now(),
            Err(_) => return,
        }
    }

    // From here on the slot's contents are being torn down; the bump tells a
    // reader in the middle of a snapshot to throw its view away.
    slot.control.generation.fetch_add(1, Ordering::Release);

    // A last look at the watermark: this is the only chance to learn how deep
    // the thread got, and the stack is still mapped.
    #[cfg(feature = "watermark")]
    {
        scan(slot);
        if slot.saturated.load(Ordering::Relaxed) {
            RETIRED.saturated.fetch_add(1, Ordering::Relaxed);
        }
    }

    RETIRED
        .peak_max
        .fetch_max(slot.peak.load(Ordering::Relaxed), Ordering::Relaxed);
    RETIRED
        .sample_count
        .fetch_add(slot.sample_count.load(Ordering::Relaxed), Ordering::Relaxed);
    RETIRED
        .sample_sum
        .fetch_add(slot.sample_sum.load(Ordering::Relaxed), Ordering::Relaxed);
    RETIRED.near_limit_count.fetch_add(
        slot.near_limit_count.load(Ordering::Relaxed),
        Ordering::Relaxed,
    );
    RETIRED.guard_trip_count.fetch_add(
        slot.guard_trip_count.load(Ordering::Relaxed),
        Ordering::Relaxed,
    );

    #[cfg(feature = "histograms")]
    for (retired, mine) in RETIRED_BUCKETS.0.iter().zip(&slot.buckets) {
        retired.fetch_add(mine.load(Ordering::Relaxed), Ordering::Relaxed);
    }

    slot.clear();
    slot.control.state.store(EMPTY, Ordering::Release);
}

/// Scans a slot's painted region and folds the result into its peak.
#[cfg(feature = "watermark")]
fn scan(slot: &Slot) -> Option<Peak> {
    let stack = slot.stack()?;
    let paint = slot.paint()?;

    // SAFETY: the paint was produced by this stack's owner, and the owner is
    // either the caller or a thread kept alive by the slot's state: `retire`
    // cannot proceed past `SCANNING`.
    let peak = unsafe { stack.peak(&paint) };

    slot.peak.fetch_max(peak.bytes(), Ordering::Relaxed);
    if peak.is_saturated() {
        slot.saturated.store(true, Ordering::Relaxed);
    }

    Some(peak)
}

/// Attaches the calling thread to the registry, returning `true` if it is
/// tracked from now on.
///
/// Every function that measures the calling thread — [`depth`], [`headroom`],
/// [`has_headroom`], [`sample`], [`current_stack`] — does this implicitly on its
/// first use, so calling it is only worthwhile to control *when* the cost is paid:
/// discovering stack bounds involves the OS, and with the `watermark` feature the
/// call also paints the unused part of the stack.
#[cfg_attr(feature = "watermark", doc = "")]
#[cfg_attr(
    feature = "watermark",
    doc = "[`peak`] and [`repaint`] attach on first use as well."
)]
///
/// The functions that only *report* — the snapshot, the thread listing, the
/// polling and the reset — never attach, which is why a thread that merely scrapes
/// metrics does not end up in the registry itself.
///
/// Attaching twice is harmless — the second call does nothing.
pub fn attach() -> bool {
    with_slot(|_| ()).is_some()
}

/// Returns `true` if the calling thread is already tracked.
///
/// Unlike every other question about the calling thread, asking this one does not
/// attach it.
pub fn is_attached() -> bool {
    SLOT.try_with(|cell| matches!(cell.get(), Some(Some(_))))
        .unwrap_or(false)
}

/// Returns the calling thread's stack, as recorded when it attached.
pub fn current_stack() -> Option<Stack> {
    with_slot(Slot::stack).flatten()
}

/// Returns how many bytes of stack the calling thread is using right now,
/// without recording anything.
pub fn depth() -> Option<usize> {
    let sp = platform::sp();
    with_slot(|slot| slot.stack()?.depth_at(sp)).flatten()
}

/// Returns how many bytes of stack the calling thread has left before it
/// overflows.
pub fn headroom() -> Option<usize> {
    let sp = platform::sp();
    with_slot(|slot| {
        let stack = slot.stack()?;
        if stack.contains(sp) {
            Some(sp - stack.limit())
        } else {
            None
        }
    })
    .flatten()
}

/// Returns `true` if the calling thread can afford to use `bytes` more stack.
///
/// This is the always-on guard for code whose depth depends on its input —
/// recursive descent parsers, tree walks, deserializers — which can trade a
/// segmentation fault for an error return by asking first. Denials are counted
/// in [`crate::Stats::guard_trip_count`], so a load that keeps coming close is
/// visible in telemetry even when nothing goes wrong.
///
/// Threads whose bounds could not be determined are given the benefit of the
/// doubt: the answer is then `true`, and no counter moves.
pub fn has_headroom(bytes: usize) -> bool {
    let sp = platform::sp();
    with_slot(|slot| {
        let limit = slot.limit.load(Ordering::Relaxed);
        if sp < limit || sp - limit >= bytes {
            return true;
        }
        slot.guard_trip_count.fetch_add(1, Ordering::Relaxed);
        false
    })
    .unwrap_or(true)
}

/// Records the calling thread's current depth and returns it.
///
/// This is the cheap, always-available way to feed the aggregates: a handful of
/// relaxed atomic operations, every one of them on the calling thread's own cache
/// lines. Put it wherever a program already has a natural tick — a request
/// handler, a scheduler loop, an event dispatch — and the depth distribution and
/// per-thread peaks follow from it.
pub fn sample() -> Option<usize> {
    let sp = platform::sp();
    with_slot(|slot| slot.observe(sp)).flatten()
}

/// Returns the deepest point the calling thread has reached since it was
/// painted, and folds the reading into its recorded peak.
///
/// Unlike [`sample`], this does not depend on having been called at the right
/// moment: the answer covers the whole interval since the thread attached (or
/// since the last [`repaint`]), including code that never heard of `stackler`.
///
/// # Cost
///
/// One pass over the untouched part of the painted region.
#[cfg(feature = "watermark")]
pub fn peak() -> Option<Peak> {
    with_slot(scan).flatten()
}

/// Folds the current watermark into the recorded peak and paints the stack
/// afresh, starting a new observation window.
///
/// Returns the newly painted region, which is what makes it possible to express
/// a later reading relative to this point — see [`crate::measure_peak`].
#[cfg(feature = "watermark")]
pub fn repaint() -> Option<Paint> {
    with_slot(|slot| {
        // Whatever the old paint knows is lost once it is overwritten, so ask it
        // first.
        scan(slot);

        let stack = slot.stack()?;
        let paint = paint_current(&stack)?;
        slot.paint_lo.store(paint.lo(), Ordering::Relaxed);
        slot.paint_hi.store(paint.hi(), Ordering::Relaxed);

        Some(paint)
    })
    .flatten()
}

/// Folds an externally obtained depth into the calling thread's peak.
#[cfg(feature = "watermark")]
pub(crate) fn record_peak(depth: usize, saturated: bool) {
    with_slot(|slot| {
        slot.peak.fetch_max(depth, Ordering::Relaxed);
        if saturated {
            slot.saturated.store(true, Ordering::Relaxed);
        }
    });
}

/// Re-reads the watermark of every tracked thread, folding what it finds into
/// their recorded peaks.
///
/// This is the reason the crate can describe threads that never call into it
/// themselves: a watermark lives in the stack, so anyone can read it. Call it
/// before taking a snapshot, or let a background
#[cfg_attr(feature = "sampler", doc = "[sampler](crate::sampler)")]
#[cfg_attr(not(feature = "sampler"), doc = "sampler")]
/// call it on a period.
///
/// # Cost
///
/// One pass over the untouched part of every tracked thread's painted region.
#[cfg(feature = "watermark")]
pub fn poll_all() {
    for slot in &SLOTS {
        // Only a live slot may be scanned, and marking it as being scanned keeps
        // its owner from exiting underneath the read.
        if slot
            .control
            .state
            .compare_exchange(LIVE, SCANNING, Ordering::Acquire, Ordering::Relaxed)
            .is_err()
        {
            continue;
        }

        scan(slot);

        slot.control.state.store(LIVE, Ordering::Release);
    }
}

/// Calls `f` with a snapshot of every tracked thread.
///
/// Threads are visited in slot order, which has no relationship to the order in
/// which they were created. The snapshots are taken one thread at a time, so
/// they do not all describe the same instant.
pub fn for_each_thread(mut f: impl FnMut(crate::ThreadStats)) {
    for (idx, slot) in SLOTS.iter().enumerate() {
        if let Some(thread) = slot.snapshot(idx) {
            f(thread);
        }
    }
}

/// Returns a snapshot of every tracked thread.
pub fn threads() -> Vec<crate::ThreadStats> {
    let mut out = Vec::new();
    for_each_thread(|thread| out.push(thread));
    out
}

/// The lifetime totals of threads that have exited.
pub(crate) struct RetiredTotals {
    pub(crate) peak_max: usize,
    pub(crate) sample_count: u64,
    pub(crate) sample_sum: u64,
    pub(crate) near_limit_count: u64,
    pub(crate) guard_trip_count: u64,
    #[cfg(feature = "watermark")]
    pub(crate) saturated: u64,
}

pub(crate) fn retired_totals() -> RetiredTotals {
    RetiredTotals {
        peak_max: RETIRED.peak_max.load(Ordering::Relaxed),
        sample_count: RETIRED.sample_count.load(Ordering::Relaxed),
        sample_sum: RETIRED.sample_sum.load(Ordering::Relaxed),
        near_limit_count: RETIRED.near_limit_count.load(Ordering::Relaxed),
        guard_trip_count: RETIRED.guard_trip_count.load(Ordering::Relaxed),
        #[cfg(feature = "watermark")]
        saturated: RETIRED.saturated.load(Ordering::Relaxed),
    }
}

pub(crate) fn threads_seen() -> u64 {
    THREADS_SEEN.load(Ordering::Relaxed)
}

pub(crate) fn untracked() -> u64 {
    UNTRACKED.load(Ordering::Relaxed)
}

#[cfg(feature = "histograms")]
pub(crate) fn depth_histogram() -> crate::Histogram {
    let mut buckets = [0u64; 64];

    for (bucket, retired) in buckets.iter_mut().zip(&RETIRED_BUCKETS.0) {
        *bucket = retired.load(Ordering::Relaxed);
    }

    for slot in SLOTS.iter().filter(|slot| slot.is_occupied()) {
        for (bucket, mine) in buckets.iter_mut().zip(&slot.buckets) {
            *bucket += mine.load(Ordering::Relaxed);
        }
    }

    crate::Histogram::from_buckets(buckets)
}

/// Clears the counters, and lowers every thread's recorded peak to the last
/// depth observed for it.
///
/// **Concurrency note:** `reset` writes many counters belonging to many threads,
/// and is not synchronized with any of them. While one is in flight:
///
/// - an observation may land on a counter that is about to be cleared, and be
///   lost, or on one just cleared, and survive;
/// - a snapshot may be internally incoherent — most visibly,
///   [`Stats::sample_avg`](crate::Stats::sample_avg) can exceed
///   [`Stats::peak_max`](crate::Stats::peak_max), because the samples and the peak
///   they came from were cleared at different moments.
///
/// A peak may also be re-learned immediately afterwards: only the owner of a stack
/// can repaint it, so the marks left by an earlier deep excursion are still there
/// for the next scan to find.
///
/// For measuring a specific operation, prefer [`crate::measure`], which diffs
/// snapshots instead of touching shared state
#[cfg_attr(
    feature = "watermark",
    doc = ", or [`crate::measure_peak`], which repaints."
)]
#[cfg_attr(not(feature = "watermark"), doc = ".")]
pub fn reset() {
    RETIRED.peak_max.store(0, Ordering::Relaxed);
    RETIRED.sample_count.store(0, Ordering::Relaxed);
    RETIRED.sample_sum.store(0, Ordering::Relaxed);
    RETIRED.near_limit_count.store(0, Ordering::Relaxed);
    RETIRED.guard_trip_count.store(0, Ordering::Relaxed);
    RETIRED.saturated.store(0, Ordering::Relaxed);

    // The threads alive right now are the ones that count as seen from here on.
    // As with the peaks below, a compare-exchange keeps a thread that attaches
    // while this is being computed from being forgotten — which would leave the
    // registry reporting more live threads than it ever saw.
    let observed = THREADS_SEEN.load(Ordering::Relaxed);
    let _ =
        THREADS_SEEN.compare_exchange(observed, count_live(), Ordering::Relaxed, Ordering::Relaxed);

    UNTRACKED.store(0, Ordering::Relaxed);

    #[cfg(feature = "histograms")]
    for bucket in &RETIRED_BUCKETS.0 {
        bucket.store(0, Ordering::Relaxed);
    }

    for slot in &SLOTS {
        if !slot.is_occupied() {
            continue;
        }

        #[cfg(feature = "histograms")]
        for bucket in &slot.buckets {
            bucket.store(0, Ordering::Relaxed);
        }

        slot.sample_count.store(0, Ordering::Relaxed);
        slot.sample_sum.store(0, Ordering::Relaxed);
        slot.near_limit_count.store(0, Ordering::Relaxed);
        slot.guard_trip_count.store(0, Ordering::Relaxed);

        // Lower the peak last, and only if nothing raised it in the meantime: a
        // plain store here would erase a genuine observation made by the owner
        // while this loop was running.
        let observed = slot.peak.load(Ordering::Relaxed);
        let _ = slot.peak.compare_exchange(
            observed,
            slot.depth_last.load(Ordering::Relaxed),
            Ordering::Relaxed,
            Ordering::Relaxed,
        );
        #[cfg(feature = "watermark")]
        slot.saturated.store(false, Ordering::Relaxed);
    }
}

fn count_live() -> u64 {
    SLOTS.iter().filter(|slot| slot.is_occupied()).count() as u64
}