Skip to main content

keyhog_profile/
runtime.rs

1use crate::schema::Stage;
2use crate::schema_v2::{
3    AnnotationV2, BatchRouteV2, BlockedWaitRecordV2, CacheEffectivenessV2, DistributionBucketV2,
4    Evidence, EvidenceGap, IndexedCounterRecordV2, LatencyBucketV2, LatencyDistributionV2,
5    MetricDistributionV2, PointEventV2, QueueDepthV2, QueueLinkV2, RetryRecordV2, SpanRecordV2,
6    StageConcurrencyV2, TypedMetricRecordV2, WorkOrigin, WorkerImbalanceV2, WorkerLoadV2,
7    WorkerOccupancyRowV2, WorkerOccupancyV2,
8};
9use std::cell::{Cell, RefCell};
10use std::collections::HashMap;
11use std::future::{poll_fn, Future};
12use std::marker::PhantomData;
13use std::rc::Rc;
14use std::sync::atomic::{AtomicU64, AtomicU8, AtomicUsize, Ordering};
15use std::sync::{Arc, Mutex, Weak};
16use std::time::Instant;
17
18pub(crate) const STAGE_COUNT: usize = Stage::ALL.len();
19
20/// Maximum number of causal span records retained by one profiling runtime.
21pub const MAX_RECORDED_SPANS: usize = 65_536;
22const MAX_NESTED_SPANS: usize = 64;
23const LATENCY_BUCKET_COUNT: usize = 65;
24pub const MAX_POINT_EVENTS: usize = 16_384;
25pub const MAX_ANNOTATIONS: usize = 16_384;
26/// Maximum pending enqueues and completed links retained per runtime.
27pub const MAX_QUEUE_LINKS: usize = 16_384;
28/// Hard cap on retained batch-route records; further routes count as drops.
29pub const MAX_BATCH_ROUTES: usize = 16_384;
30
31/// Exact reasons queue causality records were not retained.
32#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
33pub struct QueueLinkLossCounts {
34    /// Pending enqueues dropped at capacity or displaced by a duplicate sequence.
35    pub dropped_enqueues: u64,
36    /// Completed links dropped at capacity.
37    pub dropped_links: u64,
38    /// Dequeues with no recorded matching enqueue.
39    pub unmatched_dequeues: u64,
40    /// Pending enqueues never matched by a dequeue before the drain.
41    pub unconsumed_enqueues: u64,
42}
43
44/// Portable causal parent captured from a runtime's current span context.
45///
46/// The token is plain data: pass it across crate, thread, or spawn boundaries
47/// and attach it with [`span_with_parent`] or [`instrument_future_with_parent`]
48/// where thread-local propagation cannot reach. A token only applies inside
49/// the runtime it was captured from; elsewhere the span records as a root.
50#[derive(Clone, Copy, Debug, Eq, PartialEq)]
51pub struct CausalParent {
52    context_id: u64,
53    span_id: u64,
54}
55
56impl CausalParent {
57    /// Process-local identity of the runtime this token was captured from.
58    pub const fn context_id(self) -> u64 {
59        self.context_id
60    }
61
62    /// Parent span identifier, or zero when captured outside any span.
63    pub const fn span_id(self) -> u64 {
64        self.span_id
65    }
66
67    /// Whether this token names the runtime root instead of a live span.
68    pub const fn is_root(self) -> bool {
69        self.span_id == 0
70    }
71}
72
73/// Deterministic bounded policy for retaining expensive detail events.
74#[derive(Clone, Copy, Debug, Eq, PartialEq)]
75pub struct SamplingPolicy {
76    initial_events: u64,
77    every_nth_after_initial: u64,
78    maximum_retained: u64,
79}
80
81impl SamplingPolicy {
82    /// Retain `initial_events`, then every Nth observation, up to an absolute bound.
83    pub const fn bounded(
84        initial_events: u64,
85        every_nth_after_initial: u64,
86        maximum_retained: u64,
87    ) -> Self {
88        Self {
89            initial_events,
90            every_nth_after_initial: if every_nth_after_initial == 0 {
91                1
92            } else {
93                every_nth_after_initial
94            },
95            maximum_retained,
96        }
97    }
98
99    fn selects(self, observation: u64) -> bool {
100        observation < self.initial_events
101            || (observation - self.initial_events).is_multiple_of(self.every_nth_after_initial)
102    }
103}
104
105/// Exact reasons typed timeline records were not retained.
106#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
107pub struct EventLossCounts {
108    pub point_events: u64,
109    pub annotations: u64,
110    pub sampled_out_events: u64,
111}
112
113impl EventLossCounts {
114    /// Total capacity loss. Policy sampling is reported separately.
115    pub const fn capacity_drops(self) -> u64 {
116        self.point_events.saturating_add(self.annotations)
117    }
118}
119
120#[derive(Clone, Copy)]
121struct ActiveSpan {
122    runtime_key: usize,
123    span_id: u64,
124    child_elapsed_ns: u64,
125    parent_slot: Option<usize>,
126}
127
128#[derive(Clone, Copy)]
129struct SpanTrace {
130    record_index: usize,
131    span_id: u64,
132}
133
134/// One completed span's recording payload, assembled on the guard's drop path.
135#[derive(Clone, Copy)]
136struct SpanOutcome {
137    start_offset_ns: u64,
138    elapsed_ns: u64,
139    self_ns: u64,
140    blocked: bool,
141    serial: bool,
142    outermost: bool,
143}
144
145struct RawSpanRecord {
146    span_id: u64,
147    parent_span_id: u64,
148    metric_id: crate::MetricId,
149    start_ns: u64,
150    inclusive_ns: u64,
151    thread_id: u64,
152    worker_id: u64,
153    task_id: u64,
154    work_origin: WorkOrigin,
155    completed: bool,
156    hardware: RawSpanHardware,
157}
158
159/// Raw counter readings captured at span edges; the cold path joins them.
160#[derive(Clone, Copy, Default)]
161struct RawSpanHardware {
162    cycles_begin: u64,
163    cycles_end: u64,
164    instructions_begin: u64,
165    instructions_end: u64,
166    has_cycles: bool,
167    has_instructions: bool,
168    finished: bool,
169}
170
171impl RawSpanHardware {
172    fn begin() -> Self {
173        let mut hardware = Self::default();
174        if let Some(reading) = crate::hardware::span_counter_reading() {
175            if let Some(cycles) = reading.cycles {
176                hardware.cycles_begin = cycles;
177                hardware.has_cycles = true;
178            }
179            if let Some(instructions) = reading.instructions {
180                hardware.instructions_begin = instructions;
181                hardware.has_instructions = true;
182            }
183        }
184        hardware
185    }
186
187    fn finish(&mut self) {
188        if let Some(reading) = crate::hardware::span_counter_reading() {
189            if self.has_cycles {
190                if let Some(cycles) = reading.cycles {
191                    self.cycles_end = cycles;
192                }
193            }
194            if self.has_instructions {
195                if let Some(instructions) = reading.instructions {
196                    self.instructions_end = instructions;
197                }
198            }
199        }
200        self.finished = true;
201    }
202
203    fn into_evidence(self) -> Evidence<crate::hardware::SpanHardwareV2> {
204        if !self.finished || (!self.has_cycles && !self.has_instructions) {
205            return Evidence::unavailable(EvidenceGap::Unavailable);
206        }
207        let pair = |present: bool, begin: u64, end: u64| {
208            if present {
209                (Evidence::recorded(begin), Evidence::recorded(end))
210            } else {
211                (
212                    Evidence::unavailable(EvidenceGap::Unsupported),
213                    Evidence::unavailable(EvidenceGap::Unsupported),
214                )
215            }
216        };
217        let (cycles_begin, cycles_end) = pair(self.has_cycles, self.cycles_begin, self.cycles_end);
218        let (instructions_begin, instructions_end) = pair(
219            self.has_instructions,
220            self.instructions_begin,
221            self.instructions_end,
222        );
223        Evidence::recorded(crate::hardware::SpanHardwareV2 {
224            version: crate::hardware::SPAN_HARDWARE_V2_VERSION,
225            cycles_begin,
226            cycles_end,
227            instructions_begin,
228            instructions_end,
229        })
230    }
231}
232
233struct PendingQueueEnqueue {
234    thread_id: u64,
235    elapsed_ns: u64,
236}
237
238struct WorkerShard {
239    sequence: u64,
240    elapsed_ns: [AtomicU64; STAGE_COUNT],
241    calls: [AtomicU64; STAGE_COUNT],
242    attributed_ns: [AtomicU64; STAGE_COUNT],
243    blocked_ns: [AtomicU64; STAGE_COUNT],
244    blocked_calls: [AtomicU64; STAGE_COUNT],
245    legacy_elapsed_ns: [AtomicU64; STAGE_COUNT],
246    legacy_calls: [AtomicU64; STAGE_COUNT],
247    legacy_attributed_ns: [AtomicU64; STAGE_COUNT],
248    latency_buckets: [[AtomicU64; LATENCY_BUCKET_COUNT]; STAGE_COUNT],
249    latency_min_ns: [AtomicU64; STAGE_COUNT],
250    latency_max_ns: [AtomicU64; STAGE_COUNT],
251    counter_values: [AtomicU64; crate::MetricId::COUNT],
252    input_bytes: AtomicU64,
253    input_units: AtomicU64,
254    legacy_input_bytes: AtomicU64,
255    legacy_input_units: AtomicU64,
256    derived_decoder_bytes: AtomicU64,
257    backend_dispatched_bytes: AtomicU64,
258    stage_first_start_ns: [AtomicU64; STAGE_COUNT],
259    stage_last_end_ns: [AtomicU64; STAGE_COUNT],
260    stage_bytes: [AtomicU64; STAGE_COUNT],
261    serial_ns: [AtomicU64; STAGE_COUNT],
262    serial_calls: [AtomicU64; STAGE_COUNT],
263    top_level_busy_ns: AtomicU64,
264    top_level_blocked_ns: AtomicU64,
265    top_level_calls: AtomicU64,
266    cache_hits: [AtomicU64; crate::CacheId::COUNT],
267    cache_misses: [AtomicU64; crate::CacheId::COUNT],
268    indexed_counters: [[AtomicU64; crate::INDEXED_COUNTER_SLOTS]; crate::IndexedCounterId::COUNT],
269    indexed_counter_dropped: AtomicU64,
270    retries: [AtomicU64; crate::RetryCause::COUNT],
271    compile_surface_invocations:
272        [[AtomicU64; crate::CompilePhase::COUNT]; crate::CompileSurfaceId::COUNT],
273    compile_surface_loads: [AtomicU64; crate::CompileSurfaceId::COUNT],
274}
275
276const fn zero_counters() -> [AtomicU64; STAGE_COUNT] {
277    [const { AtomicU64::new(0) }; STAGE_COUNT]
278}
279
280const fn zero_indexed_counters(
281) -> [[AtomicU64; crate::INDEXED_COUNTER_SLOTS]; crate::IndexedCounterId::COUNT] {
282    [const { [const { AtomicU64::new(0) }; crate::INDEXED_COUNTER_SLOTS] };
283        crate::IndexedCounterId::COUNT]
284}
285
286const fn zero_event_values() -> [AtomicU64; crate::EventId::COUNT] {
287    [const { AtomicU64::new(0) }; crate::EventId::COUNT]
288}
289
290const fn zero_metric_values() -> [AtomicU64; crate::MetricId::COUNT] {
291    [const { AtomicU64::new(0) }; crate::MetricId::COUNT]
292}
293
294const GAUGE_PRESENT_WORDS: usize = crate::MetricId::COUNT.div_ceil(64);
295
296const fn zero_latency_buckets() -> [[AtomicU64; LATENCY_BUCKET_COUNT]; STAGE_COUNT] {
297    [const { [const { AtomicU64::new(0) }; LATENCY_BUCKET_COUNT] }; STAGE_COUNT]
298}
299
300const fn max_counters() -> [AtomicU64; STAGE_COUNT] {
301    [const { AtomicU64::new(u64::MAX) }; STAGE_COUNT]
302}
303
304const fn zero_cache_counters() -> [AtomicU64; crate::CacheId::COUNT] {
305    [const { AtomicU64::new(0) }; crate::CacheId::COUNT]
306}
307const fn zero_compile_surface_invocations(
308) -> [[AtomicU64; crate::CompilePhase::COUNT]; crate::CompileSurfaceId::COUNT] {
309    [const { [const { AtomicU64::new(0) }; crate::CompilePhase::COUNT] };
310        crate::CompileSurfaceId::COUNT]
311}
312
313const fn zero_compile_surface_loads() -> [AtomicU64; crate::CompileSurfaceId::COUNT] {
314    [const { AtomicU64::new(0) }; crate::CompileSurfaceId::COUNT]
315}
316
317impl WorkerShard {
318    fn new(sequence: u64) -> Self {
319        Self {
320            sequence,
321            elapsed_ns: zero_counters(),
322            calls: zero_counters(),
323            attributed_ns: zero_counters(),
324            blocked_ns: zero_counters(),
325            blocked_calls: zero_counters(),
326            legacy_elapsed_ns: zero_counters(),
327            legacy_calls: zero_counters(),
328            legacy_attributed_ns: zero_counters(),
329            latency_buckets: zero_latency_buckets(),
330            latency_min_ns: max_counters(),
331            latency_max_ns: zero_counters(),
332            counter_values: zero_metric_values(),
333            input_bytes: AtomicU64::new(0),
334            input_units: AtomicU64::new(0),
335            legacy_input_bytes: AtomicU64::new(0),
336            legacy_input_units: AtomicU64::new(0),
337            derived_decoder_bytes: AtomicU64::new(0),
338            backend_dispatched_bytes: AtomicU64::new(0),
339            stage_first_start_ns: max_counters(),
340            stage_last_end_ns: zero_counters(),
341            stage_bytes: zero_counters(),
342            serial_ns: zero_counters(),
343            serial_calls: zero_counters(),
344            top_level_busy_ns: AtomicU64::new(0),
345            top_level_blocked_ns: AtomicU64::new(0),
346            top_level_calls: AtomicU64::new(0),
347            cache_hits: zero_cache_counters(),
348            cache_misses: zero_cache_counters(),
349            indexed_counters: zero_indexed_counters(),
350            indexed_counter_dropped: AtomicU64::new(0),
351            retries: [const { AtomicU64::new(0) }; crate::RetryCause::COUNT],
352            compile_surface_invocations: zero_compile_surface_invocations(),
353            compile_surface_loads: zero_compile_surface_loads(),
354        }
355    }
356}
357
358fn percentile_upper_bound(
359    buckets: &[LatencyBucketV2],
360    call_count: u64,
361    percentile: u64,
362    maximum_ns: u64,
363) -> u64 {
364    let rank = (u128::from(call_count) * u128::from(percentile)).div_ceil(100);
365    let mut cumulative = 0_u128;
366    for bucket in buckets {
367        cumulative += u128::from(bucket.count);
368        if cumulative >= rank {
369            return bucket.upper_bound_ns.min(maximum_ns);
370        }
371    }
372    maximum_ns
373}
374
375#[inline]
376fn latency_bucket_index(elapsed_ns: u64) -> usize {
377    if elapsed_ns == 0 {
378        0
379    } else {
380        (u64::BITS - elapsed_ns.leading_zeros()) as usize
381    }
382}
383
384fn latency_bucket_bounds(index: usize) -> (u64, u64) {
385    match index {
386        0 => (0, 0),
387        64 => (1_u64 << 63, u64::MAX),
388        _ => (1_u64 << (index - 1), (1_u64 << index) - 1),
389    }
390}
391
392pub(crate) struct RawStageCounters {
393    pub(crate) elapsed_ns: [u64; STAGE_COUNT],
394    pub(crate) calls: [u64; STAGE_COUNT],
395    pub(crate) attributed_ns: [u64; STAGE_COUNT],
396}
397
398static ACTIVE_CONTEXTS: AtomicUsize = AtomicUsize::new(0);
399static NEXT_THREAD_ID: AtomicU64 = AtomicU64::new(1);
400static NEXT_CONTEXT_ID: AtomicU64 = AtomicU64::new(1);
401static PROCESS_ACTIVE_COMPILE_PHASE: AtomicU8 = AtomicU8::new(crate::CompilePhase::Scan as u8);
402
403struct RuntimeInner {
404    context_id: u64,
405    next_shard_sequence: AtomicU64,
406    elapsed_ns: [AtomicU64; STAGE_COUNT],
407    calls: [AtomicU64; STAGE_COUNT],
408    attributed_ns: [AtomicU64; STAGE_COUNT],
409    session_shards: Mutex<Vec<Arc<WorkerShard>>>,
410    session_gauge_values: [AtomicU64; crate::MetricId::COUNT],
411    session_gauge_present: [AtomicU64; GAUGE_PRESENT_WORDS],
412    input_bytes: AtomicU64,
413    input_units: AtomicU64,
414    session_recording: bool,
415    session_route_sequence: AtomicU64,
416    session_batch_routes: Mutex<Vec<BatchRouteV2>>,
417    session_dropped_batch_routes: AtomicU64,
418    started: Instant,
419    session_span_sequence: AtomicU64,
420    session_span_reservations: AtomicUsize,
421    session_spans: Mutex<Vec<RawSpanRecord>>,
422    session_dropped_spans: AtomicU64,
423    session_event_sequence: AtomicU64,
424    session_point_events: Mutex<Vec<PointEventV2>>,
425    session_annotations: Mutex<Vec<AnnotationV2>>,
426    session_dropped_point_events: AtomicU64,
427    session_dropped_annotations: AtomicU64,
428    session_sample_observations: [AtomicU64; crate::EventId::COUNT],
429    session_sample_retained: [AtomicU64; crate::EventId::COUNT],
430    session_sampled_out_events: AtomicU64,
431    queue_pending: Mutex<HashMap<(u8, u64), PendingQueueEnqueue>>,
432    queue_links: Mutex<Vec<QueueLinkV2>>,
433    queue_dropped_enqueues: AtomicU64,
434    queue_dropped_links: AtomicU64,
435    queue_unmatched_dequeues: AtomicU64,
436    queue_depth_current: [AtomicU64; crate::QueueId::COUNT],
437    queue_depth_high_water: [AtomicU64; crate::QueueId::COUNT],
438    queue_depth_enqueues: [AtomicU64; crate::QueueId::COUNT],
439    queue_depth_dequeues: [AtomicU64; crate::QueueId::COUNT],
440    legacy_typed_counters: [AtomicU64; crate::MetricId::COUNT],
441    distribution_buckets: [[AtomicU64; LATENCY_BUCKET_COUNT]; crate::MetricId::COUNT],
442    distribution_min: [AtomicU64; crate::MetricId::COUNT],
443    distribution_max: [AtomicU64; crate::MetricId::COUNT],
444    active_compile_phase: AtomicU8,
445    legacy_compile_surface_invocations:
446        [[AtomicU64; crate::CompilePhase::COUNT]; crate::CompileSurfaceId::COUNT],
447    legacy_compile_surface_loads: [AtomicU64; crate::CompileSurfaceId::COUNT],
448}
449
450const fn zero_queue_depths() -> [AtomicU64; crate::QueueId::COUNT] {
451    [const { AtomicU64::new(0) }; crate::QueueId::COUNT]
452}
453
454const fn zero_distribution_buckets() -> [[AtomicU64; LATENCY_BUCKET_COUNT]; crate::MetricId::COUNT]
455{
456    [const { [const { AtomicU64::new(0) }; LATENCY_BUCKET_COUNT] }; crate::MetricId::COUNT]
457}
458
459const fn zero_distribution_mins() -> [AtomicU64; crate::MetricId::COUNT] {
460    [const { AtomicU64::new(u64::MAX) }; crate::MetricId::COUNT]
461}
462
463const fn zero_distribution_maxes() -> [AtomicU64; crate::MetricId::COUNT] {
464    [const { AtomicU64::new(0) }; crate::MetricId::COUNT]
465}
466
467impl RuntimeInner {
468    fn new(session_recording: bool, started: Instant) -> Self {
469        Self {
470            context_id: NEXT_CONTEXT_ID.fetch_add(1, Ordering::Relaxed),
471            next_shard_sequence: AtomicU64::new(1),
472            elapsed_ns: zero_counters(),
473            calls: zero_counters(),
474            attributed_ns: zero_counters(),
475            session_shards: Mutex::new(Vec::new()),
476            session_gauge_values: zero_metric_values(),
477            session_gauge_present: [const { AtomicU64::new(0) }; GAUGE_PRESENT_WORDS],
478            input_bytes: AtomicU64::new(0),
479            input_units: AtomicU64::new(0),
480            session_recording,
481            session_route_sequence: AtomicU64::new(0),
482            session_batch_routes: Mutex::new(Vec::new()),
483            session_dropped_batch_routes: AtomicU64::new(0),
484            started,
485            session_span_sequence: AtomicU64::new(0),
486            session_span_reservations: AtomicUsize::new(0),
487            session_spans: Mutex::new(Vec::new()),
488            session_dropped_spans: AtomicU64::new(0),
489            session_event_sequence: AtomicU64::new(0),
490            session_point_events: Mutex::new(Vec::new()),
491            session_annotations: Mutex::new(Vec::new()),
492            session_dropped_point_events: AtomicU64::new(0),
493            session_dropped_annotations: AtomicU64::new(0),
494            session_sample_observations: zero_event_values(),
495            session_sample_retained: zero_event_values(),
496            session_sampled_out_events: AtomicU64::new(0),
497            queue_pending: Mutex::new(HashMap::new()),
498            queue_links: Mutex::new(Vec::new()),
499            queue_dropped_enqueues: AtomicU64::new(0),
500            queue_dropped_links: AtomicU64::new(0),
501            queue_unmatched_dequeues: AtomicU64::new(0),
502            queue_depth_current: zero_queue_depths(),
503            queue_depth_high_water: zero_queue_depths(),
504            queue_depth_enqueues: zero_queue_depths(),
505            queue_depth_dequeues: zero_queue_depths(),
506            legacy_typed_counters: zero_metric_values(),
507            distribution_buckets: zero_distribution_buckets(),
508            distribution_min: zero_distribution_mins(),
509            distribution_max: zero_distribution_maxes(),
510            active_compile_phase: AtomicU8::new(
511                PROCESS_ACTIVE_COMPILE_PHASE.load(Ordering::Relaxed),
512            ),
513            legacy_compile_surface_invocations: zero_compile_surface_invocations(),
514            legacy_compile_surface_loads: zero_compile_surface_loads(),
515        }
516    }
517
518    fn sorted_shards(&self) -> Vec<Arc<WorkerShard>> {
519        let shards = match self.session_shards.lock() {
520            Ok(shards) => shards,
521            Err(poisoned) => poisoned.into_inner(),
522        };
523        let mut sorted: Vec<Arc<WorkerShard>> = shards.to_vec();
524        sorted.sort_by_key(|shard| shard.sequence);
525        sorted
526    }
527}
528
529struct ThreadShardAssignment {
530    runtime: Weak<RuntimeInner>,
531    shard: Arc<WorkerShard>,
532}
533
534/// Owned fixed-stage metric storage that can be propagated across worker boundaries.
535#[derive(Clone)]
536pub struct Runtime {
537    inner: Arc<RuntimeInner>,
538}
539
540impl Runtime {
541    /// Create an isolated runtime whose measurements can back one profiling session.
542    pub fn new() -> Self {
543        Self::new_at(Instant::now())
544    }
545
546    /// Process-local identity that distinguishes this runtime from every peer.
547    pub fn context_id(&self) -> u64 {
548        self.inner.context_id
549    }
550
551    pub(crate) fn new_at(started: Instant) -> Self {
552        Self {
553            inner: Arc::new(RuntimeInner::new(true, started)),
554        }
555    }
556
557    fn legacy() -> Self {
558        Self {
559            inner: Arc::new(RuntimeInner::new(false, Instant::now())),
560        }
561    }
562
563    fn worker_shard(&self) -> Option<Arc<WorkerShard>> {
564        if !self.inner.session_recording {
565            return None;
566        }
567        THREAD_SHARDS.with(|assignments| {
568            let mut assignments = assignments.borrow_mut();
569            assignments.retain(|assignment| assignment.runtime.strong_count() != 0);
570            if let Some(assignment) = assignments.iter().find(|assignment| {
571                std::ptr::eq(assignment.runtime.as_ptr(), Arc::as_ptr(&self.inner))
572            }) {
573                return Some(assignment.shard.clone());
574            }
575            let shard = Arc::new(WorkerShard::new(
576                self.inner
577                    .next_shard_sequence
578                    .fetch_add(1, Ordering::Relaxed),
579            ));
580            match self.inner.session_shards.lock() {
581                Ok(mut shards) => shards.push(shard.clone()),
582                Err(poisoned) => poisoned.into_inner().push(shard.clone()),
583            }
584            assignments.push(ThreadShardAssignment {
585                runtime: Arc::downgrade(&self.inner),
586                shard: shard.clone(),
587            });
588            Some(shard)
589        })
590    }
591
592    /// Number of isolated worker counter shards registered by this runtime.
593    pub fn worker_shard_count(&self) -> usize {
594        match self.inner.session_shards.lock() {
595            Ok(shards) => shards.len(),
596            Err(poisoned) => poisoned.into_inner().len(),
597        }
598    }
599
600    /// Shard sequence of the calling thread without registering a new shard.
601    fn peek_worker_shard_sequence(&self) -> u64 {
602        THREAD_SHARDS.with(|assignments| {
603            assignments
604                .borrow()
605                .iter()
606                .find(|assignment| {
607                    std::ptr::eq(assignment.runtime.as_ptr(), Arc::as_ptr(&self.inner))
608                })
609                .map_or(0, |assignment| assignment.shard.sequence)
610        })
611    }
612
613    /// Capture a portable token naming this runtime's current causal parent.
614    pub fn causal_parent(&self) -> CausalParent {
615        CausalParent {
616            context_id: self.inner.context_id,
617            span_id: self.current_parent_span_id(),
618        }
619    }
620
621    /// Make this runtime current on the calling thread until the guard is dropped.
622    pub fn enter(&self) -> ContextGuard {
623        let _ = self.worker_shard();
624        CURRENT.with(|stack| stack.borrow_mut().push(self.clone()));
625        ACTIVE_CONTEXTS.fetch_add(1, Ordering::Relaxed);
626        ContextGuard {
627            runtime: self.clone(),
628            not_send: PhantomData,
629        }
630    }
631
632    fn enter_async_parent(&self, span_id: u64) -> Option<AsyncParentGuard> {
633        let stack_slot =
634            ASYNC_PARENT_SPANS.with(|stack| stack.borrow().iter().position(Option::is_none))?;
635        let runtime_key = Arc::as_ptr(&self.inner) as usize;
636        ASYNC_PARENT_SPANS.with(|stack| {
637            stack.borrow_mut()[stack_slot] = Some(ActiveSpan {
638                runtime_key,
639                span_id,
640                child_elapsed_ns: 0,
641                parent_slot: None,
642            });
643        });
644        Some(AsyncParentGuard {
645            runtime_key,
646            span_id,
647            stack_slot,
648            not_send: PhantomData,
649        })
650    }
651
652    /// Run one synchronous closure with this runtime as its current context.
653    pub fn scope<T>(&self, operation: impl FnOnce() -> T) -> T {
654        let _guard = self.enter();
655        operation()
656    }
657
658    fn elapsed_ns(&self) -> u64 {
659        u64::try_from(self.inner.started.elapsed().as_nanos()).unwrap_or(u64::MAX)
660    }
661
662    pub(crate) fn add_counter(&self, counter: crate::CounterId, delta: u64) {
663        if let Some(shard) = self.worker_shard() {
664            shard.counter_values[counter.metric_id() as usize].fetch_add(delta, Ordering::Relaxed);
665        } else if !self.inner.session_recording {
666            self.inner.legacy_typed_counters[counter.metric_id() as usize]
667                .fetch_add(delta, Ordering::Relaxed);
668        }
669    }
670
671    fn record_distribution(&self, metric_id: crate::MetricId, value: u64) {
672        let index = metric_id as usize;
673        let bucket = latency_bucket_index(value);
674        self.inner.distribution_buckets[index][bucket].fetch_add(1, Ordering::Relaxed);
675        self.inner.distribution_min[index].fetch_min(value, Ordering::Relaxed);
676        self.inner.distribution_max[index].fetch_max(value, Ordering::Relaxed);
677    }
678
679    /// Count retry annotations recorded so far without draining them.
680    pub(crate) fn retry_annotation_count(&self) -> u64 {
681        if !self.inner.session_recording {
682            return 0;
683        }
684        let annotations = match self.inner.session_annotations.lock() {
685            Ok(annotations) => annotations,
686            Err(poisoned) => poisoned.into_inner(),
687        };
688        annotations
689            .iter()
690            .filter(|annotation| annotation.annotation_id == crate::AnnotationId::RetryAttempt)
691            .count() as u64
692    }
693
694    /// Read one session gauge without clearing it; `None` when never set.
695    pub(crate) fn session_gauge(&self, gauge: crate::GaugeId) -> Option<u64> {
696        if !self.inner.session_recording {
697            return None;
698        }
699        let index = gauge.metric_id() as usize;
700        let present = self.inner.session_gauge_present[index / 64].load(Ordering::Relaxed);
701        (present & (1_u64 << (index % 64)) != 0)
702            .then(|| self.inner.session_gauge_values[index].load(Ordering::Relaxed))
703    }
704
705    /// Record the current retained-buffer level and its running high water.
706    pub(crate) fn record_retained_buffer_bytes(&self, bytes: u64) {
707        if !self.inner.session_recording {
708            return;
709        }
710        self.set_gauge(crate::GaugeId::RetainedBufferBytes, bytes);
711        let peak = crate::GaugeId::RetainedBufferPeakBytes.metric_id() as usize;
712        self.inner.session_gauge_values[peak].fetch_max(bytes, Ordering::Relaxed);
713        self.inner.session_gauge_present[peak / 64]
714            .fetch_or(1_u64 << (peak % 64), Ordering::Relaxed);
715    }
716
717    /// Drain caller-recorded value distributions in stable metric order.
718    pub fn take_metric_distributions(&self) -> Vec<MetricDistributionV2> {
719        let mut records = Vec::new();
720        for index in 0..crate::MetricId::COUNT {
721            let mut call_count = 0_u64;
722            let buckets: Vec<DistributionBucketV2> = (0..LATENCY_BUCKET_COUNT)
723                .filter_map(|bucket| {
724                    let count =
725                        self.inner.distribution_buckets[index][bucket].swap(0, Ordering::Relaxed);
726                    call_count = call_count.saturating_add(count);
727                    if count == 0 {
728                        return None;
729                    }
730                    let (lower_bound, upper_bound) = latency_bucket_bounds(bucket);
731                    Some(DistributionBucketV2 {
732                        version: 1,
733                        lower_bound,
734                        upper_bound,
735                        count,
736                    })
737                })
738                .collect();
739            if call_count == 0 {
740                continue;
741            }
742            let minimum = self.inner.distribution_min[index].swap(u64::MAX, Ordering::Relaxed);
743            let maximum = self.inner.distribution_max[index].swap(0, Ordering::Relaxed);
744            records.push(MetricDistributionV2 {
745                version: 1,
746                metric_id: crate::METRICS[index].id,
747                call_count,
748                minimum,
749                maximum,
750                buckets,
751            });
752        }
753        records
754    }
755
756    /// Drain typed counters recorded by the standalone (non-session) runtime.
757    pub fn take_legacy_typed_metrics(&self) -> Vec<TypedMetricRecordV2> {
758        if self.inner.session_recording {
759            return self.take_session_typed_metrics();
760        }
761        let mut records = Vec::new();
762        for counter in crate::CounterId::ALL {
763            let metric_id = counter.metric_id();
764            let value =
765                self.inner.legacy_typed_counters[metric_id as usize].swap(0, Ordering::Relaxed);
766            if value != 0 {
767                records.push(TypedMetricRecordV2 {
768                    version: 1,
769                    metric_id,
770                    kind: crate::MetricKind::Counter,
771                    value,
772                });
773            }
774        }
775        records
776    }
777
778    pub(crate) fn set_gauge(&self, gauge: crate::GaugeId, value: u64) {
779        if !self.inner.session_recording {
780            return;
781        }
782        let index = gauge.metric_id() as usize;
783        self.inner.session_gauge_values[index].store(value, Ordering::Relaxed);
784        self.inner.session_gauge_present[index / 64]
785            .fetch_or(1_u64 << (index % 64), Ordering::Relaxed);
786    }
787
788    fn record_event(&self, event_id: crate::EventId, value: u64) -> bool {
789        if !self.inner.session_recording {
790            return false;
791        }
792        let sequence = self
793            .inner
794            .session_event_sequence
795            .fetch_add(1, Ordering::Relaxed);
796        let event = PointEventV2 {
797            version: 2,
798            sequence,
799            event_id,
800            elapsed_ns: self.elapsed_ns(),
801            thread_id: numeric_thread_id(),
802            value,
803            task_id: evidence_or_unavailable(current_task_id()),
804            worker_id: evidence_or_unavailable(self.peek_worker_shard_sequence()),
805        };
806        let mut events = match self.inner.session_point_events.lock() {
807            Ok(events) => events,
808            Err(poisoned) => poisoned.into_inner(),
809        };
810        if events.len() == MAX_POINT_EVENTS {
811            self.inner
812                .session_dropped_point_events
813                .fetch_add(1, Ordering::Relaxed);
814            false
815        } else {
816            events.push(event);
817            true
818        }
819    }
820
821    fn record_sampled_event(
822        &self,
823        event_id: crate::EventId,
824        value: u64,
825        policy: SamplingPolicy,
826    ) -> bool {
827        if !self.inner.session_recording {
828            return false;
829        }
830        let index = event_id.index();
831        let observation =
832            self.inner.session_sample_observations[index].fetch_add(1, Ordering::Relaxed);
833        let mut retained = false;
834        if policy.selects(observation) {
835            let counter = &self.inner.session_sample_retained[index];
836            let mut current = counter.load(Ordering::Relaxed);
837            loop {
838                if current >= policy.maximum_retained {
839                    break;
840                }
841                match counter.compare_exchange_weak(
842                    current,
843                    current + 1,
844                    Ordering::Relaxed,
845                    Ordering::Relaxed,
846                ) {
847                    Ok(_) => {
848                        retained = true;
849                        break;
850                    }
851                    Err(observed) => current = observed,
852                }
853            }
854        }
855        if retained {
856            self.record_event(event_id, value)
857        } else {
858            self.inner
859                .session_sampled_out_events
860                .fetch_add(1, Ordering::Relaxed);
861            false
862        }
863    }
864
865    fn record_annotation(&self, annotation_id: crate::AnnotationId, value: u64) {
866        if !self.inner.session_recording {
867            return;
868        }
869        let sequence = self
870            .inner
871            .session_event_sequence
872            .fetch_add(1, Ordering::Relaxed);
873        let annotation = AnnotationV2 {
874            version: 2,
875            sequence,
876            annotation_id,
877            elapsed_ns: self.elapsed_ns(),
878            thread_id: numeric_thread_id(),
879            value,
880            task_id: evidence_or_unavailable(current_task_id()),
881            worker_id: evidence_or_unavailable(self.peek_worker_shard_sequence()),
882        };
883        let mut annotations = match self.inner.session_annotations.lock() {
884            Ok(annotations) => annotations,
885            Err(poisoned) => poisoned.into_inner(),
886        };
887        if annotations.len() == MAX_ANNOTATIONS {
888            self.inner
889                .session_dropped_annotations
890                .fetch_add(1, Ordering::Relaxed);
891        } else {
892            annotations.push(annotation);
893        }
894    }
895
896    /// Drain typed counter and gauge records in stable metric order.
897    pub fn take_session_typed_metrics(&self) -> Vec<TypedMetricRecordV2> {
898        let mut records =
899            Vec::with_capacity(crate::CounterId::ALL.len() + crate::GaugeId::ALL.len());
900        let shards = self.inner.sorted_shards();
901        for counter in crate::CounterId::ALL {
902            let metric_id = counter.metric_id();
903            let value = shards.iter().fold(0_u64, |total, shard| {
904                total.saturating_add(
905                    shard.counter_values[metric_id as usize].swap(0, Ordering::Relaxed),
906                )
907            });
908            if value != 0 {
909                records.push(TypedMetricRecordV2 {
910                    version: 1,
911                    metric_id,
912                    kind: crate::MetricKind::Counter,
913                    value,
914                });
915            }
916        }
917        drop(shards);
918        let present: [u64; GAUGE_PRESENT_WORDS] =
919            std::array::from_fn(|i| self.inner.session_gauge_present[i].swap(0, Ordering::Relaxed));
920        for gauge in crate::GaugeId::ALL {
921            let metric_id = gauge.metric_id();
922            let index = metric_id as usize;
923            if present[index / 64] & (1_u64 << (index % 64)) != 0 {
924                records.push(TypedMetricRecordV2 {
925                    version: 1,
926                    metric_id,
927                    kind: crate::MetricKind::Gauge,
928                    value: self.inner.session_gauge_values[metric_id as usize]
929                        .swap(0, Ordering::Relaxed),
930                });
931            }
932        }
933        records.sort_unstable_by_key(|record| record.metric_id);
934        records
935    }
936
937    /// Drain typed timeline records and exact loss counts by cause.
938    pub fn take_session_typed_events(
939        &self,
940    ) -> (Vec<PointEventV2>, Vec<AnnotationV2>, EventLossCounts) {
941        let mut events = match self.inner.session_point_events.lock() {
942            Ok(events) => events,
943            Err(poisoned) => poisoned.into_inner(),
944        };
945        let mut events = std::mem::take(&mut *events);
946        let mut annotations = match self.inner.session_annotations.lock() {
947            Ok(annotations) => annotations,
948            Err(poisoned) => poisoned.into_inner(),
949        };
950        let mut annotations = std::mem::take(&mut *annotations);
951        events.sort_unstable_by_key(|event| event.sequence);
952        annotations.sort_unstable_by_key(|annotation| annotation.sequence);
953        let loss = EventLossCounts {
954            point_events: self
955                .inner
956                .session_dropped_point_events
957                .swap(0, Ordering::Relaxed),
958            annotations: self
959                .inner
960                .session_dropped_annotations
961                .swap(0, Ordering::Relaxed),
962            sampled_out_events: self
963                .inner
964                .session_sampled_out_events
965                .swap(0, Ordering::Relaxed),
966        };
967        (events, annotations, loss)
968    }
969
970    fn current_parent(&self) -> (Option<usize>, u64) {
971        let runtime_key = Arc::as_ptr(&self.inner) as usize;
972        let active = ACTIVE_SPANS.with(|stack| {
973            let stack = stack.borrow();
974            stack.iter().enumerate().rev().find_map(|(slot, active)| {
975                active
976                    .and_then(|a| (a.runtime_key == runtime_key).then_some((Some(slot), a.span_id)))
977            })
978        });
979        if let Some(pair) = active {
980            return pair;
981        }
982        let async_span_id = ASYNC_PARENT_SPANS.with(|stack| {
983            stack
984                .borrow()
985                .iter()
986                .rev()
987                .flatten()
988                .find(|active| active.runtime_key == runtime_key)
989                .map(|active| active.span_id)
990        });
991        (None, async_span_id.unwrap_or(0))
992    }
993
994    fn current_parent_span_id(&self) -> u64 {
995        self.current_parent().1
996    }
997
998    fn reserve_span_with_id(
999        &self,
1000        span_id: u64,
1001        stage: Stage,
1002        started: Instant,
1003        parent_span_id: u64,
1004        _stack_slot: Option<usize>,
1005        worker_id: u64,
1006    ) -> Option<SpanTrace> {
1007        if !self.inner.session_recording {
1008            return None;
1009        }
1010        let reservation = self
1011            .inner
1012            .session_span_reservations
1013            .fetch_add(1, Ordering::Relaxed);
1014        if reservation >= MAX_RECORDED_SPANS {
1015            self.inner
1016                .session_dropped_spans
1017                .fetch_add(1, Ordering::Relaxed);
1018            return None;
1019        }
1020        let mut records = match self.inner.session_spans.lock() {
1021            Ok(records) => records,
1022            Err(poisoned) => poisoned.into_inner(),
1023        };
1024        let record_index = records.len();
1025        records.push(RawSpanRecord {
1026            span_id,
1027            parent_span_id,
1028            metric_id: stage.into(),
1029            start_ns: u64::try_from(
1030                started
1031                    .checked_duration_since(self.inner.started)
1032                    .unwrap_or_default()
1033                    .as_nanos(),
1034            )
1035            .unwrap_or(u64::MAX),
1036            inclusive_ns: 0,
1037            thread_id: numeric_thread_id(),
1038            worker_id,
1039            task_id: current_task_id(),
1040            work_origin: current_work_origin(),
1041            completed: false,
1042            hardware: RawSpanHardware::begin(),
1043        });
1044        Some(SpanTrace {
1045            record_index,
1046            span_id,
1047        })
1048    }
1049
1050    fn reserve_span(
1051        &self,
1052        stage: Stage,
1053        started: Instant,
1054        parent_span_id: u64,
1055        stack_slot: Option<usize>,
1056        worker_id: u64,
1057    ) -> Option<SpanTrace> {
1058        let span_id = self
1059            .inner
1060            .session_span_sequence
1061            .fetch_add(1, Ordering::Relaxed)
1062            .saturating_add(1);
1063        self.reserve_span_with_id(
1064            span_id,
1065            stage,
1066            started,
1067            parent_span_id,
1068            stack_slot,
1069            worker_id,
1070        )
1071    }
1072
1073    fn begin_span_with(
1074        &self,
1075        stage: Stage,
1076        started: Instant,
1077        parent_span_id: u64,
1078        parent_slot: Option<usize>,
1079        worker_id: u64,
1080    ) -> (Option<SpanTrace>, Option<usize>, u64) {
1081        let span_id = self
1082            .inner
1083            .session_span_sequence
1084            .fetch_add(1, Ordering::Relaxed)
1085            .saturating_add(1);
1086        let stack_slot = ACTIVE_SPANS.with(|stack| stack.borrow().iter().position(Option::is_none));
1087        let Some(stack_slot) = stack_slot else {
1088            self.inner
1089                .session_dropped_spans
1090                .fetch_add(1, Ordering::Relaxed);
1091            return (None, None, span_id);
1092        };
1093        let trace = if self.inner.session_recording {
1094            self.reserve_span_with_id(
1095                span_id,
1096                stage,
1097                started,
1098                parent_span_id,
1099                Some(stack_slot),
1100                worker_id,
1101            )
1102        } else {
1103            None
1104        };
1105        let runtime_key = Arc::as_ptr(&self.inner) as usize;
1106        ACTIVE_SPANS.with(|stack| {
1107            stack.borrow_mut()[stack_slot] = Some(ActiveSpan {
1108                runtime_key,
1109                span_id,
1110                child_elapsed_ns: 0,
1111                parent_slot,
1112            });
1113        });
1114        (trace, Some(stack_slot), span_id)
1115    }
1116
1117    fn pop_active_span(
1118        &self,
1119        stack_slot: Option<usize>,
1120        expected_span_id: u64,
1121        elapsed_ns: u64,
1122    ) -> u64 {
1123        let Some(stack_slot) = stack_slot else {
1124            return elapsed_ns;
1125        };
1126        let runtime_key = Arc::as_ptr(&self.inner) as usize;
1127        let child_elapsed_ns = ACTIVE_SPANS.with(|stack| {
1128            let mut stack = stack.borrow_mut();
1129            if let Some(active) = &stack[stack_slot] {
1130                if active.runtime_key == runtime_key && active.span_id == expected_span_id {
1131                    let active = stack[stack_slot].take().unwrap();
1132                    if let Some(parent_slot) = active.parent_slot {
1133                        if let Some(parent) = stack[parent_slot].as_mut() {
1134                            if parent.runtime_key == runtime_key {
1135                                parent.child_elapsed_ns =
1136                                    parent.child_elapsed_ns.saturating_add(elapsed_ns);
1137                            }
1138                        }
1139                    }
1140                    return active.child_elapsed_ns;
1141                }
1142            }
1143            0
1144        });
1145        elapsed_ns.saturating_sub(child_elapsed_ns)
1146    }
1147    fn begin_async_span(
1148        &self,
1149        stage: Stage,
1150        started: Instant,
1151        parent_span_id: u64,
1152        worker_id: u64,
1153    ) -> Option<SpanTrace> {
1154        if !self.inner.session_recording {
1155            return None;
1156        }
1157        self.reserve_span(stage, started, parent_span_id, None, worker_id)
1158    }
1159
1160    fn finish_span(&self, trace: SpanTrace, inclusive_ns: u64) {
1161        let mut records = match self.inner.session_spans.lock() {
1162            Ok(records) => records,
1163            Err(poisoned) => poisoned.into_inner(),
1164        };
1165        if let Some(record) = records.get_mut(trace.record_index) {
1166            record.inclusive_ns = inclusive_ns;
1167            record.completed = true;
1168            record.hardware.finish();
1169        }
1170    }
1171
1172    /// Drain bounded causal spans and the exact count omitted or unfinished.
1173    pub fn take_session_span_records(&self) -> (Vec<SpanRecordV2>, u64) {
1174        let mut records = match self.inner.session_spans.lock() {
1175            Ok(records) => records,
1176            Err(poisoned) => poisoned.into_inner(),
1177        };
1178        let mut raw = std::mem::take(&mut *records);
1179        drop(records);
1180        raw.sort_unstable_by_key(|record| record.span_id);
1181        let unfinished = raw.iter().filter(|record| !record.completed).count() as u64;
1182        raw.retain(|record| record.completed);
1183        let positions: HashMap<u64, usize> = raw
1184            .iter()
1185            .enumerate()
1186            .map(|(index, record)| (record.span_id, index))
1187            .collect();
1188        let mut exclusive: Vec<u64> = raw.iter().map(|record| record.inclusive_ns).collect();
1189        for record in &raw {
1190            if let Some(parent_index) = positions.get(&record.parent_span_id).copied() {
1191                exclusive[parent_index] =
1192                    exclusive[parent_index].saturating_sub(record.inclusive_ns);
1193            }
1194        }
1195        let spans = raw
1196            .into_iter()
1197            .enumerate()
1198            .map(|(index, record)| SpanRecordV2 {
1199                version: 3,
1200                span_id: record.span_id,
1201                parent_span_id: if positions.contains_key(&record.parent_span_id) {
1202                    Evidence::recorded(record.parent_span_id)
1203                } else {
1204                    Evidence::unavailable(EvidenceGap::Unavailable)
1205                },
1206                metric_id: record.metric_id,
1207                start_ns: record.start_ns,
1208                inclusive_ns: record.inclusive_ns,
1209                exclusive_ns: exclusive[index],
1210                thread_id: record.thread_id,
1211                task_id: evidence_or_unavailable(record.task_id),
1212                worker_id: evidence_or_unavailable(record.worker_id),
1213                work_origin: record.work_origin,
1214                hardware: record.hardware.into_evidence(),
1215            })
1216            .collect();
1217        let dropped = self
1218            .inner
1219            .session_dropped_spans
1220            .swap(0, Ordering::Relaxed)
1221            .saturating_add(unfinished);
1222        (spans, dropped)
1223    }
1224
1225    /// Drain per-micro-function logarithmic latency distributions.
1226    pub fn take_session_latency_distributions(&self) -> Vec<LatencyDistributionV2> {
1227        let shards = self.inner.sorted_shards();
1228        Stage::ALL
1229            .into_iter()
1230            .filter_map(|stage| {
1231                let mut call_count = 0_u64;
1232                let buckets: Vec<LatencyBucketV2> = (0..LATENCY_BUCKET_COUNT)
1233                    .filter_map(|index| {
1234                        let count = shards.iter().fold(0_u64, |total, shard| {
1235                            total.saturating_add(
1236                                shard.latency_buckets[stage.index()][index]
1237                                    .swap(0, Ordering::Relaxed),
1238                            )
1239                        });
1240                        call_count = call_count.saturating_add(count);
1241                        (count != 0).then(|| {
1242                            let (lower_bound_ns, upper_bound_ns) = latency_bucket_bounds(index);
1243                            LatencyBucketV2 {
1244                                version: 1,
1245                                lower_bound_ns,
1246                                upper_bound_ns,
1247                                count,
1248                            }
1249                        })
1250                    })
1251                    .collect();
1252                if call_count == 0 {
1253                    return None;
1254                }
1255                let minimum_ns = shards.iter().fold(u64::MAX, |minimum, shard| {
1256                    minimum
1257                        .min(shard.latency_min_ns[stage.index()].swap(u64::MAX, Ordering::Relaxed))
1258                });
1259                let maximum_ns = shards.iter().fold(0_u64, |maximum, shard| {
1260                    maximum.max(shard.latency_max_ns[stage.index()].swap(0, Ordering::Relaxed))
1261                });
1262                Some(LatencyDistributionV2 {
1263                    version: 2,
1264                    metric_id: stage.metric_id(),
1265                    macro_stage_id: stage.macro_stage_id(),
1266                    call_count,
1267                    minimum_ns,
1268                    maximum_ns,
1269                    p50_ns: percentile_upper_bound(&buckets, call_count, 50, maximum_ns),
1270                    p90_ns: percentile_upper_bound(&buckets, call_count, 90, maximum_ns),
1271                    p95_ns: percentile_upper_bound(&buckets, call_count, 95, maximum_ns),
1272                    p99_ns: percentile_upper_bound(&buckets, call_count, 99, maximum_ns),
1273                    buckets,
1274                })
1275            })
1276            .collect()
1277    }
1278
1279    fn record(&self, shard: Option<&WorkerShard>, stage: Stage, outcome: SpanOutcome) {
1280        let index = stage.index();
1281        let elapsed_ns = outcome.elapsed_ns;
1282        let self_ns = outcome.self_ns;
1283        if self.inner.session_recording {
1284            let Some(shard) = shard else {
1285                return;
1286            };
1287            shard.elapsed_ns[index].fetch_add(elapsed_ns, Ordering::Relaxed);
1288            shard.calls[index].fetch_add(1, Ordering::Relaxed);
1289            shard.legacy_elapsed_ns[index].fetch_add(elapsed_ns, Ordering::Relaxed);
1290            shard.legacy_calls[index].fetch_add(1, Ordering::Relaxed);
1291            let bucket = latency_bucket_index(elapsed_ns);
1292            shard.latency_buckets[index][bucket].fetch_add(1, Ordering::Relaxed);
1293            shard.latency_min_ns[index].fetch_min(elapsed_ns, Ordering::Relaxed);
1294            shard.latency_max_ns[index].fetch_max(elapsed_ns, Ordering::Relaxed);
1295            shard.stage_first_start_ns[index].fetch_min(outcome.start_offset_ns, Ordering::Relaxed);
1296            shard.stage_last_end_ns[index].fetch_max(
1297                outcome.start_offset_ns.saturating_add(elapsed_ns),
1298                Ordering::Relaxed,
1299            );
1300            // Every non-blocked stage carries its self-time (exclusive elapsed time) in attributed_ns.
1301            // Blocked wait and container stages are never attributed execution.
1302            if !outcome.blocked && !stage.is_container() {
1303                shard.attributed_ns[index].fetch_add(self_ns, Ordering::Relaxed);
1304                shard.legacy_attributed_ns[index].fetch_add(self_ns, Ordering::Relaxed);
1305            }
1306
1307            if outcome.blocked {
1308                shard.blocked_ns[index].fetch_add(elapsed_ns, Ordering::Relaxed);
1309                shard.blocked_calls[index].fetch_add(1, Ordering::Relaxed);
1310            }
1311            if outcome.serial {
1312                shard.serial_ns[index].fetch_add(elapsed_ns, Ordering::Relaxed);
1313                shard.serial_calls[index].fetch_add(1, Ordering::Relaxed);
1314            }
1315            // Worker occupancy is computed from self-time: a worker is credited once
1316            // for each nanosecond it actually spends executing or waiting in blocked state.
1317            // Container stages wrapping multi-threaded pipelines do not add to worker occupancy.
1318            if !stage.is_container() {
1319                if outcome.outermost {
1320                    shard.top_level_calls.fetch_add(1, Ordering::Relaxed);
1321                }
1322                if outcome.blocked {
1323                    shard
1324                        .top_level_blocked_ns
1325                        .fetch_add(self_ns, Ordering::Relaxed);
1326                } else {
1327                    shard
1328                        .top_level_busy_ns
1329                        .fetch_add(self_ns, Ordering::Relaxed);
1330                }
1331            }
1332            return;
1333        }
1334        self.inner.elapsed_ns[index].fetch_add(elapsed_ns, Ordering::Relaxed);
1335        self.inner.calls[index].fetch_add(1, Ordering::Relaxed);
1336        if !outcome.blocked && !stage.is_container() {
1337            self.inner.attributed_ns[index].fetch_add(self_ns, Ordering::Relaxed);
1338        }
1339    }
1340
1341    fn add_stage_bytes(&self, stage: Stage, bytes: u64) {
1342        if let Some(shard) = self.worker_shard() {
1343            shard.stage_bytes[stage.index()].fetch_add(bytes, Ordering::Relaxed);
1344        }
1345    }
1346
1347    fn record_cache_outcome(&self, cache: crate::CacheId, hit: bool) {
1348        let Some(shard) = self.worker_shard() else {
1349            return;
1350        };
1351        let slot = if hit {
1352            &shard.cache_hits[cache.index()]
1353        } else {
1354            &shard.cache_misses[cache.index()]
1355        };
1356        slot.fetch_add(1, Ordering::Relaxed);
1357    }
1358
1359    /// Read total cache hits across all worker shards for `cache`.
1360    #[inline]
1361    #[must_use]
1362    pub fn cache_hits(&self, cache: crate::CacheId) -> u64 {
1363        let index = cache.index();
1364        let mut total = 0u64;
1365        for shard in self.inner.sorted_shards() {
1366            total = total.saturating_add(shard.cache_hits[index].load(Ordering::Relaxed));
1367        }
1368        total
1369    }
1370
1371    /// Read total cache misses across all worker shards for `cache`.
1372    #[inline]
1373    #[must_use]
1374    pub fn cache_misses(&self, cache: crate::CacheId) -> u64 {
1375        let index = cache.index();
1376        let mut total = 0u64;
1377        for shard in self.inner.sorted_shards() {
1378            total = total.saturating_add(shard.cache_misses[index].load(Ordering::Relaxed));
1379        }
1380        total
1381    }
1382
1383    fn record_retry(&self, cause: crate::RetryCause) {
1384        if let Some(shard) = self.worker_shard() {
1385            shard.retries[cause.index()].fetch_add(1, Ordering::Relaxed);
1386        }
1387    }
1388
1389    fn add_indexed_counter(&self, counter: crate::IndexedCounterId, slot: u16, delta: u64) {
1390        let Some(shard) = self.worker_shard() else {
1391            return;
1392        };
1393        // Folding an out-of-range slot into the last one would attribute one
1394        // caller's cost to another. Count it as dropped instead.
1395        match shard.indexed_counters[counter.index()].get(usize::from(slot)) {
1396            Some(cell) => {
1397                cell.fetch_add(delta, Ordering::Relaxed);
1398            }
1399            None => {
1400                shard
1401                    .indexed_counter_dropped
1402                    .fetch_add(1, Ordering::Relaxed);
1403            }
1404        }
1405    }
1406
1407    fn add_input_bytes(&self, bytes: u64) {
1408        if let Some(shard) = self.worker_shard() {
1409            shard.input_bytes.fetch_add(bytes, Ordering::Relaxed);
1410            shard.legacy_input_bytes.fetch_add(bytes, Ordering::Relaxed);
1411            shard.counter_values[crate::MetricId::InputBytes as usize]
1412                .fetch_add(bytes, Ordering::Relaxed);
1413        } else {
1414            self.inner.input_bytes.fetch_add(bytes, Ordering::Relaxed);
1415            self.inner.legacy_typed_counters[crate::MetricId::InputBytes as usize]
1416                .fetch_add(bytes, Ordering::Relaxed);
1417        }
1418    }
1419
1420    fn add_input_units(&self, units: u64) {
1421        if let Some(shard) = self.worker_shard() {
1422            shard.input_units.fetch_add(units, Ordering::Relaxed);
1423            shard.legacy_input_units.fetch_add(units, Ordering::Relaxed);
1424            shard.counter_values[crate::MetricId::InputUnits as usize]
1425                .fetch_add(units, Ordering::Relaxed);
1426        } else {
1427            self.inner.input_units.fetch_add(units, Ordering::Relaxed);
1428            self.inner.legacy_typed_counters[crate::MetricId::InputUnits as usize]
1429                .fetch_add(units, Ordering::Relaxed);
1430        }
1431    }
1432
1433    fn add_derived_decoder_bytes(&self, bytes: u64) {
1434        if let Some(shard) = self.worker_shard() {
1435            shard
1436                .derived_decoder_bytes
1437                .fetch_add(bytes, Ordering::Relaxed);
1438        }
1439    }
1440
1441    fn add_backend_dispatched_bytes(&self, bytes: u64) {
1442        if let Some(shard) = self.worker_shard() {
1443            shard
1444                .backend_dispatched_bytes
1445                .fetch_add(bytes, Ordering::Relaxed);
1446        }
1447    }
1448    pub fn set_compile_phase(&self, phase: crate::CompilePhase) {
1449        self.inner
1450            .active_compile_phase
1451            .store(phase as u8, Ordering::Relaxed);
1452    }
1453
1454    pub fn active_compile_phase(&self) -> crate::CompilePhase {
1455        match self.inner.active_compile_phase.load(Ordering::Relaxed) {
1456            0 => crate::CompilePhase::Install,
1457            1 => crate::CompilePhase::Update,
1458            2 => crate::CompilePhase::Scan,
1459            _ => crate::CompilePhase::Developer,
1460        }
1461    }
1462
1463    pub fn record_compile_surface_invocation(&self, surface: crate::CompileSurfaceId) {
1464        let phase = self.active_compile_phase();
1465        self.record_compile_surface_invocation_with_phase(surface, phase);
1466    }
1467
1468    pub fn record_compile_surface_invocation_with_phase(
1469        &self,
1470        surface: crate::CompileSurfaceId,
1471        phase: crate::CompilePhase,
1472    ) {
1473        if let Some(shard) = self.worker_shard() {
1474            shard.compile_surface_invocations[surface.index()][phase.index()]
1475                .fetch_add(1, Ordering::Relaxed);
1476        } else {
1477            self.inner.legacy_compile_surface_invocations[surface.index()][phase.index()]
1478                .fetch_add(1, Ordering::Relaxed);
1479        }
1480    }
1481
1482    pub fn record_compile_surface_load(&self, surface: crate::CompileSurfaceId) {
1483        if let Some(shard) = self.worker_shard() {
1484            shard.compile_surface_loads[surface.index()].fetch_add(1, Ordering::Relaxed);
1485        } else {
1486            self.inner.legacy_compile_surface_loads[surface.index()]
1487                .fetch_add(1, Ordering::Relaxed);
1488        }
1489    }
1490
1491    pub fn total_runtime_compiles(&self) -> u64 {
1492        let mut total = 0u64;
1493        for surface in crate::CompileSurfaceId::ALL {
1494            total = total.saturating_add(
1495                self.compile_surface_invocations(surface, crate::CompilePhase::Scan),
1496            );
1497            total = total.saturating_add(
1498                self.compile_surface_invocations(surface, crate::CompilePhase::Developer),
1499            );
1500        }
1501        total
1502    }
1503
1504    pub fn compile_surface_invocations(
1505        &self,
1506        surface: crate::CompileSurfaceId,
1507        phase: crate::CompilePhase,
1508    ) -> u64 {
1509        let mut count = self.inner.legacy_compile_surface_invocations[surface.index()]
1510            [phase.index()]
1511        .load(Ordering::Relaxed);
1512        let shards = self.inner.sorted_shards();
1513        for shard in shards.iter() {
1514            count = count.saturating_add(
1515                shard.compile_surface_invocations[surface.index()][phase.index()]
1516                    .load(Ordering::Relaxed),
1517            );
1518        }
1519        count
1520    }
1521
1522    pub fn compile_surface_loads(&self, surface: crate::CompileSurfaceId) -> u64 {
1523        let mut count =
1524            self.inner.legacy_compile_surface_loads[surface.index()].load(Ordering::Relaxed);
1525        let shards = self.inner.sorted_shards();
1526        for shard in shards.iter() {
1527            count = count.saturating_add(
1528                shard.compile_surface_loads[surface.index()].load(Ordering::Relaxed),
1529            );
1530        }
1531        count
1532    }
1533
1534    pub fn compile_surface_reports(&self) -> Vec<crate::schema_v2::CompileSurfaceRecordV2> {
1535        crate::CompileSurfaceId::ALL
1536            .iter()
1537            .map(|&surface| {
1538                let install_compiles =
1539                    self.compile_surface_invocations(surface, crate::CompilePhase::Install);
1540                let update_compiles =
1541                    self.compile_surface_invocations(surface, crate::CompilePhase::Update);
1542                let scan_compiles =
1543                    self.compile_surface_invocations(surface, crate::CompilePhase::Scan);
1544                let developer_compiles =
1545                    self.compile_surface_invocations(surface, crate::CompilePhase::Developer);
1546                let runtime_compiles = scan_compiles.saturating_add(developer_compiles);
1547                let loads = self.compile_surface_loads(surface);
1548                crate::schema_v2::CompileSurfaceRecordV2 {
1549                    version: crate::schema_v2::COMPILE_SURFACE_RECORD_V2_VERSION,
1550                    surface,
1551                    name: surface.as_str().to_string(),
1552                    runtime_compiles,
1553                    loads,
1554                    install_compiles,
1555                    update_compiles,
1556                    developer_compiles,
1557                }
1558            })
1559            .collect()
1560    }
1561
1562    pub(crate) fn drain_stage_counters(&self, session: bool) -> RawStageCounters {
1563        if !session && !self.inner.session_recording {
1564            return RawStageCounters {
1565                elapsed_ns: std::array::from_fn(|index| {
1566                    self.inner.elapsed_ns[index].swap(0, Ordering::Relaxed)
1567                }),
1568                calls: std::array::from_fn(|index| {
1569                    self.inner.calls[index].swap(0, Ordering::Relaxed)
1570                }),
1571                attributed_ns: std::array::from_fn(|index| {
1572                    self.inner.attributed_ns[index].swap(0, Ordering::Relaxed)
1573                }),
1574            };
1575        }
1576        let shards = self.inner.sorted_shards();
1577        if !session {
1578            return RawStageCounters {
1579                elapsed_ns: std::array::from_fn(|index| {
1580                    shards.iter().fold(0_u64, |total, shard| {
1581                        total.saturating_add(
1582                            shard.legacy_elapsed_ns[index].swap(0, Ordering::Relaxed),
1583                        )
1584                    })
1585                }),
1586                calls: std::array::from_fn(|index| {
1587                    shards.iter().fold(0_u64, |total, shard| {
1588                        total.saturating_add(shard.legacy_calls[index].swap(0, Ordering::Relaxed))
1589                    })
1590                }),
1591                attributed_ns: std::array::from_fn(|index| {
1592                    shards.iter().fold(0_u64, |total, shard| {
1593                        total.saturating_add(
1594                            shard.legacy_attributed_ns[index].swap(0, Ordering::Relaxed),
1595                        )
1596                    })
1597                }),
1598            };
1599        }
1600        RawStageCounters {
1601            elapsed_ns: std::array::from_fn(|index| {
1602                shards.iter().fold(0_u64, |total, shard| {
1603                    total.saturating_add(shard.elapsed_ns[index].swap(0, Ordering::Relaxed))
1604                })
1605            }),
1606            calls: std::array::from_fn(|index| {
1607                shards.iter().fold(0_u64, |total, shard| {
1608                    total.saturating_add(shard.calls[index].swap(0, Ordering::Relaxed))
1609                })
1610            }),
1611            attributed_ns: std::array::from_fn(|index| {
1612                shards.iter().fold(0_u64, |total, shard| {
1613                    total.saturating_add(shard.attributed_ns[index].swap(0, Ordering::Relaxed))
1614                })
1615            }),
1616        }
1617    }
1618
1619    fn take_input_totals(&self) -> (u64, u64) {
1620        if !self.inner.session_recording {
1621            return (
1622                self.inner.input_bytes.swap(0, Ordering::Relaxed),
1623                self.inner.input_units.swap(0, Ordering::Relaxed),
1624            );
1625        }
1626        let shards = self.inner.sorted_shards();
1627        shards.iter().fold((0_u64, 0_u64), |totals, shard| {
1628            (
1629                totals
1630                    .0
1631                    .saturating_add(shard.legacy_input_bytes.swap(0, Ordering::Relaxed)),
1632                totals
1633                    .1
1634                    .saturating_add(shard.legacy_input_units.swap(0, Ordering::Relaxed)),
1635            )
1636        })
1637    }
1638
1639    pub(crate) fn take_session_input_totals(&self) -> (u64, u64) {
1640        let shards = self.inner.sorted_shards();
1641        shards.iter().fold((0_u64, 0_u64), |totals, shard| {
1642            (
1643                totals
1644                    .0
1645                    .saturating_add(shard.input_bytes.swap(0, Ordering::Relaxed)),
1646                totals
1647                    .1
1648                    .saturating_add(shard.input_units.swap(0, Ordering::Relaxed)),
1649            )
1650        })
1651    }
1652
1653    pub(crate) fn take_session_workload_totals(&self) -> (u64, u64) {
1654        let shards = self.inner.sorted_shards();
1655        shards.iter().fold((0_u64, 0_u64), |totals, shard| {
1656            (
1657                totals
1658                    .0
1659                    .saturating_add(shard.derived_decoder_bytes.swap(0, Ordering::Relaxed)),
1660                totals
1661                    .1
1662                    .saturating_add(shard.backend_dispatched_bytes.swap(0, Ordering::Relaxed)),
1663            )
1664        })
1665    }
1666
1667    fn record_batch_route(
1668        &self,
1669        workload_key_digest: &str,
1670        requested_backend: &str,
1671        selected_backend: &str,
1672        completed_backend: &str,
1673        recovered_from_backend: Option<&str>,
1674    ) {
1675        if !self.inner.session_recording {
1676            return;
1677        }
1678        let batch_sequence = self
1679            .inner
1680            .session_route_sequence
1681            .fetch_add(1, Ordering::Relaxed);
1682        self.record_event(crate::EventId::BackendBatchCompleted, batch_sequence);
1683        if recovered_from_backend.is_some() {
1684            self.record_event(crate::EventId::BackendRecovered, batch_sequence);
1685        }
1686        let record = BatchRouteV2 {
1687            version: 1,
1688            batch_sequence,
1689            workload_key_digest: workload_key_digest.to_owned(),
1690            requested_backend: requested_backend.to_owned(),
1691            selected_backend: selected_backend.to_owned(),
1692            completed_backend: completed_backend.to_owned(),
1693            recovered_from_backend: recovered_from_backend.map_or_else(
1694                || Evidence::unavailable(crate::schema_v2::EvidenceGap::Unavailable),
1695                |backend| Evidence::recorded(backend.to_owned()),
1696            ),
1697        };
1698        let mut records = match self.inner.session_batch_routes.lock() {
1699            Ok(records) => records,
1700            Err(poisoned) => poisoned.into_inner(),
1701        };
1702        if records.len() >= MAX_BATCH_ROUTES {
1703            self.inner
1704                .session_dropped_batch_routes
1705                .fetch_add(1, Ordering::Relaxed);
1706            return;
1707        }
1708        records.push(record);
1709    }
1710
1711    /// Drain completed batch-route records from this profiling runtime.
1712    pub fn take_session_batch_routes(&self) -> Vec<BatchRouteV2> {
1713        let mut records = match self.inner.session_batch_routes.lock() {
1714            Ok(records) => records,
1715            Err(poisoned) => poisoned.into_inner(),
1716        };
1717        let mut drained = std::mem::take(&mut *records);
1718        drained.sort_unstable_by_key(|record| record.batch_sequence);
1719        drained
1720    }
1721
1722    /// Drain the count of batch routes dropped after [`MAX_BATCH_ROUTES`].
1723    pub fn take_session_dropped_batch_routes(&self) -> u64 {
1724        self.inner
1725            .session_dropped_batch_routes
1726            .swap(0, Ordering::Relaxed)
1727    }
1728
1729    fn record_queue_enqueue(&self, queue: crate::QueueId, sequence: u64) {
1730        if !self.inner.session_recording {
1731            return;
1732        }
1733        let enqueue = PendingQueueEnqueue {
1734            thread_id: numeric_thread_id(),
1735            elapsed_ns: self.elapsed_ns(),
1736        };
1737        let mut pending = match self.inner.queue_pending.lock() {
1738            Ok(pending) => pending,
1739            Err(poisoned) => poisoned.into_inner(),
1740        };
1741        if pending.len() == MAX_QUEUE_LINKS && !pending.contains_key(&(queue as u8, sequence)) {
1742            self.inner
1743                .queue_dropped_enqueues
1744                .fetch_add(1, Ordering::Relaxed);
1745            return;
1746        }
1747        if pending.insert((queue as u8, sequence), enqueue).is_some() {
1748            // A duplicate (queue, sequence) enqueue displaces the earlier record.
1749            self.inner
1750                .queue_dropped_enqueues
1751                .fetch_add(1, Ordering::Relaxed);
1752        }
1753    }
1754
1755    fn record_queue_dequeue(&self, queue: crate::QueueId, sequence: u64) {
1756        if !self.inner.session_recording {
1757            return;
1758        }
1759        let enqueue = {
1760            let mut pending = match self.inner.queue_pending.lock() {
1761                Ok(pending) => pending,
1762                Err(poisoned) => poisoned.into_inner(),
1763            };
1764            pending.remove(&(queue as u8, sequence))
1765        };
1766        let Some(enqueue) = enqueue else {
1767            self.inner
1768                .queue_unmatched_dequeues
1769                .fetch_add(1, Ordering::Relaxed);
1770            return;
1771        };
1772        let link = QueueLinkV2 {
1773            version: 1,
1774            queue,
1775            sequence,
1776            producer_thread_id: enqueue.thread_id,
1777            producer_elapsed_ns: enqueue.elapsed_ns,
1778            consumer_thread_id: numeric_thread_id(),
1779            consumer_elapsed_ns: self.elapsed_ns(),
1780        };
1781        let mut links = match self.inner.queue_links.lock() {
1782            Ok(links) => links,
1783            Err(poisoned) => poisoned.into_inner(),
1784        };
1785        if links.len() == MAX_QUEUE_LINKS {
1786            self.inner
1787                .queue_dropped_links
1788                .fetch_add(1, Ordering::Relaxed);
1789        } else {
1790            links.push(link);
1791        }
1792    }
1793
1794    /// Drain matched queue links in stable (queue, sequence) order with exact loss counts.
1795    pub fn take_session_queue_links(&self) -> (Vec<QueueLinkV2>, QueueLinkLossCounts) {
1796        let mut links = match self.inner.queue_links.lock() {
1797            Ok(links) => links,
1798            Err(poisoned) => poisoned.into_inner(),
1799        };
1800        let mut links = std::mem::take(&mut *links);
1801        links.sort_unstable_by_key(|link| (link.queue, link.sequence));
1802        let unconsumed = {
1803            let mut pending = match self.inner.queue_pending.lock() {
1804                Ok(pending) => pending,
1805                Err(poisoned) => poisoned.into_inner(),
1806            };
1807            let pending = std::mem::take(&mut *pending);
1808            u64::try_from(pending.len()).unwrap_or(u64::MAX)
1809        };
1810        let loss = QueueLinkLossCounts {
1811            dropped_enqueues: self.inner.queue_dropped_enqueues.swap(0, Ordering::Relaxed),
1812            dropped_links: self.inner.queue_dropped_links.swap(0, Ordering::Relaxed),
1813            unmatched_dequeues: self
1814                .inner
1815                .queue_unmatched_dequeues
1816                .swap(0, Ordering::Relaxed),
1817            unconsumed_enqueues: unconsumed,
1818        };
1819        (links, loss)
1820    }
1821
1822    fn queue_depth_enqueue(&self, queue: crate::QueueId) {
1823        if !self.inner.session_recording {
1824            return;
1825        }
1826        let index = queue.index();
1827        let depth = self.inner.queue_depth_current[index].fetch_add(1, Ordering::Relaxed) + 1;
1828        self.inner.queue_depth_high_water[index].fetch_max(depth, Ordering::Relaxed);
1829        self.inner.queue_depth_enqueues[index].fetch_add(1, Ordering::Relaxed);
1830    }
1831
1832    fn queue_depth_dequeue(&self, queue: crate::QueueId) {
1833        if !self.inner.session_recording {
1834            return;
1835        }
1836        let index = queue.index();
1837        let counter = &self.inner.queue_depth_current[index];
1838        let mut current = counter.load(Ordering::Relaxed);
1839        loop {
1840            if current == 0 {
1841                break;
1842            }
1843            match counter.compare_exchange_weak(
1844                current,
1845                current - 1,
1846                Ordering::Relaxed,
1847                Ordering::Relaxed,
1848            ) {
1849                Ok(_) => break,
1850                Err(observed) => current = observed,
1851            }
1852        }
1853        self.inner.queue_depth_dequeues[index].fetch_add(1, Ordering::Relaxed);
1854    }
1855
1856    fn set_queue_depth(&self, queue: crate::QueueId, depth: u64) {
1857        if !self.inner.session_recording {
1858            return;
1859        }
1860        let index = queue.index();
1861        self.inner.queue_depth_current[index].store(depth, Ordering::Relaxed);
1862        self.inner.queue_depth_high_water[index].fetch_max(depth, Ordering::Relaxed);
1863    }
1864
1865    /// Drain queue depth occupancy and high-water records in stable queue order.
1866    ///
1867    /// The current depth is reported without being reset; the high-water mark
1868    /// restarts from the current depth and the enqueue/dequeue totals reset.
1869    pub fn take_session_queue_depths(&self) -> Vec<QueueDepthV2> {
1870        if !self.inner.session_recording {
1871            return Vec::new();
1872        }
1873        crate::QueueId::ALL
1874            .into_iter()
1875            .filter_map(|queue| {
1876                let index = queue.index();
1877                let current = self.inner.queue_depth_current[index].load(Ordering::Relaxed);
1878                let high_water =
1879                    self.inner.queue_depth_high_water[index].swap(current, Ordering::Relaxed);
1880                let enqueues = self.inner.queue_depth_enqueues[index].swap(0, Ordering::Relaxed);
1881                let dequeues = self.inner.queue_depth_dequeues[index].swap(0, Ordering::Relaxed);
1882                (current != 0 || high_water != 0 || enqueues != 0 || dequeues != 0).then_some(
1883                    QueueDepthV2 {
1884                        version: 1,
1885                        queue,
1886                        current,
1887                        high_water,
1888                        enqueues,
1889                        dequeues,
1890                    },
1891                )
1892            })
1893            .collect()
1894    }
1895
1896    /// Drain per-worker load and imbalance evidence merged in sorted shard order.
1897    ///
1898    /// This drains the same per-stage call and elapsed counters as
1899    /// `take_session_stage_measurements`; call it once at session drain.
1900    pub fn take_session_worker_imbalance(&self) -> WorkerImbalanceV2 {
1901        let shards = self.inner.sorted_shards();
1902        let workers: Vec<WorkerLoadV2> = shards
1903            .iter()
1904            .map(|shard| {
1905                let mut calls = 0_u64;
1906                let mut elapsed_ns = 0_u64;
1907                for index in 0..STAGE_COUNT {
1908                    calls = calls.saturating_add(shard.calls[index].swap(0, Ordering::Relaxed));
1909                    elapsed_ns = elapsed_ns
1910                        .saturating_add(shard.elapsed_ns[index].swap(0, Ordering::Relaxed));
1911                }
1912                WorkerLoadV2 {
1913                    version: 1,
1914                    worker_id: shard.sequence,
1915                    calls,
1916                    elapsed_ns,
1917                }
1918            })
1919            .collect();
1920        let worker_count = u64::try_from(workers.len()).unwrap_or(u64::MAX);
1921        let total_calls = workers
1922            .iter()
1923            .fold(0_u64, |total, worker| total.saturating_add(worker.calls));
1924        let total_elapsed_ns = workers.iter().fold(0_u64, |total, worker| {
1925            total.saturating_add(worker.elapsed_ns)
1926        });
1927        let ppm = |part: u64, whole: u64| -> u64 {
1928            if whole == 0 {
1929                0
1930            } else {
1931                u64::try_from((u128::from(part) * 1_000_000) / u128::from(whole))
1932                    .unwrap_or(u64::MAX)
1933            }
1934        };
1935        let max_calls = workers.iter().map(|worker| worker.calls).max().unwrap_or(0);
1936        let mut sorted_calls: Vec<u64> = workers.iter().map(|worker| worker.calls).collect();
1937        sorted_calls.sort_unstable();
1938        let median_calls = if sorted_calls.is_empty() {
1939            0
1940        } else {
1941            sorted_calls[sorted_calls.len() / 2]
1942        };
1943        let idle_workers = workers.iter().filter(|worker| worker.calls == 0).count() as u64;
1944        WorkerImbalanceV2 {
1945            version: 1,
1946            worker_count,
1947            total_calls,
1948            total_elapsed_ns,
1949            max_share_ppm: ppm(max_calls, total_calls),
1950            median_share_ppm: ppm(median_calls, total_calls),
1951            idle_share_ppm: ppm(idle_workers, worker_count),
1952            workers,
1953        }
1954    }
1955
1956    /// Drain blocked-wait records merged in sorted shard order.
1957    pub fn take_session_blocked_waits(&self) -> Vec<BlockedWaitRecordV2> {
1958        let shards = self.inner.sorted_shards();
1959        Stage::ALL
1960            .into_iter()
1961            .filter_map(|stage| {
1962                let index = stage.index();
1963                let calls = shards.iter().fold(0_u64, |total, shard| {
1964                    total.saturating_add(shard.blocked_calls[index].swap(0, Ordering::Relaxed))
1965                });
1966                if calls == 0 {
1967                    return None;
1968                }
1969                let blocked_ns = shards.iter().fold(0_u64, |total, shard| {
1970                    total.saturating_add(shard.blocked_ns[index].swap(0, Ordering::Relaxed))
1971                });
1972                Some(BlockedWaitRecordV2 {
1973                    version: 1,
1974                    metric_id: stage.metric_id(),
1975                    macro_stage_id: stage.macro_stage_id(),
1976                    calls,
1977                    blocked_ns,
1978                })
1979            })
1980            .collect()
1981    }
1982
1983    /// Offset of one instant from this runtime's start, saturating at zero.
1984    fn offset_ns(&self, at: Instant) -> u64 {
1985        u64::try_from(
1986            at.checked_duration_since(self.inner.started)
1987                .unwrap_or_default()
1988                .as_nanos(),
1989        )
1990        .unwrap_or(u64::MAX)
1991    }
1992
1993    /// Drain per-micro-function wall-clock occupancy merged across workers.
1994    ///
1995    /// Call this before `Session::finish`, which drains the shared per-stage
1996    /// call and elapsed counters this record reads.
1997    pub fn take_session_stage_concurrency(&self) -> Vec<StageConcurrencyV2> {
1998        let shards = self.inner.sorted_shards();
1999        Stage::ALL
2000            .into_iter()
2001            .filter_map(|stage| {
2002                let index = stage.index();
2003                let mut calls = 0_u64;
2004                let mut elapsed_ns = 0_u64;
2005                let mut max_worker_elapsed_ns = 0_u64;
2006                let mut worker_count = 0_u64;
2007                let mut first_start_ns = u64::MAX;
2008                let mut last_end_ns = 0_u64;
2009                let mut declared_serial_ns = 0_u64;
2010                let mut declared_serial_calls = 0_u64;
2011                let mut bytes = 0_u64;
2012                for shard in &shards {
2013                    let shard_calls = shard.calls[index].load(Ordering::Relaxed);
2014                    let shard_elapsed = shard.elapsed_ns[index].load(Ordering::Relaxed);
2015                    if shard_calls != 0 {
2016                        worker_count += 1;
2017                    }
2018                    calls = calls.saturating_add(shard_calls);
2019                    elapsed_ns = elapsed_ns.saturating_add(shard_elapsed);
2020                    max_worker_elapsed_ns = max_worker_elapsed_ns.max(shard_elapsed);
2021                    first_start_ns = first_start_ns
2022                        .min(shard.stage_first_start_ns[index].load(Ordering::Relaxed));
2023                    last_end_ns =
2024                        last_end_ns.max(shard.stage_last_end_ns[index].load(Ordering::Relaxed));
2025                    declared_serial_ns = declared_serial_ns
2026                        .saturating_add(shard.serial_ns[index].swap(0, Ordering::Relaxed));
2027                    declared_serial_calls = declared_serial_calls
2028                        .saturating_add(shard.serial_calls[index].swap(0, Ordering::Relaxed));
2029                    bytes =
2030                        bytes.saturating_add(shard.stage_bytes[index].swap(0, Ordering::Relaxed));
2031                }
2032                if calls == 0 {
2033                    return None;
2034                }
2035                let first_start_ns = if first_start_ns == u64::MAX {
2036                    0
2037                } else {
2038                    first_start_ns
2039                };
2040                let window_ns = last_end_ns.saturating_sub(first_start_ns);
2041                // A window of zero means every call fell inside one clock tick;
2042                // report the calls as serial rather than inventing concurrency.
2043                // A stage entered recursively on one thread sums its nested
2044                // time, so raw elapsed can exceed the wall the thread spent
2045                // there. Average concurrency can never exceed the number of
2046                // workers that entered the stage, so cap it there rather than
2047                // report a single-threaded recursion as parallel.
2048                let ceiling_milli = worker_count.saturating_mul(1_000);
2049                let concurrency_milli = if window_ns == 0 {
2050                    1_000
2051                } else {
2052                    u64::try_from((u128::from(elapsed_ns) * 1_000) / u128::from(window_ns))
2053                        .unwrap_or(u64::MAX)
2054                        .min(ceiling_milli)
2055                };
2056                Some(StageConcurrencyV2 {
2057                    version: crate::schema_v2::STAGE_CONCURRENCY_V2_VERSION,
2058                    metric_id: stage.metric_id(),
2059                    macro_stage_id: stage.macro_stage_id(),
2060                    calls,
2061                    elapsed_ns,
2062                    window_ns,
2063                    first_start_ns,
2064                    last_end_ns,
2065                    worker_count,
2066                    max_worker_elapsed_ns,
2067                    concurrency_milli,
2068                    declared_serial_ns,
2069                    declared_serial_calls,
2070                    bytes,
2071                })
2072            })
2073            .collect()
2074    }
2075
2076    /// Drain per-worker busy and blocked time merged in sorted shard order.
2077    pub fn take_session_worker_occupancy(&self) -> WorkerOccupancyV2 {
2078        let shards = self.inner.sorted_shards();
2079        let workers: Vec<WorkerOccupancyRowV2> = shards
2080            .iter()
2081            .map(|shard| WorkerOccupancyRowV2 {
2082                version: crate::schema_v2::WORKER_OCCUPANCY_V2_VERSION,
2083                worker_id: shard.sequence,
2084                busy_ns: shard.top_level_busy_ns.swap(0, Ordering::Relaxed),
2085                blocked_ns: shard.top_level_blocked_ns.swap(0, Ordering::Relaxed),
2086                calls: shard.top_level_calls.swap(0, Ordering::Relaxed),
2087            })
2088            .collect();
2089        let busy_ns = workers
2090            .iter()
2091            .fold(0_u64, |total, worker| total.saturating_add(worker.busy_ns));
2092        let blocked_ns = workers.iter().fold(0_u64, |total, worker| {
2093            total.saturating_add(worker.blocked_ns)
2094        });
2095        let calls = workers
2096            .iter()
2097            .fold(0_u64, |total, worker| total.saturating_add(worker.calls));
2098        let mut sorted_busy: Vec<u64> = workers.iter().map(|worker| worker.busy_ns).collect();
2099        sorted_busy.sort_unstable();
2100        let median_busy_ns = sorted_busy
2101            .get(sorted_busy.len() / 2)
2102            .copied()
2103            .unwrap_or_default();
2104        WorkerOccupancyV2 {
2105            version: crate::schema_v2::WORKER_OCCUPANCY_V2_VERSION,
2106            worker_count: u64::try_from(workers.len()).unwrap_or(u64::MAX),
2107            active_worker_count: workers.iter().filter(|worker| worker.calls != 0).count() as u64,
2108            busy_ns,
2109            blocked_ns,
2110            calls,
2111            busiest_busy_ns: sorted_busy.last().copied().unwrap_or_default(),
2112            median_busy_ns,
2113            workers,
2114        }
2115    }
2116
2117    /// Drain reuse-cache hit and miss counts merged in sorted shard order.
2118    pub fn take_session_cache_effectiveness(&self) -> Vec<CacheEffectivenessV2> {
2119        let shards = self.inner.sorted_shards();
2120        crate::CacheId::ALL
2121            .into_iter()
2122            .filter_map(|cache| {
2123                let index = cache.index();
2124                let mut hits = 0_u64;
2125                let mut misses = 0_u64;
2126                for shard in &shards {
2127                    hits = hits.saturating_add(shard.cache_hits[index].swap(0, Ordering::Relaxed));
2128                    misses =
2129                        misses.saturating_add(shard.cache_misses[index].swap(0, Ordering::Relaxed));
2130                }
2131                let total = hits.saturating_add(misses);
2132                if total == 0 {
2133                    return None;
2134                }
2135                Some(CacheEffectivenessV2 {
2136                    version: crate::schema_v2::CACHE_EFFECTIVENESS_V2_VERSION,
2137                    cache,
2138                    hits,
2139                    misses,
2140                    hit_rate_ppm: u64::try_from((u128::from(hits) * 1_000_000) / u128::from(total))
2141                        .unwrap_or(u64::MAX),
2142                })
2143            })
2144            .collect()
2145    }
2146
2147    /// Drain retry attempts by cause, merged in sorted shard order.
2148    pub fn take_session_retries(&self) -> Vec<RetryRecordV2> {
2149        let shards = self.inner.sorted_shards();
2150        crate::RetryCause::ALL
2151            .into_iter()
2152            .filter_map(|cause| {
2153                let attempts = shards.iter().fold(0_u64, |total, shard| {
2154                    total.saturating_add(shard.retries[cause.index()].swap(0, Ordering::Relaxed))
2155                });
2156                (attempts != 0).then_some(RetryRecordV2 {
2157                    version: crate::schema_v2::RETRY_RECORD_V2_VERSION,
2158                    cause,
2159                    attempts,
2160                })
2161            })
2162            .collect()
2163    }
2164
2165    /// Drain indexed counter families merged in sorted shard order.
2166    pub fn take_session_indexed_counters(&self) -> Vec<IndexedCounterRecordV2> {
2167        let shards = self.inner.sorted_shards();
2168        let dropped_out_of_range = shards.iter().fold(0_u64, |total, shard| {
2169            total.saturating_add(shard.indexed_counter_dropped.swap(0, Ordering::Relaxed))
2170        });
2171        crate::IndexedCounterId::ALL
2172            .into_iter()
2173            .filter_map(|counter| {
2174                let index = counter.index();
2175                let mut slots = [0_u64; crate::INDEXED_COUNTER_SLOTS];
2176                for shard in &shards {
2177                    for (slot, total) in slots.iter_mut().enumerate() {
2178                        *total = total.saturating_add(
2179                            shard.indexed_counters[index][slot].swap(0, Ordering::Relaxed),
2180                        );
2181                    }
2182                }
2183                if slots.iter().all(|value| *value == 0) && dropped_out_of_range == 0 {
2184                    return None;
2185                }
2186                Some(IndexedCounterRecordV2 {
2187                    version: crate::schema_v2::INDEXED_COUNTER_V2_VERSION,
2188                    counter,
2189                    slots: slots.to_vec(),
2190                    dropped_out_of_range,
2191                })
2192            })
2193            .collect()
2194    }
2195
2196    /// Discard every per-run accumulator this runtime owns.
2197    ///
2198    /// Benchmarks call this between measured rounds to drop warm-up, so
2199    /// anything left behind is reported as part of the next round. That makes
2200    /// a partial reset a wrong number presented as a right one, which is why
2201    /// this clears the per-worker shards as well as the runtime-level stores.
2202    fn reset(&self) {
2203        // Session drains clear their own storage; the legacy drain clears the
2204        // legacy mirrors. Both are needed because a session runtime keeps two.
2205        let _ = self.drain_stage_counters(false);
2206        let _ = self.drain_stage_counters(true);
2207        let _ = self.take_session_worker_occupancy();
2208        let _ = self.take_session_blocked_waits();
2209        let _ = self.take_session_stage_concurrency();
2210        let _ = self.take_session_cache_effectiveness();
2211        let _ = self.take_session_indexed_counters();
2212        let _ = self.take_session_retries();
2213        let _ = self.take_session_queue_depths();
2214        self.inner.input_bytes.store(0, Ordering::Relaxed);
2215        self.inner.input_units.store(0, Ordering::Relaxed);
2216        for index in 0..crate::MetricId::COUNT {
2217            self.inner.legacy_typed_counters[index].store(0, Ordering::Relaxed);
2218            self.inner.distribution_min[index].store(u64::MAX, Ordering::Relaxed);
2219            self.inner.distribution_max[index].store(0, Ordering::Relaxed);
2220            for bucket in 0..LATENCY_BUCKET_COUNT {
2221                self.inner.distribution_buckets[index][bucket].store(0, Ordering::Relaxed);
2222            }
2223        }
2224        self.inner
2225            .active_compile_phase
2226            .store(crate::CompilePhase::Scan as u8, Ordering::Relaxed);
2227        for surface in 0..crate::CompileSurfaceId::COUNT {
2228            self.inner.legacy_compile_surface_loads[surface].store(0, Ordering::Relaxed);
2229            for phase in 0..crate::CompilePhase::COUNT {
2230                self.inner.legacy_compile_surface_invocations[surface][phase]
2231                    .store(0, Ordering::Relaxed);
2232            }
2233        }
2234        for shard in self.inner.sorted_shards() {
2235            shard.input_bytes.store(0, Ordering::Relaxed);
2236            shard.input_units.store(0, Ordering::Relaxed);
2237            shard.legacy_input_bytes.store(0, Ordering::Relaxed);
2238            shard.legacy_input_units.store(0, Ordering::Relaxed);
2239            shard.derived_decoder_bytes.store(0, Ordering::Relaxed);
2240            shard.backend_dispatched_bytes.store(0, Ordering::Relaxed);
2241            for index in 0..crate::MetricId::COUNT {
2242                shard.counter_values[index].store(0, Ordering::Relaxed);
2243            }
2244            for index in 0..STAGE_COUNT {
2245                shard.latency_min_ns[index].store(u64::MAX, Ordering::Relaxed);
2246                shard.latency_max_ns[index].store(0, Ordering::Relaxed);
2247                shard.stage_first_start_ns[index].store(u64::MAX, Ordering::Relaxed);
2248                shard.stage_last_end_ns[index].store(0, Ordering::Relaxed);
2249                for bucket in 0..LATENCY_BUCKET_COUNT {
2250                    shard.latency_buckets[index][bucket].store(0, Ordering::Relaxed);
2251                }
2252            }
2253            for surface in 0..crate::CompileSurfaceId::COUNT {
2254                shard.compile_surface_loads[surface].store(0, Ordering::Relaxed);
2255                for phase in 0..crate::CompilePhase::COUNT {
2256                    shard.compile_surface_invocations[surface][phase].store(0, Ordering::Relaxed);
2257                }
2258            }
2259        }
2260    }
2261}
2262
2263impl Default for Runtime {
2264    fn default() -> Self {
2265        Self::new()
2266    }
2267}
2268
2269/// Thread context guard returned by [`Runtime::enter`].
2270pub struct ContextGuard {
2271    runtime: Runtime,
2272    not_send: PhantomData<Rc<()>>,
2273}
2274
2275impl Drop for ContextGuard {
2276    fn drop(&mut self) {
2277        CURRENT.with(|stack| {
2278            let mut stack = stack.borrow_mut();
2279            if stack
2280                .last()
2281                .is_some_and(|runtime| Arc::ptr_eq(&runtime.inner, &self.runtime.inner))
2282            {
2283                stack.pop();
2284            } else if let Some(position) = stack
2285                .iter()
2286                .rposition(|runtime| Arc::ptr_eq(&runtime.inner, &self.runtime.inner))
2287            {
2288                stack.remove(position);
2289            }
2290        });
2291        ACTIVE_CONTEXTS.fetch_sub(1, Ordering::Relaxed);
2292    }
2293}
2294
2295struct AsyncParentGuard {
2296    runtime_key: usize,
2297    span_id: u64,
2298    stack_slot: usize,
2299    not_send: PhantomData<Rc<()>>,
2300}
2301
2302impl Drop for AsyncParentGuard {
2303    fn drop(&mut self) {
2304        ASYNC_PARENT_SPANS.with(|stack| {
2305            let mut stack = stack.borrow_mut();
2306            if stack[self.stack_slot].is_some_and(|active| {
2307                active.runtime_key == self.runtime_key && active.span_id == self.span_id
2308            }) {
2309                stack[self.stack_slot] = None;
2310            }
2311        });
2312    }
2313}
2314
2315struct WorkOriginGuard {
2316    previous: WorkOrigin,
2317}
2318
2319impl WorkOriginGuard {
2320    fn enter(origin: WorkOrigin) -> Self {
2321        Self {
2322            previous: WORK_ORIGIN.with(|slot| slot.replace(origin)),
2323        }
2324    }
2325}
2326
2327impl Drop for WorkOriginGuard {
2328    fn drop(&mut self) {
2329        WORK_ORIGIN.with(|slot| slot.set(self.previous));
2330    }
2331}
2332
2333struct TaskGuard {
2334    previous: u64,
2335}
2336
2337impl TaskGuard {
2338    fn enter(task_id: u64) -> Self {
2339        Self {
2340            previous: TASK_ID.with(|slot| slot.replace(task_id)),
2341        }
2342    }
2343}
2344
2345impl Drop for TaskGuard {
2346    fn drop(&mut self) {
2347        TASK_ID.with(|slot| slot.set(self.previous));
2348    }
2349}
2350
2351struct LegacyRuntime {
2352    runtime: Runtime,
2353    enabled: bool,
2354}
2355
2356impl LegacyRuntime {
2357    fn new() -> Self {
2358        Self {
2359            runtime: Runtime::legacy(),
2360            enabled: false,
2361        }
2362    }
2363}
2364
2365/// Optional attribution for work performed inside a derived input.
2366#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
2367#[repr(u8)]
2368pub enum Attribution {
2369    #[default]
2370    Root = 0,
2371    Decoded = 1,
2372}
2373
2374impl From<Attribution> for WorkOrigin {
2375    fn from(attribution: Attribution) -> Self {
2376        match attribution {
2377            Attribution::Root => Self::Root,
2378            Attribution::Decoded => Self::Decoded,
2379        }
2380    }
2381}
2382
2383thread_local! {
2384    static CURRENT: RefCell<Vec<Runtime>> = const { RefCell::new(Vec::new()) };
2385    static WORK_ORIGIN: Cell<WorkOrigin> = const { Cell::new(WorkOrigin::Root) };
2386    static TASK_ID: Cell<u64> = const { Cell::new(0) };
2387    static SPAN_DEPTH: Cell<u32> = const { Cell::new(0) };
2388    static LEGACY: RefCell<LegacyRuntime> = RefCell::new(LegacyRuntime::new());
2389    static ACTIVE_SPANS: RefCell<[Option<ActiveSpan>; MAX_NESTED_SPANS]> =
2390        const { RefCell::new([None; MAX_NESTED_SPANS]) };
2391    static ASYNC_PARENT_SPANS: RefCell<[Option<ActiveSpan>; MAX_NESTED_SPANS]> =
2392        const { RefCell::new([None; MAX_NESTED_SPANS]) };
2393    static NUMERIC_THREAD_ID: u64 = NEXT_THREAD_ID.fetch_add(1, Ordering::Relaxed);
2394    static THREAD_SHARDS: RefCell<Vec<ThreadShardAssignment>> = const { RefCell::new(Vec::new()) };
2395}
2396
2397fn numeric_thread_id() -> u64 {
2398    NUMERIC_THREAD_ID.with(|thread_id| *thread_id)
2399}
2400
2401/// Return a clone of the runtime current on this thread.
2402pub fn current_runtime() -> Option<Runtime> {
2403    if ACTIVE_CONTEXTS.load(Ordering::Relaxed) == 0 {
2404        return None;
2405    }
2406    CURRENT
2407        .with(|stack| stack.borrow().last().cloned())
2408        .or_else(|| {
2409            LEGACY.with(|legacy| {
2410                let legacy = legacy.borrow();
2411                legacy.enabled.then(|| legacy.runtime.clone())
2412            })
2413        })
2414}
2415
2416pub(crate) fn runtime_for_drain() -> Runtime {
2417    CURRENT
2418        .with(|stack| stack.borrow().last().cloned())
2419        .unwrap_or_else(|| LEGACY.with(|legacy| legacy.borrow().runtime.clone()))
2420}
2421
2422/// Replace this thread's attribution and return its previous value.
2423///
2424/// Legacy projection of [`set_work_origin`]: the returned value is `Decoded`
2425/// whenever the previous origin was any attributed (non-root) work.
2426pub fn set_attribution(attribution: Attribution) -> Attribution {
2427    let previous = set_work_origin(attribution.into());
2428    if previous.is_attributed_work() {
2429        Attribution::Decoded
2430    } else {
2431        Attribution::Root
2432    }
2433}
2434
2435/// Replace this thread's causal work origin and return its previous value.
2436pub fn set_work_origin(origin: WorkOrigin) -> WorkOrigin {
2437    WORK_ORIGIN.with(|slot| slot.replace(origin))
2438}
2439
2440/// Current thread's causal work origin.
2441pub fn current_work_origin() -> WorkOrigin {
2442    WORK_ORIGIN.with(|slot| slot.get())
2443}
2444
2445/// Replace this thread's caller-assigned task identity and return the previous.
2446///
2447/// Zero clears the identity. The value is a caller-managed task or name index;
2448/// [`instrument_future`] propagates it across polls and worker threads.
2449pub fn set_task_id(task_id: u64) -> u64 {
2450    TASK_ID.with(|slot| slot.replace(task_id))
2451}
2452
2453/// Current thread's caller-assigned task identity, or zero when unset.
2454pub fn current_task_id() -> u64 {
2455    TASK_ID.with(|slot| slot.get())
2456}
2457
2458fn evidence_or_unavailable(value: u64) -> Evidence<u64> {
2459    if value == 0 {
2460        Evidence::unavailable(EvidenceGap::Unavailable)
2461    } else {
2462        Evidence::recorded(value)
2463    }
2464}
2465
2466/// Capture a portable token naming the current runtime's causal parent.
2467///
2468/// Returns `None` when no runtime is current on this thread.
2469pub fn current_causal_parent() -> Option<CausalParent> {
2470    current_runtime().map(|runtime| runtime.causal_parent())
2471}
2472
2473/// Return whether fixed-stage profiling is active on the calling thread.
2474#[inline]
2475pub fn enabled() -> bool {
2476    if ACTIVE_CONTEXTS.load(Ordering::Relaxed) == 0 {
2477        return false;
2478    }
2479    CURRENT.with(|stack| !stack.borrow().is_empty())
2480        || LEGACY.with(|legacy| legacy.borrow().enabled)
2481}
2482
2483/// Enable or disable the calling thread's standalone profiling runtime.
2484///
2485/// Prefer [`crate::Session::start`] for operator runs because it also captures
2486/// identity, resources, and state transitions. This switch remains available
2487/// to libraries and microbenchmarks that only need stage counters.
2488pub fn set_enabled(enabled: bool) {
2489    LEGACY.with(|legacy| {
2490        let mut legacy = legacy.borrow_mut();
2491        if legacy.enabled == enabled {
2492            return;
2493        }
2494        legacy.enabled = enabled;
2495        if enabled {
2496            ACTIVE_CONTEXTS.fetch_add(1, Ordering::Relaxed);
2497        } else {
2498            ACTIVE_CONTEXTS.fetch_sub(1, Ordering::Relaxed);
2499        }
2500    });
2501}
2502
2503struct AsyncSpan {
2504    runtime: Option<Runtime>,
2505    shard: Option<Arc<WorkerShard>>,
2506    stage: Stage,
2507    started: Option<Instant>,
2508    trace: Option<SpanTrace>,
2509}
2510
2511impl Drop for AsyncSpan {
2512    fn drop(&mut self) {
2513        let (Some(runtime), Some(started)) = (&self.runtime, self.started) else {
2514            return;
2515        };
2516        let elapsed_ns = u64::try_from(started.elapsed().as_nanos()).unwrap_or(u64::MAX);
2517        runtime.record(
2518            self.shard.as_deref(),
2519            self.stage,
2520            SpanOutcome {
2521                start_offset_ns: runtime.offset_ns(started),
2522                elapsed_ns,
2523                self_ns: 0,
2524                blocked: false,
2525                serial: false,
2526                // An async span can be polled on any thread, so it never
2527                // owns a worker's outermost interval.
2528                outermost: false,
2529            },
2530        );
2531        if let Some(trace) = self.trace {
2532            runtime.finish_span(trace, elapsed_ns);
2533        }
2534    }
2535}
2536
2537fn instrument_impl<F>(
2538    stage: Stage,
2539    future: F,
2540    parent_override: Option<CausalParent>,
2541) -> impl Future<Output = F::Output>
2542where
2543    F: Future,
2544{
2545    let runtime = current_runtime();
2546    let shard = runtime.as_ref().and_then(Runtime::worker_shard);
2547    let worker_id = shard.as_ref().map_or(0, |shard| shard.sequence);
2548    let started = runtime.as_ref().map(|_| Instant::now());
2549    let parent_span_id = match (runtime.as_ref(), parent_override) {
2550        (Some(runtime), Some(parent)) if runtime.context_id() == parent.context_id() => {
2551            parent.span_id()
2552        }
2553        (Some(_), Some(_)) => 0,
2554        (Some(runtime), None) => runtime.current_parent_span_id(),
2555        (None, _) => 0,
2556    };
2557    let trace = runtime
2558        .as_ref()
2559        .zip(started)
2560        .and_then(|(runtime, started)| {
2561            runtime.begin_async_span(stage, started, parent_span_id, worker_id)
2562        });
2563    let span_id = trace.map(|trace| trace.span_id);
2564    let origin = current_work_origin();
2565    let task_id = current_task_id();
2566    let poll_runtime = runtime.clone();
2567
2568    async move {
2569        let _span = AsyncSpan {
2570            runtime,
2571            shard,
2572            stage,
2573            started,
2574            trace,
2575        };
2576        let mut future = std::pin::pin!(future);
2577        poll_fn(|context| {
2578            let _context_guard = poll_runtime.as_ref().map(Runtime::enter);
2579            let _parent_guard = poll_runtime
2580                .as_ref()
2581                .zip(span_id)
2582                .and_then(|(runtime, span_id)| runtime.enter_async_parent(span_id));
2583            let _origin_guard = WorkOriginGuard::enter(origin);
2584            let _task_guard = TaskGuard::enter(task_id);
2585            if span_id.is_some() {
2586                crate::allocation::stage_context_push(stage);
2587            }
2588            let result = future.as_mut().poll(context);
2589            if span_id.is_some() {
2590                crate::allocation::stage_context_pop();
2591            }
2592            result
2593        })
2594        .await
2595    }
2596}
2597
2598/// Propagate the current runtime and causal parent while polling one future.
2599///
2600/// The returned future records one wall-time span from wrapper construction
2601/// through completion or cancellation. Runtime and parent guards exist only
2602/// during each poll, so the returned future remains `Send` when `F` is `Send`.
2603pub fn instrument_future<F>(stage: Stage, future: F) -> impl Future<Output = F::Output>
2604where
2605    F: Future,
2606{
2607    instrument_impl(stage, future, None)
2608}
2609
2610/// Propagate the current runtime with an explicit portable causal parent.
2611///
2612/// Use this when the future crosses a spawn boundary where thread-local
2613/// parentage cannot reach. A token captured from another runtime records the
2614/// future's span as a root of the current runtime instead.
2615pub fn instrument_future_with_parent<F>(
2616    parent: CausalParent,
2617    stage: Stage,
2618    future: F,
2619) -> impl Future<Output = F::Output>
2620where
2621    F: Future,
2622{
2623    instrument_impl(stage, future, Some(parent))
2624}
2625
2626/// Allocation-free stage guard. It contains no start timestamp while disabled.
2627#[must_use]
2628pub struct Span {
2629    runtime: Option<Runtime>,
2630    shard: Option<Arc<WorkerShard>>,
2631    stage: Stage,
2632    started: Option<Instant>,
2633    trace: Option<SpanTrace>,
2634    stack_slot: Option<usize>,
2635    span_id: u64,
2636    blocked: bool,
2637    serial: bool,
2638}
2639
2640impl Span {
2641    /// Whether this span reads and will record a clock measurement.
2642    pub fn is_recording(&self) -> bool {
2643        self.started.is_some()
2644    }
2645}
2646
2647fn span_impl(
2648    stage: Stage,
2649    parent_override: Option<CausalParent>,
2650    blocked: bool,
2651    serial: bool,
2652) -> Span {
2653    if ACTIVE_CONTEXTS.load(Ordering::Relaxed) == 0 {
2654        return Span {
2655            runtime: None,
2656            shard: None,
2657            stage,
2658            started: None,
2659            trace: None,
2660            stack_slot: None,
2661            span_id: 0,
2662            blocked: false,
2663            serial: false,
2664        };
2665    }
2666    let runtime = current_runtime();
2667    let shard = runtime.as_ref().and_then(Runtime::worker_shard);
2668    let worker_id = shard.as_ref().map_or(0, |shard| shard.sequence);
2669    let started = runtime.as_ref().map(|_| Instant::now());
2670    if started.is_some() {
2671        SPAN_DEPTH.with(|depth| depth.set(depth.get() + 1));
2672    }
2673    let (trace, stack_slot, span_id) = match (runtime.as_ref(), started) {
2674        (Some(runtime), Some(started)) => {
2675            let (parent_slot, current_parent_id) = runtime.current_parent();
2676            let parent_span_id = match parent_override {
2677                Some(parent) if runtime.context_id() == parent.context_id() => parent.span_id(),
2678                _ => current_parent_id,
2679            };
2680            runtime.begin_span_with(stage, started, parent_span_id, parent_slot, worker_id)
2681        }
2682        _ => (None, None, 0),
2683    };
2684    if trace.is_some() {
2685        crate::allocation::stage_context_push(stage);
2686    }
2687    Span {
2688        runtime,
2689        shard,
2690        stage,
2691        started,
2692        trace,
2693        stack_slot,
2694        span_id,
2695        blocked,
2696        serial,
2697    }
2698}
2699
2700/// Start one fixed-stage measurement.
2701#[inline]
2702pub fn span(stage: Stage) -> Span {
2703    span_impl(stage, None, false, false)
2704}
2705
2706/// Start one fixed-stage measurement with an explicit portable causal parent.
2707///
2708/// The token replaces thread-local parent lookup, so the caller can carry
2709/// parentage across crate, thread, or spawn boundaries. A token captured from
2710/// another runtime records this span as a root of the current runtime.
2711pub fn span_with_parent(parent: CausalParent, stage: Stage) -> Span {
2712    span_impl(stage, Some(parent), false, false)
2713}
2714
2715/// Declare that this region runs with the worker pool idle.
2716///
2717/// Use it for barriers such as enumeration, plan compilation, or final merge,
2718/// where no other worker can make progress. The guard measures wall time like
2719/// [`span`] and additionally accumulates declared-serial time, which the
2720/// profiler reports as the Amdahl floor on any speedup from more threads.
2721/// The profiler also derives serial phases from observed concurrency, so a
2722/// region left undeclared is still detected; declaring it makes the report
2723/// state the intent rather than infer it.
2724pub fn serial_span(stage: Stage) -> Span {
2725    span_impl(stage, None, false, true)
2726}
2727
2728/// Record one blocked wait interval separately from runnable execution.
2729///
2730/// The guard measures wall time like [`span`] but additionally accumulates
2731/// per-stage blocked time drained by `Runtime::take_session_blocked_waits`,
2732/// and it never counts as attributed (decoded or derived) execution. Reuse
2733/// wait stages such as [`Stage::SourceQueueWait`] and [`Stage::ScannerQueueWait`].
2734pub fn blocked(stage: Stage) -> Span {
2735    span_impl(stage, None, true, false)
2736}
2737
2738/// Time a region whose measurement drives a decision, profiled or not.
2739///
2740/// [`span`] deliberately measures nothing while profiling is off, which is
2741/// right for reporting and wrong for a measurement the product acts on. A
2742/// timer whose value picks a backend must produce the same value whether or
2743/// not an operator passed `--profile`, or the flag would change routing.
2744///
2745/// This guard therefore reads the clock unconditionally, returns the elapsed
2746/// duration to the caller, and additionally records it as a [`span`] would
2747/// when a runtime is current. The name says "decision" so a reader knows at
2748/// the call site that it costs a clock read even when profiling is off. On a
2749/// hot path where nothing acts on the value, use [`span`].
2750#[must_use = "a decision timer only measures when finished"]
2751pub struct DecisionTimer {
2752    stage: Stage,
2753    started: Instant,
2754}
2755
2756/// Start a decision-driving measurement of one micro-function.
2757pub fn decision_timer(stage: Stage) -> DecisionTimer {
2758    DecisionTimer {
2759        stage,
2760        started: Instant::now(),
2761    }
2762}
2763
2764impl DecisionTimer {
2765    /// Stop the timer, record it when profiling is on, and return the elapsed time.
2766    pub fn finish(self) -> std::time::Duration {
2767        let elapsed = self.started.elapsed();
2768        if ACTIVE_CONTEXTS.load(Ordering::Relaxed) != 0 {
2769            if let Some(runtime) = current_runtime() {
2770                let shard = runtime.worker_shard();
2771                runtime.record(
2772                    shard.as_deref(),
2773                    self.stage,
2774                    SpanOutcome {
2775                        start_offset_ns: runtime.offset_ns(self.started),
2776                        elapsed_ns: u64::try_from(elapsed.as_nanos()).unwrap_or(u64::MAX),
2777                        self_ns: 0,
2778                        blocked: false,
2779                        serial: false,
2780                        // The caller owns the enclosing span, if any; a decision
2781                        // timer never claims a worker's outermost interval.
2782                        outermost: false,
2783                    },
2784                );
2785            }
2786        }
2787        elapsed
2788    }
2789}
2790
2791impl Drop for Span {
2792    #[inline]
2793    fn drop(&mut self) {
2794        let (Some(runtime), Some(started)) = (&self.runtime, self.started) else {
2795            return;
2796        };
2797        let outermost = SPAN_DEPTH.with(|depth| {
2798            let next = depth.get().saturating_sub(1);
2799            depth.set(next);
2800            next == 0
2801        });
2802        let elapsed_ns = u64::try_from(started.elapsed().as_nanos()).unwrap_or(u64::MAX);
2803        let self_ns = runtime.pop_active_span(self.stack_slot, self.span_id, elapsed_ns);
2804        runtime.record(
2805            self.shard.as_deref(),
2806            self.stage,
2807            SpanOutcome {
2808                start_offset_ns: runtime.offset_ns(started),
2809                elapsed_ns,
2810                self_ns,
2811                blocked: self.blocked,
2812                serial: self.serial,
2813                outermost,
2814            },
2815        );
2816        if let Some(trace) = self.trace {
2817            runtime.finish_span(trace, elapsed_ns);
2818            crate::allocation::stage_context_pop();
2819        }
2820    }
2821}
2822
2823/// Time a sub-stage region into a [`crate::CounterId`] instead of a stage.
2824///
2825/// Some measurements sit strictly inside a stage leaf. Recording them as
2826/// spans would add their time to that leaf's inclusive total a second time,
2827/// so they belong in a counter. This guard is [`span`] with a counter sink:
2828/// same enabled gate, same clock, no allocation on drop.
2829#[must_use = "a counter span measures nothing until it is dropped"]
2830pub struct CounterSpan {
2831    counter: crate::CounterId,
2832    started: Option<Instant>,
2833}
2834
2835/// Start a sub-stage measurement that accumulates into one counter.
2836#[inline]
2837pub fn counter_span(counter: crate::CounterId) -> CounterSpan {
2838    CounterSpan {
2839        counter,
2840        started: (ACTIVE_CONTEXTS.load(Ordering::Relaxed) != 0).then(Instant::now),
2841    }
2842}
2843
2844impl Drop for CounterSpan {
2845    #[inline]
2846    fn drop(&mut self) {
2847        let Some(started) = self.started else {
2848            return;
2849        };
2850        add_counter(
2851            self.counter,
2852            u64::try_from(started.elapsed().as_nanos()).unwrap_or(u64::MAX),
2853        );
2854    }
2855}
2856
2857/// Count one retry attempt, whether or not the retry eventually succeeded.
2858///
2859/// Count every attempt, not every operation: a path that retries a thousand
2860/// times must read as a thousand. The profiler reports retries as a finding
2861/// because a retry that fires means a failure the product did not design out.
2862#[inline]
2863pub fn record_retry(cause: crate::RetryCause) {
2864    if let Some(runtime) = current_runtime() {
2865        runtime.record_retry(cause);
2866    }
2867}
2868
2869/// Add to one slot of an indexed counter family.
2870///
2871/// A slot at or beyond [`crate::INDEXED_COUNTER_SLOTS`] is counted as dropped
2872/// on the drained record rather than folded into another slot.
2873#[inline]
2874pub fn add_indexed_counter(counter: crate::IndexedCounterId, slot: u16, delta: u64) {
2875    if let Some(runtime) = current_runtime() {
2876        runtime.add_indexed_counter(counter, slot, delta);
2877    }
2878}
2879
2880/// Attribute bytes to one micro-function so its throughput can be reported.
2881///
2882/// Record the bytes the named stage actually moved or examined, not the run's
2883/// input size. A stage that sees each byte twice reports twice the bytes, and
2884/// that is the honest number for its own throughput.
2885#[inline]
2886pub fn add_stage_bytes(stage: crate::Stage, bytes: u64) {
2887    if let Some(runtime) = current_runtime() {
2888        runtime.add_stage_bytes(stage, bytes);
2889    }
2890}
2891
2892/// Count one consultation of a reuse cache that was served from the cache.
2893#[inline]
2894pub fn record_cache_hit(cache: crate::CacheId) {
2895    if let Some(runtime) = current_runtime() {
2896        runtime.record_cache_outcome(cache, true);
2897    }
2898}
2899
2900/// Count one consultation of a reuse cache that had to recompute or refetch.
2901#[inline]
2902pub fn record_cache_miss(cache: crate::CacheId) {
2903    if let Some(runtime) = current_runtime() {
2904        runtime.record_cache_outcome(cache, false);
2905    }
2906}
2907
2908/// Set the current compile phase (Install, Update, Scan, Developer) on the active runtime.
2909#[inline]
2910pub fn set_compile_phase(phase: crate::CompilePhase) {
2911    PROCESS_ACTIVE_COMPILE_PHASE.store(phase as u8, Ordering::Relaxed);
2912    if let Some(runtime) = current_runtime() {
2913        runtime.set_compile_phase(phase);
2914    }
2915}
2916
2917/// Retrieve the active compile phase from the active runtime (defaults to Scan).
2918#[inline]
2919pub fn active_compile_phase() -> crate::CompilePhase {
2920    if let Some(runtime) = current_runtime() {
2921        runtime.active_compile_phase()
2922    } else {
2923        match PROCESS_ACTIVE_COMPILE_PHASE.load(Ordering::Relaxed) {
2924            0 => crate::CompilePhase::Install,
2925            1 => crate::CompilePhase::Update,
2926            2 => crate::CompilePhase::Scan,
2927            3 => crate::CompilePhase::Developer,
2928            _ => crate::CompilePhase::Scan,
2929        }
2930    }
2931}
2932
2933/// Record one invocation of a compiler surface in the active compile phase.
2934#[inline]
2935pub fn record_compile_surface_invocation(surface: crate::CompileSurfaceId) {
2936    if let Some(runtime) = current_runtime() {
2937        runtime.record_compile_surface_invocation(surface);
2938    }
2939}
2940
2941/// Record one invocation of a compiler surface in an explicit compile phase.
2942#[inline]
2943pub fn record_compile_surface_invocation_with_phase(
2944    surface: crate::CompileSurfaceId,
2945    phase: crate::CompilePhase,
2946) {
2947    if let Some(runtime) = current_runtime() {
2948        runtime.record_compile_surface_invocation_with_phase(surface, phase);
2949    }
2950}
2951
2952/// Record one load of a prepared/installed artifact for a compile surface class.
2953#[inline]
2954pub fn record_compile_surface_load(surface: crate::CompileSurfaceId) {
2955    if let Some(runtime) = current_runtime() {
2956        runtime.record_compile_surface_load(surface);
2957    }
2958}
2959
2960/// Sum total runtime compilations across all surfaces in Scan and Developer phases.
2961#[inline]
2962pub fn total_runtime_compiles() -> u64 {
2963    if let Some(runtime) = current_runtime() {
2964        runtime.total_runtime_compiles()
2965    } else {
2966        0
2967    }
2968}
2969
2970/// Derive compile surface reports for all 13 compile surface classes.
2971#[inline]
2972pub fn compile_surface_reports() -> Vec<crate::schema_v2::CompileSurfaceRecordV2> {
2973    if let Some(runtime) = current_runtime() {
2974        runtime.compile_surface_reports()
2975    } else {
2976        crate::CompileSurfaceId::ALL
2977            .iter()
2978            .map(|&surface| crate::schema_v2::CompileSurfaceRecordV2 {
2979                version: crate::schema_v2::COMPILE_SURFACE_RECORD_V2_VERSION,
2980                surface,
2981                name: surface.as_str().to_string(),
2982                runtime_compiles: 0,
2983                loads: 0,
2984                install_compiles: 0,
2985                update_compiles: 0,
2986                developer_compiles: 0,
2987            })
2988            .collect()
2989    }
2990}
2991
2992/// Read total cache hits recorded in the active profile for `cache`.
2993#[inline]
2994#[must_use]
2995pub fn cache_hits(cache: crate::CacheId) -> u64 {
2996    current_runtime().map_or(0, |runtime| runtime.cache_hits(cache))
2997}
2998
2999/// Read total cache misses recorded in the active profile for `cache`.
3000#[inline]
3001#[must_use]
3002pub fn cache_misses(cache: crate::CacheId) -> u64 {
3003    current_runtime().map_or(0, |runtime| runtime.cache_misses(cache))
3004}
3005
3006/// Add source bytes processed by the current profile.
3007#[inline]
3008pub fn add_input_bytes(bytes: u64) {
3009    if let Some(runtime) = current_runtime() {
3010        runtime.add_input_bytes(bytes);
3011    }
3012}
3013
3014/// Add source units such as files, objects, responses, or chunks.
3015#[inline]
3016pub fn add_input_units(units: u64) {
3017    if let Some(runtime) = current_runtime() {
3018        runtime.add_input_units(units);
3019    }
3020}
3021
3022/// Record one expensive detail event under a deterministic bounded sampling policy.
3023#[inline]
3024pub fn record_sampled_event(event: crate::EventId, value: u64, policy: SamplingPolicy) -> bool {
3025    current_runtime().is_some_and(|runtime| runtime.record_sampled_event(event, value, policy))
3026}
3027/// Add bytes produced by accepted decode-through work in the current profile.
3028#[inline]
3029pub fn add_derived_decoder_bytes(bytes: u64) {
3030    if let Some(runtime) = current_runtime() {
3031        runtime.add_derived_decoder_bytes(bytes);
3032    }
3033}
3034
3035/// Add bytes submitted once to the completed backend route in the current profile.
3036#[inline]
3037pub fn add_backend_dispatched_bytes(bytes: u64) {
3038    if let Some(runtime) = current_runtime() {
3039        runtime.add_backend_dispatched_bytes(bytes);
3040    }
3041}
3042
3043/// Increment one typed monotonic counter in the current profiling runtime.
3044#[inline]
3045pub fn add_counter(counter: crate::CounterId, delta: u64) {
3046    if let Some(runtime) = current_runtime() {
3047        runtime.add_counter(counter, delta);
3048    }
3049}
3050
3051/// Replace one typed latest-value gauge in the current profiling runtime.
3052#[inline]
3053pub fn set_gauge(gauge: crate::GaugeId, value: u64) {
3054    if let Some(runtime) = current_runtime() {
3055        runtime.set_gauge(gauge, value);
3056    }
3057}
3058
3059/// Record one typed instantaneous event with a numeric payload.
3060#[inline]
3061pub fn record_event(event: crate::EventId, value: u64) {
3062    if let Some(runtime) = current_runtime() {
3063        runtime.record_event(event, value);
3064    }
3065}
3066
3067/// Record one producer enqueue for later matching by [`record_queue_dequeue`].
3068///
3069/// The `(queue, sequence)` pair must be unique per in-flight item. Retention
3070/// is bounded by [`MAX_QUEUE_LINKS`]; loss is counted explicitly.
3071pub fn record_queue_enqueue(queue: crate::QueueId, sequence: u64) {
3072    if let Some(runtime) = current_runtime() {
3073        runtime.record_queue_enqueue(queue, sequence);
3074    }
3075}
3076
3077/// Record the consumer dequeue matching one earlier [`record_queue_enqueue`].
3078///
3079/// A dequeue with no recorded pending enqueue increments the unmatched count.
3080pub fn record_queue_dequeue(queue: crate::QueueId, sequence: u64) {
3081    if let Some(runtime) = current_runtime() {
3082        runtime.record_queue_dequeue(queue, sequence);
3083    }
3084}
3085
3086/// Increment one queue's depth gauge and refresh its high-water mark.
3087pub fn record_queue_depth_enqueue(queue: crate::QueueId) {
3088    if let Some(runtime) = current_runtime() {
3089        runtime.queue_depth_enqueue(queue);
3090    }
3091}
3092
3093/// Decrement one queue's depth gauge, saturating at zero.
3094pub fn record_queue_depth_dequeue(queue: crate::QueueId) {
3095    if let Some(runtime) = current_runtime() {
3096        runtime.queue_depth_dequeue(queue);
3097    }
3098}
3099
3100/// Replace one queue's depth gauge and refresh its high-water mark.
3101pub fn set_queue_depth(queue: crate::QueueId, depth: u64) {
3102    if let Some(runtime) = current_runtime() {
3103        runtime.set_queue_depth(queue, depth);
3104    }
3105}
3106
3107/// Record one observed value into a metric's bounded logarithmic distribution.
3108#[inline]
3109pub fn record_distribution(metric: crate::MetricId, value: u64) {
3110    if let Some(runtime) = current_runtime() {
3111        runtime.record_distribution(metric, value);
3112    }
3113}
3114
3115/// Record one filesystem open latency inside a [`Stage::SourceWalk`] or
3116/// [`Stage::SourceRead`] instrumented path.
3117#[inline]
3118pub fn record_fs_open_latency_ns(elapsed_ns: u64) {
3119    record_distribution(crate::MetricId::FsOpenLatencyNs, elapsed_ns);
3120}
3121
3122/// Record one filesystem read latency inside a [`Stage::SourceRead`]
3123/// instrumented path.
3124#[inline]
3125pub fn record_fs_read_latency_ns(elapsed_ns: u64) {
3126    record_distribution(crate::MetricId::FsReadLatencyNs, elapsed_ns);
3127}
3128
3129/// Record one filesystem metadata (stat/readdir) latency inside a
3130/// [`Stage::SourceWalk`] instrumented path.
3131#[inline]
3132pub fn record_fs_metadata_latency_ns(elapsed_ns: u64) {
3133    record_distribution(crate::MetricId::FsMetadataLatencyNs, elapsed_ns);
3134}
3135
3136/// Record one network request latency observed by a caller.
3137#[inline]
3138pub fn record_network_latency_ns(elapsed_ns: u64) {
3139    record_distribution(crate::MetricId::NetworkLatencyNs, elapsed_ns);
3140}
3141
3142/// Add network bytes a caller read and wrote; process-level counters are not
3143/// visible to the profiler on every host, so callers report their own IO.
3144#[inline]
3145pub fn record_network_bytes(read_bytes: u64, written_bytes: u64) {
3146    if read_bytes > 0 {
3147        add_counter(crate::CounterId::NetworkBytesRead, read_bytes);
3148    }
3149    if written_bytes > 0 {
3150        add_counter(crate::CounterId::NetworkBytesWritten, written_bytes);
3151    }
3152}
3153
3154/// Count one completed network request.
3155#[inline]
3156pub fn record_network_request() {
3157    add_counter(crate::CounterId::NetworkRequests, 1);
3158}
3159
3160/// Record one explicitly observed page-cache state for IO work.
3161///
3162/// The observation becomes one [`crate::AnnotationId::IoCacheState`] timeline
3163/// record and increments the matching observation counter. The profiler
3164/// never infers cache state from latency; only caller knowledge is recorded.
3165#[inline]
3166pub fn record_io_cache_state(state: crate::IoCacheStateV2) {
3167    record_annotation(crate::AnnotationId::IoCacheState, state.as_value());
3168    let counter = match state {
3169        crate::IoCacheStateV2::Cold => crate::CounterId::PageCacheColdObservations,
3170        crate::IoCacheStateV2::Warm => crate::CounterId::PageCacheWarmObservations,
3171        crate::IoCacheStateV2::Direct => crate::CounterId::PageCacheDirectObservations,
3172    };
3173    add_counter(counter, 1);
3174}
3175
3176/// Record the current retained-buffer level in bytes; the runtime keeps the
3177/// running high water alongside the latest value.
3178#[inline]
3179pub fn record_retained_buffer_bytes(bytes: u64) {
3180    if let Some(runtime) = current_runtime() {
3181        runtime.record_retained_buffer_bytes(bytes);
3182    }
3183}
3184
3185/// Drain typed counters from the current session or standalone runtime.
3186///
3187/// Under a [`Session`] this equals `Runtime::take_session_typed_metrics`;
3188/// under the standalone `set_enabled` runtime it drains the legacy store.
3189pub fn take_typed_metrics() -> Vec<TypedMetricRecordV2> {
3190    runtime_for_drain().take_legacy_typed_metrics()
3191}
3192
3193/// Drain caller-recorded value distributions from the current runtime.
3194///
3195/// Works under both a [`Session`] and the standalone `set_enabled` runtime.
3196pub fn take_metric_distributions() -> Vec<MetricDistributionV2> {
3197    runtime_for_drain().take_metric_distributions()
3198}
3199
3200/// Record one typed numeric annotation on the current run timeline.
3201#[inline]
3202pub fn record_annotation(annotation: crate::AnnotationId, value: u64) {
3203    if let Some(runtime) = current_runtime() {
3204        runtime.record_annotation(annotation, value);
3205    }
3206}
3207
3208/// Record the requested, selected, and completed route for one completed batch.
3209///
3210/// A recovered batch records the failed selected backend in
3211/// `recovered_from_backend` and the replay backend in `completed_backend`.
3212pub fn record_batch_route(
3213    workload_key_digest: &str,
3214    requested_backend: &str,
3215    selected_backend: &str,
3216    completed_backend: &str,
3217    recovered_from_backend: Option<&str>,
3218) {
3219    if let Some(runtime) = current_runtime() {
3220        runtime.record_batch_route(
3221            workload_key_digest,
3222            requested_backend,
3223            selected_backend,
3224            completed_backend,
3225            recovered_from_backend,
3226        );
3227    }
3228}
3229
3230/// Atomically read and clear aggregate input bytes and units.
3231pub fn take_input_totals() -> (u64, u64) {
3232    runtime_for_drain().take_input_totals()
3233}
3234
3235/// Discard fixed-stage counters and input totals in the current runtime.
3236pub fn reset() {
3237    runtime_for_drain().reset();
3238}