1use crate::schema::Stage;
2use crate::schema_v2::{
3 AnnotationV2, BatchRouteV2, BlockedWaitRecordV2, CacheEffectivenessV2, DistributionBucketV2,
4 Evidence, EvidenceGap, IndexedCounterRecordV2, LatencyBucketV2, LatencyDistributionV2,
5 MetricDistributionV2, PointEventV2, QueueDepthV2, QueueLinkV2, RetryRecordV2, SpanRecordV2,
6 StageConcurrencyV2, TypedMetricRecordV2, WorkOrigin, WorkerImbalanceV2, WorkerLoadV2,
7 WorkerOccupancyRowV2, WorkerOccupancyV2,
8};
9use std::cell::{Cell, RefCell};
10use std::collections::HashMap;
11use std::future::{poll_fn, Future};
12use std::marker::PhantomData;
13use std::rc::Rc;
14use std::sync::atomic::{AtomicU64, AtomicU8, AtomicUsize, Ordering};
15use std::sync::{Arc, Mutex, Weak};
16use std::time::Instant;
17
18pub(crate) const STAGE_COUNT: usize = Stage::ALL.len();
19
20pub const MAX_RECORDED_SPANS: usize = 65_536;
22const MAX_NESTED_SPANS: usize = 64;
23const LATENCY_BUCKET_COUNT: usize = 65;
24pub const MAX_POINT_EVENTS: usize = 16_384;
25pub const MAX_ANNOTATIONS: usize = 16_384;
26pub const MAX_QUEUE_LINKS: usize = 16_384;
28pub const MAX_BATCH_ROUTES: usize = 16_384;
30
31#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
33pub struct QueueLinkLossCounts {
34 pub dropped_enqueues: u64,
36 pub dropped_links: u64,
38 pub unmatched_dequeues: u64,
40 pub unconsumed_enqueues: u64,
42}
43
44#[derive(Clone, Copy, Debug, Eq, PartialEq)]
51pub struct CausalParent {
52 context_id: u64,
53 span_id: u64,
54}
55
56impl CausalParent {
57 pub const fn context_id(self) -> u64 {
59 self.context_id
60 }
61
62 pub const fn span_id(self) -> u64 {
64 self.span_id
65 }
66
67 pub const fn is_root(self) -> bool {
69 self.span_id == 0
70 }
71}
72
73#[derive(Clone, Copy, Debug, Eq, PartialEq)]
75pub struct SamplingPolicy {
76 initial_events: u64,
77 every_nth_after_initial: u64,
78 maximum_retained: u64,
79}
80
81impl SamplingPolicy {
82 pub const fn bounded(
84 initial_events: u64,
85 every_nth_after_initial: u64,
86 maximum_retained: u64,
87 ) -> Self {
88 Self {
89 initial_events,
90 every_nth_after_initial: if every_nth_after_initial == 0 {
91 1
92 } else {
93 every_nth_after_initial
94 },
95 maximum_retained,
96 }
97 }
98
99 fn selects(self, observation: u64) -> bool {
100 observation < self.initial_events
101 || (observation - self.initial_events).is_multiple_of(self.every_nth_after_initial)
102 }
103}
104
105#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
107pub struct EventLossCounts {
108 pub point_events: u64,
109 pub annotations: u64,
110 pub sampled_out_events: u64,
111}
112
113impl EventLossCounts {
114 pub const fn capacity_drops(self) -> u64 {
116 self.point_events.saturating_add(self.annotations)
117 }
118}
119
120#[derive(Clone, Copy)]
121struct ActiveSpan {
122 runtime_key: usize,
123 span_id: u64,
124 child_elapsed_ns: u64,
125 parent_slot: Option<usize>,
126}
127
128#[derive(Clone, Copy)]
129struct SpanTrace {
130 record_index: usize,
131 span_id: u64,
132}
133
134#[derive(Clone, Copy)]
136struct SpanOutcome {
137 start_offset_ns: u64,
138 elapsed_ns: u64,
139 self_ns: u64,
140 blocked: bool,
141 serial: bool,
142 outermost: bool,
143}
144
145struct RawSpanRecord {
146 span_id: u64,
147 parent_span_id: u64,
148 metric_id: crate::MetricId,
149 start_ns: u64,
150 inclusive_ns: u64,
151 thread_id: u64,
152 worker_id: u64,
153 task_id: u64,
154 work_origin: WorkOrigin,
155 completed: bool,
156 hardware: RawSpanHardware,
157}
158
159#[derive(Clone, Copy, Default)]
161struct RawSpanHardware {
162 cycles_begin: u64,
163 cycles_end: u64,
164 instructions_begin: u64,
165 instructions_end: u64,
166 has_cycles: bool,
167 has_instructions: bool,
168 finished: bool,
169}
170
171impl RawSpanHardware {
172 fn begin() -> Self {
173 let mut hardware = Self::default();
174 if let Some(reading) = crate::hardware::span_counter_reading() {
175 if let Some(cycles) = reading.cycles {
176 hardware.cycles_begin = cycles;
177 hardware.has_cycles = true;
178 }
179 if let Some(instructions) = reading.instructions {
180 hardware.instructions_begin = instructions;
181 hardware.has_instructions = true;
182 }
183 }
184 hardware
185 }
186
187 fn finish(&mut self) {
188 if let Some(reading) = crate::hardware::span_counter_reading() {
189 if self.has_cycles {
190 if let Some(cycles) = reading.cycles {
191 self.cycles_end = cycles;
192 }
193 }
194 if self.has_instructions {
195 if let Some(instructions) = reading.instructions {
196 self.instructions_end = instructions;
197 }
198 }
199 }
200 self.finished = true;
201 }
202
203 fn into_evidence(self) -> Evidence<crate::hardware::SpanHardwareV2> {
204 if !self.finished || (!self.has_cycles && !self.has_instructions) {
205 return Evidence::unavailable(EvidenceGap::Unavailable);
206 }
207 let pair = |present: bool, begin: u64, end: u64| {
208 if present {
209 (Evidence::recorded(begin), Evidence::recorded(end))
210 } else {
211 (
212 Evidence::unavailable(EvidenceGap::Unsupported),
213 Evidence::unavailable(EvidenceGap::Unsupported),
214 )
215 }
216 };
217 let (cycles_begin, cycles_end) = pair(self.has_cycles, self.cycles_begin, self.cycles_end);
218 let (instructions_begin, instructions_end) = pair(
219 self.has_instructions,
220 self.instructions_begin,
221 self.instructions_end,
222 );
223 Evidence::recorded(crate::hardware::SpanHardwareV2 {
224 version: crate::hardware::SPAN_HARDWARE_V2_VERSION,
225 cycles_begin,
226 cycles_end,
227 instructions_begin,
228 instructions_end,
229 })
230 }
231}
232
233struct PendingQueueEnqueue {
234 thread_id: u64,
235 elapsed_ns: u64,
236}
237
238struct WorkerShard {
239 sequence: u64,
240 elapsed_ns: [AtomicU64; STAGE_COUNT],
241 calls: [AtomicU64; STAGE_COUNT],
242 attributed_ns: [AtomicU64; STAGE_COUNT],
243 blocked_ns: [AtomicU64; STAGE_COUNT],
244 blocked_calls: [AtomicU64; STAGE_COUNT],
245 legacy_elapsed_ns: [AtomicU64; STAGE_COUNT],
246 legacy_calls: [AtomicU64; STAGE_COUNT],
247 legacy_attributed_ns: [AtomicU64; STAGE_COUNT],
248 latency_buckets: [[AtomicU64; LATENCY_BUCKET_COUNT]; STAGE_COUNT],
249 latency_min_ns: [AtomicU64; STAGE_COUNT],
250 latency_max_ns: [AtomicU64; STAGE_COUNT],
251 counter_values: [AtomicU64; crate::MetricId::COUNT],
252 input_bytes: AtomicU64,
253 input_units: AtomicU64,
254 legacy_input_bytes: AtomicU64,
255 legacy_input_units: AtomicU64,
256 derived_decoder_bytes: AtomicU64,
257 backend_dispatched_bytes: AtomicU64,
258 stage_first_start_ns: [AtomicU64; STAGE_COUNT],
259 stage_last_end_ns: [AtomicU64; STAGE_COUNT],
260 stage_bytes: [AtomicU64; STAGE_COUNT],
261 serial_ns: [AtomicU64; STAGE_COUNT],
262 serial_calls: [AtomicU64; STAGE_COUNT],
263 top_level_busy_ns: AtomicU64,
264 top_level_blocked_ns: AtomicU64,
265 top_level_calls: AtomicU64,
266 cache_hits: [AtomicU64; crate::CacheId::COUNT],
267 cache_misses: [AtomicU64; crate::CacheId::COUNT],
268 indexed_counters: [[AtomicU64; crate::INDEXED_COUNTER_SLOTS]; crate::IndexedCounterId::COUNT],
269 indexed_counter_dropped: AtomicU64,
270 retries: [AtomicU64; crate::RetryCause::COUNT],
271 compile_surface_invocations:
272 [[AtomicU64; crate::CompilePhase::COUNT]; crate::CompileSurfaceId::COUNT],
273 compile_surface_loads: [AtomicU64; crate::CompileSurfaceId::COUNT],
274}
275
276const fn zero_counters() -> [AtomicU64; STAGE_COUNT] {
277 [const { AtomicU64::new(0) }; STAGE_COUNT]
278}
279
280const fn zero_indexed_counters(
281) -> [[AtomicU64; crate::INDEXED_COUNTER_SLOTS]; crate::IndexedCounterId::COUNT] {
282 [const { [const { AtomicU64::new(0) }; crate::INDEXED_COUNTER_SLOTS] };
283 crate::IndexedCounterId::COUNT]
284}
285
286const fn zero_event_values() -> [AtomicU64; crate::EventId::COUNT] {
287 [const { AtomicU64::new(0) }; crate::EventId::COUNT]
288}
289
290const fn zero_metric_values() -> [AtomicU64; crate::MetricId::COUNT] {
291 [const { AtomicU64::new(0) }; crate::MetricId::COUNT]
292}
293
294const GAUGE_PRESENT_WORDS: usize = crate::MetricId::COUNT.div_ceil(64);
295
296const fn zero_latency_buckets() -> [[AtomicU64; LATENCY_BUCKET_COUNT]; STAGE_COUNT] {
297 [const { [const { AtomicU64::new(0) }; LATENCY_BUCKET_COUNT] }; STAGE_COUNT]
298}
299
300const fn max_counters() -> [AtomicU64; STAGE_COUNT] {
301 [const { AtomicU64::new(u64::MAX) }; STAGE_COUNT]
302}
303
304const fn zero_cache_counters() -> [AtomicU64; crate::CacheId::COUNT] {
305 [const { AtomicU64::new(0) }; crate::CacheId::COUNT]
306}
307const fn zero_compile_surface_invocations(
308) -> [[AtomicU64; crate::CompilePhase::COUNT]; crate::CompileSurfaceId::COUNT] {
309 [const { [const { AtomicU64::new(0) }; crate::CompilePhase::COUNT] };
310 crate::CompileSurfaceId::COUNT]
311}
312
313const fn zero_compile_surface_loads() -> [AtomicU64; crate::CompileSurfaceId::COUNT] {
314 [const { AtomicU64::new(0) }; crate::CompileSurfaceId::COUNT]
315}
316
317impl WorkerShard {
318 fn new(sequence: u64) -> Self {
319 Self {
320 sequence,
321 elapsed_ns: zero_counters(),
322 calls: zero_counters(),
323 attributed_ns: zero_counters(),
324 blocked_ns: zero_counters(),
325 blocked_calls: zero_counters(),
326 legacy_elapsed_ns: zero_counters(),
327 legacy_calls: zero_counters(),
328 legacy_attributed_ns: zero_counters(),
329 latency_buckets: zero_latency_buckets(),
330 latency_min_ns: max_counters(),
331 latency_max_ns: zero_counters(),
332 counter_values: zero_metric_values(),
333 input_bytes: AtomicU64::new(0),
334 input_units: AtomicU64::new(0),
335 legacy_input_bytes: AtomicU64::new(0),
336 legacy_input_units: AtomicU64::new(0),
337 derived_decoder_bytes: AtomicU64::new(0),
338 backend_dispatched_bytes: AtomicU64::new(0),
339 stage_first_start_ns: max_counters(),
340 stage_last_end_ns: zero_counters(),
341 stage_bytes: zero_counters(),
342 serial_ns: zero_counters(),
343 serial_calls: zero_counters(),
344 top_level_busy_ns: AtomicU64::new(0),
345 top_level_blocked_ns: AtomicU64::new(0),
346 top_level_calls: AtomicU64::new(0),
347 cache_hits: zero_cache_counters(),
348 cache_misses: zero_cache_counters(),
349 indexed_counters: zero_indexed_counters(),
350 indexed_counter_dropped: AtomicU64::new(0),
351 retries: [const { AtomicU64::new(0) }; crate::RetryCause::COUNT],
352 compile_surface_invocations: zero_compile_surface_invocations(),
353 compile_surface_loads: zero_compile_surface_loads(),
354 }
355 }
356}
357
358fn percentile_upper_bound(
359 buckets: &[LatencyBucketV2],
360 call_count: u64,
361 percentile: u64,
362 maximum_ns: u64,
363) -> u64 {
364 let rank = (u128::from(call_count) * u128::from(percentile)).div_ceil(100);
365 let mut cumulative = 0_u128;
366 for bucket in buckets {
367 cumulative += u128::from(bucket.count);
368 if cumulative >= rank {
369 return bucket.upper_bound_ns.min(maximum_ns);
370 }
371 }
372 maximum_ns
373}
374
375#[inline]
376fn latency_bucket_index(elapsed_ns: u64) -> usize {
377 if elapsed_ns == 0 {
378 0
379 } else {
380 (u64::BITS - elapsed_ns.leading_zeros()) as usize
381 }
382}
383
384fn latency_bucket_bounds(index: usize) -> (u64, u64) {
385 match index {
386 0 => (0, 0),
387 64 => (1_u64 << 63, u64::MAX),
388 _ => (1_u64 << (index - 1), (1_u64 << index) - 1),
389 }
390}
391
392pub(crate) struct RawStageCounters {
393 pub(crate) elapsed_ns: [u64; STAGE_COUNT],
394 pub(crate) calls: [u64; STAGE_COUNT],
395 pub(crate) attributed_ns: [u64; STAGE_COUNT],
396}
397
398static ACTIVE_CONTEXTS: AtomicUsize = AtomicUsize::new(0);
399static NEXT_THREAD_ID: AtomicU64 = AtomicU64::new(1);
400static NEXT_CONTEXT_ID: AtomicU64 = AtomicU64::new(1);
401static PROCESS_ACTIVE_COMPILE_PHASE: AtomicU8 = AtomicU8::new(crate::CompilePhase::Scan as u8);
402
403struct RuntimeInner {
404 context_id: u64,
405 next_shard_sequence: AtomicU64,
406 elapsed_ns: [AtomicU64; STAGE_COUNT],
407 calls: [AtomicU64; STAGE_COUNT],
408 attributed_ns: [AtomicU64; STAGE_COUNT],
409 session_shards: Mutex<Vec<Arc<WorkerShard>>>,
410 session_gauge_values: [AtomicU64; crate::MetricId::COUNT],
411 session_gauge_present: [AtomicU64; GAUGE_PRESENT_WORDS],
412 input_bytes: AtomicU64,
413 input_units: AtomicU64,
414 session_recording: bool,
415 session_route_sequence: AtomicU64,
416 session_batch_routes: Mutex<Vec<BatchRouteV2>>,
417 session_dropped_batch_routes: AtomicU64,
418 started: Instant,
419 session_span_sequence: AtomicU64,
420 session_span_reservations: AtomicUsize,
421 session_spans: Mutex<Vec<RawSpanRecord>>,
422 session_dropped_spans: AtomicU64,
423 session_event_sequence: AtomicU64,
424 session_point_events: Mutex<Vec<PointEventV2>>,
425 session_annotations: Mutex<Vec<AnnotationV2>>,
426 session_dropped_point_events: AtomicU64,
427 session_dropped_annotations: AtomicU64,
428 session_sample_observations: [AtomicU64; crate::EventId::COUNT],
429 session_sample_retained: [AtomicU64; crate::EventId::COUNT],
430 session_sampled_out_events: AtomicU64,
431 queue_pending: Mutex<HashMap<(u8, u64), PendingQueueEnqueue>>,
432 queue_links: Mutex<Vec<QueueLinkV2>>,
433 queue_dropped_enqueues: AtomicU64,
434 queue_dropped_links: AtomicU64,
435 queue_unmatched_dequeues: AtomicU64,
436 queue_depth_current: [AtomicU64; crate::QueueId::COUNT],
437 queue_depth_high_water: [AtomicU64; crate::QueueId::COUNT],
438 queue_depth_enqueues: [AtomicU64; crate::QueueId::COUNT],
439 queue_depth_dequeues: [AtomicU64; crate::QueueId::COUNT],
440 legacy_typed_counters: [AtomicU64; crate::MetricId::COUNT],
441 distribution_buckets: [[AtomicU64; LATENCY_BUCKET_COUNT]; crate::MetricId::COUNT],
442 distribution_min: [AtomicU64; crate::MetricId::COUNT],
443 distribution_max: [AtomicU64; crate::MetricId::COUNT],
444 active_compile_phase: AtomicU8,
445 legacy_compile_surface_invocations:
446 [[AtomicU64; crate::CompilePhase::COUNT]; crate::CompileSurfaceId::COUNT],
447 legacy_compile_surface_loads: [AtomicU64; crate::CompileSurfaceId::COUNT],
448}
449
450const fn zero_queue_depths() -> [AtomicU64; crate::QueueId::COUNT] {
451 [const { AtomicU64::new(0) }; crate::QueueId::COUNT]
452}
453
454const fn zero_distribution_buckets() -> [[AtomicU64; LATENCY_BUCKET_COUNT]; crate::MetricId::COUNT]
455{
456 [const { [const { AtomicU64::new(0) }; LATENCY_BUCKET_COUNT] }; crate::MetricId::COUNT]
457}
458
459const fn zero_distribution_mins() -> [AtomicU64; crate::MetricId::COUNT] {
460 [const { AtomicU64::new(u64::MAX) }; crate::MetricId::COUNT]
461}
462
463const fn zero_distribution_maxes() -> [AtomicU64; crate::MetricId::COUNT] {
464 [const { AtomicU64::new(0) }; crate::MetricId::COUNT]
465}
466
467impl RuntimeInner {
468 fn new(session_recording: bool, started: Instant) -> Self {
469 Self {
470 context_id: NEXT_CONTEXT_ID.fetch_add(1, Ordering::Relaxed),
471 next_shard_sequence: AtomicU64::new(1),
472 elapsed_ns: zero_counters(),
473 calls: zero_counters(),
474 attributed_ns: zero_counters(),
475 session_shards: Mutex::new(Vec::new()),
476 session_gauge_values: zero_metric_values(),
477 session_gauge_present: [const { AtomicU64::new(0) }; GAUGE_PRESENT_WORDS],
478 input_bytes: AtomicU64::new(0),
479 input_units: AtomicU64::new(0),
480 session_recording,
481 session_route_sequence: AtomicU64::new(0),
482 session_batch_routes: Mutex::new(Vec::new()),
483 session_dropped_batch_routes: AtomicU64::new(0),
484 started,
485 session_span_sequence: AtomicU64::new(0),
486 session_span_reservations: AtomicUsize::new(0),
487 session_spans: Mutex::new(Vec::new()),
488 session_dropped_spans: AtomicU64::new(0),
489 session_event_sequence: AtomicU64::new(0),
490 session_point_events: Mutex::new(Vec::new()),
491 session_annotations: Mutex::new(Vec::new()),
492 session_dropped_point_events: AtomicU64::new(0),
493 session_dropped_annotations: AtomicU64::new(0),
494 session_sample_observations: zero_event_values(),
495 session_sample_retained: zero_event_values(),
496 session_sampled_out_events: AtomicU64::new(0),
497 queue_pending: Mutex::new(HashMap::new()),
498 queue_links: Mutex::new(Vec::new()),
499 queue_dropped_enqueues: AtomicU64::new(0),
500 queue_dropped_links: AtomicU64::new(0),
501 queue_unmatched_dequeues: AtomicU64::new(0),
502 queue_depth_current: zero_queue_depths(),
503 queue_depth_high_water: zero_queue_depths(),
504 queue_depth_enqueues: zero_queue_depths(),
505 queue_depth_dequeues: zero_queue_depths(),
506 legacy_typed_counters: zero_metric_values(),
507 distribution_buckets: zero_distribution_buckets(),
508 distribution_min: zero_distribution_mins(),
509 distribution_max: zero_distribution_maxes(),
510 active_compile_phase: AtomicU8::new(
511 PROCESS_ACTIVE_COMPILE_PHASE.load(Ordering::Relaxed),
512 ),
513 legacy_compile_surface_invocations: zero_compile_surface_invocations(),
514 legacy_compile_surface_loads: zero_compile_surface_loads(),
515 }
516 }
517
518 fn sorted_shards(&self) -> Vec<Arc<WorkerShard>> {
519 let shards = match self.session_shards.lock() {
520 Ok(shards) => shards,
521 Err(poisoned) => poisoned.into_inner(),
522 };
523 let mut sorted: Vec<Arc<WorkerShard>> = shards.to_vec();
524 sorted.sort_by_key(|shard| shard.sequence);
525 sorted
526 }
527}
528
529struct ThreadShardAssignment {
530 runtime: Weak<RuntimeInner>,
531 shard: Arc<WorkerShard>,
532}
533
534#[derive(Clone)]
536pub struct Runtime {
537 inner: Arc<RuntimeInner>,
538}
539
540impl Runtime {
541 pub fn new() -> Self {
543 Self::new_at(Instant::now())
544 }
545
546 pub fn context_id(&self) -> u64 {
548 self.inner.context_id
549 }
550
551 pub(crate) fn new_at(started: Instant) -> Self {
552 Self {
553 inner: Arc::new(RuntimeInner::new(true, started)),
554 }
555 }
556
557 fn legacy() -> Self {
558 Self {
559 inner: Arc::new(RuntimeInner::new(false, Instant::now())),
560 }
561 }
562
563 fn worker_shard(&self) -> Option<Arc<WorkerShard>> {
564 if !self.inner.session_recording {
565 return None;
566 }
567 THREAD_SHARDS.with(|assignments| {
568 let mut assignments = assignments.borrow_mut();
569 assignments.retain(|assignment| assignment.runtime.strong_count() != 0);
570 if let Some(assignment) = assignments.iter().find(|assignment| {
571 std::ptr::eq(assignment.runtime.as_ptr(), Arc::as_ptr(&self.inner))
572 }) {
573 return Some(assignment.shard.clone());
574 }
575 let shard = Arc::new(WorkerShard::new(
576 self.inner
577 .next_shard_sequence
578 .fetch_add(1, Ordering::Relaxed),
579 ));
580 match self.inner.session_shards.lock() {
581 Ok(mut shards) => shards.push(shard.clone()),
582 Err(poisoned) => poisoned.into_inner().push(shard.clone()),
583 }
584 assignments.push(ThreadShardAssignment {
585 runtime: Arc::downgrade(&self.inner),
586 shard: shard.clone(),
587 });
588 Some(shard)
589 })
590 }
591
592 pub fn worker_shard_count(&self) -> usize {
594 match self.inner.session_shards.lock() {
595 Ok(shards) => shards.len(),
596 Err(poisoned) => poisoned.into_inner().len(),
597 }
598 }
599
600 fn peek_worker_shard_sequence(&self) -> u64 {
602 THREAD_SHARDS.with(|assignments| {
603 assignments
604 .borrow()
605 .iter()
606 .find(|assignment| {
607 std::ptr::eq(assignment.runtime.as_ptr(), Arc::as_ptr(&self.inner))
608 })
609 .map_or(0, |assignment| assignment.shard.sequence)
610 })
611 }
612
613 pub fn causal_parent(&self) -> CausalParent {
615 CausalParent {
616 context_id: self.inner.context_id,
617 span_id: self.current_parent_span_id(),
618 }
619 }
620
621 pub fn enter(&self) -> ContextGuard {
623 let _ = self.worker_shard();
624 CURRENT.with(|stack| stack.borrow_mut().push(self.clone()));
625 ACTIVE_CONTEXTS.fetch_add(1, Ordering::Relaxed);
626 ContextGuard {
627 runtime: self.clone(),
628 not_send: PhantomData,
629 }
630 }
631
632 fn enter_async_parent(&self, span_id: u64) -> Option<AsyncParentGuard> {
633 let stack_slot =
634 ASYNC_PARENT_SPANS.with(|stack| stack.borrow().iter().position(Option::is_none))?;
635 let runtime_key = Arc::as_ptr(&self.inner) as usize;
636 ASYNC_PARENT_SPANS.with(|stack| {
637 stack.borrow_mut()[stack_slot] = Some(ActiveSpan {
638 runtime_key,
639 span_id,
640 child_elapsed_ns: 0,
641 parent_slot: None,
642 });
643 });
644 Some(AsyncParentGuard {
645 runtime_key,
646 span_id,
647 stack_slot,
648 not_send: PhantomData,
649 })
650 }
651
652 pub fn scope<T>(&self, operation: impl FnOnce() -> T) -> T {
654 let _guard = self.enter();
655 operation()
656 }
657
658 fn elapsed_ns(&self) -> u64 {
659 u64::try_from(self.inner.started.elapsed().as_nanos()).unwrap_or(u64::MAX)
660 }
661
662 pub(crate) fn add_counter(&self, counter: crate::CounterId, delta: u64) {
663 if let Some(shard) = self.worker_shard() {
664 shard.counter_values[counter.metric_id() as usize].fetch_add(delta, Ordering::Relaxed);
665 } else if !self.inner.session_recording {
666 self.inner.legacy_typed_counters[counter.metric_id() as usize]
667 .fetch_add(delta, Ordering::Relaxed);
668 }
669 }
670
671 fn record_distribution(&self, metric_id: crate::MetricId, value: u64) {
672 let index = metric_id as usize;
673 let bucket = latency_bucket_index(value);
674 self.inner.distribution_buckets[index][bucket].fetch_add(1, Ordering::Relaxed);
675 self.inner.distribution_min[index].fetch_min(value, Ordering::Relaxed);
676 self.inner.distribution_max[index].fetch_max(value, Ordering::Relaxed);
677 }
678
679 pub(crate) fn retry_annotation_count(&self) -> u64 {
681 if !self.inner.session_recording {
682 return 0;
683 }
684 let annotations = match self.inner.session_annotations.lock() {
685 Ok(annotations) => annotations,
686 Err(poisoned) => poisoned.into_inner(),
687 };
688 annotations
689 .iter()
690 .filter(|annotation| annotation.annotation_id == crate::AnnotationId::RetryAttempt)
691 .count() as u64
692 }
693
694 pub(crate) fn session_gauge(&self, gauge: crate::GaugeId) -> Option<u64> {
696 if !self.inner.session_recording {
697 return None;
698 }
699 let index = gauge.metric_id() as usize;
700 let present = self.inner.session_gauge_present[index / 64].load(Ordering::Relaxed);
701 (present & (1_u64 << (index % 64)) != 0)
702 .then(|| self.inner.session_gauge_values[index].load(Ordering::Relaxed))
703 }
704
705 pub(crate) fn record_retained_buffer_bytes(&self, bytes: u64) {
707 if !self.inner.session_recording {
708 return;
709 }
710 self.set_gauge(crate::GaugeId::RetainedBufferBytes, bytes);
711 let peak = crate::GaugeId::RetainedBufferPeakBytes.metric_id() as usize;
712 self.inner.session_gauge_values[peak].fetch_max(bytes, Ordering::Relaxed);
713 self.inner.session_gauge_present[peak / 64]
714 .fetch_or(1_u64 << (peak % 64), Ordering::Relaxed);
715 }
716
717 pub fn take_metric_distributions(&self) -> Vec<MetricDistributionV2> {
719 let mut records = Vec::new();
720 for index in 0..crate::MetricId::COUNT {
721 let mut call_count = 0_u64;
722 let buckets: Vec<DistributionBucketV2> = (0..LATENCY_BUCKET_COUNT)
723 .filter_map(|bucket| {
724 let count =
725 self.inner.distribution_buckets[index][bucket].swap(0, Ordering::Relaxed);
726 call_count = call_count.saturating_add(count);
727 if count == 0 {
728 return None;
729 }
730 let (lower_bound, upper_bound) = latency_bucket_bounds(bucket);
731 Some(DistributionBucketV2 {
732 version: 1,
733 lower_bound,
734 upper_bound,
735 count,
736 })
737 })
738 .collect();
739 if call_count == 0 {
740 continue;
741 }
742 let minimum = self.inner.distribution_min[index].swap(u64::MAX, Ordering::Relaxed);
743 let maximum = self.inner.distribution_max[index].swap(0, Ordering::Relaxed);
744 records.push(MetricDistributionV2 {
745 version: 1,
746 metric_id: crate::METRICS[index].id,
747 call_count,
748 minimum,
749 maximum,
750 buckets,
751 });
752 }
753 records
754 }
755
756 pub fn take_legacy_typed_metrics(&self) -> Vec<TypedMetricRecordV2> {
758 if self.inner.session_recording {
759 return self.take_session_typed_metrics();
760 }
761 let mut records = Vec::new();
762 for counter in crate::CounterId::ALL {
763 let metric_id = counter.metric_id();
764 let value =
765 self.inner.legacy_typed_counters[metric_id as usize].swap(0, Ordering::Relaxed);
766 if value != 0 {
767 records.push(TypedMetricRecordV2 {
768 version: 1,
769 metric_id,
770 kind: crate::MetricKind::Counter,
771 value,
772 });
773 }
774 }
775 records
776 }
777
778 pub(crate) fn set_gauge(&self, gauge: crate::GaugeId, value: u64) {
779 if !self.inner.session_recording {
780 return;
781 }
782 let index = gauge.metric_id() as usize;
783 self.inner.session_gauge_values[index].store(value, Ordering::Relaxed);
784 self.inner.session_gauge_present[index / 64]
785 .fetch_or(1_u64 << (index % 64), Ordering::Relaxed);
786 }
787
788 fn record_event(&self, event_id: crate::EventId, value: u64) -> bool {
789 if !self.inner.session_recording {
790 return false;
791 }
792 let sequence = self
793 .inner
794 .session_event_sequence
795 .fetch_add(1, Ordering::Relaxed);
796 let event = PointEventV2 {
797 version: 2,
798 sequence,
799 event_id,
800 elapsed_ns: self.elapsed_ns(),
801 thread_id: numeric_thread_id(),
802 value,
803 task_id: evidence_or_unavailable(current_task_id()),
804 worker_id: evidence_or_unavailable(self.peek_worker_shard_sequence()),
805 };
806 let mut events = match self.inner.session_point_events.lock() {
807 Ok(events) => events,
808 Err(poisoned) => poisoned.into_inner(),
809 };
810 if events.len() == MAX_POINT_EVENTS {
811 self.inner
812 .session_dropped_point_events
813 .fetch_add(1, Ordering::Relaxed);
814 false
815 } else {
816 events.push(event);
817 true
818 }
819 }
820
821 fn record_sampled_event(
822 &self,
823 event_id: crate::EventId,
824 value: u64,
825 policy: SamplingPolicy,
826 ) -> bool {
827 if !self.inner.session_recording {
828 return false;
829 }
830 let index = event_id.index();
831 let observation =
832 self.inner.session_sample_observations[index].fetch_add(1, Ordering::Relaxed);
833 let mut retained = false;
834 if policy.selects(observation) {
835 let counter = &self.inner.session_sample_retained[index];
836 let mut current = counter.load(Ordering::Relaxed);
837 loop {
838 if current >= policy.maximum_retained {
839 break;
840 }
841 match counter.compare_exchange_weak(
842 current,
843 current + 1,
844 Ordering::Relaxed,
845 Ordering::Relaxed,
846 ) {
847 Ok(_) => {
848 retained = true;
849 break;
850 }
851 Err(observed) => current = observed,
852 }
853 }
854 }
855 if retained {
856 self.record_event(event_id, value)
857 } else {
858 self.inner
859 .session_sampled_out_events
860 .fetch_add(1, Ordering::Relaxed);
861 false
862 }
863 }
864
865 fn record_annotation(&self, annotation_id: crate::AnnotationId, value: u64) {
866 if !self.inner.session_recording {
867 return;
868 }
869 let sequence = self
870 .inner
871 .session_event_sequence
872 .fetch_add(1, Ordering::Relaxed);
873 let annotation = AnnotationV2 {
874 version: 2,
875 sequence,
876 annotation_id,
877 elapsed_ns: self.elapsed_ns(),
878 thread_id: numeric_thread_id(),
879 value,
880 task_id: evidence_or_unavailable(current_task_id()),
881 worker_id: evidence_or_unavailable(self.peek_worker_shard_sequence()),
882 };
883 let mut annotations = match self.inner.session_annotations.lock() {
884 Ok(annotations) => annotations,
885 Err(poisoned) => poisoned.into_inner(),
886 };
887 if annotations.len() == MAX_ANNOTATIONS {
888 self.inner
889 .session_dropped_annotations
890 .fetch_add(1, Ordering::Relaxed);
891 } else {
892 annotations.push(annotation);
893 }
894 }
895
896 pub fn take_session_typed_metrics(&self) -> Vec<TypedMetricRecordV2> {
898 let mut records =
899 Vec::with_capacity(crate::CounterId::ALL.len() + crate::GaugeId::ALL.len());
900 let shards = self.inner.sorted_shards();
901 for counter in crate::CounterId::ALL {
902 let metric_id = counter.metric_id();
903 let value = shards.iter().fold(0_u64, |total, shard| {
904 total.saturating_add(
905 shard.counter_values[metric_id as usize].swap(0, Ordering::Relaxed),
906 )
907 });
908 if value != 0 {
909 records.push(TypedMetricRecordV2 {
910 version: 1,
911 metric_id,
912 kind: crate::MetricKind::Counter,
913 value,
914 });
915 }
916 }
917 drop(shards);
918 let present: [u64; GAUGE_PRESENT_WORDS] =
919 std::array::from_fn(|i| self.inner.session_gauge_present[i].swap(0, Ordering::Relaxed));
920 for gauge in crate::GaugeId::ALL {
921 let metric_id = gauge.metric_id();
922 let index = metric_id as usize;
923 if present[index / 64] & (1_u64 << (index % 64)) != 0 {
924 records.push(TypedMetricRecordV2 {
925 version: 1,
926 metric_id,
927 kind: crate::MetricKind::Gauge,
928 value: self.inner.session_gauge_values[metric_id as usize]
929 .swap(0, Ordering::Relaxed),
930 });
931 }
932 }
933 records.sort_unstable_by_key(|record| record.metric_id);
934 records
935 }
936
937 pub fn take_session_typed_events(
939 &self,
940 ) -> (Vec<PointEventV2>, Vec<AnnotationV2>, EventLossCounts) {
941 let mut events = match self.inner.session_point_events.lock() {
942 Ok(events) => events,
943 Err(poisoned) => poisoned.into_inner(),
944 };
945 let mut events = std::mem::take(&mut *events);
946 let mut annotations = match self.inner.session_annotations.lock() {
947 Ok(annotations) => annotations,
948 Err(poisoned) => poisoned.into_inner(),
949 };
950 let mut annotations = std::mem::take(&mut *annotations);
951 events.sort_unstable_by_key(|event| event.sequence);
952 annotations.sort_unstable_by_key(|annotation| annotation.sequence);
953 let loss = EventLossCounts {
954 point_events: self
955 .inner
956 .session_dropped_point_events
957 .swap(0, Ordering::Relaxed),
958 annotations: self
959 .inner
960 .session_dropped_annotations
961 .swap(0, Ordering::Relaxed),
962 sampled_out_events: self
963 .inner
964 .session_sampled_out_events
965 .swap(0, Ordering::Relaxed),
966 };
967 (events, annotations, loss)
968 }
969
970 fn current_parent(&self) -> (Option<usize>, u64) {
971 let runtime_key = Arc::as_ptr(&self.inner) as usize;
972 let active = ACTIVE_SPANS.with(|stack| {
973 let stack = stack.borrow();
974 stack.iter().enumerate().rev().find_map(|(slot, active)| {
975 active
976 .and_then(|a| (a.runtime_key == runtime_key).then_some((Some(slot), a.span_id)))
977 })
978 });
979 if let Some(pair) = active {
980 return pair;
981 }
982 let async_span_id = ASYNC_PARENT_SPANS.with(|stack| {
983 stack
984 .borrow()
985 .iter()
986 .rev()
987 .flatten()
988 .find(|active| active.runtime_key == runtime_key)
989 .map(|active| active.span_id)
990 });
991 (None, async_span_id.unwrap_or(0))
992 }
993
994 fn current_parent_span_id(&self) -> u64 {
995 self.current_parent().1
996 }
997
998 fn reserve_span_with_id(
999 &self,
1000 span_id: u64,
1001 stage: Stage,
1002 started: Instant,
1003 parent_span_id: u64,
1004 _stack_slot: Option<usize>,
1005 worker_id: u64,
1006 ) -> Option<SpanTrace> {
1007 if !self.inner.session_recording {
1008 return None;
1009 }
1010 let reservation = self
1011 .inner
1012 .session_span_reservations
1013 .fetch_add(1, Ordering::Relaxed);
1014 if reservation >= MAX_RECORDED_SPANS {
1015 self.inner
1016 .session_dropped_spans
1017 .fetch_add(1, Ordering::Relaxed);
1018 return None;
1019 }
1020 let mut records = match self.inner.session_spans.lock() {
1021 Ok(records) => records,
1022 Err(poisoned) => poisoned.into_inner(),
1023 };
1024 let record_index = records.len();
1025 records.push(RawSpanRecord {
1026 span_id,
1027 parent_span_id,
1028 metric_id: stage.into(),
1029 start_ns: u64::try_from(
1030 started
1031 .checked_duration_since(self.inner.started)
1032 .unwrap_or_default()
1033 .as_nanos(),
1034 )
1035 .unwrap_or(u64::MAX),
1036 inclusive_ns: 0,
1037 thread_id: numeric_thread_id(),
1038 worker_id,
1039 task_id: current_task_id(),
1040 work_origin: current_work_origin(),
1041 completed: false,
1042 hardware: RawSpanHardware::begin(),
1043 });
1044 Some(SpanTrace {
1045 record_index,
1046 span_id,
1047 })
1048 }
1049
1050 fn reserve_span(
1051 &self,
1052 stage: Stage,
1053 started: Instant,
1054 parent_span_id: u64,
1055 stack_slot: Option<usize>,
1056 worker_id: u64,
1057 ) -> Option<SpanTrace> {
1058 let span_id = self
1059 .inner
1060 .session_span_sequence
1061 .fetch_add(1, Ordering::Relaxed)
1062 .saturating_add(1);
1063 self.reserve_span_with_id(
1064 span_id,
1065 stage,
1066 started,
1067 parent_span_id,
1068 stack_slot,
1069 worker_id,
1070 )
1071 }
1072
1073 fn begin_span_with(
1074 &self,
1075 stage: Stage,
1076 started: Instant,
1077 parent_span_id: u64,
1078 parent_slot: Option<usize>,
1079 worker_id: u64,
1080 ) -> (Option<SpanTrace>, Option<usize>, u64) {
1081 let span_id = self
1082 .inner
1083 .session_span_sequence
1084 .fetch_add(1, Ordering::Relaxed)
1085 .saturating_add(1);
1086 let stack_slot = ACTIVE_SPANS.with(|stack| stack.borrow().iter().position(Option::is_none));
1087 let Some(stack_slot) = stack_slot else {
1088 self.inner
1089 .session_dropped_spans
1090 .fetch_add(1, Ordering::Relaxed);
1091 return (None, None, span_id);
1092 };
1093 let trace = if self.inner.session_recording {
1094 self.reserve_span_with_id(
1095 span_id,
1096 stage,
1097 started,
1098 parent_span_id,
1099 Some(stack_slot),
1100 worker_id,
1101 )
1102 } else {
1103 None
1104 };
1105 let runtime_key = Arc::as_ptr(&self.inner) as usize;
1106 ACTIVE_SPANS.with(|stack| {
1107 stack.borrow_mut()[stack_slot] = Some(ActiveSpan {
1108 runtime_key,
1109 span_id,
1110 child_elapsed_ns: 0,
1111 parent_slot,
1112 });
1113 });
1114 (trace, Some(stack_slot), span_id)
1115 }
1116
1117 fn pop_active_span(
1118 &self,
1119 stack_slot: Option<usize>,
1120 expected_span_id: u64,
1121 elapsed_ns: u64,
1122 ) -> u64 {
1123 let Some(stack_slot) = stack_slot else {
1124 return elapsed_ns;
1125 };
1126 let runtime_key = Arc::as_ptr(&self.inner) as usize;
1127 let child_elapsed_ns = ACTIVE_SPANS.with(|stack| {
1128 let mut stack = stack.borrow_mut();
1129 if let Some(active) = &stack[stack_slot] {
1130 if active.runtime_key == runtime_key && active.span_id == expected_span_id {
1131 let active = stack[stack_slot].take().unwrap();
1132 if let Some(parent_slot) = active.parent_slot {
1133 if let Some(parent) = stack[parent_slot].as_mut() {
1134 if parent.runtime_key == runtime_key {
1135 parent.child_elapsed_ns =
1136 parent.child_elapsed_ns.saturating_add(elapsed_ns);
1137 }
1138 }
1139 }
1140 return active.child_elapsed_ns;
1141 }
1142 }
1143 0
1144 });
1145 elapsed_ns.saturating_sub(child_elapsed_ns)
1146 }
1147 fn begin_async_span(
1148 &self,
1149 stage: Stage,
1150 started: Instant,
1151 parent_span_id: u64,
1152 worker_id: u64,
1153 ) -> Option<SpanTrace> {
1154 if !self.inner.session_recording {
1155 return None;
1156 }
1157 self.reserve_span(stage, started, parent_span_id, None, worker_id)
1158 }
1159
1160 fn finish_span(&self, trace: SpanTrace, inclusive_ns: u64) {
1161 let mut records = match self.inner.session_spans.lock() {
1162 Ok(records) => records,
1163 Err(poisoned) => poisoned.into_inner(),
1164 };
1165 if let Some(record) = records.get_mut(trace.record_index) {
1166 record.inclusive_ns = inclusive_ns;
1167 record.completed = true;
1168 record.hardware.finish();
1169 }
1170 }
1171
1172 pub fn take_session_span_records(&self) -> (Vec<SpanRecordV2>, u64) {
1174 let mut records = match self.inner.session_spans.lock() {
1175 Ok(records) => records,
1176 Err(poisoned) => poisoned.into_inner(),
1177 };
1178 let mut raw = std::mem::take(&mut *records);
1179 drop(records);
1180 raw.sort_unstable_by_key(|record| record.span_id);
1181 let unfinished = raw.iter().filter(|record| !record.completed).count() as u64;
1182 raw.retain(|record| record.completed);
1183 let positions: HashMap<u64, usize> = raw
1184 .iter()
1185 .enumerate()
1186 .map(|(index, record)| (record.span_id, index))
1187 .collect();
1188 let mut exclusive: Vec<u64> = raw.iter().map(|record| record.inclusive_ns).collect();
1189 for record in &raw {
1190 if let Some(parent_index) = positions.get(&record.parent_span_id).copied() {
1191 exclusive[parent_index] =
1192 exclusive[parent_index].saturating_sub(record.inclusive_ns);
1193 }
1194 }
1195 let spans = raw
1196 .into_iter()
1197 .enumerate()
1198 .map(|(index, record)| SpanRecordV2 {
1199 version: 3,
1200 span_id: record.span_id,
1201 parent_span_id: if positions.contains_key(&record.parent_span_id) {
1202 Evidence::recorded(record.parent_span_id)
1203 } else {
1204 Evidence::unavailable(EvidenceGap::Unavailable)
1205 },
1206 metric_id: record.metric_id,
1207 start_ns: record.start_ns,
1208 inclusive_ns: record.inclusive_ns,
1209 exclusive_ns: exclusive[index],
1210 thread_id: record.thread_id,
1211 task_id: evidence_or_unavailable(record.task_id),
1212 worker_id: evidence_or_unavailable(record.worker_id),
1213 work_origin: record.work_origin,
1214 hardware: record.hardware.into_evidence(),
1215 })
1216 .collect();
1217 let dropped = self
1218 .inner
1219 .session_dropped_spans
1220 .swap(0, Ordering::Relaxed)
1221 .saturating_add(unfinished);
1222 (spans, dropped)
1223 }
1224
1225 pub fn take_session_latency_distributions(&self) -> Vec<LatencyDistributionV2> {
1227 let shards = self.inner.sorted_shards();
1228 Stage::ALL
1229 .into_iter()
1230 .filter_map(|stage| {
1231 let mut call_count = 0_u64;
1232 let buckets: Vec<LatencyBucketV2> = (0..LATENCY_BUCKET_COUNT)
1233 .filter_map(|index| {
1234 let count = shards.iter().fold(0_u64, |total, shard| {
1235 total.saturating_add(
1236 shard.latency_buckets[stage.index()][index]
1237 .swap(0, Ordering::Relaxed),
1238 )
1239 });
1240 call_count = call_count.saturating_add(count);
1241 (count != 0).then(|| {
1242 let (lower_bound_ns, upper_bound_ns) = latency_bucket_bounds(index);
1243 LatencyBucketV2 {
1244 version: 1,
1245 lower_bound_ns,
1246 upper_bound_ns,
1247 count,
1248 }
1249 })
1250 })
1251 .collect();
1252 if call_count == 0 {
1253 return None;
1254 }
1255 let minimum_ns = shards.iter().fold(u64::MAX, |minimum, shard| {
1256 minimum
1257 .min(shard.latency_min_ns[stage.index()].swap(u64::MAX, Ordering::Relaxed))
1258 });
1259 let maximum_ns = shards.iter().fold(0_u64, |maximum, shard| {
1260 maximum.max(shard.latency_max_ns[stage.index()].swap(0, Ordering::Relaxed))
1261 });
1262 Some(LatencyDistributionV2 {
1263 version: 2,
1264 metric_id: stage.metric_id(),
1265 macro_stage_id: stage.macro_stage_id(),
1266 call_count,
1267 minimum_ns,
1268 maximum_ns,
1269 p50_ns: percentile_upper_bound(&buckets, call_count, 50, maximum_ns),
1270 p90_ns: percentile_upper_bound(&buckets, call_count, 90, maximum_ns),
1271 p95_ns: percentile_upper_bound(&buckets, call_count, 95, maximum_ns),
1272 p99_ns: percentile_upper_bound(&buckets, call_count, 99, maximum_ns),
1273 buckets,
1274 })
1275 })
1276 .collect()
1277 }
1278
1279 fn record(&self, shard: Option<&WorkerShard>, stage: Stage, outcome: SpanOutcome) {
1280 let index = stage.index();
1281 let elapsed_ns = outcome.elapsed_ns;
1282 let self_ns = outcome.self_ns;
1283 if self.inner.session_recording {
1284 let Some(shard) = shard else {
1285 return;
1286 };
1287 shard.elapsed_ns[index].fetch_add(elapsed_ns, Ordering::Relaxed);
1288 shard.calls[index].fetch_add(1, Ordering::Relaxed);
1289 shard.legacy_elapsed_ns[index].fetch_add(elapsed_ns, Ordering::Relaxed);
1290 shard.legacy_calls[index].fetch_add(1, Ordering::Relaxed);
1291 let bucket = latency_bucket_index(elapsed_ns);
1292 shard.latency_buckets[index][bucket].fetch_add(1, Ordering::Relaxed);
1293 shard.latency_min_ns[index].fetch_min(elapsed_ns, Ordering::Relaxed);
1294 shard.latency_max_ns[index].fetch_max(elapsed_ns, Ordering::Relaxed);
1295 shard.stage_first_start_ns[index].fetch_min(outcome.start_offset_ns, Ordering::Relaxed);
1296 shard.stage_last_end_ns[index].fetch_max(
1297 outcome.start_offset_ns.saturating_add(elapsed_ns),
1298 Ordering::Relaxed,
1299 );
1300 if !outcome.blocked && !stage.is_container() {
1303 shard.attributed_ns[index].fetch_add(self_ns, Ordering::Relaxed);
1304 shard.legacy_attributed_ns[index].fetch_add(self_ns, Ordering::Relaxed);
1305 }
1306
1307 if outcome.blocked {
1308 shard.blocked_ns[index].fetch_add(elapsed_ns, Ordering::Relaxed);
1309 shard.blocked_calls[index].fetch_add(1, Ordering::Relaxed);
1310 }
1311 if outcome.serial {
1312 shard.serial_ns[index].fetch_add(elapsed_ns, Ordering::Relaxed);
1313 shard.serial_calls[index].fetch_add(1, Ordering::Relaxed);
1314 }
1315 if !stage.is_container() {
1319 if outcome.outermost {
1320 shard.top_level_calls.fetch_add(1, Ordering::Relaxed);
1321 }
1322 if outcome.blocked {
1323 shard
1324 .top_level_blocked_ns
1325 .fetch_add(self_ns, Ordering::Relaxed);
1326 } else {
1327 shard
1328 .top_level_busy_ns
1329 .fetch_add(self_ns, Ordering::Relaxed);
1330 }
1331 }
1332 return;
1333 }
1334 self.inner.elapsed_ns[index].fetch_add(elapsed_ns, Ordering::Relaxed);
1335 self.inner.calls[index].fetch_add(1, Ordering::Relaxed);
1336 if !outcome.blocked && !stage.is_container() {
1337 self.inner.attributed_ns[index].fetch_add(self_ns, Ordering::Relaxed);
1338 }
1339 }
1340
1341 fn add_stage_bytes(&self, stage: Stage, bytes: u64) {
1342 if let Some(shard) = self.worker_shard() {
1343 shard.stage_bytes[stage.index()].fetch_add(bytes, Ordering::Relaxed);
1344 }
1345 }
1346
1347 fn record_cache_outcome(&self, cache: crate::CacheId, hit: bool) {
1348 let Some(shard) = self.worker_shard() else {
1349 return;
1350 };
1351 let slot = if hit {
1352 &shard.cache_hits[cache.index()]
1353 } else {
1354 &shard.cache_misses[cache.index()]
1355 };
1356 slot.fetch_add(1, Ordering::Relaxed);
1357 }
1358
1359 #[inline]
1361 #[must_use]
1362 pub fn cache_hits(&self, cache: crate::CacheId) -> u64 {
1363 let index = cache.index();
1364 let mut total = 0u64;
1365 for shard in self.inner.sorted_shards() {
1366 total = total.saturating_add(shard.cache_hits[index].load(Ordering::Relaxed));
1367 }
1368 total
1369 }
1370
1371 #[inline]
1373 #[must_use]
1374 pub fn cache_misses(&self, cache: crate::CacheId) -> u64 {
1375 let index = cache.index();
1376 let mut total = 0u64;
1377 for shard in self.inner.sorted_shards() {
1378 total = total.saturating_add(shard.cache_misses[index].load(Ordering::Relaxed));
1379 }
1380 total
1381 }
1382
1383 fn record_retry(&self, cause: crate::RetryCause) {
1384 if let Some(shard) = self.worker_shard() {
1385 shard.retries[cause.index()].fetch_add(1, Ordering::Relaxed);
1386 }
1387 }
1388
1389 fn add_indexed_counter(&self, counter: crate::IndexedCounterId, slot: u16, delta: u64) {
1390 let Some(shard) = self.worker_shard() else {
1391 return;
1392 };
1393 match shard.indexed_counters[counter.index()].get(usize::from(slot)) {
1396 Some(cell) => {
1397 cell.fetch_add(delta, Ordering::Relaxed);
1398 }
1399 None => {
1400 shard
1401 .indexed_counter_dropped
1402 .fetch_add(1, Ordering::Relaxed);
1403 }
1404 }
1405 }
1406
1407 fn add_input_bytes(&self, bytes: u64) {
1408 if let Some(shard) = self.worker_shard() {
1409 shard.input_bytes.fetch_add(bytes, Ordering::Relaxed);
1410 shard.legacy_input_bytes.fetch_add(bytes, Ordering::Relaxed);
1411 shard.counter_values[crate::MetricId::InputBytes as usize]
1412 .fetch_add(bytes, Ordering::Relaxed);
1413 } else {
1414 self.inner.input_bytes.fetch_add(bytes, Ordering::Relaxed);
1415 self.inner.legacy_typed_counters[crate::MetricId::InputBytes as usize]
1416 .fetch_add(bytes, Ordering::Relaxed);
1417 }
1418 }
1419
1420 fn add_input_units(&self, units: u64) {
1421 if let Some(shard) = self.worker_shard() {
1422 shard.input_units.fetch_add(units, Ordering::Relaxed);
1423 shard.legacy_input_units.fetch_add(units, Ordering::Relaxed);
1424 shard.counter_values[crate::MetricId::InputUnits as usize]
1425 .fetch_add(units, Ordering::Relaxed);
1426 } else {
1427 self.inner.input_units.fetch_add(units, Ordering::Relaxed);
1428 self.inner.legacy_typed_counters[crate::MetricId::InputUnits as usize]
1429 .fetch_add(units, Ordering::Relaxed);
1430 }
1431 }
1432
1433 fn add_derived_decoder_bytes(&self, bytes: u64) {
1434 if let Some(shard) = self.worker_shard() {
1435 shard
1436 .derived_decoder_bytes
1437 .fetch_add(bytes, Ordering::Relaxed);
1438 }
1439 }
1440
1441 fn add_backend_dispatched_bytes(&self, bytes: u64) {
1442 if let Some(shard) = self.worker_shard() {
1443 shard
1444 .backend_dispatched_bytes
1445 .fetch_add(bytes, Ordering::Relaxed);
1446 }
1447 }
1448 pub fn set_compile_phase(&self, phase: crate::CompilePhase) {
1449 self.inner
1450 .active_compile_phase
1451 .store(phase as u8, Ordering::Relaxed);
1452 }
1453
1454 pub fn active_compile_phase(&self) -> crate::CompilePhase {
1455 match self.inner.active_compile_phase.load(Ordering::Relaxed) {
1456 0 => crate::CompilePhase::Install,
1457 1 => crate::CompilePhase::Update,
1458 2 => crate::CompilePhase::Scan,
1459 _ => crate::CompilePhase::Developer,
1460 }
1461 }
1462
1463 pub fn record_compile_surface_invocation(&self, surface: crate::CompileSurfaceId) {
1464 let phase = self.active_compile_phase();
1465 self.record_compile_surface_invocation_with_phase(surface, phase);
1466 }
1467
1468 pub fn record_compile_surface_invocation_with_phase(
1469 &self,
1470 surface: crate::CompileSurfaceId,
1471 phase: crate::CompilePhase,
1472 ) {
1473 if let Some(shard) = self.worker_shard() {
1474 shard.compile_surface_invocations[surface.index()][phase.index()]
1475 .fetch_add(1, Ordering::Relaxed);
1476 } else {
1477 self.inner.legacy_compile_surface_invocations[surface.index()][phase.index()]
1478 .fetch_add(1, Ordering::Relaxed);
1479 }
1480 }
1481
1482 pub fn record_compile_surface_load(&self, surface: crate::CompileSurfaceId) {
1483 if let Some(shard) = self.worker_shard() {
1484 shard.compile_surface_loads[surface.index()].fetch_add(1, Ordering::Relaxed);
1485 } else {
1486 self.inner.legacy_compile_surface_loads[surface.index()]
1487 .fetch_add(1, Ordering::Relaxed);
1488 }
1489 }
1490
1491 pub fn total_runtime_compiles(&self) -> u64 {
1492 let mut total = 0u64;
1493 for surface in crate::CompileSurfaceId::ALL {
1494 total = total.saturating_add(
1495 self.compile_surface_invocations(surface, crate::CompilePhase::Scan),
1496 );
1497 total = total.saturating_add(
1498 self.compile_surface_invocations(surface, crate::CompilePhase::Developer),
1499 );
1500 }
1501 total
1502 }
1503
1504 pub fn compile_surface_invocations(
1505 &self,
1506 surface: crate::CompileSurfaceId,
1507 phase: crate::CompilePhase,
1508 ) -> u64 {
1509 let mut count = self.inner.legacy_compile_surface_invocations[surface.index()]
1510 [phase.index()]
1511 .load(Ordering::Relaxed);
1512 let shards = self.inner.sorted_shards();
1513 for shard in shards.iter() {
1514 count = count.saturating_add(
1515 shard.compile_surface_invocations[surface.index()][phase.index()]
1516 .load(Ordering::Relaxed),
1517 );
1518 }
1519 count
1520 }
1521
1522 pub fn compile_surface_loads(&self, surface: crate::CompileSurfaceId) -> u64 {
1523 let mut count =
1524 self.inner.legacy_compile_surface_loads[surface.index()].load(Ordering::Relaxed);
1525 let shards = self.inner.sorted_shards();
1526 for shard in shards.iter() {
1527 count = count.saturating_add(
1528 shard.compile_surface_loads[surface.index()].load(Ordering::Relaxed),
1529 );
1530 }
1531 count
1532 }
1533
1534 pub fn compile_surface_reports(&self) -> Vec<crate::schema_v2::CompileSurfaceRecordV2> {
1535 crate::CompileSurfaceId::ALL
1536 .iter()
1537 .map(|&surface| {
1538 let install_compiles =
1539 self.compile_surface_invocations(surface, crate::CompilePhase::Install);
1540 let update_compiles =
1541 self.compile_surface_invocations(surface, crate::CompilePhase::Update);
1542 let scan_compiles =
1543 self.compile_surface_invocations(surface, crate::CompilePhase::Scan);
1544 let developer_compiles =
1545 self.compile_surface_invocations(surface, crate::CompilePhase::Developer);
1546 let runtime_compiles = scan_compiles.saturating_add(developer_compiles);
1547 let loads = self.compile_surface_loads(surface);
1548 crate::schema_v2::CompileSurfaceRecordV2 {
1549 version: crate::schema_v2::COMPILE_SURFACE_RECORD_V2_VERSION,
1550 surface,
1551 name: surface.as_str().to_string(),
1552 runtime_compiles,
1553 loads,
1554 install_compiles,
1555 update_compiles,
1556 developer_compiles,
1557 }
1558 })
1559 .collect()
1560 }
1561
1562 pub(crate) fn drain_stage_counters(&self, session: bool) -> RawStageCounters {
1563 if !session && !self.inner.session_recording {
1564 return RawStageCounters {
1565 elapsed_ns: std::array::from_fn(|index| {
1566 self.inner.elapsed_ns[index].swap(0, Ordering::Relaxed)
1567 }),
1568 calls: std::array::from_fn(|index| {
1569 self.inner.calls[index].swap(0, Ordering::Relaxed)
1570 }),
1571 attributed_ns: std::array::from_fn(|index| {
1572 self.inner.attributed_ns[index].swap(0, Ordering::Relaxed)
1573 }),
1574 };
1575 }
1576 let shards = self.inner.sorted_shards();
1577 if !session {
1578 return RawStageCounters {
1579 elapsed_ns: std::array::from_fn(|index| {
1580 shards.iter().fold(0_u64, |total, shard| {
1581 total.saturating_add(
1582 shard.legacy_elapsed_ns[index].swap(0, Ordering::Relaxed),
1583 )
1584 })
1585 }),
1586 calls: std::array::from_fn(|index| {
1587 shards.iter().fold(0_u64, |total, shard| {
1588 total.saturating_add(shard.legacy_calls[index].swap(0, Ordering::Relaxed))
1589 })
1590 }),
1591 attributed_ns: std::array::from_fn(|index| {
1592 shards.iter().fold(0_u64, |total, shard| {
1593 total.saturating_add(
1594 shard.legacy_attributed_ns[index].swap(0, Ordering::Relaxed),
1595 )
1596 })
1597 }),
1598 };
1599 }
1600 RawStageCounters {
1601 elapsed_ns: std::array::from_fn(|index| {
1602 shards.iter().fold(0_u64, |total, shard| {
1603 total.saturating_add(shard.elapsed_ns[index].swap(0, Ordering::Relaxed))
1604 })
1605 }),
1606 calls: std::array::from_fn(|index| {
1607 shards.iter().fold(0_u64, |total, shard| {
1608 total.saturating_add(shard.calls[index].swap(0, Ordering::Relaxed))
1609 })
1610 }),
1611 attributed_ns: std::array::from_fn(|index| {
1612 shards.iter().fold(0_u64, |total, shard| {
1613 total.saturating_add(shard.attributed_ns[index].swap(0, Ordering::Relaxed))
1614 })
1615 }),
1616 }
1617 }
1618
1619 fn take_input_totals(&self) -> (u64, u64) {
1620 if !self.inner.session_recording {
1621 return (
1622 self.inner.input_bytes.swap(0, Ordering::Relaxed),
1623 self.inner.input_units.swap(0, Ordering::Relaxed),
1624 );
1625 }
1626 let shards = self.inner.sorted_shards();
1627 shards.iter().fold((0_u64, 0_u64), |totals, shard| {
1628 (
1629 totals
1630 .0
1631 .saturating_add(shard.legacy_input_bytes.swap(0, Ordering::Relaxed)),
1632 totals
1633 .1
1634 .saturating_add(shard.legacy_input_units.swap(0, Ordering::Relaxed)),
1635 )
1636 })
1637 }
1638
1639 pub(crate) fn take_session_input_totals(&self) -> (u64, u64) {
1640 let shards = self.inner.sorted_shards();
1641 shards.iter().fold((0_u64, 0_u64), |totals, shard| {
1642 (
1643 totals
1644 .0
1645 .saturating_add(shard.input_bytes.swap(0, Ordering::Relaxed)),
1646 totals
1647 .1
1648 .saturating_add(shard.input_units.swap(0, Ordering::Relaxed)),
1649 )
1650 })
1651 }
1652
1653 pub(crate) fn take_session_workload_totals(&self) -> (u64, u64) {
1654 let shards = self.inner.sorted_shards();
1655 shards.iter().fold((0_u64, 0_u64), |totals, shard| {
1656 (
1657 totals
1658 .0
1659 .saturating_add(shard.derived_decoder_bytes.swap(0, Ordering::Relaxed)),
1660 totals
1661 .1
1662 .saturating_add(shard.backend_dispatched_bytes.swap(0, Ordering::Relaxed)),
1663 )
1664 })
1665 }
1666
1667 fn record_batch_route(
1668 &self,
1669 workload_key_digest: &str,
1670 requested_backend: &str,
1671 selected_backend: &str,
1672 completed_backend: &str,
1673 recovered_from_backend: Option<&str>,
1674 ) {
1675 if !self.inner.session_recording {
1676 return;
1677 }
1678 let batch_sequence = self
1679 .inner
1680 .session_route_sequence
1681 .fetch_add(1, Ordering::Relaxed);
1682 self.record_event(crate::EventId::BackendBatchCompleted, batch_sequence);
1683 if recovered_from_backend.is_some() {
1684 self.record_event(crate::EventId::BackendRecovered, batch_sequence);
1685 }
1686 let record = BatchRouteV2 {
1687 version: 1,
1688 batch_sequence,
1689 workload_key_digest: workload_key_digest.to_owned(),
1690 requested_backend: requested_backend.to_owned(),
1691 selected_backend: selected_backend.to_owned(),
1692 completed_backend: completed_backend.to_owned(),
1693 recovered_from_backend: recovered_from_backend.map_or_else(
1694 || Evidence::unavailable(crate::schema_v2::EvidenceGap::Unavailable),
1695 |backend| Evidence::recorded(backend.to_owned()),
1696 ),
1697 };
1698 let mut records = match self.inner.session_batch_routes.lock() {
1699 Ok(records) => records,
1700 Err(poisoned) => poisoned.into_inner(),
1701 };
1702 if records.len() >= MAX_BATCH_ROUTES {
1703 self.inner
1704 .session_dropped_batch_routes
1705 .fetch_add(1, Ordering::Relaxed);
1706 return;
1707 }
1708 records.push(record);
1709 }
1710
1711 pub fn take_session_batch_routes(&self) -> Vec<BatchRouteV2> {
1713 let mut records = match self.inner.session_batch_routes.lock() {
1714 Ok(records) => records,
1715 Err(poisoned) => poisoned.into_inner(),
1716 };
1717 let mut drained = std::mem::take(&mut *records);
1718 drained.sort_unstable_by_key(|record| record.batch_sequence);
1719 drained
1720 }
1721
1722 pub fn take_session_dropped_batch_routes(&self) -> u64 {
1724 self.inner
1725 .session_dropped_batch_routes
1726 .swap(0, Ordering::Relaxed)
1727 }
1728
1729 fn record_queue_enqueue(&self, queue: crate::QueueId, sequence: u64) {
1730 if !self.inner.session_recording {
1731 return;
1732 }
1733 let enqueue = PendingQueueEnqueue {
1734 thread_id: numeric_thread_id(),
1735 elapsed_ns: self.elapsed_ns(),
1736 };
1737 let mut pending = match self.inner.queue_pending.lock() {
1738 Ok(pending) => pending,
1739 Err(poisoned) => poisoned.into_inner(),
1740 };
1741 if pending.len() == MAX_QUEUE_LINKS && !pending.contains_key(&(queue as u8, sequence)) {
1742 self.inner
1743 .queue_dropped_enqueues
1744 .fetch_add(1, Ordering::Relaxed);
1745 return;
1746 }
1747 if pending.insert((queue as u8, sequence), enqueue).is_some() {
1748 self.inner
1750 .queue_dropped_enqueues
1751 .fetch_add(1, Ordering::Relaxed);
1752 }
1753 }
1754
1755 fn record_queue_dequeue(&self, queue: crate::QueueId, sequence: u64) {
1756 if !self.inner.session_recording {
1757 return;
1758 }
1759 let enqueue = {
1760 let mut pending = match self.inner.queue_pending.lock() {
1761 Ok(pending) => pending,
1762 Err(poisoned) => poisoned.into_inner(),
1763 };
1764 pending.remove(&(queue as u8, sequence))
1765 };
1766 let Some(enqueue) = enqueue else {
1767 self.inner
1768 .queue_unmatched_dequeues
1769 .fetch_add(1, Ordering::Relaxed);
1770 return;
1771 };
1772 let link = QueueLinkV2 {
1773 version: 1,
1774 queue,
1775 sequence,
1776 producer_thread_id: enqueue.thread_id,
1777 producer_elapsed_ns: enqueue.elapsed_ns,
1778 consumer_thread_id: numeric_thread_id(),
1779 consumer_elapsed_ns: self.elapsed_ns(),
1780 };
1781 let mut links = match self.inner.queue_links.lock() {
1782 Ok(links) => links,
1783 Err(poisoned) => poisoned.into_inner(),
1784 };
1785 if links.len() == MAX_QUEUE_LINKS {
1786 self.inner
1787 .queue_dropped_links
1788 .fetch_add(1, Ordering::Relaxed);
1789 } else {
1790 links.push(link);
1791 }
1792 }
1793
1794 pub fn take_session_queue_links(&self) -> (Vec<QueueLinkV2>, QueueLinkLossCounts) {
1796 let mut links = match self.inner.queue_links.lock() {
1797 Ok(links) => links,
1798 Err(poisoned) => poisoned.into_inner(),
1799 };
1800 let mut links = std::mem::take(&mut *links);
1801 links.sort_unstable_by_key(|link| (link.queue, link.sequence));
1802 let unconsumed = {
1803 let mut pending = match self.inner.queue_pending.lock() {
1804 Ok(pending) => pending,
1805 Err(poisoned) => poisoned.into_inner(),
1806 };
1807 let pending = std::mem::take(&mut *pending);
1808 u64::try_from(pending.len()).unwrap_or(u64::MAX)
1809 };
1810 let loss = QueueLinkLossCounts {
1811 dropped_enqueues: self.inner.queue_dropped_enqueues.swap(0, Ordering::Relaxed),
1812 dropped_links: self.inner.queue_dropped_links.swap(0, Ordering::Relaxed),
1813 unmatched_dequeues: self
1814 .inner
1815 .queue_unmatched_dequeues
1816 .swap(0, Ordering::Relaxed),
1817 unconsumed_enqueues: unconsumed,
1818 };
1819 (links, loss)
1820 }
1821
1822 fn queue_depth_enqueue(&self, queue: crate::QueueId) {
1823 if !self.inner.session_recording {
1824 return;
1825 }
1826 let index = queue.index();
1827 let depth = self.inner.queue_depth_current[index].fetch_add(1, Ordering::Relaxed) + 1;
1828 self.inner.queue_depth_high_water[index].fetch_max(depth, Ordering::Relaxed);
1829 self.inner.queue_depth_enqueues[index].fetch_add(1, Ordering::Relaxed);
1830 }
1831
1832 fn queue_depth_dequeue(&self, queue: crate::QueueId) {
1833 if !self.inner.session_recording {
1834 return;
1835 }
1836 let index = queue.index();
1837 let counter = &self.inner.queue_depth_current[index];
1838 let mut current = counter.load(Ordering::Relaxed);
1839 loop {
1840 if current == 0 {
1841 break;
1842 }
1843 match counter.compare_exchange_weak(
1844 current,
1845 current - 1,
1846 Ordering::Relaxed,
1847 Ordering::Relaxed,
1848 ) {
1849 Ok(_) => break,
1850 Err(observed) => current = observed,
1851 }
1852 }
1853 self.inner.queue_depth_dequeues[index].fetch_add(1, Ordering::Relaxed);
1854 }
1855
1856 fn set_queue_depth(&self, queue: crate::QueueId, depth: u64) {
1857 if !self.inner.session_recording {
1858 return;
1859 }
1860 let index = queue.index();
1861 self.inner.queue_depth_current[index].store(depth, Ordering::Relaxed);
1862 self.inner.queue_depth_high_water[index].fetch_max(depth, Ordering::Relaxed);
1863 }
1864
1865 pub fn take_session_queue_depths(&self) -> Vec<QueueDepthV2> {
1870 if !self.inner.session_recording {
1871 return Vec::new();
1872 }
1873 crate::QueueId::ALL
1874 .into_iter()
1875 .filter_map(|queue| {
1876 let index = queue.index();
1877 let current = self.inner.queue_depth_current[index].load(Ordering::Relaxed);
1878 let high_water =
1879 self.inner.queue_depth_high_water[index].swap(current, Ordering::Relaxed);
1880 let enqueues = self.inner.queue_depth_enqueues[index].swap(0, Ordering::Relaxed);
1881 let dequeues = self.inner.queue_depth_dequeues[index].swap(0, Ordering::Relaxed);
1882 (current != 0 || high_water != 0 || enqueues != 0 || dequeues != 0).then_some(
1883 QueueDepthV2 {
1884 version: 1,
1885 queue,
1886 current,
1887 high_water,
1888 enqueues,
1889 dequeues,
1890 },
1891 )
1892 })
1893 .collect()
1894 }
1895
1896 pub fn take_session_worker_imbalance(&self) -> WorkerImbalanceV2 {
1901 let shards = self.inner.sorted_shards();
1902 let workers: Vec<WorkerLoadV2> = shards
1903 .iter()
1904 .map(|shard| {
1905 let mut calls = 0_u64;
1906 let mut elapsed_ns = 0_u64;
1907 for index in 0..STAGE_COUNT {
1908 calls = calls.saturating_add(shard.calls[index].swap(0, Ordering::Relaxed));
1909 elapsed_ns = elapsed_ns
1910 .saturating_add(shard.elapsed_ns[index].swap(0, Ordering::Relaxed));
1911 }
1912 WorkerLoadV2 {
1913 version: 1,
1914 worker_id: shard.sequence,
1915 calls,
1916 elapsed_ns,
1917 }
1918 })
1919 .collect();
1920 let worker_count = u64::try_from(workers.len()).unwrap_or(u64::MAX);
1921 let total_calls = workers
1922 .iter()
1923 .fold(0_u64, |total, worker| total.saturating_add(worker.calls));
1924 let total_elapsed_ns = workers.iter().fold(0_u64, |total, worker| {
1925 total.saturating_add(worker.elapsed_ns)
1926 });
1927 let ppm = |part: u64, whole: u64| -> u64 {
1928 if whole == 0 {
1929 0
1930 } else {
1931 u64::try_from((u128::from(part) * 1_000_000) / u128::from(whole))
1932 .unwrap_or(u64::MAX)
1933 }
1934 };
1935 let max_calls = workers.iter().map(|worker| worker.calls).max().unwrap_or(0);
1936 let mut sorted_calls: Vec<u64> = workers.iter().map(|worker| worker.calls).collect();
1937 sorted_calls.sort_unstable();
1938 let median_calls = if sorted_calls.is_empty() {
1939 0
1940 } else {
1941 sorted_calls[sorted_calls.len() / 2]
1942 };
1943 let idle_workers = workers.iter().filter(|worker| worker.calls == 0).count() as u64;
1944 WorkerImbalanceV2 {
1945 version: 1,
1946 worker_count,
1947 total_calls,
1948 total_elapsed_ns,
1949 max_share_ppm: ppm(max_calls, total_calls),
1950 median_share_ppm: ppm(median_calls, total_calls),
1951 idle_share_ppm: ppm(idle_workers, worker_count),
1952 workers,
1953 }
1954 }
1955
1956 pub fn take_session_blocked_waits(&self) -> Vec<BlockedWaitRecordV2> {
1958 let shards = self.inner.sorted_shards();
1959 Stage::ALL
1960 .into_iter()
1961 .filter_map(|stage| {
1962 let index = stage.index();
1963 let calls = shards.iter().fold(0_u64, |total, shard| {
1964 total.saturating_add(shard.blocked_calls[index].swap(0, Ordering::Relaxed))
1965 });
1966 if calls == 0 {
1967 return None;
1968 }
1969 let blocked_ns = shards.iter().fold(0_u64, |total, shard| {
1970 total.saturating_add(shard.blocked_ns[index].swap(0, Ordering::Relaxed))
1971 });
1972 Some(BlockedWaitRecordV2 {
1973 version: 1,
1974 metric_id: stage.metric_id(),
1975 macro_stage_id: stage.macro_stage_id(),
1976 calls,
1977 blocked_ns,
1978 })
1979 })
1980 .collect()
1981 }
1982
1983 fn offset_ns(&self, at: Instant) -> u64 {
1985 u64::try_from(
1986 at.checked_duration_since(self.inner.started)
1987 .unwrap_or_default()
1988 .as_nanos(),
1989 )
1990 .unwrap_or(u64::MAX)
1991 }
1992
1993 pub fn take_session_stage_concurrency(&self) -> Vec<StageConcurrencyV2> {
1998 let shards = self.inner.sorted_shards();
1999 Stage::ALL
2000 .into_iter()
2001 .filter_map(|stage| {
2002 let index = stage.index();
2003 let mut calls = 0_u64;
2004 let mut elapsed_ns = 0_u64;
2005 let mut max_worker_elapsed_ns = 0_u64;
2006 let mut worker_count = 0_u64;
2007 let mut first_start_ns = u64::MAX;
2008 let mut last_end_ns = 0_u64;
2009 let mut declared_serial_ns = 0_u64;
2010 let mut declared_serial_calls = 0_u64;
2011 let mut bytes = 0_u64;
2012 for shard in &shards {
2013 let shard_calls = shard.calls[index].load(Ordering::Relaxed);
2014 let shard_elapsed = shard.elapsed_ns[index].load(Ordering::Relaxed);
2015 if shard_calls != 0 {
2016 worker_count += 1;
2017 }
2018 calls = calls.saturating_add(shard_calls);
2019 elapsed_ns = elapsed_ns.saturating_add(shard_elapsed);
2020 max_worker_elapsed_ns = max_worker_elapsed_ns.max(shard_elapsed);
2021 first_start_ns = first_start_ns
2022 .min(shard.stage_first_start_ns[index].load(Ordering::Relaxed));
2023 last_end_ns =
2024 last_end_ns.max(shard.stage_last_end_ns[index].load(Ordering::Relaxed));
2025 declared_serial_ns = declared_serial_ns
2026 .saturating_add(shard.serial_ns[index].swap(0, Ordering::Relaxed));
2027 declared_serial_calls = declared_serial_calls
2028 .saturating_add(shard.serial_calls[index].swap(0, Ordering::Relaxed));
2029 bytes =
2030 bytes.saturating_add(shard.stage_bytes[index].swap(0, Ordering::Relaxed));
2031 }
2032 if calls == 0 {
2033 return None;
2034 }
2035 let first_start_ns = if first_start_ns == u64::MAX {
2036 0
2037 } else {
2038 first_start_ns
2039 };
2040 let window_ns = last_end_ns.saturating_sub(first_start_ns);
2041 let ceiling_milli = worker_count.saturating_mul(1_000);
2049 let concurrency_milli = if window_ns == 0 {
2050 1_000
2051 } else {
2052 u64::try_from((u128::from(elapsed_ns) * 1_000) / u128::from(window_ns))
2053 .unwrap_or(u64::MAX)
2054 .min(ceiling_milli)
2055 };
2056 Some(StageConcurrencyV2 {
2057 version: crate::schema_v2::STAGE_CONCURRENCY_V2_VERSION,
2058 metric_id: stage.metric_id(),
2059 macro_stage_id: stage.macro_stage_id(),
2060 calls,
2061 elapsed_ns,
2062 window_ns,
2063 first_start_ns,
2064 last_end_ns,
2065 worker_count,
2066 max_worker_elapsed_ns,
2067 concurrency_milli,
2068 declared_serial_ns,
2069 declared_serial_calls,
2070 bytes,
2071 })
2072 })
2073 .collect()
2074 }
2075
2076 pub fn take_session_worker_occupancy(&self) -> WorkerOccupancyV2 {
2078 let shards = self.inner.sorted_shards();
2079 let workers: Vec<WorkerOccupancyRowV2> = shards
2080 .iter()
2081 .map(|shard| WorkerOccupancyRowV2 {
2082 version: crate::schema_v2::WORKER_OCCUPANCY_V2_VERSION,
2083 worker_id: shard.sequence,
2084 busy_ns: shard.top_level_busy_ns.swap(0, Ordering::Relaxed),
2085 blocked_ns: shard.top_level_blocked_ns.swap(0, Ordering::Relaxed),
2086 calls: shard.top_level_calls.swap(0, Ordering::Relaxed),
2087 })
2088 .collect();
2089 let busy_ns = workers
2090 .iter()
2091 .fold(0_u64, |total, worker| total.saturating_add(worker.busy_ns));
2092 let blocked_ns = workers.iter().fold(0_u64, |total, worker| {
2093 total.saturating_add(worker.blocked_ns)
2094 });
2095 let calls = workers
2096 .iter()
2097 .fold(0_u64, |total, worker| total.saturating_add(worker.calls));
2098 let mut sorted_busy: Vec<u64> = workers.iter().map(|worker| worker.busy_ns).collect();
2099 sorted_busy.sort_unstable();
2100 let median_busy_ns = sorted_busy
2101 .get(sorted_busy.len() / 2)
2102 .copied()
2103 .unwrap_or_default();
2104 WorkerOccupancyV2 {
2105 version: crate::schema_v2::WORKER_OCCUPANCY_V2_VERSION,
2106 worker_count: u64::try_from(workers.len()).unwrap_or(u64::MAX),
2107 active_worker_count: workers.iter().filter(|worker| worker.calls != 0).count() as u64,
2108 busy_ns,
2109 blocked_ns,
2110 calls,
2111 busiest_busy_ns: sorted_busy.last().copied().unwrap_or_default(),
2112 median_busy_ns,
2113 workers,
2114 }
2115 }
2116
2117 pub fn take_session_cache_effectiveness(&self) -> Vec<CacheEffectivenessV2> {
2119 let shards = self.inner.sorted_shards();
2120 crate::CacheId::ALL
2121 .into_iter()
2122 .filter_map(|cache| {
2123 let index = cache.index();
2124 let mut hits = 0_u64;
2125 let mut misses = 0_u64;
2126 for shard in &shards {
2127 hits = hits.saturating_add(shard.cache_hits[index].swap(0, Ordering::Relaxed));
2128 misses =
2129 misses.saturating_add(shard.cache_misses[index].swap(0, Ordering::Relaxed));
2130 }
2131 let total = hits.saturating_add(misses);
2132 if total == 0 {
2133 return None;
2134 }
2135 Some(CacheEffectivenessV2 {
2136 version: crate::schema_v2::CACHE_EFFECTIVENESS_V2_VERSION,
2137 cache,
2138 hits,
2139 misses,
2140 hit_rate_ppm: u64::try_from((u128::from(hits) * 1_000_000) / u128::from(total))
2141 .unwrap_or(u64::MAX),
2142 })
2143 })
2144 .collect()
2145 }
2146
2147 pub fn take_session_retries(&self) -> Vec<RetryRecordV2> {
2149 let shards = self.inner.sorted_shards();
2150 crate::RetryCause::ALL
2151 .into_iter()
2152 .filter_map(|cause| {
2153 let attempts = shards.iter().fold(0_u64, |total, shard| {
2154 total.saturating_add(shard.retries[cause.index()].swap(0, Ordering::Relaxed))
2155 });
2156 (attempts != 0).then_some(RetryRecordV2 {
2157 version: crate::schema_v2::RETRY_RECORD_V2_VERSION,
2158 cause,
2159 attempts,
2160 })
2161 })
2162 .collect()
2163 }
2164
2165 pub fn take_session_indexed_counters(&self) -> Vec<IndexedCounterRecordV2> {
2167 let shards = self.inner.sorted_shards();
2168 let dropped_out_of_range = shards.iter().fold(0_u64, |total, shard| {
2169 total.saturating_add(shard.indexed_counter_dropped.swap(0, Ordering::Relaxed))
2170 });
2171 crate::IndexedCounterId::ALL
2172 .into_iter()
2173 .filter_map(|counter| {
2174 let index = counter.index();
2175 let mut slots = [0_u64; crate::INDEXED_COUNTER_SLOTS];
2176 for shard in &shards {
2177 for (slot, total) in slots.iter_mut().enumerate() {
2178 *total = total.saturating_add(
2179 shard.indexed_counters[index][slot].swap(0, Ordering::Relaxed),
2180 );
2181 }
2182 }
2183 if slots.iter().all(|value| *value == 0) && dropped_out_of_range == 0 {
2184 return None;
2185 }
2186 Some(IndexedCounterRecordV2 {
2187 version: crate::schema_v2::INDEXED_COUNTER_V2_VERSION,
2188 counter,
2189 slots: slots.to_vec(),
2190 dropped_out_of_range,
2191 })
2192 })
2193 .collect()
2194 }
2195
2196 fn reset(&self) {
2203 let _ = self.drain_stage_counters(false);
2206 let _ = self.drain_stage_counters(true);
2207 let _ = self.take_session_worker_occupancy();
2208 let _ = self.take_session_blocked_waits();
2209 let _ = self.take_session_stage_concurrency();
2210 let _ = self.take_session_cache_effectiveness();
2211 let _ = self.take_session_indexed_counters();
2212 let _ = self.take_session_retries();
2213 let _ = self.take_session_queue_depths();
2214 self.inner.input_bytes.store(0, Ordering::Relaxed);
2215 self.inner.input_units.store(0, Ordering::Relaxed);
2216 for index in 0..crate::MetricId::COUNT {
2217 self.inner.legacy_typed_counters[index].store(0, Ordering::Relaxed);
2218 self.inner.distribution_min[index].store(u64::MAX, Ordering::Relaxed);
2219 self.inner.distribution_max[index].store(0, Ordering::Relaxed);
2220 for bucket in 0..LATENCY_BUCKET_COUNT {
2221 self.inner.distribution_buckets[index][bucket].store(0, Ordering::Relaxed);
2222 }
2223 }
2224 self.inner
2225 .active_compile_phase
2226 .store(crate::CompilePhase::Scan as u8, Ordering::Relaxed);
2227 for surface in 0..crate::CompileSurfaceId::COUNT {
2228 self.inner.legacy_compile_surface_loads[surface].store(0, Ordering::Relaxed);
2229 for phase in 0..crate::CompilePhase::COUNT {
2230 self.inner.legacy_compile_surface_invocations[surface][phase]
2231 .store(0, Ordering::Relaxed);
2232 }
2233 }
2234 for shard in self.inner.sorted_shards() {
2235 shard.input_bytes.store(0, Ordering::Relaxed);
2236 shard.input_units.store(0, Ordering::Relaxed);
2237 shard.legacy_input_bytes.store(0, Ordering::Relaxed);
2238 shard.legacy_input_units.store(0, Ordering::Relaxed);
2239 shard.derived_decoder_bytes.store(0, Ordering::Relaxed);
2240 shard.backend_dispatched_bytes.store(0, Ordering::Relaxed);
2241 for index in 0..crate::MetricId::COUNT {
2242 shard.counter_values[index].store(0, Ordering::Relaxed);
2243 }
2244 for index in 0..STAGE_COUNT {
2245 shard.latency_min_ns[index].store(u64::MAX, Ordering::Relaxed);
2246 shard.latency_max_ns[index].store(0, Ordering::Relaxed);
2247 shard.stage_first_start_ns[index].store(u64::MAX, Ordering::Relaxed);
2248 shard.stage_last_end_ns[index].store(0, Ordering::Relaxed);
2249 for bucket in 0..LATENCY_BUCKET_COUNT {
2250 shard.latency_buckets[index][bucket].store(0, Ordering::Relaxed);
2251 }
2252 }
2253 for surface in 0..crate::CompileSurfaceId::COUNT {
2254 shard.compile_surface_loads[surface].store(0, Ordering::Relaxed);
2255 for phase in 0..crate::CompilePhase::COUNT {
2256 shard.compile_surface_invocations[surface][phase].store(0, Ordering::Relaxed);
2257 }
2258 }
2259 }
2260 }
2261}
2262
2263impl Default for Runtime {
2264 fn default() -> Self {
2265 Self::new()
2266 }
2267}
2268
2269pub struct ContextGuard {
2271 runtime: Runtime,
2272 not_send: PhantomData<Rc<()>>,
2273}
2274
2275impl Drop for ContextGuard {
2276 fn drop(&mut self) {
2277 CURRENT.with(|stack| {
2278 let mut stack = stack.borrow_mut();
2279 if stack
2280 .last()
2281 .is_some_and(|runtime| Arc::ptr_eq(&runtime.inner, &self.runtime.inner))
2282 {
2283 stack.pop();
2284 } else if let Some(position) = stack
2285 .iter()
2286 .rposition(|runtime| Arc::ptr_eq(&runtime.inner, &self.runtime.inner))
2287 {
2288 stack.remove(position);
2289 }
2290 });
2291 ACTIVE_CONTEXTS.fetch_sub(1, Ordering::Relaxed);
2292 }
2293}
2294
2295struct AsyncParentGuard {
2296 runtime_key: usize,
2297 span_id: u64,
2298 stack_slot: usize,
2299 not_send: PhantomData<Rc<()>>,
2300}
2301
2302impl Drop for AsyncParentGuard {
2303 fn drop(&mut self) {
2304 ASYNC_PARENT_SPANS.with(|stack| {
2305 let mut stack = stack.borrow_mut();
2306 if stack[self.stack_slot].is_some_and(|active| {
2307 active.runtime_key == self.runtime_key && active.span_id == self.span_id
2308 }) {
2309 stack[self.stack_slot] = None;
2310 }
2311 });
2312 }
2313}
2314
2315struct WorkOriginGuard {
2316 previous: WorkOrigin,
2317}
2318
2319impl WorkOriginGuard {
2320 fn enter(origin: WorkOrigin) -> Self {
2321 Self {
2322 previous: WORK_ORIGIN.with(|slot| slot.replace(origin)),
2323 }
2324 }
2325}
2326
2327impl Drop for WorkOriginGuard {
2328 fn drop(&mut self) {
2329 WORK_ORIGIN.with(|slot| slot.set(self.previous));
2330 }
2331}
2332
2333struct TaskGuard {
2334 previous: u64,
2335}
2336
2337impl TaskGuard {
2338 fn enter(task_id: u64) -> Self {
2339 Self {
2340 previous: TASK_ID.with(|slot| slot.replace(task_id)),
2341 }
2342 }
2343}
2344
2345impl Drop for TaskGuard {
2346 fn drop(&mut self) {
2347 TASK_ID.with(|slot| slot.set(self.previous));
2348 }
2349}
2350
2351struct LegacyRuntime {
2352 runtime: Runtime,
2353 enabled: bool,
2354}
2355
2356impl LegacyRuntime {
2357 fn new() -> Self {
2358 Self {
2359 runtime: Runtime::legacy(),
2360 enabled: false,
2361 }
2362 }
2363}
2364
2365#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
2367#[repr(u8)]
2368pub enum Attribution {
2369 #[default]
2370 Root = 0,
2371 Decoded = 1,
2372}
2373
2374impl From<Attribution> for WorkOrigin {
2375 fn from(attribution: Attribution) -> Self {
2376 match attribution {
2377 Attribution::Root => Self::Root,
2378 Attribution::Decoded => Self::Decoded,
2379 }
2380 }
2381}
2382
2383thread_local! {
2384 static CURRENT: RefCell<Vec<Runtime>> = const { RefCell::new(Vec::new()) };
2385 static WORK_ORIGIN: Cell<WorkOrigin> = const { Cell::new(WorkOrigin::Root) };
2386 static TASK_ID: Cell<u64> = const { Cell::new(0) };
2387 static SPAN_DEPTH: Cell<u32> = const { Cell::new(0) };
2388 static LEGACY: RefCell<LegacyRuntime> = RefCell::new(LegacyRuntime::new());
2389 static ACTIVE_SPANS: RefCell<[Option<ActiveSpan>; MAX_NESTED_SPANS]> =
2390 const { RefCell::new([None; MAX_NESTED_SPANS]) };
2391 static ASYNC_PARENT_SPANS: RefCell<[Option<ActiveSpan>; MAX_NESTED_SPANS]> =
2392 const { RefCell::new([None; MAX_NESTED_SPANS]) };
2393 static NUMERIC_THREAD_ID: u64 = NEXT_THREAD_ID.fetch_add(1, Ordering::Relaxed);
2394 static THREAD_SHARDS: RefCell<Vec<ThreadShardAssignment>> = const { RefCell::new(Vec::new()) };
2395}
2396
2397fn numeric_thread_id() -> u64 {
2398 NUMERIC_THREAD_ID.with(|thread_id| *thread_id)
2399}
2400
2401pub fn current_runtime() -> Option<Runtime> {
2403 if ACTIVE_CONTEXTS.load(Ordering::Relaxed) == 0 {
2404 return None;
2405 }
2406 CURRENT
2407 .with(|stack| stack.borrow().last().cloned())
2408 .or_else(|| {
2409 LEGACY.with(|legacy| {
2410 let legacy = legacy.borrow();
2411 legacy.enabled.then(|| legacy.runtime.clone())
2412 })
2413 })
2414}
2415
2416pub(crate) fn runtime_for_drain() -> Runtime {
2417 CURRENT
2418 .with(|stack| stack.borrow().last().cloned())
2419 .unwrap_or_else(|| LEGACY.with(|legacy| legacy.borrow().runtime.clone()))
2420}
2421
2422pub fn set_attribution(attribution: Attribution) -> Attribution {
2427 let previous = set_work_origin(attribution.into());
2428 if previous.is_attributed_work() {
2429 Attribution::Decoded
2430 } else {
2431 Attribution::Root
2432 }
2433}
2434
2435pub fn set_work_origin(origin: WorkOrigin) -> WorkOrigin {
2437 WORK_ORIGIN.with(|slot| slot.replace(origin))
2438}
2439
2440pub fn current_work_origin() -> WorkOrigin {
2442 WORK_ORIGIN.with(|slot| slot.get())
2443}
2444
2445pub fn set_task_id(task_id: u64) -> u64 {
2450 TASK_ID.with(|slot| slot.replace(task_id))
2451}
2452
2453pub fn current_task_id() -> u64 {
2455 TASK_ID.with(|slot| slot.get())
2456}
2457
2458fn evidence_or_unavailable(value: u64) -> Evidence<u64> {
2459 if value == 0 {
2460 Evidence::unavailable(EvidenceGap::Unavailable)
2461 } else {
2462 Evidence::recorded(value)
2463 }
2464}
2465
2466pub fn current_causal_parent() -> Option<CausalParent> {
2470 current_runtime().map(|runtime| runtime.causal_parent())
2471}
2472
2473#[inline]
2475pub fn enabled() -> bool {
2476 if ACTIVE_CONTEXTS.load(Ordering::Relaxed) == 0 {
2477 return false;
2478 }
2479 CURRENT.with(|stack| !stack.borrow().is_empty())
2480 || LEGACY.with(|legacy| legacy.borrow().enabled)
2481}
2482
2483pub fn set_enabled(enabled: bool) {
2489 LEGACY.with(|legacy| {
2490 let mut legacy = legacy.borrow_mut();
2491 if legacy.enabled == enabled {
2492 return;
2493 }
2494 legacy.enabled = enabled;
2495 if enabled {
2496 ACTIVE_CONTEXTS.fetch_add(1, Ordering::Relaxed);
2497 } else {
2498 ACTIVE_CONTEXTS.fetch_sub(1, Ordering::Relaxed);
2499 }
2500 });
2501}
2502
2503struct AsyncSpan {
2504 runtime: Option<Runtime>,
2505 shard: Option<Arc<WorkerShard>>,
2506 stage: Stage,
2507 started: Option<Instant>,
2508 trace: Option<SpanTrace>,
2509}
2510
2511impl Drop for AsyncSpan {
2512 fn drop(&mut self) {
2513 let (Some(runtime), Some(started)) = (&self.runtime, self.started) else {
2514 return;
2515 };
2516 let elapsed_ns = u64::try_from(started.elapsed().as_nanos()).unwrap_or(u64::MAX);
2517 runtime.record(
2518 self.shard.as_deref(),
2519 self.stage,
2520 SpanOutcome {
2521 start_offset_ns: runtime.offset_ns(started),
2522 elapsed_ns,
2523 self_ns: 0,
2524 blocked: false,
2525 serial: false,
2526 outermost: false,
2529 },
2530 );
2531 if let Some(trace) = self.trace {
2532 runtime.finish_span(trace, elapsed_ns);
2533 }
2534 }
2535}
2536
2537fn instrument_impl<F>(
2538 stage: Stage,
2539 future: F,
2540 parent_override: Option<CausalParent>,
2541) -> impl Future<Output = F::Output>
2542where
2543 F: Future,
2544{
2545 let runtime = current_runtime();
2546 let shard = runtime.as_ref().and_then(Runtime::worker_shard);
2547 let worker_id = shard.as_ref().map_or(0, |shard| shard.sequence);
2548 let started = runtime.as_ref().map(|_| Instant::now());
2549 let parent_span_id = match (runtime.as_ref(), parent_override) {
2550 (Some(runtime), Some(parent)) if runtime.context_id() == parent.context_id() => {
2551 parent.span_id()
2552 }
2553 (Some(_), Some(_)) => 0,
2554 (Some(runtime), None) => runtime.current_parent_span_id(),
2555 (None, _) => 0,
2556 };
2557 let trace = runtime
2558 .as_ref()
2559 .zip(started)
2560 .and_then(|(runtime, started)| {
2561 runtime.begin_async_span(stage, started, parent_span_id, worker_id)
2562 });
2563 let span_id = trace.map(|trace| trace.span_id);
2564 let origin = current_work_origin();
2565 let task_id = current_task_id();
2566 let poll_runtime = runtime.clone();
2567
2568 async move {
2569 let _span = AsyncSpan {
2570 runtime,
2571 shard,
2572 stage,
2573 started,
2574 trace,
2575 };
2576 let mut future = std::pin::pin!(future);
2577 poll_fn(|context| {
2578 let _context_guard = poll_runtime.as_ref().map(Runtime::enter);
2579 let _parent_guard = poll_runtime
2580 .as_ref()
2581 .zip(span_id)
2582 .and_then(|(runtime, span_id)| runtime.enter_async_parent(span_id));
2583 let _origin_guard = WorkOriginGuard::enter(origin);
2584 let _task_guard = TaskGuard::enter(task_id);
2585 if span_id.is_some() {
2586 crate::allocation::stage_context_push(stage);
2587 }
2588 let result = future.as_mut().poll(context);
2589 if span_id.is_some() {
2590 crate::allocation::stage_context_pop();
2591 }
2592 result
2593 })
2594 .await
2595 }
2596}
2597
2598pub fn instrument_future<F>(stage: Stage, future: F) -> impl Future<Output = F::Output>
2604where
2605 F: Future,
2606{
2607 instrument_impl(stage, future, None)
2608}
2609
2610pub fn instrument_future_with_parent<F>(
2616 parent: CausalParent,
2617 stage: Stage,
2618 future: F,
2619) -> impl Future<Output = F::Output>
2620where
2621 F: Future,
2622{
2623 instrument_impl(stage, future, Some(parent))
2624}
2625
2626#[must_use]
2628pub struct Span {
2629 runtime: Option<Runtime>,
2630 shard: Option<Arc<WorkerShard>>,
2631 stage: Stage,
2632 started: Option<Instant>,
2633 trace: Option<SpanTrace>,
2634 stack_slot: Option<usize>,
2635 span_id: u64,
2636 blocked: bool,
2637 serial: bool,
2638}
2639
2640impl Span {
2641 pub fn is_recording(&self) -> bool {
2643 self.started.is_some()
2644 }
2645}
2646
2647fn span_impl(
2648 stage: Stage,
2649 parent_override: Option<CausalParent>,
2650 blocked: bool,
2651 serial: bool,
2652) -> Span {
2653 if ACTIVE_CONTEXTS.load(Ordering::Relaxed) == 0 {
2654 return Span {
2655 runtime: None,
2656 shard: None,
2657 stage,
2658 started: None,
2659 trace: None,
2660 stack_slot: None,
2661 span_id: 0,
2662 blocked: false,
2663 serial: false,
2664 };
2665 }
2666 let runtime = current_runtime();
2667 let shard = runtime.as_ref().and_then(Runtime::worker_shard);
2668 let worker_id = shard.as_ref().map_or(0, |shard| shard.sequence);
2669 let started = runtime.as_ref().map(|_| Instant::now());
2670 if started.is_some() {
2671 SPAN_DEPTH.with(|depth| depth.set(depth.get() + 1));
2672 }
2673 let (trace, stack_slot, span_id) = match (runtime.as_ref(), started) {
2674 (Some(runtime), Some(started)) => {
2675 let (parent_slot, current_parent_id) = runtime.current_parent();
2676 let parent_span_id = match parent_override {
2677 Some(parent) if runtime.context_id() == parent.context_id() => parent.span_id(),
2678 _ => current_parent_id,
2679 };
2680 runtime.begin_span_with(stage, started, parent_span_id, parent_slot, worker_id)
2681 }
2682 _ => (None, None, 0),
2683 };
2684 if trace.is_some() {
2685 crate::allocation::stage_context_push(stage);
2686 }
2687 Span {
2688 runtime,
2689 shard,
2690 stage,
2691 started,
2692 trace,
2693 stack_slot,
2694 span_id,
2695 blocked,
2696 serial,
2697 }
2698}
2699
2700#[inline]
2702pub fn span(stage: Stage) -> Span {
2703 span_impl(stage, None, false, false)
2704}
2705
2706pub fn span_with_parent(parent: CausalParent, stage: Stage) -> Span {
2712 span_impl(stage, Some(parent), false, false)
2713}
2714
2715pub fn serial_span(stage: Stage) -> Span {
2725 span_impl(stage, None, false, true)
2726}
2727
2728pub fn blocked(stage: Stage) -> Span {
2735 span_impl(stage, None, true, false)
2736}
2737
2738#[must_use = "a decision timer only measures when finished"]
2751pub struct DecisionTimer {
2752 stage: Stage,
2753 started: Instant,
2754}
2755
2756pub fn decision_timer(stage: Stage) -> DecisionTimer {
2758 DecisionTimer {
2759 stage,
2760 started: Instant::now(),
2761 }
2762}
2763
2764impl DecisionTimer {
2765 pub fn finish(self) -> std::time::Duration {
2767 let elapsed = self.started.elapsed();
2768 if ACTIVE_CONTEXTS.load(Ordering::Relaxed) != 0 {
2769 if let Some(runtime) = current_runtime() {
2770 let shard = runtime.worker_shard();
2771 runtime.record(
2772 shard.as_deref(),
2773 self.stage,
2774 SpanOutcome {
2775 start_offset_ns: runtime.offset_ns(self.started),
2776 elapsed_ns: u64::try_from(elapsed.as_nanos()).unwrap_or(u64::MAX),
2777 self_ns: 0,
2778 blocked: false,
2779 serial: false,
2780 outermost: false,
2783 },
2784 );
2785 }
2786 }
2787 elapsed
2788 }
2789}
2790
2791impl Drop for Span {
2792 #[inline]
2793 fn drop(&mut self) {
2794 let (Some(runtime), Some(started)) = (&self.runtime, self.started) else {
2795 return;
2796 };
2797 let outermost = SPAN_DEPTH.with(|depth| {
2798 let next = depth.get().saturating_sub(1);
2799 depth.set(next);
2800 next == 0
2801 });
2802 let elapsed_ns = u64::try_from(started.elapsed().as_nanos()).unwrap_or(u64::MAX);
2803 let self_ns = runtime.pop_active_span(self.stack_slot, self.span_id, elapsed_ns);
2804 runtime.record(
2805 self.shard.as_deref(),
2806 self.stage,
2807 SpanOutcome {
2808 start_offset_ns: runtime.offset_ns(started),
2809 elapsed_ns,
2810 self_ns,
2811 blocked: self.blocked,
2812 serial: self.serial,
2813 outermost,
2814 },
2815 );
2816 if let Some(trace) = self.trace {
2817 runtime.finish_span(trace, elapsed_ns);
2818 crate::allocation::stage_context_pop();
2819 }
2820 }
2821}
2822
2823#[must_use = "a counter span measures nothing until it is dropped"]
2830pub struct CounterSpan {
2831 counter: crate::CounterId,
2832 started: Option<Instant>,
2833}
2834
2835#[inline]
2837pub fn counter_span(counter: crate::CounterId) -> CounterSpan {
2838 CounterSpan {
2839 counter,
2840 started: (ACTIVE_CONTEXTS.load(Ordering::Relaxed) != 0).then(Instant::now),
2841 }
2842}
2843
2844impl Drop for CounterSpan {
2845 #[inline]
2846 fn drop(&mut self) {
2847 let Some(started) = self.started else {
2848 return;
2849 };
2850 add_counter(
2851 self.counter,
2852 u64::try_from(started.elapsed().as_nanos()).unwrap_or(u64::MAX),
2853 );
2854 }
2855}
2856
2857#[inline]
2863pub fn record_retry(cause: crate::RetryCause) {
2864 if let Some(runtime) = current_runtime() {
2865 runtime.record_retry(cause);
2866 }
2867}
2868
2869#[inline]
2874pub fn add_indexed_counter(counter: crate::IndexedCounterId, slot: u16, delta: u64) {
2875 if let Some(runtime) = current_runtime() {
2876 runtime.add_indexed_counter(counter, slot, delta);
2877 }
2878}
2879
2880#[inline]
2886pub fn add_stage_bytes(stage: crate::Stage, bytes: u64) {
2887 if let Some(runtime) = current_runtime() {
2888 runtime.add_stage_bytes(stage, bytes);
2889 }
2890}
2891
2892#[inline]
2894pub fn record_cache_hit(cache: crate::CacheId) {
2895 if let Some(runtime) = current_runtime() {
2896 runtime.record_cache_outcome(cache, true);
2897 }
2898}
2899
2900#[inline]
2902pub fn record_cache_miss(cache: crate::CacheId) {
2903 if let Some(runtime) = current_runtime() {
2904 runtime.record_cache_outcome(cache, false);
2905 }
2906}
2907
2908#[inline]
2910pub fn set_compile_phase(phase: crate::CompilePhase) {
2911 PROCESS_ACTIVE_COMPILE_PHASE.store(phase as u8, Ordering::Relaxed);
2912 if let Some(runtime) = current_runtime() {
2913 runtime.set_compile_phase(phase);
2914 }
2915}
2916
2917#[inline]
2919pub fn active_compile_phase() -> crate::CompilePhase {
2920 if let Some(runtime) = current_runtime() {
2921 runtime.active_compile_phase()
2922 } else {
2923 match PROCESS_ACTIVE_COMPILE_PHASE.load(Ordering::Relaxed) {
2924 0 => crate::CompilePhase::Install,
2925 1 => crate::CompilePhase::Update,
2926 2 => crate::CompilePhase::Scan,
2927 3 => crate::CompilePhase::Developer,
2928 _ => crate::CompilePhase::Scan,
2929 }
2930 }
2931}
2932
2933#[inline]
2935pub fn record_compile_surface_invocation(surface: crate::CompileSurfaceId) {
2936 if let Some(runtime) = current_runtime() {
2937 runtime.record_compile_surface_invocation(surface);
2938 }
2939}
2940
2941#[inline]
2943pub fn record_compile_surface_invocation_with_phase(
2944 surface: crate::CompileSurfaceId,
2945 phase: crate::CompilePhase,
2946) {
2947 if let Some(runtime) = current_runtime() {
2948 runtime.record_compile_surface_invocation_with_phase(surface, phase);
2949 }
2950}
2951
2952#[inline]
2954pub fn record_compile_surface_load(surface: crate::CompileSurfaceId) {
2955 if let Some(runtime) = current_runtime() {
2956 runtime.record_compile_surface_load(surface);
2957 }
2958}
2959
2960#[inline]
2962pub fn total_runtime_compiles() -> u64 {
2963 if let Some(runtime) = current_runtime() {
2964 runtime.total_runtime_compiles()
2965 } else {
2966 0
2967 }
2968}
2969
2970#[inline]
2972pub fn compile_surface_reports() -> Vec<crate::schema_v2::CompileSurfaceRecordV2> {
2973 if let Some(runtime) = current_runtime() {
2974 runtime.compile_surface_reports()
2975 } else {
2976 crate::CompileSurfaceId::ALL
2977 .iter()
2978 .map(|&surface| crate::schema_v2::CompileSurfaceRecordV2 {
2979 version: crate::schema_v2::COMPILE_SURFACE_RECORD_V2_VERSION,
2980 surface,
2981 name: surface.as_str().to_string(),
2982 runtime_compiles: 0,
2983 loads: 0,
2984 install_compiles: 0,
2985 update_compiles: 0,
2986 developer_compiles: 0,
2987 })
2988 .collect()
2989 }
2990}
2991
2992#[inline]
2994#[must_use]
2995pub fn cache_hits(cache: crate::CacheId) -> u64 {
2996 current_runtime().map_or(0, |runtime| runtime.cache_hits(cache))
2997}
2998
2999#[inline]
3001#[must_use]
3002pub fn cache_misses(cache: crate::CacheId) -> u64 {
3003 current_runtime().map_or(0, |runtime| runtime.cache_misses(cache))
3004}
3005
3006#[inline]
3008pub fn add_input_bytes(bytes: u64) {
3009 if let Some(runtime) = current_runtime() {
3010 runtime.add_input_bytes(bytes);
3011 }
3012}
3013
3014#[inline]
3016pub fn add_input_units(units: u64) {
3017 if let Some(runtime) = current_runtime() {
3018 runtime.add_input_units(units);
3019 }
3020}
3021
3022#[inline]
3024pub fn record_sampled_event(event: crate::EventId, value: u64, policy: SamplingPolicy) -> bool {
3025 current_runtime().is_some_and(|runtime| runtime.record_sampled_event(event, value, policy))
3026}
3027#[inline]
3029pub fn add_derived_decoder_bytes(bytes: u64) {
3030 if let Some(runtime) = current_runtime() {
3031 runtime.add_derived_decoder_bytes(bytes);
3032 }
3033}
3034
3035#[inline]
3037pub fn add_backend_dispatched_bytes(bytes: u64) {
3038 if let Some(runtime) = current_runtime() {
3039 runtime.add_backend_dispatched_bytes(bytes);
3040 }
3041}
3042
3043#[inline]
3045pub fn add_counter(counter: crate::CounterId, delta: u64) {
3046 if let Some(runtime) = current_runtime() {
3047 runtime.add_counter(counter, delta);
3048 }
3049}
3050
3051#[inline]
3053pub fn set_gauge(gauge: crate::GaugeId, value: u64) {
3054 if let Some(runtime) = current_runtime() {
3055 runtime.set_gauge(gauge, value);
3056 }
3057}
3058
3059#[inline]
3061pub fn record_event(event: crate::EventId, value: u64) {
3062 if let Some(runtime) = current_runtime() {
3063 runtime.record_event(event, value);
3064 }
3065}
3066
3067pub fn record_queue_enqueue(queue: crate::QueueId, sequence: u64) {
3072 if let Some(runtime) = current_runtime() {
3073 runtime.record_queue_enqueue(queue, sequence);
3074 }
3075}
3076
3077pub fn record_queue_dequeue(queue: crate::QueueId, sequence: u64) {
3081 if let Some(runtime) = current_runtime() {
3082 runtime.record_queue_dequeue(queue, sequence);
3083 }
3084}
3085
3086pub fn record_queue_depth_enqueue(queue: crate::QueueId) {
3088 if let Some(runtime) = current_runtime() {
3089 runtime.queue_depth_enqueue(queue);
3090 }
3091}
3092
3093pub fn record_queue_depth_dequeue(queue: crate::QueueId) {
3095 if let Some(runtime) = current_runtime() {
3096 runtime.queue_depth_dequeue(queue);
3097 }
3098}
3099
3100pub fn set_queue_depth(queue: crate::QueueId, depth: u64) {
3102 if let Some(runtime) = current_runtime() {
3103 runtime.set_queue_depth(queue, depth);
3104 }
3105}
3106
3107#[inline]
3109pub fn record_distribution(metric: crate::MetricId, value: u64) {
3110 if let Some(runtime) = current_runtime() {
3111 runtime.record_distribution(metric, value);
3112 }
3113}
3114
3115#[inline]
3118pub fn record_fs_open_latency_ns(elapsed_ns: u64) {
3119 record_distribution(crate::MetricId::FsOpenLatencyNs, elapsed_ns);
3120}
3121
3122#[inline]
3125pub fn record_fs_read_latency_ns(elapsed_ns: u64) {
3126 record_distribution(crate::MetricId::FsReadLatencyNs, elapsed_ns);
3127}
3128
3129#[inline]
3132pub fn record_fs_metadata_latency_ns(elapsed_ns: u64) {
3133 record_distribution(crate::MetricId::FsMetadataLatencyNs, elapsed_ns);
3134}
3135
3136#[inline]
3138pub fn record_network_latency_ns(elapsed_ns: u64) {
3139 record_distribution(crate::MetricId::NetworkLatencyNs, elapsed_ns);
3140}
3141
3142#[inline]
3145pub fn record_network_bytes(read_bytes: u64, written_bytes: u64) {
3146 if read_bytes > 0 {
3147 add_counter(crate::CounterId::NetworkBytesRead, read_bytes);
3148 }
3149 if written_bytes > 0 {
3150 add_counter(crate::CounterId::NetworkBytesWritten, written_bytes);
3151 }
3152}
3153
3154#[inline]
3156pub fn record_network_request() {
3157 add_counter(crate::CounterId::NetworkRequests, 1);
3158}
3159
3160#[inline]
3166pub fn record_io_cache_state(state: crate::IoCacheStateV2) {
3167 record_annotation(crate::AnnotationId::IoCacheState, state.as_value());
3168 let counter = match state {
3169 crate::IoCacheStateV2::Cold => crate::CounterId::PageCacheColdObservations,
3170 crate::IoCacheStateV2::Warm => crate::CounterId::PageCacheWarmObservations,
3171 crate::IoCacheStateV2::Direct => crate::CounterId::PageCacheDirectObservations,
3172 };
3173 add_counter(counter, 1);
3174}
3175
3176#[inline]
3179pub fn record_retained_buffer_bytes(bytes: u64) {
3180 if let Some(runtime) = current_runtime() {
3181 runtime.record_retained_buffer_bytes(bytes);
3182 }
3183}
3184
3185pub fn take_typed_metrics() -> Vec<TypedMetricRecordV2> {
3190 runtime_for_drain().take_legacy_typed_metrics()
3191}
3192
3193pub fn take_metric_distributions() -> Vec<MetricDistributionV2> {
3197 runtime_for_drain().take_metric_distributions()
3198}
3199
3200#[inline]
3202pub fn record_annotation(annotation: crate::AnnotationId, value: u64) {
3203 if let Some(runtime) = current_runtime() {
3204 runtime.record_annotation(annotation, value);
3205 }
3206}
3207
3208pub fn record_batch_route(
3213 workload_key_digest: &str,
3214 requested_backend: &str,
3215 selected_backend: &str,
3216 completed_backend: &str,
3217 recovered_from_backend: Option<&str>,
3218) {
3219 if let Some(runtime) = current_runtime() {
3220 runtime.record_batch_route(
3221 workload_key_digest,
3222 requested_backend,
3223 selected_backend,
3224 completed_backend,
3225 recovered_from_backend,
3226 );
3227 }
3228}
3229
3230pub fn take_input_totals() -> (u64, u64) {
3232 runtime_for_drain().take_input_totals()
3233}
3234
3235pub fn reset() {
3237 runtime_for_drain().reset();
3238}