vthread/handoff_profile.rs
1//! Bounded owner-local handoff evidence; clocks exist only in the opt-in build.
2
3use std::time::Duration;
4
5pub use crate::handoff_channel::{ChannelCounters, ChannelDirection};
6pub use crate::handoff_mutex::MutexCounters;
7
8/// Inclusive duration histogram bounds in nanoseconds; the last bin includes all
9/// larger observations. Clock overhead is not subtracted.
10pub const HANDOFF_DURATION_BOUNDS_NS: [u64; 10] = [
11 1_000,
12 5_000,
13 10_000,
14 25_000,
15 50_000,
16 100_000,
17 250_000,
18 1_000_000,
19 10_000_000,
20 u64::MAX,
21];
22
23/// A measured region, not a disjoint CPU-time category. Nested regions overlap;
24/// elapsed time includes preemption and must not be summed as CPU consumption.
25#[derive(
26 ::core::clone::Clone,
27 ::core::marker::Copy,
28 ::core::fmt::Debug,
29 ::core::cmp::PartialEq,
30 ::core::cmp::Eq,
31)]
32pub enum HandoffStage {
33 /// Complete idle-driver invocation, including completion flushing and observation.
34 IdleEpisode,
35 /// Polling that ends with a visible start or wake, not proof of a valid dispatch.
36 PollWork,
37 /// Polling that ends with a changed control epoch but no observed task work.
38 PollControl,
39 /// Polling that exhausts the unchanged bounded probe budget.
40 PollExhausted,
41 /// Idle snapshot publication before the wait API, including capacity scans.
42 IdlePublication,
43 /// Predicate-backed wait API, including locking, arming and any native waits.
44 WaitApi,
45 /// A condition-variable invocation including reacquisition of its native lock.
46 /// This does not prove that the OS descheduled the thread.
47 NativeWait,
48 /// Selected wait claim through route publication and final claim publication.
49 WakePublication,
50 /// A finish path that observed a claim/binding phase, through its exit.
51 ClaimFinish,
52 /// Acquisition of the channel metadata lock, excluding its held section.
53 ChannelLock,
54 /// Channel metadata critical section, excluding unlock and lock acquisition.
55 ChannelHeld,
56}
57
58impl HandoffStage {
59 /// All measured regions in stable histogram order.
60 pub const ALL: [Self; 11] = [
61 Self::IdleEpisode,
62 Self::PollWork,
63 Self::PollControl,
64 Self::PollExhausted,
65 Self::IdlePublication,
66 Self::WaitApi,
67 Self::NativeWait,
68 Self::WakePublication,
69 Self::ClaimFinish,
70 Self::ChannelLock,
71 Self::ChannelHeld,
72 ];
73}
74
75/// Fixed-size elapsed-time distribution. Counts are cumulative; total nanoseconds
76/// saturate at `u64::MAX`. Observations are diagnostic, not headline latency data.
77#[derive(
78 ::core::clone::Clone,
79 ::core::marker::Copy,
80 ::core::fmt::Debug,
81 ::core::default::Default,
82 ::core::cmp::PartialEq,
83 ::core::cmp::Eq,
84)]
85pub struct HandoffDuration {
86 count: u64,
87 total_ns: u64,
88 maximum_ns: u64,
89 bins: [u64; 10],
90}
91
92impl HandoffDuration {
93 pub(crate) fn record(&mut self, elapsed: Duration) {
94 let nanos = elapsed.as_nanos().min(u128::from(u64::MAX)) as u64;
95 let bin = HANDOFF_DURATION_BOUNDS_NS.partition_point(|bound| *bound < nanos);
96 self.count += 1;
97 self.total_ns = self.total_ns.saturating_add(nanos);
98 self.maximum_ns = self.maximum_ns.max(nanos);
99 self.bins[bin] += 1;
100 }
101
102 /// Number of completed measured regions.
103 pub fn count(&self) -> u64 {
104 self.count
105 }
106 /// Saturating cumulative elapsed nanoseconds, not CPU time.
107 pub fn total_ns(&self) -> u64 {
108 self.total_ns
109 }
110 /// Largest individual elapsed duration in nanoseconds.
111 pub fn maximum_ns(&self) -> u64 {
112 self.maximum_ns
113 }
114 /// Counts corresponding to [`HANDOFF_DURATION_BOUNDS_NS`].
115 pub fn bins(&self) -> &[u64; 10] {
116 &self.bins
117 }
118}
119
120/// Cumulative evidence attributed to the executing owner carrier. Native callers
121/// without a mounted carrier route are not counted. No new shared counter atomics
122/// or per-observation allocations are used. Active snapshots can lag; shutdown
123/// publishes final totals. Enabled clocks, TLS accesses and larger snapshots affect
124/// scheduling, so this feature cannot supply uninstrumented performance claims.
125#[derive(
126 ::core::clone::Clone,
127 ::core::marker::Copy,
128 ::core::fmt::Debug,
129 ::core::default::Default,
130 ::core::cmp::PartialEq,
131 ::core::cmp::Eq,
132)]
133pub struct HandoffProfile {
134 durations: [HandoffDuration; 11],
135 pub(crate) channels: [ChannelCounters; 2],
136 pub(crate) mutex: MutexCounters,
137 pub(crate) local_publications: u64,
138 pub(crate) remote_publications: u64,
139}
140
141impl HandoffProfile {
142 pub(crate) fn record(&mut self, stage: HandoffStage, elapsed: Duration) {
143 self.durations[stage as usize].record(elapsed);
144 }
145
146 /// Elapsed distribution for one possibly overlapping region.
147 pub fn duration(&self, stage: HandoffStage) -> &HandoffDuration {
148 &self.durations[stage as usize]
149 }
150 /// Channel operations and notifications initiated on this carrier.
151 /// Actual park crossings are counted on the same non-migrating owner.
152 pub fn channel(&self, direction: ChannelDirection) -> &ChannelCounters {
153 &self.channels[direction as usize]
154 }
155 /// Useful mutex acquisitions, ticket lifetimes and selected owner identities.
156 /// Source-side grants and recipient-side receipts may belong to different carriers.
157 pub fn mutex(&self) -> &MutexCounters {
158 &self.mutex
159 }
160 /// Selected notices routed through the source carrier's local queue.
161 pub fn local_publications(&self) -> u64 {
162 self.local_publications
163 }
164 /// Selected notices routed through a shared hub, including forced shared routing.
165 /// These are not necessarily distinct-carrier destinations or successful inserts.
166 pub fn remote_publications(&self) -> u64 {
167 self.remote_publications
168 }
169}
170
171#[cfg(test)]
172#[path = "handoff_profile_test.rs"]
173mod handoff_profile_test;