1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
use std::marker::PhantomData;
use base2histogram::Histogram;
use crate::Config;
use crate::RaftTypeConfig;
use crate::core::NotificationName;
use crate::core::raft_msg::RaftMsgName;
use crate::core::runtime_stats::DisplayMode;
use crate::core::runtime_stats::RuntimeStatsDisplay;
use crate::core::runtime_stats::log_stage::LogStageHistograms;
use crate::core::runtime_stats::log_stage::LogStages;
use crate::core::stage::Stage;
use crate::engine::CommandName;
#[cfg(feature = "runtime-stats")]
use crate::type_config::TypeConfigExt;
use crate::type_config::alias::InstantOf;
/// Runtime statistics for Raft operations.
///
/// This is a volatile structure that is not persisted. It accumulates
/// statistics from the time the Raft node starts.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct RuntimeStats<C>
where C: RaftTypeConfig
{
/// Histogram tracking the distribution of log entry counts in Apply commands.
///
/// This tracks how many log entries are included in each apply command sent
/// to the state machine, helping identify batch size patterns and I/O efficiency.
pub apply_batch: Histogram,
/// Histogram tracking the distribution of log entry counts when appending to storage.
///
/// This tracks how many log entries are included in each AppendEntries command
/// submitted to the storage layer, helping identify write batch patterns and storage I/O
/// efficiency.
pub append_batch: Histogram,
/// Histogram tracking the distribution of log entry counts in replication RPCs.
///
/// This tracks how many log entries are included in each AppendEntries RPC
/// sent to followers during replication, helping identify replication batch patterns.
pub replicate_batch: Histogram,
/// Histogram tracking the distribution of RaftMsg counts processed in a batch.
///
/// This tracks how many RaftMsg are processed together before calling
/// `run_engine_commands()`, helping identify message batching efficiency.
pub raft_msg_per_run: Histogram,
/// Histogram tracking the distribution of client write entries merged per batch.
///
/// This tracks how many client write entries are merged together in each
/// `RaftMsg::ClientWrite` after batching, helping identify batching efficiency.
pub write_batch: Histogram,
/// Histogram tracking the budget for RaftMsg processing.
///
/// This tracks the maximum number of RaftMsg allowed to process in each
/// `process_raft_msg()` call, helping understand load balancing behavior.
pub raft_msg_budget: Histogram,
/// Histogram tracking the budget for Notification processing.
///
/// This tracks the maximum number of Notifications allowed to process in each
/// `process_notification()` call, helping understand load balancing behavior.
pub notification_budget: Histogram,
/// Histogram tracking RaftMsg budget utilization in permille (0-1000).
///
/// This tracks `processed * 1000 / budget` for each `process_raft_msg()` call,
/// where 1000 means 100% utilization. Helps identify if the budget is well-tuned.
pub raft_msg_usage_permille: Histogram,
/// Histogram tracking Notification budget utilization in permille (0-1000).
///
/// This tracks `processed * 1000 / budget` for each `process_notification()` call,
/// where 1000 means 100% utilization. Helps identify if the budget is well-tuned.
pub notification_usage_permille: Histogram,
/// Count of each command type executed.
///
/// This tracks how many times each command type has been executed,
/// useful for understanding workload patterns and debugging.
/// Indexed by `CommandName::index()`.
pub command_counts: Vec<u64>,
/// Count of each RaftMsg type received.
///
/// This tracks how many times each RaftMsg type has been received,
/// useful for understanding API usage patterns and debugging.
/// Indexed by `stats::RaftMsgName::index()`.
pub raft_msg_counts: Vec<u64>,
/// Count of each Notification type received.
///
/// This tracks how many times each Notification type has been received,
/// useful for understanding internal message patterns and debugging.
/// Indexed by `NotificationName::index()`.
pub notification_counts: Vec<u64>,
/// Per-entry timestamp tracking across 6 lifecycle stages
/// (Proposed, Received, Submitted, Persisted, Committed, Applied).
///
/// Uses a fixed-capacity ring buffer per stage.
/// Stage-to-stage gaps reveal where latency accumulates
/// (channel queueing, storage I/O, replication, state machine apply).
pub log_stage: LogStages<InstantOf<C>>,
/// Precomputed stage-to-stage duration histograms derived from [`log_stage`](Self::log_stage).
///
/// Rebuilt on demand via [`build_log_stage_histograms()`](Self::build_log_stage_histograms).
pub log_stage_histograms: LogStageHistograms,
_phantom: PhantomData<C>,
}
impl<C> Default for RuntimeStats<C>
where C: RaftTypeConfig
{
fn default() -> Self {
Self::new(&Config::default())
}
}
impl<C> RuntimeStats<C>
where C: RaftTypeConfig
{
pub(crate) fn new(config: &Config) -> Self {
let _ = config;
Self {
apply_batch: Histogram::<()>::new(),
append_batch: Histogram::<()>::new(),
replicate_batch: Histogram::<()>::new(),
raft_msg_per_run: Histogram::<()>::new(),
write_batch: Histogram::<()>::new(),
raft_msg_budget: Histogram::<()>::new(),
notification_budget: Histogram::<()>::new(),
raft_msg_usage_permille: Histogram::<()>::new(),
notification_usage_permille: Histogram::<()>::new(),
command_counts: vec![0; CommandName::COUNT],
raft_msg_counts: vec![0; RaftMsgName::COUNT],
notification_counts: vec![0; NotificationName::COUNT],
log_stage: LogStages::new(config.log_stage_capacity(), 0),
log_stage_histograms: LogStageHistograms::new(),
_phantom: PhantomData,
}
}
/// Record the execution of a command.
pub fn record_command(&mut self, name: CommandName) {
self.command_counts[name.index()] += 1;
}
/// Record the receipt of a RaftMsg.
pub fn record_raft_msg(&mut self, name: RaftMsgName) {
self.raft_msg_counts[name.index()] += 1;
}
/// Record the receipt of a Notification.
pub fn record_notification(&mut self, name: NotificationName) {
self.notification_counts[name.index()] += 1;
}
pub fn record_log_stage_now(&mut self, stage: Stage, index: u64) {
#[cfg(feature = "runtime-stats")]
self.record_log_stage(stage, index, C::now());
#[cfg(not(feature = "runtime-stats"))]
{
let _ = (stage, index);
}
}
#[allow(dead_code)]
pub fn record_log_stage(&mut self, stage: Stage, index: u64, now: InstantOf<C>) {
#[cfg(feature = "runtime-stats")]
self.log_stage.record_stage(stage, index, now);
#[cfg(not(feature = "runtime-stats"))]
{
let _ = (stage, index, now);
}
}
#[allow(dead_code)]
pub fn build_log_stage_histograms(&mut self) {
#[cfg(feature = "runtime-stats")]
{
self.log_stage_histograms = self.log_stage.compute_histograms();
}
}
/// Returns a displayable representation of the runtime statistics.
///
/// All values are precomputed when calling this method, so the returned
/// `RuntimeStatsDisplay` can be cheaply formatted multiple times.
///
/// Use builder methods like `.human_readable()` to change the display mode.
#[allow(dead_code)]
pub fn display(&self) -> RuntimeStatsDisplay<C> {
RuntimeStatsDisplay {
mode: DisplayMode::default(),
apply_batch: self.apply_batch.percentile_stats(),
append_batch: self.append_batch.percentile_stats(),
replicate_batch: self.replicate_batch.percentile_stats(),
raft_msg_per_run: self.raft_msg_per_run.percentile_stats(),
write_batch: self.write_batch.percentile_stats(),
raft_msg_budget: self.raft_msg_budget.percentile_stats(),
notification_budget: self.notification_budget.percentile_stats(),
raft_msg_usage_permille: self.raft_msg_usage_permille.percentile_stats(),
notification_usage_permille: self.notification_usage_permille.percentile_stats(),
command_counts: self.command_counts.clone(),
raft_msg_counts: self.raft_msg_counts.clone(),
notification_counts: self.notification_counts.clone(),
log_stage_percentiles: self.log_stage_histograms.percentile_stats_array(),
log_stage_histograms: self.log_stage_histograms.clone(),
log_stages: self.log_stage.clone(),
}
}
}