1use serde::{Deserialize, Serialize};
16use std::collections::{HashMap, VecDeque};
17use std::sync::Arc;
18use std::sync::atomic::{AtomicI64, AtomicU64, Ordering, Ordering as AtomicOrdering};
19use std::time::{Duration, Instant, SystemTime};
20use tokio::sync::Mutex;
21
22use crate::resync::ResyncStatusType;
23
24const ROLLING_WINDOW: Duration = Duration::from_secs(60);
25const FAILURE_LAST_HOUR_WINDOW: Duration = Duration::from_secs(60 * 60);
26const I64_MAX_AS_U128: u128 = 9_223_372_036_854_775_807;
27
28#[derive(Debug)]
29pub struct ExponentialMovingAverage {
30 pub alpha: f64,
31 pub value: AtomicU64,
32 pub last_update: Arc<Mutex<SystemTime>>,
33}
34
35impl ExponentialMovingAverage {
36 pub fn new() -> Self {
37 let now = SystemTime::now();
38 Self {
39 alpha: 0.1,
40 value: AtomicU64::new(0_f64.to_bits()),
41 last_update: Arc::new(Mutex::new(now)),
42 }
43 }
44
45 pub fn add_value(&self, value: f64, timestamp: SystemTime) {
46 let current_value = f64::from_bits(self.value.load(AtomicOrdering::Relaxed));
47 let new_value = if current_value == 0.0 {
48 value
49 } else {
50 self.alpha * value + (1.0 - self.alpha) * current_value
51 };
52 self.value.store(new_value.to_bits(), AtomicOrdering::Relaxed);
53
54 if let Ok(mut last_update) = self.last_update.try_lock() {
55 *last_update = timestamp;
56 }
57 }
58
59 pub fn get_current_average(&self) -> f64 {
60 f64::from_bits(self.value.load(AtomicOrdering::Relaxed))
61 }
62
63 pub fn update_exponential_moving_average(&self, now: SystemTime) {
64 if let Ok(mut last_update_guard) = self.last_update.try_lock() {
65 let last_update = *last_update_guard;
66 if let Ok(duration) = now.duration_since(last_update)
67 && duration.as_secs() > 0
68 {
69 let decay = (-duration.as_secs_f64() / 60.0).exp();
70 let current_value = f64::from_bits(self.value.load(AtomicOrdering::Relaxed));
71 self.value.store((current_value * decay).to_bits(), AtomicOrdering::Relaxed);
72 *last_update_guard = now;
73 }
74 }
75 }
76
77 pub fn merge(&self, other: &ExponentialMovingAverage) -> Self {
78 let now = SystemTime::now();
79 let self_value = f64::from_bits(self.value.load(AtomicOrdering::Relaxed));
80 let other_value = f64::from_bits(other.value.load(AtomicOrdering::Relaxed));
81 let merged_value = (self_value + other_value) / 2.0;
82
83 let self_time = self.last_update.try_lock().map(|t| *t).unwrap_or(now);
84 let other_time = other.last_update.try_lock().map(|t| *t).unwrap_or(now);
85 let merged_time = self_time.max(other_time);
86
87 Self {
88 alpha: self.alpha,
89 value: AtomicU64::new(merged_value.to_bits()),
90 last_update: Arc::new(Mutex::new(merged_time)),
91 }
92 }
93}
94
95impl Clone for ExponentialMovingAverage {
96 fn clone(&self) -> Self {
97 let now = SystemTime::now();
98 let value = self.value.load(AtomicOrdering::Relaxed);
99 let last_update = self.last_update.try_lock().map(|t| *t).unwrap_or(now);
100
101 Self {
102 alpha: self.alpha,
103 value: AtomicU64::new(value),
104 last_update: Arc::new(Mutex::new(last_update)),
105 }
106 }
107}
108
109impl Default for ExponentialMovingAverage {
110 fn default() -> Self {
111 Self::new()
112 }
113}
114
115impl Serialize for ExponentialMovingAverage {
116 fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
117 where
118 S: serde::Serializer,
119 {
120 use serde::ser::SerializeStruct;
121 let mut state = serializer.serialize_struct("ExponentialMovingAverage", 3)?;
122 state.serialize_field("alpha", &self.alpha)?;
123 state.serialize_field("value", &f64::from_bits(self.value.load(AtomicOrdering::Relaxed)))?;
124 let last_update = self.last_update.try_lock().map(|t| *t).unwrap_or(SystemTime::UNIX_EPOCH);
125 state.serialize_field("last_update", &last_update)?;
126 state.end()
127 }
128}
129
130impl<'de> Deserialize<'de> for ExponentialMovingAverage {
131 fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
132 where
133 D: serde::Deserializer<'de>,
134 {
135 #[derive(Deserialize)]
136 struct ExponentialMovingAverageData {
137 alpha: f64,
138 value: f64,
139 last_update: SystemTime,
140 }
141
142 let data = ExponentialMovingAverageData::deserialize(deserializer)?;
143 Ok(Self {
144 alpha: data.alpha,
145 value: AtomicU64::new(data.value.to_bits()),
146 last_update: Arc::new(Mutex::new(data.last_update)),
147 })
148 }
149}
150
151#[derive(Debug, Clone, Serialize, Deserialize)]
152pub struct XferStats {
153 pub avg: f64,
154 pub curr: f64,
155 pub peak: f64,
156 pub measure: ExponentialMovingAverage,
157}
158
159impl XferStats {
160 pub fn new() -> Self {
161 Self {
162 avg: 0.0,
163 curr: 0.0,
164 peak: 0.0,
165 measure: ExponentialMovingAverage::new(),
166 }
167 }
168
169 pub fn add_size(&mut self, size: i64, duration: Duration) {
170 if duration.as_nanos() > 0 {
171 let rate = (size as f64) / duration.as_secs_f64();
172 self.curr = rate;
173 if rate > self.peak {
174 self.peak = rate;
175 }
176 self.measure.add_value(rate, SystemTime::now());
177 self.avg = self.measure.get_current_average();
178 }
179 }
180
181 pub fn clone_stats(&self) -> Self {
182 Self {
183 avg: self.avg,
184 curr: self.curr,
185 peak: self.peak,
186 measure: self.measure.clone(),
187 }
188 }
189
190 pub fn merge(&self, other: &XferStats) -> Self {
191 Self {
192 avg: (self.avg + other.avg) / 2.0,
193 curr: self.curr + other.curr,
194 peak: self.peak.max(other.peak),
195 measure: self.measure.merge(&other.measure),
196 }
197 }
198
199 pub fn update_exponential_moving_average(&mut self, now: SystemTime) {
200 self.measure.update_exponential_moving_average(now);
201 self.avg = self.measure.get_current_average();
202 }
203}
204
205impl Default for XferStats {
206 fn default() -> Self {
207 Self::new()
208 }
209}
210
211#[derive(Debug, Default, Serialize, Deserialize)]
212pub struct InQueueStats {
213 pub bytes: i64,
214 pub count: i64,
215 #[serde(skip)]
216 pub now_bytes: AtomicI64,
217 #[serde(skip)]
218 pub now_count: AtomicI64,
219}
220
221#[derive(Debug, Clone)]
222struct QueueSample {
223 observed_at: Instant,
224 bytes: i64,
225 count: i64,
226}
227
228impl Clone for InQueueStats {
229 fn clone(&self) -> Self {
230 Self {
231 bytes: self.bytes,
232 count: self.count,
233 now_bytes: AtomicI64::new(self.now_bytes.load(Ordering::Relaxed)),
234 now_count: AtomicI64::new(self.now_count.load(Ordering::Relaxed)),
235 }
236 }
237}
238
239impl InQueueStats {
240 pub fn new() -> Self {
241 Self::default()
242 }
243
244 pub fn add_current(&self, bytes: i64, count: i64) {
245 self.now_bytes.fetch_add(bytes.max(0), Ordering::Relaxed);
246 self.now_count.fetch_add(count.max(0), Ordering::Relaxed);
247 }
248
249 pub fn subtract_current(&self, bytes: i64, count: i64) {
250 saturating_atomic_sub(&self.now_bytes, bytes.max(0));
251 saturating_atomic_sub(&self.now_count, count.max(0));
252 }
253
254 pub fn get_current_bytes(&self) -> i64 {
255 self.now_bytes.load(Ordering::Relaxed)
256 }
257
258 pub fn get_current_count(&self) -> i64 {
259 self.now_count.load(Ordering::Relaxed)
260 }
261}
262
263fn saturating_atomic_sub(value: &AtomicI64, delta: i64) {
264 if delta == 0 {
265 return;
266 }
267
268 let _ = value.try_update(Ordering::Relaxed, Ordering::Relaxed, |current| Some(current.saturating_sub(delta).max(0)));
269}
270
271#[derive(Debug, Clone, Default, Serialize, Deserialize)]
272pub struct InQueueMetric {
273 pub curr: InQueueStats,
274 pub avg: InQueueStats,
275 pub max: InQueueStats,
276 pub last_minute: InQueueStats,
277 #[serde(skip)]
278 samples: VecDeque<QueueSample>,
279}
280
281impl InQueueMetric {
282 fn observe(&mut self, observed_at: Instant) {
283 let bytes = self.curr.now_bytes.load(Ordering::Relaxed);
284 let count = self.curr.now_count.load(Ordering::Relaxed);
285
286 self.curr.bytes = bytes;
287 self.curr.count = count;
288 self.samples.push_back(QueueSample {
289 observed_at,
290 bytes,
291 count,
292 });
293
294 while self
295 .samples
296 .front()
297 .is_some_and(|sample| observed_at.duration_since(sample.observed_at) > ROLLING_WINDOW)
298 {
299 self.samples.pop_front();
300 }
301
302 if self.samples.is_empty() {
303 self.avg = InQueueStats::default();
304 self.max = InQueueStats::default();
305 self.last_minute = InQueueStats::default();
306 return;
307 }
308
309 let sample_count = self.samples.len() as i64;
310 let total_bytes = self.samples.iter().map(|sample| sample.bytes).sum::<i64>();
311 let total_count = self.samples.iter().map(|sample| sample.count).sum::<i64>();
312 let max_bytes = self.samples.iter().map(|sample| sample.bytes).max().unwrap_or(0);
313 let max_count = self.samples.iter().map(|sample| sample.count).max().unwrap_or(0);
314
315 self.avg.bytes = total_bytes / sample_count;
316 self.avg.count = total_count / sample_count;
317 self.max.bytes = max_bytes;
318 self.max.count = max_count;
319 self.last_minute.bytes = self.avg.bytes;
320 self.last_minute.count = self.avg.count;
321 }
322
323 pub fn snapshot(&self) -> Self {
324 let mut snapshot = self.clone();
325 snapshot.samples.clear();
326 snapshot.curr.bytes = snapshot.curr.now_bytes.load(Ordering::Relaxed);
327 snapshot.curr.count = snapshot.curr.now_count.load(Ordering::Relaxed);
328 snapshot
329 }
330
331 pub fn merge(&self, other: &InQueueMetric) -> Self {
332 Self {
333 curr: InQueueStats {
334 bytes: self.curr.bytes.saturating_add(other.curr.bytes),
335 count: self.curr.count.saturating_add(other.curr.count),
336 now_bytes: AtomicI64::new(
337 self.curr
338 .now_bytes
339 .load(Ordering::Relaxed)
340 .saturating_add(other.curr.now_bytes.load(Ordering::Relaxed)),
341 ),
342 now_count: AtomicI64::new(
343 self.curr
344 .now_count
345 .load(Ordering::Relaxed)
346 .saturating_add(other.curr.now_count.load(Ordering::Relaxed)),
347 ),
348 },
349 avg: InQueueStats {
350 bytes: self.avg.bytes.saturating_add(other.avg.bytes) / 2,
351 count: self.avg.count.saturating_add(other.avg.count) / 2,
352 ..Default::default()
353 },
354 max: InQueueStats {
355 bytes: self.max.bytes.max(other.max.bytes),
356 count: self.max.count.max(other.max.count),
357 ..Default::default()
358 },
359 last_minute: InQueueStats {
360 bytes: self.last_minute.bytes.saturating_add(other.last_minute.bytes),
361 count: self.last_minute.count.saturating_add(other.last_minute.count),
362 ..Default::default()
363 },
364 samples: VecDeque::new(),
365 }
366 }
367}
368
369#[derive(Debug, Default)]
370pub struct QueueCache {
371 pub bucket_stats: HashMap<String, InQueueMetric>,
372 pub sr_queue_stats: InQueueMetric,
373}
374
375impl QueueCache {
376 pub fn new() -> Self {
377 Self::default()
378 }
379
380 pub fn inc(&mut self, bucket: &str, size: i64) {
381 let stats = self.bucket_stats.entry(bucket.to_string()).or_default();
382 stats.curr.add_current(size, 1);
383 self.sr_queue_stats.curr.add_current(size, 1);
384 }
385
386 pub fn dec(&mut self, bucket: &str, size: i64) {
387 let stats = self.bucket_stats.entry(bucket.to_string()).or_default();
388 stats.curr.subtract_current(size, 1);
389 self.sr_queue_stats.curr.subtract_current(size, 1);
390 }
391
392 pub fn update(&mut self) {
393 let observed_at = Instant::now();
394 self.sr_queue_stats.observe(observed_at);
395 for stats in self.bucket_stats.values_mut() {
396 stats.observe(observed_at);
397 }
398 }
399
400 pub fn get_bucket_stats(&self, bucket: &str) -> InQueueMetric {
401 self.bucket_stats.get(bucket).map(InQueueMetric::snapshot).unwrap_or_default()
402 }
403
404 pub fn get_site_stats(&self) -> InQueueMetric {
405 self.sr_queue_stats.snapshot()
406 }
407}
408
409#[derive(Debug, Clone, Default, Serialize, Deserialize)]
410pub struct ProxyMetric {
411 pub get_total: i64,
412 pub get_failed: i64,
413 pub get_tag_total: i64,
414 pub get_tag_failed: i64,
415 pub put_total: i64,
416 pub put_failed: i64,
417 pub put_tag_total: i64,
418 pub put_tag_failed: i64,
419 pub delete_tag_total: i64,
420 pub delete_tag_failed: i64,
421 pub head_total: i64,
422 pub head_failed: i64,
423}
424
425impl ProxyMetric {
426 pub fn add(&mut self, other: &ProxyMetric) {
427 self.get_total += other.get_total;
428 self.get_failed += other.get_failed;
429 self.get_tag_total += other.get_tag_total;
430 self.get_tag_failed += other.get_tag_failed;
431 self.put_total += other.put_total;
432 self.put_failed += other.put_failed;
433 self.put_tag_total += other.put_tag_total;
434 self.put_tag_failed += other.put_tag_failed;
435 self.delete_tag_total += other.delete_tag_total;
436 self.delete_tag_failed += other.delete_tag_failed;
437 self.head_total += other.head_total;
438 self.head_failed += other.head_failed;
439 }
440}
441
442#[derive(Debug, Clone, Default)]
443pub struct ProxyStatsCache {
444 bucket_stats: HashMap<String, ProxyMetric>,
445}
446
447impl ProxyStatsCache {
448 pub fn new() -> Self {
449 Self::default()
450 }
451
452 pub fn inc(&mut self, bucket: &str, api: &str, is_err: bool) {
453 let metric = self.bucket_stats.entry(bucket.to_string()).or_default();
454
455 match api {
456 "GetObject" => {
457 metric.get_total += 1;
458 if is_err {
459 metric.get_failed += 1;
460 }
461 }
462 "GetObjectTagging" => {
463 metric.get_tag_total += 1;
464 if is_err {
465 metric.get_tag_failed += 1;
466 }
467 }
468 "PutObject" => {
469 metric.put_total += 1;
470 if is_err {
471 metric.put_failed += 1;
472 }
473 }
474 "PutObjectTagging" => {
475 metric.put_tag_total += 1;
476 if is_err {
477 metric.put_tag_failed += 1;
478 }
479 }
480 "HeadObject" => {
481 metric.head_total += 1;
482 if is_err {
483 metric.head_failed += 1;
484 }
485 }
486 "DeleteObjectTagging" => {
487 metric.delete_tag_total += 1;
488 if is_err {
489 metric.delete_tag_failed += 1;
490 }
491 }
492 _ => {}
493 }
494 }
495
496 pub fn get_bucket_stats(&self, bucket: &str) -> ProxyMetric {
497 self.bucket_stats.get(bucket).cloned().unwrap_or_default()
498 }
499
500 pub fn bucket_names(&self) -> impl Iterator<Item = &str> {
501 self.bucket_stats.keys().map(String::as_str)
502 }
503
504 pub fn get_site_stats(&self) -> ProxyMetric {
505 let mut total = ProxyMetric::default();
506 for metric in self.bucket_stats.values() {
507 total.add(metric);
508 }
509 total
510 }
511}
512
513#[derive(Debug, Clone)]
514struct FailureSample {
515 observed_at: Instant,
516 size: i64,
517}
518
519#[derive(Debug, Clone, Default, Serialize, Deserialize)]
520pub struct FailStats {
521 pub count: i64,
522 pub size: i64,
523 #[serde(default)]
528 pub last_minute: FailedMetric,
529 #[serde(default)]
530 pub last_hour: FailedMetric,
531 #[serde(skip)]
532 recent: VecDeque<FailureSample>,
533}
534
535impl FailStats {
536 pub fn new() -> Self {
537 Self::default()
538 }
539
540 pub fn add_size<E>(&mut self, size: i64, _err: Option<&E>) {
541 let observed_at = Instant::now();
542 self.count = self.count.saturating_add(1);
543 self.size = self.size.saturating_add(size);
544 self.recent.push_back(FailureSample { observed_at, size });
545 self.prune(observed_at);
546 }
547
548 pub fn refresh_windows(&mut self) {
555 self.last_minute = self.recent_since(Duration::from_secs(60));
556 self.last_hour = self.recent_since(Duration::from_secs(3600));
557 }
558
559 fn prune(&mut self, observed_at: Instant) {
560 while self
561 .recent
562 .front()
563 .is_some_and(|sample| observed_at.duration_since(sample.observed_at) > FAILURE_LAST_HOUR_WINDOW)
564 {
565 self.recent.pop_front();
566 }
567 }
568
569 pub fn recent_since(&self, window: Duration) -> FailedMetric {
570 let now = Instant::now();
571 let mut count = 0i64;
572 let mut size = 0i64;
573 for sample in self.recent.iter().rev() {
574 if now.duration_since(sample.observed_at) > window {
575 break;
576 }
577 count += 1;
578 size += sample.size;
579 }
580 FailedMetric { count, size }
581 }
582
583 pub fn recent_windows(&self, short: Duration, long: Duration) -> (FailedMetric, FailedMetric) {
589 let now = Instant::now();
590 let mut short_metric = FailedMetric::default();
591 let mut long_metric = FailedMetric::default();
592 for sample in self.recent.iter().rev() {
593 let age = now.duration_since(sample.observed_at);
594 if age > long {
595 break;
596 }
597 if age <= short {
598 short_metric.count += 1;
599 short_metric.size += sample.size;
600 }
601 long_metric.count += 1;
602 long_metric.size += sample.size;
603 }
604 (short_metric, long_metric)
605 }
606
607 pub fn merge(&self, other: &FailStats) -> Self {
608 Self {
609 count: self.count.saturating_add(other.count),
610 size: self.size.saturating_add(other.size),
611 last_minute: FailedMetric {
614 count: self.last_minute.count.saturating_add(other.last_minute.count),
615 size: self.last_minute.size.saturating_add(other.last_minute.size),
616 },
617 last_hour: FailedMetric {
618 count: self.last_hour.count.saturating_add(other.last_hour.count),
619 size: self.last_hour.size.saturating_add(other.last_hour.size),
620 },
621 recent: VecDeque::new(),
622 }
623 }
624
625 pub fn to_metric(&self) -> FailedMetric {
626 FailedMetric {
627 count: self.count,
628 size: self.size,
629 }
630 }
631}
632
633#[derive(Debug, Clone, Default, Serialize, Deserialize)]
634pub struct FailedMetric {
635 pub count: i64,
636 pub size: i64,
637}
638
639#[derive(Debug, Clone, Default, Serialize, Deserialize)]
640pub struct LatencyStats {
641 pub avg: f64,
642 pub curr: f64,
643 pub max: f64,
644}
645
646impl LatencyStats {
647 pub fn new() -> Self {
648 Self::default()
649 }
650
651 pub fn update(&mut self, _size: i64, duration: Duration) {
652 let latency = duration.as_millis() as f64;
653 self.curr = latency;
654 if latency > self.max {
655 self.max = latency;
656 }
657 self.avg = (self.avg + latency) / 2.0;
658 }
659
660 pub fn merge(&self, other: &LatencyStats) -> Self {
661 Self {
662 avg: (self.avg + other.avg) / 2.0,
663 curr: self.curr.max(other.curr),
664 max: self.max.max(other.max),
665 }
666 }
667}
668
669#[derive(Debug, Clone, Default, Serialize, Deserialize)]
670pub struct BucketReplicationStat {
671 pub replicated_size: i64,
672 pub replicated_count: i64,
673 pub failed: FailedMetric,
674 pub fail_stats: FailStats,
675 pub latency: LatencyStats,
676 pub xfer_rate_lrg: XferStats,
677 pub xfer_rate_sml: XferStats,
678 pub bandwidth_limit_bytes_per_sec: i64,
679 pub current_bandwidth_bytes_per_sec: f64,
680 #[serde(default)]
681 pub latency_scope: ReplicationMetricScope,
682 #[serde(default)]
683 pub bandwidth_scope: ReplicationMetricScope,
684}
685
686impl BucketReplicationStat {
687 pub fn new() -> Self {
688 Self::default()
689 }
690
691 pub fn update_xfer_rate(&mut self, size: i64, duration: Duration) {
692 if size >= crate::runtime::MIN_LARGE_OBJ_SIZE {
695 self.xfer_rate_lrg.add_size(size, duration);
696 } else {
697 self.xfer_rate_sml.add_size(size, duration);
698 }
699 }
700
701 pub fn set_node_local_bandwidth(&mut self, limit_bytes_per_sec: i64, current_bytes_per_sec: f64) {
702 self.bandwidth_limit_bytes_per_sec = limit_bytes_per_sec;
703 self.current_bandwidth_bytes_per_sec = current_bytes_per_sec;
704 self.bandwidth_scope = ReplicationMetricScope::NodeLocal;
705 }
706}
707
708#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
709#[serde(rename_all = "snake_case")]
710pub enum ReplicationMetricScope {
711 #[default]
712 Unavailable,
713 NodeLocal,
714 ClusterAggregated,
715 PartialCluster,
716}
717
718#[derive(Debug, Clone, Default, Serialize, Deserialize)]
719pub struct QueueStats {
720 pub nodes: Vec<QueueNode>,
721}
722
723#[derive(Debug, Clone, Default, Serialize, Deserialize)]
724pub struct QueueNode {
725 pub q_stats: InQueueMetric,
726}
727
728#[derive(Debug, Clone, Default, Serialize, Deserialize)]
729pub struct BucketReplicationStats {
730 pub stats: HashMap<String, BucketReplicationStat>,
731 pub replica_size: i64,
732 pub replica_count: i64,
733 pub replicated_size: i64,
734 pub replicated_count: i64,
735 pub q_stat: InQueueMetric,
736 #[serde(default)]
737 pub resync_started_count: i64,
738 #[serde(default)]
739 pub resync_completed_count: i64,
740 #[serde(default)]
741 pub resync_failed_count: i64,
742 #[serde(default)]
743 pub resync_canceled_count: i64,
744 #[serde(default)]
745 pub resync_duration_ms: i64,
746 #[serde(default)]
747 pub provider_available: bool,
748 #[serde(default)]
749 pub cluster_complete: bool,
750 #[serde(default)]
751 pub observed_node_count: u32,
752 #[serde(default)]
753 pub expected_node_count: u32,
754 #[serde(default)]
755 pub queue_scope: ReplicationMetricScope,
756}
757
758impl BucketReplicationStats {
759 pub fn new() -> Self {
760 Self::default()
761 }
762
763 pub fn is_empty(&self) -> bool {
764 self.stats.is_empty()
765 && self.replica_size == 0
766 && self.replicated_size == 0
767 && self.resync_started_count == 0
768 && self.resync_completed_count == 0
769 && self.resync_failed_count == 0
770 && self.resync_canceled_count == 0
771 && self.resync_duration_ms == 0
772 }
773
774 pub fn has_replication_usage(&self) -> bool {
775 self.replica_size > 0
776 || self.replicated_size > 0
777 || self.resync_started_count > 0
778 || self.resync_completed_count > 0
779 || self.resync_failed_count > 0
780 || self.resync_canceled_count > 0
781 || self.resync_duration_ms > 0
782 || !self.stats.is_empty()
783 }
784
785 pub fn clone_stats(&self) -> Self {
786 let mut snapshot = self.clone();
787 for stat in snapshot.stats.values_mut() {
788 stat.failed = stat.fail_stats.to_metric();
789 }
790 snapshot
791 }
792
793 pub fn mark_node_local_provider_available(&mut self) {
794 self.provider_available = true;
795 self.cluster_complete = true;
796 self.observed_node_count = 1;
797 self.expected_node_count = 1;
798 }
799
800 pub fn record_resync_status(&mut self, status: ResyncStatusType, duration: Option<Duration>) {
801 match status {
802 ResyncStatusType::ResyncStarted => self.resync_started_count += 1,
803 ResyncStatusType::ResyncCompleted => self.resync_completed_count += 1,
804 ResyncStatusType::ResyncFailed => self.resync_failed_count += 1,
805 ResyncStatusType::ResyncCanceled => self.resync_canceled_count += 1,
806 ResyncStatusType::NoResync | ResyncStatusType::ResyncPending => return,
807 }
808
809 if let Some(duration) = duration {
810 let duration_ms = duration.as_millis().min(I64_MAX_AS_U128);
811 if let Ok(duration_ms) = i64::try_from(duration_ms) {
812 self.resync_duration_ms = self.resync_duration_ms.saturating_add(duration_ms);
813 }
814 }
815 }
816}
817
818#[derive(Debug, Clone, Default, Serialize, Deserialize)]
819pub struct BucketStats {
820 pub uptime: i64,
821 pub replication_stats: BucketReplicationStats,
822 pub queue_stats: QueueStats,
823 pub proxy_stats: ProxyMetric,
824}
825
826#[derive(Debug, Clone, Default, Serialize, Deserialize)]
827pub struct SRMetricsSummary {
828 pub uptime: i64,
829 pub queued: InQueueMetric,
830 pub active_workers: ActiveWorkerStat,
831 pub metrics: HashMap<String, i64>,
832 pub proxied: ProxyMetric,
833 pub replica_size: i64,
834 pub replica_count: i64,
835}
836
837#[derive(Debug, Clone, Default, Serialize, Deserialize)]
838pub struct ActiveWorkerStat {
839 pub curr: i32,
840 pub max: i32,
841 pub avg: f64,
842 #[serde(skip)]
843 samples: VecDeque<WorkerSample>,
844}
845
846#[derive(Debug, Clone)]
847struct WorkerSample {
848 observed_at: Instant,
849 workers: i32,
850}
851
852impl ActiveWorkerStat {
853 pub fn new() -> Self {
854 Self::default()
855 }
856
857 pub fn get(&self) -> Self {
858 self.clone()
859 }
860
861 pub fn update(&mut self, curr: i32) {
862 let observed_at = Instant::now();
863 self.curr = curr;
864 self.samples.push_back(WorkerSample {
865 observed_at,
866 workers: curr,
867 });
868
869 while self
870 .samples
871 .front()
872 .is_some_and(|sample| observed_at.duration_since(sample.observed_at) > ROLLING_WINDOW)
873 {
874 self.samples.pop_front();
875 }
876
877 if self.samples.is_empty() {
878 self.max = curr;
879 self.avg = curr as f64;
880 return;
881 }
882
883 self.max = self.samples.iter().map(|sample| sample.workers).max().unwrap_or(curr);
884 let total = self.samples.iter().map(|sample| sample.workers as i64).sum::<i64>();
885 self.avg = total as f64 / self.samples.len() as f64;
886 }
887}
888
889#[cfg(test)]
890mod tests {
891 use super::*;
892
893 #[test]
894 fn in_queue_metric_observe_updates_rolling_stats() {
895 let mut metric = InQueueMetric::default();
896 metric.curr.add_current(128, 4);
897 metric.observe(Instant::now());
898
899 metric.curr.add_current(128, 2);
900 metric.observe(Instant::now());
901
902 assert_eq!(metric.curr.bytes, 256);
903 assert_eq!(metric.curr.count, 6);
904 assert_eq!(metric.max.bytes, 256);
905 assert_eq!(metric.max.count, 6);
906 assert_eq!(metric.last_minute.bytes, 192);
907 assert_eq!(metric.last_minute.count, 5);
908 }
909
910 #[test]
911 fn queue_cache_decrement_saturates_at_zero() {
912 let mut cache = QueueCache::new();
913 cache.inc("bucket", 128);
914 cache.dec("bucket", 512);
915 cache.dec("bucket", 1);
916
917 let bucket = cache.get_bucket_stats("bucket");
918 let site = cache.get_site_stats();
919 assert_eq!(bucket.curr.count, 0);
920 assert_eq!(bucket.curr.bytes, 0);
921 assert_eq!(site.curr.count, 0);
922 assert_eq!(site.curr.bytes, 0);
923 }
924
925 #[test]
926 fn fail_stats_recent_since_tracks_windows() {
927 let mut stats = FailStats::default();
928 stats.add_size(64, None::<&()>);
929 stats.add_size(32, None::<&()>);
930
931 let last_minute = stats.recent_since(Duration::from_secs(60));
932 let last_hour = stats.recent_since(Duration::from_secs(60 * 60));
933 assert_eq!(last_minute.count, 2);
934 assert_eq!(last_minute.size, 96);
935 assert_eq!(last_hour.count, 2);
936 assert_eq!(last_hour.size, 96);
937 }
938
939 #[test]
940 fn fail_stats_recent_windows_matches_two_separate_scans() {
941 let mut stats = FailStats::default();
942 stats.add_size(64, None::<&()>);
943 stats.add_size(32, None::<&()>);
944
945 let (minute, hour) = stats.recent_windows(Duration::from_secs(60), Duration::from_secs(60 * 60));
946 let expected_minute = stats.recent_since(Duration::from_secs(60));
947 let expected_hour = stats.recent_since(Duration::from_secs(60 * 60));
948
949 assert_eq!((minute.count, minute.size), (expected_minute.count, expected_minute.size));
950 assert_eq!((hour.count, hour.size), (expected_hour.count, expected_hour.size));
951 assert_eq!(minute.count, 2);
952 assert_eq!(hour.size, 96);
953
954 let empty = FailStats::default();
955 let (minute, hour) = empty.recent_windows(Duration::from_secs(60), Duration::from_secs(60 * 60));
956 assert_eq!((minute.count, minute.size, hour.count, hour.size), (0, 0, 0, 0));
957 }
958
959 #[test]
960 fn fail_stats_saturate_instead_of_wrapping() {
961 let mut stats = FailStats {
962 count: i64::MAX,
963 size: i64::MAX,
964 ..Default::default()
965 };
966
967 stats.add_size(1, None::<&()>);
968 let merged = stats.merge(&FailStats {
969 count: 1,
970 size: 1,
971 ..Default::default()
972 });
973
974 assert_eq!(stats.count, i64::MAX);
975 assert_eq!(stats.size, i64::MAX);
976 assert_eq!(merged.count, i64::MAX);
977 assert_eq!(merged.size, i64::MAX);
978 }
979
980 #[test]
981 fn active_worker_stat_update_tracks_rolling_avg_and_max() {
982 let mut stats = ActiveWorkerStat::default();
983 stats.update(2);
984 stats.update(6);
985 stats.update(4);
986
987 assert_eq!(stats.curr, 4);
988 assert_eq!(stats.max, 6);
989 assert_eq!(stats.avg, 4.0);
990 }
991}