Skip to main content

rustfs_replication/
stats.rs

1// Copyright 2024 RustFS Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use serde::{Deserialize, Serialize};
16use std::collections::{HashMap, VecDeque};
17use std::sync::Arc;
18use std::sync::atomic::{AtomicI64, AtomicU64, Ordering, Ordering as AtomicOrdering};
19use std::time::{Duration, Instant, SystemTime};
20use tokio::sync::Mutex;
21
22use crate::resync::ResyncStatusType;
23
24const ROLLING_WINDOW: Duration = Duration::from_secs(60);
25const FAILURE_LAST_HOUR_WINDOW: Duration = Duration::from_secs(60 * 60);
26const I64_MAX_AS_U128: u128 = 9_223_372_036_854_775_807;
27
28#[derive(Debug)]
29pub struct ExponentialMovingAverage {
30    pub alpha: f64,
31    pub value: AtomicU64,
32    pub last_update: Arc<Mutex<SystemTime>>,
33}
34
35impl ExponentialMovingAverage {
36    pub fn new() -> Self {
37        let now = SystemTime::now();
38        Self {
39            alpha: 0.1,
40            value: AtomicU64::new(0_f64.to_bits()),
41            last_update: Arc::new(Mutex::new(now)),
42        }
43    }
44
45    pub fn add_value(&self, value: f64, timestamp: SystemTime) {
46        let current_value = f64::from_bits(self.value.load(AtomicOrdering::Relaxed));
47        let new_value = if current_value == 0.0 {
48            value
49        } else {
50            self.alpha * value + (1.0 - self.alpha) * current_value
51        };
52        self.value.store(new_value.to_bits(), AtomicOrdering::Relaxed);
53
54        if let Ok(mut last_update) = self.last_update.try_lock() {
55            *last_update = timestamp;
56        }
57    }
58
59    pub fn get_current_average(&self) -> f64 {
60        f64::from_bits(self.value.load(AtomicOrdering::Relaxed))
61    }
62
63    pub fn update_exponential_moving_average(&self, now: SystemTime) {
64        if let Ok(mut last_update_guard) = self.last_update.try_lock() {
65            let last_update = *last_update_guard;
66            if let Ok(duration) = now.duration_since(last_update)
67                && duration.as_secs() > 0
68            {
69                let decay = (-duration.as_secs_f64() / 60.0).exp();
70                let current_value = f64::from_bits(self.value.load(AtomicOrdering::Relaxed));
71                self.value.store((current_value * decay).to_bits(), AtomicOrdering::Relaxed);
72                *last_update_guard = now;
73            }
74        }
75    }
76
77    pub fn merge(&self, other: &ExponentialMovingAverage) -> Self {
78        let now = SystemTime::now();
79        let self_value = f64::from_bits(self.value.load(AtomicOrdering::Relaxed));
80        let other_value = f64::from_bits(other.value.load(AtomicOrdering::Relaxed));
81        let merged_value = (self_value + other_value) / 2.0;
82
83        let self_time = self.last_update.try_lock().map(|t| *t).unwrap_or(now);
84        let other_time = other.last_update.try_lock().map(|t| *t).unwrap_or(now);
85        let merged_time = self_time.max(other_time);
86
87        Self {
88            alpha: self.alpha,
89            value: AtomicU64::new(merged_value.to_bits()),
90            last_update: Arc::new(Mutex::new(merged_time)),
91        }
92    }
93}
94
95impl Clone for ExponentialMovingAverage {
96    fn clone(&self) -> Self {
97        let now = SystemTime::now();
98        let value = self.value.load(AtomicOrdering::Relaxed);
99        let last_update = self.last_update.try_lock().map(|t| *t).unwrap_or(now);
100
101        Self {
102            alpha: self.alpha,
103            value: AtomicU64::new(value),
104            last_update: Arc::new(Mutex::new(last_update)),
105        }
106    }
107}
108
109impl Default for ExponentialMovingAverage {
110    fn default() -> Self {
111        Self::new()
112    }
113}
114
115impl Serialize for ExponentialMovingAverage {
116    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
117    where
118        S: serde::Serializer,
119    {
120        use serde::ser::SerializeStruct;
121        let mut state = serializer.serialize_struct("ExponentialMovingAverage", 3)?;
122        state.serialize_field("alpha", &self.alpha)?;
123        state.serialize_field("value", &f64::from_bits(self.value.load(AtomicOrdering::Relaxed)))?;
124        let last_update = self.last_update.try_lock().map(|t| *t).unwrap_or(SystemTime::UNIX_EPOCH);
125        state.serialize_field("last_update", &last_update)?;
126        state.end()
127    }
128}
129
130impl<'de> Deserialize<'de> for ExponentialMovingAverage {
131    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
132    where
133        D: serde::Deserializer<'de>,
134    {
135        #[derive(Deserialize)]
136        struct ExponentialMovingAverageData {
137            alpha: f64,
138            value: f64,
139            last_update: SystemTime,
140        }
141
142        let data = ExponentialMovingAverageData::deserialize(deserializer)?;
143        Ok(Self {
144            alpha: data.alpha,
145            value: AtomicU64::new(data.value.to_bits()),
146            last_update: Arc::new(Mutex::new(data.last_update)),
147        })
148    }
149}
150
151#[derive(Debug, Clone, Serialize, Deserialize)]
152pub struct XferStats {
153    pub avg: f64,
154    pub curr: f64,
155    pub peak: f64,
156    pub measure: ExponentialMovingAverage,
157}
158
159impl XferStats {
160    pub fn new() -> Self {
161        Self {
162            avg: 0.0,
163            curr: 0.0,
164            peak: 0.0,
165            measure: ExponentialMovingAverage::new(),
166        }
167    }
168
169    pub fn add_size(&mut self, size: i64, duration: Duration) {
170        if duration.as_nanos() > 0 {
171            let rate = (size as f64) / duration.as_secs_f64();
172            self.curr = rate;
173            if rate > self.peak {
174                self.peak = rate;
175            }
176            self.measure.add_value(rate, SystemTime::now());
177            self.avg = self.measure.get_current_average();
178        }
179    }
180
181    pub fn clone_stats(&self) -> Self {
182        Self {
183            avg: self.avg,
184            curr: self.curr,
185            peak: self.peak,
186            measure: self.measure.clone(),
187        }
188    }
189
190    pub fn merge(&self, other: &XferStats) -> Self {
191        Self {
192            avg: (self.avg + other.avg) / 2.0,
193            curr: self.curr + other.curr,
194            peak: self.peak.max(other.peak),
195            measure: self.measure.merge(&other.measure),
196        }
197    }
198
199    pub fn update_exponential_moving_average(&mut self, now: SystemTime) {
200        self.measure.update_exponential_moving_average(now);
201        self.avg = self.measure.get_current_average();
202    }
203}
204
205impl Default for XferStats {
206    fn default() -> Self {
207        Self::new()
208    }
209}
210
211#[derive(Debug, Default, Serialize, Deserialize)]
212pub struct InQueueStats {
213    pub bytes: i64,
214    pub count: i64,
215    #[serde(skip)]
216    pub now_bytes: AtomicI64,
217    #[serde(skip)]
218    pub now_count: AtomicI64,
219}
220
221#[derive(Debug, Clone)]
222struct QueueSample {
223    observed_at: Instant,
224    bytes: i64,
225    count: i64,
226}
227
228impl Clone for InQueueStats {
229    fn clone(&self) -> Self {
230        Self {
231            bytes: self.bytes,
232            count: self.count,
233            now_bytes: AtomicI64::new(self.now_bytes.load(Ordering::Relaxed)),
234            now_count: AtomicI64::new(self.now_count.load(Ordering::Relaxed)),
235        }
236    }
237}
238
239impl InQueueStats {
240    pub fn new() -> Self {
241        Self::default()
242    }
243
244    pub fn add_current(&self, bytes: i64, count: i64) {
245        self.now_bytes.fetch_add(bytes.max(0), Ordering::Relaxed);
246        self.now_count.fetch_add(count.max(0), Ordering::Relaxed);
247    }
248
249    pub fn subtract_current(&self, bytes: i64, count: i64) {
250        saturating_atomic_sub(&self.now_bytes, bytes.max(0));
251        saturating_atomic_sub(&self.now_count, count.max(0));
252    }
253
254    pub fn get_current_bytes(&self) -> i64 {
255        self.now_bytes.load(Ordering::Relaxed)
256    }
257
258    pub fn get_current_count(&self) -> i64 {
259        self.now_count.load(Ordering::Relaxed)
260    }
261}
262
263fn saturating_atomic_sub(value: &AtomicI64, delta: i64) {
264    if delta == 0 {
265        return;
266    }
267
268    let _ = value.try_update(Ordering::Relaxed, Ordering::Relaxed, |current| Some(current.saturating_sub(delta).max(0)));
269}
270
271#[derive(Debug, Clone, Default, Serialize, Deserialize)]
272pub struct InQueueMetric {
273    pub curr: InQueueStats,
274    pub avg: InQueueStats,
275    pub max: InQueueStats,
276    pub last_minute: InQueueStats,
277    #[serde(skip)]
278    samples: VecDeque<QueueSample>,
279}
280
281impl InQueueMetric {
282    fn observe(&mut self, observed_at: Instant) {
283        let bytes = self.curr.now_bytes.load(Ordering::Relaxed);
284        let count = self.curr.now_count.load(Ordering::Relaxed);
285
286        self.curr.bytes = bytes;
287        self.curr.count = count;
288        self.samples.push_back(QueueSample {
289            observed_at,
290            bytes,
291            count,
292        });
293
294        while self
295            .samples
296            .front()
297            .is_some_and(|sample| observed_at.duration_since(sample.observed_at) > ROLLING_WINDOW)
298        {
299            self.samples.pop_front();
300        }
301
302        if self.samples.is_empty() {
303            self.avg = InQueueStats::default();
304            self.max = InQueueStats::default();
305            self.last_minute = InQueueStats::default();
306            return;
307        }
308
309        let sample_count = self.samples.len() as i64;
310        let total_bytes = self.samples.iter().map(|sample| sample.bytes).sum::<i64>();
311        let total_count = self.samples.iter().map(|sample| sample.count).sum::<i64>();
312        let max_bytes = self.samples.iter().map(|sample| sample.bytes).max().unwrap_or(0);
313        let max_count = self.samples.iter().map(|sample| sample.count).max().unwrap_or(0);
314
315        self.avg.bytes = total_bytes / sample_count;
316        self.avg.count = total_count / sample_count;
317        self.max.bytes = max_bytes;
318        self.max.count = max_count;
319        self.last_minute.bytes = self.avg.bytes;
320        self.last_minute.count = self.avg.count;
321    }
322
323    pub fn snapshot(&self) -> Self {
324        let mut snapshot = self.clone();
325        snapshot.samples.clear();
326        snapshot.curr.bytes = snapshot.curr.now_bytes.load(Ordering::Relaxed);
327        snapshot.curr.count = snapshot.curr.now_count.load(Ordering::Relaxed);
328        snapshot
329    }
330
331    pub fn merge(&self, other: &InQueueMetric) -> Self {
332        Self {
333            curr: InQueueStats {
334                bytes: self.curr.bytes.saturating_add(other.curr.bytes),
335                count: self.curr.count.saturating_add(other.curr.count),
336                now_bytes: AtomicI64::new(
337                    self.curr
338                        .now_bytes
339                        .load(Ordering::Relaxed)
340                        .saturating_add(other.curr.now_bytes.load(Ordering::Relaxed)),
341                ),
342                now_count: AtomicI64::new(
343                    self.curr
344                        .now_count
345                        .load(Ordering::Relaxed)
346                        .saturating_add(other.curr.now_count.load(Ordering::Relaxed)),
347                ),
348            },
349            avg: InQueueStats {
350                bytes: self.avg.bytes.saturating_add(other.avg.bytes) / 2,
351                count: self.avg.count.saturating_add(other.avg.count) / 2,
352                ..Default::default()
353            },
354            max: InQueueStats {
355                bytes: self.max.bytes.max(other.max.bytes),
356                count: self.max.count.max(other.max.count),
357                ..Default::default()
358            },
359            last_minute: InQueueStats {
360                bytes: self.last_minute.bytes.saturating_add(other.last_minute.bytes),
361                count: self.last_minute.count.saturating_add(other.last_minute.count),
362                ..Default::default()
363            },
364            samples: VecDeque::new(),
365        }
366    }
367}
368
369#[derive(Debug, Default)]
370pub struct QueueCache {
371    pub bucket_stats: HashMap<String, InQueueMetric>,
372    pub sr_queue_stats: InQueueMetric,
373}
374
375impl QueueCache {
376    pub fn new() -> Self {
377        Self::default()
378    }
379
380    pub fn inc(&mut self, bucket: &str, size: i64) {
381        let stats = self.bucket_stats.entry(bucket.to_string()).or_default();
382        stats.curr.add_current(size, 1);
383        self.sr_queue_stats.curr.add_current(size, 1);
384    }
385
386    pub fn dec(&mut self, bucket: &str, size: i64) {
387        let stats = self.bucket_stats.entry(bucket.to_string()).or_default();
388        stats.curr.subtract_current(size, 1);
389        self.sr_queue_stats.curr.subtract_current(size, 1);
390    }
391
392    pub fn update(&mut self) {
393        let observed_at = Instant::now();
394        self.sr_queue_stats.observe(observed_at);
395        for stats in self.bucket_stats.values_mut() {
396            stats.observe(observed_at);
397        }
398    }
399
400    pub fn get_bucket_stats(&self, bucket: &str) -> InQueueMetric {
401        self.bucket_stats.get(bucket).map(InQueueMetric::snapshot).unwrap_or_default()
402    }
403
404    pub fn get_site_stats(&self) -> InQueueMetric {
405        self.sr_queue_stats.snapshot()
406    }
407}
408
409#[derive(Debug, Clone, Default, Serialize, Deserialize)]
410pub struct ProxyMetric {
411    pub get_total: i64,
412    pub get_failed: i64,
413    pub get_tag_total: i64,
414    pub get_tag_failed: i64,
415    pub put_total: i64,
416    pub put_failed: i64,
417    pub put_tag_total: i64,
418    pub put_tag_failed: i64,
419    pub delete_tag_total: i64,
420    pub delete_tag_failed: i64,
421    pub head_total: i64,
422    pub head_failed: i64,
423}
424
425impl ProxyMetric {
426    pub fn add(&mut self, other: &ProxyMetric) {
427        self.get_total += other.get_total;
428        self.get_failed += other.get_failed;
429        self.get_tag_total += other.get_tag_total;
430        self.get_tag_failed += other.get_tag_failed;
431        self.put_total += other.put_total;
432        self.put_failed += other.put_failed;
433        self.put_tag_total += other.put_tag_total;
434        self.put_tag_failed += other.put_tag_failed;
435        self.delete_tag_total += other.delete_tag_total;
436        self.delete_tag_failed += other.delete_tag_failed;
437        self.head_total += other.head_total;
438        self.head_failed += other.head_failed;
439    }
440}
441
442#[derive(Debug, Clone, Default)]
443pub struct ProxyStatsCache {
444    bucket_stats: HashMap<String, ProxyMetric>,
445}
446
447impl ProxyStatsCache {
448    pub fn new() -> Self {
449        Self::default()
450    }
451
452    pub fn inc(&mut self, bucket: &str, api: &str, is_err: bool) {
453        let metric = self.bucket_stats.entry(bucket.to_string()).or_default();
454
455        match api {
456            "GetObject" => {
457                metric.get_total += 1;
458                if is_err {
459                    metric.get_failed += 1;
460                }
461            }
462            "GetObjectTagging" => {
463                metric.get_tag_total += 1;
464                if is_err {
465                    metric.get_tag_failed += 1;
466                }
467            }
468            "PutObject" => {
469                metric.put_total += 1;
470                if is_err {
471                    metric.put_failed += 1;
472                }
473            }
474            "PutObjectTagging" => {
475                metric.put_tag_total += 1;
476                if is_err {
477                    metric.put_tag_failed += 1;
478                }
479            }
480            "HeadObject" => {
481                metric.head_total += 1;
482                if is_err {
483                    metric.head_failed += 1;
484                }
485            }
486            "DeleteObjectTagging" => {
487                metric.delete_tag_total += 1;
488                if is_err {
489                    metric.delete_tag_failed += 1;
490                }
491            }
492            _ => {}
493        }
494    }
495
496    pub fn get_bucket_stats(&self, bucket: &str) -> ProxyMetric {
497        self.bucket_stats.get(bucket).cloned().unwrap_or_default()
498    }
499
500    pub fn bucket_names(&self) -> impl Iterator<Item = &str> {
501        self.bucket_stats.keys().map(String::as_str)
502    }
503
504    pub fn get_site_stats(&self) -> ProxyMetric {
505        let mut total = ProxyMetric::default();
506        for metric in self.bucket_stats.values() {
507            total.add(metric);
508        }
509        total
510    }
511}
512
513#[derive(Debug, Clone)]
514struct FailureSample {
515    observed_at: Instant,
516    size: i64,
517}
518
519#[derive(Debug, Clone, Default, Serialize, Deserialize)]
520pub struct FailStats {
521    pub count: i64,
522    pub size: i64,
523    /// Rolling-window snapshots refreshed at collection time
524    /// ([`Self::refresh_windows`]). The raw samples (`recent`) are process
525    /// local (serde-skipped), so these fields are what survives the peer-RPC
526    /// wire and [`Self::merge`]-based cluster aggregation.
527    #[serde(default)]
528    pub last_minute: FailedMetric,
529    #[serde(default)]
530    pub last_hour: FailedMetric,
531    #[serde(skip)]
532    recent: VecDeque<FailureSample>,
533}
534
535impl FailStats {
536    pub fn new() -> Self {
537        Self::default()
538    }
539
540    pub fn add_size<E>(&mut self, size: i64, _err: Option<&E>) {
541        let observed_at = Instant::now();
542        self.count = self.count.saturating_add(1);
543        self.size = self.size.saturating_add(size);
544        self.recent.push_back(FailureSample { observed_at, size });
545        self.prune(observed_at);
546    }
547
548    /// Recompute the serializable rolling-window snapshots from the local
549    /// samples. Called at the collection point (per-node stats snapshot),
550    /// never on the failure hot path — the two deque scans are O(window) and
551    /// `add_size` runs under the bucket-stats write lock. Only meaningful on
552    /// the live per-node struct: a deserialized or merged struct has no
553    /// samples, and refreshing it would wipe the aggregated windows.
554    pub fn refresh_windows(&mut self) {
555        self.last_minute = self.recent_since(Duration::from_secs(60));
556        self.last_hour = self.recent_since(Duration::from_secs(3600));
557    }
558
559    fn prune(&mut self, observed_at: Instant) {
560        while self
561            .recent
562            .front()
563            .is_some_and(|sample| observed_at.duration_since(sample.observed_at) > FAILURE_LAST_HOUR_WINDOW)
564        {
565            self.recent.pop_front();
566        }
567    }
568
569    pub fn recent_since(&self, window: Duration) -> FailedMetric {
570        let now = Instant::now();
571        let mut count = 0i64;
572        let mut size = 0i64;
573        for sample in self.recent.iter().rev() {
574            if now.duration_since(sample.observed_at) > window {
575                break;
576            }
577            count += 1;
578            size += sample.size;
579        }
580        FailedMetric { count, size }
581    }
582
583    /// Both rolling windows from one walk of the samples. `short` must be the
584    /// narrower window; the walk stops at `long`. Callers that need both (the
585    /// per-node site snapshot) would otherwise scan the deque twice while
586    /// holding the bucket-stats read lock, and the deque is only bounded by
587    /// the one-hour window - an unreachable target under load fills it.
588    pub fn recent_windows(&self, short: Duration, long: Duration) -> (FailedMetric, FailedMetric) {
589        let now = Instant::now();
590        let mut short_metric = FailedMetric::default();
591        let mut long_metric = FailedMetric::default();
592        for sample in self.recent.iter().rev() {
593            let age = now.duration_since(sample.observed_at);
594            if age > long {
595                break;
596            }
597            if age <= short {
598                short_metric.count += 1;
599                short_metric.size += sample.size;
600            }
601            long_metric.count += 1;
602            long_metric.size += sample.size;
603        }
604        (short_metric, long_metric)
605    }
606
607    pub fn merge(&self, other: &FailStats) -> Self {
608        Self {
609            count: self.count.saturating_add(other.count),
610            size: self.size.saturating_add(other.size),
611            // The window snapshots sum across nodes; the raw samples do not
612            // travel and stay empty on aggregated structs.
613            last_minute: FailedMetric {
614                count: self.last_minute.count.saturating_add(other.last_minute.count),
615                size: self.last_minute.size.saturating_add(other.last_minute.size),
616            },
617            last_hour: FailedMetric {
618                count: self.last_hour.count.saturating_add(other.last_hour.count),
619                size: self.last_hour.size.saturating_add(other.last_hour.size),
620            },
621            recent: VecDeque::new(),
622        }
623    }
624
625    pub fn to_metric(&self) -> FailedMetric {
626        FailedMetric {
627            count: self.count,
628            size: self.size,
629        }
630    }
631}
632
633#[derive(Debug, Clone, Default, Serialize, Deserialize)]
634pub struct FailedMetric {
635    pub count: i64,
636    pub size: i64,
637}
638
639#[derive(Debug, Clone, Default, Serialize, Deserialize)]
640pub struct LatencyStats {
641    pub avg: f64,
642    pub curr: f64,
643    pub max: f64,
644}
645
646impl LatencyStats {
647    pub fn new() -> Self {
648        Self::default()
649    }
650
651    pub fn update(&mut self, _size: i64, duration: Duration) {
652        let latency = duration.as_millis() as f64;
653        self.curr = latency;
654        if latency > self.max {
655            self.max = latency;
656        }
657        self.avg = (self.avg + latency) / 2.0;
658    }
659
660    pub fn merge(&self, other: &LatencyStats) -> Self {
661        Self {
662            avg: (self.avg + other.avg) / 2.0,
663            curr: self.curr.max(other.curr),
664            max: self.max.max(other.max),
665        }
666    }
667}
668
669#[derive(Debug, Clone, Default, Serialize, Deserialize)]
670pub struct BucketReplicationStat {
671    pub replicated_size: i64,
672    pub replicated_count: i64,
673    pub failed: FailedMetric,
674    pub fail_stats: FailStats,
675    pub latency: LatencyStats,
676    pub xfer_rate_lrg: XferStats,
677    pub xfer_rate_sml: XferStats,
678    pub bandwidth_limit_bytes_per_sec: i64,
679    pub current_bandwidth_bytes_per_sec: f64,
680    #[serde(default)]
681    pub latency_scope: ReplicationMetricScope,
682    #[serde(default)]
683    pub bandwidth_scope: ReplicationMetricScope,
684}
685
686impl BucketReplicationStat {
687    pub fn new() -> Self {
688        Self::default()
689    }
690
691    pub fn update_xfer_rate(&mut self, size: i64, duration: Duration) {
692        // Same boundary as the worker-pool split and minio-go's
693        // Large/Small transfer-summary labels: >= 128 MiB is "large".
694        if size >= crate::runtime::MIN_LARGE_OBJ_SIZE {
695            self.xfer_rate_lrg.add_size(size, duration);
696        } else {
697            self.xfer_rate_sml.add_size(size, duration);
698        }
699    }
700
701    pub fn set_node_local_bandwidth(&mut self, limit_bytes_per_sec: i64, current_bytes_per_sec: f64) {
702        self.bandwidth_limit_bytes_per_sec = limit_bytes_per_sec;
703        self.current_bandwidth_bytes_per_sec = current_bytes_per_sec;
704        self.bandwidth_scope = ReplicationMetricScope::NodeLocal;
705    }
706}
707
708#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
709#[serde(rename_all = "snake_case")]
710pub enum ReplicationMetricScope {
711    #[default]
712    Unavailable,
713    NodeLocal,
714    ClusterAggregated,
715    PartialCluster,
716}
717
718#[derive(Debug, Clone, Default, Serialize, Deserialize)]
719pub struct QueueStats {
720    pub nodes: Vec<QueueNode>,
721}
722
723#[derive(Debug, Clone, Default, Serialize, Deserialize)]
724pub struct QueueNode {
725    pub q_stats: InQueueMetric,
726}
727
728#[derive(Debug, Clone, Default, Serialize, Deserialize)]
729pub struct BucketReplicationStats {
730    pub stats: HashMap<String, BucketReplicationStat>,
731    pub replica_size: i64,
732    pub replica_count: i64,
733    pub replicated_size: i64,
734    pub replicated_count: i64,
735    pub q_stat: InQueueMetric,
736    #[serde(default)]
737    pub resync_started_count: i64,
738    #[serde(default)]
739    pub resync_completed_count: i64,
740    #[serde(default)]
741    pub resync_failed_count: i64,
742    #[serde(default)]
743    pub resync_canceled_count: i64,
744    #[serde(default)]
745    pub resync_duration_ms: i64,
746    #[serde(default)]
747    pub provider_available: bool,
748    #[serde(default)]
749    pub cluster_complete: bool,
750    #[serde(default)]
751    pub observed_node_count: u32,
752    #[serde(default)]
753    pub expected_node_count: u32,
754    #[serde(default)]
755    pub queue_scope: ReplicationMetricScope,
756}
757
758impl BucketReplicationStats {
759    pub fn new() -> Self {
760        Self::default()
761    }
762
763    pub fn is_empty(&self) -> bool {
764        self.stats.is_empty()
765            && self.replica_size == 0
766            && self.replicated_size == 0
767            && self.resync_started_count == 0
768            && self.resync_completed_count == 0
769            && self.resync_failed_count == 0
770            && self.resync_canceled_count == 0
771            && self.resync_duration_ms == 0
772    }
773
774    pub fn has_replication_usage(&self) -> bool {
775        self.replica_size > 0
776            || self.replicated_size > 0
777            || self.resync_started_count > 0
778            || self.resync_completed_count > 0
779            || self.resync_failed_count > 0
780            || self.resync_canceled_count > 0
781            || self.resync_duration_ms > 0
782            || !self.stats.is_empty()
783    }
784
785    pub fn clone_stats(&self) -> Self {
786        let mut snapshot = self.clone();
787        for stat in snapshot.stats.values_mut() {
788            stat.failed = stat.fail_stats.to_metric();
789        }
790        snapshot
791    }
792
793    pub fn mark_node_local_provider_available(&mut self) {
794        self.provider_available = true;
795        self.cluster_complete = true;
796        self.observed_node_count = 1;
797        self.expected_node_count = 1;
798    }
799
800    pub fn record_resync_status(&mut self, status: ResyncStatusType, duration: Option<Duration>) {
801        match status {
802            ResyncStatusType::ResyncStarted => self.resync_started_count += 1,
803            ResyncStatusType::ResyncCompleted => self.resync_completed_count += 1,
804            ResyncStatusType::ResyncFailed => self.resync_failed_count += 1,
805            ResyncStatusType::ResyncCanceled => self.resync_canceled_count += 1,
806            ResyncStatusType::NoResync | ResyncStatusType::ResyncPending => return,
807        }
808
809        if let Some(duration) = duration {
810            let duration_ms = duration.as_millis().min(I64_MAX_AS_U128);
811            if let Ok(duration_ms) = i64::try_from(duration_ms) {
812                self.resync_duration_ms = self.resync_duration_ms.saturating_add(duration_ms);
813            }
814        }
815    }
816}
817
818#[derive(Debug, Clone, Default, Serialize, Deserialize)]
819pub struct BucketStats {
820    pub uptime: i64,
821    pub replication_stats: BucketReplicationStats,
822    pub queue_stats: QueueStats,
823    pub proxy_stats: ProxyMetric,
824}
825
826#[derive(Debug, Clone, Default, Serialize, Deserialize)]
827pub struct SRMetricsSummary {
828    pub uptime: i64,
829    pub queued: InQueueMetric,
830    pub active_workers: ActiveWorkerStat,
831    pub metrics: HashMap<String, i64>,
832    pub proxied: ProxyMetric,
833    pub replica_size: i64,
834    pub replica_count: i64,
835}
836
837#[derive(Debug, Clone, Default, Serialize, Deserialize)]
838pub struct ActiveWorkerStat {
839    pub curr: i32,
840    pub max: i32,
841    pub avg: f64,
842    #[serde(skip)]
843    samples: VecDeque<WorkerSample>,
844}
845
846#[derive(Debug, Clone)]
847struct WorkerSample {
848    observed_at: Instant,
849    workers: i32,
850}
851
852impl ActiveWorkerStat {
853    pub fn new() -> Self {
854        Self::default()
855    }
856
857    pub fn get(&self) -> Self {
858        self.clone()
859    }
860
861    pub fn update(&mut self, curr: i32) {
862        let observed_at = Instant::now();
863        self.curr = curr;
864        self.samples.push_back(WorkerSample {
865            observed_at,
866            workers: curr,
867        });
868
869        while self
870            .samples
871            .front()
872            .is_some_and(|sample| observed_at.duration_since(sample.observed_at) > ROLLING_WINDOW)
873        {
874            self.samples.pop_front();
875        }
876
877        if self.samples.is_empty() {
878            self.max = curr;
879            self.avg = curr as f64;
880            return;
881        }
882
883        self.max = self.samples.iter().map(|sample| sample.workers).max().unwrap_or(curr);
884        let total = self.samples.iter().map(|sample| sample.workers as i64).sum::<i64>();
885        self.avg = total as f64 / self.samples.len() as f64;
886    }
887}
888
889#[cfg(test)]
890mod tests {
891    use super::*;
892
893    #[test]
894    fn in_queue_metric_observe_updates_rolling_stats() {
895        let mut metric = InQueueMetric::default();
896        metric.curr.add_current(128, 4);
897        metric.observe(Instant::now());
898
899        metric.curr.add_current(128, 2);
900        metric.observe(Instant::now());
901
902        assert_eq!(metric.curr.bytes, 256);
903        assert_eq!(metric.curr.count, 6);
904        assert_eq!(metric.max.bytes, 256);
905        assert_eq!(metric.max.count, 6);
906        assert_eq!(metric.last_minute.bytes, 192);
907        assert_eq!(metric.last_minute.count, 5);
908    }
909
910    #[test]
911    fn queue_cache_decrement_saturates_at_zero() {
912        let mut cache = QueueCache::new();
913        cache.inc("bucket", 128);
914        cache.dec("bucket", 512);
915        cache.dec("bucket", 1);
916
917        let bucket = cache.get_bucket_stats("bucket");
918        let site = cache.get_site_stats();
919        assert_eq!(bucket.curr.count, 0);
920        assert_eq!(bucket.curr.bytes, 0);
921        assert_eq!(site.curr.count, 0);
922        assert_eq!(site.curr.bytes, 0);
923    }
924
925    #[test]
926    fn fail_stats_recent_since_tracks_windows() {
927        let mut stats = FailStats::default();
928        stats.add_size(64, None::<&()>);
929        stats.add_size(32, None::<&()>);
930
931        let last_minute = stats.recent_since(Duration::from_secs(60));
932        let last_hour = stats.recent_since(Duration::from_secs(60 * 60));
933        assert_eq!(last_minute.count, 2);
934        assert_eq!(last_minute.size, 96);
935        assert_eq!(last_hour.count, 2);
936        assert_eq!(last_hour.size, 96);
937    }
938
939    #[test]
940    fn fail_stats_recent_windows_matches_two_separate_scans() {
941        let mut stats = FailStats::default();
942        stats.add_size(64, None::<&()>);
943        stats.add_size(32, None::<&()>);
944
945        let (minute, hour) = stats.recent_windows(Duration::from_secs(60), Duration::from_secs(60 * 60));
946        let expected_minute = stats.recent_since(Duration::from_secs(60));
947        let expected_hour = stats.recent_since(Duration::from_secs(60 * 60));
948
949        assert_eq!((minute.count, minute.size), (expected_minute.count, expected_minute.size));
950        assert_eq!((hour.count, hour.size), (expected_hour.count, expected_hour.size));
951        assert_eq!(minute.count, 2);
952        assert_eq!(hour.size, 96);
953
954        let empty = FailStats::default();
955        let (minute, hour) = empty.recent_windows(Duration::from_secs(60), Duration::from_secs(60 * 60));
956        assert_eq!((minute.count, minute.size, hour.count, hour.size), (0, 0, 0, 0));
957    }
958
959    #[test]
960    fn fail_stats_saturate_instead_of_wrapping() {
961        let mut stats = FailStats {
962            count: i64::MAX,
963            size: i64::MAX,
964            ..Default::default()
965        };
966
967        stats.add_size(1, None::<&()>);
968        let merged = stats.merge(&FailStats {
969            count: 1,
970            size: 1,
971            ..Default::default()
972        });
973
974        assert_eq!(stats.count, i64::MAX);
975        assert_eq!(stats.size, i64::MAX);
976        assert_eq!(merged.count, i64::MAX);
977        assert_eq!(merged.size, i64::MAX);
978    }
979
980    #[test]
981    fn active_worker_stat_update_tracks_rolling_avg_and_max() {
982        let mut stats = ActiveWorkerStat::default();
983        stats.update(2);
984        stats.update(6);
985        stats.update(4);
986
987        assert_eq!(stats.curr, 4);
988        assert_eq!(stats.max, 6);
989        assert_eq!(stats.avg, 4.0);
990    }
991}