1use crate::last_minute::{AccElem, LastMinuteLatency};
16use chrono::{DateTime, Utc};
17use jiff::Timestamp;
18use rustfs_heal_contracts::heal_channel::HealScanMode;
19use serde::{Deserialize, Serialize};
20use std::{
21 collections::{BTreeSet, HashMap},
22 fmt::Display,
23 future::Future,
24 pin::Pin,
25 sync::{
26 Arc, Mutex, OnceLock,
27 atomic::{AtomicBool, AtomicU8, AtomicU64, Ordering},
28 },
29 time::{Duration, SystemTime},
30};
31use tokio::sync::RwLock;
32
33#[derive(Debug, Clone, Copy, PartialEq, Eq)]
34pub enum IlmAction {
35 NoneAction = 0,
36 DeleteAction,
37 DeleteVersionAction,
38 TransitionAction,
39 TransitionVersionAction,
40 DeleteRestoredAction,
41 DeleteRestoredVersionAction,
42 DeleteAllVersionsAction,
43 DelMarkerDeleteAllVersionsAction,
44 ActionCount,
45}
46
47impl IlmAction {
48 pub fn delete_restored(&self) -> bool {
49 *self == Self::DeleteRestoredAction || *self == Self::DeleteRestoredVersionAction
50 }
51
52 pub fn delete_versioned(&self) -> bool {
53 *self == Self::DeleteVersionAction || *self == Self::DeleteRestoredVersionAction
54 }
55
56 pub fn delete_all(&self) -> bool {
57 *self == Self::DeleteAllVersionsAction || *self == Self::DelMarkerDeleteAllVersionsAction
58 }
59
60 pub fn delete(&self) -> bool {
61 if self.delete_restored() {
62 return true;
63 }
64 *self == Self::DeleteVersionAction
65 || *self == Self::DeleteAction
66 || *self == Self::DeleteAllVersionsAction
67 || *self == Self::DelMarkerDeleteAllVersionsAction
68 }
69
70 pub fn as_str(&self) -> &'static str {
71 match self {
72 Self::NoneAction => "none",
73 Self::DeleteAction => "delete",
74 Self::DeleteVersionAction => "delete_version",
75 Self::TransitionAction => "transition",
76 Self::TransitionVersionAction => "transition_version",
77 Self::DeleteRestoredAction => "delete_restored",
78 Self::DeleteRestoredVersionAction => "delete_restored_version",
79 Self::DeleteAllVersionsAction => "delete_all_versions",
80 Self::DelMarkerDeleteAllVersionsAction => "del_marker_delete_all_versions",
81 Self::ActionCount => "action_count",
82 }
83 }
84
85 pub fn from_index(i: usize) -> Option<Self> {
86 match i {
87 0 => Some(Self::NoneAction),
88 1 => Some(Self::DeleteAction),
89 2 => Some(Self::DeleteVersionAction),
90 3 => Some(Self::TransitionAction),
91 4 => Some(Self::TransitionVersionAction),
92 5 => Some(Self::DeleteRestoredAction),
93 6 => Some(Self::DeleteRestoredVersionAction),
94 7 => Some(Self::DeleteAllVersionsAction),
95 8 => Some(Self::DelMarkerDeleteAllVersionsAction),
96 9 => Some(Self::ActionCount),
97 _ => None,
98 }
99 }
100}
101
102impl Display for IlmAction {
103 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
104 write!(f, "{self:?}")
105 }
106}
107
108pub static GLOBAL_METRICS: OnceLock<Arc<Metrics>> = OnceLock::new();
109
110pub fn global_metrics() -> &'static Arc<Metrics> {
111 GLOBAL_METRICS.get_or_init(|| Arc::new(Metrics::new()))
112}
113
114#[derive(Debug, Clone, Copy, PartialEq, Eq)]
115pub enum Metric {
116 ReadMetadata = 0,
119 CheckMissing,
120 SaveUsage,
121 ApplyAll,
122 ApplyVersion,
123 TierObjSweep,
124 HealCheck,
125 Ilm,
126 CheckReplication,
127 Yield,
128 ThrottleSleep,
129 CleanAbandoned,
130 ApplyNonCurrent,
131 HealAbandonedVersion,
132
133 QuotaCheck,
135 QuotaViolation,
136 QuotaSync,
137
138 StartTrace,
140 ScanObject, HealAbandonedObject,
142
143 LastRealtime,
145
146 ScanFolder, ScanCycle, ScanBucketDrive, CompactFolder, ScanBucketDriveStart,
152 ScanBucketDriveFailure,
153
154 Last,
156}
157
158impl Metric {
159 pub fn as_str(&self) -> &'static str {
161 match self {
162 Self::ReadMetadata => "read_metadata",
163 Self::CheckMissing => "check_missing",
164 Self::SaveUsage => "save_usage",
165 Self::ApplyAll => "apply_all",
166 Self::ApplyVersion => "apply_version",
167 Self::TierObjSweep => "tier_obj_sweep",
168 Self::HealCheck => "heal_check",
169 Self::Ilm => "ilm",
170 Self::CheckReplication => "check_replication",
171 Self::Yield => "yield",
172 Self::ThrottleSleep => "throttle_sleep",
173 Self::CleanAbandoned => "clean_abandoned",
174 Self::ApplyNonCurrent => "apply_non_current",
175 Self::HealAbandonedVersion => "heal_abandoned_version",
176 Self::QuotaCheck => "quota_check",
177 Self::QuotaViolation => "quota_violation",
178 Self::QuotaSync => "quota_sync",
179 Self::StartTrace => "start_trace",
180 Self::ScanObject => "scan_object",
181 Self::HealAbandonedObject => "heal_abandoned_object",
182 Self::LastRealtime => "last_realtime",
183 Self::ScanFolder => "scan_folder",
184 Self::ScanCycle => "scan_cycle",
185 Self::ScanBucketDrive => "scan_bucket_drive",
186 Self::CompactFolder => "compact_folder",
187 Self::ScanBucketDriveStart => "scan_bucket_drive_start",
188 Self::ScanBucketDriveFailure => "scan_bucket_drive_failure",
189 Self::Last => "last",
190 }
191 }
192
193 pub fn from_index(index: usize) -> Option<Self> {
195 if index >= Self::Last as usize {
196 return None;
197 }
198 match index {
199 0 => Some(Self::ReadMetadata),
200 1 => Some(Self::CheckMissing),
201 2 => Some(Self::SaveUsage),
202 3 => Some(Self::ApplyAll),
203 4 => Some(Self::ApplyVersion),
204 5 => Some(Self::TierObjSweep),
205 6 => Some(Self::HealCheck),
206 7 => Some(Self::Ilm),
207 8 => Some(Self::CheckReplication),
208 9 => Some(Self::Yield),
209 10 => Some(Self::ThrottleSleep),
210 11 => Some(Self::CleanAbandoned),
211 12 => Some(Self::ApplyNonCurrent),
212 13 => Some(Self::HealAbandonedVersion),
213 14 => Some(Self::QuotaCheck),
214 15 => Some(Self::QuotaViolation),
215 16 => Some(Self::QuotaSync),
216 17 => Some(Self::StartTrace),
217 18 => Some(Self::ScanObject),
218 19 => Some(Self::HealAbandonedObject),
219 20 => Some(Self::LastRealtime),
220 21 => Some(Self::ScanFolder),
221 22 => Some(Self::ScanCycle),
222 23 => Some(Self::ScanBucketDrive),
223 24 => Some(Self::CompactFolder),
224 25 => Some(Self::ScanBucketDriveStart),
225 26 => Some(Self::ScanBucketDriveFailure),
226 27 => Some(Self::Last),
227 _ => None,
228 }
229 }
230}
231
232const SCANNER_CHECKPOINT_EVENT_SET: &str = "set";
233const SCANNER_CHECKPOINT_EVENT_USED: &str = "used";
234const SCANNER_CHECKPOINT_EVENT_IGNORED: &str = "ignored";
235const SCANNER_CHECKPOINT_EVENT_STALE: &str = "stale";
236
237#[derive(Clone, Copy, Debug, PartialEq, Eq)]
238pub enum ScannerWorkSource {
239 Usage,
240 Lifecycle,
241 BucketReplication,
242 SiteReplication,
243 Heal,
244 Bitrot,
245 Alerts,
246}
247
248impl ScannerWorkSource {
249 const ALL: [Self; 7] = [
250 Self::Usage,
251 Self::Lifecycle,
252 Self::BucketReplication,
253 Self::SiteReplication,
254 Self::Heal,
255 Self::Bitrot,
256 Self::Alerts,
257 ];
258
259 pub fn as_str(self) -> &'static str {
260 match self {
261 Self::Usage => "usage",
262 Self::Lifecycle => "lifecycle",
263 Self::BucketReplication => "bucket_replication",
264 Self::SiteReplication => "site_replication",
265 Self::Heal => "heal",
266 Self::Bitrot => "bitrot",
267 Self::Alerts => "alerts",
268 }
269 }
270
271 fn code(self) -> u8 {
272 match self {
273 Self::Usage => 1,
274 Self::Lifecycle => 2,
275 Self::BucketReplication => 3,
276 Self::SiteReplication => 4,
277 Self::Heal => 5,
278 Self::Bitrot => 6,
279 Self::Alerts => 7,
280 }
281 }
282
283 fn from_code(code: u8) -> Option<Self> {
284 match code {
285 1 => Some(Self::Usage),
286 2 => Some(Self::Lifecycle),
287 3 => Some(Self::BucketReplication),
288 4 => Some(Self::SiteReplication),
289 5 => Some(Self::Heal),
290 6 => Some(Self::Bitrot),
291 7 => Some(Self::Alerts),
292 _ => None,
293 }
294 }
295
296 fn all() -> &'static [Self] {
297 &Self::ALL
298 }
299
300 fn index(self) -> usize {
301 match self {
302 Self::Usage => 0,
303 Self::Lifecycle => 1,
304 Self::BucketReplication => 2,
305 Self::SiteReplication => 3,
306 Self::Heal => 4,
307 Self::Bitrot => 5,
308 Self::Alerts => 6,
309 }
310 }
311}
312
313#[derive(Debug, Clone, Copy, PartialEq, Eq)]
314pub enum ScannerReplicationRepairKind {
315 BucketObject,
316 BucketDeleteMarker,
317 BucketVersionPurge,
318 BucketExistingObject,
319 SitePassiveRequeue,
320 SiteActiveResync,
321}
322
323#[derive(Debug, Clone, Copy, PartialEq, Eq)]
324pub enum ScannerReplicationRepairRole {
325 RepairAdmission,
326 BoundarySignal,
327}
328
329impl ScannerReplicationRepairRole {
330 pub fn as_str(self) -> &'static str {
331 match self {
332 Self::RepairAdmission => "repair_admission",
333 Self::BoundarySignal => "boundary_signal",
334 }
335 }
336}
337
338#[derive(Debug, Clone, Copy, PartialEq, Eq)]
339pub enum ScannerReplicationExecutionOwner {
340 BucketReplicationQueue,
341 SiteReplicationRuntime,
342}
343
344impl ScannerReplicationExecutionOwner {
345 pub fn as_str(self) -> &'static str {
346 match self {
347 Self::BucketReplicationQueue => "bucket_replication_queue",
348 Self::SiteReplicationRuntime => "site_replication_runtime",
349 }
350 }
351}
352
353impl ScannerReplicationRepairKind {
354 const ALL: [Self; 6] = [
355 Self::BucketObject,
356 Self::BucketDeleteMarker,
357 Self::BucketVersionPurge,
358 Self::BucketExistingObject,
359 Self::SitePassiveRequeue,
360 Self::SiteActiveResync,
361 ];
362
363 pub fn source(self) -> ScannerWorkSource {
364 match self {
365 Self::BucketObject | Self::BucketDeleteMarker | Self::BucketVersionPurge | Self::BucketExistingObject => {
366 ScannerWorkSource::BucketReplication
367 }
368 Self::SitePassiveRequeue | Self::SiteActiveResync => ScannerWorkSource::SiteReplication,
369 }
370 }
371
372 pub fn scanner_role(self) -> ScannerReplicationRepairRole {
373 match self {
374 Self::BucketObject | Self::BucketDeleteMarker | Self::BucketVersionPurge | Self::BucketExistingObject => {
375 ScannerReplicationRepairRole::RepairAdmission
376 }
377 Self::SitePassiveRequeue | Self::SiteActiveResync => ScannerReplicationRepairRole::BoundarySignal,
378 }
379 }
380
381 pub fn execution_owner(self) -> ScannerReplicationExecutionOwner {
382 match self {
383 Self::BucketObject | Self::BucketDeleteMarker | Self::BucketVersionPurge | Self::BucketExistingObject => {
384 ScannerReplicationExecutionOwner::BucketReplicationQueue
385 }
386 Self::SitePassiveRequeue | Self::SiteActiveResync => ScannerReplicationExecutionOwner::SiteReplicationRuntime,
387 }
388 }
389
390 pub fn as_str(self) -> &'static str {
391 match self {
392 Self::BucketObject => "object",
393 Self::BucketDeleteMarker => "delete_marker",
394 Self::BucketVersionPurge => "version_purge",
395 Self::BucketExistingObject => "existing_object",
396 Self::SitePassiveRequeue => "passive_requeue",
397 Self::SiteActiveResync => "active_resync",
398 }
399 }
400
401 fn all() -> &'static [Self] {
402 &Self::ALL
403 }
404
405 fn index(self) -> usize {
406 match self {
407 Self::BucketObject => 0,
408 Self::BucketDeleteMarker => 1,
409 Self::BucketVersionPurge => 2,
410 Self::BucketExistingObject => 3,
411 Self::SitePassiveRequeue => 4,
412 Self::SiteActiveResync => 5,
413 }
414 }
415}
416
417#[derive(Debug, Default)]
418struct ScannerSourceWorkCounters {
419 checked: AtomicU64,
420 queued: AtomicU64,
421 executed: AtomicU64,
422 failed: AtomicU64,
423 skipped: AtomicU64,
424 missed: AtomicU64,
425}
426
427#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
428pub struct ScannerSourceWorkUpdate {
429 pub checked: u64,
430 pub queued: u64,
431 pub executed: u64,
432 pub failed: u64,
433 pub skipped: u64,
434 pub missed: u64,
435}
436
437impl ScannerSourceWorkUpdate {
438 pub const fn checked(count: u64) -> Self {
439 Self {
440 checked: count,
441 queued: 0,
442 executed: 0,
443 failed: 0,
444 skipped: 0,
445 missed: 0,
446 }
447 }
448
449 pub const fn queued(count: u64) -> Self {
450 Self {
451 checked: 0,
452 queued: count,
453 executed: 0,
454 failed: 0,
455 skipped: 0,
456 missed: 0,
457 }
458 }
459
460 pub const fn executed(count: u64) -> Self {
461 Self {
462 checked: 0,
463 queued: 0,
464 executed: count,
465 failed: 0,
466 skipped: 0,
467 missed: 0,
468 }
469 }
470
471 pub const fn failed(count: u64) -> Self {
472 Self {
473 checked: 0,
474 queued: 0,
475 executed: 0,
476 failed: count,
477 skipped: 0,
478 missed: 0,
479 }
480 }
481
482 pub const fn missed(count: u64) -> Self {
483 Self {
484 checked: 0,
485 queued: 0,
486 executed: 0,
487 failed: 0,
488 skipped: 0,
489 missed: count,
490 }
491 }
492}
493
494impl ScannerSourceWorkCounters {
495 fn add(&self, work: ScannerSourceWorkUpdate) {
496 self.checked.fetch_add(work.checked, Ordering::Relaxed);
497 self.queued.fetch_add(work.queued, Ordering::Relaxed);
498 self.executed.fetch_add(work.executed, Ordering::Relaxed);
499 self.failed.fetch_add(work.failed, Ordering::Relaxed);
500 self.skipped.fetch_add(work.skipped, Ordering::Relaxed);
501 self.missed.fetch_add(work.missed, Ordering::Relaxed);
502 }
503
504 fn store(&self, values: ScannerSourceWorkValues) {
505 self.checked.store(values.checked, Ordering::Relaxed);
506 self.queued.store(values.queued, Ordering::Relaxed);
507 self.executed.store(values.executed, Ordering::Relaxed);
508 self.failed.store(values.failed, Ordering::Relaxed);
509 self.skipped.store(values.skipped, Ordering::Relaxed);
510 self.missed.store(values.missed, Ordering::Relaxed);
511 }
512
513 fn values(&self) -> ScannerSourceWorkValues {
514 ScannerSourceWorkValues {
515 checked: self.checked.load(Ordering::Relaxed),
516 queued: self.queued.load(Ordering::Relaxed),
517 executed: self.executed.load(Ordering::Relaxed),
518 failed: self.failed.load(Ordering::Relaxed),
519 skipped: self.skipped.load(Ordering::Relaxed),
520 missed: self.missed.load(Ordering::Relaxed),
521 }
522 }
523
524 fn snapshot(&self, source: ScannerWorkSource) -> ScannerSourceWorkSnapshot {
525 self.values().snapshot(source)
526 }
527}
528
529#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
530struct ScannerSourceWorkValues {
531 checked: u64,
532 queued: u64,
533 executed: u64,
534 failed: u64,
535 skipped: u64,
536 missed: u64,
537}
538
539impl ScannerSourceWorkValues {
540 fn saturating_sub(self, start: Self) -> Self {
541 Self {
542 checked: self.checked.saturating_sub(start.checked),
543 queued: self.queued.saturating_sub(start.queued),
544 executed: self.executed.saturating_sub(start.executed),
545 failed: self.failed.saturating_sub(start.failed),
546 skipped: self.skipped.saturating_sub(start.skipped),
547 missed: self.missed.saturating_sub(start.missed),
548 }
549 }
550
551 fn snapshot(self, source: ScannerWorkSource) -> ScannerSourceWorkSnapshot {
552 ScannerSourceWorkSnapshot {
553 source: source.as_str().to_string(),
554 checked: self.checked,
555 queued: self.queued,
556 executed: self.executed,
557 failed: self.failed,
558 skipped: self.skipped,
559 missed: self.missed,
560 }
561 }
562
563 fn replication_repair_snapshot(self, kind: ScannerReplicationRepairKind) -> ScannerReplicationRepairSnapshot {
564 ScannerReplicationRepairSnapshot {
565 source: kind.source().as_str().to_string(),
566 kind: kind.as_str().to_string(),
567 scanner_role: kind.scanner_role().as_str().to_string(),
568 execution_owner: kind.execution_owner().as_str().to_string(),
569 checked: self.checked,
570 queued: self.queued,
571 executed: self.executed,
572 failed: self.failed,
573 skipped: self.skipped,
574 missed: self.missed,
575 }
576 }
577}
578
579pub struct LockedLastMinuteLatency {
599 latency: Arc<Mutex<LastMinuteLatency>>,
603}
604
605impl Default for LockedLastMinuteLatency {
606 fn default() -> Self {
607 Self::new()
608 }
609}
610
611impl Clone for LockedLastMinuteLatency {
614 fn clone(&self) -> Self {
615 let inner = match self.latency.lock() {
616 Ok(guard) => guard.clone(),
617 Err(poisoned) => poisoned.into_inner().clone(),
618 };
619 Self {
620 latency: Arc::new(Mutex::new(inner)),
621 }
622 }
623}
624
625impl LockedLastMinuteLatency {
626 pub fn new() -> Self {
627 Self {
628 latency: Arc::new(Mutex::new(LastMinuteLatency::default())),
629 }
630 }
631
632 pub fn add(&self, duration: Duration) {
634 self.add_size(duration, 0);
635 }
636
637 pub fn add_size(&self, duration: Duration, size: u64) {
639 let now = SystemTime::now()
640 .duration_since(SystemTime::UNIX_EPOCH)
641 .unwrap_or_default()
642 .as_secs();
643
644 let elem = AccElem {
645 n: 1,
646 total: duration.as_secs(),
647 size,
648 };
649
650 match self.latency.lock() {
651 Ok(mut guard) => guard.add_all(now, &elem),
652 Err(poisoned) => poisoned.into_inner().add_all(now, &elem),
653 }
654 }
655
656 pub fn total(&self) -> AccElem {
658 match self.latency.lock() {
659 Ok(mut latency) => latency.get_total(),
660 Err(poisoned) => poisoned.into_inner().get_total(),
661 }
662 }
663}
664
665#[derive(Clone, Debug)]
671struct CurrentPathState {
672 path: String,
673 updated_at: Timestamp,
674}
675
676struct CurrentPathTracker {
677 state: Arc<RwLock<CurrentPathState>>,
678}
679
680impl CurrentPathTracker {
681 fn new(initial_path: String) -> Self {
682 Self::new_at(initial_path, Timestamp::now())
683 }
684
685 fn new_at(initial_path: String, updated_at: Timestamp) -> Self {
686 Self {
687 state: Arc::new(RwLock::new(CurrentPathState {
688 path: initial_path,
689 updated_at,
690 })),
691 }
692 }
693
694 async fn update_path(&self, path: String) {
695 let mut state = self.state.write().await;
696 state.path = path;
697 state.updated_at = Timestamp::now();
698 }
699
700 async fn get_state(&self) -> CurrentPathState {
701 self.state.read().await.clone()
702 }
703}
704
705fn chrono_to_jiff_timestamp(dt: DateTime<Utc>) -> Timestamp {
706 let seconds = dt.timestamp();
707 let nanoseconds = match i32::try_from(dt.timestamp_subsec_nanos()) {
708 Ok(nanoseconds) => nanoseconds,
709 Err(_) => {
710 return if seconds < 0 { Timestamp::MIN } else { Timestamp::MAX };
711 }
712 };
713
714 match Timestamp::new(seconds, nanoseconds) {
715 Ok(timestamp) => timestamp,
716 Err(_) => {
717 if seconds < 0 {
718 Timestamp::MIN
719 } else {
720 Timestamp::MAX
721 }
722 }
723 }
724}
725
726fn timestamp_elapsed_seconds_since(now: Timestamp, earlier: Timestamp) -> u64 {
727 let duration = now.duration_since(earlier);
728 if duration.is_negative() {
729 return 0;
730 }
731
732 u64::try_from(duration.as_secs()).unwrap_or(u64::MAX)
733}
734
735#[derive(Clone, Copy, Debug, Default)]
736struct ScannerDiskBucketScanState {
737 concurrency_limit: u64,
738 queued: u64,
739 active: u64,
740}
741
742type ScannerDiskBucketScanKey = (String, String);
743
744#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
745pub struct ScannerDiskBucketScanSnapshot {
746 pub pool: String,
747 pub set: String,
748 pub concurrency_limit: u64,
749 pub queued: u64,
750 pub active: u64,
751}
752
753#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
754struct ScannerBucketDriveResultKey {
755 bucket: String,
756 drive: String,
757 result: String,
758}
759
760impl ScannerBucketDriveResultKey {
761 fn new(bucket: impl Into<String>, drive: impl Into<String>, result: impl Into<String>) -> Self {
762 Self {
763 bucket: bucket.into(),
764 drive: drive.into(),
765 result: result.into(),
766 }
767 }
768}
769
770const MAX_SCANNER_BUCKET_DRIVE_RESULT_KEYS: usize = 4096;
771
772#[derive(Debug, Default)]
773struct ScannerBucketDriveResults {
774 counts: HashMap<ScannerBucketDriveResultKey, ScannerBucketDriveResultValue>,
775 eviction_index: BTreeSet<(u64, ScannerBucketDriveResultKey)>,
776}
777
778#[derive(Clone, Copy, Debug)]
779struct ScannerBucketDriveResultValue {
780 count: u64,
781 last_seen: u64,
782}
783
784#[derive(Clone, Debug, Eq, Hash, PartialEq)]
785struct ScannerActiveBucketDriveKey {
786 source: String,
787 bucket: String,
788 drive: String,
789}
790
791#[derive(Clone, Copy, Debug)]
792struct ScannerActiveBucketDriveValue {
793 count: u64,
794 started_at: Timestamp,
795}
796
797#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]
807pub enum TierRequestOperation {
808 Put,
809 Get,
810 Remove,
811 Probe,
812 InUse,
813}
814
815impl TierRequestOperation {
816 pub const ALL: [Self; 5] = [Self::Put, Self::Get, Self::Remove, Self::Probe, Self::InUse];
817
818 pub const fn as_label(self) -> &'static str {
819 match self {
820 Self::Put => "put",
821 Self::Get => "get",
822 Self::Remove => "remove",
823 Self::Probe => "probe",
824 Self::InUse => "in_use",
825 }
826 }
827
828 const fn index(self) -> usize {
829 self as usize
830 }
831}
832
833#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]
840pub enum TierRequestOutcome {
841 Success,
842 BackendError,
843 Timeout,
844 Cancelled,
845}
846
847impl TierRequestOutcome {
848 pub const ALL: [Self; 4] = [Self::Success, Self::BackendError, Self::Timeout, Self::Cancelled];
849
850 pub const fn as_label(self) -> &'static str {
851 match self {
852 Self::Success => "success",
853 Self::BackendError => "backend_error",
854 Self::Timeout => "timeout",
855 Self::Cancelled => "cancelled",
856 }
857 }
858
859 const fn index(self) -> usize {
860 self as usize
861 }
862
863 pub fn from_error(err: &std::io::Error) -> Self {
865 match err.kind() {
866 std::io::ErrorKind::TimedOut => Self::Timeout,
867 std::io::ErrorKind::Interrupted => Self::Cancelled,
868 _ => Self::BackendError,
869 }
870 }
871}
872
873#[derive(Clone, Copy, Debug, Eq, PartialEq)]
875pub struct TierRequestCount {
876 pub operation: TierRequestOperation,
877 pub outcome: TierRequestOutcome,
878 pub count: u64,
879}
880
881const TIER_REQUEST_COUNTER_SLOTS: usize = TierRequestOperation::ALL.len() * TierRequestOutcome::ALL.len();
882
883pub struct Metrics {
884 operations: Vec<AtomicU64>,
885 latency: Vec<LockedLastMinuteLatency>,
886 actions: Vec<AtomicU64>,
887 actions_latency: Vec<LockedLastMinuteLatency>,
888 current_paths: Arc<RwLock<HashMap<String, Arc<CurrentPathTracker>>>>,
889 cycle_info: Arc<RwLock<Option<CurrentCycle>>>,
890 current_scan_mode: AtomicU8,
891 current_scan_cycle_work_active: AtomicBool,
892 current_scan_cycle_objects_start: AtomicU64,
893 current_scan_cycle_directories_start: AtomicU64,
894 current_scan_cycle_bucket_drive_scans_start: AtomicU64,
895 current_scan_cycle_bucket_drive_failures_start: AtomicU64,
896 current_scan_cycle_yield_events_start: AtomicU64,
897 current_scan_cycle_yield_duration_millis_start: AtomicU64,
898 current_scan_cycle_throttle_sleep_events_start: AtomicU64,
899 current_scan_cycle_throttle_sleep_duration_millis_start: AtomicU64,
900 current_scan_cycle_ilm_actions_start: AtomicU64,
901 current_scan_cycle_lifecycle_expiry_actions_start: AtomicU64,
902 current_scan_cycle_lifecycle_transition_actions_start: AtomicU64,
903 current_scan_cycle_heal_objects_start: AtomicU64,
904 current_scan_cycle_replication_checks_start: AtomicU64,
905 current_scan_cycle_usage_saves_start: AtomicU64,
906 scanner_set_scan_concurrency_limit: AtomicU64,
907 scanner_set_scans_queued: AtomicU64,
908 scanner_set_scans_active: AtomicU64,
909 scanner_disk_bucket_scan_states: Mutex<HashMap<ScannerDiskBucketScanKey, ScannerDiskBucketScanState>>,
910 scanner_bucket_drive_results: Mutex<ScannerBucketDriveResults>,
911 scanner_active_bucket_drive_scans: Mutex<HashMap<ScannerActiveBucketDriveKey, ScannerActiveBucketDriveValue>>,
912 scanner_bucket_drive_result_clock: AtomicU64,
913 current_scan_cycle_bucket_drive_results_start: Mutex<HashMap<ScannerBucketDriveResultKey, u64>>,
914 last_scan_cycle_bucket_drive_results: Mutex<Vec<ScannerBucketDriveResultSnapshot>>,
915 scanner_leader_lock_state: RwLock<String>,
916 scanner_leader_lock_held: AtomicBool,
917 scanner_leader_lock_last_error: RwLock<String>,
918 scanner_leader_lock_last_update_unix_secs: AtomicU64,
919 last_scan_cycle_result: AtomicU8,
920 last_scan_cycle_partial_reason: AtomicU8,
921 last_scan_cycle_partial_source: AtomicU8,
922 last_scan_cycle_end_unix_secs: AtomicU64,
923 last_scan_cycle_duration_millis: AtomicU64,
924 last_scan_cycle_objects_scanned: AtomicU64,
925 last_scan_cycle_directories_scanned: AtomicU64,
926 lifetime_versions_scanned: AtomicU64,
931 last_scan_cycle_bucket_drive_scans: AtomicU64,
932 last_scan_cycle_bucket_drive_failures: AtomicU64,
933 last_scan_cycle_yield_events: AtomicU64,
934 last_scan_cycle_yield_duration_millis: AtomicU64,
935 last_scan_cycle_throttle_sleep_events: AtomicU64,
936 last_scan_cycle_throttle_sleep_duration_millis: AtomicU64,
937 last_scan_cycle_ilm_actions: AtomicU64,
938 last_scan_cycle_lifecycle_expiry_actions: AtomicU64,
939 last_scan_cycle_lifecycle_transition_actions: AtomicU64,
940 last_scan_cycle_heal_objects: AtomicU64,
941 last_scan_cycle_replication_checks: AtomicU64,
942 last_scan_cycle_usage_saves: AtomicU64,
943 failed_scan_cycles: AtomicU64,
944 superseded_scan_cycles: AtomicU64,
945 partial_scan_cycles_unknown: AtomicU64,
946 partial_scan_cycles_runtime: AtomicU64,
947 partial_scan_cycles_objects: AtomicU64,
948 partial_scan_cycles_directories: AtomicU64,
949 partial_scan_cycles_by_source: Vec<AtomicU64>,
950 scanner_yield_duration_millis: AtomicU64,
951 scanner_throttle_sleep_duration_millis: AtomicU64,
952 scanner_ilm_actions: AtomicU64,
953 scanner_lifecycle_expiry_actions: AtomicU64,
954 scanner_lifecycle_transition_actions: AtomicU64,
955 scanner_expiry_queue_capacity: AtomicU64,
956 scanner_expiry_queued: AtomicU64,
957 scanner_expiry_active: AtomicU64,
958 scanner_expiry_workers: AtomicU64,
959 scanner_expiry_queue_missed: AtomicU64,
960 scanner_expiry_queued_total: AtomicU64,
961 scanner_expiry_missed_total: AtomicU64,
962 scanner_expiry_blocked_total: AtomicU64,
963 scanner_expiry_not_enqueued_total: AtomicU64,
964 scanner_expiry_delete_failed_total: AtomicU64,
965 scanner_transition_queue_capacity: AtomicU64,
966 scanner_transition_queued: AtomicU64,
967 scanner_transition_active: AtomicU64,
968 scanner_transition_workers: AtomicU64,
969 scanner_transition_queue_full: AtomicU64,
970 scanner_transition_queue_send_timeout: AtomicU64,
971 scanner_transition_compensation_scheduled: AtomicU64,
972 scanner_transition_compensation_pending: AtomicU64,
973 scanner_transition_compensation_running: AtomicU64,
974 scanner_transition_queued_total: AtomicU64,
975 scanner_transition_missed_total: AtomicU64,
976 scanner_transition_completed: AtomicU64,
977 tier_requests: [AtomicU64; TIER_REQUEST_COUNTER_SLOTS],
981 scanner_transition_failed: AtomicU64,
982 scanner_throttle_idle_mode_enabled: AtomicBool,
983 scanner_throttle_sleep_factor_micros: AtomicU64,
984 scanner_throttle_max_sleep_millis: AtomicU64,
985 scanner_yield_every_n_objects: AtomicU64,
986 scanner_cycle_interval_millis: AtomicU64,
987 scanner_cycle_max_duration_millis: AtomicU64,
988 scanner_cycle_max_objects: AtomicU64,
989 scanner_cycle_max_directories: AtomicU64,
990 scanner_cycle_timeout_total: AtomicU64,
991 scanner_cycle_recovery_required_total: AtomicU64,
992 scanner_cycle_last_progress_age_seconds: AtomicU64,
993 scanner_leader_lease_without_progress: AtomicBool,
994 scanner_bitrot_cycle_enabled: AtomicBool,
995 scanner_bitrot_cycle_millis: AtomicU64,
996 scanner_checkpoint: Mutex<Option<ScannerCheckpointReport>>,
997 scanner_checkpoint_used: AtomicU64,
998 scanner_checkpoint_cleared: AtomicU64,
999 scanner_checkpoint_ignored: AtomicU64,
1000 scanner_checkpoint_stale: AtomicU64,
1001 scanner_dirty_usage_pending_buckets: AtomicU64,
1002 scanner_dirty_usage_last_mark_unix_secs: AtomicU64,
1003 scanner_dirty_usage_last_clear_unix_secs: AtomicU64,
1004 scanner_dirty_usage_last_cycle_dirty_buckets: AtomicU64,
1005 scanner_dirty_usage_last_cycle_cleared_buckets: AtomicU64,
1006 scanner_usage_last_save_unix_secs: AtomicU64,
1007 scanner_usage_last_durable_success_unix_secs: AtomicU64,
1008 scanner_usage_last_publication_unix_secs: AtomicU64,
1009 scanner_usage_last_publication_state: Mutex<String>,
1010 scanner_usage_last_publication_reason: Mutex<String>,
1011 scanner_usage_last_save_result: AtomicU8,
1012 scanner_usage_deferred_pending: AtomicBool,
1013 scanner_usage_deferred_total: AtomicU64,
1014 scanner_usage_last_deferred_unix_secs: AtomicU64,
1015 scanner_usage_last_deferred_reason: Mutex<String>,
1016 scanner_source_work: Vec<ScannerSourceWorkCounters>,
1017 current_scan_cycle_source_work_start: Vec<ScannerSourceWorkCounters>,
1018 last_scan_cycle_source_work: Vec<ScannerSourceWorkCounters>,
1019 scanner_replication_repair_work: Vec<ScannerSourceWorkCounters>,
1020 current_scan_cycle_replication_repair_work_start: Vec<ScannerSourceWorkCounters>,
1021 last_scan_cycle_replication_repair_work: Vec<ScannerSourceWorkCounters>,
1022 partial_scan_cycles: AtomicU64,
1023}
1024
1025const SCAN_CYCLE_RESULT_UNKNOWN: u8 = 0;
1026const SCAN_CYCLE_RESULT_SUCCESS: u8 = 1;
1027const SCAN_CYCLE_RESULT_ERROR: u8 = 2;
1028const SCAN_CYCLE_RESULT_PARTIAL: u8 = 3;
1029const SCAN_CYCLE_RESULT_SUPERSEDED: u8 = 4;
1030const SCAN_CYCLE_RESULT_DEFERRED: u8 = 5;
1031const SCAN_CYCLE_RESULT_UNKNOWN_LABEL: &str = "unknown";
1032const SCAN_CYCLE_RESULT_SUCCESS_LABEL: &str = "success";
1033const SCAN_CYCLE_RESULT_ERROR_LABEL: &str = "error";
1034const SCAN_CYCLE_RESULT_PARTIAL_LABEL: &str = "partial";
1035const SCAN_CYCLE_RESULT_SUPERSEDED_LABEL: &str = "superseded";
1036const SCAN_CYCLE_RESULT_DEFERRED_LABEL: &str = "deferred";
1037
1038#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
1039pub enum ScanCyclePartialReason {
1040 #[default]
1041 Unknown = 0,
1042 Runtime = 1,
1043 Objects = 2,
1044 Directories = 3,
1045}
1046
1047impl ScanCyclePartialReason {
1048 pub fn as_str(self) -> &'static str {
1049 match self {
1050 Self::Unknown => SCAN_CYCLE_RESULT_UNKNOWN_LABEL,
1051 Self::Runtime => "runtime",
1052 Self::Objects => "objects",
1053 Self::Directories => "directories",
1054 }
1055 }
1056
1057 fn from_code(code: u8) -> Self {
1058 match code {
1059 1 => Self::Runtime,
1060 2 => Self::Objects,
1061 3 => Self::Directories,
1062 _ => Self::Unknown,
1063 }
1064 }
1065}
1066
1067#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
1068pub enum ScannerUsageSaveResult {
1069 #[default]
1070 Unknown = 0,
1071 Success = 1,
1072 Failed = 2,
1073 SkippedStale = 3,
1074 EncodeFailed = 4,
1075}
1076
1077impl ScannerUsageSaveResult {
1078 pub fn as_str(self) -> &'static str {
1079 match self {
1080 Self::Unknown => SCAN_CYCLE_RESULT_UNKNOWN_LABEL,
1081 Self::Success => SCAN_CYCLE_RESULT_SUCCESS_LABEL,
1082 Self::Failed => "failed",
1083 Self::SkippedStale => "skipped_stale",
1084 Self::EncodeFailed => "encode_failed",
1085 }
1086 }
1087
1088 fn from_code(code: u8) -> Self {
1089 match code {
1090 1 => Self::Success,
1091 2 => Self::Failed,
1092 3 => Self::SkippedStale,
1093 4 => Self::EncodeFailed,
1094 _ => Self::Unknown,
1095 }
1096 }
1097}
1098
1099#[derive(Clone, Debug, Default, Serialize, Deserialize)]
1100pub struct CurrentCycle {
1101 pub current: u64,
1102 pub next: u64,
1103 pub cycle_completed: Vec<DateTime<Utc>>,
1104 pub started: DateTime<Utc>,
1105}
1106
1107#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
1108pub struct ScanCycleWorkSnapshot {
1109 objects_scanned: u64,
1110 directories_scanned: u64,
1111 bucket_drive_scans: u64,
1112 bucket_drive_failures: u64,
1113 yield_events: u64,
1114 yield_duration_millis: u64,
1115 throttle_sleep_events: u64,
1116 throttle_sleep_duration_millis: u64,
1117 ilm_actions: u64,
1118 lifecycle_expiry_actions: u64,
1119 lifecycle_transition_actions: u64,
1120 heal_objects: u64,
1121 replication_checks: u64,
1122 usage_saves: u64,
1123}
1124
1125#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
1126pub struct ScannerTimedAction {
1127 pub count: u64,
1128 pub acc_time: u64,
1129 pub bytes: u64,
1130}
1131
1132#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
1133pub struct ScannerCheckpointReport {
1134 pub version: u16,
1135 pub resume_after: String,
1136 pub reason: String,
1137 pub last_event: String,
1138}
1139
1140#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
1141pub struct ScannerSourceWorkSnapshot {
1142 pub source: String,
1143 pub checked: u64,
1144 pub queued: u64,
1145 pub executed: u64,
1146 pub failed: u64,
1147 pub skipped: u64,
1148 #[serde(default)]
1149 pub missed: u64,
1150}
1151
1152#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
1153pub struct ScannerBucketDriveResultSnapshot {
1154 pub bucket: String,
1155 pub drive: String,
1156 pub result: String,
1157 pub count: u64,
1158}
1159
1160#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
1161pub struct ScannerActiveBucketDriveSnapshot {
1162 pub source: String,
1163 pub bucket: String,
1164 pub drive: String,
1165 pub count: u64,
1166 pub age_seconds: u64,
1167}
1168
1169#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
1170pub struct ScannerReplicationRepairSnapshot {
1171 pub source: String,
1172 pub kind: String,
1173 #[serde(default)]
1174 pub scanner_role: String,
1175 #[serde(default)]
1176 pub execution_owner: String,
1177 pub checked: u64,
1178 pub queued: u64,
1179 pub executed: u64,
1180 pub failed: u64,
1181 pub skipped: u64,
1182 #[serde(default)]
1183 pub missed: u64,
1184}
1185
1186#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
1187pub struct ScannerMaintenanceSourceSnapshot {
1188 pub source: String,
1189 pub state: String,
1190 pub reason: String,
1191 pub backlog: u64,
1192 pub current_checked: u64,
1193 pub current_queued: u64,
1194 pub current_missed: u64,
1195 pub lifetime_missed: u64,
1196 pub partial_cycles: u64,
1197}
1198
1199#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
1200pub struct ScannerMaintenanceControlSnapshot {
1201 pub primary_control: String,
1202 pub sources: Vec<ScannerMaintenanceSourceSnapshot>,
1203}
1204
1205impl Default for ScannerMaintenanceControlSnapshot {
1206 fn default() -> Self {
1207 Self {
1208 primary_control: SCANNER_MAINTENANCE_CONTROL_NONE.to_string(),
1209 sources: Vec::new(),
1210 }
1211 }
1212}
1213
1214#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
1215pub struct ScannerSourceCycleSnapshot {
1216 pub source: String,
1217 pub cycles: u64,
1218}
1219
1220#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
1221pub struct ScannerPacingPressureSnapshot {
1222 pub primary_pressure: String,
1223 pub current_queued_scans: u64,
1224 pub current_active_scans: u64,
1225 pub last_cycle_budget_limited: bool,
1226 pub last_cycle_pause_observed: bool,
1227 pub last_cycle_throttle_sleep_ratio: f64,
1228 pub last_cycle_yield_ratio: f64,
1229 pub last_cycle_total_pause_ratio: f64,
1230}
1231
1232impl Default for ScannerPacingPressureSnapshot {
1233 fn default() -> Self {
1234 Self {
1235 primary_pressure: "none".to_string(),
1236 current_queued_scans: 0,
1237 current_active_scans: 0,
1238 last_cycle_budget_limited: false,
1239 last_cycle_pause_observed: false,
1240 last_cycle_throttle_sleep_ratio: 0.0,
1241 last_cycle_yield_ratio: 0.0,
1242 last_cycle_total_pause_ratio: 0.0,
1243 }
1244 }
1245}
1246
1247#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
1248pub struct ScannerLifecycleExpiryStateUpdate {
1249 pub queue_capacity: u64,
1250 pub queued: u64,
1251 pub active: u64,
1252 pub workers: u64,
1253 pub queue_missed: u64,
1254}
1255
1256#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
1257pub struct ScannerLifecycleExpirySnapshot {
1258 pub current_queue_capacity: u64,
1259 pub current_queued: u64,
1260 pub current_active: u64,
1261 pub current_workers: u64,
1262 pub queue_missed: u64,
1263 pub scanner_queued: u64,
1264 pub scanner_missed: u64,
1265 #[serde(default)]
1266 pub scanner_blocked: u64,
1267 #[serde(default)]
1268 pub scanner_not_enqueued: u64,
1269 #[serde(default)]
1270 pub delete_failed: u64,
1271}
1272
1273#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
1274pub struct ScannerLifecycleTransitionStateUpdate {
1275 pub queue_capacity: u64,
1276 pub queued: u64,
1277 pub active: u64,
1278 pub workers: u64,
1279 pub queue_full: u64,
1280 pub queue_send_timeout: u64,
1281 pub compensation_scheduled: u64,
1282 pub compensation_pending: u64,
1283 pub compensation_running: u64,
1284}
1285
1286#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
1287pub struct ScannerLifecycleTransitionSnapshot {
1288 pub current_queue_capacity: u64,
1289 pub current_queued: u64,
1290 pub current_active: u64,
1291 pub current_workers: u64,
1292 pub queue_full: u64,
1293 pub queue_send_timeout: u64,
1294 pub compensation_scheduled: u64,
1295 pub compensation_pending: u64,
1296 pub compensation_running: u64,
1297 pub scanner_queued: u64,
1298 pub scanner_missed: u64,
1299 pub completed: u64,
1300 pub failed: u64,
1301}
1302
1303#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
1304pub struct ScannerUsageFreshnessSnapshot {
1305 pub dirty_pending_buckets: u64,
1306 pub last_dirty_mark_unix_secs: u64,
1307 pub last_dirty_clear_unix_secs: u64,
1308 pub last_cycle_dirty_buckets: u64,
1309 pub last_cycle_cleared_dirty_buckets: u64,
1310 pub last_usage_save_unix_secs: u64,
1311 pub last_usage_save_result: String,
1312 pub last_usage_save_result_code: u64,
1313 #[serde(default)]
1314 pub last_durable_success_unix_secs: u64,
1315 #[serde(default)]
1316 pub last_publication_unix_secs: u64,
1317 #[serde(default)]
1318 pub last_publication_state: String,
1319 #[serde(default)]
1320 pub last_publication_reason: String,
1321 #[serde(default)]
1322 pub deferred_pending: bool,
1323 #[serde(default)]
1324 pub deferred_total: u64,
1325 #[serde(default)]
1326 pub last_deferred_unix_secs: u64,
1327 #[serde(default)]
1328 pub last_deferred_reason: String,
1329}
1330
1331#[derive(Clone, Debug, Default, Serialize, Deserialize)]
1332pub struct ScannerLastMinute {
1333 pub actions: HashMap<String, ScannerTimedAction>,
1334 pub ilm: HashMap<String, ScannerTimedAction>,
1335}
1336
1337#[derive(Clone, Debug, Default, Serialize, Deserialize)]
1338pub struct ScannerMetricsReport {
1339 pub collected_at: Timestamp,
1340 pub current_cycle: u64,
1341 #[serde(default)]
1342 pub current_cycle_active: bool,
1343 pub current_started: Timestamp,
1344 pub cycles_completed_at: Vec<Timestamp>,
1345 pub ongoing_buckets: usize,
1346 #[serde(default)]
1347 pub active_scan_paths: usize,
1348 #[serde(default)]
1349 pub oldest_active_path_age_seconds: u64,
1350 pub life_time_ops: HashMap<String, u64>,
1351 pub life_time_ilm: HashMap<String, u64>,
1352 #[serde(default)]
1354 pub versions_scanned: u64,
1355 pub last_minute: ScannerLastMinute,
1356 pub active_paths: Vec<String>,
1357 pub current_scan_mode: String,
1358 #[serde(default)]
1359 pub leader_lock_state: String,
1360 #[serde(default)]
1361 pub leader_lock_held_by_this_process: bool,
1362 #[serde(default)]
1363 pub leader_lock_last_error: String,
1364 #[serde(default)]
1365 pub leader_lock_last_update_unix_secs: u64,
1366 #[serde(default)]
1367 pub last_cycle_end_unix_secs: u64,
1368 #[serde(default)]
1369 pub current_set_scan_concurrency_limit: u64,
1370 #[serde(default)]
1371 pub current_set_scans_queued: u64,
1372 #[serde(default)]
1373 pub current_set_scans_active: u64,
1374 #[serde(default)]
1375 pub current_disk_scan_concurrency_limit: u64,
1376 #[serde(default)]
1377 pub current_disk_bucket_scans_queued: u64,
1378 #[serde(default)]
1379 pub current_disk_bucket_scans_active: u64,
1380 #[serde(default)]
1381 pub current_cycle_objects_scanned: u64,
1382 #[serde(default)]
1383 pub current_cycle_directories_scanned: u64,
1384 #[serde(default)]
1385 pub current_cycle_bucket_drive_scans: u64,
1386 #[serde(default)]
1387 pub current_cycle_bucket_drive_failures: u64,
1388 #[serde(default)]
1389 pub current_cycle_yield_events: u64,
1390 #[serde(default)]
1391 pub current_cycle_yield_duration_seconds: f64,
1392 #[serde(default)]
1393 pub current_cycle_throttle_sleep_events: u64,
1394 #[serde(default)]
1395 pub current_cycle_throttle_sleep_duration_seconds: f64,
1396 #[serde(default)]
1397 pub current_cycle_ilm_actions: u64,
1398 #[serde(default)]
1399 pub current_cycle_lifecycle_expiry_actions: u64,
1400 #[serde(default)]
1401 pub current_cycle_lifecycle_transition_actions: u64,
1402 #[serde(default)]
1403 pub current_cycle_heal_objects: u64,
1404 #[serde(default)]
1405 pub current_cycle_replication_checks: u64,
1406 #[serde(default)]
1407 pub current_cycle_usage_saves: u64,
1408 pub last_cycle_result: String,
1409 pub last_cycle_result_code: u64,
1410 #[serde(default)]
1411 pub last_cycle_partial_reason: String,
1412 #[serde(default)]
1413 pub last_cycle_partial_reason_code: u64,
1414 #[serde(default)]
1415 pub last_cycle_partial_source: String,
1416 #[serde(default)]
1417 pub last_cycle_partial_source_code: u64,
1418 pub last_cycle_duration_seconds: f64,
1419 #[serde(default)]
1420 pub last_cycle_objects_scanned: u64,
1421 #[serde(default)]
1422 pub last_cycle_directories_scanned: u64,
1423 #[serde(default)]
1424 pub last_cycle_bucket_drive_scans: u64,
1425 #[serde(default)]
1426 pub last_cycle_bucket_drive_failures: u64,
1427 #[serde(default)]
1428 pub last_cycle_yield_events: u64,
1429 #[serde(default)]
1430 pub last_cycle_yield_duration_seconds: f64,
1431 #[serde(default)]
1432 pub last_cycle_throttle_sleep_events: u64,
1433 #[serde(default)]
1434 pub last_cycle_throttle_sleep_duration_seconds: f64,
1435 #[serde(default)]
1436 pub last_cycle_ilm_actions: u64,
1437 #[serde(default)]
1438 pub last_cycle_lifecycle_expiry_actions: u64,
1439 #[serde(default)]
1440 pub last_cycle_lifecycle_transition_actions: u64,
1441 #[serde(default)]
1442 pub last_cycle_heal_objects: u64,
1443 #[serde(default)]
1444 pub last_cycle_replication_checks: u64,
1445 #[serde(default)]
1446 pub last_cycle_usage_saves: u64,
1447 pub failed_cycles: u64,
1448 #[serde(default)]
1449 pub superseded_cycles: u64,
1450 #[serde(default)]
1451 pub partial_cycles_unknown: u64,
1452 #[serde(default)]
1453 pub partial_cycles_runtime: u64,
1454 #[serde(default)]
1455 pub partial_cycles_objects: u64,
1456 #[serde(default)]
1457 pub partial_cycles_directories: u64,
1458 #[serde(default)]
1459 pub partial_cycles_by_source: Vec<ScannerSourceCycleSnapshot>,
1460 #[serde(default)]
1461 pub pacing_pressure: ScannerPacingPressureSnapshot,
1462 #[serde(default)]
1463 pub lifecycle_expiry: ScannerLifecycleExpirySnapshot,
1464 #[serde(default)]
1465 pub lifecycle_transition: ScannerLifecycleTransitionSnapshot,
1466 #[serde(default)]
1467 pub usage_freshness: ScannerUsageFreshnessSnapshot,
1468 #[serde(default)]
1469 pub maintenance_control: ScannerMaintenanceControlSnapshot,
1470 #[serde(default)]
1471 pub throttle_idle_mode_enabled: bool,
1472 #[serde(default)]
1473 pub throttle_sleep_factor: f64,
1474 #[serde(default)]
1475 pub throttle_max_sleep_seconds: f64,
1476 #[serde(default)]
1477 pub yield_every_n_objects: u64,
1478 #[serde(default)]
1479 pub cycle_interval_seconds: f64,
1480 #[serde(default)]
1481 pub cycle_max_duration_seconds: f64,
1482 #[serde(default)]
1483 pub cycle_max_objects: u64,
1484 #[serde(default)]
1485 pub cycle_max_directories: u64,
1486 #[serde(default)]
1487 pub cycle_timeout_total: u64,
1488 #[serde(default)]
1489 pub cycle_recovery_required_total: u64,
1490 #[serde(default)]
1491 pub cycle_last_progress_age: u64,
1492 #[serde(default)]
1493 pub leader_lease_without_progress: bool,
1494 #[serde(default)]
1495 pub bitrot_cycle_enabled: bool,
1496 #[serde(default)]
1497 pub bitrot_cycle_seconds: f64,
1498 #[serde(default)]
1499 #[serde(skip_serializing_if = "Option::is_none")]
1500 pub scan_checkpoint: Option<ScannerCheckpointReport>,
1501 #[serde(default)]
1502 pub scan_checkpoint_used: u64,
1503 #[serde(default)]
1504 pub scan_checkpoint_cleared: u64,
1505 #[serde(default)]
1506 pub scan_checkpoint_ignored: u64,
1507 #[serde(default)]
1508 pub scan_checkpoint_stale: u64,
1509 #[serde(default)]
1510 pub source_work: Vec<ScannerSourceWorkSnapshot>,
1511 #[serde(default)]
1512 pub current_cycle_source_work: Vec<ScannerSourceWorkSnapshot>,
1513 #[serde(default)]
1514 pub last_cycle_source_work: Vec<ScannerSourceWorkSnapshot>,
1515 #[serde(default)]
1516 pub replication_repair: Vec<ScannerReplicationRepairSnapshot>,
1517 #[serde(default)]
1518 pub current_cycle_replication_repair: Vec<ScannerReplicationRepairSnapshot>,
1519 #[serde(default)]
1520 pub last_cycle_replication_repair: Vec<ScannerReplicationRepairSnapshot>,
1521 #[serde(default)]
1522 pub partial_cycles: u64,
1523}
1524
1525#[derive(Clone, Debug, Default, Serialize, Deserialize)]
1526pub struct ScannerRuntimeDetailsReport {
1527 #[serde(default)]
1528 pub disk_bucket_scan_states: Vec<ScannerDiskBucketScanSnapshot>,
1529 #[serde(default)]
1530 pub bucket_drive_results: Vec<ScannerBucketDriveResultSnapshot>,
1531 #[serde(default)]
1532 pub current_cycle_bucket_drive_results: Vec<ScannerBucketDriveResultSnapshot>,
1533 #[serde(default)]
1534 pub last_cycle_bucket_drive_results: Vec<ScannerBucketDriveResultSnapshot>,
1535 #[serde(default)]
1536 pub active_bucket_drive_scans: Vec<ScannerActiveBucketDriveSnapshot>,
1537}
1538
1539impl CurrentCycle {
1540 pub fn unmarshal(&mut self, buf: &[u8]) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
1541 *self = rmp_serde::from_slice(buf)?;
1542 Ok(())
1543 }
1544
1545 pub fn marshal(&self) -> Result<Vec<u8>, Box<dyn std::error::Error + Send + Sync>> {
1546 Ok(rmp_serde::to_vec(self)?)
1547 }
1548}
1549
1550const OTEL_SCANNER_BUCKETS_SCANNED: &str = "rustfs_scanner_buckets_scanned_total";
1552const OTEL_SCANNER_CYCLES: &str = "rustfs_scanner_cycles_total";
1553const OTEL_SCANNER_CYCLE_DURATION_SECONDS: &str = "rustfs_scanner_cycle_duration_seconds";
1554const OTEL_SCANNER_BUCKET_DRIVE_DURATION_SECONDS: &str = "rustfs_scanner_bucket_drive_duration_seconds";
1555const OTEL_SCANNER_CYCLE_TIMEOUT_TOTAL: &str = "rustfs_scanner_cycle_timeout_total";
1556const OTEL_SCANNER_CYCLE_LAST_PROGRESS_AGE: &str = "rustfs_scanner_cycle_last_progress_age";
1557const OTEL_SCANNER_LEADER_LEASE_WITHOUT_PROGRESS: &str = "rustfs_scanner_leader_lease_without_progress";
1558
1559fn scan_cycle_result_label(result: u8) -> &'static str {
1560 match result {
1561 SCAN_CYCLE_RESULT_SUCCESS => SCAN_CYCLE_RESULT_SUCCESS_LABEL,
1562 SCAN_CYCLE_RESULT_ERROR => SCAN_CYCLE_RESULT_ERROR_LABEL,
1563 SCAN_CYCLE_RESULT_PARTIAL => SCAN_CYCLE_RESULT_PARTIAL_LABEL,
1564 SCAN_CYCLE_RESULT_SUPERSEDED => SCAN_CYCLE_RESULT_SUPERSEDED_LABEL,
1565 SCAN_CYCLE_RESULT_DEFERRED => SCAN_CYCLE_RESULT_DEFERRED_LABEL,
1566 _ => SCAN_CYCLE_RESULT_UNKNOWN_LABEL,
1567 }
1568}
1569
1570fn scanner_ratio(numerator: f64, denominator: f64) -> f64 {
1571 if denominator <= 0.0 {
1572 0.0
1573 } else {
1574 (numerator / denominator).clamp(0.0, 1.0)
1575 }
1576}
1577
1578fn unix_now_secs() -> u64 {
1579 SystemTime::now()
1580 .duration_since(SystemTime::UNIX_EPOCH)
1581 .unwrap_or_default()
1582 .as_secs()
1583}
1584
1585fn scanner_last_cycle_budget_limited(result_code: u64, partial_reason: &str) -> bool {
1586 result_code == u64::from(SCAN_CYCLE_RESULT_PARTIAL) && matches!(partial_reason, "runtime" | "objects" | "directories")
1587}
1588
1589fn scanner_primary_pressure(
1590 current_queued_scans: u64,
1591 current_active_scans: u64,
1592 last_cycle_budget_limited: bool,
1593 last_cycle_throttle_sleep_events: u64,
1594 last_cycle_yield_events: u64,
1595) -> &'static str {
1596 if current_queued_scans > 0 {
1597 "queued_scans"
1598 } else if last_cycle_budget_limited {
1599 "cycle_budget"
1600 } else if last_cycle_throttle_sleep_events > 0 || last_cycle_yield_events > 0 {
1601 "throttle_pause"
1602 } else if current_active_scans > 0 {
1603 "active_scans"
1604 } else {
1605 "none"
1606 }
1607}
1608
1609fn scanner_pacing_pressure(metrics: &ScannerMetricsReport) -> ScannerPacingPressureSnapshot {
1610 let current_queued_scans = metrics
1611 .current_set_scans_queued
1612 .saturating_add(metrics.current_disk_bucket_scans_queued);
1613 let current_active_scans = metrics
1614 .current_set_scans_active
1615 .saturating_add(metrics.current_disk_bucket_scans_active)
1616 .max(usize_to_u64_saturated(metrics.active_scan_paths));
1617 let last_cycle_budget_limited =
1618 scanner_last_cycle_budget_limited(metrics.last_cycle_result_code, metrics.last_cycle_partial_reason.as_str());
1619 let last_cycle_pause_observed = metrics.last_cycle_throttle_sleep_events > 0 || metrics.last_cycle_yield_events > 0;
1620 let last_cycle_throttle_sleep_ratio =
1621 scanner_ratio(metrics.last_cycle_throttle_sleep_duration_seconds, metrics.last_cycle_duration_seconds);
1622 let last_cycle_yield_ratio = scanner_ratio(metrics.last_cycle_yield_duration_seconds, metrics.last_cycle_duration_seconds);
1623 let last_cycle_total_pause_ratio = scanner_ratio(
1624 metrics.last_cycle_throttle_sleep_duration_seconds.max(0.0) + metrics.last_cycle_yield_duration_seconds.max(0.0),
1625 metrics.last_cycle_duration_seconds,
1626 );
1627
1628 ScannerPacingPressureSnapshot {
1629 primary_pressure: scanner_primary_pressure(
1630 current_queued_scans,
1631 current_active_scans,
1632 last_cycle_budget_limited,
1633 metrics.last_cycle_throttle_sleep_events,
1634 metrics.last_cycle_yield_events,
1635 )
1636 .to_string(),
1637 current_queued_scans,
1638 current_active_scans,
1639 last_cycle_budget_limited,
1640 last_cycle_pause_observed,
1641 last_cycle_throttle_sleep_ratio,
1642 last_cycle_yield_ratio,
1643 last_cycle_total_pause_ratio,
1644 }
1645}
1646
1647const SCANNER_MAINTENANCE_CONTROL_NONE: &str = "none";
1648const SCANNER_MAINTENANCE_CONTROL_BLOCKED_SOURCE: &str = "blocked_source";
1649const SCANNER_MAINTENANCE_CONTROL_DEFERRED_SOURCE: &str = "deferred_source";
1650const SCANNER_MAINTENANCE_CONTROL_ACTIVE_SOURCE: &str = "active_source";
1651const SCANNER_MAINTENANCE_CONTROL_PACING_PRESSURE: &str = "pacing_pressure";
1652
1653const SCANNER_MAINTENANCE_STATE_IDLE: &str = "idle";
1654const SCANNER_MAINTENANCE_STATE_ACTIVE: &str = "active";
1655const SCANNER_MAINTENANCE_STATE_DEFERRED: &str = "deferred";
1656const SCANNER_MAINTENANCE_STATE_BLOCKED: &str = "blocked";
1657
1658const SCANNER_MAINTENANCE_REASON_IDLE: &str = "idle";
1659const SCANNER_MAINTENANCE_REASON_ACTIVE_WORK: &str = "active_work";
1660const SCANNER_MAINTENANCE_REASON_QUEUED_WORK: &str = "queued_work";
1661const SCANNER_MAINTENANCE_REASON_PARTIAL_CYCLE: &str = "partial_cycle";
1662const SCANNER_MAINTENANCE_REASON_MISSED_WORK: &str = "missed_work";
1663const SCANNER_MAINTENANCE_REASON_EXPIRY_QUEUE_BACKLOG: &str = "expiry_queue_backlog";
1664const SCANNER_MAINTENANCE_REASON_TRANSITION_FAILED: &str = "transition_failed";
1665const SCANNER_MAINTENANCE_REASON_TRANSITION_COMPENSATION_BACKLOG: &str = "transition_compensation_backlog";
1666const SCANNER_MAINTENANCE_REASON_TRANSITION_QUEUE_BACKLOG: &str = "transition_queue_backlog";
1667const SCANNER_MAINTENANCE_REASON_TRANSITION_QUEUE_FULL: &str = "transition_queue_full";
1668
1669fn scanner_source_work_snapshot(work: &[ScannerSourceWorkSnapshot], source: ScannerWorkSource) -> ScannerSourceWorkSnapshot {
1670 work.iter()
1671 .find(|snapshot| snapshot.source == source.as_str())
1672 .cloned()
1673 .unwrap_or_else(|| ScannerSourceWorkSnapshot {
1674 source: source.as_str().to_string(),
1675 ..Default::default()
1676 })
1677}
1678
1679fn scanner_source_partial_cycles(metrics: &ScannerMetricsReport, source: ScannerWorkSource) -> u64 {
1680 metrics
1681 .partial_cycles_by_source
1682 .iter()
1683 .find(|snapshot| snapshot.source == source.as_str())
1684 .map(|snapshot| snapshot.cycles)
1685 .unwrap_or_default()
1686}
1687
1688fn scanner_source_has_current_work(work: &ScannerSourceWorkSnapshot) -> bool {
1689 work.checked > 0 || work.queued > 0 || work.executed > 0 || work.failed > 0 || work.skipped > 0 || work.missed > 0
1690}
1691
1692fn scanner_lifecycle_transition_backlog(metrics: &ScannerMetricsReport) -> u64 {
1693 metrics
1694 .lifecycle_transition
1695 .current_queued
1696 .saturating_add(metrics.lifecycle_transition.current_active)
1697 .saturating_add(metrics.lifecycle_transition.compensation_pending)
1698 .saturating_add(metrics.lifecycle_transition.compensation_running)
1699}
1700
1701fn scanner_lifecycle_expiry_backlog(metrics: &ScannerMetricsReport) -> u64 {
1702 metrics
1703 .lifecycle_expiry
1704 .current_queued
1705 .saturating_add(metrics.lifecycle_expiry.current_active)
1706}
1707
1708fn scanner_source_maintenance_state(
1709 source: ScannerWorkSource,
1710 current: &ScannerSourceWorkSnapshot,
1711 metrics: &ScannerMetricsReport,
1712) -> (&'static str, &'static str, u64) {
1713 if current.missed > 0 {
1714 return (SCANNER_MAINTENANCE_STATE_BLOCKED, SCANNER_MAINTENANCE_REASON_MISSED_WORK, current.missed);
1715 }
1716
1717 if source == ScannerWorkSource::Lifecycle && current.failed > 0 {
1718 return (
1719 SCANNER_MAINTENANCE_STATE_BLOCKED,
1720 SCANNER_MAINTENANCE_REASON_TRANSITION_FAILED,
1721 current.failed,
1722 );
1723 }
1724
1725 if source == ScannerWorkSource::Lifecycle {
1726 let transition_backlog = scanner_lifecycle_transition_backlog(metrics);
1727 if metrics.lifecycle_transition.current_queue_capacity > 0
1728 && metrics.lifecycle_transition.current_queued >= metrics.lifecycle_transition.current_queue_capacity
1729 {
1730 return (
1731 SCANNER_MAINTENANCE_STATE_BLOCKED,
1732 SCANNER_MAINTENANCE_REASON_TRANSITION_QUEUE_FULL,
1733 transition_backlog,
1734 );
1735 }
1736 }
1737
1738 if metrics.last_cycle_partial_source == source.as_str() && metrics.pacing_pressure.last_cycle_budget_limited {
1739 return (SCANNER_MAINTENANCE_STATE_DEFERRED, SCANNER_MAINTENANCE_REASON_PARTIAL_CYCLE, 0);
1740 }
1741
1742 if current.queued > 0 {
1743 return (SCANNER_MAINTENANCE_STATE_ACTIVE, SCANNER_MAINTENANCE_REASON_QUEUED_WORK, current.queued);
1744 }
1745
1746 if source == ScannerWorkSource::Lifecycle {
1747 let expiry_backlog = scanner_lifecycle_expiry_backlog(metrics);
1748 if expiry_backlog > 0 {
1749 return (
1750 SCANNER_MAINTENANCE_STATE_ACTIVE,
1751 SCANNER_MAINTENANCE_REASON_EXPIRY_QUEUE_BACKLOG,
1752 expiry_backlog,
1753 );
1754 }
1755
1756 let compensation_backlog = metrics
1757 .lifecycle_transition
1758 .compensation_pending
1759 .saturating_add(metrics.lifecycle_transition.compensation_running);
1760 if compensation_backlog > 0 {
1761 return (
1762 SCANNER_MAINTENANCE_STATE_ACTIVE,
1763 SCANNER_MAINTENANCE_REASON_TRANSITION_COMPENSATION_BACKLOG,
1764 compensation_backlog,
1765 );
1766 }
1767
1768 let transition_backlog = scanner_lifecycle_transition_backlog(metrics);
1769 if transition_backlog > 0 {
1770 return (
1771 SCANNER_MAINTENANCE_STATE_ACTIVE,
1772 SCANNER_MAINTENANCE_REASON_TRANSITION_QUEUE_BACKLOG,
1773 transition_backlog,
1774 );
1775 }
1776 }
1777
1778 if scanner_source_has_current_work(current) {
1779 return (SCANNER_MAINTENANCE_STATE_ACTIVE, SCANNER_MAINTENANCE_REASON_ACTIVE_WORK, 0);
1780 }
1781
1782 (SCANNER_MAINTENANCE_STATE_IDLE, SCANNER_MAINTENANCE_REASON_IDLE, 0)
1783}
1784
1785fn scanner_maintenance_source_work(metrics: &ScannerMetricsReport) -> &[ScannerSourceWorkSnapshot] {
1786 if metrics.current_cycle_source_work.is_empty() {
1787 &metrics.last_cycle_source_work
1788 } else {
1789 &metrics.current_cycle_source_work
1790 }
1791}
1792
1793fn scanner_maintenance_control(metrics: &ScannerMetricsReport) -> ScannerMaintenanceControlSnapshot {
1794 let mut has_blocked = false;
1795 let mut has_deferred = false;
1796 let mut has_active = false;
1797 let current_source_work = scanner_maintenance_source_work(metrics);
1798
1799 let sources = ScannerWorkSource::all()
1800 .iter()
1801 .map(|source| {
1802 let current = scanner_source_work_snapshot(current_source_work, *source);
1803 let lifetime = scanner_source_work_snapshot(&metrics.source_work, *source);
1804 let partial_cycles = scanner_source_partial_cycles(metrics, *source);
1805 let (state, reason, backlog) = scanner_source_maintenance_state(*source, ¤t, metrics);
1806
1807 match state {
1808 SCANNER_MAINTENANCE_STATE_BLOCKED => has_blocked = true,
1809 SCANNER_MAINTENANCE_STATE_DEFERRED => has_deferred = true,
1810 SCANNER_MAINTENANCE_STATE_ACTIVE => has_active = true,
1811 _ => {}
1812 }
1813
1814 ScannerMaintenanceSourceSnapshot {
1815 source: source.as_str().to_string(),
1816 state: state.to_string(),
1817 reason: reason.to_string(),
1818 backlog,
1819 current_checked: current.checked,
1820 current_queued: current.queued,
1821 current_missed: current.missed,
1822 lifetime_missed: lifetime.missed,
1823 partial_cycles,
1824 }
1825 })
1826 .collect();
1827
1828 let primary_control = if has_blocked {
1829 SCANNER_MAINTENANCE_CONTROL_BLOCKED_SOURCE
1830 } else if has_deferred {
1831 SCANNER_MAINTENANCE_CONTROL_DEFERRED_SOURCE
1832 } else if has_active {
1833 SCANNER_MAINTENANCE_CONTROL_ACTIVE_SOURCE
1834 } else if metrics.pacing_pressure.primary_pressure != SCANNER_MAINTENANCE_CONTROL_NONE {
1835 SCANNER_MAINTENANCE_CONTROL_PACING_PRESSURE
1836 } else {
1837 SCANNER_MAINTENANCE_CONTROL_NONE
1838 };
1839
1840 ScannerMaintenanceControlSnapshot {
1841 primary_control: primary_control.to_string(),
1842 sources,
1843 }
1844}
1845
1846fn duration_millis_saturated(duration: Duration) -> u64 {
1847 duration.as_millis().min(u64::MAX as u128) as u64
1848}
1849
1850fn usize_to_u64_saturated(value: usize) -> u64 {
1851 u64::try_from(value).unwrap_or(u64::MAX)
1852}
1853
1854fn scaled_f64_to_u64_saturated(value: f64, scale: f64) -> u64 {
1855 if !value.is_finite() || value <= 0.0 {
1856 return 0;
1857 }
1858
1859 (value * scale).round().clamp(0.0, u64::MAX as f64) as u64
1860}
1861
1862pub fn emit_scan_cycle_complete(success: bool, duration: Duration) {
1863 let result = if success {
1864 SCAN_CYCLE_RESULT_SUCCESS_LABEL
1865 } else {
1866 SCAN_CYCLE_RESULT_ERROR_LABEL
1867 };
1868 global_metrics().record_scan_cycle_complete(success, duration);
1869 metrics::counter!(OTEL_SCANNER_CYCLES, "result" => result).increment(1);
1870 if success {
1871 metrics::gauge!(OTEL_SCANNER_CYCLE_DURATION_SECONDS).set(duration.as_secs_f64());
1872 }
1873}
1874
1875pub fn emit_scan_cycle_partial(duration: Duration, reason: ScanCyclePartialReason) {
1876 global_metrics().record_scan_cycle_partial(duration, reason);
1877 metrics::counter!(OTEL_SCANNER_CYCLES, "result" => SCAN_CYCLE_RESULT_PARTIAL_LABEL).increment(1);
1878}
1879
1880pub fn emit_scan_cycle_partial_with_source(
1881 duration: Duration,
1882 reason: ScanCyclePartialReason,
1883 source: Option<ScannerWorkSource>,
1884) {
1885 global_metrics().record_scan_cycle_partial_with_source(duration, reason, source);
1886 metrics::counter!(OTEL_SCANNER_CYCLES, "result" => SCAN_CYCLE_RESULT_PARTIAL_LABEL).increment(1);
1887}
1888
1889pub fn emit_scan_cycle_superseded(duration: Duration) {
1890 global_metrics().record_scan_cycle_superseded(duration);
1891 metrics::counter!(OTEL_SCANNER_CYCLES, "result" => SCAN_CYCLE_RESULT_SUPERSEDED_LABEL).increment(1);
1892}
1893
1894pub fn emit_scan_cycle_deferred(duration: Duration) {
1895 global_metrics().record_scan_cycle_deferred(duration);
1896 metrics::counter!(OTEL_SCANNER_CYCLES, "result" => SCAN_CYCLE_RESULT_DEFERRED_LABEL).increment(1);
1897}
1898
1899pub fn emit_scan_bucket_drive_complete(_source: ScannerWorkSource, success: bool, bucket: &str, disk: &str, duration: Duration) {
1900 let result = if success { "success" } else { "error" };
1901 global_metrics().record_scanner_bucket_drive_result(bucket, disk, result);
1902 metrics::counter!(
1903 OTEL_SCANNER_BUCKETS_SCANNED,
1904 "result" => result,
1905 "bucket" => bucket.to_owned(),
1906 "disk" => disk.to_owned()
1907 )
1908 .increment(1);
1909 metrics::histogram!(
1910 OTEL_SCANNER_BUCKET_DRIVE_DURATION_SECONDS,
1911 "bucket" => bucket.to_owned(),
1912 "disk" => disk.to_owned()
1913 )
1914 .record(duration.as_secs_f64());
1915}
1916
1917pub fn emit_scan_bucket_drive_partial(_source: ScannerWorkSource, bucket: &str, disk: &str, duration: Duration) {
1918 global_metrics().record_scanner_bucket_drive_result(bucket, disk, SCAN_CYCLE_RESULT_PARTIAL_LABEL);
1919 metrics::counter!(
1920 OTEL_SCANNER_BUCKETS_SCANNED,
1921 "result" => SCAN_CYCLE_RESULT_PARTIAL_LABEL,
1922 "bucket" => bucket.to_owned(),
1923 "disk" => disk.to_owned()
1924 )
1925 .increment(1);
1926 metrics::histogram!(
1927 OTEL_SCANNER_BUCKET_DRIVE_DURATION_SECONDS,
1928 "bucket" => bucket.to_owned(),
1929 "disk" => disk.to_owned()
1930 )
1931 .record(duration.as_secs_f64());
1932}
1933
1934impl Metrics {
1935 pub fn new() -> Self {
1936 Self {
1937 operations: (0..Metric::Last as usize).map(|_| AtomicU64::new(0)).collect(),
1938 latency: (0..Metric::LastRealtime as usize)
1941 .map(|_| LockedLastMinuteLatency::new())
1942 .collect(),
1943 actions: (0..IlmAction::ActionCount as usize).map(|_| AtomicU64::new(0)).collect(),
1944 actions_latency: (0..IlmAction::ActionCount as usize)
1945 .map(|_| LockedLastMinuteLatency::new())
1946 .collect(),
1947 current_paths: Arc::new(RwLock::new(HashMap::new())),
1948 cycle_info: Arc::new(RwLock::new(None)),
1949 current_scan_mode: AtomicU8::new(HealScanMode::Unknown as u8),
1950 current_scan_cycle_work_active: AtomicBool::new(false),
1951 current_scan_cycle_objects_start: AtomicU64::new(0),
1952 current_scan_cycle_directories_start: AtomicU64::new(0),
1953 current_scan_cycle_bucket_drive_scans_start: AtomicU64::new(0),
1954 current_scan_cycle_bucket_drive_failures_start: AtomicU64::new(0),
1955 current_scan_cycle_yield_events_start: AtomicU64::new(0),
1956 current_scan_cycle_yield_duration_millis_start: AtomicU64::new(0),
1957 current_scan_cycle_throttle_sleep_events_start: AtomicU64::new(0),
1958 current_scan_cycle_throttle_sleep_duration_millis_start: AtomicU64::new(0),
1959 current_scan_cycle_ilm_actions_start: AtomicU64::new(0),
1960 current_scan_cycle_lifecycle_expiry_actions_start: AtomicU64::new(0),
1961 current_scan_cycle_lifecycle_transition_actions_start: AtomicU64::new(0),
1962 current_scan_cycle_heal_objects_start: AtomicU64::new(0),
1963 current_scan_cycle_replication_checks_start: AtomicU64::new(0),
1964 current_scan_cycle_usage_saves_start: AtomicU64::new(0),
1965 scanner_set_scan_concurrency_limit: AtomicU64::new(0),
1966 scanner_set_scans_queued: AtomicU64::new(0),
1967 scanner_set_scans_active: AtomicU64::new(0),
1968 scanner_disk_bucket_scan_states: Mutex::new(HashMap::new()),
1969 scanner_bucket_drive_results: Mutex::new(ScannerBucketDriveResults::default()),
1970 scanner_active_bucket_drive_scans: Mutex::new(HashMap::new()),
1971 scanner_bucket_drive_result_clock: AtomicU64::new(0),
1972 current_scan_cycle_bucket_drive_results_start: Mutex::new(HashMap::new()),
1973 last_scan_cycle_bucket_drive_results: Mutex::new(Vec::new()),
1974 scanner_leader_lock_state: RwLock::new("unknown".to_string()),
1975 scanner_leader_lock_held: AtomicBool::new(false),
1976 scanner_leader_lock_last_error: RwLock::new(String::new()),
1977 scanner_leader_lock_last_update_unix_secs: AtomicU64::new(0),
1978 last_scan_cycle_result: AtomicU8::new(SCAN_CYCLE_RESULT_UNKNOWN),
1979 last_scan_cycle_partial_reason: AtomicU8::new(ScanCyclePartialReason::Unknown as u8),
1980 last_scan_cycle_partial_source: AtomicU8::new(0),
1981 last_scan_cycle_end_unix_secs: AtomicU64::new(0),
1982 last_scan_cycle_duration_millis: AtomicU64::new(0),
1983 last_scan_cycle_objects_scanned: AtomicU64::new(0),
1984 last_scan_cycle_directories_scanned: AtomicU64::new(0),
1985 lifetime_versions_scanned: AtomicU64::new(0),
1986 last_scan_cycle_bucket_drive_scans: AtomicU64::new(0),
1987 last_scan_cycle_bucket_drive_failures: AtomicU64::new(0),
1988 last_scan_cycle_yield_events: AtomicU64::new(0),
1989 last_scan_cycle_yield_duration_millis: AtomicU64::new(0),
1990 last_scan_cycle_throttle_sleep_events: AtomicU64::new(0),
1991 last_scan_cycle_throttle_sleep_duration_millis: AtomicU64::new(0),
1992 last_scan_cycle_ilm_actions: AtomicU64::new(0),
1993 last_scan_cycle_lifecycle_expiry_actions: AtomicU64::new(0),
1994 last_scan_cycle_lifecycle_transition_actions: AtomicU64::new(0),
1995 last_scan_cycle_heal_objects: AtomicU64::new(0),
1996 last_scan_cycle_replication_checks: AtomicU64::new(0),
1997 last_scan_cycle_usage_saves: AtomicU64::new(0),
1998 failed_scan_cycles: AtomicU64::new(0),
1999 superseded_scan_cycles: AtomicU64::new(0),
2000 partial_scan_cycles_unknown: AtomicU64::new(0),
2001 partial_scan_cycles_runtime: AtomicU64::new(0),
2002 partial_scan_cycles_objects: AtomicU64::new(0),
2003 partial_scan_cycles_directories: AtomicU64::new(0),
2004 partial_scan_cycles_by_source: ScannerWorkSource::all().iter().map(|_| AtomicU64::new(0)).collect(),
2005 scanner_yield_duration_millis: AtomicU64::new(0),
2006 scanner_throttle_sleep_duration_millis: AtomicU64::new(0),
2007 scanner_ilm_actions: AtomicU64::new(0),
2008 scanner_lifecycle_expiry_actions: AtomicU64::new(0),
2009 scanner_lifecycle_transition_actions: AtomicU64::new(0),
2010 scanner_expiry_queue_capacity: AtomicU64::new(0),
2011 scanner_expiry_queued: AtomicU64::new(0),
2012 scanner_expiry_active: AtomicU64::new(0),
2013 scanner_expiry_workers: AtomicU64::new(0),
2014 scanner_expiry_queue_missed: AtomicU64::new(0),
2015 scanner_expiry_queued_total: AtomicU64::new(0),
2016 scanner_expiry_missed_total: AtomicU64::new(0),
2017 scanner_expiry_blocked_total: AtomicU64::new(0),
2018 scanner_expiry_not_enqueued_total: AtomicU64::new(0),
2019 scanner_expiry_delete_failed_total: AtomicU64::new(0),
2020 scanner_transition_queue_capacity: AtomicU64::new(0),
2021 scanner_transition_queued: AtomicU64::new(0),
2022 scanner_transition_active: AtomicU64::new(0),
2023 scanner_transition_workers: AtomicU64::new(0),
2024 scanner_transition_queue_full: AtomicU64::new(0),
2025 scanner_transition_queue_send_timeout: AtomicU64::new(0),
2026 scanner_transition_compensation_scheduled: AtomicU64::new(0),
2027 scanner_transition_compensation_pending: AtomicU64::new(0),
2028 scanner_transition_compensation_running: AtomicU64::new(0),
2029 scanner_transition_queued_total: AtomicU64::new(0),
2030 scanner_transition_missed_total: AtomicU64::new(0),
2031 scanner_transition_completed: AtomicU64::new(0),
2032 tier_requests: std::array::from_fn(|_| AtomicU64::new(0)),
2033 scanner_transition_failed: AtomicU64::new(0),
2034 scanner_throttle_idle_mode_enabled: AtomicBool::new(false),
2035 scanner_throttle_sleep_factor_micros: AtomicU64::new(0),
2036 scanner_throttle_max_sleep_millis: AtomicU64::new(0),
2037 scanner_yield_every_n_objects: AtomicU64::new(0),
2038 scanner_cycle_interval_millis: AtomicU64::new(0),
2039 scanner_cycle_max_duration_millis: AtomicU64::new(0),
2040 scanner_cycle_max_objects: AtomicU64::new(0),
2041 scanner_cycle_max_directories: AtomicU64::new(0),
2042 scanner_cycle_timeout_total: AtomicU64::new(0),
2043 scanner_cycle_recovery_required_total: AtomicU64::new(0),
2044 scanner_cycle_last_progress_age_seconds: AtomicU64::new(0),
2045 scanner_leader_lease_without_progress: AtomicBool::new(false),
2046 scanner_bitrot_cycle_enabled: AtomicBool::new(false),
2047 scanner_bitrot_cycle_millis: AtomicU64::new(0),
2048 scanner_checkpoint: Mutex::new(None),
2049 scanner_checkpoint_used: AtomicU64::new(0),
2050 scanner_checkpoint_cleared: AtomicU64::new(0),
2051 scanner_checkpoint_ignored: AtomicU64::new(0),
2052 scanner_checkpoint_stale: AtomicU64::new(0),
2053 scanner_dirty_usage_pending_buckets: AtomicU64::new(0),
2054 scanner_dirty_usage_last_mark_unix_secs: AtomicU64::new(0),
2055 scanner_dirty_usage_last_clear_unix_secs: AtomicU64::new(0),
2056 scanner_dirty_usage_last_cycle_dirty_buckets: AtomicU64::new(0),
2057 scanner_dirty_usage_last_cycle_cleared_buckets: AtomicU64::new(0),
2058 scanner_usage_last_save_unix_secs: AtomicU64::new(0),
2059 scanner_usage_last_durable_success_unix_secs: AtomicU64::new(0),
2060 scanner_usage_last_publication_unix_secs: AtomicU64::new(0),
2061 scanner_usage_last_publication_state: Mutex::new(String::new()),
2062 scanner_usage_last_publication_reason: Mutex::new(String::new()),
2063 scanner_usage_last_save_result: AtomicU8::new(ScannerUsageSaveResult::Unknown as u8),
2064 scanner_usage_deferred_pending: AtomicBool::new(false),
2065 scanner_usage_deferred_total: AtomicU64::new(0),
2066 scanner_usage_last_deferred_unix_secs: AtomicU64::new(0),
2067 scanner_usage_last_deferred_reason: Mutex::new(String::new()),
2068 scanner_source_work: ScannerWorkSource::all()
2069 .iter()
2070 .map(|_| ScannerSourceWorkCounters::default())
2071 .collect(),
2072 current_scan_cycle_source_work_start: ScannerWorkSource::all()
2073 .iter()
2074 .map(|_| ScannerSourceWorkCounters::default())
2075 .collect(),
2076 last_scan_cycle_source_work: ScannerWorkSource::all()
2077 .iter()
2078 .map(|_| ScannerSourceWorkCounters::default())
2079 .collect(),
2080 scanner_replication_repair_work: ScannerReplicationRepairKind::all()
2081 .iter()
2082 .map(|_| ScannerSourceWorkCounters::default())
2083 .collect(),
2084 current_scan_cycle_replication_repair_work_start: ScannerReplicationRepairKind::all()
2085 .iter()
2086 .map(|_| ScannerSourceWorkCounters::default())
2087 .collect(),
2088 last_scan_cycle_replication_repair_work: ScannerReplicationRepairKind::all()
2089 .iter()
2090 .map(|_| ScannerSourceWorkCounters::default())
2091 .collect(),
2092 partial_scan_cycles: AtomicU64::new(0),
2093 }
2094 }
2095
2096 pub fn log(metric: Metric) -> impl Fn(&HashMap<String, String>) {
2107 let metric_idx = metric as usize;
2108 let start = SystemTime::now();
2109 move |_custom: &HashMap<String, String>| {
2110 let duration = SystemTime::now().duration_since(start).unwrap_or_default();
2111 global_metrics().operations[metric_idx].fetch_add(1, Ordering::Relaxed);
2112 global_metrics().record_source_work_for_metric(metric, 1);
2113 if metric_idx < Metric::LastRealtime as usize {
2114 global_metrics().latency[metric_idx].add(duration);
2115 }
2116 }
2117 }
2118
2119 pub fn time_size(metric: Metric) -> impl Fn(u64) {
2122 let metric_idx = metric as usize;
2123 let start = SystemTime::now();
2124 move |size: u64| {
2125 let duration = SystemTime::now().duration_since(start).unwrap_or_default();
2126 global_metrics().operations[metric_idx].fetch_add(1, Ordering::Relaxed);
2127 global_metrics().record_source_work_for_metric(metric, 1);
2128 if metric_idx < Metric::LastRealtime as usize {
2129 global_metrics().latency[metric_idx].add_size(duration, size);
2130 }
2131 }
2132 }
2133
2134 pub fn time(metric: Metric) -> impl Fn() {
2137 let metric_idx = metric as usize;
2138 let start = SystemTime::now();
2139 move || {
2140 let duration = SystemTime::now().duration_since(start).unwrap_or_default();
2141 global_metrics().operations[metric_idx].fetch_add(1, Ordering::Relaxed);
2142 global_metrics().record_source_work_for_metric(metric, 1);
2143 if metric_idx < Metric::LastRealtime as usize {
2144 global_metrics().latency[metric_idx].add(duration);
2145 }
2146 }
2147 }
2148
2149 pub fn time_n(metric: Metric) -> Box<dyn Fn(usize) -> Box<dyn Fn() + Send + Sync> + Send + Sync> {
2152 let metric_idx = metric as usize;
2153 let start = SystemTime::now();
2154 Box::new(move |count: usize| {
2155 Box::new(move || {
2156 let duration = SystemTime::now().duration_since(start).unwrap_or_default();
2157 let count = usize_to_u64_saturated(count);
2158 global_metrics().operations[metric_idx].fetch_add(count, Ordering::Relaxed);
2159 global_metrics().record_source_work_for_metric(metric, count);
2160 if metric_idx < Metric::LastRealtime as usize {
2161 global_metrics().latency[metric_idx].add(duration);
2162 }
2163 })
2164 })
2165 }
2166
2167 pub fn time_ilm(a: IlmAction) -> Box<dyn Fn(u64) -> Box<dyn Fn() + Send + Sync> + Send + Sync> {
2170 let a_idx = a as usize;
2171 if a_idx == IlmAction::NoneAction as usize || a_idx >= IlmAction::ActionCount as usize {
2172 return Box::new(|_| Box::new(|| {}));
2173 }
2174 let start = SystemTime::now();
2175 Box::new(move |versions: u64| {
2176 Box::new(move || {
2177 let duration = SystemTime::now().duration_since(start).unwrap_or_default();
2178 let metric_idx = Metric::Ilm as usize;
2179 global_metrics().operations[metric_idx].fetch_add(versions, Ordering::Relaxed);
2180 global_metrics().actions[a_idx].fetch_add(versions, Ordering::Relaxed);
2181 global_metrics().actions_latency[a_idx].add(duration);
2182 })
2183 })
2184 }
2185
2186 pub fn inc_time(metric: Metric, duration: Duration) {
2189 let metric_idx = metric as usize;
2190 global_metrics().operations[metric_idx].fetch_add(1, Ordering::Relaxed);
2191 global_metrics().record_source_work_for_metric(metric, 1);
2192 if metric_idx < Metric::LastRealtime as usize {
2193 global_metrics().latency[metric_idx].add(duration);
2194 }
2195 }
2196
2197 pub fn record_scanner_yield(&self, duration: Duration) {
2198 let metric_idx = Metric::Yield as usize;
2199 self.operations[metric_idx].fetch_add(1, Ordering::Relaxed);
2200 if metric_idx < Metric::LastRealtime as usize {
2201 self.latency[metric_idx].add(duration);
2202 }
2203 let duration_millis = duration_millis_saturated(duration);
2204 let _ = self
2205 .scanner_yield_duration_millis
2206 .try_update(Ordering::Relaxed, Ordering::Relaxed, |current| {
2207 Some(current.saturating_add(duration_millis))
2208 });
2209 }
2210
2211 pub fn record_scanner_throttle_sleep(&self, duration: Duration) {
2212 let metric_idx = Metric::ThrottleSleep as usize;
2213 self.operations[metric_idx].fetch_add(1, Ordering::Relaxed);
2214 if metric_idx < Metric::LastRealtime as usize {
2215 self.latency[metric_idx].add(duration);
2216 }
2217 let duration_millis = duration_millis_saturated(duration);
2218 let _ = self
2219 .scanner_throttle_sleep_duration_millis
2220 .try_update(Ordering::Relaxed, Ordering::Relaxed, |current| {
2221 Some(current.saturating_add(duration_millis))
2222 });
2223 }
2224
2225 pub fn record_scanner_ilm_action(&self, count: u64) {
2226 self.scanner_ilm_actions.fetch_add(count, Ordering::Relaxed);
2227 self.record_scanner_source_executed(ScannerWorkSource::Lifecycle, count);
2228 }
2229
2230 pub fn record_scanner_lifecycle_action(&self, action: IlmAction, count: u64) {
2231 self.record_scanner_ilm_action(count);
2232 match action {
2233 IlmAction::TransitionAction | IlmAction::TransitionVersionAction => {
2234 self.scanner_lifecycle_transition_actions.fetch_add(count, Ordering::Relaxed);
2235 }
2236 _ if action.delete() => {
2237 self.scanner_lifecycle_expiry_actions.fetch_add(count, Ordering::Relaxed);
2238 }
2239 _ => {}
2240 }
2241 }
2242
2243 pub fn record_scanner_ilm_enqueue_result(&self, count: u64, queued: bool) {
2244 if queued {
2245 self.record_scanner_source_queued(ScannerWorkSource::Lifecycle, count);
2246 } else {
2247 self.record_scanner_source_missed(ScannerWorkSource::Lifecycle, count);
2248 }
2249 }
2250
2251 pub fn record_scanner_expiry_enqueue_result(&self, count: u64, queued: bool) {
2252 self.record_scanner_ilm_enqueue_result(count, queued);
2253 if queued {
2254 self.scanner_expiry_queued_total.fetch_add(count, Ordering::Relaxed);
2255 } else {
2256 self.scanner_expiry_missed_total.fetch_add(count, Ordering::Relaxed);
2257 self.scanner_expiry_not_enqueued_total.fetch_add(count, Ordering::Relaxed);
2258 }
2259 }
2260
2261 pub fn record_scanner_expiry_blocked(&self, count: u64) {
2262 self.scanner_expiry_blocked_total.fetch_add(count, Ordering::Relaxed);
2263 }
2264
2265 pub fn record_scanner_expiry_delete_failed(&self, count: u64) {
2266 self.scanner_expiry_delete_failed_total.fetch_add(count, Ordering::Relaxed);
2267 }
2268
2269 pub fn record_scanner_transition_enqueue_result(&self, count: u64, queued: bool) {
2270 self.record_scanner_ilm_enqueue_result(count, queued);
2271 if queued {
2272 self.scanner_transition_queued_total.fetch_add(count, Ordering::Relaxed);
2273 } else {
2274 self.scanner_transition_missed_total.fetch_add(count, Ordering::Relaxed);
2275 }
2276 }
2277
2278 pub fn record_scanner_lifecycle_expiry_state(&self, state: ScannerLifecycleExpiryStateUpdate) {
2279 self.scanner_expiry_queue_capacity
2280 .store(state.queue_capacity, Ordering::Relaxed);
2281 self.scanner_expiry_queued.store(state.queued, Ordering::Relaxed);
2282 self.scanner_expiry_active.store(state.active, Ordering::Relaxed);
2283 self.scanner_expiry_workers.store(state.workers, Ordering::Relaxed);
2284 self.scanner_expiry_queue_missed.store(state.queue_missed, Ordering::Relaxed);
2285 }
2286
2287 pub fn record_scanner_lifecycle_transition_state(&self, state: ScannerLifecycleTransitionStateUpdate) {
2288 self.scanner_transition_queue_capacity
2289 .store(state.queue_capacity, Ordering::Relaxed);
2290 self.scanner_transition_queued.store(state.queued, Ordering::Relaxed);
2291 self.scanner_transition_active.store(state.active, Ordering::Relaxed);
2292 self.scanner_transition_workers.store(state.workers, Ordering::Relaxed);
2293 self.scanner_transition_queue_full.store(state.queue_full, Ordering::Relaxed);
2294 self.scanner_transition_queue_send_timeout
2295 .store(state.queue_send_timeout, Ordering::Relaxed);
2296 self.scanner_transition_compensation_scheduled
2297 .store(state.compensation_scheduled, Ordering::Relaxed);
2298 self.scanner_transition_compensation_pending
2299 .store(state.compensation_pending, Ordering::Relaxed);
2300 self.scanner_transition_compensation_running
2301 .store(state.compensation_running, Ordering::Relaxed);
2302 }
2303
2304 pub fn record_tier_request(&self, operation: TierRequestOperation, outcome: TierRequestOutcome) {
2310 let slot = operation.index() * TierRequestOutcome::ALL.len() + outcome.index();
2311 self.tier_requests[slot].fetch_add(1, Ordering::Relaxed);
2312 }
2313
2314 pub fn tier_request_counts(&self) -> Vec<TierRequestCount> {
2317 let mut counts = Vec::with_capacity(TIER_REQUEST_COUNTER_SLOTS);
2318 for operation in TierRequestOperation::ALL {
2319 for outcome in TierRequestOutcome::ALL {
2320 let slot = operation.index() * TierRequestOutcome::ALL.len() + outcome.index();
2321 counts.push(TierRequestCount {
2322 operation,
2323 outcome,
2324 count: self.tier_requests[slot].load(Ordering::Relaxed),
2325 });
2326 }
2327 }
2328 counts
2329 }
2330
2331 pub fn record_scanner_transition_completed(&self, count: u64) {
2332 self.scanner_transition_completed.fetch_add(count, Ordering::Relaxed);
2333 }
2334
2335 pub fn record_scanner_transition_failed(&self, count: u64) {
2336 self.scanner_transition_failed.fetch_add(count, Ordering::Relaxed);
2337 self.record_scanner_source_failed(ScannerWorkSource::Lifecycle, count);
2338 if !self.current_scan_cycle_work_active.load(Ordering::Acquire) {
2339 self.record_last_cycle_scanner_source_work(ScannerWorkSource::Lifecycle, ScannerSourceWorkUpdate::failed(count));
2340 }
2341 }
2342
2343 pub fn record_scanner_checkpoint_set(&self, version: u16, resume_after: impl Into<String>, reason: impl Into<String>) {
2344 let checkpoint = ScannerCheckpointReport {
2345 version,
2346 resume_after: resume_after.into(),
2347 reason: reason.into(),
2348 last_event: SCANNER_CHECKPOINT_EVENT_SET.to_string(),
2349 };
2350 match self.scanner_checkpoint.lock() {
2351 Ok(mut current) => *current = Some(checkpoint),
2352 Err(poisoned) => *poisoned.into_inner() = Some(checkpoint),
2353 }
2354 }
2355
2356 pub fn record_scanner_checkpoint_used(&self) {
2357 self.scanner_checkpoint_used.fetch_add(1, Ordering::Relaxed);
2358 self.update_scanner_checkpoint_event(SCANNER_CHECKPOINT_EVENT_USED);
2359 }
2360
2361 pub fn record_scanner_checkpoint_cleared(&self) {
2362 self.scanner_checkpoint_cleared.fetch_add(1, Ordering::Relaxed);
2363 match self.scanner_checkpoint.lock() {
2364 Ok(mut current) => *current = None,
2365 Err(poisoned) => *poisoned.into_inner() = None,
2366 }
2367 }
2368
2369 pub fn record_scanner_checkpoint_ignored(&self) {
2370 self.scanner_checkpoint_ignored.fetch_add(1, Ordering::Relaxed);
2371 self.update_scanner_checkpoint_event(SCANNER_CHECKPOINT_EVENT_IGNORED);
2372 }
2373
2374 pub fn record_scanner_checkpoint_stale(&self) {
2375 self.scanner_checkpoint_stale.fetch_add(1, Ordering::Relaxed);
2376 self.update_scanner_checkpoint_event(SCANNER_CHECKPOINT_EVENT_STALE);
2377 }
2378
2379 pub fn record_scanner_dirty_usage_pending(&self, pending_buckets: u64) {
2380 self.scanner_dirty_usage_pending_buckets
2381 .store(pending_buckets, Ordering::Relaxed);
2382 if pending_buckets > 0 {
2383 self.scanner_dirty_usage_last_mark_unix_secs
2384 .store(unix_now_secs(), Ordering::Relaxed);
2385 }
2386 }
2387
2388 pub fn record_scanner_dirty_usage_clear(&self, pending_buckets: u64) {
2389 self.scanner_dirty_usage_pending_buckets
2390 .store(pending_buckets, Ordering::Relaxed);
2391 self.scanner_dirty_usage_last_clear_unix_secs
2392 .store(unix_now_secs(), Ordering::Relaxed);
2393 }
2394
2395 pub fn record_scanner_dirty_usage_cycle_snapshot(&self, dirty_buckets: u64) {
2396 self.scanner_dirty_usage_last_cycle_dirty_buckets
2397 .store(dirty_buckets, Ordering::Relaxed);
2398 self.scanner_dirty_usage_last_cycle_cleared_buckets
2399 .store(0, Ordering::Relaxed);
2400 }
2401
2402 pub fn record_scanner_dirty_usage_cycle_clear(&self, cleared_buckets: u64, pending_buckets: u64) {
2403 self.scanner_dirty_usage_last_cycle_cleared_buckets
2404 .store(cleared_buckets, Ordering::Relaxed);
2405 self.scanner_dirty_usage_pending_buckets
2406 .store(pending_buckets, Ordering::Relaxed);
2407 if cleared_buckets > 0 {
2408 self.scanner_dirty_usage_last_clear_unix_secs
2409 .store(unix_now_secs(), Ordering::Relaxed);
2410 }
2411 }
2412
2413 pub fn record_scanner_usage_save_result(&self, result: ScannerUsageSaveResult) {
2414 self.scanner_usage_last_save_result.store(result as u8, Ordering::Relaxed);
2415 self.scanner_usage_last_save_unix_secs
2416 .store(unix_now_secs(), Ordering::Relaxed);
2417 }
2418
2419 pub fn record_scanner_usage_deferred(&self, reason: impl Into<String>) {
2422 let reason = reason.into();
2423 self.record_scanner_usage_publication("deferred", reason.clone());
2424 self.scanner_usage_deferred_pending.store(true, Ordering::Release);
2425 self.scanner_usage_deferred_total.fetch_add(1, Ordering::Relaxed);
2426 self.scanner_usage_last_deferred_unix_secs
2427 .store(unix_now_secs(), Ordering::Relaxed);
2428 let mut last_reason = match self.scanner_usage_last_deferred_reason.lock() {
2429 Ok(guard) => guard,
2430 Err(poisoned) => poisoned.into_inner(),
2431 };
2432 *last_reason = reason;
2433 }
2434
2435 pub fn record_scanner_usage_durable_success(&self) {
2436 self.record_scanner_usage_publication("success", "");
2437 self.scanner_usage_last_durable_success_unix_secs
2438 .store(unix_now_secs(), Ordering::Relaxed);
2439 self.scanner_usage_deferred_pending.store(false, Ordering::Release);
2440 }
2441
2442 pub fn record_scanner_usage_publication(&self, state: &str, reason: impl Into<String>) {
2443 self.scanner_usage_last_publication_unix_secs
2444 .store(unix_now_secs(), Ordering::Relaxed);
2445 let mut publication_state = match self.scanner_usage_last_publication_state.lock() {
2446 Ok(guard) => guard,
2447 Err(poisoned) => poisoned.into_inner(),
2448 };
2449 *publication_state = state.to_string();
2450 let mut publication_reason = match self.scanner_usage_last_publication_reason.lock() {
2451 Ok(guard) => guard,
2452 Err(poisoned) => poisoned.into_inner(),
2453 };
2454 *publication_reason = reason.into();
2455 }
2456
2457 pub fn record_scanner_source_work(&self, source: ScannerWorkSource, work: ScannerSourceWorkUpdate) {
2458 if let Some(counters) = self.scanner_source_work.get(source.index()) {
2459 counters.add(work);
2460 }
2461 }
2462
2463 pub fn record_scanner_replication_repair_work(&self, kind: ScannerReplicationRepairKind, work: ScannerSourceWorkUpdate) {
2464 if let Some(counters) = self.scanner_replication_repair_work.get(kind.index()) {
2465 counters.add(work);
2466 }
2467 }
2468
2469 fn record_last_cycle_scanner_source_work(&self, source: ScannerWorkSource, work: ScannerSourceWorkUpdate) {
2470 if let Some(counters) = self.last_scan_cycle_source_work.get(source.index()) {
2471 counters.add(work);
2472 }
2473 }
2474
2475 pub fn record_scanner_source_checked(&self, source: ScannerWorkSource, count: u64) {
2476 self.record_scanner_source_work(source, ScannerSourceWorkUpdate::checked(count));
2477 }
2478
2479 pub fn record_scanner_versions_scanned(&self, count: u64) {
2487 self.lifetime_versions_scanned.fetch_add(count, Ordering::Relaxed);
2488 }
2489
2490 pub fn record_scanner_source_queued(&self, source: ScannerWorkSource, count: u64) {
2491 self.record_scanner_source_work(source, ScannerSourceWorkUpdate::queued(count));
2492 }
2493
2494 pub fn record_scanner_source_executed(&self, source: ScannerWorkSource, count: u64) {
2495 self.record_scanner_source_work(source, ScannerSourceWorkUpdate::executed(count));
2496 }
2497
2498 pub fn record_scanner_source_failed(&self, source: ScannerWorkSource, count: u64) {
2499 self.record_scanner_source_work(source, ScannerSourceWorkUpdate::failed(count));
2500 }
2501
2502 pub fn record_scanner_source_missed(&self, source: ScannerWorkSource, count: u64) {
2503 self.record_scanner_source_work(source, ScannerSourceWorkUpdate::missed(count));
2504 }
2505
2506 fn update_scanner_checkpoint_event(&self, event: &str) {
2507 match self.scanner_checkpoint.lock() {
2508 Ok(mut current) => {
2509 if let Some(checkpoint) = current.as_mut() {
2510 checkpoint.last_event = event.to_string();
2511 }
2512 }
2513 Err(poisoned) => {
2514 let mut current = poisoned.into_inner();
2515 if let Some(checkpoint) = current.as_mut() {
2516 checkpoint.last_event = event.to_string();
2517 }
2518 }
2519 }
2520 }
2521
2522 fn record_source_work_for_metric(&self, metric: Metric, count: u64) {
2523 match metric {
2524 Metric::ScanObject | Metric::ScanFolder => {
2525 self.record_scanner_source_checked(ScannerWorkSource::Usage, count);
2526 }
2527 Metric::SaveUsage => {
2528 self.record_scanner_source_executed(ScannerWorkSource::Usage, count);
2529 }
2530 Metric::CheckReplication => {
2531 self.record_scanner_source_checked(ScannerWorkSource::BucketReplication, count);
2532 }
2533 Metric::HealCheck => {
2534 self.record_scanner_source_checked(ScannerWorkSource::Heal, count);
2535 }
2536 _ => {}
2537 }
2538 }
2539
2540 pub fn record_scan_bucket_drive_start(&self, source: ScannerWorkSource, bucket: &str, drive: &str) {
2541 self.operations[Metric::ScanBucketDriveStart as usize].fetch_add(1, Ordering::Relaxed);
2542 if bucket.is_empty() || drive.is_empty() {
2543 return;
2544 }
2545 let key = ScannerActiveBucketDriveKey {
2546 source: source.as_str().to_string(),
2547 bucket: bucket.to_string(),
2548 drive: drive.to_string(),
2549 };
2550 let mut active = self
2551 .scanner_active_bucket_drive_scans
2552 .lock()
2553 .unwrap_or_else(|poisoned| poisoned.into_inner());
2554 active
2555 .entry(key)
2556 .and_modify(|value| value.count = value.count.saturating_add(1))
2557 .or_insert(ScannerActiveBucketDriveValue {
2558 count: 1,
2559 started_at: Timestamp::now(),
2560 });
2561 }
2562
2563 pub fn record_scan_bucket_drive_end(&self, source: ScannerWorkSource, bucket: &str, drive: &str) {
2564 let key = ScannerActiveBucketDriveKey {
2565 source: source.as_str().to_string(),
2566 bucket: bucket.to_string(),
2567 drive: drive.to_string(),
2568 };
2569 let mut active = self
2570 .scanner_active_bucket_drive_scans
2571 .lock()
2572 .unwrap_or_else(|poisoned| poisoned.into_inner());
2573 if let Some(value) = active.get_mut(&key) {
2574 value.count = value.count.saturating_sub(1);
2575 if value.count == 0 {
2576 active.remove(&key);
2577 }
2578 }
2579 }
2580
2581 pub fn record_scan_bucket_drive_failure(&self) {
2582 self.operations[Metric::ScanBucketDriveFailure as usize].fetch_add(1, Ordering::Relaxed);
2583 }
2584
2585 pub fn record_scanner_throttle_config(
2586 &self,
2587 idle_mode_enabled: bool,
2588 sleep_factor: f64,
2589 max_sleep: Duration,
2590 yield_every_n_objects: u64,
2591 ) {
2592 self.scanner_throttle_idle_mode_enabled
2593 .store(idle_mode_enabled, Ordering::Relaxed);
2594 self.scanner_throttle_sleep_factor_micros
2595 .store(scaled_f64_to_u64_saturated(sleep_factor, 1_000_000.0), Ordering::Relaxed);
2596 self.scanner_throttle_max_sleep_millis
2597 .store(duration_millis_saturated(max_sleep), Ordering::Relaxed);
2598 self.scanner_yield_every_n_objects
2599 .store(yield_every_n_objects, Ordering::Relaxed);
2600 }
2601
2602 pub fn record_scanner_cycle_config(
2603 &self,
2604 cycle_interval: Duration,
2605 bitrot_cycle: Option<Duration>,
2606 cycle_max_duration: Option<Duration>,
2607 cycle_max_objects: Option<u64>,
2608 cycle_max_directories: Option<u64>,
2609 ) {
2610 self.scanner_cycle_interval_millis
2611 .store(duration_millis_saturated(cycle_interval), Ordering::Relaxed);
2612 self.scanner_cycle_max_duration_millis
2613 .store(cycle_max_duration.map(duration_millis_saturated).unwrap_or_default(), Ordering::Relaxed);
2614 self.scanner_cycle_max_objects
2615 .store(cycle_max_objects.unwrap_or_default(), Ordering::Relaxed);
2616 self.scanner_cycle_max_directories
2617 .store(cycle_max_directories.unwrap_or_default(), Ordering::Relaxed);
2618 self.scanner_leader_lease_without_progress.store(false, Ordering::Relaxed);
2619 self.scanner_cycle_last_progress_age_seconds.store(0, Ordering::Relaxed);
2620 metrics::gauge!(OTEL_SCANNER_LEADER_LEASE_WITHOUT_PROGRESS).set(0.0);
2621 metrics::gauge!(OTEL_SCANNER_CYCLE_LAST_PROGRESS_AGE).set(0.0);
2622 self.scanner_bitrot_cycle_enabled
2623 .store(bitrot_cycle.is_some(), Ordering::Relaxed);
2624 self.scanner_bitrot_cycle_millis
2625 .store(bitrot_cycle.map(duration_millis_saturated).unwrap_or_default(), Ordering::Relaxed);
2626 }
2627
2628 pub fn record_scanner_cycle_timeout(&self, recovery_required: bool, progress_age: Duration) {
2629 self.scanner_cycle_timeout_total.fetch_add(1, Ordering::Relaxed);
2630 if recovery_required {
2631 self.scanner_cycle_recovery_required_total.fetch_add(1, Ordering::Relaxed);
2632 }
2633 self.scanner_cycle_last_progress_age_seconds
2634 .store(progress_age.as_secs(), Ordering::Relaxed);
2635 self.scanner_leader_lease_without_progress.store(true, Ordering::Relaxed);
2636 metrics::counter!(OTEL_SCANNER_CYCLE_TIMEOUT_TOTAL).increment(1);
2637 metrics::gauge!(OTEL_SCANNER_CYCLE_LAST_PROGRESS_AGE).set(progress_age.as_secs_f64());
2638 metrics::gauge!(OTEL_SCANNER_LEADER_LEASE_WITHOUT_PROGRESS).set(1.0);
2639 }
2640
2641 pub fn record_scanner_set_scan_state(&self, concurrency_limit: Option<usize>, queued: Option<usize>, active: Option<usize>) {
2642 if let Some(concurrency_limit) = concurrency_limit {
2643 self.scanner_set_scan_concurrency_limit
2644 .store(concurrency_limit as u64, Ordering::Relaxed);
2645 }
2646 if let Some(queued) = queued {
2647 self.scanner_set_scans_queued.store(queued as u64, Ordering::Relaxed);
2648 }
2649 if let Some(active) = active {
2650 self.scanner_set_scans_active.store(active as u64, Ordering::Relaxed);
2651 }
2652 }
2653
2654 pub fn reset_scanner_set_scan_state(&self) {
2655 self.record_scanner_set_scan_state(Some(0), Some(0), Some(0));
2656 match self.scanner_disk_bucket_scan_states.lock() {
2657 Ok(mut states) => states.clear(),
2658 Err(poisoned) => poisoned.into_inner().clear(),
2659 }
2660 }
2661
2662 pub fn record_scanner_disk_bucket_scan_state(
2663 &self,
2664 pool: &str,
2665 set: &str,
2666 concurrency_limit: Option<usize>,
2667 queued: Option<usize>,
2668 active: Option<usize>,
2669 ) {
2670 let key = (pool.to_string(), set.to_string());
2671 let mut states = self
2672 .scanner_disk_bucket_scan_states
2673 .lock()
2674 .unwrap_or_else(|poisoned| poisoned.into_inner());
2675 let state = states.entry(key).or_default();
2676 if let Some(concurrency_limit) = concurrency_limit {
2677 state.concurrency_limit = concurrency_limit as u64;
2678 }
2679 if let Some(queued) = queued {
2680 state.queued = queued as u64;
2681 }
2682 if let Some(active) = active {
2683 state.active = active as u64;
2684 }
2685 }
2686
2687 pub fn record_scanner_bucket_drive_result(&self, bucket: &str, drive: &str, result: &str) {
2688 if bucket.is_empty() || drive.is_empty() || result.is_empty() {
2689 return;
2690 }
2691 let key = ScannerBucketDriveResultKey::new(bucket, drive, result);
2692 let mut results = self
2693 .scanner_bucket_drive_results
2694 .lock()
2695 .unwrap_or_else(|poisoned| poisoned.into_inner());
2696 let last_seen = self.scanner_bucket_drive_result_clock.fetch_add(1, Ordering::Relaxed);
2697 if let Some(previous_last_seen) = results.counts.get_mut(&key).map(|value| {
2698 let previous_last_seen = value.last_seen;
2699 value.count = value.count.saturating_add(1);
2700 value.last_seen = last_seen;
2701 previous_last_seen
2702 }) {
2703 results.eviction_index.remove(&(previous_last_seen, key.clone()));
2704 results.eviction_index.insert((last_seen, key));
2705 return;
2706 }
2707
2708 if results.counts.len() >= MAX_SCANNER_BUCKET_DRIVE_RESULT_KEYS
2709 && let Some((_, stale_key)) = results.eviction_index.pop_first()
2710 {
2711 results.counts.remove(&stale_key);
2712 }
2713
2714 if results.counts.len() < MAX_SCANNER_BUCKET_DRIVE_RESULT_KEYS {
2715 results
2716 .counts
2717 .insert(key.clone(), ScannerBucketDriveResultValue { count: 1, last_seen });
2718 results.eviction_index.insert((last_seen, key));
2719 }
2720 }
2721
2722 pub fn lifetime(&self, metric: Metric) -> u64 {
2728 let idx = metric as usize;
2729 if idx >= Metric::Last as usize {
2730 return 0;
2731 }
2732 self.operations[idx].load(Ordering::Relaxed)
2733 }
2734
2735 pub fn last_minute(&self, metric: Metric) -> AccElem {
2738 let idx = metric as usize;
2739 if idx >= Metric::LastRealtime as usize {
2740 return AccElem::default();
2741 }
2742 self.latency[idx].total()
2743 }
2744
2745 pub async fn set_cycle(&self, cycle: Option<CurrentCycle>) {
2747 *self.cycle_info.write().await = cycle;
2748 }
2749
2750 pub async fn start_scan_cycle_work_with_cycle(&self, cycle: CurrentCycle) -> ScanCycleWorkSnapshot {
2752 let mut current_cycle = self.cycle_info.write().await;
2753 let snapshot = self.start_scan_cycle_work();
2754 *current_cycle = Some(cycle);
2755 snapshot
2756 }
2757
2758 pub async fn finish_scan_cycle_work_with_cycle(&self, start: ScanCycleWorkSnapshot, cycle: CurrentCycle) {
2760 let mut current_cycle = self.cycle_info.write().await;
2761 self.finish_scan_cycle_work(start);
2762 *current_cycle = Some(cycle);
2763 }
2764
2765 pub async fn get_cycle(&self) -> Option<CurrentCycle> {
2767 self.cycle_info.read().await.clone()
2768 }
2769
2770 pub fn set_current_scan_mode(&self, scan_mode: HealScanMode) {
2771 self.current_scan_mode.store(scan_mode as u8, Ordering::Relaxed);
2772 }
2773
2774 pub fn clear_current_scan_mode(&self) {
2775 self.set_current_scan_mode(HealScanMode::Unknown);
2776 }
2777
2778 pub fn current_scan_mode(&self) -> HealScanMode {
2779 HealScanMode::from_u8(self.current_scan_mode.load(Ordering::Relaxed)).unwrap_or(HealScanMode::Unknown)
2780 }
2781
2782 pub async fn record_scanner_leader_liveness(&self, state: impl Into<String>, held: bool, error: impl Into<String>) {
2783 *self.scanner_leader_lock_state.write().await = state.into();
2784 *self.scanner_leader_lock_last_error.write().await = error.into();
2785 self.scanner_leader_lock_held.store(held, Ordering::Relaxed);
2786 self.scanner_leader_lock_last_update_unix_secs
2787 .store(Utc::now().timestamp().max(0) as u64, Ordering::Relaxed);
2788 }
2789
2790 pub fn record_scanner_cycle_end_time(&self) {
2791 self.last_scan_cycle_end_unix_secs
2792 .store(Utc::now().timestamp().max(0) as u64, Ordering::Relaxed);
2793 }
2794
2795 pub fn record_scan_cycle_complete(&self, success: bool, duration: Duration) {
2796 let result = if success {
2797 SCAN_CYCLE_RESULT_SUCCESS
2798 } else {
2799 self.failed_scan_cycles.fetch_add(1, Ordering::Relaxed);
2800 SCAN_CYCLE_RESULT_ERROR
2801 };
2802 self.record_scanner_cycle_end_time();
2803 self.last_scan_cycle_result.store(result, Ordering::Relaxed);
2804 self.last_scan_cycle_partial_reason
2805 .store(ScanCyclePartialReason::Unknown as u8, Ordering::Relaxed);
2806 self.last_scan_cycle_partial_source.store(0, Ordering::Relaxed);
2807 self.last_scan_cycle_duration_millis
2808 .store(duration_millis_saturated(duration), Ordering::Relaxed);
2809 }
2810
2811 pub fn record_scan_cycle_superseded(&self, duration: Duration) {
2812 self.record_scanner_cycle_end_time();
2813 self.superseded_scan_cycles.fetch_add(1, Ordering::Relaxed);
2814 self.last_scan_cycle_result
2815 .store(SCAN_CYCLE_RESULT_SUPERSEDED, Ordering::Relaxed);
2816 self.last_scan_cycle_partial_reason
2817 .store(ScanCyclePartialReason::Unknown as u8, Ordering::Relaxed);
2818 self.last_scan_cycle_partial_source.store(0, Ordering::Relaxed);
2819 self.last_scan_cycle_duration_millis
2820 .store(duration_millis_saturated(duration), Ordering::Relaxed);
2821 }
2822
2823 pub fn record_scan_cycle_deferred(&self, duration: Duration) {
2824 self.record_scanner_cycle_end_time();
2825 self.last_scan_cycle_result
2826 .store(SCAN_CYCLE_RESULT_DEFERRED, Ordering::Relaxed);
2827 self.last_scan_cycle_partial_reason
2828 .store(ScanCyclePartialReason::Unknown as u8, Ordering::Relaxed);
2829 self.last_scan_cycle_partial_source.store(0, Ordering::Relaxed);
2830 self.last_scan_cycle_duration_millis
2831 .store(duration_millis_saturated(duration), Ordering::Relaxed);
2832 }
2833
2834 pub fn record_scan_cycle_partial(&self, duration: Duration, reason: ScanCyclePartialReason) {
2835 self.record_scan_cycle_partial_with_source(duration, reason, None);
2836 }
2837
2838 pub fn record_scan_cycle_partial_with_source(
2839 &self,
2840 duration: Duration,
2841 reason: ScanCyclePartialReason,
2842 source: Option<ScannerWorkSource>,
2843 ) {
2844 self.record_scanner_cycle_end_time();
2845 self.partial_scan_cycles.fetch_add(1, Ordering::Relaxed);
2846 match reason {
2847 ScanCyclePartialReason::Unknown => &self.partial_scan_cycles_unknown,
2848 ScanCyclePartialReason::Runtime => &self.partial_scan_cycles_runtime,
2849 ScanCyclePartialReason::Objects => &self.partial_scan_cycles_objects,
2850 ScanCyclePartialReason::Directories => &self.partial_scan_cycles_directories,
2851 }
2852 .fetch_add(1, Ordering::Relaxed);
2853 self.last_scan_cycle_result
2854 .store(SCAN_CYCLE_RESULT_PARTIAL, Ordering::Relaxed);
2855 self.last_scan_cycle_partial_reason.store(reason as u8, Ordering::Relaxed);
2856 self.last_scan_cycle_partial_source
2857 .store(source.map(ScannerWorkSource::code).unwrap_or_default(), Ordering::Relaxed);
2858 if let Some(source) = source
2859 && let Some(cycles) = self.partial_scan_cycles_by_source.get(source.index())
2860 {
2861 cycles.fetch_add(1, Ordering::Relaxed);
2862 }
2863 self.last_scan_cycle_duration_millis
2864 .store(duration_millis_saturated(duration), Ordering::Relaxed);
2865 }
2866
2867 pub fn start_scan_cycle_work(&self) -> ScanCycleWorkSnapshot {
2868 let snapshot = self.scan_cycle_work_snapshot();
2869 let source_snapshot = self.scanner_source_work_values();
2870 let replication_repair_snapshot = self.scanner_replication_repair_work_values();
2871 self.current_scan_cycle_objects_start
2872 .store(snapshot.objects_scanned, Ordering::Relaxed);
2873 self.current_scan_cycle_directories_start
2874 .store(snapshot.directories_scanned, Ordering::Relaxed);
2875 self.current_scan_cycle_bucket_drive_scans_start
2876 .store(snapshot.bucket_drive_scans, Ordering::Relaxed);
2877 self.current_scan_cycle_bucket_drive_failures_start
2878 .store(snapshot.bucket_drive_failures, Ordering::Relaxed);
2879 self.current_scan_cycle_yield_events_start
2880 .store(snapshot.yield_events, Ordering::Relaxed);
2881 self.current_scan_cycle_yield_duration_millis_start
2882 .store(snapshot.yield_duration_millis, Ordering::Relaxed);
2883 self.current_scan_cycle_throttle_sleep_events_start
2884 .store(snapshot.throttle_sleep_events, Ordering::Relaxed);
2885 self.current_scan_cycle_throttle_sleep_duration_millis_start
2886 .store(snapshot.throttle_sleep_duration_millis, Ordering::Relaxed);
2887 self.current_scan_cycle_ilm_actions_start
2888 .store(snapshot.ilm_actions, Ordering::Relaxed);
2889 self.current_scan_cycle_lifecycle_expiry_actions_start
2890 .store(snapshot.lifecycle_expiry_actions, Ordering::Relaxed);
2891 self.current_scan_cycle_lifecycle_transition_actions_start
2892 .store(snapshot.lifecycle_transition_actions, Ordering::Relaxed);
2893 self.current_scan_cycle_heal_objects_start
2894 .store(snapshot.heal_objects, Ordering::Relaxed);
2895 self.current_scan_cycle_replication_checks_start
2896 .store(snapshot.replication_checks, Ordering::Relaxed);
2897 self.current_scan_cycle_usage_saves_start
2898 .store(snapshot.usage_saves, Ordering::Relaxed);
2899 self.store_scanner_source_work_values(&self.current_scan_cycle_source_work_start, &source_snapshot);
2900 self.store_scanner_replication_repair_work_values(
2901 &self.current_scan_cycle_replication_repair_work_start,
2902 &replication_repair_snapshot,
2903 );
2904 let bucket_drive_results = self.scanner_bucket_drive_result_counts();
2905 match self.current_scan_cycle_bucket_drive_results_start.lock() {
2906 Ok(mut start) => *start = bucket_drive_results,
2907 Err(poisoned) => *poisoned.into_inner() = bucket_drive_results,
2908 }
2909 self.current_scan_cycle_work_active.store(true, Ordering::Release);
2910 snapshot
2911 }
2912
2913 pub fn finish_scan_cycle_work(&self, start: ScanCycleWorkSnapshot) {
2914 let work = self.scan_cycle_work_since(start);
2915 let source_work = self.scanner_source_work_since(&self.current_scan_cycle_source_work_start_values());
2916 let replication_repair_work =
2917 self.scanner_replication_repair_work_since(&self.current_scan_cycle_replication_repair_work_start_values());
2918 self.record_scan_cycle_work(work);
2919 self.record_scan_cycle_source_work(&source_work);
2920 self.record_scan_cycle_replication_repair_work(&replication_repair_work);
2921 let bucket_drive_results = self.current_cycle_bucket_drive_result_snapshots();
2922 match self.last_scan_cycle_bucket_drive_results.lock() {
2923 Ok(mut last) => *last = bucket_drive_results,
2924 Err(poisoned) => *poisoned.into_inner() = bucket_drive_results,
2925 }
2926 self.current_scan_cycle_work_active.store(false, Ordering::Release);
2927 }
2928
2929 pub fn current_scan_cycle_has_unresolved_heal_work(&self) -> bool {
2930 if !self.current_scan_cycle_work_active.load(Ordering::Acquire) {
2931 return false;
2932 }
2933
2934 let source_work = self.scanner_source_work_since(&self.current_scan_cycle_source_work_start_values());
2935 [ScannerWorkSource::Heal, ScannerWorkSource::Bitrot]
2936 .into_iter()
2937 .filter_map(|source| source_work.get(source.index()))
2938 .any(|work| work.queued > 0 || work.skipped > 0 || work.failed > 0 || work.missed > 0)
2939 }
2940
2941 fn scan_cycle_work_snapshot(&self) -> ScanCycleWorkSnapshot {
2942 ScanCycleWorkSnapshot {
2943 objects_scanned: self.lifetime(Metric::ScanObject),
2944 directories_scanned: self.lifetime(Metric::ScanFolder),
2945 bucket_drive_scans: self.lifetime(Metric::ScanBucketDrive),
2946 bucket_drive_failures: self.lifetime(Metric::ScanBucketDriveFailure),
2947 yield_events: self.lifetime(Metric::Yield),
2948 yield_duration_millis: self.scanner_yield_duration_millis.load(Ordering::Relaxed),
2949 throttle_sleep_events: self.lifetime(Metric::ThrottleSleep),
2950 throttle_sleep_duration_millis: self.scanner_throttle_sleep_duration_millis.load(Ordering::Relaxed),
2951 ilm_actions: self.scanner_ilm_actions.load(Ordering::Relaxed),
2952 lifecycle_expiry_actions: self.scanner_lifecycle_expiry_actions.load(Ordering::Relaxed),
2953 lifecycle_transition_actions: self.scanner_lifecycle_transition_actions.load(Ordering::Relaxed),
2954 heal_objects: self.lifetime(Metric::HealAbandonedObject),
2955 replication_checks: self.lifetime(Metric::CheckReplication),
2956 usage_saves: self.lifetime(Metric::SaveUsage),
2957 }
2958 }
2959
2960 fn current_scan_cycle_work_start(&self) -> ScanCycleWorkSnapshot {
2961 ScanCycleWorkSnapshot {
2962 objects_scanned: self.current_scan_cycle_objects_start.load(Ordering::Relaxed),
2963 directories_scanned: self.current_scan_cycle_directories_start.load(Ordering::Relaxed),
2964 bucket_drive_scans: self.current_scan_cycle_bucket_drive_scans_start.load(Ordering::Relaxed),
2965 bucket_drive_failures: self.current_scan_cycle_bucket_drive_failures_start.load(Ordering::Relaxed),
2966 yield_events: self.current_scan_cycle_yield_events_start.load(Ordering::Relaxed),
2967 yield_duration_millis: self.current_scan_cycle_yield_duration_millis_start.load(Ordering::Relaxed),
2968 throttle_sleep_events: self.current_scan_cycle_throttle_sleep_events_start.load(Ordering::Relaxed),
2969 throttle_sleep_duration_millis: self
2970 .current_scan_cycle_throttle_sleep_duration_millis_start
2971 .load(Ordering::Relaxed),
2972 ilm_actions: self.current_scan_cycle_ilm_actions_start.load(Ordering::Relaxed),
2973 lifecycle_expiry_actions: self.current_scan_cycle_lifecycle_expiry_actions_start.load(Ordering::Relaxed),
2974 lifecycle_transition_actions: self
2975 .current_scan_cycle_lifecycle_transition_actions_start
2976 .load(Ordering::Relaxed),
2977 heal_objects: self.current_scan_cycle_heal_objects_start.load(Ordering::Relaxed),
2978 replication_checks: self.current_scan_cycle_replication_checks_start.load(Ordering::Relaxed),
2979 usage_saves: self.current_scan_cycle_usage_saves_start.load(Ordering::Relaxed),
2980 }
2981 }
2982
2983 fn scan_cycle_work_since(&self, start: ScanCycleWorkSnapshot) -> ScanCycleWorkSnapshot {
2984 let current = self.scan_cycle_work_snapshot();
2985 ScanCycleWorkSnapshot {
2986 objects_scanned: current.objects_scanned.saturating_sub(start.objects_scanned),
2987 directories_scanned: current.directories_scanned.saturating_sub(start.directories_scanned),
2988 bucket_drive_scans: current.bucket_drive_scans.saturating_sub(start.bucket_drive_scans),
2989 bucket_drive_failures: current.bucket_drive_failures.saturating_sub(start.bucket_drive_failures),
2990 yield_events: current.yield_events.saturating_sub(start.yield_events),
2991 yield_duration_millis: current.yield_duration_millis.saturating_sub(start.yield_duration_millis),
2992 throttle_sleep_events: current.throttle_sleep_events.saturating_sub(start.throttle_sleep_events),
2993 throttle_sleep_duration_millis: current
2994 .throttle_sleep_duration_millis
2995 .saturating_sub(start.throttle_sleep_duration_millis),
2996 ilm_actions: current.ilm_actions.saturating_sub(start.ilm_actions),
2997 lifecycle_expiry_actions: current
2998 .lifecycle_expiry_actions
2999 .saturating_sub(start.lifecycle_expiry_actions),
3000 lifecycle_transition_actions: current
3001 .lifecycle_transition_actions
3002 .saturating_sub(start.lifecycle_transition_actions),
3003 heal_objects: current.heal_objects.saturating_sub(start.heal_objects),
3004 replication_checks: current.replication_checks.saturating_sub(start.replication_checks),
3005 usage_saves: current.usage_saves.saturating_sub(start.usage_saves),
3006 }
3007 }
3008
3009 fn scanner_bucket_drive_result_counts(&self) -> HashMap<ScannerBucketDriveResultKey, u64> {
3010 self.scanner_bucket_drive_results
3011 .lock()
3012 .unwrap_or_else(|poisoned| poisoned.into_inner())
3013 .counts
3014 .iter()
3015 .map(|(key, value)| (key.clone(), value.count))
3016 .collect()
3017 }
3018
3019 fn scanner_bucket_drive_result_snapshots(
3020 counts: impl IntoIterator<Item = (ScannerBucketDriveResultKey, u64)>,
3021 ) -> Vec<ScannerBucketDriveResultSnapshot> {
3022 let mut snapshots = counts
3023 .into_iter()
3024 .filter(|(_, count)| *count > 0)
3025 .map(|(key, count)| ScannerBucketDriveResultSnapshot {
3026 bucket: key.bucket,
3027 drive: key.drive,
3028 result: key.result,
3029 count,
3030 })
3031 .collect::<Vec<_>>();
3032 snapshots.sort_by(|left, right| {
3033 left.bucket
3034 .cmp(&right.bucket)
3035 .then_with(|| left.drive.cmp(&right.drive))
3036 .then_with(|| left.result.cmp(&right.result))
3037 });
3038 snapshots
3039 }
3040
3041 fn scanner_bucket_drive_result_counter_snapshots(&self) -> Vec<ScannerBucketDriveResultSnapshot> {
3042 Self::scanner_bucket_drive_result_snapshots(self.scanner_bucket_drive_result_counts())
3043 }
3044
3045 fn current_cycle_bucket_drive_result_snapshots(&self) -> Vec<ScannerBucketDriveResultSnapshot> {
3046 let current = self.scanner_bucket_drive_result_counts();
3047 let start = self
3048 .current_scan_cycle_bucket_drive_results_start
3049 .lock()
3050 .unwrap_or_else(|poisoned| poisoned.into_inner())
3051 .clone();
3052 Self::scanner_bucket_drive_result_snapshots(current.into_iter().filter_map(|(key, count)| {
3053 let delta = count.saturating_sub(start.get(&key).copied().unwrap_or_default());
3054 (delta > 0).then_some((key, delta))
3055 }))
3056 }
3057
3058 pub fn scanner_runtime_details_report(&self) -> ScannerRuntimeDetailsReport {
3059 self.scanner_runtime_details_report_for_active(self.current_scan_cycle_work_active.load(Ordering::Acquire))
3060 }
3061
3062 fn scanner_runtime_details_report_for_active(&self, current_cycle_active: bool) -> ScannerRuntimeDetailsReport {
3063 let current_cycle_bucket_drive_results = if current_cycle_active {
3064 self.current_cycle_bucket_drive_result_snapshots()
3065 } else {
3066 Vec::new()
3067 };
3068 let now = Timestamp::now();
3069 let mut active_bucket_drive_scans = self
3070 .scanner_active_bucket_drive_scans
3071 .lock()
3072 .unwrap_or_else(|poisoned| poisoned.into_inner())
3073 .iter()
3074 .map(|(key, value)| ScannerActiveBucketDriveSnapshot {
3075 source: key.source.clone(),
3076 bucket: key.bucket.clone(),
3077 drive: key.drive.clone(),
3078 count: value.count,
3079 age_seconds: timestamp_elapsed_seconds_since(now, value.started_at),
3080 })
3081 .collect::<Vec<_>>();
3082 active_bucket_drive_scans.sort_by(|left, right| {
3083 left.source
3084 .cmp(&right.source)
3085 .then_with(|| left.bucket.cmp(&right.bucket))
3086 .then_with(|| left.drive.cmp(&right.drive))
3087 });
3088 ScannerRuntimeDetailsReport {
3089 disk_bucket_scan_states: self.scanner_disk_bucket_scan_state_snapshots(),
3090 bucket_drive_results: self.scanner_bucket_drive_result_counter_snapshots(),
3091 current_cycle_bucket_drive_results,
3092 last_cycle_bucket_drive_results: self
3093 .last_scan_cycle_bucket_drive_results
3094 .lock()
3095 .unwrap_or_else(|poisoned| poisoned.into_inner())
3096 .clone(),
3097 active_bucket_drive_scans,
3098 }
3099 }
3100
3101 fn scanner_disk_bucket_scan_state_snapshots(&self) -> Vec<ScannerDiskBucketScanSnapshot> {
3102 let mut disk_bucket_scan_states = match self.scanner_disk_bucket_scan_states.lock() {
3103 Ok(states) => states
3104 .iter()
3105 .map(|((pool, set), state)| ScannerDiskBucketScanSnapshot {
3106 pool: pool.clone(),
3107 set: set.clone(),
3108 concurrency_limit: state.concurrency_limit,
3109 queued: state.queued,
3110 active: state.active,
3111 })
3112 .collect::<Vec<_>>(),
3113 Err(poisoned) => poisoned
3114 .into_inner()
3115 .iter()
3116 .map(|((pool, set), state)| ScannerDiskBucketScanSnapshot {
3117 pool: pool.clone(),
3118 set: set.clone(),
3119 concurrency_limit: state.concurrency_limit,
3120 queued: state.queued,
3121 active: state.active,
3122 })
3123 .collect::<Vec<_>>(),
3124 };
3125 disk_bucket_scan_states.sort_by(|left, right| left.pool.cmp(&right.pool).then_with(|| left.set.cmp(&right.set)));
3126 disk_bucket_scan_states
3127 }
3128
3129 fn scanner_source_work_values(&self) -> Vec<ScannerSourceWorkValues> {
3130 ScannerWorkSource::all()
3131 .iter()
3132 .filter_map(|source| {
3133 self.scanner_source_work
3134 .get(source.index())
3135 .map(ScannerSourceWorkCounters::values)
3136 })
3137 .collect()
3138 }
3139
3140 fn current_scan_cycle_source_work_start_values(&self) -> Vec<ScannerSourceWorkValues> {
3141 ScannerWorkSource::all()
3142 .iter()
3143 .filter_map(|source| {
3144 self.current_scan_cycle_source_work_start
3145 .get(source.index())
3146 .map(ScannerSourceWorkCounters::values)
3147 })
3148 .collect()
3149 }
3150
3151 fn scanner_source_work_since(&self, start: &[ScannerSourceWorkValues]) -> Vec<ScannerSourceWorkValues> {
3152 self.scanner_source_work_values()
3153 .into_iter()
3154 .enumerate()
3155 .map(|(index, current)| current.saturating_sub(start.get(index).copied().unwrap_or_default()))
3156 .collect()
3157 }
3158
3159 fn scanner_source_work_snapshots(&self, values: &[ScannerSourceWorkValues]) -> Vec<ScannerSourceWorkSnapshot> {
3160 ScannerWorkSource::all()
3161 .iter()
3162 .filter_map(|source| values.get(source.index()).map(|values| values.snapshot(*source)))
3163 .collect()
3164 }
3165
3166 fn scanner_source_work_counter_snapshots(&self, counters: &[ScannerSourceWorkCounters]) -> Vec<ScannerSourceWorkSnapshot> {
3167 ScannerWorkSource::all()
3168 .iter()
3169 .filter_map(|source| counters.get(source.index()).map(|counters| counters.snapshot(*source)))
3170 .collect()
3171 }
3172
3173 fn scanner_replication_repair_work_values(&self) -> Vec<ScannerSourceWorkValues> {
3174 ScannerReplicationRepairKind::all()
3175 .iter()
3176 .filter_map(|kind| {
3177 self.scanner_replication_repair_work
3178 .get(kind.index())
3179 .map(ScannerSourceWorkCounters::values)
3180 })
3181 .collect()
3182 }
3183
3184 fn current_scan_cycle_replication_repair_work_start_values(&self) -> Vec<ScannerSourceWorkValues> {
3185 ScannerReplicationRepairKind::all()
3186 .iter()
3187 .filter_map(|kind| {
3188 self.current_scan_cycle_replication_repair_work_start
3189 .get(kind.index())
3190 .map(ScannerSourceWorkCounters::values)
3191 })
3192 .collect()
3193 }
3194
3195 fn scanner_replication_repair_work_since(&self, start: &[ScannerSourceWorkValues]) -> Vec<ScannerSourceWorkValues> {
3196 self.scanner_replication_repair_work_values()
3197 .into_iter()
3198 .enumerate()
3199 .map(|(index, current)| current.saturating_sub(start.get(index).copied().unwrap_or_default()))
3200 .collect()
3201 }
3202
3203 fn scanner_replication_repair_work_snapshots(
3204 &self,
3205 values: &[ScannerSourceWorkValues],
3206 ) -> Vec<ScannerReplicationRepairSnapshot> {
3207 ScannerReplicationRepairKind::all()
3208 .iter()
3209 .filter_map(|kind| {
3210 values
3211 .get(kind.index())
3212 .map(|values| values.replication_repair_snapshot(*kind))
3213 })
3214 .collect()
3215 }
3216
3217 fn scanner_replication_repair_work_counter_snapshots(
3218 &self,
3219 counters: &[ScannerSourceWorkCounters],
3220 ) -> Vec<ScannerReplicationRepairSnapshot> {
3221 ScannerReplicationRepairKind::all()
3222 .iter()
3223 .filter_map(|kind| {
3224 counters
3225 .get(kind.index())
3226 .map(|counters| counters.values().replication_repair_snapshot(*kind))
3227 })
3228 .collect()
3229 }
3230
3231 fn store_scanner_source_work_values(&self, counters: &[ScannerSourceWorkCounters], values: &[ScannerSourceWorkValues]) {
3232 for source in ScannerWorkSource::all() {
3233 if let (Some(counter), Some(values)) = (counters.get(source.index()), values.get(source.index())) {
3234 counter.store(*values);
3235 }
3236 }
3237 }
3238
3239 fn store_scanner_replication_repair_work_values(
3240 &self,
3241 counters: &[ScannerSourceWorkCounters],
3242 values: &[ScannerSourceWorkValues],
3243 ) {
3244 for kind in ScannerReplicationRepairKind::all() {
3245 if let (Some(counter), Some(values)) = (counters.get(kind.index()), values.get(kind.index())) {
3246 counter.store(*values);
3247 }
3248 }
3249 }
3250
3251 pub fn record_scan_cycle_work(&self, work: ScanCycleWorkSnapshot) {
3252 self.last_scan_cycle_objects_scanned
3255 .store(work.objects_scanned, Ordering::Relaxed);
3256 self.last_scan_cycle_directories_scanned
3257 .store(work.directories_scanned, Ordering::Relaxed);
3258 self.last_scan_cycle_bucket_drive_scans
3259 .store(work.bucket_drive_scans, Ordering::Relaxed);
3260 self.last_scan_cycle_bucket_drive_failures
3261 .store(work.bucket_drive_failures, Ordering::Relaxed);
3262 self.last_scan_cycle_yield_events.store(work.yield_events, Ordering::Relaxed);
3263 self.last_scan_cycle_yield_duration_millis
3264 .store(work.yield_duration_millis, Ordering::Relaxed);
3265 self.last_scan_cycle_throttle_sleep_events
3266 .store(work.throttle_sleep_events, Ordering::Relaxed);
3267 self.last_scan_cycle_throttle_sleep_duration_millis
3268 .store(work.throttle_sleep_duration_millis, Ordering::Relaxed);
3269 self.last_scan_cycle_ilm_actions.store(work.ilm_actions, Ordering::Relaxed);
3270 self.last_scan_cycle_lifecycle_expiry_actions
3271 .store(work.lifecycle_expiry_actions, Ordering::Relaxed);
3272 self.last_scan_cycle_lifecycle_transition_actions
3273 .store(work.lifecycle_transition_actions, Ordering::Relaxed);
3274 self.last_scan_cycle_heal_objects.store(work.heal_objects, Ordering::Relaxed);
3275 self.last_scan_cycle_replication_checks
3276 .store(work.replication_checks, Ordering::Relaxed);
3277 self.last_scan_cycle_usage_saves.store(work.usage_saves, Ordering::Relaxed);
3278 }
3279
3280 fn record_scan_cycle_source_work(&self, work: &[ScannerSourceWorkValues]) {
3281 self.store_scanner_source_work_values(&self.last_scan_cycle_source_work, work);
3282 }
3283
3284 fn record_scan_cycle_replication_repair_work(&self, work: &[ScannerSourceWorkValues]) {
3285 self.store_scanner_replication_repair_work_values(&self.last_scan_cycle_replication_repair_work, work);
3286 }
3287
3288 pub async fn get_current_paths(&self) -> Vec<String> {
3290 self.current_path_snapshots()
3291 .await
3292 .into_iter()
3293 .map(|(disk, state)| format!("{disk}/{}", state.path))
3294 .collect()
3295 }
3296
3297 async fn current_path_snapshots(&self) -> Vec<(String, CurrentPathState)> {
3298 let paths = self.current_paths.read().await;
3299 let mut result = Vec::with_capacity(paths.len());
3300 for (disk, tracker) in paths.iter() {
3301 result.push((disk.clone(), tracker.get_state().await));
3302 }
3303 result.sort_by(|(left, _), (right, _)| left.cmp(right));
3304 result
3305 }
3306
3307 pub async fn active_drives(&self) -> usize {
3309 self.current_paths.read().await.len()
3310 }
3311
3312 pub async fn report(&self) -> ScannerMetricsReport {
3314 self.report_with_runtime_details().await.0
3315 }
3316
3317 pub async fn report_with_runtime_details(&self) -> (ScannerMetricsReport, ScannerRuntimeDetailsReport) {
3318 let mut m = ScannerMetricsReport::default();
3319 let runtime_details;
3320
3321 let has_cycle = {
3322 let cycle = self.cycle_info.read().await;
3323 let has_cycle = if let Some(cycle) = cycle.as_ref() {
3324 m.current_cycle = cycle.current;
3325 m.cycles_completed_at = cycle.cycle_completed.iter().copied().map(chrono_to_jiff_timestamp).collect();
3326 m.current_started = chrono_to_jiff_timestamp(cycle.started);
3327 true
3328 } else {
3329 false
3330 };
3331 m.current_cycle_active = self.current_scan_cycle_work_active.load(Ordering::Acquire);
3332 if m.current_cycle_active {
3333 let current_work = self.scan_cycle_work_since(self.current_scan_cycle_work_start());
3335 let current_source_work = self.scanner_source_work_since(&self.current_scan_cycle_source_work_start_values());
3336 let current_replication_repair_work =
3337 self.scanner_replication_repair_work_since(&self.current_scan_cycle_replication_repair_work_start_values());
3338 m.current_cycle_objects_scanned = current_work.objects_scanned;
3339 m.current_cycle_directories_scanned = current_work.directories_scanned;
3340 m.current_cycle_bucket_drive_scans = current_work.bucket_drive_scans;
3341 m.current_cycle_bucket_drive_failures = current_work.bucket_drive_failures;
3342 m.current_cycle_yield_events = current_work.yield_events;
3343 m.current_cycle_yield_duration_seconds = current_work.yield_duration_millis as f64 / 1000.0;
3344 m.current_cycle_throttle_sleep_events = current_work.throttle_sleep_events;
3345 m.current_cycle_throttle_sleep_duration_seconds = current_work.throttle_sleep_duration_millis as f64 / 1000.0;
3346 m.current_cycle_ilm_actions = current_work.ilm_actions;
3347 m.current_cycle_lifecycle_expiry_actions = current_work.lifecycle_expiry_actions;
3348 m.current_cycle_lifecycle_transition_actions = current_work.lifecycle_transition_actions;
3349 m.current_cycle_heal_objects = current_work.heal_objects;
3350 m.current_cycle_replication_checks = current_work.replication_checks;
3351 m.current_cycle_usage_saves = current_work.usage_saves;
3352 m.current_cycle_source_work = self.scanner_source_work_snapshots(¤t_source_work);
3353 m.current_cycle_replication_repair =
3354 self.scanner_replication_repair_work_snapshots(¤t_replication_repair_work);
3355 }
3356 runtime_details = self.scanner_runtime_details_report_for_active(m.current_cycle_active);
3357 has_cycle
3358 };
3359
3360 if !has_cycle && let Some(init_time) = crate::init_time::get_global_init_time().await {
3361 m.current_started = chrono_to_jiff_timestamp(init_time);
3362 }
3363
3364 m.collected_at = Timestamp::now();
3365 let current_path_snapshots = self.current_path_snapshots().await;
3366 m.active_scan_paths = current_path_snapshots.len();
3367 m.oldest_active_path_age_seconds = current_path_snapshots
3368 .iter()
3369 .map(|(_, state)| timestamp_elapsed_seconds_since(m.collected_at, state.updated_at))
3370 .max()
3371 .unwrap_or_default();
3372 m.active_paths = current_path_snapshots
3373 .into_iter()
3374 .map(|(disk, state)| format!("{disk}/{}", state.path))
3375 .collect();
3376 m.current_scan_mode = self.current_scan_mode().as_str().to_string();
3377 m.versions_scanned = self.lifetime_versions_scanned.load(Ordering::Relaxed);
3378 m.leader_lock_state = self.scanner_leader_lock_state.read().await.clone();
3379 m.leader_lock_held_by_this_process = self.scanner_leader_lock_held.load(Ordering::Relaxed);
3380 m.leader_lock_last_error = self.scanner_leader_lock_last_error.read().await.clone();
3381 m.leader_lock_last_update_unix_secs = self.scanner_leader_lock_last_update_unix_secs.load(Ordering::Relaxed);
3382 m.last_cycle_end_unix_secs = self.last_scan_cycle_end_unix_secs.load(Ordering::Relaxed);
3383 m.current_set_scan_concurrency_limit = self.scanner_set_scan_concurrency_limit.load(Ordering::Relaxed);
3384 m.current_set_scans_queued = self.scanner_set_scans_queued.load(Ordering::Relaxed);
3385 m.current_set_scans_active = self.scanner_set_scans_active.load(Ordering::Relaxed);
3386 let disk_bucket_scan_states = self.scanner_disk_bucket_scan_state_snapshots();
3387 let (disk_scan_concurrency_limit, disk_bucket_scans_queued, disk_bucket_scans_active) =
3388 disk_bucket_scan_states.iter().fold((0, 0, 0), |acc, state| {
3389 (acc.0 + state.concurrency_limit, acc.1 + state.queued, acc.2 + state.active)
3390 });
3391 m.current_disk_scan_concurrency_limit = disk_scan_concurrency_limit;
3392 m.current_disk_bucket_scans_queued = disk_bucket_scans_queued;
3393 m.current_disk_bucket_scans_active = disk_bucket_scans_active;
3394 let last_cycle_result = self.last_scan_cycle_result.load(Ordering::Relaxed);
3395 m.last_cycle_result = scan_cycle_result_label(last_cycle_result).to_string();
3396 m.last_cycle_result_code = last_cycle_result as u64;
3397 let partial_reason = ScanCyclePartialReason::from_code(self.last_scan_cycle_partial_reason.load(Ordering::Relaxed));
3398 m.last_cycle_partial_reason = partial_reason.as_str().to_string();
3399 m.last_cycle_partial_reason_code = partial_reason as u8 as u64;
3400 let partial_source = ScannerWorkSource::from_code(self.last_scan_cycle_partial_source.load(Ordering::Relaxed));
3401 m.last_cycle_partial_source = partial_source
3402 .map(ScannerWorkSource::as_str)
3403 .unwrap_or(SCAN_CYCLE_RESULT_UNKNOWN_LABEL)
3404 .to_string();
3405 m.last_cycle_partial_source_code = partial_source.map(|source| source.code().into()).unwrap_or_default();
3406 m.last_cycle_duration_seconds = self.last_scan_cycle_duration_millis.load(Ordering::Relaxed) as f64 / 1000.0;
3407 m.last_cycle_objects_scanned = self.last_scan_cycle_objects_scanned.load(Ordering::Relaxed);
3408 m.last_cycle_directories_scanned = self.last_scan_cycle_directories_scanned.load(Ordering::Relaxed);
3409 m.last_cycle_bucket_drive_scans = self.last_scan_cycle_bucket_drive_scans.load(Ordering::Relaxed);
3410 m.last_cycle_bucket_drive_failures = self.last_scan_cycle_bucket_drive_failures.load(Ordering::Relaxed);
3411 m.last_cycle_yield_events = self.last_scan_cycle_yield_events.load(Ordering::Relaxed);
3412 m.last_cycle_yield_duration_seconds = self.last_scan_cycle_yield_duration_millis.load(Ordering::Relaxed) as f64 / 1000.0;
3413 m.last_cycle_throttle_sleep_events = self.last_scan_cycle_throttle_sleep_events.load(Ordering::Relaxed);
3414 m.last_cycle_throttle_sleep_duration_seconds =
3415 self.last_scan_cycle_throttle_sleep_duration_millis.load(Ordering::Relaxed) as f64 / 1000.0;
3416 m.last_cycle_ilm_actions = self.last_scan_cycle_ilm_actions.load(Ordering::Relaxed);
3417 m.last_cycle_lifecycle_expiry_actions = self.last_scan_cycle_lifecycle_expiry_actions.load(Ordering::Relaxed);
3418 m.last_cycle_lifecycle_transition_actions = self.last_scan_cycle_lifecycle_transition_actions.load(Ordering::Relaxed);
3419 m.last_cycle_heal_objects = self.last_scan_cycle_heal_objects.load(Ordering::Relaxed);
3420 m.last_cycle_replication_checks = self.last_scan_cycle_replication_checks.load(Ordering::Relaxed);
3421 m.last_cycle_usage_saves = self.last_scan_cycle_usage_saves.load(Ordering::Relaxed);
3422 m.last_cycle_source_work = self.scanner_source_work_counter_snapshots(&self.last_scan_cycle_source_work);
3423 m.last_cycle_replication_repair =
3424 self.scanner_replication_repair_work_counter_snapshots(&self.last_scan_cycle_replication_repair_work);
3425 m.failed_cycles = self.failed_scan_cycles.load(Ordering::Relaxed);
3426 m.superseded_cycles = self.superseded_scan_cycles.load(Ordering::Relaxed);
3427 m.partial_cycles_unknown = self.partial_scan_cycles_unknown.load(Ordering::Relaxed);
3428 m.partial_cycles_runtime = self.partial_scan_cycles_runtime.load(Ordering::Relaxed);
3429 m.partial_cycles_objects = self.partial_scan_cycles_objects.load(Ordering::Relaxed);
3430 m.partial_cycles_directories = self.partial_scan_cycles_directories.load(Ordering::Relaxed);
3431 m.partial_cycles_by_source = ScannerWorkSource::all()
3432 .iter()
3433 .filter_map(|source| {
3434 self.partial_scan_cycles_by_source
3435 .get(source.index())
3436 .map(|cycles| ScannerSourceCycleSnapshot {
3437 source: source.as_str().to_string(),
3438 cycles: cycles.load(Ordering::Relaxed),
3439 })
3440 })
3441 .collect();
3442 m.lifecycle_expiry = ScannerLifecycleExpirySnapshot {
3443 current_queue_capacity: self.scanner_expiry_queue_capacity.load(Ordering::Relaxed),
3444 current_queued: self.scanner_expiry_queued.load(Ordering::Relaxed),
3445 current_active: self.scanner_expiry_active.load(Ordering::Relaxed),
3446 current_workers: self.scanner_expiry_workers.load(Ordering::Relaxed),
3447 queue_missed: self.scanner_expiry_queue_missed.load(Ordering::Relaxed),
3448 scanner_queued: self.scanner_expiry_queued_total.load(Ordering::Relaxed),
3449 scanner_missed: self.scanner_expiry_missed_total.load(Ordering::Relaxed),
3450 scanner_blocked: self.scanner_expiry_blocked_total.load(Ordering::Relaxed),
3451 scanner_not_enqueued: self.scanner_expiry_not_enqueued_total.load(Ordering::Relaxed),
3452 delete_failed: self.scanner_expiry_delete_failed_total.load(Ordering::Relaxed),
3453 };
3454 m.lifecycle_transition = ScannerLifecycleTransitionSnapshot {
3455 current_queue_capacity: self.scanner_transition_queue_capacity.load(Ordering::Relaxed),
3456 current_queued: self.scanner_transition_queued.load(Ordering::Relaxed),
3457 current_active: self.scanner_transition_active.load(Ordering::Relaxed),
3458 current_workers: self.scanner_transition_workers.load(Ordering::Relaxed),
3459 queue_full: self.scanner_transition_queue_full.load(Ordering::Relaxed),
3460 queue_send_timeout: self.scanner_transition_queue_send_timeout.load(Ordering::Relaxed),
3461 compensation_scheduled: self.scanner_transition_compensation_scheduled.load(Ordering::Relaxed),
3462 compensation_pending: self.scanner_transition_compensation_pending.load(Ordering::Relaxed),
3463 compensation_running: self.scanner_transition_compensation_running.load(Ordering::Relaxed),
3464 scanner_queued: self.scanner_transition_queued_total.load(Ordering::Relaxed),
3465 scanner_missed: self.scanner_transition_missed_total.load(Ordering::Relaxed),
3466 completed: self.scanner_transition_completed.load(Ordering::Relaxed),
3467 failed: self.scanner_transition_failed.load(Ordering::Relaxed),
3468 };
3469 let usage_save_result = ScannerUsageSaveResult::from_code(self.scanner_usage_last_save_result.load(Ordering::Relaxed));
3470 m.usage_freshness = ScannerUsageFreshnessSnapshot {
3471 dirty_pending_buckets: self.scanner_dirty_usage_pending_buckets.load(Ordering::Relaxed),
3472 last_dirty_mark_unix_secs: self.scanner_dirty_usage_last_mark_unix_secs.load(Ordering::Relaxed),
3473 last_dirty_clear_unix_secs: self.scanner_dirty_usage_last_clear_unix_secs.load(Ordering::Relaxed),
3474 last_cycle_dirty_buckets: self.scanner_dirty_usage_last_cycle_dirty_buckets.load(Ordering::Relaxed),
3475 last_cycle_cleared_dirty_buckets: self.scanner_dirty_usage_last_cycle_cleared_buckets.load(Ordering::Relaxed),
3476 last_usage_save_unix_secs: self.scanner_usage_last_save_unix_secs.load(Ordering::Relaxed),
3477 last_usage_save_result: usage_save_result.as_str().to_string(),
3478 last_usage_save_result_code: usage_save_result as u8 as u64,
3479 last_durable_success_unix_secs: self.scanner_usage_last_durable_success_unix_secs.load(Ordering::Relaxed),
3480 last_publication_unix_secs: self.scanner_usage_last_publication_unix_secs.load(Ordering::Relaxed),
3481 last_publication_state: match self.scanner_usage_last_publication_state.lock() {
3482 Ok(state) => state.clone(),
3483 Err(poisoned) => poisoned.into_inner().clone(),
3484 },
3485 last_publication_reason: match self.scanner_usage_last_publication_reason.lock() {
3486 Ok(reason) => reason.clone(),
3487 Err(poisoned) => poisoned.into_inner().clone(),
3488 },
3489 deferred_pending: self.scanner_usage_deferred_pending.load(Ordering::Acquire),
3490 deferred_total: self.scanner_usage_deferred_total.load(Ordering::Relaxed),
3491 last_deferred_unix_secs: self.scanner_usage_last_deferred_unix_secs.load(Ordering::Relaxed),
3492 last_deferred_reason: match self.scanner_usage_last_deferred_reason.lock() {
3493 Ok(reason) => reason.clone(),
3494 Err(poisoned) => poisoned.into_inner().clone(),
3495 },
3496 };
3497 m.throttle_idle_mode_enabled = self.scanner_throttle_idle_mode_enabled.load(Ordering::Relaxed);
3498 m.throttle_sleep_factor = self.scanner_throttle_sleep_factor_micros.load(Ordering::Relaxed) as f64 / 1_000_000.0;
3499 m.throttle_max_sleep_seconds = self.scanner_throttle_max_sleep_millis.load(Ordering::Relaxed) as f64 / 1000.0;
3500 m.yield_every_n_objects = self.scanner_yield_every_n_objects.load(Ordering::Relaxed);
3501 m.cycle_interval_seconds = self.scanner_cycle_interval_millis.load(Ordering::Relaxed) as f64 / 1000.0;
3502 m.cycle_max_duration_seconds = self.scanner_cycle_max_duration_millis.load(Ordering::Relaxed) as f64 / 1000.0;
3503 m.cycle_max_objects = self.scanner_cycle_max_objects.load(Ordering::Relaxed);
3504 m.cycle_max_directories = self.scanner_cycle_max_directories.load(Ordering::Relaxed);
3505 m.cycle_timeout_total = self.scanner_cycle_timeout_total.load(Ordering::Relaxed);
3506 m.cycle_recovery_required_total = self.scanner_cycle_recovery_required_total.load(Ordering::Relaxed);
3507 m.cycle_last_progress_age = self.scanner_cycle_last_progress_age_seconds.load(Ordering::Relaxed);
3508 m.leader_lease_without_progress = self.scanner_leader_lease_without_progress.load(Ordering::Relaxed);
3509 m.bitrot_cycle_enabled = self.scanner_bitrot_cycle_enabled.load(Ordering::Relaxed);
3510 m.bitrot_cycle_seconds = self.scanner_bitrot_cycle_millis.load(Ordering::Relaxed) as f64 / 1000.0;
3511 m.scan_checkpoint = match self.scanner_checkpoint.lock() {
3512 Ok(checkpoint) => checkpoint.clone(),
3513 Err(poisoned) => poisoned.into_inner().clone(),
3514 };
3515 m.scan_checkpoint_used = self.scanner_checkpoint_used.load(Ordering::Relaxed);
3516 m.scan_checkpoint_cleared = self.scanner_checkpoint_cleared.load(Ordering::Relaxed);
3517 m.scan_checkpoint_ignored = self.scanner_checkpoint_ignored.load(Ordering::Relaxed);
3518 m.scan_checkpoint_stale = self.scanner_checkpoint_stale.load(Ordering::Relaxed);
3519 m.source_work = self.scanner_source_work_counter_snapshots(&self.scanner_source_work);
3520 m.replication_repair = self.scanner_replication_repair_work_counter_snapshots(&self.scanner_replication_repair_work);
3521 m.partial_cycles = self.partial_scan_cycles.load(Ordering::Relaxed);
3522
3523 for i in 0..Metric::Last as usize {
3525 let count = self.operations[i].load(Ordering::Relaxed);
3526 if count > 0
3527 && let Some(metric) = Metric::from_index(i)
3528 {
3529 m.life_time_ops.insert(metric.as_str().to_string(), count);
3530 }
3531 }
3532
3533 for i in 0..Metric::LastRealtime as usize {
3535 let last_min = self.latency[i].total();
3536 if last_min.n > 0
3537 && let Some(metric) = Metric::from_index(i)
3538 {
3539 m.last_minute.actions.insert(
3540 metric.as_str().to_string(),
3541 ScannerTimedAction {
3542 count: last_min.n,
3543 acc_time: last_min.total,
3544 bytes: last_min.size,
3545 },
3546 );
3547 }
3548 }
3549
3550 for i in 0..IlmAction::ActionCount as usize {
3552 let count = self.actions[i].load(Ordering::Relaxed);
3553 if count > 0
3554 && let Some(action) = IlmAction::from_index(i)
3555 {
3556 m.life_time_ilm.insert(action.as_str().to_string(), count);
3557 }
3558 }
3559
3560 for i in 0..IlmAction::ActionCount as usize {
3562 let last_min = self.actions_latency[i].total();
3563 if last_min.n > 0
3564 && let Some(action) = IlmAction::from_index(i)
3565 {
3566 m.last_minute.ilm.insert(
3567 action.as_str().to_string(),
3568 ScannerTimedAction {
3569 count: last_min.n,
3570 acc_time: last_min.total,
3571 bytes: last_min.size,
3572 },
3573 );
3574 }
3575 }
3576
3577 m.pacing_pressure = scanner_pacing_pressure(&m);
3578 m.maintenance_control = scanner_maintenance_control(&m);
3579
3580 (m, runtime_details)
3581 }
3582}
3583
3584impl Default for Metrics {
3585 fn default() -> Self {
3586 Self::new()
3587 }
3588}
3589
3590pub type UpdateCurrentPathFn = Arc<dyn Fn(&str) -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + Sync>;
3595pub type CloseDiskFn = Arc<dyn Fn() -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + Sync>;
3596
3597pub async fn current_path_updater(disk: &str, initial: &str) -> (UpdateCurrentPathFn, CloseDiskFn) {
3600 let tracker = Arc::new(CurrentPathTracker::new(initial.to_string()));
3601 let disk_name = disk.to_string();
3602
3603 global_metrics()
3604 .current_paths
3605 .write()
3606 .await
3607 .insert(disk_name.clone(), Arc::clone(&tracker));
3608
3609 let update_fn: UpdateCurrentPathFn = {
3610 let tracker = Arc::clone(&tracker);
3611 Arc::new(move |path: &str| {
3612 let tracker = Arc::clone(&tracker);
3613 let path = path.to_string();
3614 Box::pin(async move { tracker.update_path(path).await })
3615 })
3616 };
3617
3618 let done_fn: CloseDiskFn = {
3619 let disk = disk_name;
3620 Arc::new(move || {
3621 let disk = disk.clone();
3622 Box::pin(async move {
3623 global_metrics().current_paths.write().await.remove(&disk);
3624 })
3625 })
3626 };
3627
3628 (update_fn, done_fn)
3629}
3630
3631pub struct CloseDiskGuard(Option<CloseDiskFn>);
3636
3637impl CloseDiskGuard {
3638 pub fn new(close_disk: CloseDiskFn) -> Self {
3639 Self(Some(close_disk))
3640 }
3641
3642 pub async fn close(&mut self) {
3643 let Some(close_disk) = self.0.clone() else {
3644 return;
3645 };
3646 close_disk().await;
3647 self.0 = None;
3648 }
3649}
3650
3651impl Drop for CloseDiskGuard {
3652 fn drop(&mut self) {
3653 if let Some(close_disk) = self.0.take()
3654 && let Ok(handle) = tokio::runtime::Handle::try_current()
3655 {
3656 handle.spawn(close_disk());
3657 }
3658 }
3660}
3661
3662#[cfg(test)]
3663mod tests {
3664 use super::*;
3665
3666 fn tier_cell(metrics: &Metrics, operation: TierRequestOperation, outcome: TierRequestOutcome) -> u64 {
3667 metrics
3668 .tier_request_counts()
3669 .into_iter()
3670 .find(|count| count.operation == operation && count.outcome == outcome)
3671 .map(|count| count.count)
3672 .expect("every operation/outcome cell must be reported")
3673 }
3674
3675 #[test]
3676 fn a_tier_request_lands_in_exactly_one_operation_outcome_cell() {
3677 let metrics = Metrics::default();
3678 metrics.record_tier_request(TierRequestOperation::Put, TierRequestOutcome::Success);
3679
3680 assert_eq!(tier_cell(&metrics, TierRequestOperation::Put, TierRequestOutcome::Success), 1);
3681 assert_eq!(
3682 metrics
3683 .tier_request_counts()
3684 .into_iter()
3685 .map(|count| count.count)
3686 .sum::<u64>(),
3687 1,
3688 "one request must not increment a second cell"
3689 );
3690 }
3691
3692 #[test]
3693 fn every_operation_outcome_pair_has_its_own_cell() {
3694 let metrics = Metrics::default();
3695 for operation in TierRequestOperation::ALL {
3696 for outcome in TierRequestOutcome::ALL {
3697 metrics.record_tier_request(operation, outcome);
3698 }
3699 }
3700
3701 let counts = metrics.tier_request_counts();
3702 assert_eq!(
3703 counts.len(),
3704 TierRequestOperation::ALL.len() * TierRequestOutcome::ALL.len(),
3705 "the series set is fixed by the two label enums"
3706 );
3707 assert!(
3708 counts.iter().all(|count| count.count == 1),
3709 "two different pairs must not share a counter slot"
3710 );
3711 }
3712
3713 #[test]
3714 fn a_backend_error_is_classified_without_reading_its_message() {
3715 assert_eq!(
3716 TierRequestOutcome::from_error(&std::io::Error::new(std::io::ErrorKind::TimedOut, "s3.example.com:9000 timed out")),
3717 TierRequestOutcome::Timeout
3718 );
3719 assert_eq!(
3720 TierRequestOutcome::from_error(&std::io::Error::new(std::io::ErrorKind::Interrupted, "cancelled")),
3721 TierRequestOutcome::Cancelled
3722 );
3723 assert_eq!(
3724 TierRequestOutcome::from_error(&std::io::Error::other("AccessDenied")),
3725 TierRequestOutcome::BackendError
3726 );
3727 }
3728
3729 #[test]
3730 fn tier_request_labels_are_stable_identifiers() {
3731 assert_eq!(TierRequestOperation::Put.as_label(), "put");
3732 assert_eq!(TierRequestOperation::InUse.as_label(), "in_use");
3733 assert_eq!(TierRequestOutcome::BackendError.as_label(), "backend_error");
3734 }
3735
3736 #[test]
3737 fn scanner_metrics_report_timestamps_serialize_as_rfc3339_utc() {
3738 let report = ScannerMetricsReport {
3739 collected_at: Timestamp::constant(1_700_000_000, 123_456_000),
3740 current_started: Timestamp::constant(1_699_999_940, 0),
3741 cycles_completed_at: vec![Timestamp::constant(1_700_000_060, 987_654_000)],
3742 ..Default::default()
3743 };
3744
3745 let value = serde_json::to_value(&report).expect("scanner metrics report should serialize");
3746
3747 assert_eq!(value["collected_at"].as_str(), Some("2023-11-14T22:13:20.123456Z"));
3748 assert_eq!(value["current_started"].as_str(), Some("2023-11-14T22:12:20Z"));
3749 assert_eq!(value["cycles_completed_at"][0].as_str(), Some("2023-11-14T22:14:20.987654Z"));
3750 }
3751
3752 #[tokio::test]
3753 async fn close_disk_guard_runs_cleanup_when_an_early_return_drops_it() {
3754 let (closed_tx, closed_rx) = tokio::sync::oneshot::channel();
3755 let closed_tx = Arc::new(std::sync::Mutex::new(Some(closed_tx)));
3756 let close_disk: CloseDiskFn = {
3757 let closed_tx = Arc::clone(&closed_tx);
3758 Arc::new(move || {
3759 let closed_tx = closed_tx.lock().expect("close callback lock").take();
3760 Box::pin(async move {
3761 if let Some(closed_tx) = closed_tx {
3762 let _ = closed_tx.send(());
3763 }
3764 })
3765 })
3766 };
3767
3768 let guard = CloseDiskGuard::new(close_disk);
3769 drop(guard);
3770
3771 tokio::time::timeout(std::time::Duration::from_secs(1), closed_rx)
3772 .await
3773 .expect("drop cleanup should run")
3774 .expect("drop cleanup should signal");
3775 }
3776
3777 #[tokio::test]
3778 async fn close_disk_guard_runs_explicit_cleanup_once() {
3779 let close_count = Arc::new(std::sync::atomic::AtomicUsize::new(0));
3780 let close_disk: CloseDiskFn = {
3781 let close_count = Arc::clone(&close_count);
3782 Arc::new(move || {
3783 close_count.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
3784 Box::pin(std::future::ready(()))
3785 })
3786 };
3787
3788 let mut guard = CloseDiskGuard::new(close_disk);
3789 guard.close().await;
3790 drop(guard);
3791 tokio::time::sleep(std::time::Duration::from_millis(10)).await;
3792
3793 assert_eq!(close_count.load(std::sync::atomic::Ordering::Relaxed), 1);
3794 }
3795
3796 #[tokio::test]
3797 async fn current_path_updater_registers_before_return() {
3798 let disk = format!("test-disk-{}", uuid::Uuid::new_v4());
3799 let (_update_path, close_disk) = current_path_updater(&disk, "bucket-a").await;
3800
3801 assert!(global_metrics().current_paths.read().await.contains_key(&disk));
3802
3803 close_disk().await;
3804 assert!(!global_metrics().current_paths.read().await.contains_key(&disk));
3805 }
3806
3807 #[tokio::test]
3808 async fn report_counts_active_scan_paths() {
3809 let metrics = Metrics::new();
3810 let updated_at = Timestamp::now() - jiff::SignedDuration::from_secs(12);
3811 metrics.current_paths.write().await.insert(
3812 "disk-a".to_string(),
3813 Arc::new(CurrentPathTracker::new_at("bucket-a".to_string(), updated_at)),
3814 );
3815
3816 let report = metrics.report().await;
3817
3818 assert_eq!(report.active_scan_paths, 1);
3819 assert!(
3820 report.oldest_active_path_age_seconds >= 12,
3821 "expected active path age to reflect last path update"
3822 );
3823 assert_eq!(report.ongoing_buckets, 0);
3824 assert_eq!(report.active_paths, vec!["disk-a/bucket-a".to_string()]);
3825 }
3826
3827 #[tokio::test]
3828 async fn report_active_path_age_uses_last_path_update() {
3829 let metrics = Metrics::new();
3830 let tracker = Arc::new(CurrentPathTracker::new_at(
3831 "bucket-a".to_string(),
3832 Timestamp::now() - jiff::SignedDuration::from_secs(60 * 60),
3833 ));
3834 metrics
3835 .current_paths
3836 .write()
3837 .await
3838 .insert("disk-a".to_string(), Arc::clone(&tracker));
3839
3840 tracker.update_path("bucket-a/prefix".to_string()).await;
3841
3842 let report = metrics.report().await;
3843
3844 assert_eq!(report.active_paths, vec!["disk-a/bucket-a/prefix".to_string()]);
3845 assert!(
3846 report.oldest_active_path_age_seconds < 5,
3847 "expected active path age to reset after path update"
3848 );
3849 }
3850
3851 #[tokio::test]
3852 async fn report_includes_scanner_queue_state() {
3853 let metrics = Metrics::new();
3854 metrics.record_scanner_set_scan_state(Some(3), Some(5), Some(2));
3855 metrics.record_scanner_disk_bucket_scan_state("0", "0", Some(2), Some(7), Some(1));
3856 metrics.record_scanner_disk_bucket_scan_state("0", "1", Some(4), Some(11), Some(3));
3857
3858 let report = metrics.report().await;
3859
3860 assert_eq!(report.current_set_scan_concurrency_limit, 3);
3861 assert_eq!(report.current_set_scans_queued, 5);
3862 assert_eq!(report.current_set_scans_active, 2);
3863 assert_eq!(report.current_disk_scan_concurrency_limit, 6);
3864 assert_eq!(report.current_disk_bucket_scans_queued, 18);
3865 assert_eq!(report.current_disk_bucket_scans_active, 4);
3866
3867 metrics.reset_scanner_set_scan_state();
3868 let report = metrics.report().await;
3869
3870 assert_eq!(report.current_set_scan_concurrency_limit, 0);
3871 assert_eq!(report.current_set_scans_queued, 0);
3872 assert_eq!(report.current_set_scans_active, 0);
3873 assert_eq!(report.current_disk_scan_concurrency_limit, 0);
3874 assert_eq!(report.current_disk_bucket_scans_queued, 0);
3875 assert_eq!(report.current_disk_bucket_scans_active, 0);
3876 }
3877
3878 #[tokio::test]
3879 async fn report_counts_scanned_versions_independent_of_lifecycle() {
3880 let metrics = Metrics::new();
3881
3882 metrics.record_scanner_versions_scanned(3);
3885 metrics.record_scanner_versions_scanned(5);
3886
3887 let report = metrics.report().await;
3888
3889 assert_eq!(report.versions_scanned, 8);
3890 let lifecycle_checked = report
3893 .source_work
3894 .iter()
3895 .find(|s| s.source == "lifecycle")
3896 .map(|s| s.checked)
3897 .unwrap_or_default();
3898 assert_eq!(lifecycle_checked, 0);
3899 }
3900
3901 #[tokio::test]
3902 async fn report_tracks_lifecycle_checked_versions_separately() {
3903 let metrics = Metrics::new();
3904
3905 metrics.record_scanner_versions_scanned(10);
3908 metrics.record_scanner_source_checked(ScannerWorkSource::Lifecycle, 4);
3909
3910 let report = metrics.report().await;
3911
3912 assert_eq!(report.versions_scanned, 10, "total scanned versions independent of ILM");
3913 let lifecycle_checked = report
3914 .source_work
3915 .iter()
3916 .find(|s| s.source == "lifecycle")
3917 .map(|s| s.checked)
3918 .unwrap_or_default();
3919 assert_eq!(lifecycle_checked, 4, "ILM-checked versions tracked on the Lifecycle source");
3920 }
3921
3922 #[tokio::test]
3923 async fn report_derives_scanner_pacing_pressure() {
3924 let metrics = Metrics::new();
3925 metrics.record_scanner_set_scan_state(Some(2), Some(3), Some(1));
3926 metrics.record_scanner_disk_bucket_scan_state("0", "0", Some(1), Some(4), Some(1));
3927 metrics.record_scan_cycle_partial_with_source(
3928 Duration::from_secs(10),
3929 ScanCyclePartialReason::Runtime,
3930 Some(ScannerWorkSource::Usage),
3931 );
3932 metrics.record_scan_cycle_work(ScanCycleWorkSnapshot {
3933 yield_events: 2,
3934 yield_duration_millis: 500,
3935 throttle_sleep_events: 4,
3936 throttle_sleep_duration_millis: 2500,
3937 ..Default::default()
3938 });
3939
3940 let report = metrics.report().await;
3941
3942 assert_eq!(report.pacing_pressure.primary_pressure, "queued_scans");
3943 assert_eq!(report.pacing_pressure.current_queued_scans, 7);
3944 assert_eq!(report.pacing_pressure.current_active_scans, 2);
3945 assert!(report.pacing_pressure.last_cycle_budget_limited);
3946 assert!(report.pacing_pressure.last_cycle_pause_observed);
3947 assert_eq!(report.pacing_pressure.last_cycle_throttle_sleep_ratio, 0.25);
3948 assert_eq!(report.pacing_pressure.last_cycle_yield_ratio, 0.05);
3949 assert_eq!(report.pacing_pressure.last_cycle_total_pause_ratio, 0.3);
3950 }
3951
3952 #[tokio::test]
3953 async fn report_includes_scanner_checkpoint_status() {
3954 let metrics = Metrics::new();
3955 metrics.record_scanner_checkpoint_set(1, "bucket/child-a", "directories");
3956 metrics.record_scanner_checkpoint_used();
3957
3958 let report = metrics.report().await;
3959
3960 let checkpoint = report.scan_checkpoint.as_ref().expect("scanner checkpoint should be visible");
3961 assert_eq!(checkpoint.version, 1);
3962 assert_eq!(checkpoint.resume_after, "bucket/child-a");
3963 assert_eq!(checkpoint.reason, "directories");
3964 assert_eq!(checkpoint.last_event, "used");
3965 assert_eq!(report.scan_checkpoint_used, 1);
3966 assert_eq!(report.scan_checkpoint_cleared, 0);
3967
3968 metrics.record_scanner_checkpoint_stale();
3969 metrics.record_scanner_checkpoint_cleared();
3970 let report = metrics.report().await;
3971
3972 assert!(report.scan_checkpoint.is_none());
3973 assert_eq!(report.scan_checkpoint_used, 1);
3974 assert_eq!(report.scan_checkpoint_stale, 1);
3975 assert_eq!(report.scan_checkpoint_cleared, 1);
3976 }
3977
3978 #[tokio::test]
3979 async fn report_includes_scanner_source_work() {
3980 let metrics = Metrics::new();
3981 metrics.record_scanner_source_work(
3982 ScannerWorkSource::Usage,
3983 ScannerSourceWorkUpdate {
3984 checked: 3,
3985 executed: 1,
3986 ..Default::default()
3987 },
3988 );
3989 metrics.record_scanner_source_work(
3990 ScannerWorkSource::Lifecycle,
3991 ScannerSourceWorkUpdate {
3992 queued: 2,
3993 executed: 1,
3994 failed: 1,
3995 missed: 3,
3996 ..Default::default()
3997 },
3998 );
3999 metrics.record_scanner_ilm_enqueue_result(4, true);
4000 metrics.record_scanner_ilm_enqueue_result(5, false);
4001
4002 let report = metrics.report().await;
4003
4004 let usage = report
4005 .source_work
4006 .iter()
4007 .find(|work| work.source == ScannerWorkSource::Usage.as_str())
4008 .expect("usage source work should be visible");
4009 assert_eq!(usage.checked, 3);
4010 assert_eq!(usage.executed, 1);
4011
4012 let lifecycle = report
4013 .source_work
4014 .iter()
4015 .find(|work| work.source == ScannerWorkSource::Lifecycle.as_str())
4016 .expect("lifecycle source work should be visible");
4017 assert_eq!(lifecycle.queued, 6);
4018 assert_eq!(lifecycle.executed, 1);
4019 assert_eq!(lifecycle.failed, 1);
4020 assert_eq!(lifecycle.missed, 8);
4021 }
4022
4023 #[tokio::test]
4024 async fn report_splits_scanner_lifecycle_actions_by_expiry_and_transition() {
4025 let metrics = Metrics::new();
4026 let start = metrics.start_scan_cycle_work();
4027
4028 metrics.record_scanner_lifecycle_action(IlmAction::DeleteAction, 2);
4029 metrics.record_scanner_lifecycle_action(IlmAction::DeleteRestoredVersionAction, 3);
4030 metrics.record_scanner_lifecycle_action(IlmAction::TransitionAction, 5);
4031 metrics.record_scanner_lifecycle_action(IlmAction::TransitionVersionAction, 7);
4032
4033 let report = metrics.report().await;
4034
4035 assert_eq!(report.current_cycle_ilm_actions, 17);
4036 assert_eq!(report.current_cycle_lifecycle_expiry_actions, 5);
4037 assert_eq!(report.current_cycle_lifecycle_transition_actions, 12);
4038
4039 metrics.finish_scan_cycle_work(start);
4040 let report = metrics.report().await;
4041
4042 assert_eq!(report.last_cycle_ilm_actions, 17);
4043 assert_eq!(report.last_cycle_lifecycle_expiry_actions, 5);
4044 assert_eq!(report.last_cycle_lifecycle_transition_actions, 12);
4045 }
4046
4047 #[tokio::test]
4048 async fn report_includes_scanner_lifecycle_transition_status() {
4049 let metrics = Metrics::new();
4050 metrics.record_scanner_lifecycle_transition_state(ScannerLifecycleTransitionStateUpdate {
4051 queue_capacity: 16,
4052 queued: 5,
4053 active: 2,
4054 workers: 4,
4055 queue_full: 3,
4056 queue_send_timeout: 1,
4057 compensation_scheduled: 2,
4058 compensation_pending: 3,
4059 compensation_running: 1,
4060 });
4061 metrics.record_scanner_transition_enqueue_result(6, true);
4062 metrics.record_scanner_transition_enqueue_result(2, false);
4063 metrics.record_scanner_transition_completed(4);
4064 metrics.record_scanner_transition_failed(1);
4065
4066 let report = metrics.report().await;
4067
4068 assert_eq!(report.lifecycle_transition.current_queue_capacity, 16);
4069 assert_eq!(report.lifecycle_transition.current_queued, 5);
4070 assert_eq!(report.lifecycle_transition.current_active, 2);
4071 assert_eq!(report.lifecycle_transition.current_workers, 4);
4072 assert_eq!(report.lifecycle_transition.queue_full, 3);
4073 assert_eq!(report.lifecycle_transition.queue_send_timeout, 1);
4074 assert_eq!(report.lifecycle_transition.compensation_scheduled, 2);
4075 assert_eq!(report.lifecycle_transition.compensation_pending, 3);
4076 assert_eq!(report.lifecycle_transition.compensation_running, 1);
4077 assert_eq!(report.lifecycle_transition.scanner_queued, 6);
4078 assert_eq!(report.lifecycle_transition.scanner_missed, 2);
4079 assert_eq!(report.lifecycle_transition.completed, 4);
4080 assert_eq!(report.lifecycle_transition.failed, 1);
4081 }
4082
4083 #[tokio::test]
4084 async fn report_includes_scanner_lifecycle_expiry_status() {
4085 let metrics = Metrics::new();
4086 metrics.record_scanner_lifecycle_expiry_state(ScannerLifecycleExpiryStateUpdate {
4087 queue_capacity: 16,
4088 queued: 5,
4089 active: 2,
4090 workers: 4,
4091 queue_missed: 3,
4092 });
4093 metrics.record_scanner_expiry_enqueue_result(6, true);
4094 metrics.record_scanner_expiry_enqueue_result(2, false);
4095 metrics.record_scanner_expiry_blocked(4);
4096 metrics.record_scanner_expiry_delete_failed(1);
4097
4098 let report = metrics.report().await;
4099
4100 assert_eq!(report.lifecycle_expiry.current_queue_capacity, 16);
4101 assert_eq!(report.lifecycle_expiry.current_queued, 5);
4102 assert_eq!(report.lifecycle_expiry.current_active, 2);
4103 assert_eq!(report.lifecycle_expiry.current_workers, 4);
4104 assert_eq!(report.lifecycle_expiry.queue_missed, 3);
4105 assert_eq!(report.lifecycle_expiry.scanner_queued, 6);
4106 assert_eq!(report.lifecycle_expiry.scanner_missed, 2);
4107 assert_eq!(report.lifecycle_expiry.scanner_blocked, 4);
4108 assert_eq!(report.lifecycle_expiry.scanner_not_enqueued, 2);
4109 assert_eq!(report.lifecycle_expiry.delete_failed, 1);
4110 }
4111
4112 #[tokio::test]
4113 async fn report_derives_scanner_maintenance_control_status() {
4114 let metrics = Metrics::new();
4115 let start = metrics.start_scan_cycle_work();
4116 metrics.record_scanner_source_checked(ScannerWorkSource::Usage, 3);
4117 metrics.record_scanner_source_missed(ScannerWorkSource::Lifecycle, 2);
4118 metrics.record_scanner_source_queued(ScannerWorkSource::BucketReplication, 1);
4119 metrics.record_scan_cycle_partial_with_source(
4120 Duration::from_secs(5),
4121 ScanCyclePartialReason::Objects,
4122 Some(ScannerWorkSource::Usage),
4123 );
4124 metrics.record_scanner_lifecycle_transition_state(ScannerLifecycleTransitionStateUpdate {
4125 queue_capacity: 4,
4126 queued: 2,
4127 queue_full: 1,
4128 ..Default::default()
4129 });
4130
4131 let report = metrics.report().await;
4132
4133 assert_eq!(report.maintenance_control.primary_control, "blocked_source");
4134 let usage = report
4135 .maintenance_control
4136 .sources
4137 .iter()
4138 .find(|source| source.source == ScannerWorkSource::Usage.as_str())
4139 .expect("usage source control should be visible");
4140 assert_eq!(usage.state, "deferred");
4141 assert_eq!(usage.reason, "partial_cycle");
4142 assert_eq!(usage.current_checked, 3);
4143 assert_eq!(usage.partial_cycles, 1);
4144
4145 let lifecycle = report
4146 .maintenance_control
4147 .sources
4148 .iter()
4149 .find(|source| source.source == ScannerWorkSource::Lifecycle.as_str())
4150 .expect("lifecycle source control should be visible");
4151 assert_eq!(lifecycle.state, "blocked");
4152 assert_eq!(lifecycle.reason, "missed_work");
4153 assert_eq!(lifecycle.current_missed, 2);
4154 assert_eq!(lifecycle.backlog, 2);
4155
4156 let bucket_replication = report
4157 .maintenance_control
4158 .sources
4159 .iter()
4160 .find(|source| source.source == ScannerWorkSource::BucketReplication.as_str())
4161 .expect("bucket replication source control should be visible");
4162 assert_eq!(bucket_replication.state, "active");
4163 assert_eq!(bucket_replication.reason, "queued_work");
4164 assert_eq!(bucket_replication.current_queued, 1);
4165
4166 metrics.finish_scan_cycle_work(start);
4167 }
4168
4169 #[test]
4170 fn unresolved_heal_work_only_reflects_the_active_cycle() {
4171 let metrics = Metrics::new();
4172 assert!(!metrics.current_scan_cycle_has_unresolved_heal_work());
4173
4174 let start = metrics.start_scan_cycle_work();
4175 metrics.record_scanner_source_missed(ScannerWorkSource::Heal, 1);
4176 assert!(metrics.current_scan_cycle_has_unresolved_heal_work());
4177
4178 metrics.finish_scan_cycle_work(start);
4179 assert!(!metrics.current_scan_cycle_has_unresolved_heal_work());
4180
4181 let start = metrics.start_scan_cycle_work();
4182 metrics.record_scanner_source_queued(ScannerWorkSource::Heal, 1);
4183 assert!(metrics.current_scan_cycle_has_unresolved_heal_work());
4184 metrics.finish_scan_cycle_work(start);
4185
4186 let start = metrics.start_scan_cycle_work();
4187 metrics.record_scanner_source_work(
4188 ScannerWorkSource::Bitrot,
4189 ScannerSourceWorkUpdate {
4190 skipped: 1,
4191 ..Default::default()
4192 },
4193 );
4194 assert!(metrics.current_scan_cycle_has_unresolved_heal_work());
4195 metrics.finish_scan_cycle_work(start);
4196
4197 let start = metrics.start_scan_cycle_work();
4198 metrics.record_scanner_source_failed(ScannerWorkSource::Bitrot, 1);
4199 assert!(metrics.current_scan_cycle_has_unresolved_heal_work());
4200 metrics.finish_scan_cycle_work(start);
4201 }
4202
4203 #[tokio::test]
4204 async fn report_marks_transition_failures_as_blocked_lifecycle_control() {
4205 let metrics = Metrics::new();
4206 let _start = metrics.start_scan_cycle_work();
4207
4208 metrics.record_scanner_transition_failed(2);
4209 let report = metrics.report().await;
4210
4211 let lifecycle_work = report
4212 .current_cycle_source_work
4213 .iter()
4214 .find(|work| work.source == ScannerWorkSource::Lifecycle.as_str())
4215 .expect("current lifecycle source work should be visible");
4216 assert_eq!(lifecycle_work.failed, 2);
4217
4218 let lifecycle = report
4219 .maintenance_control
4220 .sources
4221 .iter()
4222 .find(|source| source.source == ScannerWorkSource::Lifecycle.as_str())
4223 .expect("lifecycle source control should be visible");
4224 assert_eq!(report.maintenance_control.primary_control, "blocked_source");
4225 assert_eq!(lifecycle.state, "blocked");
4226 assert_eq!(lifecycle.reason, "transition_failed");
4227 assert_eq!(lifecycle.backlog, 2);
4228 }
4229
4230 #[tokio::test]
4231 async fn report_marks_off_cycle_transition_failures_as_blocked_lifecycle_control() {
4232 let metrics = Metrics::new();
4233 let start = metrics.start_scan_cycle_work();
4234 metrics.finish_scan_cycle_work(start);
4235
4236 metrics.record_scanner_transition_failed(1);
4237 let report = metrics.report().await;
4238
4239 assert!(report.current_cycle_source_work.is_empty());
4240 let lifecycle_work = report
4241 .last_cycle_source_work
4242 .iter()
4243 .find(|work| work.source == ScannerWorkSource::Lifecycle.as_str())
4244 .expect("last-cycle lifecycle source work should be visible");
4245 assert_eq!(lifecycle_work.failed, 1);
4246
4247 let lifecycle = report
4248 .maintenance_control
4249 .sources
4250 .iter()
4251 .find(|source| source.source == ScannerWorkSource::Lifecycle.as_str())
4252 .expect("lifecycle source control should be visible");
4253 assert_eq!(report.maintenance_control.primary_control, "blocked_source");
4254 assert_eq!(lifecycle.state, "blocked");
4255 assert_eq!(lifecycle.reason, "transition_failed");
4256 assert_eq!(lifecycle.backlog, 1);
4257 }
4258
4259 #[tokio::test]
4260 async fn report_marks_expiry_queue_backlog_as_lifecycle_control() {
4261 let metrics = Metrics::new();
4262 metrics.record_scanner_lifecycle_expiry_state(ScannerLifecycleExpiryStateUpdate {
4263 queued: 2,
4264 active: 1,
4265 ..Default::default()
4266 });
4267
4268 let report = metrics.report().await;
4269
4270 let lifecycle = report
4271 .maintenance_control
4272 .sources
4273 .iter()
4274 .find(|source| source.source == ScannerWorkSource::Lifecycle.as_str())
4275 .expect("lifecycle source control should be visible");
4276 assert_eq!(report.maintenance_control.primary_control, "active_source");
4277 assert_eq!(lifecycle.state, "active");
4278 assert_eq!(lifecycle.reason, "expiry_queue_backlog");
4279 assert_eq!(lifecycle.backlog, 3);
4280 }
4281
4282 #[tokio::test]
4283 async fn report_marks_running_transition_compensation_as_lifecycle_backlog() {
4284 let metrics = Metrics::new();
4285 metrics.record_scanner_lifecycle_transition_state(ScannerLifecycleTransitionStateUpdate {
4286 compensation_running: 1,
4287 ..Default::default()
4288 });
4289
4290 let report = metrics.report().await;
4291
4292 let lifecycle = report
4293 .maintenance_control
4294 .sources
4295 .iter()
4296 .find(|source| source.source == ScannerWorkSource::Lifecycle.as_str())
4297 .expect("lifecycle source control should be visible");
4298 assert_eq!(report.maintenance_control.primary_control, "active_source");
4299 assert_eq!(lifecycle.state, "active");
4300 assert_eq!(lifecycle.reason, "transition_compensation_backlog");
4301 assert_eq!(lifecycle.backlog, 1);
4302 }
4303
4304 #[tokio::test]
4305 async fn report_marks_pending_transition_compensation_as_lifecycle_backlog() {
4306 let metrics = Metrics::new();
4307 metrics.record_scanner_lifecycle_transition_state(ScannerLifecycleTransitionStateUpdate {
4308 compensation_pending: 2,
4309 ..Default::default()
4310 });
4311
4312 let report = metrics.report().await;
4313
4314 let lifecycle = report
4315 .maintenance_control
4316 .sources
4317 .iter()
4318 .find(|source| source.source == ScannerWorkSource::Lifecycle.as_str())
4319 .expect("lifecycle source control should be visible");
4320 assert_eq!(report.maintenance_control.primary_control, "active_source");
4321 assert_eq!(lifecycle.state, "active");
4322 assert_eq!(lifecycle.reason, "transition_compensation_backlog");
4323 assert_eq!(lifecycle.backlog, 2);
4324 }
4325
4326 #[tokio::test]
4327 async fn report_uses_last_cycle_source_work_for_maintenance_control_between_cycles() {
4328 let metrics = Metrics::new();
4329 let start = metrics.start_scan_cycle_work();
4330 metrics.record_scanner_source_missed(ScannerWorkSource::Lifecycle, 2);
4331
4332 metrics.finish_scan_cycle_work(start);
4333 let report = metrics.report().await;
4334
4335 assert!(report.current_cycle_source_work.is_empty());
4336 let lifecycle = report
4337 .maintenance_control
4338 .sources
4339 .iter()
4340 .find(|source| source.source == ScannerWorkSource::Lifecycle.as_str())
4341 .expect("lifecycle source control should be visible");
4342 assert_eq!(report.maintenance_control.primary_control, "blocked_source");
4343 assert_eq!(lifecycle.state, "blocked");
4344 assert_eq!(lifecycle.reason, "missed_work");
4345 assert_eq!(lifecycle.current_missed, 2);
4346 assert_eq!(lifecycle.backlog, 2);
4347 }
4348
4349 #[tokio::test]
4350 async fn scanner_metrics_update_source_work() {
4351 let metrics = Metrics::new();
4352 metrics.record_source_work_for_metric(Metric::ScanObject, 3);
4353 metrics.record_source_work_for_metric(Metric::ScanFolder, 2);
4354 metrics.record_source_work_for_metric(Metric::SaveUsage, 1);
4355 metrics.record_source_work_for_metric(Metric::CheckReplication, 4);
4356 metrics.record_source_work_for_metric(Metric::HealCheck, 5);
4357 metrics.record_source_work_for_metric(Metric::HealAbandonedObject, 6);
4358
4359 let report = metrics.report().await;
4360
4361 let usage = report
4362 .source_work
4363 .iter()
4364 .find(|work| work.source == ScannerWorkSource::Usage.as_str())
4365 .expect("usage source work should be visible");
4366 assert_eq!(usage.checked, 5);
4367 assert_eq!(usage.executed, 1);
4368
4369 let bucket_replication = report
4370 .source_work
4371 .iter()
4372 .find(|work| work.source == ScannerWorkSource::BucketReplication.as_str())
4373 .expect("bucket replication source work should be visible");
4374 assert_eq!(bucket_replication.checked, 4);
4375
4376 let heal = report
4377 .source_work
4378 .iter()
4379 .find(|work| work.source == ScannerWorkSource::Heal.as_str())
4380 .expect("heal source work should be visible");
4381 assert_eq!(heal.checked, 5);
4382 assert_eq!(heal.executed, 0);
4383 }
4384
4385 #[tokio::test]
4386 async fn report_includes_scanner_replication_repair_work() {
4387 let metrics = Metrics::new();
4388
4389 metrics.record_scanner_replication_repair_work(
4390 ScannerReplicationRepairKind::BucketObject,
4391 ScannerSourceWorkUpdate::queued(2),
4392 );
4393 metrics.record_scanner_replication_repair_work(
4394 ScannerReplicationRepairKind::BucketDeleteMarker,
4395 ScannerSourceWorkUpdate::missed(1),
4396 );
4397 metrics.record_scanner_replication_repair_work(
4398 ScannerReplicationRepairKind::BucketVersionPurge,
4399 ScannerSourceWorkUpdate {
4400 skipped: 3,
4401 ..Default::default()
4402 },
4403 );
4404 metrics.record_scanner_replication_repair_work(
4405 ScannerReplicationRepairKind::BucketExistingObject,
4406 ScannerSourceWorkUpdate::queued(4),
4407 );
4408 metrics.record_scanner_replication_repair_work(
4409 ScannerReplicationRepairKind::SitePassiveRequeue,
4410 ScannerSourceWorkUpdate::missed(5),
4411 );
4412 metrics.record_scanner_replication_repair_work(
4413 ScannerReplicationRepairKind::SiteActiveResync,
4414 ScannerSourceWorkUpdate::queued(6),
4415 );
4416
4417 let report = metrics.report().await;
4418 let object = report
4419 .replication_repair
4420 .iter()
4421 .find(|repair| repair.source == "bucket_replication" && repair.kind == "object")
4422 .expect("bucket object repair work should be visible");
4423 assert_eq!(object.scanner_role, "repair_admission");
4424 assert_eq!(object.execution_owner, "bucket_replication_queue");
4425 assert_eq!(object.queued, 2);
4426
4427 let delete_marker = report
4428 .replication_repair
4429 .iter()
4430 .find(|repair| repair.source == "bucket_replication" && repair.kind == "delete_marker")
4431 .expect("bucket delete-marker repair work should be visible");
4432 assert_eq!(delete_marker.scanner_role, "repair_admission");
4433 assert_eq!(delete_marker.execution_owner, "bucket_replication_queue");
4434 assert_eq!(delete_marker.missed, 1);
4435
4436 let version_purge = report
4437 .replication_repair
4438 .iter()
4439 .find(|repair| repair.source == "bucket_replication" && repair.kind == "version_purge")
4440 .expect("bucket version purge repair work should be visible");
4441 assert_eq!(version_purge.scanner_role, "repair_admission");
4442 assert_eq!(version_purge.execution_owner, "bucket_replication_queue");
4443 assert_eq!(version_purge.skipped, 3);
4444
4445 let existing_object = report
4446 .replication_repair
4447 .iter()
4448 .find(|repair| repair.source == "bucket_replication" && repair.kind == "existing_object")
4449 .expect("bucket existing object repair work should be visible");
4450 assert_eq!(existing_object.scanner_role, "repair_admission");
4451 assert_eq!(existing_object.execution_owner, "bucket_replication_queue");
4452 assert_eq!(existing_object.queued, 4);
4453
4454 let passive_requeue = report
4455 .replication_repair
4456 .iter()
4457 .find(|repair| repair.source == "site_replication" && repair.kind == "passive_requeue")
4458 .expect("site passive requeue boundary work should be visible");
4459 assert_eq!(passive_requeue.scanner_role, "boundary_signal");
4460 assert_eq!(passive_requeue.execution_owner, "site_replication_runtime");
4461 assert_eq!(passive_requeue.missed, 5);
4462
4463 let active_resync = report
4464 .replication_repair
4465 .iter()
4466 .find(|repair| repair.source == "site_replication" && repair.kind == "active_resync")
4467 .expect("site active resync boundary work should be visible");
4468 assert_eq!(active_resync.scanner_role, "boundary_signal");
4469 assert_eq!(active_resync.execution_owner, "site_replication_runtime");
4470 assert_eq!(active_resync.queued, 6);
4471 }
4472
4473 #[test]
4474 fn scanner_replication_repair_kind_sources_are_stable() {
4475 let bucket_kinds = [
4476 ScannerReplicationRepairKind::BucketObject,
4477 ScannerReplicationRepairKind::BucketDeleteMarker,
4478 ScannerReplicationRepairKind::BucketVersionPurge,
4479 ScannerReplicationRepairKind::BucketExistingObject,
4480 ];
4481 for kind in bucket_kinds {
4482 assert_eq!(kind.source(), ScannerWorkSource::BucketReplication);
4483 assert_eq!(kind.scanner_role(), ScannerReplicationRepairRole::RepairAdmission);
4484 assert_eq!(kind.execution_owner(), ScannerReplicationExecutionOwner::BucketReplicationQueue);
4485 }
4486
4487 assert_eq!(
4488 ScannerReplicationRepairKind::SitePassiveRequeue.source(),
4489 ScannerWorkSource::SiteReplication
4490 );
4491 assert_eq!(
4492 ScannerReplicationRepairKind::SiteActiveResync.source(),
4493 ScannerWorkSource::SiteReplication
4494 );
4495 assert_eq!(
4496 ScannerReplicationRepairKind::SitePassiveRequeue.scanner_role(),
4497 ScannerReplicationRepairRole::BoundarySignal
4498 );
4499 assert_eq!(
4500 ScannerReplicationRepairKind::SiteActiveResync.scanner_role(),
4501 ScannerReplicationRepairRole::BoundarySignal
4502 );
4503 assert_eq!(
4504 ScannerReplicationRepairKind::SitePassiveRequeue.execution_owner(),
4505 ScannerReplicationExecutionOwner::SiteReplicationRuntime
4506 );
4507 assert_eq!(
4508 ScannerReplicationRepairKind::SiteActiveResync.execution_owner(),
4509 ScannerReplicationExecutionOwner::SiteReplicationRuntime
4510 );
4511 assert_eq!(ScannerReplicationRepairKind::SitePassiveRequeue.as_str(), "passive_requeue");
4512 assert_eq!(ScannerReplicationRepairKind::SiteActiveResync.as_str(), "active_resync");
4513 }
4514
4515 #[tokio::test]
4516 async fn report_includes_scan_cycle_source_work() {
4517 let metrics = Metrics::new();
4518 metrics.record_scanner_source_work(
4519 ScannerWorkSource::Usage,
4520 ScannerSourceWorkUpdate {
4521 checked: 10,
4522 executed: 1,
4523 ..Default::default()
4524 },
4525 );
4526
4527 let start = metrics.start_scan_cycle_work();
4528 metrics.record_scanner_source_work(
4529 ScannerWorkSource::Usage,
4530 ScannerSourceWorkUpdate {
4531 checked: 3,
4532 executed: 2,
4533 ..Default::default()
4534 },
4535 );
4536 metrics.record_scanner_source_work(
4537 ScannerWorkSource::Lifecycle,
4538 ScannerSourceWorkUpdate {
4539 queued: 1,
4540 executed: 1,
4541 missed: 2,
4542 ..Default::default()
4543 },
4544 );
4545
4546 let report = metrics.report().await;
4547 let usage = report
4548 .current_cycle_source_work
4549 .iter()
4550 .find(|work| work.source == ScannerWorkSource::Usage.as_str())
4551 .expect("current cycle usage source work should be visible");
4552 assert_eq!(usage.checked, 3);
4553 assert_eq!(usage.executed, 2);
4554
4555 let lifecycle = report
4556 .current_cycle_source_work
4557 .iter()
4558 .find(|work| work.source == ScannerWorkSource::Lifecycle.as_str())
4559 .expect("current cycle lifecycle source work should be visible");
4560 assert_eq!(lifecycle.queued, 1);
4561 assert_eq!(lifecycle.executed, 1);
4562 assert_eq!(lifecycle.missed, 2);
4563
4564 metrics.finish_scan_cycle_work(start);
4565 let report = metrics.report().await;
4566 assert!(report.current_cycle_source_work.is_empty());
4567
4568 let usage = report
4569 .last_cycle_source_work
4570 .iter()
4571 .find(|work| work.source == ScannerWorkSource::Usage.as_str())
4572 .expect("last cycle usage source work should be visible");
4573 assert_eq!(usage.checked, 3);
4574 assert_eq!(usage.executed, 2);
4575
4576 let lifecycle = report
4577 .last_cycle_source_work
4578 .iter()
4579 .find(|work| work.source == ScannerWorkSource::Lifecycle.as_str())
4580 .expect("last cycle lifecycle source work should be visible");
4581 assert_eq!(lifecycle.queued, 1);
4582 assert_eq!(lifecycle.executed, 1);
4583 assert_eq!(lifecycle.missed, 2);
4584 }
4585
4586 #[tokio::test]
4587 async fn report_preserves_current_cycle_started_time() {
4588 let previous_init_time = *crate::init_time::GLOBAL_INIT_TIME.read().await;
4589 let init_time = Utc::now() - chrono::Duration::hours(1);
4590 let cycle_started = Utc::now();
4591 *crate::init_time::GLOBAL_INIT_TIME.write().await = Some(init_time);
4592
4593 let metrics = Metrics::new();
4594 metrics
4595 .set_cycle(Some(CurrentCycle {
4596 current: 7,
4597 started: cycle_started,
4598 ..Default::default()
4599 }))
4600 .await;
4601
4602 let report = metrics.report().await;
4603 *crate::init_time::GLOBAL_INIT_TIME.write().await = previous_init_time;
4604
4605 assert_eq!(report.current_started, chrono_to_jiff_timestamp(cycle_started));
4606 }
4607
4608 #[tokio::test]
4609 async fn report_includes_current_scan_mode() {
4610 let metrics = Metrics::new();
4611 metrics.set_current_scan_mode(HealScanMode::Deep);
4612
4613 let report = metrics.report().await;
4614
4615 assert_eq!(report.current_scan_mode, HealScanMode::Deep.as_str());
4616 }
4617
4618 #[tokio::test]
4619 async fn report_includes_scanner_leader_liveness() {
4620 let metrics = Metrics::new();
4621
4622 metrics.record_scanner_leader_liveness("contended", false, "lock busy").await;
4623 metrics.record_scanner_cycle_end_time();
4624 let report = metrics.report().await;
4625
4626 assert_eq!(report.leader_lock_state, "contended");
4627 assert!(!report.leader_lock_held_by_this_process);
4628 assert_eq!(report.leader_lock_last_error, "lock busy");
4629 assert!(report.leader_lock_last_update_unix_secs > 0);
4630 assert!(report.last_cycle_end_unix_secs > 0);
4631 }
4632
4633 #[tokio::test]
4634 async fn report_includes_last_scan_cycle_result() {
4635 let metrics = Metrics::new();
4636 metrics.record_scan_cycle_complete(false, Duration::from_millis(1500));
4637
4638 let report = metrics.report().await;
4639
4640 assert_eq!(report.last_cycle_result, SCAN_CYCLE_RESULT_ERROR_LABEL);
4641 assert_eq!(report.last_cycle_result_code, SCAN_CYCLE_RESULT_ERROR as u64);
4642 assert_eq!(report.last_cycle_duration_seconds, 1.5);
4643 assert_eq!(report.failed_cycles, 1);
4644 }
4645
4646 #[tokio::test]
4647 async fn report_tracks_superseded_cycle_without_failed_increment() {
4648 let metrics = Metrics::new();
4649 metrics.record_scan_cycle_superseded(Duration::from_millis(750));
4650
4651 let report = metrics.report().await;
4652
4653 assert_eq!(report.last_cycle_result, SCAN_CYCLE_RESULT_SUPERSEDED_LABEL);
4654 assert_eq!(report.last_cycle_result_code, u64::from(SCAN_CYCLE_RESULT_SUPERSEDED));
4655 assert_eq!(report.last_cycle_duration_seconds, 0.75);
4656 assert_eq!(report.failed_cycles, 0);
4657 assert_eq!(report.superseded_cycles, 1);
4658 assert_eq!(report.partial_cycles, 0);
4659 }
4660
4661 #[tokio::test]
4662 async fn report_tracks_deferred_cycle_without_failed_increment() {
4663 let metrics = Metrics::new();
4664 metrics.record_scan_cycle_deferred(Duration::from_millis(250));
4665
4666 let report = metrics.report().await;
4667
4668 assert_eq!(report.last_cycle_result, SCAN_CYCLE_RESULT_DEFERRED_LABEL);
4669 assert_eq!(report.last_cycle_result_code, u64::from(SCAN_CYCLE_RESULT_DEFERRED));
4670 assert_eq!(report.last_cycle_duration_seconds, 0.25);
4671 assert_eq!(report.failed_cycles, 0);
4672 assert_eq!(report.superseded_cycles, 0);
4673 assert_eq!(report.partial_cycles, 0);
4674 }
4675
4676 #[tokio::test]
4677 async fn report_tracks_successful_scan_cycle_without_failed_increment() {
4678 let metrics = Metrics::new();
4679 metrics.record_scan_cycle_partial_with_source(
4680 Duration::from_millis(500),
4681 ScanCyclePartialReason::Runtime,
4682 Some(ScannerWorkSource::Heal),
4683 );
4684 metrics.record_scan_cycle_complete(true, Duration::from_secs(2));
4685
4686 let report = metrics.report().await;
4687
4688 assert_eq!(report.last_cycle_result, SCAN_CYCLE_RESULT_SUCCESS_LABEL);
4689 assert_eq!(report.last_cycle_result_code, SCAN_CYCLE_RESULT_SUCCESS as u64);
4690 assert_eq!(report.last_cycle_partial_reason, SCAN_CYCLE_RESULT_UNKNOWN_LABEL);
4691 assert_eq!(report.last_cycle_partial_reason_code, ScanCyclePartialReason::Unknown as u8 as u64);
4692 assert_eq!(report.last_cycle_partial_source, SCAN_CYCLE_RESULT_UNKNOWN_LABEL);
4693 assert_eq!(report.last_cycle_partial_source_code, 0);
4694 assert_eq!(report.last_cycle_duration_seconds, 2.0);
4695 assert_eq!(report.failed_cycles, 0);
4696 assert_eq!(report.partial_cycles, 1);
4697 }
4698
4699 #[tokio::test]
4700 async fn report_tracks_partial_scan_cycle_without_failed_increment() {
4701 let metrics = Metrics::new();
4702 metrics.record_scan_cycle_partial_with_source(
4703 Duration::from_millis(2500),
4704 ScanCyclePartialReason::Directories,
4705 Some(ScannerWorkSource::Usage),
4706 );
4707
4708 let report = metrics.report().await;
4709
4710 assert_eq!(report.last_cycle_result, SCAN_CYCLE_RESULT_PARTIAL_LABEL);
4711 assert_eq!(report.last_cycle_result_code, SCAN_CYCLE_RESULT_PARTIAL as u64);
4712 assert_eq!(report.last_cycle_partial_reason, "directories");
4713 assert_eq!(report.last_cycle_partial_reason_code, ScanCyclePartialReason::Directories as u8 as u64);
4714 assert_eq!(report.last_cycle_partial_source, ScannerWorkSource::Usage.as_str());
4715 assert_eq!(report.last_cycle_partial_source_code, 1);
4716 assert_eq!(report.last_cycle_duration_seconds, 2.5);
4717 assert_eq!(report.failed_cycles, 0);
4718 assert_eq!(report.partial_cycles, 1);
4719 assert_eq!(report.partial_cycles_directories, 1);
4720 assert_eq!(report.partial_cycles_runtime, 0);
4721 assert_eq!(report.partial_cycles_objects, 0);
4722 assert_eq!(report.partial_cycles_unknown, 0);
4723 let usage = report
4724 .partial_cycles_by_source
4725 .iter()
4726 .find(|source| source.source == ScannerWorkSource::Usage.as_str())
4727 .expect("usage partial source count should be visible");
4728 assert_eq!(usage.cycles, 1);
4729 }
4730
4731 #[tokio::test]
4732 async fn report_includes_last_scan_cycle_work() {
4733 let metrics = Metrics::new();
4734 metrics.record_scan_cycle_work(ScanCycleWorkSnapshot {
4735 objects_scanned: 11,
4736 directories_scanned: 7,
4737 bucket_drive_scans: 3,
4738 bucket_drive_failures: 2,
4739 yield_events: 5,
4740 yield_duration_millis: 250,
4741 throttle_sleep_events: 7,
4742 throttle_sleep_duration_millis: 500,
4743 ilm_actions: 13,
4744 heal_objects: 2,
4745 replication_checks: 4,
4746 usage_saves: 6,
4747 ..Default::default()
4748 });
4749
4750 let report = metrics.report().await;
4751
4752 assert_eq!(report.last_cycle_objects_scanned, 11);
4753 assert_eq!(report.last_cycle_directories_scanned, 7);
4754 assert_eq!(report.last_cycle_bucket_drive_scans, 3);
4755 assert_eq!(report.last_cycle_bucket_drive_failures, 2);
4756 assert_eq!(report.last_cycle_yield_events, 5);
4757 assert_eq!(report.last_cycle_yield_duration_seconds, 0.25);
4758 assert_eq!(report.last_cycle_throttle_sleep_events, 7);
4759 assert_eq!(report.last_cycle_throttle_sleep_duration_seconds, 0.5);
4760 assert_eq!(report.last_cycle_ilm_actions, 13);
4761 assert_eq!(report.last_cycle_heal_objects, 2);
4762 assert_eq!(report.last_cycle_replication_checks, 4);
4763 assert_eq!(report.last_cycle_usage_saves, 6);
4764 }
4765
4766 #[tokio::test]
4767 async fn report_includes_bucket_drive_scan_starts() {
4768 let metrics = Metrics::new();
4769 metrics.record_scan_bucket_drive_start(ScannerWorkSource::Usage, "bucket-a", "/mnt/data/1");
4770 metrics.record_scan_bucket_drive_failure();
4771
4772 let report = metrics.report().await;
4773
4774 assert_eq!(report.life_time_ops.get("scan_bucket_drive_start"), Some(&1));
4775 assert_eq!(report.life_time_ops.get("scan_bucket_drive_failure"), Some(&1));
4776 }
4777
4778 #[tokio::test]
4779 async fn active_bucket_drive_snapshot_is_structured_and_retired_on_end() {
4780 let metrics = Metrics::new();
4781 metrics.record_scan_bucket_drive_start(ScannerWorkSource::Usage, "bucket-a", "/mnt/data/1");
4782 metrics.record_scan_bucket_drive_start(ScannerWorkSource::Usage, "bucket-a", "/mnt/data/1");
4783 let active = metrics.scanner_runtime_details_report().active_bucket_drive_scans;
4784 assert_eq!(active.len(), 1);
4785 assert_eq!(active[0].source, ScannerWorkSource::Usage.as_str());
4786 assert_eq!(active[0].bucket, "bucket-a");
4787 assert_eq!(active[0].drive, "/mnt/data/1");
4788 assert_eq!(active[0].count, 2);
4789
4790 metrics.record_scan_bucket_drive_end(ScannerWorkSource::Usage, "bucket-a", "/mnt/data/1");
4791 assert_eq!(metrics.scanner_runtime_details_report().active_bucket_drive_scans[0].count, 1);
4792 metrics.record_scan_bucket_drive_end(ScannerWorkSource::Usage, "bucket-a", "/mnt/data/1");
4793 assert!(metrics.scanner_runtime_details_report().active_bucket_drive_scans.is_empty());
4794
4795 metrics.record_scan_bucket_drive_start(ScannerWorkSource::Usage, "", "/mnt/data/1");
4796 assert!(metrics.scanner_runtime_details_report().active_bucket_drive_scans.is_empty());
4797 }
4798
4799 #[tokio::test]
4800 async fn report_includes_structured_bucket_drive_results() {
4801 let metrics = Metrics::new();
4802 metrics.record_scanner_bucket_drive_result("photos", "/data1", "success");
4803
4804 let cycle_start = metrics.start_scan_cycle_work();
4805 metrics.record_scanner_bucket_drive_result("photos", "/data1", "partial");
4806
4807 let active_report = metrics.scanner_runtime_details_report();
4808 assert_eq!(
4809 active_report.current_cycle_bucket_drive_results,
4810 vec![ScannerBucketDriveResultSnapshot {
4811 bucket: "photos".to_string(),
4812 drive: "/data1".to_string(),
4813 result: "partial".to_string(),
4814 count: 1,
4815 }]
4816 );
4817
4818 metrics.finish_scan_cycle_work(cycle_start);
4819
4820 let report = metrics.scanner_runtime_details_report();
4821
4822 assert_eq!(
4823 report.bucket_drive_results,
4824 vec![
4825 ScannerBucketDriveResultSnapshot {
4826 bucket: "photos".to_string(),
4827 drive: "/data1".to_string(),
4828 result: "partial".to_string(),
4829 count: 1,
4830 },
4831 ScannerBucketDriveResultSnapshot {
4832 bucket: "photos".to_string(),
4833 drive: "/data1".to_string(),
4834 result: "success".to_string(),
4835 count: 1,
4836 },
4837 ]
4838 );
4839 assert!(report.current_cycle_bucket_drive_results.is_empty());
4840 assert_eq!(
4841 report.last_cycle_bucket_drive_results,
4842 vec![ScannerBucketDriveResultSnapshot {
4843 bucket: "photos".to_string(),
4844 drive: "/data1".to_string(),
4845 result: "partial".to_string(),
4846 count: 1,
4847 }]
4848 );
4849 }
4850
4851 #[tokio::test]
4852 async fn scanner_bucket_drive_results_are_bounded() {
4853 let metrics = Metrics::new();
4854 for index in 0..MAX_SCANNER_BUCKET_DRIVE_RESULT_KEYS {
4855 metrics.record_scanner_bucket_drive_result(&format!("bucket-{index}"), "/data1", "success");
4856 }
4857 metrics.record_scanner_bucket_drive_result("overflow", "/data1", "success");
4858
4859 let report = metrics.scanner_runtime_details_report();
4860
4861 assert_eq!(report.bucket_drive_results.len(), MAX_SCANNER_BUCKET_DRIVE_RESULT_KEYS);
4862 assert!(
4863 report
4864 .bucket_drive_results
4865 .iter()
4866 .any(|snapshot| snapshot.bucket == "overflow")
4867 );
4868 assert!(
4869 report
4870 .bucket_drive_results
4871 .iter()
4872 .all(|snapshot| snapshot.bucket != "bucket-0")
4873 );
4874 }
4875
4876 #[tokio::test]
4877 async fn scanner_bucket_drive_result_eviction_keeps_recent_keys() {
4878 let metrics = Metrics::new();
4879 for index in 0..MAX_SCANNER_BUCKET_DRIVE_RESULT_KEYS {
4880 metrics.record_scanner_bucket_drive_result(&format!("bucket-{index}"), "/data1", "success");
4881 }
4882 metrics.record_scanner_bucket_drive_result("bucket-0", "/data1", "success");
4883 metrics.record_scanner_bucket_drive_result("overflow", "/data1", "success");
4884
4885 let report = metrics.scanner_runtime_details_report();
4886
4887 assert_eq!(report.bucket_drive_results.len(), MAX_SCANNER_BUCKET_DRIVE_RESULT_KEYS);
4888 assert!(
4889 report
4890 .bucket_drive_results
4891 .iter()
4892 .any(|snapshot| snapshot.bucket == "bucket-0" && snapshot.count == 2)
4893 );
4894 assert!(
4895 report
4896 .bucket_drive_results
4897 .iter()
4898 .all(|snapshot| snapshot.bucket != "bucket-1")
4899 );
4900 }
4901
4902 #[tokio::test]
4903 async fn scanner_bucket_drive_result_eviction_survives_full_refresh() {
4904 let metrics = Metrics::new();
4905 for index in 0..MAX_SCANNER_BUCKET_DRIVE_RESULT_KEYS {
4906 metrics.record_scanner_bucket_drive_result(&format!("bucket-{index}"), "/data1", "success");
4907 }
4908 for index in 0..MAX_SCANNER_BUCKET_DRIVE_RESULT_KEYS {
4909 metrics.record_scanner_bucket_drive_result(&format!("bucket-{index}"), "/data1", "success");
4910 }
4911 metrics.record_scanner_bucket_drive_result("overflow", "/data1", "success");
4912
4913 let report = metrics.scanner_runtime_details_report();
4914
4915 assert_eq!(report.bucket_drive_results.len(), MAX_SCANNER_BUCKET_DRIVE_RESULT_KEYS);
4916 assert!(
4917 report
4918 .bucket_drive_results
4919 .iter()
4920 .any(|snapshot| snapshot.bucket == "overflow")
4921 );
4922 assert!(
4923 report
4924 .bucket_drive_results
4925 .iter()
4926 .all(|snapshot| snapshot.bucket != "bucket-0")
4927 );
4928 }
4929
4930 #[tokio::test]
4931 async fn report_includes_usage_freshness_status() {
4932 let metrics = Metrics::new();
4933 metrics.record_scanner_dirty_usage_pending(2);
4934 metrics.record_scanner_dirty_usage_cycle_snapshot(1);
4935 metrics.record_scanner_dirty_usage_cycle_clear(1, 1);
4936 metrics.record_scanner_usage_save_result(ScannerUsageSaveResult::Success);
4937 metrics.record_scanner_usage_deferred("data_movement");
4938
4939 let report = metrics.report().await;
4940
4941 assert_eq!(report.usage_freshness.dirty_pending_buckets, 1);
4942 assert!(report.usage_freshness.last_dirty_mark_unix_secs > 0);
4943 assert!(report.usage_freshness.last_dirty_clear_unix_secs > 0);
4944 assert_eq!(report.usage_freshness.last_cycle_dirty_buckets, 1);
4945 assert_eq!(report.usage_freshness.last_cycle_cleared_dirty_buckets, 1);
4946 assert!(report.usage_freshness.last_usage_save_unix_secs > 0);
4947 assert_eq!(report.usage_freshness.last_usage_save_result, "success");
4948 assert_eq!(report.usage_freshness.last_usage_save_result_code, 1);
4949 assert!(report.usage_freshness.deferred_pending);
4950 assert_eq!(report.usage_freshness.deferred_total, 1);
4951 assert!(report.usage_freshness.last_deferred_unix_secs > 0);
4952 assert_eq!(report.usage_freshness.last_deferred_reason, "data_movement");
4953
4954 metrics.record_scanner_usage_durable_success();
4955 let report = metrics.report().await;
4956 assert!(!report.usage_freshness.deferred_pending);
4957 assert_eq!(report.usage_freshness.deferred_total, 1);
4958 assert!(report.usage_freshness.last_durable_success_unix_secs > 0);
4959 assert_eq!(report.usage_freshness.last_publication_state, "success");
4960
4961 metrics.record_scanner_usage_publication("no_update", "no_update");
4962 let report = metrics.report().await;
4963 assert_eq!(report.usage_freshness.last_publication_state, "no_update");
4964 assert_eq!(report.usage_freshness.last_publication_reason, "no_update");
4965 }
4966
4967 #[tokio::test]
4968 async fn dirty_usage_cycle_snapshot_resets_stale_cleared_count() {
4969 let metrics = Metrics::new();
4970 metrics.record_scanner_dirty_usage_cycle_snapshot(2);
4971 metrics.record_scanner_dirty_usage_cycle_clear(2, 0);
4972 metrics.record_scanner_dirty_usage_cycle_snapshot(1);
4973
4974 let report = metrics.report().await;
4975
4976 assert_eq!(report.usage_freshness.last_cycle_dirty_buckets, 1);
4977 assert_eq!(report.usage_freshness.last_cycle_cleared_dirty_buckets, 0);
4978 }
4979
4980 #[tokio::test]
4981 async fn report_includes_active_scan_cycle_work() {
4982 let metrics = Metrics::new();
4983 metrics.operations[Metric::ScanObject as usize].store(10, Ordering::Relaxed);
4984 metrics.operations[Metric::ScanFolder as usize].store(5, Ordering::Relaxed);
4985 metrics.operations[Metric::ScanBucketDrive as usize].store(1, Ordering::Relaxed);
4986 metrics.operations[Metric::ScanBucketDriveFailure as usize].store(1, Ordering::Relaxed);
4987 metrics.operations[Metric::Yield as usize].store(2, Ordering::Relaxed);
4988 metrics.operations[Metric::ThrottleSleep as usize].store(3, Ordering::Relaxed);
4989 metrics.operations[Metric::Ilm as usize].store(6, Ordering::Relaxed);
4990 metrics.record_scanner_ilm_action(6);
4991 metrics.operations[Metric::HealAbandonedObject as usize].store(4, Ordering::Relaxed);
4992 metrics.operations[Metric::CheckReplication as usize].store(5, Ordering::Relaxed);
4993 metrics.operations[Metric::SaveUsage as usize].store(3, Ordering::Relaxed);
4994 metrics.scanner_yield_duration_millis.store(100, Ordering::Relaxed);
4995 metrics.scanner_throttle_sleep_duration_millis.store(200, Ordering::Relaxed);
4996
4997 let start = metrics.start_scan_cycle_work();
4998 metrics.operations[Metric::ScanObject as usize].store(17, Ordering::Relaxed);
4999 metrics.operations[Metric::ScanFolder as usize].store(8, Ordering::Relaxed);
5000 metrics.operations[Metric::ScanBucketDrive as usize].store(3, Ordering::Relaxed);
5001 metrics.operations[Metric::ScanBucketDriveFailure as usize].store(4, Ordering::Relaxed);
5002 metrics.operations[Metric::Ilm as usize].store(15, Ordering::Relaxed);
5003 metrics.record_scanner_ilm_action(9);
5004 metrics.operations[Metric::HealAbandonedObject as usize].store(7, Ordering::Relaxed);
5005 metrics.operations[Metric::CheckReplication as usize].store(11, Ordering::Relaxed);
5006 metrics.operations[Metric::SaveUsage as usize].store(5, Ordering::Relaxed);
5007 metrics.record_scanner_yield(Duration::from_millis(150));
5008 metrics.record_scanner_throttle_sleep(Duration::from_millis(250));
5009
5010 let report = metrics.report().await;
5011
5012 assert!(report.current_cycle_active);
5013 assert_eq!(report.current_cycle, 0);
5014 assert_eq!(report.current_cycle_objects_scanned, 7);
5015 assert_eq!(report.current_cycle_directories_scanned, 3);
5016 assert_eq!(report.current_cycle_bucket_drive_scans, 2);
5017 assert_eq!(report.current_cycle_bucket_drive_failures, 3);
5018 assert_eq!(report.current_cycle_yield_events, 1);
5019 assert_eq!(report.current_cycle_yield_duration_seconds, 0.15);
5020 assert_eq!(report.current_cycle_throttle_sleep_events, 1);
5021 assert_eq!(report.current_cycle_throttle_sleep_duration_seconds, 0.25);
5022 assert_eq!(report.current_cycle_ilm_actions, 9);
5023 assert_eq!(report.current_cycle_heal_objects, 3);
5024 assert_eq!(report.current_cycle_replication_checks, 6);
5025 assert_eq!(report.current_cycle_usage_saves, 2);
5026
5027 metrics.finish_scan_cycle_work(start);
5028 let report = metrics.report().await;
5029
5030 assert!(!report.current_cycle_active);
5031 assert_eq!(report.current_cycle, 0);
5032 assert_eq!(report.current_cycle_objects_scanned, 0);
5033 assert_eq!(report.current_cycle_directories_scanned, 0);
5034 assert_eq!(report.current_cycle_bucket_drive_scans, 0);
5035 assert_eq!(report.current_cycle_bucket_drive_failures, 0);
5036 assert_eq!(report.current_cycle_yield_events, 0);
5037 assert_eq!(report.current_cycle_yield_duration_seconds, 0.0);
5038 assert_eq!(report.current_cycle_throttle_sleep_events, 0);
5039 assert_eq!(report.current_cycle_throttle_sleep_duration_seconds, 0.0);
5040 assert_eq!(report.current_cycle_ilm_actions, 0);
5041 assert_eq!(report.current_cycle_heal_objects, 0);
5042 assert_eq!(report.current_cycle_replication_checks, 0);
5043 assert_eq!(report.current_cycle_usage_saves, 0);
5044 assert_eq!(report.last_cycle_objects_scanned, 7);
5045 assert_eq!(report.last_cycle_directories_scanned, 3);
5046 assert_eq!(report.last_cycle_bucket_drive_scans, 2);
5047 assert_eq!(report.last_cycle_bucket_drive_failures, 3);
5048 assert_eq!(report.last_cycle_yield_events, 1);
5049 assert_eq!(report.last_cycle_yield_duration_seconds, 0.15);
5050 assert_eq!(report.last_cycle_throttle_sleep_events, 1);
5051 assert_eq!(report.last_cycle_throttle_sleep_duration_seconds, 0.25);
5052 assert_eq!(report.last_cycle_ilm_actions, 9);
5053 assert_eq!(report.last_cycle_heal_objects, 3);
5054 assert_eq!(report.last_cycle_replication_checks, 6);
5055 assert_eq!(report.last_cycle_usage_saves, 2);
5056 }
5057
5058 #[tokio::test]
5059 async fn scan_cycle_activity_and_cycle_state_publish_together() {
5060 let metrics = Metrics::new();
5061 let cycle_started = Utc::now() - chrono::Duration::seconds(5);
5062 let active_cycle = CurrentCycle {
5063 current: 12,
5064 next: 13,
5065 started: cycle_started,
5066 ..Default::default()
5067 };
5068
5069 let cycle_state = metrics.cycle_info.read().await;
5070 let mut start_transition = Box::pin(metrics.start_scan_cycle_work_with_cycle(active_cycle));
5071 let waker = std::task::Waker::noop();
5072 let mut context = std::task::Context::from_waker(waker);
5073 assert!(start_transition.as_mut().poll(&mut context).is_pending());
5074 assert!(!metrics.current_scan_cycle_work_active.load(Ordering::Acquire));
5075 drop(cycle_state);
5076
5077 let start = start_transition.await;
5078 let active = metrics.report().await;
5079 assert!(active.current_cycle_active);
5080 assert_eq!(active.current_cycle, 12);
5081 assert_eq!(active.current_started, chrono_to_jiff_timestamp(cycle_started));
5082
5083 let idle_cycle = CurrentCycle {
5084 current: 0,
5085 next: 13,
5086 started: cycle_started,
5087 ..Default::default()
5088 };
5089 let cycle_state = metrics.cycle_info.read().await;
5090 let mut finish_transition = Box::pin(metrics.finish_scan_cycle_work_with_cycle(start, idle_cycle));
5091 assert!(finish_transition.as_mut().poll(&mut context).is_pending());
5092 assert!(metrics.current_scan_cycle_work_active.load(Ordering::Acquire));
5093 drop(cycle_state);
5094
5095 finish_transition.await;
5096 let idle = metrics.report().await;
5097 assert!(!idle.current_cycle_active);
5098 assert_eq!(idle.current_cycle, 0);
5099 }
5100
5101 #[tokio::test]
5102 async fn report_keeps_cycle_identity_and_work_in_one_snapshot() {
5103 let metrics = Metrics::new();
5104 let cycle_ten = CurrentCycle {
5105 current: 10,
5106 next: 11,
5107 started: Utc::now() - chrono::Duration::seconds(10),
5108 ..Default::default()
5109 };
5110 let cycle_ten_start = metrics.start_scan_cycle_work_with_cycle(cycle_ten.clone()).await;
5111 metrics.operations[Metric::ScanObject as usize].store(1, Ordering::Relaxed);
5112 metrics.record_scanner_bucket_drive_result("cycle-ten", "/data1", "partial");
5113
5114 let paths = metrics.current_paths.write().await;
5115 let mut report = Box::pin(metrics.report_with_runtime_details());
5116 let waker = std::task::Waker::noop();
5117 let mut context = std::task::Context::from_waker(waker);
5118 assert!(report.as_mut().poll(&mut context).is_pending());
5119
5120 metrics
5121 .finish_scan_cycle_work_with_cycle(cycle_ten_start, CurrentCycle { current: 0, ..cycle_ten })
5122 .await;
5123 let cycle_eleven_start = metrics
5124 .start_scan_cycle_work_with_cycle(CurrentCycle {
5125 current: 11,
5126 next: 12,
5127 started: Utc::now(),
5128 ..Default::default()
5129 })
5130 .await;
5131 metrics.operations[Metric::ScanObject as usize].store(101, Ordering::Relaxed);
5132 metrics.record_scanner_bucket_drive_result("cycle-eleven", "/data1", "partial");
5133
5134 drop(paths);
5135 let (snapshot, runtime_details) = report.await;
5136
5137 assert_eq!(snapshot.current_cycle, 10);
5138 assert_eq!(snapshot.current_cycle_objects_scanned, 1);
5139 assert_eq!(
5140 runtime_details.current_cycle_bucket_drive_results,
5141 vec![ScannerBucketDriveResultSnapshot {
5142 bucket: "cycle-ten".to_string(),
5143 drive: "/data1".to_string(),
5144 result: "partial".to_string(),
5145 count: 1,
5146 }]
5147 );
5148
5149 metrics
5150 .finish_scan_cycle_work_with_cycle(cycle_eleven_start, CurrentCycle::default())
5151 .await;
5152 }
5153
5154 #[tokio::test]
5155 async fn scanner_cycle_ilm_actions_ignore_global_ilm_work() {
5156 let metrics = Metrics::new();
5157 metrics.operations[Metric::Ilm as usize].store(4, Ordering::Relaxed);
5158
5159 let start = metrics.start_scan_cycle_work();
5160 metrics.operations[Metric::Ilm as usize].store(11, Ordering::Relaxed);
5161
5162 let report = metrics.report().await;
5163
5164 assert_eq!(report.current_cycle_ilm_actions, 0);
5165
5166 metrics.record_scanner_ilm_action(3);
5167 let report = metrics.report().await;
5168
5169 assert_eq!(report.current_cycle_ilm_actions, 3);
5170
5171 metrics.finish_scan_cycle_work(start);
5172 let report = metrics.report().await;
5173
5174 assert_eq!(report.last_cycle_ilm_actions, 3);
5175 }
5176
5177 #[test]
5178 fn record_scanner_yield_tracks_count_and_duration() {
5179 let metrics = Metrics::new();
5180
5181 metrics.record_scanner_yield(Duration::from_millis(42));
5182
5183 assert_eq!(metrics.lifetime(Metric::Yield), 1);
5184 assert_eq!(metrics.scanner_yield_duration_millis.load(Ordering::Relaxed), 42);
5185 assert_eq!(metrics.last_minute(Metric::Yield).n, 1);
5186 assert_eq!(metrics.lifetime(Metric::ThrottleSleep), 0);
5187 }
5188
5189 #[test]
5190 fn record_scanner_throttle_sleep_tracks_count_and_duration() {
5191 let metrics = Metrics::new();
5192
5193 metrics.record_scanner_throttle_sleep(Duration::from_millis(42));
5194
5195 assert_eq!(metrics.lifetime(Metric::ThrottleSleep), 1);
5196 assert_eq!(metrics.scanner_throttle_sleep_duration_millis.load(Ordering::Relaxed), 42);
5197 assert_eq!(metrics.last_minute(Metric::ThrottleSleep).n, 1);
5198 assert_eq!(metrics.lifetime(Metric::Yield), 0);
5199 }
5200
5201 #[tokio::test]
5202 async fn report_includes_scanner_throttle_config() {
5203 let metrics = Metrics::new();
5204
5205 metrics.record_scanner_throttle_config(true, 2.5, Duration::from_millis(1500), 128);
5206 let report = metrics.report().await;
5207
5208 assert!(report.throttle_idle_mode_enabled);
5209 assert_eq!(report.throttle_sleep_factor, 2.5);
5210 assert_eq!(report.throttle_max_sleep_seconds, 1.5);
5211 assert_eq!(report.yield_every_n_objects, 128);
5212 }
5213
5214 #[tokio::test]
5215 async fn scanner_throttle_config_rejects_invalid_sleep_factor() {
5216 let metrics = Metrics::new();
5217
5218 metrics.record_scanner_throttle_config(false, f64::NAN, Duration::ZERO, 0);
5219 let report = metrics.report().await;
5220
5221 assert!(!report.throttle_idle_mode_enabled);
5222 assert_eq!(report.throttle_sleep_factor, 0.0);
5223 assert_eq!(report.throttle_max_sleep_seconds, 0.0);
5224 assert_eq!(report.yield_every_n_objects, 0);
5225 }
5226
5227 #[tokio::test]
5228 async fn report_includes_scanner_cycle_config() {
5229 let metrics = Metrics::new();
5230
5231 metrics.record_scanner_cycle_config(
5232 Duration::from_secs(3600),
5233 Some(Duration::from_secs(86400)),
5234 Some(Duration::from_secs(1800)),
5235 Some(1_000_000),
5236 Some(100_000),
5237 );
5238 let report = metrics.report().await;
5239
5240 assert_eq!(report.cycle_interval_seconds, 3600.0);
5241 assert_eq!(report.cycle_max_duration_seconds, 1800.0);
5242 assert_eq!(report.cycle_max_objects, 1_000_000);
5243 assert_eq!(report.cycle_max_directories, 100_000);
5244 assert!(report.bitrot_cycle_enabled);
5245 assert_eq!(report.bitrot_cycle_seconds, 86400.0);
5246
5247 metrics.record_scanner_cycle_config(Duration::from_secs(60), None, None, None, None);
5248 let report = metrics.report().await;
5249
5250 assert_eq!(report.cycle_interval_seconds, 60.0);
5251 assert_eq!(report.cycle_max_duration_seconds, 0.0);
5252 assert_eq!(report.cycle_max_objects, 0);
5253 assert_eq!(report.cycle_max_directories, 0);
5254 assert!(!report.bitrot_cycle_enabled);
5255 assert_eq!(report.bitrot_cycle_seconds, 0.0);
5256 }
5257
5258 #[tokio::test]
5259 async fn scanner_cycle_timeout_metrics_reset_for_a_new_cycle() {
5260 let metrics = Metrics::new();
5261 metrics.record_scanner_cycle_timeout(true, Duration::from_secs(17));
5262 let timed_out = metrics.report().await;
5263 assert_eq!(timed_out.cycle_timeout_total, 1);
5264 assert_eq!(timed_out.cycle_last_progress_age, 17);
5265 assert!(timed_out.leader_lease_without_progress);
5266
5267 metrics.record_scanner_cycle_config(Duration::from_secs(60), None, Some(Duration::from_secs(1)), None, None);
5268 let current = metrics.report().await;
5269 assert_eq!(current.cycle_timeout_total, 1);
5270 assert_eq!(current.cycle_last_progress_age, 0);
5271 assert!(!current.leader_lease_without_progress);
5272 }
5273}