Skip to main content

rustfs_scanner_metrics/
metrics.rs

1// Copyright 2024 RustFS Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use crate::last_minute::{AccElem, LastMinuteLatency};
16use chrono::{DateTime, Utc};
17use jiff::Timestamp;
18use rustfs_heal_contracts::heal_channel::HealScanMode;
19use serde::{Deserialize, Serialize};
20use std::{
21    collections::{BTreeSet, HashMap},
22    fmt::Display,
23    future::Future,
24    pin::Pin,
25    sync::{
26        Arc, Mutex, OnceLock,
27        atomic::{AtomicBool, AtomicU8, AtomicU64, Ordering},
28    },
29    time::{Duration, SystemTime},
30};
31use tokio::sync::RwLock;
32
33#[derive(Debug, Clone, Copy, PartialEq, Eq)]
34pub enum IlmAction {
35    NoneAction = 0,
36    DeleteAction,
37    DeleteVersionAction,
38    TransitionAction,
39    TransitionVersionAction,
40    DeleteRestoredAction,
41    DeleteRestoredVersionAction,
42    DeleteAllVersionsAction,
43    DelMarkerDeleteAllVersionsAction,
44    ActionCount,
45}
46
47impl IlmAction {
48    pub fn delete_restored(&self) -> bool {
49        *self == Self::DeleteRestoredAction || *self == Self::DeleteRestoredVersionAction
50    }
51
52    pub fn delete_versioned(&self) -> bool {
53        *self == Self::DeleteVersionAction || *self == Self::DeleteRestoredVersionAction
54    }
55
56    pub fn delete_all(&self) -> bool {
57        *self == Self::DeleteAllVersionsAction || *self == Self::DelMarkerDeleteAllVersionsAction
58    }
59
60    pub fn delete(&self) -> bool {
61        if self.delete_restored() {
62            return true;
63        }
64        *self == Self::DeleteVersionAction
65            || *self == Self::DeleteAction
66            || *self == Self::DeleteAllVersionsAction
67            || *self == Self::DelMarkerDeleteAllVersionsAction
68    }
69
70    pub fn as_str(&self) -> &'static str {
71        match self {
72            Self::NoneAction => "none",
73            Self::DeleteAction => "delete",
74            Self::DeleteVersionAction => "delete_version",
75            Self::TransitionAction => "transition",
76            Self::TransitionVersionAction => "transition_version",
77            Self::DeleteRestoredAction => "delete_restored",
78            Self::DeleteRestoredVersionAction => "delete_restored_version",
79            Self::DeleteAllVersionsAction => "delete_all_versions",
80            Self::DelMarkerDeleteAllVersionsAction => "del_marker_delete_all_versions",
81            Self::ActionCount => "action_count",
82        }
83    }
84
85    pub fn from_index(i: usize) -> Option<Self> {
86        match i {
87            0 => Some(Self::NoneAction),
88            1 => Some(Self::DeleteAction),
89            2 => Some(Self::DeleteVersionAction),
90            3 => Some(Self::TransitionAction),
91            4 => Some(Self::TransitionVersionAction),
92            5 => Some(Self::DeleteRestoredAction),
93            6 => Some(Self::DeleteRestoredVersionAction),
94            7 => Some(Self::DeleteAllVersionsAction),
95            8 => Some(Self::DelMarkerDeleteAllVersionsAction),
96            9 => Some(Self::ActionCount),
97            _ => None,
98        }
99    }
100}
101
102impl Display for IlmAction {
103    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
104        write!(f, "{self:?}")
105    }
106}
107
108pub static GLOBAL_METRICS: OnceLock<Arc<Metrics>> = OnceLock::new();
109
110pub fn global_metrics() -> &'static Arc<Metrics> {
111    GLOBAL_METRICS.get_or_init(|| Arc::new(Metrics::new()))
112}
113
114#[derive(Debug, Clone, Copy, PartialEq, Eq)]
115pub enum Metric {
116    // START Realtime metrics, that only records
117    // last minute latencies and total operation count.
118    ReadMetadata = 0,
119    CheckMissing,
120    SaveUsage,
121    ApplyAll,
122    ApplyVersion,
123    TierObjSweep,
124    HealCheck,
125    Ilm,
126    CheckReplication,
127    Yield,
128    ThrottleSleep,
129    CleanAbandoned,
130    ApplyNonCurrent,
131    HealAbandonedVersion,
132
133    // Quota metrics:
134    QuotaCheck,
135    QuotaViolation,
136    QuotaSync,
137
138    // START Trace metrics:
139    StartTrace,
140    ScanObject, // Scan object. All operations included.
141    HealAbandonedObject,
142
143    // END realtime metrics:
144    LastRealtime,
145
146    // Trace only metrics:
147    ScanFolder,      // Scan a folder on disk, recursively.
148    ScanCycle,       // Full cycle, cluster global.
149    ScanBucketDrive, // Single bucket on one drive.
150    CompactFolder,   // Folder compacted.
151    ScanBucketDriveStart,
152    ScanBucketDriveFailure,
153
154    // Must be last:
155    Last,
156}
157
158impl Metric {
159    /// Convert to string representation for metrics
160    pub fn as_str(&self) -> &'static str {
161        match self {
162            Self::ReadMetadata => "read_metadata",
163            Self::CheckMissing => "check_missing",
164            Self::SaveUsage => "save_usage",
165            Self::ApplyAll => "apply_all",
166            Self::ApplyVersion => "apply_version",
167            Self::TierObjSweep => "tier_obj_sweep",
168            Self::HealCheck => "heal_check",
169            Self::Ilm => "ilm",
170            Self::CheckReplication => "check_replication",
171            Self::Yield => "yield",
172            Self::ThrottleSleep => "throttle_sleep",
173            Self::CleanAbandoned => "clean_abandoned",
174            Self::ApplyNonCurrent => "apply_non_current",
175            Self::HealAbandonedVersion => "heal_abandoned_version",
176            Self::QuotaCheck => "quota_check",
177            Self::QuotaViolation => "quota_violation",
178            Self::QuotaSync => "quota_sync",
179            Self::StartTrace => "start_trace",
180            Self::ScanObject => "scan_object",
181            Self::HealAbandonedObject => "heal_abandoned_object",
182            Self::LastRealtime => "last_realtime",
183            Self::ScanFolder => "scan_folder",
184            Self::ScanCycle => "scan_cycle",
185            Self::ScanBucketDrive => "scan_bucket_drive",
186            Self::CompactFolder => "compact_folder",
187            Self::ScanBucketDriveStart => "scan_bucket_drive_start",
188            Self::ScanBucketDriveFailure => "scan_bucket_drive_failure",
189            Self::Last => "last",
190        }
191    }
192
193    /// Convert from index back to enum (safe version)
194    pub fn from_index(index: usize) -> Option<Self> {
195        if index >= Self::Last as usize {
196            return None;
197        }
198        match index {
199            0 => Some(Self::ReadMetadata),
200            1 => Some(Self::CheckMissing),
201            2 => Some(Self::SaveUsage),
202            3 => Some(Self::ApplyAll),
203            4 => Some(Self::ApplyVersion),
204            5 => Some(Self::TierObjSweep),
205            6 => Some(Self::HealCheck),
206            7 => Some(Self::Ilm),
207            8 => Some(Self::CheckReplication),
208            9 => Some(Self::Yield),
209            10 => Some(Self::ThrottleSleep),
210            11 => Some(Self::CleanAbandoned),
211            12 => Some(Self::ApplyNonCurrent),
212            13 => Some(Self::HealAbandonedVersion),
213            14 => Some(Self::QuotaCheck),
214            15 => Some(Self::QuotaViolation),
215            16 => Some(Self::QuotaSync),
216            17 => Some(Self::StartTrace),
217            18 => Some(Self::ScanObject),
218            19 => Some(Self::HealAbandonedObject),
219            20 => Some(Self::LastRealtime),
220            21 => Some(Self::ScanFolder),
221            22 => Some(Self::ScanCycle),
222            23 => Some(Self::ScanBucketDrive),
223            24 => Some(Self::CompactFolder),
224            25 => Some(Self::ScanBucketDriveStart),
225            26 => Some(Self::ScanBucketDriveFailure),
226            27 => Some(Self::Last),
227            _ => None,
228        }
229    }
230}
231
232const SCANNER_CHECKPOINT_EVENT_SET: &str = "set";
233const SCANNER_CHECKPOINT_EVENT_USED: &str = "used";
234const SCANNER_CHECKPOINT_EVENT_IGNORED: &str = "ignored";
235const SCANNER_CHECKPOINT_EVENT_STALE: &str = "stale";
236
237#[derive(Clone, Copy, Debug, PartialEq, Eq)]
238pub enum ScannerWorkSource {
239    Usage,
240    Lifecycle,
241    BucketReplication,
242    SiteReplication,
243    Heal,
244    Bitrot,
245    Alerts,
246}
247
248impl ScannerWorkSource {
249    const ALL: [Self; 7] = [
250        Self::Usage,
251        Self::Lifecycle,
252        Self::BucketReplication,
253        Self::SiteReplication,
254        Self::Heal,
255        Self::Bitrot,
256        Self::Alerts,
257    ];
258
259    pub fn as_str(self) -> &'static str {
260        match self {
261            Self::Usage => "usage",
262            Self::Lifecycle => "lifecycle",
263            Self::BucketReplication => "bucket_replication",
264            Self::SiteReplication => "site_replication",
265            Self::Heal => "heal",
266            Self::Bitrot => "bitrot",
267            Self::Alerts => "alerts",
268        }
269    }
270
271    fn code(self) -> u8 {
272        match self {
273            Self::Usage => 1,
274            Self::Lifecycle => 2,
275            Self::BucketReplication => 3,
276            Self::SiteReplication => 4,
277            Self::Heal => 5,
278            Self::Bitrot => 6,
279            Self::Alerts => 7,
280        }
281    }
282
283    fn from_code(code: u8) -> Option<Self> {
284        match code {
285            1 => Some(Self::Usage),
286            2 => Some(Self::Lifecycle),
287            3 => Some(Self::BucketReplication),
288            4 => Some(Self::SiteReplication),
289            5 => Some(Self::Heal),
290            6 => Some(Self::Bitrot),
291            7 => Some(Self::Alerts),
292            _ => None,
293        }
294    }
295
296    fn all() -> &'static [Self] {
297        &Self::ALL
298    }
299
300    fn index(self) -> usize {
301        match self {
302            Self::Usage => 0,
303            Self::Lifecycle => 1,
304            Self::BucketReplication => 2,
305            Self::SiteReplication => 3,
306            Self::Heal => 4,
307            Self::Bitrot => 5,
308            Self::Alerts => 6,
309        }
310    }
311}
312
313#[derive(Debug, Clone, Copy, PartialEq, Eq)]
314pub enum ScannerReplicationRepairKind {
315    BucketObject,
316    BucketDeleteMarker,
317    BucketVersionPurge,
318    BucketExistingObject,
319    SitePassiveRequeue,
320    SiteActiveResync,
321}
322
323#[derive(Debug, Clone, Copy, PartialEq, Eq)]
324pub enum ScannerReplicationRepairRole {
325    RepairAdmission,
326    BoundarySignal,
327}
328
329impl ScannerReplicationRepairRole {
330    pub fn as_str(self) -> &'static str {
331        match self {
332            Self::RepairAdmission => "repair_admission",
333            Self::BoundarySignal => "boundary_signal",
334        }
335    }
336}
337
338#[derive(Debug, Clone, Copy, PartialEq, Eq)]
339pub enum ScannerReplicationExecutionOwner {
340    BucketReplicationQueue,
341    SiteReplicationRuntime,
342}
343
344impl ScannerReplicationExecutionOwner {
345    pub fn as_str(self) -> &'static str {
346        match self {
347            Self::BucketReplicationQueue => "bucket_replication_queue",
348            Self::SiteReplicationRuntime => "site_replication_runtime",
349        }
350    }
351}
352
353impl ScannerReplicationRepairKind {
354    const ALL: [Self; 6] = [
355        Self::BucketObject,
356        Self::BucketDeleteMarker,
357        Self::BucketVersionPurge,
358        Self::BucketExistingObject,
359        Self::SitePassiveRequeue,
360        Self::SiteActiveResync,
361    ];
362
363    pub fn source(self) -> ScannerWorkSource {
364        match self {
365            Self::BucketObject | Self::BucketDeleteMarker | Self::BucketVersionPurge | Self::BucketExistingObject => {
366                ScannerWorkSource::BucketReplication
367            }
368            Self::SitePassiveRequeue | Self::SiteActiveResync => ScannerWorkSource::SiteReplication,
369        }
370    }
371
372    pub fn scanner_role(self) -> ScannerReplicationRepairRole {
373        match self {
374            Self::BucketObject | Self::BucketDeleteMarker | Self::BucketVersionPurge | Self::BucketExistingObject => {
375                ScannerReplicationRepairRole::RepairAdmission
376            }
377            Self::SitePassiveRequeue | Self::SiteActiveResync => ScannerReplicationRepairRole::BoundarySignal,
378        }
379    }
380
381    pub fn execution_owner(self) -> ScannerReplicationExecutionOwner {
382        match self {
383            Self::BucketObject | Self::BucketDeleteMarker | Self::BucketVersionPurge | Self::BucketExistingObject => {
384                ScannerReplicationExecutionOwner::BucketReplicationQueue
385            }
386            Self::SitePassiveRequeue | Self::SiteActiveResync => ScannerReplicationExecutionOwner::SiteReplicationRuntime,
387        }
388    }
389
390    pub fn as_str(self) -> &'static str {
391        match self {
392            Self::BucketObject => "object",
393            Self::BucketDeleteMarker => "delete_marker",
394            Self::BucketVersionPurge => "version_purge",
395            Self::BucketExistingObject => "existing_object",
396            Self::SitePassiveRequeue => "passive_requeue",
397            Self::SiteActiveResync => "active_resync",
398        }
399    }
400
401    fn all() -> &'static [Self] {
402        &Self::ALL
403    }
404
405    fn index(self) -> usize {
406        match self {
407            Self::BucketObject => 0,
408            Self::BucketDeleteMarker => 1,
409            Self::BucketVersionPurge => 2,
410            Self::BucketExistingObject => 3,
411            Self::SitePassiveRequeue => 4,
412            Self::SiteActiveResync => 5,
413        }
414    }
415}
416
417#[derive(Debug, Default)]
418struct ScannerSourceWorkCounters {
419    checked: AtomicU64,
420    queued: AtomicU64,
421    executed: AtomicU64,
422    failed: AtomicU64,
423    skipped: AtomicU64,
424    missed: AtomicU64,
425}
426
427#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
428pub struct ScannerSourceWorkUpdate {
429    pub checked: u64,
430    pub queued: u64,
431    pub executed: u64,
432    pub failed: u64,
433    pub skipped: u64,
434    pub missed: u64,
435}
436
437impl ScannerSourceWorkUpdate {
438    pub const fn checked(count: u64) -> Self {
439        Self {
440            checked: count,
441            queued: 0,
442            executed: 0,
443            failed: 0,
444            skipped: 0,
445            missed: 0,
446        }
447    }
448
449    pub const fn queued(count: u64) -> Self {
450        Self {
451            checked: 0,
452            queued: count,
453            executed: 0,
454            failed: 0,
455            skipped: 0,
456            missed: 0,
457        }
458    }
459
460    pub const fn executed(count: u64) -> Self {
461        Self {
462            checked: 0,
463            queued: 0,
464            executed: count,
465            failed: 0,
466            skipped: 0,
467            missed: 0,
468        }
469    }
470
471    pub const fn failed(count: u64) -> Self {
472        Self {
473            checked: 0,
474            queued: 0,
475            executed: 0,
476            failed: count,
477            skipped: 0,
478            missed: 0,
479        }
480    }
481
482    pub const fn missed(count: u64) -> Self {
483        Self {
484            checked: 0,
485            queued: 0,
486            executed: 0,
487            failed: 0,
488            skipped: 0,
489            missed: count,
490        }
491    }
492}
493
494impl ScannerSourceWorkCounters {
495    fn add(&self, work: ScannerSourceWorkUpdate) {
496        self.checked.fetch_add(work.checked, Ordering::Relaxed);
497        self.queued.fetch_add(work.queued, Ordering::Relaxed);
498        self.executed.fetch_add(work.executed, Ordering::Relaxed);
499        self.failed.fetch_add(work.failed, Ordering::Relaxed);
500        self.skipped.fetch_add(work.skipped, Ordering::Relaxed);
501        self.missed.fetch_add(work.missed, Ordering::Relaxed);
502    }
503
504    fn store(&self, values: ScannerSourceWorkValues) {
505        self.checked.store(values.checked, Ordering::Relaxed);
506        self.queued.store(values.queued, Ordering::Relaxed);
507        self.executed.store(values.executed, Ordering::Relaxed);
508        self.failed.store(values.failed, Ordering::Relaxed);
509        self.skipped.store(values.skipped, Ordering::Relaxed);
510        self.missed.store(values.missed, Ordering::Relaxed);
511    }
512
513    fn values(&self) -> ScannerSourceWorkValues {
514        ScannerSourceWorkValues {
515            checked: self.checked.load(Ordering::Relaxed),
516            queued: self.queued.load(Ordering::Relaxed),
517            executed: self.executed.load(Ordering::Relaxed),
518            failed: self.failed.load(Ordering::Relaxed),
519            skipped: self.skipped.load(Ordering::Relaxed),
520            missed: self.missed.load(Ordering::Relaxed),
521        }
522    }
523
524    fn snapshot(&self, source: ScannerWorkSource) -> ScannerSourceWorkSnapshot {
525        self.values().snapshot(source)
526    }
527}
528
529#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
530struct ScannerSourceWorkValues {
531    checked: u64,
532    queued: u64,
533    executed: u64,
534    failed: u64,
535    skipped: u64,
536    missed: u64,
537}
538
539impl ScannerSourceWorkValues {
540    fn saturating_sub(self, start: Self) -> Self {
541        Self {
542            checked: self.checked.saturating_sub(start.checked),
543            queued: self.queued.saturating_sub(start.queued),
544            executed: self.executed.saturating_sub(start.executed),
545            failed: self.failed.saturating_sub(start.failed),
546            skipped: self.skipped.saturating_sub(start.skipped),
547            missed: self.missed.saturating_sub(start.missed),
548        }
549    }
550
551    fn snapshot(self, source: ScannerWorkSource) -> ScannerSourceWorkSnapshot {
552        ScannerSourceWorkSnapshot {
553            source: source.as_str().to_string(),
554            checked: self.checked,
555            queued: self.queued,
556            executed: self.executed,
557            failed: self.failed,
558            skipped: self.skipped,
559            missed: self.missed,
560        }
561    }
562
563    fn replication_repair_snapshot(self, kind: ScannerReplicationRepairKind) -> ScannerReplicationRepairSnapshot {
564        ScannerReplicationRepairSnapshot {
565            source: kind.source().as_str().to_string(),
566            kind: kind.as_str().to_string(),
567            scanner_role: kind.scanner_role().as_str().to_string(),
568            execution_owner: kind.execution_owner().as_str().to_string(),
569            checked: self.checked,
570            queued: self.queued,
571            executed: self.executed,
572            failed: self.failed,
573            skipped: self.skipped,
574            missed: self.missed,
575        }
576    }
577}
578
579// ---------------------------------------------------------------------------
580// LockedLastMinuteLatency
581// ---------------------------------------------------------------------------
582//
583// Uses std::sync::Mutex instead of tokio::sync::Mutex.
584//
585// Rationale: the critical section is a handful of integer additions inside
586// LastMinuteLatency::add_all / get_total — no I/O, no blocking syscalls, no
587// awaiting.  A std blocking mutex is cheaper (no task-wakeup overhead) and,
588// crucially, lets every caller be *synchronous*.  That eliminates the need to
589// spawn a background task just to record a duration, which was the only reason
590// tokio::spawn appeared in log/time/time_size/time_n/time_ilm.
591//
592// Note: vec![LockedLastMinuteLatency::default(); N] with the old Arc-based
593// Clone made every element share the *same* inner mutex — a latent bug where
594// all metrics slots wrote to one counter.  The new Clone creates a fresh
595// independent Mutex per element, matching the intent.
596
597/// Thread-safe wrapper for LastMinuteLatency backed by a std blocking mutex.
598pub struct LockedLastMinuteLatency {
599    // Arc so Clone is cheap *and* each cloned value stays independent (its own
600    // allocation).  We never hand out the Arc to two Metrics at once; the Arc
601    // is purely a convenience for the Clone impl below.
602    latency: Arc<Mutex<LastMinuteLatency>>,
603}
604
605impl Default for LockedLastMinuteLatency {
606    fn default() -> Self {
607        Self::new()
608    }
609}
610
611// Produce a fresh, independent slot — *not* a shared alias.  This is what
612// vec![val; N] and #[derive(Clone)] on the parent struct both need.
613impl Clone for LockedLastMinuteLatency {
614    fn clone(&self) -> Self {
615        let inner = match self.latency.lock() {
616            Ok(guard) => guard.clone(),
617            Err(poisoned) => poisoned.into_inner().clone(),
618        };
619        Self {
620            latency: Arc::new(Mutex::new(inner)),
621        }
622    }
623}
624
625impl LockedLastMinuteLatency {
626    pub fn new() -> Self {
627        Self {
628            latency: Arc::new(Mutex::new(LastMinuteLatency::default())),
629        }
630    }
631
632    /// Record a duration sample (no size).
633    pub fn add(&self, duration: Duration) {
634        self.add_size(duration, 0);
635    }
636
637    /// Record a duration sample with an associated byte count.
638    pub fn add_size(&self, duration: Duration, size: u64) {
639        let now = SystemTime::now()
640            .duration_since(SystemTime::UNIX_EPOCH)
641            .unwrap_or_default()
642            .as_secs();
643
644        let elem = AccElem {
645            n: 1,
646            total: duration.as_secs(),
647            size,
648        };
649
650        match self.latency.lock() {
651            Ok(mut guard) => guard.add_all(now, &elem),
652            Err(poisoned) => poisoned.into_inner().add_all(now, &elem),
653        }
654    }
655
656    /// Return accumulated totals for the last minute window.
657    pub fn total(&self) -> AccElem {
658        match self.latency.lock() {
659            Ok(mut latency) => latency.get_total(),
660            Err(poisoned) => poisoned.into_inner().get_total(),
661        }
662    }
663}
664
665// ---------------------------------------------------------------------------
666// CurrentPathTracker — unchanged, still uses tokio::sync::RwLock because it
667// lives inside async path-update callbacks.
668// ---------------------------------------------------------------------------
669
670#[derive(Clone, Debug)]
671struct CurrentPathState {
672    path: String,
673    updated_at: Timestamp,
674}
675
676struct CurrentPathTracker {
677    state: Arc<RwLock<CurrentPathState>>,
678}
679
680impl CurrentPathTracker {
681    fn new(initial_path: String) -> Self {
682        Self::new_at(initial_path, Timestamp::now())
683    }
684
685    fn new_at(initial_path: String, updated_at: Timestamp) -> Self {
686        Self {
687            state: Arc::new(RwLock::new(CurrentPathState {
688                path: initial_path,
689                updated_at,
690            })),
691        }
692    }
693
694    async fn update_path(&self, path: String) {
695        let mut state = self.state.write().await;
696        state.path = path;
697        state.updated_at = Timestamp::now();
698    }
699
700    async fn get_state(&self) -> CurrentPathState {
701        self.state.read().await.clone()
702    }
703}
704
705fn chrono_to_jiff_timestamp(dt: DateTime<Utc>) -> Timestamp {
706    let seconds = dt.timestamp();
707    let nanoseconds = match i32::try_from(dt.timestamp_subsec_nanos()) {
708        Ok(nanoseconds) => nanoseconds,
709        Err(_) => {
710            return if seconds < 0 { Timestamp::MIN } else { Timestamp::MAX };
711        }
712    };
713
714    match Timestamp::new(seconds, nanoseconds) {
715        Ok(timestamp) => timestamp,
716        Err(_) => {
717            if seconds < 0 {
718                Timestamp::MIN
719            } else {
720                Timestamp::MAX
721            }
722        }
723    }
724}
725
726fn timestamp_elapsed_seconds_since(now: Timestamp, earlier: Timestamp) -> u64 {
727    let duration = now.duration_since(earlier);
728    if duration.is_negative() {
729        return 0;
730    }
731
732    u64::try_from(duration.as_secs()).unwrap_or(u64::MAX)
733}
734
735#[derive(Clone, Copy, Debug, Default)]
736struct ScannerDiskBucketScanState {
737    concurrency_limit: u64,
738    queued: u64,
739    active: u64,
740}
741
742type ScannerDiskBucketScanKey = (String, String);
743
744#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
745pub struct ScannerDiskBucketScanSnapshot {
746    pub pool: String,
747    pub set: String,
748    pub concurrency_limit: u64,
749    pub queued: u64,
750    pub active: u64,
751}
752
753#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
754struct ScannerBucketDriveResultKey {
755    bucket: String,
756    drive: String,
757    result: String,
758}
759
760impl ScannerBucketDriveResultKey {
761    fn new(bucket: impl Into<String>, drive: impl Into<String>, result: impl Into<String>) -> Self {
762        Self {
763            bucket: bucket.into(),
764            drive: drive.into(),
765            result: result.into(),
766        }
767    }
768}
769
770const MAX_SCANNER_BUCKET_DRIVE_RESULT_KEYS: usize = 4096;
771
772#[derive(Debug, Default)]
773struct ScannerBucketDriveResults {
774    counts: HashMap<ScannerBucketDriveResultKey, ScannerBucketDriveResultValue>,
775    eviction_index: BTreeSet<(u64, ScannerBucketDriveResultKey)>,
776}
777
778#[derive(Clone, Copy, Debug)]
779struct ScannerBucketDriveResultValue {
780    count: u64,
781    last_seen: u64,
782}
783
784#[derive(Clone, Debug, Eq, Hash, PartialEq)]
785struct ScannerActiveBucketDriveKey {
786    source: String,
787    bucket: String,
788    drive: String,
789}
790
791#[derive(Clone, Copy, Debug)]
792struct ScannerActiveBucketDriveValue {
793    count: u64,
794    started_at: Timestamp,
795}
796
797// ---------------------------------------------------------------------------
798// Metrics
799// ---------------------------------------------------------------------------
800
801/// A remote-tier request a warm backend issues on the cluster's behalf.
802///
803/// The set is closed on purpose: these values become metric labels, so a new
804/// operation is a deliberate schema change rather than something a call site
805/// can invent.
806#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]
807pub enum TierRequestOperation {
808    Put,
809    Get,
810    Remove,
811    Probe,
812    InUse,
813}
814
815impl TierRequestOperation {
816    pub const ALL: [Self; 5] = [Self::Put, Self::Get, Self::Remove, Self::Probe, Self::InUse];
817
818    pub const fn as_label(self) -> &'static str {
819        match self {
820            Self::Put => "put",
821            Self::Get => "get",
822            Self::Remove => "remove",
823            Self::Probe => "probe",
824            Self::InUse => "in_use",
825        }
826    }
827
828    const fn index(self) -> usize {
829        self as usize
830    }
831}
832
833/// How a remote-tier request ended.
834///
835/// This classifies the request only. A transition whose remote PUT succeeded
836/// and whose local commit then failed is a `Success` here: the remote service
837/// did perform the request, and reporting it as a tier failure would hide a
838/// leaked remote object behind an apparent backend outage.
839#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]
840pub enum TierRequestOutcome {
841    Success,
842    BackendError,
843    Timeout,
844    Cancelled,
845}
846
847impl TierRequestOutcome {
848    pub const ALL: [Self; 4] = [Self::Success, Self::BackendError, Self::Timeout, Self::Cancelled];
849
850    pub const fn as_label(self) -> &'static str {
851        match self {
852            Self::Success => "success",
853            Self::BackendError => "backend_error",
854            Self::Timeout => "timeout",
855            Self::Cancelled => "cancelled",
856        }
857    }
858
859    const fn index(self) -> usize {
860        self as usize
861    }
862
863    /// Classify a warm backend error without letting its message into a label.
864    pub fn from_error(err: &std::io::Error) -> Self {
865        match err.kind() {
866            std::io::ErrorKind::TimedOut => Self::Timeout,
867            std::io::ErrorKind::Interrupted => Self::Cancelled,
868            _ => Self::BackendError,
869        }
870    }
871}
872
873/// One fixed operation/outcome cell of the tier request counters.
874#[derive(Clone, Copy, Debug, Eq, PartialEq)]
875pub struct TierRequestCount {
876    pub operation: TierRequestOperation,
877    pub outcome: TierRequestOutcome,
878    pub count: u64,
879}
880
881const TIER_REQUEST_COUNTER_SLOTS: usize = TierRequestOperation::ALL.len() * TierRequestOutcome::ALL.len();
882
883pub struct Metrics {
884    operations: Vec<AtomicU64>,
885    latency: Vec<LockedLastMinuteLatency>,
886    actions: Vec<AtomicU64>,
887    actions_latency: Vec<LockedLastMinuteLatency>,
888    current_paths: Arc<RwLock<HashMap<String, Arc<CurrentPathTracker>>>>,
889    cycle_info: Arc<RwLock<Option<CurrentCycle>>>,
890    current_scan_mode: AtomicU8,
891    current_scan_cycle_work_active: AtomicBool,
892    current_scan_cycle_objects_start: AtomicU64,
893    current_scan_cycle_directories_start: AtomicU64,
894    current_scan_cycle_bucket_drive_scans_start: AtomicU64,
895    current_scan_cycle_bucket_drive_failures_start: AtomicU64,
896    current_scan_cycle_yield_events_start: AtomicU64,
897    current_scan_cycle_yield_duration_millis_start: AtomicU64,
898    current_scan_cycle_throttle_sleep_events_start: AtomicU64,
899    current_scan_cycle_throttle_sleep_duration_millis_start: AtomicU64,
900    current_scan_cycle_ilm_actions_start: AtomicU64,
901    current_scan_cycle_lifecycle_expiry_actions_start: AtomicU64,
902    current_scan_cycle_lifecycle_transition_actions_start: AtomicU64,
903    current_scan_cycle_heal_objects_start: AtomicU64,
904    current_scan_cycle_replication_checks_start: AtomicU64,
905    current_scan_cycle_usage_saves_start: AtomicU64,
906    scanner_set_scan_concurrency_limit: AtomicU64,
907    scanner_set_scans_queued: AtomicU64,
908    scanner_set_scans_active: AtomicU64,
909    scanner_disk_bucket_scan_states: Mutex<HashMap<ScannerDiskBucketScanKey, ScannerDiskBucketScanState>>,
910    scanner_bucket_drive_results: Mutex<ScannerBucketDriveResults>,
911    scanner_active_bucket_drive_scans: Mutex<HashMap<ScannerActiveBucketDriveKey, ScannerActiveBucketDriveValue>>,
912    scanner_bucket_drive_result_clock: AtomicU64,
913    current_scan_cycle_bucket_drive_results_start: Mutex<HashMap<ScannerBucketDriveResultKey, u64>>,
914    last_scan_cycle_bucket_drive_results: Mutex<Vec<ScannerBucketDriveResultSnapshot>>,
915    scanner_leader_lock_state: RwLock<String>,
916    scanner_leader_lock_held: AtomicBool,
917    scanner_leader_lock_last_error: RwLock<String>,
918    scanner_leader_lock_last_update_unix_secs: AtomicU64,
919    last_scan_cycle_result: AtomicU8,
920    last_scan_cycle_partial_reason: AtomicU8,
921    last_scan_cycle_partial_source: AtomicU8,
922    last_scan_cycle_end_unix_secs: AtomicU64,
923    last_scan_cycle_duration_millis: AtomicU64,
924    last_scan_cycle_objects_scanned: AtomicU64,
925    last_scan_cycle_directories_scanned: AtomicU64,
926    /// Lifetime count of object versions walked by the data scanner, counted
927    /// for every version regardless of whether any lifecycle rule applies.
928    /// This is the honest source for `rustfs_scanner_versions_scanned_total`;
929    /// ILM-checked versions are tracked separately on the `Lifecycle` source.
930    lifetime_versions_scanned: AtomicU64,
931    last_scan_cycle_bucket_drive_scans: AtomicU64,
932    last_scan_cycle_bucket_drive_failures: AtomicU64,
933    last_scan_cycle_yield_events: AtomicU64,
934    last_scan_cycle_yield_duration_millis: AtomicU64,
935    last_scan_cycle_throttle_sleep_events: AtomicU64,
936    last_scan_cycle_throttle_sleep_duration_millis: AtomicU64,
937    last_scan_cycle_ilm_actions: AtomicU64,
938    last_scan_cycle_lifecycle_expiry_actions: AtomicU64,
939    last_scan_cycle_lifecycle_transition_actions: AtomicU64,
940    last_scan_cycle_heal_objects: AtomicU64,
941    last_scan_cycle_replication_checks: AtomicU64,
942    last_scan_cycle_usage_saves: AtomicU64,
943    failed_scan_cycles: AtomicU64,
944    superseded_scan_cycles: AtomicU64,
945    partial_scan_cycles_unknown: AtomicU64,
946    partial_scan_cycles_runtime: AtomicU64,
947    partial_scan_cycles_objects: AtomicU64,
948    partial_scan_cycles_directories: AtomicU64,
949    partial_scan_cycles_by_source: Vec<AtomicU64>,
950    scanner_yield_duration_millis: AtomicU64,
951    scanner_throttle_sleep_duration_millis: AtomicU64,
952    scanner_ilm_actions: AtomicU64,
953    scanner_lifecycle_expiry_actions: AtomicU64,
954    scanner_lifecycle_transition_actions: AtomicU64,
955    scanner_expiry_queue_capacity: AtomicU64,
956    scanner_expiry_queued: AtomicU64,
957    scanner_expiry_active: AtomicU64,
958    scanner_expiry_workers: AtomicU64,
959    scanner_expiry_queue_missed: AtomicU64,
960    scanner_expiry_queued_total: AtomicU64,
961    scanner_expiry_missed_total: AtomicU64,
962    scanner_expiry_blocked_total: AtomicU64,
963    scanner_expiry_not_enqueued_total: AtomicU64,
964    scanner_expiry_delete_failed_total: AtomicU64,
965    scanner_transition_queue_capacity: AtomicU64,
966    scanner_transition_queued: AtomicU64,
967    scanner_transition_active: AtomicU64,
968    scanner_transition_workers: AtomicU64,
969    scanner_transition_queue_full: AtomicU64,
970    scanner_transition_queue_send_timeout: AtomicU64,
971    scanner_transition_compensation_scheduled: AtomicU64,
972    scanner_transition_compensation_pending: AtomicU64,
973    scanner_transition_compensation_running: AtomicU64,
974    scanner_transition_queued_total: AtomicU64,
975    scanner_transition_missed_total: AtomicU64,
976    scanner_transition_completed: AtomicU64,
977    /// Remote tier requests by operation and outcome. The width is fixed by
978    /// the two label enums, so the series count cannot grow with tier names,
979    /// endpoints or object keys.
980    tier_requests: [AtomicU64; TIER_REQUEST_COUNTER_SLOTS],
981    scanner_transition_failed: AtomicU64,
982    scanner_throttle_idle_mode_enabled: AtomicBool,
983    scanner_throttle_sleep_factor_micros: AtomicU64,
984    scanner_throttle_max_sleep_millis: AtomicU64,
985    scanner_yield_every_n_objects: AtomicU64,
986    scanner_cycle_interval_millis: AtomicU64,
987    scanner_cycle_max_duration_millis: AtomicU64,
988    scanner_cycle_max_objects: AtomicU64,
989    scanner_cycle_max_directories: AtomicU64,
990    scanner_cycle_timeout_total: AtomicU64,
991    scanner_cycle_recovery_required_total: AtomicU64,
992    scanner_cycle_last_progress_age_seconds: AtomicU64,
993    scanner_leader_lease_without_progress: AtomicBool,
994    scanner_bitrot_cycle_enabled: AtomicBool,
995    scanner_bitrot_cycle_millis: AtomicU64,
996    scanner_checkpoint: Mutex<Option<ScannerCheckpointReport>>,
997    scanner_checkpoint_used: AtomicU64,
998    scanner_checkpoint_cleared: AtomicU64,
999    scanner_checkpoint_ignored: AtomicU64,
1000    scanner_checkpoint_stale: AtomicU64,
1001    scanner_dirty_usage_pending_buckets: AtomicU64,
1002    scanner_dirty_usage_last_mark_unix_secs: AtomicU64,
1003    scanner_dirty_usage_last_clear_unix_secs: AtomicU64,
1004    scanner_dirty_usage_last_cycle_dirty_buckets: AtomicU64,
1005    scanner_dirty_usage_last_cycle_cleared_buckets: AtomicU64,
1006    scanner_usage_last_save_unix_secs: AtomicU64,
1007    scanner_usage_last_durable_success_unix_secs: AtomicU64,
1008    scanner_usage_last_publication_unix_secs: AtomicU64,
1009    scanner_usage_last_publication_state: Mutex<String>,
1010    scanner_usage_last_publication_reason: Mutex<String>,
1011    scanner_usage_last_save_result: AtomicU8,
1012    scanner_usage_deferred_pending: AtomicBool,
1013    scanner_usage_deferred_total: AtomicU64,
1014    scanner_usage_last_deferred_unix_secs: AtomicU64,
1015    scanner_usage_last_deferred_reason: Mutex<String>,
1016    scanner_source_work: Vec<ScannerSourceWorkCounters>,
1017    current_scan_cycle_source_work_start: Vec<ScannerSourceWorkCounters>,
1018    last_scan_cycle_source_work: Vec<ScannerSourceWorkCounters>,
1019    scanner_replication_repair_work: Vec<ScannerSourceWorkCounters>,
1020    current_scan_cycle_replication_repair_work_start: Vec<ScannerSourceWorkCounters>,
1021    last_scan_cycle_replication_repair_work: Vec<ScannerSourceWorkCounters>,
1022    partial_scan_cycles: AtomicU64,
1023}
1024
1025const SCAN_CYCLE_RESULT_UNKNOWN: u8 = 0;
1026const SCAN_CYCLE_RESULT_SUCCESS: u8 = 1;
1027const SCAN_CYCLE_RESULT_ERROR: u8 = 2;
1028const SCAN_CYCLE_RESULT_PARTIAL: u8 = 3;
1029const SCAN_CYCLE_RESULT_SUPERSEDED: u8 = 4;
1030const SCAN_CYCLE_RESULT_DEFERRED: u8 = 5;
1031const SCAN_CYCLE_RESULT_UNKNOWN_LABEL: &str = "unknown";
1032const SCAN_CYCLE_RESULT_SUCCESS_LABEL: &str = "success";
1033const SCAN_CYCLE_RESULT_ERROR_LABEL: &str = "error";
1034const SCAN_CYCLE_RESULT_PARTIAL_LABEL: &str = "partial";
1035const SCAN_CYCLE_RESULT_SUPERSEDED_LABEL: &str = "superseded";
1036const SCAN_CYCLE_RESULT_DEFERRED_LABEL: &str = "deferred";
1037
1038#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
1039pub enum ScanCyclePartialReason {
1040    #[default]
1041    Unknown = 0,
1042    Runtime = 1,
1043    Objects = 2,
1044    Directories = 3,
1045}
1046
1047impl ScanCyclePartialReason {
1048    pub fn as_str(self) -> &'static str {
1049        match self {
1050            Self::Unknown => SCAN_CYCLE_RESULT_UNKNOWN_LABEL,
1051            Self::Runtime => "runtime",
1052            Self::Objects => "objects",
1053            Self::Directories => "directories",
1054        }
1055    }
1056
1057    fn from_code(code: u8) -> Self {
1058        match code {
1059            1 => Self::Runtime,
1060            2 => Self::Objects,
1061            3 => Self::Directories,
1062            _ => Self::Unknown,
1063        }
1064    }
1065}
1066
1067#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
1068pub enum ScannerUsageSaveResult {
1069    #[default]
1070    Unknown = 0,
1071    Success = 1,
1072    Failed = 2,
1073    SkippedStale = 3,
1074    EncodeFailed = 4,
1075}
1076
1077impl ScannerUsageSaveResult {
1078    pub fn as_str(self) -> &'static str {
1079        match self {
1080            Self::Unknown => SCAN_CYCLE_RESULT_UNKNOWN_LABEL,
1081            Self::Success => SCAN_CYCLE_RESULT_SUCCESS_LABEL,
1082            Self::Failed => "failed",
1083            Self::SkippedStale => "skipped_stale",
1084            Self::EncodeFailed => "encode_failed",
1085        }
1086    }
1087
1088    fn from_code(code: u8) -> Self {
1089        match code {
1090            1 => Self::Success,
1091            2 => Self::Failed,
1092            3 => Self::SkippedStale,
1093            4 => Self::EncodeFailed,
1094            _ => Self::Unknown,
1095        }
1096    }
1097}
1098
1099#[derive(Clone, Debug, Default, Serialize, Deserialize)]
1100pub struct CurrentCycle {
1101    pub current: u64,
1102    pub next: u64,
1103    pub cycle_completed: Vec<DateTime<Utc>>,
1104    pub started: DateTime<Utc>,
1105}
1106
1107#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
1108pub struct ScanCycleWorkSnapshot {
1109    objects_scanned: u64,
1110    directories_scanned: u64,
1111    bucket_drive_scans: u64,
1112    bucket_drive_failures: u64,
1113    yield_events: u64,
1114    yield_duration_millis: u64,
1115    throttle_sleep_events: u64,
1116    throttle_sleep_duration_millis: u64,
1117    ilm_actions: u64,
1118    lifecycle_expiry_actions: u64,
1119    lifecycle_transition_actions: u64,
1120    heal_objects: u64,
1121    replication_checks: u64,
1122    usage_saves: u64,
1123}
1124
1125#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
1126pub struct ScannerTimedAction {
1127    pub count: u64,
1128    pub acc_time: u64,
1129    pub bytes: u64,
1130}
1131
1132#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
1133pub struct ScannerCheckpointReport {
1134    pub version: u16,
1135    pub resume_after: String,
1136    pub reason: String,
1137    pub last_event: String,
1138}
1139
1140#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
1141pub struct ScannerSourceWorkSnapshot {
1142    pub source: String,
1143    pub checked: u64,
1144    pub queued: u64,
1145    pub executed: u64,
1146    pub failed: u64,
1147    pub skipped: u64,
1148    #[serde(default)]
1149    pub missed: u64,
1150}
1151
1152#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
1153pub struct ScannerBucketDriveResultSnapshot {
1154    pub bucket: String,
1155    pub drive: String,
1156    pub result: String,
1157    pub count: u64,
1158}
1159
1160#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
1161pub struct ScannerActiveBucketDriveSnapshot {
1162    pub source: String,
1163    pub bucket: String,
1164    pub drive: String,
1165    pub count: u64,
1166    pub age_seconds: u64,
1167}
1168
1169#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
1170pub struct ScannerReplicationRepairSnapshot {
1171    pub source: String,
1172    pub kind: String,
1173    #[serde(default)]
1174    pub scanner_role: String,
1175    #[serde(default)]
1176    pub execution_owner: String,
1177    pub checked: u64,
1178    pub queued: u64,
1179    pub executed: u64,
1180    pub failed: u64,
1181    pub skipped: u64,
1182    #[serde(default)]
1183    pub missed: u64,
1184}
1185
1186#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
1187pub struct ScannerMaintenanceSourceSnapshot {
1188    pub source: String,
1189    pub state: String,
1190    pub reason: String,
1191    pub backlog: u64,
1192    pub current_checked: u64,
1193    pub current_queued: u64,
1194    pub current_missed: u64,
1195    pub lifetime_missed: u64,
1196    pub partial_cycles: u64,
1197}
1198
1199#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
1200pub struct ScannerMaintenanceControlSnapshot {
1201    pub primary_control: String,
1202    pub sources: Vec<ScannerMaintenanceSourceSnapshot>,
1203}
1204
1205impl Default for ScannerMaintenanceControlSnapshot {
1206    fn default() -> Self {
1207        Self {
1208            primary_control: SCANNER_MAINTENANCE_CONTROL_NONE.to_string(),
1209            sources: Vec::new(),
1210        }
1211    }
1212}
1213
1214#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
1215pub struct ScannerSourceCycleSnapshot {
1216    pub source: String,
1217    pub cycles: u64,
1218}
1219
1220#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
1221pub struct ScannerPacingPressureSnapshot {
1222    pub primary_pressure: String,
1223    pub current_queued_scans: u64,
1224    pub current_active_scans: u64,
1225    pub last_cycle_budget_limited: bool,
1226    pub last_cycle_pause_observed: bool,
1227    pub last_cycle_throttle_sleep_ratio: f64,
1228    pub last_cycle_yield_ratio: f64,
1229    pub last_cycle_total_pause_ratio: f64,
1230}
1231
1232impl Default for ScannerPacingPressureSnapshot {
1233    fn default() -> Self {
1234        Self {
1235            primary_pressure: "none".to_string(),
1236            current_queued_scans: 0,
1237            current_active_scans: 0,
1238            last_cycle_budget_limited: false,
1239            last_cycle_pause_observed: false,
1240            last_cycle_throttle_sleep_ratio: 0.0,
1241            last_cycle_yield_ratio: 0.0,
1242            last_cycle_total_pause_ratio: 0.0,
1243        }
1244    }
1245}
1246
1247#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
1248pub struct ScannerLifecycleExpiryStateUpdate {
1249    pub queue_capacity: u64,
1250    pub queued: u64,
1251    pub active: u64,
1252    pub workers: u64,
1253    pub queue_missed: u64,
1254}
1255
1256#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
1257pub struct ScannerLifecycleExpirySnapshot {
1258    pub current_queue_capacity: u64,
1259    pub current_queued: u64,
1260    pub current_active: u64,
1261    pub current_workers: u64,
1262    pub queue_missed: u64,
1263    pub scanner_queued: u64,
1264    pub scanner_missed: u64,
1265    #[serde(default)]
1266    pub scanner_blocked: u64,
1267    #[serde(default)]
1268    pub scanner_not_enqueued: u64,
1269    #[serde(default)]
1270    pub delete_failed: u64,
1271}
1272
1273#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
1274pub struct ScannerLifecycleTransitionStateUpdate {
1275    pub queue_capacity: u64,
1276    pub queued: u64,
1277    pub active: u64,
1278    pub workers: u64,
1279    pub queue_full: u64,
1280    pub queue_send_timeout: u64,
1281    pub compensation_scheduled: u64,
1282    pub compensation_pending: u64,
1283    pub compensation_running: u64,
1284}
1285
1286#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
1287pub struct ScannerLifecycleTransitionSnapshot {
1288    pub current_queue_capacity: u64,
1289    pub current_queued: u64,
1290    pub current_active: u64,
1291    pub current_workers: u64,
1292    pub queue_full: u64,
1293    pub queue_send_timeout: u64,
1294    pub compensation_scheduled: u64,
1295    pub compensation_pending: u64,
1296    pub compensation_running: u64,
1297    pub scanner_queued: u64,
1298    pub scanner_missed: u64,
1299    pub completed: u64,
1300    pub failed: u64,
1301}
1302
1303#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
1304pub struct ScannerUsageFreshnessSnapshot {
1305    pub dirty_pending_buckets: u64,
1306    pub last_dirty_mark_unix_secs: u64,
1307    pub last_dirty_clear_unix_secs: u64,
1308    pub last_cycle_dirty_buckets: u64,
1309    pub last_cycle_cleared_dirty_buckets: u64,
1310    pub last_usage_save_unix_secs: u64,
1311    pub last_usage_save_result: String,
1312    pub last_usage_save_result_code: u64,
1313    #[serde(default)]
1314    pub last_durable_success_unix_secs: u64,
1315    #[serde(default)]
1316    pub last_publication_unix_secs: u64,
1317    #[serde(default)]
1318    pub last_publication_state: String,
1319    #[serde(default)]
1320    pub last_publication_reason: String,
1321    #[serde(default)]
1322    pub deferred_pending: bool,
1323    #[serde(default)]
1324    pub deferred_total: u64,
1325    #[serde(default)]
1326    pub last_deferred_unix_secs: u64,
1327    #[serde(default)]
1328    pub last_deferred_reason: String,
1329}
1330
1331#[derive(Clone, Debug, Default, Serialize, Deserialize)]
1332pub struct ScannerLastMinute {
1333    pub actions: HashMap<String, ScannerTimedAction>,
1334    pub ilm: HashMap<String, ScannerTimedAction>,
1335}
1336
1337#[derive(Clone, Debug, Default, Serialize, Deserialize)]
1338pub struct ScannerMetricsReport {
1339    pub collected_at: Timestamp,
1340    pub current_cycle: u64,
1341    #[serde(default)]
1342    pub current_cycle_active: bool,
1343    pub current_started: Timestamp,
1344    pub cycles_completed_at: Vec<Timestamp>,
1345    pub ongoing_buckets: usize,
1346    #[serde(default)]
1347    pub active_scan_paths: usize,
1348    #[serde(default)]
1349    pub oldest_active_path_age_seconds: u64,
1350    pub life_time_ops: HashMap<String, u64>,
1351    pub life_time_ilm: HashMap<String, u64>,
1352    /// Lifetime object versions scanned, independent of ILM configuration.
1353    #[serde(default)]
1354    pub versions_scanned: u64,
1355    pub last_minute: ScannerLastMinute,
1356    pub active_paths: Vec<String>,
1357    pub current_scan_mode: String,
1358    #[serde(default)]
1359    pub leader_lock_state: String,
1360    #[serde(default)]
1361    pub leader_lock_held_by_this_process: bool,
1362    #[serde(default)]
1363    pub leader_lock_last_error: String,
1364    #[serde(default)]
1365    pub leader_lock_last_update_unix_secs: u64,
1366    #[serde(default)]
1367    pub last_cycle_end_unix_secs: u64,
1368    #[serde(default)]
1369    pub current_set_scan_concurrency_limit: u64,
1370    #[serde(default)]
1371    pub current_set_scans_queued: u64,
1372    #[serde(default)]
1373    pub current_set_scans_active: u64,
1374    #[serde(default)]
1375    pub current_disk_scan_concurrency_limit: u64,
1376    #[serde(default)]
1377    pub current_disk_bucket_scans_queued: u64,
1378    #[serde(default)]
1379    pub current_disk_bucket_scans_active: u64,
1380    #[serde(default)]
1381    pub current_cycle_objects_scanned: u64,
1382    #[serde(default)]
1383    pub current_cycle_directories_scanned: u64,
1384    #[serde(default)]
1385    pub current_cycle_bucket_drive_scans: u64,
1386    #[serde(default)]
1387    pub current_cycle_bucket_drive_failures: u64,
1388    #[serde(default)]
1389    pub current_cycle_yield_events: u64,
1390    #[serde(default)]
1391    pub current_cycle_yield_duration_seconds: f64,
1392    #[serde(default)]
1393    pub current_cycle_throttle_sleep_events: u64,
1394    #[serde(default)]
1395    pub current_cycle_throttle_sleep_duration_seconds: f64,
1396    #[serde(default)]
1397    pub current_cycle_ilm_actions: u64,
1398    #[serde(default)]
1399    pub current_cycle_lifecycle_expiry_actions: u64,
1400    #[serde(default)]
1401    pub current_cycle_lifecycle_transition_actions: u64,
1402    #[serde(default)]
1403    pub current_cycle_heal_objects: u64,
1404    #[serde(default)]
1405    pub current_cycle_replication_checks: u64,
1406    #[serde(default)]
1407    pub current_cycle_usage_saves: u64,
1408    pub last_cycle_result: String,
1409    pub last_cycle_result_code: u64,
1410    #[serde(default)]
1411    pub last_cycle_partial_reason: String,
1412    #[serde(default)]
1413    pub last_cycle_partial_reason_code: u64,
1414    #[serde(default)]
1415    pub last_cycle_partial_source: String,
1416    #[serde(default)]
1417    pub last_cycle_partial_source_code: u64,
1418    pub last_cycle_duration_seconds: f64,
1419    #[serde(default)]
1420    pub last_cycle_objects_scanned: u64,
1421    #[serde(default)]
1422    pub last_cycle_directories_scanned: u64,
1423    #[serde(default)]
1424    pub last_cycle_bucket_drive_scans: u64,
1425    #[serde(default)]
1426    pub last_cycle_bucket_drive_failures: u64,
1427    #[serde(default)]
1428    pub last_cycle_yield_events: u64,
1429    #[serde(default)]
1430    pub last_cycle_yield_duration_seconds: f64,
1431    #[serde(default)]
1432    pub last_cycle_throttle_sleep_events: u64,
1433    #[serde(default)]
1434    pub last_cycle_throttle_sleep_duration_seconds: f64,
1435    #[serde(default)]
1436    pub last_cycle_ilm_actions: u64,
1437    #[serde(default)]
1438    pub last_cycle_lifecycle_expiry_actions: u64,
1439    #[serde(default)]
1440    pub last_cycle_lifecycle_transition_actions: u64,
1441    #[serde(default)]
1442    pub last_cycle_heal_objects: u64,
1443    #[serde(default)]
1444    pub last_cycle_replication_checks: u64,
1445    #[serde(default)]
1446    pub last_cycle_usage_saves: u64,
1447    pub failed_cycles: u64,
1448    #[serde(default)]
1449    pub superseded_cycles: u64,
1450    #[serde(default)]
1451    pub partial_cycles_unknown: u64,
1452    #[serde(default)]
1453    pub partial_cycles_runtime: u64,
1454    #[serde(default)]
1455    pub partial_cycles_objects: u64,
1456    #[serde(default)]
1457    pub partial_cycles_directories: u64,
1458    #[serde(default)]
1459    pub partial_cycles_by_source: Vec<ScannerSourceCycleSnapshot>,
1460    #[serde(default)]
1461    pub pacing_pressure: ScannerPacingPressureSnapshot,
1462    #[serde(default)]
1463    pub lifecycle_expiry: ScannerLifecycleExpirySnapshot,
1464    #[serde(default)]
1465    pub lifecycle_transition: ScannerLifecycleTransitionSnapshot,
1466    #[serde(default)]
1467    pub usage_freshness: ScannerUsageFreshnessSnapshot,
1468    #[serde(default)]
1469    pub maintenance_control: ScannerMaintenanceControlSnapshot,
1470    #[serde(default)]
1471    pub throttle_idle_mode_enabled: bool,
1472    #[serde(default)]
1473    pub throttle_sleep_factor: f64,
1474    #[serde(default)]
1475    pub throttle_max_sleep_seconds: f64,
1476    #[serde(default)]
1477    pub yield_every_n_objects: u64,
1478    #[serde(default)]
1479    pub cycle_interval_seconds: f64,
1480    #[serde(default)]
1481    pub cycle_max_duration_seconds: f64,
1482    #[serde(default)]
1483    pub cycle_max_objects: u64,
1484    #[serde(default)]
1485    pub cycle_max_directories: u64,
1486    #[serde(default)]
1487    pub cycle_timeout_total: u64,
1488    #[serde(default)]
1489    pub cycle_recovery_required_total: u64,
1490    #[serde(default)]
1491    pub cycle_last_progress_age: u64,
1492    #[serde(default)]
1493    pub leader_lease_without_progress: bool,
1494    #[serde(default)]
1495    pub bitrot_cycle_enabled: bool,
1496    #[serde(default)]
1497    pub bitrot_cycle_seconds: f64,
1498    #[serde(default)]
1499    #[serde(skip_serializing_if = "Option::is_none")]
1500    pub scan_checkpoint: Option<ScannerCheckpointReport>,
1501    #[serde(default)]
1502    pub scan_checkpoint_used: u64,
1503    #[serde(default)]
1504    pub scan_checkpoint_cleared: u64,
1505    #[serde(default)]
1506    pub scan_checkpoint_ignored: u64,
1507    #[serde(default)]
1508    pub scan_checkpoint_stale: u64,
1509    #[serde(default)]
1510    pub source_work: Vec<ScannerSourceWorkSnapshot>,
1511    #[serde(default)]
1512    pub current_cycle_source_work: Vec<ScannerSourceWorkSnapshot>,
1513    #[serde(default)]
1514    pub last_cycle_source_work: Vec<ScannerSourceWorkSnapshot>,
1515    #[serde(default)]
1516    pub replication_repair: Vec<ScannerReplicationRepairSnapshot>,
1517    #[serde(default)]
1518    pub current_cycle_replication_repair: Vec<ScannerReplicationRepairSnapshot>,
1519    #[serde(default)]
1520    pub last_cycle_replication_repair: Vec<ScannerReplicationRepairSnapshot>,
1521    #[serde(default)]
1522    pub partial_cycles: u64,
1523}
1524
1525#[derive(Clone, Debug, Default, Serialize, Deserialize)]
1526pub struct ScannerRuntimeDetailsReport {
1527    #[serde(default)]
1528    pub disk_bucket_scan_states: Vec<ScannerDiskBucketScanSnapshot>,
1529    #[serde(default)]
1530    pub bucket_drive_results: Vec<ScannerBucketDriveResultSnapshot>,
1531    #[serde(default)]
1532    pub current_cycle_bucket_drive_results: Vec<ScannerBucketDriveResultSnapshot>,
1533    #[serde(default)]
1534    pub last_cycle_bucket_drive_results: Vec<ScannerBucketDriveResultSnapshot>,
1535    #[serde(default)]
1536    pub active_bucket_drive_scans: Vec<ScannerActiveBucketDriveSnapshot>,
1537}
1538
1539impl CurrentCycle {
1540    pub fn unmarshal(&mut self, buf: &[u8]) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
1541        *self = rmp_serde::from_slice(buf)?;
1542        Ok(())
1543    }
1544
1545    pub fn marshal(&self) -> Result<Vec<u8>, Box<dyn std::error::Error + Send + Sync>> {
1546        Ok(rmp_serde::to_vec(self)?)
1547    }
1548}
1549
1550/// OTEL metric name constants for scanner metrics
1551const OTEL_SCANNER_BUCKETS_SCANNED: &str = "rustfs_scanner_buckets_scanned_total";
1552const OTEL_SCANNER_CYCLES: &str = "rustfs_scanner_cycles_total";
1553const OTEL_SCANNER_CYCLE_DURATION_SECONDS: &str = "rustfs_scanner_cycle_duration_seconds";
1554const OTEL_SCANNER_BUCKET_DRIVE_DURATION_SECONDS: &str = "rustfs_scanner_bucket_drive_duration_seconds";
1555const OTEL_SCANNER_CYCLE_TIMEOUT_TOTAL: &str = "rustfs_scanner_cycle_timeout_total";
1556const OTEL_SCANNER_CYCLE_LAST_PROGRESS_AGE: &str = "rustfs_scanner_cycle_last_progress_age";
1557const OTEL_SCANNER_LEADER_LEASE_WITHOUT_PROGRESS: &str = "rustfs_scanner_leader_lease_without_progress";
1558
1559fn scan_cycle_result_label(result: u8) -> &'static str {
1560    match result {
1561        SCAN_CYCLE_RESULT_SUCCESS => SCAN_CYCLE_RESULT_SUCCESS_LABEL,
1562        SCAN_CYCLE_RESULT_ERROR => SCAN_CYCLE_RESULT_ERROR_LABEL,
1563        SCAN_CYCLE_RESULT_PARTIAL => SCAN_CYCLE_RESULT_PARTIAL_LABEL,
1564        SCAN_CYCLE_RESULT_SUPERSEDED => SCAN_CYCLE_RESULT_SUPERSEDED_LABEL,
1565        SCAN_CYCLE_RESULT_DEFERRED => SCAN_CYCLE_RESULT_DEFERRED_LABEL,
1566        _ => SCAN_CYCLE_RESULT_UNKNOWN_LABEL,
1567    }
1568}
1569
1570fn scanner_ratio(numerator: f64, denominator: f64) -> f64 {
1571    if denominator <= 0.0 {
1572        0.0
1573    } else {
1574        (numerator / denominator).clamp(0.0, 1.0)
1575    }
1576}
1577
1578fn unix_now_secs() -> u64 {
1579    SystemTime::now()
1580        .duration_since(SystemTime::UNIX_EPOCH)
1581        .unwrap_or_default()
1582        .as_secs()
1583}
1584
1585fn scanner_last_cycle_budget_limited(result_code: u64, partial_reason: &str) -> bool {
1586    result_code == u64::from(SCAN_CYCLE_RESULT_PARTIAL) && matches!(partial_reason, "runtime" | "objects" | "directories")
1587}
1588
1589fn scanner_primary_pressure(
1590    current_queued_scans: u64,
1591    current_active_scans: u64,
1592    last_cycle_budget_limited: bool,
1593    last_cycle_throttle_sleep_events: u64,
1594    last_cycle_yield_events: u64,
1595) -> &'static str {
1596    if current_queued_scans > 0 {
1597        "queued_scans"
1598    } else if last_cycle_budget_limited {
1599        "cycle_budget"
1600    } else if last_cycle_throttle_sleep_events > 0 || last_cycle_yield_events > 0 {
1601        "throttle_pause"
1602    } else if current_active_scans > 0 {
1603        "active_scans"
1604    } else {
1605        "none"
1606    }
1607}
1608
1609fn scanner_pacing_pressure(metrics: &ScannerMetricsReport) -> ScannerPacingPressureSnapshot {
1610    let current_queued_scans = metrics
1611        .current_set_scans_queued
1612        .saturating_add(metrics.current_disk_bucket_scans_queued);
1613    let current_active_scans = metrics
1614        .current_set_scans_active
1615        .saturating_add(metrics.current_disk_bucket_scans_active)
1616        .max(usize_to_u64_saturated(metrics.active_scan_paths));
1617    let last_cycle_budget_limited =
1618        scanner_last_cycle_budget_limited(metrics.last_cycle_result_code, metrics.last_cycle_partial_reason.as_str());
1619    let last_cycle_pause_observed = metrics.last_cycle_throttle_sleep_events > 0 || metrics.last_cycle_yield_events > 0;
1620    let last_cycle_throttle_sleep_ratio =
1621        scanner_ratio(metrics.last_cycle_throttle_sleep_duration_seconds, metrics.last_cycle_duration_seconds);
1622    let last_cycle_yield_ratio = scanner_ratio(metrics.last_cycle_yield_duration_seconds, metrics.last_cycle_duration_seconds);
1623    let last_cycle_total_pause_ratio = scanner_ratio(
1624        metrics.last_cycle_throttle_sleep_duration_seconds.max(0.0) + metrics.last_cycle_yield_duration_seconds.max(0.0),
1625        metrics.last_cycle_duration_seconds,
1626    );
1627
1628    ScannerPacingPressureSnapshot {
1629        primary_pressure: scanner_primary_pressure(
1630            current_queued_scans,
1631            current_active_scans,
1632            last_cycle_budget_limited,
1633            metrics.last_cycle_throttle_sleep_events,
1634            metrics.last_cycle_yield_events,
1635        )
1636        .to_string(),
1637        current_queued_scans,
1638        current_active_scans,
1639        last_cycle_budget_limited,
1640        last_cycle_pause_observed,
1641        last_cycle_throttle_sleep_ratio,
1642        last_cycle_yield_ratio,
1643        last_cycle_total_pause_ratio,
1644    }
1645}
1646
1647const SCANNER_MAINTENANCE_CONTROL_NONE: &str = "none";
1648const SCANNER_MAINTENANCE_CONTROL_BLOCKED_SOURCE: &str = "blocked_source";
1649const SCANNER_MAINTENANCE_CONTROL_DEFERRED_SOURCE: &str = "deferred_source";
1650const SCANNER_MAINTENANCE_CONTROL_ACTIVE_SOURCE: &str = "active_source";
1651const SCANNER_MAINTENANCE_CONTROL_PACING_PRESSURE: &str = "pacing_pressure";
1652
1653const SCANNER_MAINTENANCE_STATE_IDLE: &str = "idle";
1654const SCANNER_MAINTENANCE_STATE_ACTIVE: &str = "active";
1655const SCANNER_MAINTENANCE_STATE_DEFERRED: &str = "deferred";
1656const SCANNER_MAINTENANCE_STATE_BLOCKED: &str = "blocked";
1657
1658const SCANNER_MAINTENANCE_REASON_IDLE: &str = "idle";
1659const SCANNER_MAINTENANCE_REASON_ACTIVE_WORK: &str = "active_work";
1660const SCANNER_MAINTENANCE_REASON_QUEUED_WORK: &str = "queued_work";
1661const SCANNER_MAINTENANCE_REASON_PARTIAL_CYCLE: &str = "partial_cycle";
1662const SCANNER_MAINTENANCE_REASON_MISSED_WORK: &str = "missed_work";
1663const SCANNER_MAINTENANCE_REASON_EXPIRY_QUEUE_BACKLOG: &str = "expiry_queue_backlog";
1664const SCANNER_MAINTENANCE_REASON_TRANSITION_FAILED: &str = "transition_failed";
1665const SCANNER_MAINTENANCE_REASON_TRANSITION_COMPENSATION_BACKLOG: &str = "transition_compensation_backlog";
1666const SCANNER_MAINTENANCE_REASON_TRANSITION_QUEUE_BACKLOG: &str = "transition_queue_backlog";
1667const SCANNER_MAINTENANCE_REASON_TRANSITION_QUEUE_FULL: &str = "transition_queue_full";
1668
1669fn scanner_source_work_snapshot(work: &[ScannerSourceWorkSnapshot], source: ScannerWorkSource) -> ScannerSourceWorkSnapshot {
1670    work.iter()
1671        .find(|snapshot| snapshot.source == source.as_str())
1672        .cloned()
1673        .unwrap_or_else(|| ScannerSourceWorkSnapshot {
1674            source: source.as_str().to_string(),
1675            ..Default::default()
1676        })
1677}
1678
1679fn scanner_source_partial_cycles(metrics: &ScannerMetricsReport, source: ScannerWorkSource) -> u64 {
1680    metrics
1681        .partial_cycles_by_source
1682        .iter()
1683        .find(|snapshot| snapshot.source == source.as_str())
1684        .map(|snapshot| snapshot.cycles)
1685        .unwrap_or_default()
1686}
1687
1688fn scanner_source_has_current_work(work: &ScannerSourceWorkSnapshot) -> bool {
1689    work.checked > 0 || work.queued > 0 || work.executed > 0 || work.failed > 0 || work.skipped > 0 || work.missed > 0
1690}
1691
1692fn scanner_lifecycle_transition_backlog(metrics: &ScannerMetricsReport) -> u64 {
1693    metrics
1694        .lifecycle_transition
1695        .current_queued
1696        .saturating_add(metrics.lifecycle_transition.current_active)
1697        .saturating_add(metrics.lifecycle_transition.compensation_pending)
1698        .saturating_add(metrics.lifecycle_transition.compensation_running)
1699}
1700
1701fn scanner_lifecycle_expiry_backlog(metrics: &ScannerMetricsReport) -> u64 {
1702    metrics
1703        .lifecycle_expiry
1704        .current_queued
1705        .saturating_add(metrics.lifecycle_expiry.current_active)
1706}
1707
1708fn scanner_source_maintenance_state(
1709    source: ScannerWorkSource,
1710    current: &ScannerSourceWorkSnapshot,
1711    metrics: &ScannerMetricsReport,
1712) -> (&'static str, &'static str, u64) {
1713    if current.missed > 0 {
1714        return (SCANNER_MAINTENANCE_STATE_BLOCKED, SCANNER_MAINTENANCE_REASON_MISSED_WORK, current.missed);
1715    }
1716
1717    if source == ScannerWorkSource::Lifecycle && current.failed > 0 {
1718        return (
1719            SCANNER_MAINTENANCE_STATE_BLOCKED,
1720            SCANNER_MAINTENANCE_REASON_TRANSITION_FAILED,
1721            current.failed,
1722        );
1723    }
1724
1725    if source == ScannerWorkSource::Lifecycle {
1726        let transition_backlog = scanner_lifecycle_transition_backlog(metrics);
1727        if metrics.lifecycle_transition.current_queue_capacity > 0
1728            && metrics.lifecycle_transition.current_queued >= metrics.lifecycle_transition.current_queue_capacity
1729        {
1730            return (
1731                SCANNER_MAINTENANCE_STATE_BLOCKED,
1732                SCANNER_MAINTENANCE_REASON_TRANSITION_QUEUE_FULL,
1733                transition_backlog,
1734            );
1735        }
1736    }
1737
1738    if metrics.last_cycle_partial_source == source.as_str() && metrics.pacing_pressure.last_cycle_budget_limited {
1739        return (SCANNER_MAINTENANCE_STATE_DEFERRED, SCANNER_MAINTENANCE_REASON_PARTIAL_CYCLE, 0);
1740    }
1741
1742    if current.queued > 0 {
1743        return (SCANNER_MAINTENANCE_STATE_ACTIVE, SCANNER_MAINTENANCE_REASON_QUEUED_WORK, current.queued);
1744    }
1745
1746    if source == ScannerWorkSource::Lifecycle {
1747        let expiry_backlog = scanner_lifecycle_expiry_backlog(metrics);
1748        if expiry_backlog > 0 {
1749            return (
1750                SCANNER_MAINTENANCE_STATE_ACTIVE,
1751                SCANNER_MAINTENANCE_REASON_EXPIRY_QUEUE_BACKLOG,
1752                expiry_backlog,
1753            );
1754        }
1755
1756        let compensation_backlog = metrics
1757            .lifecycle_transition
1758            .compensation_pending
1759            .saturating_add(metrics.lifecycle_transition.compensation_running);
1760        if compensation_backlog > 0 {
1761            return (
1762                SCANNER_MAINTENANCE_STATE_ACTIVE,
1763                SCANNER_MAINTENANCE_REASON_TRANSITION_COMPENSATION_BACKLOG,
1764                compensation_backlog,
1765            );
1766        }
1767
1768        let transition_backlog = scanner_lifecycle_transition_backlog(metrics);
1769        if transition_backlog > 0 {
1770            return (
1771                SCANNER_MAINTENANCE_STATE_ACTIVE,
1772                SCANNER_MAINTENANCE_REASON_TRANSITION_QUEUE_BACKLOG,
1773                transition_backlog,
1774            );
1775        }
1776    }
1777
1778    if scanner_source_has_current_work(current) {
1779        return (SCANNER_MAINTENANCE_STATE_ACTIVE, SCANNER_MAINTENANCE_REASON_ACTIVE_WORK, 0);
1780    }
1781
1782    (SCANNER_MAINTENANCE_STATE_IDLE, SCANNER_MAINTENANCE_REASON_IDLE, 0)
1783}
1784
1785fn scanner_maintenance_source_work(metrics: &ScannerMetricsReport) -> &[ScannerSourceWorkSnapshot] {
1786    if metrics.current_cycle_source_work.is_empty() {
1787        &metrics.last_cycle_source_work
1788    } else {
1789        &metrics.current_cycle_source_work
1790    }
1791}
1792
1793fn scanner_maintenance_control(metrics: &ScannerMetricsReport) -> ScannerMaintenanceControlSnapshot {
1794    let mut has_blocked = false;
1795    let mut has_deferred = false;
1796    let mut has_active = false;
1797    let current_source_work = scanner_maintenance_source_work(metrics);
1798
1799    let sources = ScannerWorkSource::all()
1800        .iter()
1801        .map(|source| {
1802            let current = scanner_source_work_snapshot(current_source_work, *source);
1803            let lifetime = scanner_source_work_snapshot(&metrics.source_work, *source);
1804            let partial_cycles = scanner_source_partial_cycles(metrics, *source);
1805            let (state, reason, backlog) = scanner_source_maintenance_state(*source, &current, metrics);
1806
1807            match state {
1808                SCANNER_MAINTENANCE_STATE_BLOCKED => has_blocked = true,
1809                SCANNER_MAINTENANCE_STATE_DEFERRED => has_deferred = true,
1810                SCANNER_MAINTENANCE_STATE_ACTIVE => has_active = true,
1811                _ => {}
1812            }
1813
1814            ScannerMaintenanceSourceSnapshot {
1815                source: source.as_str().to_string(),
1816                state: state.to_string(),
1817                reason: reason.to_string(),
1818                backlog,
1819                current_checked: current.checked,
1820                current_queued: current.queued,
1821                current_missed: current.missed,
1822                lifetime_missed: lifetime.missed,
1823                partial_cycles,
1824            }
1825        })
1826        .collect();
1827
1828    let primary_control = if has_blocked {
1829        SCANNER_MAINTENANCE_CONTROL_BLOCKED_SOURCE
1830    } else if has_deferred {
1831        SCANNER_MAINTENANCE_CONTROL_DEFERRED_SOURCE
1832    } else if has_active {
1833        SCANNER_MAINTENANCE_CONTROL_ACTIVE_SOURCE
1834    } else if metrics.pacing_pressure.primary_pressure != SCANNER_MAINTENANCE_CONTROL_NONE {
1835        SCANNER_MAINTENANCE_CONTROL_PACING_PRESSURE
1836    } else {
1837        SCANNER_MAINTENANCE_CONTROL_NONE
1838    };
1839
1840    ScannerMaintenanceControlSnapshot {
1841        primary_control: primary_control.to_string(),
1842        sources,
1843    }
1844}
1845
1846fn duration_millis_saturated(duration: Duration) -> u64 {
1847    duration.as_millis().min(u64::MAX as u128) as u64
1848}
1849
1850fn usize_to_u64_saturated(value: usize) -> u64 {
1851    u64::try_from(value).unwrap_or(u64::MAX)
1852}
1853
1854fn scaled_f64_to_u64_saturated(value: f64, scale: f64) -> u64 {
1855    if !value.is_finite() || value <= 0.0 {
1856        return 0;
1857    }
1858
1859    (value * scale).round().clamp(0.0, u64::MAX as f64) as u64
1860}
1861
1862pub fn emit_scan_cycle_complete(success: bool, duration: Duration) {
1863    let result = if success {
1864        SCAN_CYCLE_RESULT_SUCCESS_LABEL
1865    } else {
1866        SCAN_CYCLE_RESULT_ERROR_LABEL
1867    };
1868    global_metrics().record_scan_cycle_complete(success, duration);
1869    metrics::counter!(OTEL_SCANNER_CYCLES, "result" => result).increment(1);
1870    if success {
1871        metrics::gauge!(OTEL_SCANNER_CYCLE_DURATION_SECONDS).set(duration.as_secs_f64());
1872    }
1873}
1874
1875pub fn emit_scan_cycle_partial(duration: Duration, reason: ScanCyclePartialReason) {
1876    global_metrics().record_scan_cycle_partial(duration, reason);
1877    metrics::counter!(OTEL_SCANNER_CYCLES, "result" => SCAN_CYCLE_RESULT_PARTIAL_LABEL).increment(1);
1878}
1879
1880pub fn emit_scan_cycle_partial_with_source(
1881    duration: Duration,
1882    reason: ScanCyclePartialReason,
1883    source: Option<ScannerWorkSource>,
1884) {
1885    global_metrics().record_scan_cycle_partial_with_source(duration, reason, source);
1886    metrics::counter!(OTEL_SCANNER_CYCLES, "result" => SCAN_CYCLE_RESULT_PARTIAL_LABEL).increment(1);
1887}
1888
1889pub fn emit_scan_cycle_superseded(duration: Duration) {
1890    global_metrics().record_scan_cycle_superseded(duration);
1891    metrics::counter!(OTEL_SCANNER_CYCLES, "result" => SCAN_CYCLE_RESULT_SUPERSEDED_LABEL).increment(1);
1892}
1893
1894pub fn emit_scan_cycle_deferred(duration: Duration) {
1895    global_metrics().record_scan_cycle_deferred(duration);
1896    metrics::counter!(OTEL_SCANNER_CYCLES, "result" => SCAN_CYCLE_RESULT_DEFERRED_LABEL).increment(1);
1897}
1898
1899pub fn emit_scan_bucket_drive_complete(_source: ScannerWorkSource, success: bool, bucket: &str, disk: &str, duration: Duration) {
1900    let result = if success { "success" } else { "error" };
1901    global_metrics().record_scanner_bucket_drive_result(bucket, disk, result);
1902    metrics::counter!(
1903        OTEL_SCANNER_BUCKETS_SCANNED,
1904        "result" => result,
1905        "bucket" => bucket.to_owned(),
1906        "disk" => disk.to_owned()
1907    )
1908    .increment(1);
1909    metrics::histogram!(
1910        OTEL_SCANNER_BUCKET_DRIVE_DURATION_SECONDS,
1911        "bucket" => bucket.to_owned(),
1912        "disk" => disk.to_owned()
1913    )
1914    .record(duration.as_secs_f64());
1915}
1916
1917pub fn emit_scan_bucket_drive_partial(_source: ScannerWorkSource, bucket: &str, disk: &str, duration: Duration) {
1918    global_metrics().record_scanner_bucket_drive_result(bucket, disk, SCAN_CYCLE_RESULT_PARTIAL_LABEL);
1919    metrics::counter!(
1920        OTEL_SCANNER_BUCKETS_SCANNED,
1921        "result" => SCAN_CYCLE_RESULT_PARTIAL_LABEL,
1922        "bucket" => bucket.to_owned(),
1923        "disk" => disk.to_owned()
1924    )
1925    .increment(1);
1926    metrics::histogram!(
1927        OTEL_SCANNER_BUCKET_DRIVE_DURATION_SECONDS,
1928        "bucket" => bucket.to_owned(),
1929        "disk" => disk.to_owned()
1930    )
1931    .record(duration.as_secs_f64());
1932}
1933
1934impl Metrics {
1935    pub fn new() -> Self {
1936        Self {
1937            operations: (0..Metric::Last as usize).map(|_| AtomicU64::new(0)).collect(),
1938            // Each slot gets its own fresh LockedLastMinuteLatency so that
1939            // different metrics never accidentally share state.
1940            latency: (0..Metric::LastRealtime as usize)
1941                .map(|_| LockedLastMinuteLatency::new())
1942                .collect(),
1943            actions: (0..IlmAction::ActionCount as usize).map(|_| AtomicU64::new(0)).collect(),
1944            actions_latency: (0..IlmAction::ActionCount as usize)
1945                .map(|_| LockedLastMinuteLatency::new())
1946                .collect(),
1947            current_paths: Arc::new(RwLock::new(HashMap::new())),
1948            cycle_info: Arc::new(RwLock::new(None)),
1949            current_scan_mode: AtomicU8::new(HealScanMode::Unknown as u8),
1950            current_scan_cycle_work_active: AtomicBool::new(false),
1951            current_scan_cycle_objects_start: AtomicU64::new(0),
1952            current_scan_cycle_directories_start: AtomicU64::new(0),
1953            current_scan_cycle_bucket_drive_scans_start: AtomicU64::new(0),
1954            current_scan_cycle_bucket_drive_failures_start: AtomicU64::new(0),
1955            current_scan_cycle_yield_events_start: AtomicU64::new(0),
1956            current_scan_cycle_yield_duration_millis_start: AtomicU64::new(0),
1957            current_scan_cycle_throttle_sleep_events_start: AtomicU64::new(0),
1958            current_scan_cycle_throttle_sleep_duration_millis_start: AtomicU64::new(0),
1959            current_scan_cycle_ilm_actions_start: AtomicU64::new(0),
1960            current_scan_cycle_lifecycle_expiry_actions_start: AtomicU64::new(0),
1961            current_scan_cycle_lifecycle_transition_actions_start: AtomicU64::new(0),
1962            current_scan_cycle_heal_objects_start: AtomicU64::new(0),
1963            current_scan_cycle_replication_checks_start: AtomicU64::new(0),
1964            current_scan_cycle_usage_saves_start: AtomicU64::new(0),
1965            scanner_set_scan_concurrency_limit: AtomicU64::new(0),
1966            scanner_set_scans_queued: AtomicU64::new(0),
1967            scanner_set_scans_active: AtomicU64::new(0),
1968            scanner_disk_bucket_scan_states: Mutex::new(HashMap::new()),
1969            scanner_bucket_drive_results: Mutex::new(ScannerBucketDriveResults::default()),
1970            scanner_active_bucket_drive_scans: Mutex::new(HashMap::new()),
1971            scanner_bucket_drive_result_clock: AtomicU64::new(0),
1972            current_scan_cycle_bucket_drive_results_start: Mutex::new(HashMap::new()),
1973            last_scan_cycle_bucket_drive_results: Mutex::new(Vec::new()),
1974            scanner_leader_lock_state: RwLock::new("unknown".to_string()),
1975            scanner_leader_lock_held: AtomicBool::new(false),
1976            scanner_leader_lock_last_error: RwLock::new(String::new()),
1977            scanner_leader_lock_last_update_unix_secs: AtomicU64::new(0),
1978            last_scan_cycle_result: AtomicU8::new(SCAN_CYCLE_RESULT_UNKNOWN),
1979            last_scan_cycle_partial_reason: AtomicU8::new(ScanCyclePartialReason::Unknown as u8),
1980            last_scan_cycle_partial_source: AtomicU8::new(0),
1981            last_scan_cycle_end_unix_secs: AtomicU64::new(0),
1982            last_scan_cycle_duration_millis: AtomicU64::new(0),
1983            last_scan_cycle_objects_scanned: AtomicU64::new(0),
1984            last_scan_cycle_directories_scanned: AtomicU64::new(0),
1985            lifetime_versions_scanned: AtomicU64::new(0),
1986            last_scan_cycle_bucket_drive_scans: AtomicU64::new(0),
1987            last_scan_cycle_bucket_drive_failures: AtomicU64::new(0),
1988            last_scan_cycle_yield_events: AtomicU64::new(0),
1989            last_scan_cycle_yield_duration_millis: AtomicU64::new(0),
1990            last_scan_cycle_throttle_sleep_events: AtomicU64::new(0),
1991            last_scan_cycle_throttle_sleep_duration_millis: AtomicU64::new(0),
1992            last_scan_cycle_ilm_actions: AtomicU64::new(0),
1993            last_scan_cycle_lifecycle_expiry_actions: AtomicU64::new(0),
1994            last_scan_cycle_lifecycle_transition_actions: AtomicU64::new(0),
1995            last_scan_cycle_heal_objects: AtomicU64::new(0),
1996            last_scan_cycle_replication_checks: AtomicU64::new(0),
1997            last_scan_cycle_usage_saves: AtomicU64::new(0),
1998            failed_scan_cycles: AtomicU64::new(0),
1999            superseded_scan_cycles: AtomicU64::new(0),
2000            partial_scan_cycles_unknown: AtomicU64::new(0),
2001            partial_scan_cycles_runtime: AtomicU64::new(0),
2002            partial_scan_cycles_objects: AtomicU64::new(0),
2003            partial_scan_cycles_directories: AtomicU64::new(0),
2004            partial_scan_cycles_by_source: ScannerWorkSource::all().iter().map(|_| AtomicU64::new(0)).collect(),
2005            scanner_yield_duration_millis: AtomicU64::new(0),
2006            scanner_throttle_sleep_duration_millis: AtomicU64::new(0),
2007            scanner_ilm_actions: AtomicU64::new(0),
2008            scanner_lifecycle_expiry_actions: AtomicU64::new(0),
2009            scanner_lifecycle_transition_actions: AtomicU64::new(0),
2010            scanner_expiry_queue_capacity: AtomicU64::new(0),
2011            scanner_expiry_queued: AtomicU64::new(0),
2012            scanner_expiry_active: AtomicU64::new(0),
2013            scanner_expiry_workers: AtomicU64::new(0),
2014            scanner_expiry_queue_missed: AtomicU64::new(0),
2015            scanner_expiry_queued_total: AtomicU64::new(0),
2016            scanner_expiry_missed_total: AtomicU64::new(0),
2017            scanner_expiry_blocked_total: AtomicU64::new(0),
2018            scanner_expiry_not_enqueued_total: AtomicU64::new(0),
2019            scanner_expiry_delete_failed_total: AtomicU64::new(0),
2020            scanner_transition_queue_capacity: AtomicU64::new(0),
2021            scanner_transition_queued: AtomicU64::new(0),
2022            scanner_transition_active: AtomicU64::new(0),
2023            scanner_transition_workers: AtomicU64::new(0),
2024            scanner_transition_queue_full: AtomicU64::new(0),
2025            scanner_transition_queue_send_timeout: AtomicU64::new(0),
2026            scanner_transition_compensation_scheduled: AtomicU64::new(0),
2027            scanner_transition_compensation_pending: AtomicU64::new(0),
2028            scanner_transition_compensation_running: AtomicU64::new(0),
2029            scanner_transition_queued_total: AtomicU64::new(0),
2030            scanner_transition_missed_total: AtomicU64::new(0),
2031            scanner_transition_completed: AtomicU64::new(0),
2032            tier_requests: std::array::from_fn(|_| AtomicU64::new(0)),
2033            scanner_transition_failed: AtomicU64::new(0),
2034            scanner_throttle_idle_mode_enabled: AtomicBool::new(false),
2035            scanner_throttle_sleep_factor_micros: AtomicU64::new(0),
2036            scanner_throttle_max_sleep_millis: AtomicU64::new(0),
2037            scanner_yield_every_n_objects: AtomicU64::new(0),
2038            scanner_cycle_interval_millis: AtomicU64::new(0),
2039            scanner_cycle_max_duration_millis: AtomicU64::new(0),
2040            scanner_cycle_max_objects: AtomicU64::new(0),
2041            scanner_cycle_max_directories: AtomicU64::new(0),
2042            scanner_cycle_timeout_total: AtomicU64::new(0),
2043            scanner_cycle_recovery_required_total: AtomicU64::new(0),
2044            scanner_cycle_last_progress_age_seconds: AtomicU64::new(0),
2045            scanner_leader_lease_without_progress: AtomicBool::new(false),
2046            scanner_bitrot_cycle_enabled: AtomicBool::new(false),
2047            scanner_bitrot_cycle_millis: AtomicU64::new(0),
2048            scanner_checkpoint: Mutex::new(None),
2049            scanner_checkpoint_used: AtomicU64::new(0),
2050            scanner_checkpoint_cleared: AtomicU64::new(0),
2051            scanner_checkpoint_ignored: AtomicU64::new(0),
2052            scanner_checkpoint_stale: AtomicU64::new(0),
2053            scanner_dirty_usage_pending_buckets: AtomicU64::new(0),
2054            scanner_dirty_usage_last_mark_unix_secs: AtomicU64::new(0),
2055            scanner_dirty_usage_last_clear_unix_secs: AtomicU64::new(0),
2056            scanner_dirty_usage_last_cycle_dirty_buckets: AtomicU64::new(0),
2057            scanner_dirty_usage_last_cycle_cleared_buckets: AtomicU64::new(0),
2058            scanner_usage_last_save_unix_secs: AtomicU64::new(0),
2059            scanner_usage_last_durable_success_unix_secs: AtomicU64::new(0),
2060            scanner_usage_last_publication_unix_secs: AtomicU64::new(0),
2061            scanner_usage_last_publication_state: Mutex::new(String::new()),
2062            scanner_usage_last_publication_reason: Mutex::new(String::new()),
2063            scanner_usage_last_save_result: AtomicU8::new(ScannerUsageSaveResult::Unknown as u8),
2064            scanner_usage_deferred_pending: AtomicBool::new(false),
2065            scanner_usage_deferred_total: AtomicU64::new(0),
2066            scanner_usage_last_deferred_unix_secs: AtomicU64::new(0),
2067            scanner_usage_last_deferred_reason: Mutex::new(String::new()),
2068            scanner_source_work: ScannerWorkSource::all()
2069                .iter()
2070                .map(|_| ScannerSourceWorkCounters::default())
2071                .collect(),
2072            current_scan_cycle_source_work_start: ScannerWorkSource::all()
2073                .iter()
2074                .map(|_| ScannerSourceWorkCounters::default())
2075                .collect(),
2076            last_scan_cycle_source_work: ScannerWorkSource::all()
2077                .iter()
2078                .map(|_| ScannerSourceWorkCounters::default())
2079                .collect(),
2080            scanner_replication_repair_work: ScannerReplicationRepairKind::all()
2081                .iter()
2082                .map(|_| ScannerSourceWorkCounters::default())
2083                .collect(),
2084            current_scan_cycle_replication_repair_work_start: ScannerReplicationRepairKind::all()
2085                .iter()
2086                .map(|_| ScannerSourceWorkCounters::default())
2087                .collect(),
2088            last_scan_cycle_replication_repair_work: ScannerReplicationRepairKind::all()
2089                .iter()
2090                .map(|_| ScannerSourceWorkCounters::default())
2091                .collect(),
2092            partial_scan_cycles: AtomicU64::new(0),
2093        }
2094    }
2095
2096    // -----------------------------------------------------------------------
2097    // Metric recording helpers
2098    //
2099    // All of these are now pure sync closures.  No tokio::spawn, no heap-
2100    // allocated future, no scheduler overhead — just an atomic increment and a
2101    // std-mutex lock for fewer than ~10 ns of work.
2102    // -----------------------------------------------------------------------
2103
2104    /// Return a closure that records one observation of `metric` (with
2105    /// optional caller-supplied metadata).  Call it once the operation ends.
2106    pub fn log(metric: Metric) -> impl Fn(&HashMap<String, String>) {
2107        let metric_idx = metric as usize;
2108        let start = SystemTime::now();
2109        move |_custom: &HashMap<String, String>| {
2110            let duration = SystemTime::now().duration_since(start).unwrap_or_default();
2111            global_metrics().operations[metric_idx].fetch_add(1, Ordering::Relaxed);
2112            global_metrics().record_source_work_for_metric(metric, 1);
2113            if metric_idx < Metric::LastRealtime as usize {
2114                global_metrics().latency[metric_idx].add(duration);
2115            }
2116        }
2117    }
2118
2119    /// Return a closure that records one observation of `metric` together with
2120    /// a byte count.  Call `done(size_bytes)` when the operation ends.
2121    pub fn time_size(metric: Metric) -> impl Fn(u64) {
2122        let metric_idx = metric as usize;
2123        let start = SystemTime::now();
2124        move |size: u64| {
2125            let duration = SystemTime::now().duration_since(start).unwrap_or_default();
2126            global_metrics().operations[metric_idx].fetch_add(1, Ordering::Relaxed);
2127            global_metrics().record_source_work_for_metric(metric, 1);
2128            if metric_idx < Metric::LastRealtime as usize {
2129                global_metrics().latency[metric_idx].add_size(duration, size);
2130            }
2131        }
2132    }
2133
2134    /// Return a closure that records one observation of `metric`.
2135    /// Call `done()` when the operation ends.
2136    pub fn time(metric: Metric) -> impl Fn() {
2137        let metric_idx = metric as usize;
2138        let start = SystemTime::now();
2139        move || {
2140            let duration = SystemTime::now().duration_since(start).unwrap_or_default();
2141            global_metrics().operations[metric_idx].fetch_add(1, Ordering::Relaxed);
2142            global_metrics().record_source_work_for_metric(metric, 1);
2143            if metric_idx < Metric::LastRealtime as usize {
2144                global_metrics().latency[metric_idx].add(duration);
2145            }
2146        }
2147    }
2148
2149    /// Return a two-stage closure: first call takes an item count, second call
2150    /// (returned closure) fires when the batch of `count` operations ends.
2151    pub fn time_n(metric: Metric) -> Box<dyn Fn(usize) -> Box<dyn Fn() + Send + Sync> + Send + Sync> {
2152        let metric_idx = metric as usize;
2153        let start = SystemTime::now();
2154        Box::new(move |count: usize| {
2155            Box::new(move || {
2156                let duration = SystemTime::now().duration_since(start).unwrap_or_default();
2157                let count = usize_to_u64_saturated(count);
2158                global_metrics().operations[metric_idx].fetch_add(count, Ordering::Relaxed);
2159                global_metrics().record_source_work_for_metric(metric, count);
2160                if metric_idx < Metric::LastRealtime as usize {
2161                    global_metrics().latency[metric_idx].add(duration);
2162                }
2163            })
2164        })
2165    }
2166
2167    /// Return a two-stage closure for ILM actions: first call takes a version
2168    /// count, second call fires when the action completes.
2169    pub fn time_ilm(a: IlmAction) -> Box<dyn Fn(u64) -> Box<dyn Fn() + Send + Sync> + Send + Sync> {
2170        let a_idx = a as usize;
2171        if a_idx == IlmAction::NoneAction as usize || a_idx >= IlmAction::ActionCount as usize {
2172            return Box::new(|_| Box::new(|| {}));
2173        }
2174        let start = SystemTime::now();
2175        Box::new(move |versions: u64| {
2176            Box::new(move || {
2177                let duration = SystemTime::now().duration_since(start).unwrap_or_default();
2178                let metric_idx = Metric::Ilm as usize;
2179                global_metrics().operations[metric_idx].fetch_add(versions, Ordering::Relaxed);
2180                global_metrics().actions[a_idx].fetch_add(versions, Ordering::Relaxed);
2181                global_metrics().actions_latency[a_idx].add(duration);
2182            })
2183        })
2184    }
2185
2186    /// Record a single observation of `metric` with a caller-supplied duration.
2187    /// No longer async — nothing inside requires it.
2188    pub fn inc_time(metric: Metric, duration: Duration) {
2189        let metric_idx = metric as usize;
2190        global_metrics().operations[metric_idx].fetch_add(1, Ordering::Relaxed);
2191        global_metrics().record_source_work_for_metric(metric, 1);
2192        if metric_idx < Metric::LastRealtime as usize {
2193            global_metrics().latency[metric_idx].add(duration);
2194        }
2195    }
2196
2197    pub fn record_scanner_yield(&self, duration: Duration) {
2198        let metric_idx = Metric::Yield as usize;
2199        self.operations[metric_idx].fetch_add(1, Ordering::Relaxed);
2200        if metric_idx < Metric::LastRealtime as usize {
2201            self.latency[metric_idx].add(duration);
2202        }
2203        let duration_millis = duration_millis_saturated(duration);
2204        let _ = self
2205            .scanner_yield_duration_millis
2206            .try_update(Ordering::Relaxed, Ordering::Relaxed, |current| {
2207                Some(current.saturating_add(duration_millis))
2208            });
2209    }
2210
2211    pub fn record_scanner_throttle_sleep(&self, duration: Duration) {
2212        let metric_idx = Metric::ThrottleSleep as usize;
2213        self.operations[metric_idx].fetch_add(1, Ordering::Relaxed);
2214        if metric_idx < Metric::LastRealtime as usize {
2215            self.latency[metric_idx].add(duration);
2216        }
2217        let duration_millis = duration_millis_saturated(duration);
2218        let _ = self
2219            .scanner_throttle_sleep_duration_millis
2220            .try_update(Ordering::Relaxed, Ordering::Relaxed, |current| {
2221                Some(current.saturating_add(duration_millis))
2222            });
2223    }
2224
2225    pub fn record_scanner_ilm_action(&self, count: u64) {
2226        self.scanner_ilm_actions.fetch_add(count, Ordering::Relaxed);
2227        self.record_scanner_source_executed(ScannerWorkSource::Lifecycle, count);
2228    }
2229
2230    pub fn record_scanner_lifecycle_action(&self, action: IlmAction, count: u64) {
2231        self.record_scanner_ilm_action(count);
2232        match action {
2233            IlmAction::TransitionAction | IlmAction::TransitionVersionAction => {
2234                self.scanner_lifecycle_transition_actions.fetch_add(count, Ordering::Relaxed);
2235            }
2236            _ if action.delete() => {
2237                self.scanner_lifecycle_expiry_actions.fetch_add(count, Ordering::Relaxed);
2238            }
2239            _ => {}
2240        }
2241    }
2242
2243    pub fn record_scanner_ilm_enqueue_result(&self, count: u64, queued: bool) {
2244        if queued {
2245            self.record_scanner_source_queued(ScannerWorkSource::Lifecycle, count);
2246        } else {
2247            self.record_scanner_source_missed(ScannerWorkSource::Lifecycle, count);
2248        }
2249    }
2250
2251    pub fn record_scanner_expiry_enqueue_result(&self, count: u64, queued: bool) {
2252        self.record_scanner_ilm_enqueue_result(count, queued);
2253        if queued {
2254            self.scanner_expiry_queued_total.fetch_add(count, Ordering::Relaxed);
2255        } else {
2256            self.scanner_expiry_missed_total.fetch_add(count, Ordering::Relaxed);
2257            self.scanner_expiry_not_enqueued_total.fetch_add(count, Ordering::Relaxed);
2258        }
2259    }
2260
2261    pub fn record_scanner_expiry_blocked(&self, count: u64) {
2262        self.scanner_expiry_blocked_total.fetch_add(count, Ordering::Relaxed);
2263    }
2264
2265    pub fn record_scanner_expiry_delete_failed(&self, count: u64) {
2266        self.scanner_expiry_delete_failed_total.fetch_add(count, Ordering::Relaxed);
2267    }
2268
2269    pub fn record_scanner_transition_enqueue_result(&self, count: u64, queued: bool) {
2270        self.record_scanner_ilm_enqueue_result(count, queued);
2271        if queued {
2272            self.scanner_transition_queued_total.fetch_add(count, Ordering::Relaxed);
2273        } else {
2274            self.scanner_transition_missed_total.fetch_add(count, Ordering::Relaxed);
2275        }
2276    }
2277
2278    pub fn record_scanner_lifecycle_expiry_state(&self, state: ScannerLifecycleExpiryStateUpdate) {
2279        self.scanner_expiry_queue_capacity
2280            .store(state.queue_capacity, Ordering::Relaxed);
2281        self.scanner_expiry_queued.store(state.queued, Ordering::Relaxed);
2282        self.scanner_expiry_active.store(state.active, Ordering::Relaxed);
2283        self.scanner_expiry_workers.store(state.workers, Ordering::Relaxed);
2284        self.scanner_expiry_queue_missed.store(state.queue_missed, Ordering::Relaxed);
2285    }
2286
2287    pub fn record_scanner_lifecycle_transition_state(&self, state: ScannerLifecycleTransitionStateUpdate) {
2288        self.scanner_transition_queue_capacity
2289            .store(state.queue_capacity, Ordering::Relaxed);
2290        self.scanner_transition_queued.store(state.queued, Ordering::Relaxed);
2291        self.scanner_transition_active.store(state.active, Ordering::Relaxed);
2292        self.scanner_transition_workers.store(state.workers, Ordering::Relaxed);
2293        self.scanner_transition_queue_full.store(state.queue_full, Ordering::Relaxed);
2294        self.scanner_transition_queue_send_timeout
2295            .store(state.queue_send_timeout, Ordering::Relaxed);
2296        self.scanner_transition_compensation_scheduled
2297            .store(state.compensation_scheduled, Ordering::Relaxed);
2298        self.scanner_transition_compensation_pending
2299            .store(state.compensation_pending, Ordering::Relaxed);
2300        self.scanner_transition_compensation_running
2301            .store(state.compensation_running, Ordering::Relaxed);
2302    }
2303
2304    /// Count one completed remote tier request.
2305    ///
2306    /// Called once per request at the single point where every warm backend
2307    /// call returns, so a retried request is counted once per attempt and a
2308    /// successful attempt is never also counted as a failure.
2309    pub fn record_tier_request(&self, operation: TierRequestOperation, outcome: TierRequestOutcome) {
2310        let slot = operation.index() * TierRequestOutcome::ALL.len() + outcome.index();
2311        self.tier_requests[slot].fetch_add(1, Ordering::Relaxed);
2312    }
2313
2314    /// Every operation/outcome cell, including the zero ones, so a scrape has
2315    /// a stable series set from the first request onward.
2316    pub fn tier_request_counts(&self) -> Vec<TierRequestCount> {
2317        let mut counts = Vec::with_capacity(TIER_REQUEST_COUNTER_SLOTS);
2318        for operation in TierRequestOperation::ALL {
2319            for outcome in TierRequestOutcome::ALL {
2320                let slot = operation.index() * TierRequestOutcome::ALL.len() + outcome.index();
2321                counts.push(TierRequestCount {
2322                    operation,
2323                    outcome,
2324                    count: self.tier_requests[slot].load(Ordering::Relaxed),
2325                });
2326            }
2327        }
2328        counts
2329    }
2330
2331    pub fn record_scanner_transition_completed(&self, count: u64) {
2332        self.scanner_transition_completed.fetch_add(count, Ordering::Relaxed);
2333    }
2334
2335    pub fn record_scanner_transition_failed(&self, count: u64) {
2336        self.scanner_transition_failed.fetch_add(count, Ordering::Relaxed);
2337        self.record_scanner_source_failed(ScannerWorkSource::Lifecycle, count);
2338        if !self.current_scan_cycle_work_active.load(Ordering::Acquire) {
2339            self.record_last_cycle_scanner_source_work(ScannerWorkSource::Lifecycle, ScannerSourceWorkUpdate::failed(count));
2340        }
2341    }
2342
2343    pub fn record_scanner_checkpoint_set(&self, version: u16, resume_after: impl Into<String>, reason: impl Into<String>) {
2344        let checkpoint = ScannerCheckpointReport {
2345            version,
2346            resume_after: resume_after.into(),
2347            reason: reason.into(),
2348            last_event: SCANNER_CHECKPOINT_EVENT_SET.to_string(),
2349        };
2350        match self.scanner_checkpoint.lock() {
2351            Ok(mut current) => *current = Some(checkpoint),
2352            Err(poisoned) => *poisoned.into_inner() = Some(checkpoint),
2353        }
2354    }
2355
2356    pub fn record_scanner_checkpoint_used(&self) {
2357        self.scanner_checkpoint_used.fetch_add(1, Ordering::Relaxed);
2358        self.update_scanner_checkpoint_event(SCANNER_CHECKPOINT_EVENT_USED);
2359    }
2360
2361    pub fn record_scanner_checkpoint_cleared(&self) {
2362        self.scanner_checkpoint_cleared.fetch_add(1, Ordering::Relaxed);
2363        match self.scanner_checkpoint.lock() {
2364            Ok(mut current) => *current = None,
2365            Err(poisoned) => *poisoned.into_inner() = None,
2366        }
2367    }
2368
2369    pub fn record_scanner_checkpoint_ignored(&self) {
2370        self.scanner_checkpoint_ignored.fetch_add(1, Ordering::Relaxed);
2371        self.update_scanner_checkpoint_event(SCANNER_CHECKPOINT_EVENT_IGNORED);
2372    }
2373
2374    pub fn record_scanner_checkpoint_stale(&self) {
2375        self.scanner_checkpoint_stale.fetch_add(1, Ordering::Relaxed);
2376        self.update_scanner_checkpoint_event(SCANNER_CHECKPOINT_EVENT_STALE);
2377    }
2378
2379    pub fn record_scanner_dirty_usage_pending(&self, pending_buckets: u64) {
2380        self.scanner_dirty_usage_pending_buckets
2381            .store(pending_buckets, Ordering::Relaxed);
2382        if pending_buckets > 0 {
2383            self.scanner_dirty_usage_last_mark_unix_secs
2384                .store(unix_now_secs(), Ordering::Relaxed);
2385        }
2386    }
2387
2388    pub fn record_scanner_dirty_usage_clear(&self, pending_buckets: u64) {
2389        self.scanner_dirty_usage_pending_buckets
2390            .store(pending_buckets, Ordering::Relaxed);
2391        self.scanner_dirty_usage_last_clear_unix_secs
2392            .store(unix_now_secs(), Ordering::Relaxed);
2393    }
2394
2395    pub fn record_scanner_dirty_usage_cycle_snapshot(&self, dirty_buckets: u64) {
2396        self.scanner_dirty_usage_last_cycle_dirty_buckets
2397            .store(dirty_buckets, Ordering::Relaxed);
2398        self.scanner_dirty_usage_last_cycle_cleared_buckets
2399            .store(0, Ordering::Relaxed);
2400    }
2401
2402    pub fn record_scanner_dirty_usage_cycle_clear(&self, cleared_buckets: u64, pending_buckets: u64) {
2403        self.scanner_dirty_usage_last_cycle_cleared_buckets
2404            .store(cleared_buckets, Ordering::Relaxed);
2405        self.scanner_dirty_usage_pending_buckets
2406            .store(pending_buckets, Ordering::Relaxed);
2407        if cleared_buckets > 0 {
2408            self.scanner_dirty_usage_last_clear_unix_secs
2409                .store(unix_now_secs(), Ordering::Relaxed);
2410        }
2411    }
2412
2413    pub fn record_scanner_usage_save_result(&self, result: ScannerUsageSaveResult) {
2414        self.scanner_usage_last_save_result.store(result as u8, Ordering::Relaxed);
2415        self.scanner_usage_last_save_unix_secs
2416            .store(unix_now_secs(), Ordering::Relaxed);
2417    }
2418
2419    /// Record an intentional retryable usage publication deferral separately
2420    /// from the last durable save result.
2421    pub fn record_scanner_usage_deferred(&self, reason: impl Into<String>) {
2422        let reason = reason.into();
2423        self.record_scanner_usage_publication("deferred", reason.clone());
2424        self.scanner_usage_deferred_pending.store(true, Ordering::Release);
2425        self.scanner_usage_deferred_total.fetch_add(1, Ordering::Relaxed);
2426        self.scanner_usage_last_deferred_unix_secs
2427            .store(unix_now_secs(), Ordering::Relaxed);
2428        let mut last_reason = match self.scanner_usage_last_deferred_reason.lock() {
2429            Ok(guard) => guard,
2430            Err(poisoned) => poisoned.into_inner(),
2431        };
2432        *last_reason = reason;
2433    }
2434
2435    pub fn record_scanner_usage_durable_success(&self) {
2436        self.record_scanner_usage_publication("success", "");
2437        self.scanner_usage_last_durable_success_unix_secs
2438            .store(unix_now_secs(), Ordering::Relaxed);
2439        self.scanner_usage_deferred_pending.store(false, Ordering::Release);
2440    }
2441
2442    pub fn record_scanner_usage_publication(&self, state: &str, reason: impl Into<String>) {
2443        self.scanner_usage_last_publication_unix_secs
2444            .store(unix_now_secs(), Ordering::Relaxed);
2445        let mut publication_state = match self.scanner_usage_last_publication_state.lock() {
2446            Ok(guard) => guard,
2447            Err(poisoned) => poisoned.into_inner(),
2448        };
2449        *publication_state = state.to_string();
2450        let mut publication_reason = match self.scanner_usage_last_publication_reason.lock() {
2451            Ok(guard) => guard,
2452            Err(poisoned) => poisoned.into_inner(),
2453        };
2454        *publication_reason = reason.into();
2455    }
2456
2457    pub fn record_scanner_source_work(&self, source: ScannerWorkSource, work: ScannerSourceWorkUpdate) {
2458        if let Some(counters) = self.scanner_source_work.get(source.index()) {
2459            counters.add(work);
2460        }
2461    }
2462
2463    pub fn record_scanner_replication_repair_work(&self, kind: ScannerReplicationRepairKind, work: ScannerSourceWorkUpdate) {
2464        if let Some(counters) = self.scanner_replication_repair_work.get(kind.index()) {
2465            counters.add(work);
2466        }
2467    }
2468
2469    fn record_last_cycle_scanner_source_work(&self, source: ScannerWorkSource, work: ScannerSourceWorkUpdate) {
2470        if let Some(counters) = self.last_scan_cycle_source_work.get(source.index()) {
2471            counters.add(work);
2472        }
2473    }
2474
2475    pub fn record_scanner_source_checked(&self, source: ScannerWorkSource, count: u64) {
2476        self.record_scanner_source_work(source, ScannerSourceWorkUpdate::checked(count));
2477    }
2478
2479    /// Record `count` object versions walked by the data scanner.
2480    ///
2481    /// Every version the scanner visits is counted here, independent of
2482    /// lifecycle configuration, so `rustfs_scanner_versions_scanned_total`
2483    /// reflects real scan coverage even on clusters with no ILM rules. ILM
2484    /// evaluation coverage is recorded separately via the `Lifecycle` source's
2485    /// `checked` counter and surfaces as `rustfs_ilm_versions_scanned_total`.
2486    pub fn record_scanner_versions_scanned(&self, count: u64) {
2487        self.lifetime_versions_scanned.fetch_add(count, Ordering::Relaxed);
2488    }
2489
2490    pub fn record_scanner_source_queued(&self, source: ScannerWorkSource, count: u64) {
2491        self.record_scanner_source_work(source, ScannerSourceWorkUpdate::queued(count));
2492    }
2493
2494    pub fn record_scanner_source_executed(&self, source: ScannerWorkSource, count: u64) {
2495        self.record_scanner_source_work(source, ScannerSourceWorkUpdate::executed(count));
2496    }
2497
2498    pub fn record_scanner_source_failed(&self, source: ScannerWorkSource, count: u64) {
2499        self.record_scanner_source_work(source, ScannerSourceWorkUpdate::failed(count));
2500    }
2501
2502    pub fn record_scanner_source_missed(&self, source: ScannerWorkSource, count: u64) {
2503        self.record_scanner_source_work(source, ScannerSourceWorkUpdate::missed(count));
2504    }
2505
2506    fn update_scanner_checkpoint_event(&self, event: &str) {
2507        match self.scanner_checkpoint.lock() {
2508            Ok(mut current) => {
2509                if let Some(checkpoint) = current.as_mut() {
2510                    checkpoint.last_event = event.to_string();
2511                }
2512            }
2513            Err(poisoned) => {
2514                let mut current = poisoned.into_inner();
2515                if let Some(checkpoint) = current.as_mut() {
2516                    checkpoint.last_event = event.to_string();
2517                }
2518            }
2519        }
2520    }
2521
2522    fn record_source_work_for_metric(&self, metric: Metric, count: u64) {
2523        match metric {
2524            Metric::ScanObject | Metric::ScanFolder => {
2525                self.record_scanner_source_checked(ScannerWorkSource::Usage, count);
2526            }
2527            Metric::SaveUsage => {
2528                self.record_scanner_source_executed(ScannerWorkSource::Usage, count);
2529            }
2530            Metric::CheckReplication => {
2531                self.record_scanner_source_checked(ScannerWorkSource::BucketReplication, count);
2532            }
2533            Metric::HealCheck => {
2534                self.record_scanner_source_checked(ScannerWorkSource::Heal, count);
2535            }
2536            _ => {}
2537        }
2538    }
2539
2540    pub fn record_scan_bucket_drive_start(&self, source: ScannerWorkSource, bucket: &str, drive: &str) {
2541        self.operations[Metric::ScanBucketDriveStart as usize].fetch_add(1, Ordering::Relaxed);
2542        if bucket.is_empty() || drive.is_empty() {
2543            return;
2544        }
2545        let key = ScannerActiveBucketDriveKey {
2546            source: source.as_str().to_string(),
2547            bucket: bucket.to_string(),
2548            drive: drive.to_string(),
2549        };
2550        let mut active = self
2551            .scanner_active_bucket_drive_scans
2552            .lock()
2553            .unwrap_or_else(|poisoned| poisoned.into_inner());
2554        active
2555            .entry(key)
2556            .and_modify(|value| value.count = value.count.saturating_add(1))
2557            .or_insert(ScannerActiveBucketDriveValue {
2558                count: 1,
2559                started_at: Timestamp::now(),
2560            });
2561    }
2562
2563    pub fn record_scan_bucket_drive_end(&self, source: ScannerWorkSource, bucket: &str, drive: &str) {
2564        let key = ScannerActiveBucketDriveKey {
2565            source: source.as_str().to_string(),
2566            bucket: bucket.to_string(),
2567            drive: drive.to_string(),
2568        };
2569        let mut active = self
2570            .scanner_active_bucket_drive_scans
2571            .lock()
2572            .unwrap_or_else(|poisoned| poisoned.into_inner());
2573        if let Some(value) = active.get_mut(&key) {
2574            value.count = value.count.saturating_sub(1);
2575            if value.count == 0 {
2576                active.remove(&key);
2577            }
2578        }
2579    }
2580
2581    pub fn record_scan_bucket_drive_failure(&self) {
2582        self.operations[Metric::ScanBucketDriveFailure as usize].fetch_add(1, Ordering::Relaxed);
2583    }
2584
2585    pub fn record_scanner_throttle_config(
2586        &self,
2587        idle_mode_enabled: bool,
2588        sleep_factor: f64,
2589        max_sleep: Duration,
2590        yield_every_n_objects: u64,
2591    ) {
2592        self.scanner_throttle_idle_mode_enabled
2593            .store(idle_mode_enabled, Ordering::Relaxed);
2594        self.scanner_throttle_sleep_factor_micros
2595            .store(scaled_f64_to_u64_saturated(sleep_factor, 1_000_000.0), Ordering::Relaxed);
2596        self.scanner_throttle_max_sleep_millis
2597            .store(duration_millis_saturated(max_sleep), Ordering::Relaxed);
2598        self.scanner_yield_every_n_objects
2599            .store(yield_every_n_objects, Ordering::Relaxed);
2600    }
2601
2602    pub fn record_scanner_cycle_config(
2603        &self,
2604        cycle_interval: Duration,
2605        bitrot_cycle: Option<Duration>,
2606        cycle_max_duration: Option<Duration>,
2607        cycle_max_objects: Option<u64>,
2608        cycle_max_directories: Option<u64>,
2609    ) {
2610        self.scanner_cycle_interval_millis
2611            .store(duration_millis_saturated(cycle_interval), Ordering::Relaxed);
2612        self.scanner_cycle_max_duration_millis
2613            .store(cycle_max_duration.map(duration_millis_saturated).unwrap_or_default(), Ordering::Relaxed);
2614        self.scanner_cycle_max_objects
2615            .store(cycle_max_objects.unwrap_or_default(), Ordering::Relaxed);
2616        self.scanner_cycle_max_directories
2617            .store(cycle_max_directories.unwrap_or_default(), Ordering::Relaxed);
2618        self.scanner_leader_lease_without_progress.store(false, Ordering::Relaxed);
2619        self.scanner_cycle_last_progress_age_seconds.store(0, Ordering::Relaxed);
2620        metrics::gauge!(OTEL_SCANNER_LEADER_LEASE_WITHOUT_PROGRESS).set(0.0);
2621        metrics::gauge!(OTEL_SCANNER_CYCLE_LAST_PROGRESS_AGE).set(0.0);
2622        self.scanner_bitrot_cycle_enabled
2623            .store(bitrot_cycle.is_some(), Ordering::Relaxed);
2624        self.scanner_bitrot_cycle_millis
2625            .store(bitrot_cycle.map(duration_millis_saturated).unwrap_or_default(), Ordering::Relaxed);
2626    }
2627
2628    pub fn record_scanner_cycle_timeout(&self, recovery_required: bool, progress_age: Duration) {
2629        self.scanner_cycle_timeout_total.fetch_add(1, Ordering::Relaxed);
2630        if recovery_required {
2631            self.scanner_cycle_recovery_required_total.fetch_add(1, Ordering::Relaxed);
2632        }
2633        self.scanner_cycle_last_progress_age_seconds
2634            .store(progress_age.as_secs(), Ordering::Relaxed);
2635        self.scanner_leader_lease_without_progress.store(true, Ordering::Relaxed);
2636        metrics::counter!(OTEL_SCANNER_CYCLE_TIMEOUT_TOTAL).increment(1);
2637        metrics::gauge!(OTEL_SCANNER_CYCLE_LAST_PROGRESS_AGE).set(progress_age.as_secs_f64());
2638        metrics::gauge!(OTEL_SCANNER_LEADER_LEASE_WITHOUT_PROGRESS).set(1.0);
2639    }
2640
2641    pub fn record_scanner_set_scan_state(&self, concurrency_limit: Option<usize>, queued: Option<usize>, active: Option<usize>) {
2642        if let Some(concurrency_limit) = concurrency_limit {
2643            self.scanner_set_scan_concurrency_limit
2644                .store(concurrency_limit as u64, Ordering::Relaxed);
2645        }
2646        if let Some(queued) = queued {
2647            self.scanner_set_scans_queued.store(queued as u64, Ordering::Relaxed);
2648        }
2649        if let Some(active) = active {
2650            self.scanner_set_scans_active.store(active as u64, Ordering::Relaxed);
2651        }
2652    }
2653
2654    pub fn reset_scanner_set_scan_state(&self) {
2655        self.record_scanner_set_scan_state(Some(0), Some(0), Some(0));
2656        match self.scanner_disk_bucket_scan_states.lock() {
2657            Ok(mut states) => states.clear(),
2658            Err(poisoned) => poisoned.into_inner().clear(),
2659        }
2660    }
2661
2662    pub fn record_scanner_disk_bucket_scan_state(
2663        &self,
2664        pool: &str,
2665        set: &str,
2666        concurrency_limit: Option<usize>,
2667        queued: Option<usize>,
2668        active: Option<usize>,
2669    ) {
2670        let key = (pool.to_string(), set.to_string());
2671        let mut states = self
2672            .scanner_disk_bucket_scan_states
2673            .lock()
2674            .unwrap_or_else(|poisoned| poisoned.into_inner());
2675        let state = states.entry(key).or_default();
2676        if let Some(concurrency_limit) = concurrency_limit {
2677            state.concurrency_limit = concurrency_limit as u64;
2678        }
2679        if let Some(queued) = queued {
2680            state.queued = queued as u64;
2681        }
2682        if let Some(active) = active {
2683            state.active = active as u64;
2684        }
2685    }
2686
2687    pub fn record_scanner_bucket_drive_result(&self, bucket: &str, drive: &str, result: &str) {
2688        if bucket.is_empty() || drive.is_empty() || result.is_empty() {
2689            return;
2690        }
2691        let key = ScannerBucketDriveResultKey::new(bucket, drive, result);
2692        let mut results = self
2693            .scanner_bucket_drive_results
2694            .lock()
2695            .unwrap_or_else(|poisoned| poisoned.into_inner());
2696        let last_seen = self.scanner_bucket_drive_result_clock.fetch_add(1, Ordering::Relaxed);
2697        if let Some(previous_last_seen) = results.counts.get_mut(&key).map(|value| {
2698            let previous_last_seen = value.last_seen;
2699            value.count = value.count.saturating_add(1);
2700            value.last_seen = last_seen;
2701            previous_last_seen
2702        }) {
2703            results.eviction_index.remove(&(previous_last_seen, key.clone()));
2704            results.eviction_index.insert((last_seen, key));
2705            return;
2706        }
2707
2708        if results.counts.len() >= MAX_SCANNER_BUCKET_DRIVE_RESULT_KEYS
2709            && let Some((_, stale_key)) = results.eviction_index.pop_first()
2710        {
2711            results.counts.remove(&stale_key);
2712        }
2713
2714        if results.counts.len() < MAX_SCANNER_BUCKET_DRIVE_RESULT_KEYS {
2715            results
2716                .counts
2717                .insert(key.clone(), ScannerBucketDriveResultValue { count: 1, last_seen });
2718            results.eviction_index.insert((last_seen, key));
2719        }
2720    }
2721
2722    // -----------------------------------------------------------------------
2723    // Read-side helpers
2724    // -----------------------------------------------------------------------
2725
2726    /// Lifetime operation count for `metric`.
2727    pub fn lifetime(&self, metric: Metric) -> u64 {
2728        let idx = metric as usize;
2729        if idx >= Metric::Last as usize {
2730            return 0;
2731        }
2732        self.operations[idx].load(Ordering::Relaxed)
2733    }
2734
2735    /// Last-minute accumulated stats for a realtime metric.
2736    /// No longer async — LockedLastMinuteLatency::total() is now synchronous.
2737    pub fn last_minute(&self, metric: Metric) -> AccElem {
2738        let idx = metric as usize;
2739        if idx >= Metric::LastRealtime as usize {
2740            return AccElem::default();
2741        }
2742        self.latency[idx].total()
2743    }
2744
2745    /// Replace the current cycle record.
2746    pub async fn set_cycle(&self, cycle: Option<CurrentCycle>) {
2747        *self.cycle_info.write().await = cycle;
2748    }
2749
2750    /// Publish a scanner cycle and its work-accounting baseline as one state transition.
2751    pub async fn start_scan_cycle_work_with_cycle(&self, cycle: CurrentCycle) -> ScanCycleWorkSnapshot {
2752        let mut current_cycle = self.cycle_info.write().await;
2753        let snapshot = self.start_scan_cycle_work();
2754        *current_cycle = Some(cycle);
2755        snapshot
2756    }
2757
2758    /// Publish the completed work snapshot and idle cycle state as one state transition.
2759    pub async fn finish_scan_cycle_work_with_cycle(&self, start: ScanCycleWorkSnapshot, cycle: CurrentCycle) {
2760        let mut current_cycle = self.cycle_info.write().await;
2761        self.finish_scan_cycle_work(start);
2762        *current_cycle = Some(cycle);
2763    }
2764
2765    /// Read the current cycle record.
2766    pub async fn get_cycle(&self) -> Option<CurrentCycle> {
2767        self.cycle_info.read().await.clone()
2768    }
2769
2770    pub fn set_current_scan_mode(&self, scan_mode: HealScanMode) {
2771        self.current_scan_mode.store(scan_mode as u8, Ordering::Relaxed);
2772    }
2773
2774    pub fn clear_current_scan_mode(&self) {
2775        self.set_current_scan_mode(HealScanMode::Unknown);
2776    }
2777
2778    pub fn current_scan_mode(&self) -> HealScanMode {
2779        HealScanMode::from_u8(self.current_scan_mode.load(Ordering::Relaxed)).unwrap_or(HealScanMode::Unknown)
2780    }
2781
2782    pub async fn record_scanner_leader_liveness(&self, state: impl Into<String>, held: bool, error: impl Into<String>) {
2783        *self.scanner_leader_lock_state.write().await = state.into();
2784        *self.scanner_leader_lock_last_error.write().await = error.into();
2785        self.scanner_leader_lock_held.store(held, Ordering::Relaxed);
2786        self.scanner_leader_lock_last_update_unix_secs
2787            .store(Utc::now().timestamp().max(0) as u64, Ordering::Relaxed);
2788    }
2789
2790    pub fn record_scanner_cycle_end_time(&self) {
2791        self.last_scan_cycle_end_unix_secs
2792            .store(Utc::now().timestamp().max(0) as u64, Ordering::Relaxed);
2793    }
2794
2795    pub fn record_scan_cycle_complete(&self, success: bool, duration: Duration) {
2796        let result = if success {
2797            SCAN_CYCLE_RESULT_SUCCESS
2798        } else {
2799            self.failed_scan_cycles.fetch_add(1, Ordering::Relaxed);
2800            SCAN_CYCLE_RESULT_ERROR
2801        };
2802        self.record_scanner_cycle_end_time();
2803        self.last_scan_cycle_result.store(result, Ordering::Relaxed);
2804        self.last_scan_cycle_partial_reason
2805            .store(ScanCyclePartialReason::Unknown as u8, Ordering::Relaxed);
2806        self.last_scan_cycle_partial_source.store(0, Ordering::Relaxed);
2807        self.last_scan_cycle_duration_millis
2808            .store(duration_millis_saturated(duration), Ordering::Relaxed);
2809    }
2810
2811    pub fn record_scan_cycle_superseded(&self, duration: Duration) {
2812        self.record_scanner_cycle_end_time();
2813        self.superseded_scan_cycles.fetch_add(1, Ordering::Relaxed);
2814        self.last_scan_cycle_result
2815            .store(SCAN_CYCLE_RESULT_SUPERSEDED, Ordering::Relaxed);
2816        self.last_scan_cycle_partial_reason
2817            .store(ScanCyclePartialReason::Unknown as u8, Ordering::Relaxed);
2818        self.last_scan_cycle_partial_source.store(0, Ordering::Relaxed);
2819        self.last_scan_cycle_duration_millis
2820            .store(duration_millis_saturated(duration), Ordering::Relaxed);
2821    }
2822
2823    pub fn record_scan_cycle_deferred(&self, duration: Duration) {
2824        self.record_scanner_cycle_end_time();
2825        self.last_scan_cycle_result
2826            .store(SCAN_CYCLE_RESULT_DEFERRED, Ordering::Relaxed);
2827        self.last_scan_cycle_partial_reason
2828            .store(ScanCyclePartialReason::Unknown as u8, Ordering::Relaxed);
2829        self.last_scan_cycle_partial_source.store(0, Ordering::Relaxed);
2830        self.last_scan_cycle_duration_millis
2831            .store(duration_millis_saturated(duration), Ordering::Relaxed);
2832    }
2833
2834    pub fn record_scan_cycle_partial(&self, duration: Duration, reason: ScanCyclePartialReason) {
2835        self.record_scan_cycle_partial_with_source(duration, reason, None);
2836    }
2837
2838    pub fn record_scan_cycle_partial_with_source(
2839        &self,
2840        duration: Duration,
2841        reason: ScanCyclePartialReason,
2842        source: Option<ScannerWorkSource>,
2843    ) {
2844        self.record_scanner_cycle_end_time();
2845        self.partial_scan_cycles.fetch_add(1, Ordering::Relaxed);
2846        match reason {
2847            ScanCyclePartialReason::Unknown => &self.partial_scan_cycles_unknown,
2848            ScanCyclePartialReason::Runtime => &self.partial_scan_cycles_runtime,
2849            ScanCyclePartialReason::Objects => &self.partial_scan_cycles_objects,
2850            ScanCyclePartialReason::Directories => &self.partial_scan_cycles_directories,
2851        }
2852        .fetch_add(1, Ordering::Relaxed);
2853        self.last_scan_cycle_result
2854            .store(SCAN_CYCLE_RESULT_PARTIAL, Ordering::Relaxed);
2855        self.last_scan_cycle_partial_reason.store(reason as u8, Ordering::Relaxed);
2856        self.last_scan_cycle_partial_source
2857            .store(source.map(ScannerWorkSource::code).unwrap_or_default(), Ordering::Relaxed);
2858        if let Some(source) = source
2859            && let Some(cycles) = self.partial_scan_cycles_by_source.get(source.index())
2860        {
2861            cycles.fetch_add(1, Ordering::Relaxed);
2862        }
2863        self.last_scan_cycle_duration_millis
2864            .store(duration_millis_saturated(duration), Ordering::Relaxed);
2865    }
2866
2867    pub fn start_scan_cycle_work(&self) -> ScanCycleWorkSnapshot {
2868        let snapshot = self.scan_cycle_work_snapshot();
2869        let source_snapshot = self.scanner_source_work_values();
2870        let replication_repair_snapshot = self.scanner_replication_repair_work_values();
2871        self.current_scan_cycle_objects_start
2872            .store(snapshot.objects_scanned, Ordering::Relaxed);
2873        self.current_scan_cycle_directories_start
2874            .store(snapshot.directories_scanned, Ordering::Relaxed);
2875        self.current_scan_cycle_bucket_drive_scans_start
2876            .store(snapshot.bucket_drive_scans, Ordering::Relaxed);
2877        self.current_scan_cycle_bucket_drive_failures_start
2878            .store(snapshot.bucket_drive_failures, Ordering::Relaxed);
2879        self.current_scan_cycle_yield_events_start
2880            .store(snapshot.yield_events, Ordering::Relaxed);
2881        self.current_scan_cycle_yield_duration_millis_start
2882            .store(snapshot.yield_duration_millis, Ordering::Relaxed);
2883        self.current_scan_cycle_throttle_sleep_events_start
2884            .store(snapshot.throttle_sleep_events, Ordering::Relaxed);
2885        self.current_scan_cycle_throttle_sleep_duration_millis_start
2886            .store(snapshot.throttle_sleep_duration_millis, Ordering::Relaxed);
2887        self.current_scan_cycle_ilm_actions_start
2888            .store(snapshot.ilm_actions, Ordering::Relaxed);
2889        self.current_scan_cycle_lifecycle_expiry_actions_start
2890            .store(snapshot.lifecycle_expiry_actions, Ordering::Relaxed);
2891        self.current_scan_cycle_lifecycle_transition_actions_start
2892            .store(snapshot.lifecycle_transition_actions, Ordering::Relaxed);
2893        self.current_scan_cycle_heal_objects_start
2894            .store(snapshot.heal_objects, Ordering::Relaxed);
2895        self.current_scan_cycle_replication_checks_start
2896            .store(snapshot.replication_checks, Ordering::Relaxed);
2897        self.current_scan_cycle_usage_saves_start
2898            .store(snapshot.usage_saves, Ordering::Relaxed);
2899        self.store_scanner_source_work_values(&self.current_scan_cycle_source_work_start, &source_snapshot);
2900        self.store_scanner_replication_repair_work_values(
2901            &self.current_scan_cycle_replication_repair_work_start,
2902            &replication_repair_snapshot,
2903        );
2904        let bucket_drive_results = self.scanner_bucket_drive_result_counts();
2905        match self.current_scan_cycle_bucket_drive_results_start.lock() {
2906            Ok(mut start) => *start = bucket_drive_results,
2907            Err(poisoned) => *poisoned.into_inner() = bucket_drive_results,
2908        }
2909        self.current_scan_cycle_work_active.store(true, Ordering::Release);
2910        snapshot
2911    }
2912
2913    pub fn finish_scan_cycle_work(&self, start: ScanCycleWorkSnapshot) {
2914        let work = self.scan_cycle_work_since(start);
2915        let source_work = self.scanner_source_work_since(&self.current_scan_cycle_source_work_start_values());
2916        let replication_repair_work =
2917            self.scanner_replication_repair_work_since(&self.current_scan_cycle_replication_repair_work_start_values());
2918        self.record_scan_cycle_work(work);
2919        self.record_scan_cycle_source_work(&source_work);
2920        self.record_scan_cycle_replication_repair_work(&replication_repair_work);
2921        let bucket_drive_results = self.current_cycle_bucket_drive_result_snapshots();
2922        match self.last_scan_cycle_bucket_drive_results.lock() {
2923            Ok(mut last) => *last = bucket_drive_results,
2924            Err(poisoned) => *poisoned.into_inner() = bucket_drive_results,
2925        }
2926        self.current_scan_cycle_work_active.store(false, Ordering::Release);
2927    }
2928
2929    pub fn current_scan_cycle_has_unresolved_heal_work(&self) -> bool {
2930        if !self.current_scan_cycle_work_active.load(Ordering::Acquire) {
2931            return false;
2932        }
2933
2934        let source_work = self.scanner_source_work_since(&self.current_scan_cycle_source_work_start_values());
2935        [ScannerWorkSource::Heal, ScannerWorkSource::Bitrot]
2936            .into_iter()
2937            .filter_map(|source| source_work.get(source.index()))
2938            .any(|work| work.queued > 0 || work.skipped > 0 || work.failed > 0 || work.missed > 0)
2939    }
2940
2941    fn scan_cycle_work_snapshot(&self) -> ScanCycleWorkSnapshot {
2942        ScanCycleWorkSnapshot {
2943            objects_scanned: self.lifetime(Metric::ScanObject),
2944            directories_scanned: self.lifetime(Metric::ScanFolder),
2945            bucket_drive_scans: self.lifetime(Metric::ScanBucketDrive),
2946            bucket_drive_failures: self.lifetime(Metric::ScanBucketDriveFailure),
2947            yield_events: self.lifetime(Metric::Yield),
2948            yield_duration_millis: self.scanner_yield_duration_millis.load(Ordering::Relaxed),
2949            throttle_sleep_events: self.lifetime(Metric::ThrottleSleep),
2950            throttle_sleep_duration_millis: self.scanner_throttle_sleep_duration_millis.load(Ordering::Relaxed),
2951            ilm_actions: self.scanner_ilm_actions.load(Ordering::Relaxed),
2952            lifecycle_expiry_actions: self.scanner_lifecycle_expiry_actions.load(Ordering::Relaxed),
2953            lifecycle_transition_actions: self.scanner_lifecycle_transition_actions.load(Ordering::Relaxed),
2954            heal_objects: self.lifetime(Metric::HealAbandonedObject),
2955            replication_checks: self.lifetime(Metric::CheckReplication),
2956            usage_saves: self.lifetime(Metric::SaveUsage),
2957        }
2958    }
2959
2960    fn current_scan_cycle_work_start(&self) -> ScanCycleWorkSnapshot {
2961        ScanCycleWorkSnapshot {
2962            objects_scanned: self.current_scan_cycle_objects_start.load(Ordering::Relaxed),
2963            directories_scanned: self.current_scan_cycle_directories_start.load(Ordering::Relaxed),
2964            bucket_drive_scans: self.current_scan_cycle_bucket_drive_scans_start.load(Ordering::Relaxed),
2965            bucket_drive_failures: self.current_scan_cycle_bucket_drive_failures_start.load(Ordering::Relaxed),
2966            yield_events: self.current_scan_cycle_yield_events_start.load(Ordering::Relaxed),
2967            yield_duration_millis: self.current_scan_cycle_yield_duration_millis_start.load(Ordering::Relaxed),
2968            throttle_sleep_events: self.current_scan_cycle_throttle_sleep_events_start.load(Ordering::Relaxed),
2969            throttle_sleep_duration_millis: self
2970                .current_scan_cycle_throttle_sleep_duration_millis_start
2971                .load(Ordering::Relaxed),
2972            ilm_actions: self.current_scan_cycle_ilm_actions_start.load(Ordering::Relaxed),
2973            lifecycle_expiry_actions: self.current_scan_cycle_lifecycle_expiry_actions_start.load(Ordering::Relaxed),
2974            lifecycle_transition_actions: self
2975                .current_scan_cycle_lifecycle_transition_actions_start
2976                .load(Ordering::Relaxed),
2977            heal_objects: self.current_scan_cycle_heal_objects_start.load(Ordering::Relaxed),
2978            replication_checks: self.current_scan_cycle_replication_checks_start.load(Ordering::Relaxed),
2979            usage_saves: self.current_scan_cycle_usage_saves_start.load(Ordering::Relaxed),
2980        }
2981    }
2982
2983    fn scan_cycle_work_since(&self, start: ScanCycleWorkSnapshot) -> ScanCycleWorkSnapshot {
2984        let current = self.scan_cycle_work_snapshot();
2985        ScanCycleWorkSnapshot {
2986            objects_scanned: current.objects_scanned.saturating_sub(start.objects_scanned),
2987            directories_scanned: current.directories_scanned.saturating_sub(start.directories_scanned),
2988            bucket_drive_scans: current.bucket_drive_scans.saturating_sub(start.bucket_drive_scans),
2989            bucket_drive_failures: current.bucket_drive_failures.saturating_sub(start.bucket_drive_failures),
2990            yield_events: current.yield_events.saturating_sub(start.yield_events),
2991            yield_duration_millis: current.yield_duration_millis.saturating_sub(start.yield_duration_millis),
2992            throttle_sleep_events: current.throttle_sleep_events.saturating_sub(start.throttle_sleep_events),
2993            throttle_sleep_duration_millis: current
2994                .throttle_sleep_duration_millis
2995                .saturating_sub(start.throttle_sleep_duration_millis),
2996            ilm_actions: current.ilm_actions.saturating_sub(start.ilm_actions),
2997            lifecycle_expiry_actions: current
2998                .lifecycle_expiry_actions
2999                .saturating_sub(start.lifecycle_expiry_actions),
3000            lifecycle_transition_actions: current
3001                .lifecycle_transition_actions
3002                .saturating_sub(start.lifecycle_transition_actions),
3003            heal_objects: current.heal_objects.saturating_sub(start.heal_objects),
3004            replication_checks: current.replication_checks.saturating_sub(start.replication_checks),
3005            usage_saves: current.usage_saves.saturating_sub(start.usage_saves),
3006        }
3007    }
3008
3009    fn scanner_bucket_drive_result_counts(&self) -> HashMap<ScannerBucketDriveResultKey, u64> {
3010        self.scanner_bucket_drive_results
3011            .lock()
3012            .unwrap_or_else(|poisoned| poisoned.into_inner())
3013            .counts
3014            .iter()
3015            .map(|(key, value)| (key.clone(), value.count))
3016            .collect()
3017    }
3018
3019    fn scanner_bucket_drive_result_snapshots(
3020        counts: impl IntoIterator<Item = (ScannerBucketDriveResultKey, u64)>,
3021    ) -> Vec<ScannerBucketDriveResultSnapshot> {
3022        let mut snapshots = counts
3023            .into_iter()
3024            .filter(|(_, count)| *count > 0)
3025            .map(|(key, count)| ScannerBucketDriveResultSnapshot {
3026                bucket: key.bucket,
3027                drive: key.drive,
3028                result: key.result,
3029                count,
3030            })
3031            .collect::<Vec<_>>();
3032        snapshots.sort_by(|left, right| {
3033            left.bucket
3034                .cmp(&right.bucket)
3035                .then_with(|| left.drive.cmp(&right.drive))
3036                .then_with(|| left.result.cmp(&right.result))
3037        });
3038        snapshots
3039    }
3040
3041    fn scanner_bucket_drive_result_counter_snapshots(&self) -> Vec<ScannerBucketDriveResultSnapshot> {
3042        Self::scanner_bucket_drive_result_snapshots(self.scanner_bucket_drive_result_counts())
3043    }
3044
3045    fn current_cycle_bucket_drive_result_snapshots(&self) -> Vec<ScannerBucketDriveResultSnapshot> {
3046        let current = self.scanner_bucket_drive_result_counts();
3047        let start = self
3048            .current_scan_cycle_bucket_drive_results_start
3049            .lock()
3050            .unwrap_or_else(|poisoned| poisoned.into_inner())
3051            .clone();
3052        Self::scanner_bucket_drive_result_snapshots(current.into_iter().filter_map(|(key, count)| {
3053            let delta = count.saturating_sub(start.get(&key).copied().unwrap_or_default());
3054            (delta > 0).then_some((key, delta))
3055        }))
3056    }
3057
3058    pub fn scanner_runtime_details_report(&self) -> ScannerRuntimeDetailsReport {
3059        self.scanner_runtime_details_report_for_active(self.current_scan_cycle_work_active.load(Ordering::Acquire))
3060    }
3061
3062    fn scanner_runtime_details_report_for_active(&self, current_cycle_active: bool) -> ScannerRuntimeDetailsReport {
3063        let current_cycle_bucket_drive_results = if current_cycle_active {
3064            self.current_cycle_bucket_drive_result_snapshots()
3065        } else {
3066            Vec::new()
3067        };
3068        let now = Timestamp::now();
3069        let mut active_bucket_drive_scans = self
3070            .scanner_active_bucket_drive_scans
3071            .lock()
3072            .unwrap_or_else(|poisoned| poisoned.into_inner())
3073            .iter()
3074            .map(|(key, value)| ScannerActiveBucketDriveSnapshot {
3075                source: key.source.clone(),
3076                bucket: key.bucket.clone(),
3077                drive: key.drive.clone(),
3078                count: value.count,
3079                age_seconds: timestamp_elapsed_seconds_since(now, value.started_at),
3080            })
3081            .collect::<Vec<_>>();
3082        active_bucket_drive_scans.sort_by(|left, right| {
3083            left.source
3084                .cmp(&right.source)
3085                .then_with(|| left.bucket.cmp(&right.bucket))
3086                .then_with(|| left.drive.cmp(&right.drive))
3087        });
3088        ScannerRuntimeDetailsReport {
3089            disk_bucket_scan_states: self.scanner_disk_bucket_scan_state_snapshots(),
3090            bucket_drive_results: self.scanner_bucket_drive_result_counter_snapshots(),
3091            current_cycle_bucket_drive_results,
3092            last_cycle_bucket_drive_results: self
3093                .last_scan_cycle_bucket_drive_results
3094                .lock()
3095                .unwrap_or_else(|poisoned| poisoned.into_inner())
3096                .clone(),
3097            active_bucket_drive_scans,
3098        }
3099    }
3100
3101    fn scanner_disk_bucket_scan_state_snapshots(&self) -> Vec<ScannerDiskBucketScanSnapshot> {
3102        let mut disk_bucket_scan_states = match self.scanner_disk_bucket_scan_states.lock() {
3103            Ok(states) => states
3104                .iter()
3105                .map(|((pool, set), state)| ScannerDiskBucketScanSnapshot {
3106                    pool: pool.clone(),
3107                    set: set.clone(),
3108                    concurrency_limit: state.concurrency_limit,
3109                    queued: state.queued,
3110                    active: state.active,
3111                })
3112                .collect::<Vec<_>>(),
3113            Err(poisoned) => poisoned
3114                .into_inner()
3115                .iter()
3116                .map(|((pool, set), state)| ScannerDiskBucketScanSnapshot {
3117                    pool: pool.clone(),
3118                    set: set.clone(),
3119                    concurrency_limit: state.concurrency_limit,
3120                    queued: state.queued,
3121                    active: state.active,
3122                })
3123                .collect::<Vec<_>>(),
3124        };
3125        disk_bucket_scan_states.sort_by(|left, right| left.pool.cmp(&right.pool).then_with(|| left.set.cmp(&right.set)));
3126        disk_bucket_scan_states
3127    }
3128
3129    fn scanner_source_work_values(&self) -> Vec<ScannerSourceWorkValues> {
3130        ScannerWorkSource::all()
3131            .iter()
3132            .filter_map(|source| {
3133                self.scanner_source_work
3134                    .get(source.index())
3135                    .map(ScannerSourceWorkCounters::values)
3136            })
3137            .collect()
3138    }
3139
3140    fn current_scan_cycle_source_work_start_values(&self) -> Vec<ScannerSourceWorkValues> {
3141        ScannerWorkSource::all()
3142            .iter()
3143            .filter_map(|source| {
3144                self.current_scan_cycle_source_work_start
3145                    .get(source.index())
3146                    .map(ScannerSourceWorkCounters::values)
3147            })
3148            .collect()
3149    }
3150
3151    fn scanner_source_work_since(&self, start: &[ScannerSourceWorkValues]) -> Vec<ScannerSourceWorkValues> {
3152        self.scanner_source_work_values()
3153            .into_iter()
3154            .enumerate()
3155            .map(|(index, current)| current.saturating_sub(start.get(index).copied().unwrap_or_default()))
3156            .collect()
3157    }
3158
3159    fn scanner_source_work_snapshots(&self, values: &[ScannerSourceWorkValues]) -> Vec<ScannerSourceWorkSnapshot> {
3160        ScannerWorkSource::all()
3161            .iter()
3162            .filter_map(|source| values.get(source.index()).map(|values| values.snapshot(*source)))
3163            .collect()
3164    }
3165
3166    fn scanner_source_work_counter_snapshots(&self, counters: &[ScannerSourceWorkCounters]) -> Vec<ScannerSourceWorkSnapshot> {
3167        ScannerWorkSource::all()
3168            .iter()
3169            .filter_map(|source| counters.get(source.index()).map(|counters| counters.snapshot(*source)))
3170            .collect()
3171    }
3172
3173    fn scanner_replication_repair_work_values(&self) -> Vec<ScannerSourceWorkValues> {
3174        ScannerReplicationRepairKind::all()
3175            .iter()
3176            .filter_map(|kind| {
3177                self.scanner_replication_repair_work
3178                    .get(kind.index())
3179                    .map(ScannerSourceWorkCounters::values)
3180            })
3181            .collect()
3182    }
3183
3184    fn current_scan_cycle_replication_repair_work_start_values(&self) -> Vec<ScannerSourceWorkValues> {
3185        ScannerReplicationRepairKind::all()
3186            .iter()
3187            .filter_map(|kind| {
3188                self.current_scan_cycle_replication_repair_work_start
3189                    .get(kind.index())
3190                    .map(ScannerSourceWorkCounters::values)
3191            })
3192            .collect()
3193    }
3194
3195    fn scanner_replication_repair_work_since(&self, start: &[ScannerSourceWorkValues]) -> Vec<ScannerSourceWorkValues> {
3196        self.scanner_replication_repair_work_values()
3197            .into_iter()
3198            .enumerate()
3199            .map(|(index, current)| current.saturating_sub(start.get(index).copied().unwrap_or_default()))
3200            .collect()
3201    }
3202
3203    fn scanner_replication_repair_work_snapshots(
3204        &self,
3205        values: &[ScannerSourceWorkValues],
3206    ) -> Vec<ScannerReplicationRepairSnapshot> {
3207        ScannerReplicationRepairKind::all()
3208            .iter()
3209            .filter_map(|kind| {
3210                values
3211                    .get(kind.index())
3212                    .map(|values| values.replication_repair_snapshot(*kind))
3213            })
3214            .collect()
3215    }
3216
3217    fn scanner_replication_repair_work_counter_snapshots(
3218        &self,
3219        counters: &[ScannerSourceWorkCounters],
3220    ) -> Vec<ScannerReplicationRepairSnapshot> {
3221        ScannerReplicationRepairKind::all()
3222            .iter()
3223            .filter_map(|kind| {
3224                counters
3225                    .get(kind.index())
3226                    .map(|counters| counters.values().replication_repair_snapshot(*kind))
3227            })
3228            .collect()
3229    }
3230
3231    fn store_scanner_source_work_values(&self, counters: &[ScannerSourceWorkCounters], values: &[ScannerSourceWorkValues]) {
3232        for source in ScannerWorkSource::all() {
3233            if let (Some(counter), Some(values)) = (counters.get(source.index()), values.get(source.index())) {
3234                counter.store(*values);
3235            }
3236        }
3237    }
3238
3239    fn store_scanner_replication_repair_work_values(
3240        &self,
3241        counters: &[ScannerSourceWorkCounters],
3242        values: &[ScannerSourceWorkValues],
3243    ) {
3244        for kind in ScannerReplicationRepairKind::all() {
3245            if let (Some(counter), Some(values)) = (counters.get(kind.index()), values.get(kind.index())) {
3246                counter.store(*values);
3247            }
3248        }
3249    }
3250
3251    pub fn record_scan_cycle_work(&self, work: ScanCycleWorkSnapshot) {
3252        // Telemetry-only gauges: readers may observe a transient mixed snapshot
3253        // while these independent atomic fields are updated.
3254        self.last_scan_cycle_objects_scanned
3255            .store(work.objects_scanned, Ordering::Relaxed);
3256        self.last_scan_cycle_directories_scanned
3257            .store(work.directories_scanned, Ordering::Relaxed);
3258        self.last_scan_cycle_bucket_drive_scans
3259            .store(work.bucket_drive_scans, Ordering::Relaxed);
3260        self.last_scan_cycle_bucket_drive_failures
3261            .store(work.bucket_drive_failures, Ordering::Relaxed);
3262        self.last_scan_cycle_yield_events.store(work.yield_events, Ordering::Relaxed);
3263        self.last_scan_cycle_yield_duration_millis
3264            .store(work.yield_duration_millis, Ordering::Relaxed);
3265        self.last_scan_cycle_throttle_sleep_events
3266            .store(work.throttle_sleep_events, Ordering::Relaxed);
3267        self.last_scan_cycle_throttle_sleep_duration_millis
3268            .store(work.throttle_sleep_duration_millis, Ordering::Relaxed);
3269        self.last_scan_cycle_ilm_actions.store(work.ilm_actions, Ordering::Relaxed);
3270        self.last_scan_cycle_lifecycle_expiry_actions
3271            .store(work.lifecycle_expiry_actions, Ordering::Relaxed);
3272        self.last_scan_cycle_lifecycle_transition_actions
3273            .store(work.lifecycle_transition_actions, Ordering::Relaxed);
3274        self.last_scan_cycle_heal_objects.store(work.heal_objects, Ordering::Relaxed);
3275        self.last_scan_cycle_replication_checks
3276            .store(work.replication_checks, Ordering::Relaxed);
3277        self.last_scan_cycle_usage_saves.store(work.usage_saves, Ordering::Relaxed);
3278    }
3279
3280    fn record_scan_cycle_source_work(&self, work: &[ScannerSourceWorkValues]) {
3281        self.store_scanner_source_work_values(&self.last_scan_cycle_source_work, work);
3282    }
3283
3284    fn record_scan_cycle_replication_repair_work(&self, work: &[ScannerSourceWorkValues]) {
3285        self.store_scanner_replication_repair_work_values(&self.last_scan_cycle_replication_repair_work, work);
3286    }
3287
3288    /// Snapshot of every path currently being scanned.
3289    pub async fn get_current_paths(&self) -> Vec<String> {
3290        self.current_path_snapshots()
3291            .await
3292            .into_iter()
3293            .map(|(disk, state)| format!("{disk}/{}", state.path))
3294            .collect()
3295    }
3296
3297    async fn current_path_snapshots(&self) -> Vec<(String, CurrentPathState)> {
3298        let paths = self.current_paths.read().await;
3299        let mut result = Vec::with_capacity(paths.len());
3300        for (disk, tracker) in paths.iter() {
3301            result.push((disk.clone(), tracker.get_state().await));
3302        }
3303        result.sort_by(|(left, _), (right, _)| left.cmp(right));
3304        result
3305    }
3306
3307    /// Number of drives with an active scan in progress.
3308    pub async fn active_drives(&self) -> usize {
3309        self.current_paths.read().await.len()
3310    }
3311
3312    /// Build a full metrics report snapshot.
3313    pub async fn report(&self) -> ScannerMetricsReport {
3314        self.report_with_runtime_details().await.0
3315    }
3316
3317    pub async fn report_with_runtime_details(&self) -> (ScannerMetricsReport, ScannerRuntimeDetailsReport) {
3318        let mut m = ScannerMetricsReport::default();
3319        let runtime_details;
3320
3321        let has_cycle = {
3322            let cycle = self.cycle_info.read().await;
3323            let has_cycle = if let Some(cycle) = cycle.as_ref() {
3324                m.current_cycle = cycle.current;
3325                m.cycles_completed_at = cycle.cycle_completed.iter().copied().map(chrono_to_jiff_timestamp).collect();
3326                m.current_started = chrono_to_jiff_timestamp(cycle.started);
3327                true
3328            } else {
3329                false
3330            };
3331            m.current_cycle_active = self.current_scan_cycle_work_active.load(Ordering::Acquire);
3332            if m.current_cycle_active {
3333                // Keep cycle_info before cycle-baseline locks so active scrapes cannot mix two cycle identities.
3334                let current_work = self.scan_cycle_work_since(self.current_scan_cycle_work_start());
3335                let current_source_work = self.scanner_source_work_since(&self.current_scan_cycle_source_work_start_values());
3336                let current_replication_repair_work =
3337                    self.scanner_replication_repair_work_since(&self.current_scan_cycle_replication_repair_work_start_values());
3338                m.current_cycle_objects_scanned = current_work.objects_scanned;
3339                m.current_cycle_directories_scanned = current_work.directories_scanned;
3340                m.current_cycle_bucket_drive_scans = current_work.bucket_drive_scans;
3341                m.current_cycle_bucket_drive_failures = current_work.bucket_drive_failures;
3342                m.current_cycle_yield_events = current_work.yield_events;
3343                m.current_cycle_yield_duration_seconds = current_work.yield_duration_millis as f64 / 1000.0;
3344                m.current_cycle_throttle_sleep_events = current_work.throttle_sleep_events;
3345                m.current_cycle_throttle_sleep_duration_seconds = current_work.throttle_sleep_duration_millis as f64 / 1000.0;
3346                m.current_cycle_ilm_actions = current_work.ilm_actions;
3347                m.current_cycle_lifecycle_expiry_actions = current_work.lifecycle_expiry_actions;
3348                m.current_cycle_lifecycle_transition_actions = current_work.lifecycle_transition_actions;
3349                m.current_cycle_heal_objects = current_work.heal_objects;
3350                m.current_cycle_replication_checks = current_work.replication_checks;
3351                m.current_cycle_usage_saves = current_work.usage_saves;
3352                m.current_cycle_source_work = self.scanner_source_work_snapshots(&current_source_work);
3353                m.current_cycle_replication_repair =
3354                    self.scanner_replication_repair_work_snapshots(&current_replication_repair_work);
3355            }
3356            runtime_details = self.scanner_runtime_details_report_for_active(m.current_cycle_active);
3357            has_cycle
3358        };
3359
3360        if !has_cycle && let Some(init_time) = crate::init_time::get_global_init_time().await {
3361            m.current_started = chrono_to_jiff_timestamp(init_time);
3362        }
3363
3364        m.collected_at = Timestamp::now();
3365        let current_path_snapshots = self.current_path_snapshots().await;
3366        m.active_scan_paths = current_path_snapshots.len();
3367        m.oldest_active_path_age_seconds = current_path_snapshots
3368            .iter()
3369            .map(|(_, state)| timestamp_elapsed_seconds_since(m.collected_at, state.updated_at))
3370            .max()
3371            .unwrap_or_default();
3372        m.active_paths = current_path_snapshots
3373            .into_iter()
3374            .map(|(disk, state)| format!("{disk}/{}", state.path))
3375            .collect();
3376        m.current_scan_mode = self.current_scan_mode().as_str().to_string();
3377        m.versions_scanned = self.lifetime_versions_scanned.load(Ordering::Relaxed);
3378        m.leader_lock_state = self.scanner_leader_lock_state.read().await.clone();
3379        m.leader_lock_held_by_this_process = self.scanner_leader_lock_held.load(Ordering::Relaxed);
3380        m.leader_lock_last_error = self.scanner_leader_lock_last_error.read().await.clone();
3381        m.leader_lock_last_update_unix_secs = self.scanner_leader_lock_last_update_unix_secs.load(Ordering::Relaxed);
3382        m.last_cycle_end_unix_secs = self.last_scan_cycle_end_unix_secs.load(Ordering::Relaxed);
3383        m.current_set_scan_concurrency_limit = self.scanner_set_scan_concurrency_limit.load(Ordering::Relaxed);
3384        m.current_set_scans_queued = self.scanner_set_scans_queued.load(Ordering::Relaxed);
3385        m.current_set_scans_active = self.scanner_set_scans_active.load(Ordering::Relaxed);
3386        let disk_bucket_scan_states = self.scanner_disk_bucket_scan_state_snapshots();
3387        let (disk_scan_concurrency_limit, disk_bucket_scans_queued, disk_bucket_scans_active) =
3388            disk_bucket_scan_states.iter().fold((0, 0, 0), |acc, state| {
3389                (acc.0 + state.concurrency_limit, acc.1 + state.queued, acc.2 + state.active)
3390            });
3391        m.current_disk_scan_concurrency_limit = disk_scan_concurrency_limit;
3392        m.current_disk_bucket_scans_queued = disk_bucket_scans_queued;
3393        m.current_disk_bucket_scans_active = disk_bucket_scans_active;
3394        let last_cycle_result = self.last_scan_cycle_result.load(Ordering::Relaxed);
3395        m.last_cycle_result = scan_cycle_result_label(last_cycle_result).to_string();
3396        m.last_cycle_result_code = last_cycle_result as u64;
3397        let partial_reason = ScanCyclePartialReason::from_code(self.last_scan_cycle_partial_reason.load(Ordering::Relaxed));
3398        m.last_cycle_partial_reason = partial_reason.as_str().to_string();
3399        m.last_cycle_partial_reason_code = partial_reason as u8 as u64;
3400        let partial_source = ScannerWorkSource::from_code(self.last_scan_cycle_partial_source.load(Ordering::Relaxed));
3401        m.last_cycle_partial_source = partial_source
3402            .map(ScannerWorkSource::as_str)
3403            .unwrap_or(SCAN_CYCLE_RESULT_UNKNOWN_LABEL)
3404            .to_string();
3405        m.last_cycle_partial_source_code = partial_source.map(|source| source.code().into()).unwrap_or_default();
3406        m.last_cycle_duration_seconds = self.last_scan_cycle_duration_millis.load(Ordering::Relaxed) as f64 / 1000.0;
3407        m.last_cycle_objects_scanned = self.last_scan_cycle_objects_scanned.load(Ordering::Relaxed);
3408        m.last_cycle_directories_scanned = self.last_scan_cycle_directories_scanned.load(Ordering::Relaxed);
3409        m.last_cycle_bucket_drive_scans = self.last_scan_cycle_bucket_drive_scans.load(Ordering::Relaxed);
3410        m.last_cycle_bucket_drive_failures = self.last_scan_cycle_bucket_drive_failures.load(Ordering::Relaxed);
3411        m.last_cycle_yield_events = self.last_scan_cycle_yield_events.load(Ordering::Relaxed);
3412        m.last_cycle_yield_duration_seconds = self.last_scan_cycle_yield_duration_millis.load(Ordering::Relaxed) as f64 / 1000.0;
3413        m.last_cycle_throttle_sleep_events = self.last_scan_cycle_throttle_sleep_events.load(Ordering::Relaxed);
3414        m.last_cycle_throttle_sleep_duration_seconds =
3415            self.last_scan_cycle_throttle_sleep_duration_millis.load(Ordering::Relaxed) as f64 / 1000.0;
3416        m.last_cycle_ilm_actions = self.last_scan_cycle_ilm_actions.load(Ordering::Relaxed);
3417        m.last_cycle_lifecycle_expiry_actions = self.last_scan_cycle_lifecycle_expiry_actions.load(Ordering::Relaxed);
3418        m.last_cycle_lifecycle_transition_actions = self.last_scan_cycle_lifecycle_transition_actions.load(Ordering::Relaxed);
3419        m.last_cycle_heal_objects = self.last_scan_cycle_heal_objects.load(Ordering::Relaxed);
3420        m.last_cycle_replication_checks = self.last_scan_cycle_replication_checks.load(Ordering::Relaxed);
3421        m.last_cycle_usage_saves = self.last_scan_cycle_usage_saves.load(Ordering::Relaxed);
3422        m.last_cycle_source_work = self.scanner_source_work_counter_snapshots(&self.last_scan_cycle_source_work);
3423        m.last_cycle_replication_repair =
3424            self.scanner_replication_repair_work_counter_snapshots(&self.last_scan_cycle_replication_repair_work);
3425        m.failed_cycles = self.failed_scan_cycles.load(Ordering::Relaxed);
3426        m.superseded_cycles = self.superseded_scan_cycles.load(Ordering::Relaxed);
3427        m.partial_cycles_unknown = self.partial_scan_cycles_unknown.load(Ordering::Relaxed);
3428        m.partial_cycles_runtime = self.partial_scan_cycles_runtime.load(Ordering::Relaxed);
3429        m.partial_cycles_objects = self.partial_scan_cycles_objects.load(Ordering::Relaxed);
3430        m.partial_cycles_directories = self.partial_scan_cycles_directories.load(Ordering::Relaxed);
3431        m.partial_cycles_by_source = ScannerWorkSource::all()
3432            .iter()
3433            .filter_map(|source| {
3434                self.partial_scan_cycles_by_source
3435                    .get(source.index())
3436                    .map(|cycles| ScannerSourceCycleSnapshot {
3437                        source: source.as_str().to_string(),
3438                        cycles: cycles.load(Ordering::Relaxed),
3439                    })
3440            })
3441            .collect();
3442        m.lifecycle_expiry = ScannerLifecycleExpirySnapshot {
3443            current_queue_capacity: self.scanner_expiry_queue_capacity.load(Ordering::Relaxed),
3444            current_queued: self.scanner_expiry_queued.load(Ordering::Relaxed),
3445            current_active: self.scanner_expiry_active.load(Ordering::Relaxed),
3446            current_workers: self.scanner_expiry_workers.load(Ordering::Relaxed),
3447            queue_missed: self.scanner_expiry_queue_missed.load(Ordering::Relaxed),
3448            scanner_queued: self.scanner_expiry_queued_total.load(Ordering::Relaxed),
3449            scanner_missed: self.scanner_expiry_missed_total.load(Ordering::Relaxed),
3450            scanner_blocked: self.scanner_expiry_blocked_total.load(Ordering::Relaxed),
3451            scanner_not_enqueued: self.scanner_expiry_not_enqueued_total.load(Ordering::Relaxed),
3452            delete_failed: self.scanner_expiry_delete_failed_total.load(Ordering::Relaxed),
3453        };
3454        m.lifecycle_transition = ScannerLifecycleTransitionSnapshot {
3455            current_queue_capacity: self.scanner_transition_queue_capacity.load(Ordering::Relaxed),
3456            current_queued: self.scanner_transition_queued.load(Ordering::Relaxed),
3457            current_active: self.scanner_transition_active.load(Ordering::Relaxed),
3458            current_workers: self.scanner_transition_workers.load(Ordering::Relaxed),
3459            queue_full: self.scanner_transition_queue_full.load(Ordering::Relaxed),
3460            queue_send_timeout: self.scanner_transition_queue_send_timeout.load(Ordering::Relaxed),
3461            compensation_scheduled: self.scanner_transition_compensation_scheduled.load(Ordering::Relaxed),
3462            compensation_pending: self.scanner_transition_compensation_pending.load(Ordering::Relaxed),
3463            compensation_running: self.scanner_transition_compensation_running.load(Ordering::Relaxed),
3464            scanner_queued: self.scanner_transition_queued_total.load(Ordering::Relaxed),
3465            scanner_missed: self.scanner_transition_missed_total.load(Ordering::Relaxed),
3466            completed: self.scanner_transition_completed.load(Ordering::Relaxed),
3467            failed: self.scanner_transition_failed.load(Ordering::Relaxed),
3468        };
3469        let usage_save_result = ScannerUsageSaveResult::from_code(self.scanner_usage_last_save_result.load(Ordering::Relaxed));
3470        m.usage_freshness = ScannerUsageFreshnessSnapshot {
3471            dirty_pending_buckets: self.scanner_dirty_usage_pending_buckets.load(Ordering::Relaxed),
3472            last_dirty_mark_unix_secs: self.scanner_dirty_usage_last_mark_unix_secs.load(Ordering::Relaxed),
3473            last_dirty_clear_unix_secs: self.scanner_dirty_usage_last_clear_unix_secs.load(Ordering::Relaxed),
3474            last_cycle_dirty_buckets: self.scanner_dirty_usage_last_cycle_dirty_buckets.load(Ordering::Relaxed),
3475            last_cycle_cleared_dirty_buckets: self.scanner_dirty_usage_last_cycle_cleared_buckets.load(Ordering::Relaxed),
3476            last_usage_save_unix_secs: self.scanner_usage_last_save_unix_secs.load(Ordering::Relaxed),
3477            last_usage_save_result: usage_save_result.as_str().to_string(),
3478            last_usage_save_result_code: usage_save_result as u8 as u64,
3479            last_durable_success_unix_secs: self.scanner_usage_last_durable_success_unix_secs.load(Ordering::Relaxed),
3480            last_publication_unix_secs: self.scanner_usage_last_publication_unix_secs.load(Ordering::Relaxed),
3481            last_publication_state: match self.scanner_usage_last_publication_state.lock() {
3482                Ok(state) => state.clone(),
3483                Err(poisoned) => poisoned.into_inner().clone(),
3484            },
3485            last_publication_reason: match self.scanner_usage_last_publication_reason.lock() {
3486                Ok(reason) => reason.clone(),
3487                Err(poisoned) => poisoned.into_inner().clone(),
3488            },
3489            deferred_pending: self.scanner_usage_deferred_pending.load(Ordering::Acquire),
3490            deferred_total: self.scanner_usage_deferred_total.load(Ordering::Relaxed),
3491            last_deferred_unix_secs: self.scanner_usage_last_deferred_unix_secs.load(Ordering::Relaxed),
3492            last_deferred_reason: match self.scanner_usage_last_deferred_reason.lock() {
3493                Ok(reason) => reason.clone(),
3494                Err(poisoned) => poisoned.into_inner().clone(),
3495            },
3496        };
3497        m.throttle_idle_mode_enabled = self.scanner_throttle_idle_mode_enabled.load(Ordering::Relaxed);
3498        m.throttle_sleep_factor = self.scanner_throttle_sleep_factor_micros.load(Ordering::Relaxed) as f64 / 1_000_000.0;
3499        m.throttle_max_sleep_seconds = self.scanner_throttle_max_sleep_millis.load(Ordering::Relaxed) as f64 / 1000.0;
3500        m.yield_every_n_objects = self.scanner_yield_every_n_objects.load(Ordering::Relaxed);
3501        m.cycle_interval_seconds = self.scanner_cycle_interval_millis.load(Ordering::Relaxed) as f64 / 1000.0;
3502        m.cycle_max_duration_seconds = self.scanner_cycle_max_duration_millis.load(Ordering::Relaxed) as f64 / 1000.0;
3503        m.cycle_max_objects = self.scanner_cycle_max_objects.load(Ordering::Relaxed);
3504        m.cycle_max_directories = self.scanner_cycle_max_directories.load(Ordering::Relaxed);
3505        m.cycle_timeout_total = self.scanner_cycle_timeout_total.load(Ordering::Relaxed);
3506        m.cycle_recovery_required_total = self.scanner_cycle_recovery_required_total.load(Ordering::Relaxed);
3507        m.cycle_last_progress_age = self.scanner_cycle_last_progress_age_seconds.load(Ordering::Relaxed);
3508        m.leader_lease_without_progress = self.scanner_leader_lease_without_progress.load(Ordering::Relaxed);
3509        m.bitrot_cycle_enabled = self.scanner_bitrot_cycle_enabled.load(Ordering::Relaxed);
3510        m.bitrot_cycle_seconds = self.scanner_bitrot_cycle_millis.load(Ordering::Relaxed) as f64 / 1000.0;
3511        m.scan_checkpoint = match self.scanner_checkpoint.lock() {
3512            Ok(checkpoint) => checkpoint.clone(),
3513            Err(poisoned) => poisoned.into_inner().clone(),
3514        };
3515        m.scan_checkpoint_used = self.scanner_checkpoint_used.load(Ordering::Relaxed);
3516        m.scan_checkpoint_cleared = self.scanner_checkpoint_cleared.load(Ordering::Relaxed);
3517        m.scan_checkpoint_ignored = self.scanner_checkpoint_ignored.load(Ordering::Relaxed);
3518        m.scan_checkpoint_stale = self.scanner_checkpoint_stale.load(Ordering::Relaxed);
3519        m.source_work = self.scanner_source_work_counter_snapshots(&self.scanner_source_work);
3520        m.replication_repair = self.scanner_replication_repair_work_counter_snapshots(&self.scanner_replication_repair_work);
3521        m.partial_cycles = self.partial_scan_cycles.load(Ordering::Relaxed);
3522
3523        // Lifetime operation counts
3524        for i in 0..Metric::Last as usize {
3525            let count = self.operations[i].load(Ordering::Relaxed);
3526            if count > 0
3527                && let Some(metric) = Metric::from_index(i)
3528            {
3529                m.life_time_ops.insert(metric.as_str().to_string(), count);
3530            }
3531        }
3532
3533        // Last-minute stats for realtime metrics — now plain sync calls
3534        for i in 0..Metric::LastRealtime as usize {
3535            let last_min = self.latency[i].total();
3536            if last_min.n > 0
3537                && let Some(metric) = Metric::from_index(i)
3538            {
3539                m.last_minute.actions.insert(
3540                    metric.as_str().to_string(),
3541                    ScannerTimedAction {
3542                        count: last_min.n,
3543                        acc_time: last_min.total,
3544                        bytes: last_min.size,
3545                    },
3546                );
3547            }
3548        }
3549
3550        // Lifetime ILM counts
3551        for i in 0..IlmAction::ActionCount as usize {
3552            let count = self.actions[i].load(Ordering::Relaxed);
3553            if count > 0
3554                && let Some(action) = IlmAction::from_index(i)
3555            {
3556                m.life_time_ilm.insert(action.as_str().to_string(), count);
3557            }
3558        }
3559
3560        // Last-minute ILM latency — plain sync calls
3561        for i in 0..IlmAction::ActionCount as usize {
3562            let last_min = self.actions_latency[i].total();
3563            if last_min.n > 0
3564                && let Some(action) = IlmAction::from_index(i)
3565            {
3566                m.last_minute.ilm.insert(
3567                    action.as_str().to_string(),
3568                    ScannerTimedAction {
3569                        count: last_min.n,
3570                        acc_time: last_min.total,
3571                        bytes: last_min.size,
3572                    },
3573                );
3574            }
3575        }
3576
3577        m.pacing_pressure = scanner_pacing_pressure(&m);
3578        m.maintenance_control = scanner_maintenance_control(&m);
3579
3580        (m, runtime_details)
3581    }
3582}
3583
3584impl Default for Metrics {
3585    fn default() -> Self {
3586        Self::new()
3587    }
3588}
3589
3590// ---------------------------------------------------------------------------
3591// Path tracking helpers
3592// ---------------------------------------------------------------------------
3593
3594pub type UpdateCurrentPathFn = Arc<dyn Fn(&str) -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + Sync>;
3595pub type CloseDiskFn = Arc<dyn Fn() -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + Sync>;
3596
3597/// Register a new disk in the global path tracker and return two callbacks:
3598/// one to update the current path and one to deregister the disk when done.
3599pub async fn current_path_updater(disk: &str, initial: &str) -> (UpdateCurrentPathFn, CloseDiskFn) {
3600    let tracker = Arc::new(CurrentPathTracker::new(initial.to_string()));
3601    let disk_name = disk.to_string();
3602
3603    global_metrics()
3604        .current_paths
3605        .write()
3606        .await
3607        .insert(disk_name.clone(), Arc::clone(&tracker));
3608
3609    let update_fn: UpdateCurrentPathFn = {
3610        let tracker = Arc::clone(&tracker);
3611        Arc::new(move |path: &str| {
3612            let tracker = Arc::clone(&tracker);
3613            let path = path.to_string();
3614            Box::pin(async move { tracker.update_path(path).await })
3615        })
3616    };
3617
3618    let done_fn: CloseDiskFn = {
3619        let disk = disk_name;
3620        Arc::new(move || {
3621            let disk = disk.clone();
3622            Box::pin(async move {
3623                global_metrics().current_paths.write().await.remove(&disk);
3624            })
3625        })
3626    };
3627
3628    (update_fn, done_fn)
3629}
3630
3631// ---------------------------------------------------------------------------
3632// CloseDiskGuard
3633// ---------------------------------------------------------------------------
3634
3635pub struct CloseDiskGuard(Option<CloseDiskFn>);
3636
3637impl CloseDiskGuard {
3638    pub fn new(close_disk: CloseDiskFn) -> Self {
3639        Self(Some(close_disk))
3640    }
3641
3642    pub async fn close(&mut self) {
3643        let Some(close_disk) = self.0.clone() else {
3644            return;
3645        };
3646        close_disk().await;
3647        self.0 = None;
3648    }
3649}
3650
3651impl Drop for CloseDiskGuard {
3652    fn drop(&mut self) {
3653        if let Some(close_disk) = self.0.take()
3654            && let Ok(handle) = tokio::runtime::Handle::try_current()
3655        {
3656            handle.spawn(close_disk());
3657        }
3658        // If there is no runtime we are in a test or shutdown path; skip cleanup.
3659    }
3660}
3661
3662#[cfg(test)]
3663mod tests {
3664    use super::*;
3665
3666    fn tier_cell(metrics: &Metrics, operation: TierRequestOperation, outcome: TierRequestOutcome) -> u64 {
3667        metrics
3668            .tier_request_counts()
3669            .into_iter()
3670            .find(|count| count.operation == operation && count.outcome == outcome)
3671            .map(|count| count.count)
3672            .expect("every operation/outcome cell must be reported")
3673    }
3674
3675    #[test]
3676    fn a_tier_request_lands_in_exactly_one_operation_outcome_cell() {
3677        let metrics = Metrics::default();
3678        metrics.record_tier_request(TierRequestOperation::Put, TierRequestOutcome::Success);
3679
3680        assert_eq!(tier_cell(&metrics, TierRequestOperation::Put, TierRequestOutcome::Success), 1);
3681        assert_eq!(
3682            metrics
3683                .tier_request_counts()
3684                .into_iter()
3685                .map(|count| count.count)
3686                .sum::<u64>(),
3687            1,
3688            "one request must not increment a second cell"
3689        );
3690    }
3691
3692    #[test]
3693    fn every_operation_outcome_pair_has_its_own_cell() {
3694        let metrics = Metrics::default();
3695        for operation in TierRequestOperation::ALL {
3696            for outcome in TierRequestOutcome::ALL {
3697                metrics.record_tier_request(operation, outcome);
3698            }
3699        }
3700
3701        let counts = metrics.tier_request_counts();
3702        assert_eq!(
3703            counts.len(),
3704            TierRequestOperation::ALL.len() * TierRequestOutcome::ALL.len(),
3705            "the series set is fixed by the two label enums"
3706        );
3707        assert!(
3708            counts.iter().all(|count| count.count == 1),
3709            "two different pairs must not share a counter slot"
3710        );
3711    }
3712
3713    #[test]
3714    fn a_backend_error_is_classified_without_reading_its_message() {
3715        assert_eq!(
3716            TierRequestOutcome::from_error(&std::io::Error::new(std::io::ErrorKind::TimedOut, "s3.example.com:9000 timed out")),
3717            TierRequestOutcome::Timeout
3718        );
3719        assert_eq!(
3720            TierRequestOutcome::from_error(&std::io::Error::new(std::io::ErrorKind::Interrupted, "cancelled")),
3721            TierRequestOutcome::Cancelled
3722        );
3723        assert_eq!(
3724            TierRequestOutcome::from_error(&std::io::Error::other("AccessDenied")),
3725            TierRequestOutcome::BackendError
3726        );
3727    }
3728
3729    #[test]
3730    fn tier_request_labels_are_stable_identifiers() {
3731        assert_eq!(TierRequestOperation::Put.as_label(), "put");
3732        assert_eq!(TierRequestOperation::InUse.as_label(), "in_use");
3733        assert_eq!(TierRequestOutcome::BackendError.as_label(), "backend_error");
3734    }
3735
3736    #[test]
3737    fn scanner_metrics_report_timestamps_serialize_as_rfc3339_utc() {
3738        let report = ScannerMetricsReport {
3739            collected_at: Timestamp::constant(1_700_000_000, 123_456_000),
3740            current_started: Timestamp::constant(1_699_999_940, 0),
3741            cycles_completed_at: vec![Timestamp::constant(1_700_000_060, 987_654_000)],
3742            ..Default::default()
3743        };
3744
3745        let value = serde_json::to_value(&report).expect("scanner metrics report should serialize");
3746
3747        assert_eq!(value["collected_at"].as_str(), Some("2023-11-14T22:13:20.123456Z"));
3748        assert_eq!(value["current_started"].as_str(), Some("2023-11-14T22:12:20Z"));
3749        assert_eq!(value["cycles_completed_at"][0].as_str(), Some("2023-11-14T22:14:20.987654Z"));
3750    }
3751
3752    #[tokio::test]
3753    async fn close_disk_guard_runs_cleanup_when_an_early_return_drops_it() {
3754        let (closed_tx, closed_rx) = tokio::sync::oneshot::channel();
3755        let closed_tx = Arc::new(std::sync::Mutex::new(Some(closed_tx)));
3756        let close_disk: CloseDiskFn = {
3757            let closed_tx = Arc::clone(&closed_tx);
3758            Arc::new(move || {
3759                let closed_tx = closed_tx.lock().expect("close callback lock").take();
3760                Box::pin(async move {
3761                    if let Some(closed_tx) = closed_tx {
3762                        let _ = closed_tx.send(());
3763                    }
3764                })
3765            })
3766        };
3767
3768        let guard = CloseDiskGuard::new(close_disk);
3769        drop(guard);
3770
3771        tokio::time::timeout(std::time::Duration::from_secs(1), closed_rx)
3772            .await
3773            .expect("drop cleanup should run")
3774            .expect("drop cleanup should signal");
3775    }
3776
3777    #[tokio::test]
3778    async fn close_disk_guard_runs_explicit_cleanup_once() {
3779        let close_count = Arc::new(std::sync::atomic::AtomicUsize::new(0));
3780        let close_disk: CloseDiskFn = {
3781            let close_count = Arc::clone(&close_count);
3782            Arc::new(move || {
3783                close_count.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
3784                Box::pin(std::future::ready(()))
3785            })
3786        };
3787
3788        let mut guard = CloseDiskGuard::new(close_disk);
3789        guard.close().await;
3790        drop(guard);
3791        tokio::time::sleep(std::time::Duration::from_millis(10)).await;
3792
3793        assert_eq!(close_count.load(std::sync::atomic::Ordering::Relaxed), 1);
3794    }
3795
3796    #[tokio::test]
3797    async fn current_path_updater_registers_before_return() {
3798        let disk = format!("test-disk-{}", uuid::Uuid::new_v4());
3799        let (_update_path, close_disk) = current_path_updater(&disk, "bucket-a").await;
3800
3801        assert!(global_metrics().current_paths.read().await.contains_key(&disk));
3802
3803        close_disk().await;
3804        assert!(!global_metrics().current_paths.read().await.contains_key(&disk));
3805    }
3806
3807    #[tokio::test]
3808    async fn report_counts_active_scan_paths() {
3809        let metrics = Metrics::new();
3810        let updated_at = Timestamp::now() - jiff::SignedDuration::from_secs(12);
3811        metrics.current_paths.write().await.insert(
3812            "disk-a".to_string(),
3813            Arc::new(CurrentPathTracker::new_at("bucket-a".to_string(), updated_at)),
3814        );
3815
3816        let report = metrics.report().await;
3817
3818        assert_eq!(report.active_scan_paths, 1);
3819        assert!(
3820            report.oldest_active_path_age_seconds >= 12,
3821            "expected active path age to reflect last path update"
3822        );
3823        assert_eq!(report.ongoing_buckets, 0);
3824        assert_eq!(report.active_paths, vec!["disk-a/bucket-a".to_string()]);
3825    }
3826
3827    #[tokio::test]
3828    async fn report_active_path_age_uses_last_path_update() {
3829        let metrics = Metrics::new();
3830        let tracker = Arc::new(CurrentPathTracker::new_at(
3831            "bucket-a".to_string(),
3832            Timestamp::now() - jiff::SignedDuration::from_secs(60 * 60),
3833        ));
3834        metrics
3835            .current_paths
3836            .write()
3837            .await
3838            .insert("disk-a".to_string(), Arc::clone(&tracker));
3839
3840        tracker.update_path("bucket-a/prefix".to_string()).await;
3841
3842        let report = metrics.report().await;
3843
3844        assert_eq!(report.active_paths, vec!["disk-a/bucket-a/prefix".to_string()]);
3845        assert!(
3846            report.oldest_active_path_age_seconds < 5,
3847            "expected active path age to reset after path update"
3848        );
3849    }
3850
3851    #[tokio::test]
3852    async fn report_includes_scanner_queue_state() {
3853        let metrics = Metrics::new();
3854        metrics.record_scanner_set_scan_state(Some(3), Some(5), Some(2));
3855        metrics.record_scanner_disk_bucket_scan_state("0", "0", Some(2), Some(7), Some(1));
3856        metrics.record_scanner_disk_bucket_scan_state("0", "1", Some(4), Some(11), Some(3));
3857
3858        let report = metrics.report().await;
3859
3860        assert_eq!(report.current_set_scan_concurrency_limit, 3);
3861        assert_eq!(report.current_set_scans_queued, 5);
3862        assert_eq!(report.current_set_scans_active, 2);
3863        assert_eq!(report.current_disk_scan_concurrency_limit, 6);
3864        assert_eq!(report.current_disk_bucket_scans_queued, 18);
3865        assert_eq!(report.current_disk_bucket_scans_active, 4);
3866
3867        metrics.reset_scanner_set_scan_state();
3868        let report = metrics.report().await;
3869
3870        assert_eq!(report.current_set_scan_concurrency_limit, 0);
3871        assert_eq!(report.current_set_scans_queued, 0);
3872        assert_eq!(report.current_set_scans_active, 0);
3873        assert_eq!(report.current_disk_scan_concurrency_limit, 0);
3874        assert_eq!(report.current_disk_bucket_scans_queued, 0);
3875        assert_eq!(report.current_disk_bucket_scans_active, 0);
3876    }
3877
3878    #[tokio::test]
3879    async fn report_counts_scanned_versions_independent_of_lifecycle() {
3880        let metrics = Metrics::new();
3881
3882        // No ILM configuration touched: scanned versions must still accrue so
3883        // rustfs_scanner_versions_scanned_total reflects real coverage.
3884        metrics.record_scanner_versions_scanned(3);
3885        metrics.record_scanner_versions_scanned(5);
3886
3887        let report = metrics.report().await;
3888
3889        assert_eq!(report.versions_scanned, 8);
3890        // ILM-checked versions live on the Lifecycle source and stay zero when
3891        // no lifecycle evaluation ran.
3892        let lifecycle_checked = report
3893            .source_work
3894            .iter()
3895            .find(|s| s.source == "lifecycle")
3896            .map(|s| s.checked)
3897            .unwrap_or_default();
3898        assert_eq!(lifecycle_checked, 0);
3899    }
3900
3901    #[tokio::test]
3902    async fn report_tracks_lifecycle_checked_versions_separately() {
3903        let metrics = Metrics::new();
3904
3905        // Simulate the scanner walking versions and, on a lifecycle-configured
3906        // bucket, handing a subset to the ILM evaluator.
3907        metrics.record_scanner_versions_scanned(10);
3908        metrics.record_scanner_source_checked(ScannerWorkSource::Lifecycle, 4);
3909
3910        let report = metrics.report().await;
3911
3912        assert_eq!(report.versions_scanned, 10, "total scanned versions independent of ILM");
3913        let lifecycle_checked = report
3914            .source_work
3915            .iter()
3916            .find(|s| s.source == "lifecycle")
3917            .map(|s| s.checked)
3918            .unwrap_or_default();
3919        assert_eq!(lifecycle_checked, 4, "ILM-checked versions tracked on the Lifecycle source");
3920    }
3921
3922    #[tokio::test]
3923    async fn report_derives_scanner_pacing_pressure() {
3924        let metrics = Metrics::new();
3925        metrics.record_scanner_set_scan_state(Some(2), Some(3), Some(1));
3926        metrics.record_scanner_disk_bucket_scan_state("0", "0", Some(1), Some(4), Some(1));
3927        metrics.record_scan_cycle_partial_with_source(
3928            Duration::from_secs(10),
3929            ScanCyclePartialReason::Runtime,
3930            Some(ScannerWorkSource::Usage),
3931        );
3932        metrics.record_scan_cycle_work(ScanCycleWorkSnapshot {
3933            yield_events: 2,
3934            yield_duration_millis: 500,
3935            throttle_sleep_events: 4,
3936            throttle_sleep_duration_millis: 2500,
3937            ..Default::default()
3938        });
3939
3940        let report = metrics.report().await;
3941
3942        assert_eq!(report.pacing_pressure.primary_pressure, "queued_scans");
3943        assert_eq!(report.pacing_pressure.current_queued_scans, 7);
3944        assert_eq!(report.pacing_pressure.current_active_scans, 2);
3945        assert!(report.pacing_pressure.last_cycle_budget_limited);
3946        assert!(report.pacing_pressure.last_cycle_pause_observed);
3947        assert_eq!(report.pacing_pressure.last_cycle_throttle_sleep_ratio, 0.25);
3948        assert_eq!(report.pacing_pressure.last_cycle_yield_ratio, 0.05);
3949        assert_eq!(report.pacing_pressure.last_cycle_total_pause_ratio, 0.3);
3950    }
3951
3952    #[tokio::test]
3953    async fn report_includes_scanner_checkpoint_status() {
3954        let metrics = Metrics::new();
3955        metrics.record_scanner_checkpoint_set(1, "bucket/child-a", "directories");
3956        metrics.record_scanner_checkpoint_used();
3957
3958        let report = metrics.report().await;
3959
3960        let checkpoint = report.scan_checkpoint.as_ref().expect("scanner checkpoint should be visible");
3961        assert_eq!(checkpoint.version, 1);
3962        assert_eq!(checkpoint.resume_after, "bucket/child-a");
3963        assert_eq!(checkpoint.reason, "directories");
3964        assert_eq!(checkpoint.last_event, "used");
3965        assert_eq!(report.scan_checkpoint_used, 1);
3966        assert_eq!(report.scan_checkpoint_cleared, 0);
3967
3968        metrics.record_scanner_checkpoint_stale();
3969        metrics.record_scanner_checkpoint_cleared();
3970        let report = metrics.report().await;
3971
3972        assert!(report.scan_checkpoint.is_none());
3973        assert_eq!(report.scan_checkpoint_used, 1);
3974        assert_eq!(report.scan_checkpoint_stale, 1);
3975        assert_eq!(report.scan_checkpoint_cleared, 1);
3976    }
3977
3978    #[tokio::test]
3979    async fn report_includes_scanner_source_work() {
3980        let metrics = Metrics::new();
3981        metrics.record_scanner_source_work(
3982            ScannerWorkSource::Usage,
3983            ScannerSourceWorkUpdate {
3984                checked: 3,
3985                executed: 1,
3986                ..Default::default()
3987            },
3988        );
3989        metrics.record_scanner_source_work(
3990            ScannerWorkSource::Lifecycle,
3991            ScannerSourceWorkUpdate {
3992                queued: 2,
3993                executed: 1,
3994                failed: 1,
3995                missed: 3,
3996                ..Default::default()
3997            },
3998        );
3999        metrics.record_scanner_ilm_enqueue_result(4, true);
4000        metrics.record_scanner_ilm_enqueue_result(5, false);
4001
4002        let report = metrics.report().await;
4003
4004        let usage = report
4005            .source_work
4006            .iter()
4007            .find(|work| work.source == ScannerWorkSource::Usage.as_str())
4008            .expect("usage source work should be visible");
4009        assert_eq!(usage.checked, 3);
4010        assert_eq!(usage.executed, 1);
4011
4012        let lifecycle = report
4013            .source_work
4014            .iter()
4015            .find(|work| work.source == ScannerWorkSource::Lifecycle.as_str())
4016            .expect("lifecycle source work should be visible");
4017        assert_eq!(lifecycle.queued, 6);
4018        assert_eq!(lifecycle.executed, 1);
4019        assert_eq!(lifecycle.failed, 1);
4020        assert_eq!(lifecycle.missed, 8);
4021    }
4022
4023    #[tokio::test]
4024    async fn report_splits_scanner_lifecycle_actions_by_expiry_and_transition() {
4025        let metrics = Metrics::new();
4026        let start = metrics.start_scan_cycle_work();
4027
4028        metrics.record_scanner_lifecycle_action(IlmAction::DeleteAction, 2);
4029        metrics.record_scanner_lifecycle_action(IlmAction::DeleteRestoredVersionAction, 3);
4030        metrics.record_scanner_lifecycle_action(IlmAction::TransitionAction, 5);
4031        metrics.record_scanner_lifecycle_action(IlmAction::TransitionVersionAction, 7);
4032
4033        let report = metrics.report().await;
4034
4035        assert_eq!(report.current_cycle_ilm_actions, 17);
4036        assert_eq!(report.current_cycle_lifecycle_expiry_actions, 5);
4037        assert_eq!(report.current_cycle_lifecycle_transition_actions, 12);
4038
4039        metrics.finish_scan_cycle_work(start);
4040        let report = metrics.report().await;
4041
4042        assert_eq!(report.last_cycle_ilm_actions, 17);
4043        assert_eq!(report.last_cycle_lifecycle_expiry_actions, 5);
4044        assert_eq!(report.last_cycle_lifecycle_transition_actions, 12);
4045    }
4046
4047    #[tokio::test]
4048    async fn report_includes_scanner_lifecycle_transition_status() {
4049        let metrics = Metrics::new();
4050        metrics.record_scanner_lifecycle_transition_state(ScannerLifecycleTransitionStateUpdate {
4051            queue_capacity: 16,
4052            queued: 5,
4053            active: 2,
4054            workers: 4,
4055            queue_full: 3,
4056            queue_send_timeout: 1,
4057            compensation_scheduled: 2,
4058            compensation_pending: 3,
4059            compensation_running: 1,
4060        });
4061        metrics.record_scanner_transition_enqueue_result(6, true);
4062        metrics.record_scanner_transition_enqueue_result(2, false);
4063        metrics.record_scanner_transition_completed(4);
4064        metrics.record_scanner_transition_failed(1);
4065
4066        let report = metrics.report().await;
4067
4068        assert_eq!(report.lifecycle_transition.current_queue_capacity, 16);
4069        assert_eq!(report.lifecycle_transition.current_queued, 5);
4070        assert_eq!(report.lifecycle_transition.current_active, 2);
4071        assert_eq!(report.lifecycle_transition.current_workers, 4);
4072        assert_eq!(report.lifecycle_transition.queue_full, 3);
4073        assert_eq!(report.lifecycle_transition.queue_send_timeout, 1);
4074        assert_eq!(report.lifecycle_transition.compensation_scheduled, 2);
4075        assert_eq!(report.lifecycle_transition.compensation_pending, 3);
4076        assert_eq!(report.lifecycle_transition.compensation_running, 1);
4077        assert_eq!(report.lifecycle_transition.scanner_queued, 6);
4078        assert_eq!(report.lifecycle_transition.scanner_missed, 2);
4079        assert_eq!(report.lifecycle_transition.completed, 4);
4080        assert_eq!(report.lifecycle_transition.failed, 1);
4081    }
4082
4083    #[tokio::test]
4084    async fn report_includes_scanner_lifecycle_expiry_status() {
4085        let metrics = Metrics::new();
4086        metrics.record_scanner_lifecycle_expiry_state(ScannerLifecycleExpiryStateUpdate {
4087            queue_capacity: 16,
4088            queued: 5,
4089            active: 2,
4090            workers: 4,
4091            queue_missed: 3,
4092        });
4093        metrics.record_scanner_expiry_enqueue_result(6, true);
4094        metrics.record_scanner_expiry_enqueue_result(2, false);
4095        metrics.record_scanner_expiry_blocked(4);
4096        metrics.record_scanner_expiry_delete_failed(1);
4097
4098        let report = metrics.report().await;
4099
4100        assert_eq!(report.lifecycle_expiry.current_queue_capacity, 16);
4101        assert_eq!(report.lifecycle_expiry.current_queued, 5);
4102        assert_eq!(report.lifecycle_expiry.current_active, 2);
4103        assert_eq!(report.lifecycle_expiry.current_workers, 4);
4104        assert_eq!(report.lifecycle_expiry.queue_missed, 3);
4105        assert_eq!(report.lifecycle_expiry.scanner_queued, 6);
4106        assert_eq!(report.lifecycle_expiry.scanner_missed, 2);
4107        assert_eq!(report.lifecycle_expiry.scanner_blocked, 4);
4108        assert_eq!(report.lifecycle_expiry.scanner_not_enqueued, 2);
4109        assert_eq!(report.lifecycle_expiry.delete_failed, 1);
4110    }
4111
4112    #[tokio::test]
4113    async fn report_derives_scanner_maintenance_control_status() {
4114        let metrics = Metrics::new();
4115        let start = metrics.start_scan_cycle_work();
4116        metrics.record_scanner_source_checked(ScannerWorkSource::Usage, 3);
4117        metrics.record_scanner_source_missed(ScannerWorkSource::Lifecycle, 2);
4118        metrics.record_scanner_source_queued(ScannerWorkSource::BucketReplication, 1);
4119        metrics.record_scan_cycle_partial_with_source(
4120            Duration::from_secs(5),
4121            ScanCyclePartialReason::Objects,
4122            Some(ScannerWorkSource::Usage),
4123        );
4124        metrics.record_scanner_lifecycle_transition_state(ScannerLifecycleTransitionStateUpdate {
4125            queue_capacity: 4,
4126            queued: 2,
4127            queue_full: 1,
4128            ..Default::default()
4129        });
4130
4131        let report = metrics.report().await;
4132
4133        assert_eq!(report.maintenance_control.primary_control, "blocked_source");
4134        let usage = report
4135            .maintenance_control
4136            .sources
4137            .iter()
4138            .find(|source| source.source == ScannerWorkSource::Usage.as_str())
4139            .expect("usage source control should be visible");
4140        assert_eq!(usage.state, "deferred");
4141        assert_eq!(usage.reason, "partial_cycle");
4142        assert_eq!(usage.current_checked, 3);
4143        assert_eq!(usage.partial_cycles, 1);
4144
4145        let lifecycle = report
4146            .maintenance_control
4147            .sources
4148            .iter()
4149            .find(|source| source.source == ScannerWorkSource::Lifecycle.as_str())
4150            .expect("lifecycle source control should be visible");
4151        assert_eq!(lifecycle.state, "blocked");
4152        assert_eq!(lifecycle.reason, "missed_work");
4153        assert_eq!(lifecycle.current_missed, 2);
4154        assert_eq!(lifecycle.backlog, 2);
4155
4156        let bucket_replication = report
4157            .maintenance_control
4158            .sources
4159            .iter()
4160            .find(|source| source.source == ScannerWorkSource::BucketReplication.as_str())
4161            .expect("bucket replication source control should be visible");
4162        assert_eq!(bucket_replication.state, "active");
4163        assert_eq!(bucket_replication.reason, "queued_work");
4164        assert_eq!(bucket_replication.current_queued, 1);
4165
4166        metrics.finish_scan_cycle_work(start);
4167    }
4168
4169    #[test]
4170    fn unresolved_heal_work_only_reflects_the_active_cycle() {
4171        let metrics = Metrics::new();
4172        assert!(!metrics.current_scan_cycle_has_unresolved_heal_work());
4173
4174        let start = metrics.start_scan_cycle_work();
4175        metrics.record_scanner_source_missed(ScannerWorkSource::Heal, 1);
4176        assert!(metrics.current_scan_cycle_has_unresolved_heal_work());
4177
4178        metrics.finish_scan_cycle_work(start);
4179        assert!(!metrics.current_scan_cycle_has_unresolved_heal_work());
4180
4181        let start = metrics.start_scan_cycle_work();
4182        metrics.record_scanner_source_queued(ScannerWorkSource::Heal, 1);
4183        assert!(metrics.current_scan_cycle_has_unresolved_heal_work());
4184        metrics.finish_scan_cycle_work(start);
4185
4186        let start = metrics.start_scan_cycle_work();
4187        metrics.record_scanner_source_work(
4188            ScannerWorkSource::Bitrot,
4189            ScannerSourceWorkUpdate {
4190                skipped: 1,
4191                ..Default::default()
4192            },
4193        );
4194        assert!(metrics.current_scan_cycle_has_unresolved_heal_work());
4195        metrics.finish_scan_cycle_work(start);
4196
4197        let start = metrics.start_scan_cycle_work();
4198        metrics.record_scanner_source_failed(ScannerWorkSource::Bitrot, 1);
4199        assert!(metrics.current_scan_cycle_has_unresolved_heal_work());
4200        metrics.finish_scan_cycle_work(start);
4201    }
4202
4203    #[tokio::test]
4204    async fn report_marks_transition_failures_as_blocked_lifecycle_control() {
4205        let metrics = Metrics::new();
4206        let _start = metrics.start_scan_cycle_work();
4207
4208        metrics.record_scanner_transition_failed(2);
4209        let report = metrics.report().await;
4210
4211        let lifecycle_work = report
4212            .current_cycle_source_work
4213            .iter()
4214            .find(|work| work.source == ScannerWorkSource::Lifecycle.as_str())
4215            .expect("current lifecycle source work should be visible");
4216        assert_eq!(lifecycle_work.failed, 2);
4217
4218        let lifecycle = report
4219            .maintenance_control
4220            .sources
4221            .iter()
4222            .find(|source| source.source == ScannerWorkSource::Lifecycle.as_str())
4223            .expect("lifecycle source control should be visible");
4224        assert_eq!(report.maintenance_control.primary_control, "blocked_source");
4225        assert_eq!(lifecycle.state, "blocked");
4226        assert_eq!(lifecycle.reason, "transition_failed");
4227        assert_eq!(lifecycle.backlog, 2);
4228    }
4229
4230    #[tokio::test]
4231    async fn report_marks_off_cycle_transition_failures_as_blocked_lifecycle_control() {
4232        let metrics = Metrics::new();
4233        let start = metrics.start_scan_cycle_work();
4234        metrics.finish_scan_cycle_work(start);
4235
4236        metrics.record_scanner_transition_failed(1);
4237        let report = metrics.report().await;
4238
4239        assert!(report.current_cycle_source_work.is_empty());
4240        let lifecycle_work = report
4241            .last_cycle_source_work
4242            .iter()
4243            .find(|work| work.source == ScannerWorkSource::Lifecycle.as_str())
4244            .expect("last-cycle lifecycle source work should be visible");
4245        assert_eq!(lifecycle_work.failed, 1);
4246
4247        let lifecycle = report
4248            .maintenance_control
4249            .sources
4250            .iter()
4251            .find(|source| source.source == ScannerWorkSource::Lifecycle.as_str())
4252            .expect("lifecycle source control should be visible");
4253        assert_eq!(report.maintenance_control.primary_control, "blocked_source");
4254        assert_eq!(lifecycle.state, "blocked");
4255        assert_eq!(lifecycle.reason, "transition_failed");
4256        assert_eq!(lifecycle.backlog, 1);
4257    }
4258
4259    #[tokio::test]
4260    async fn report_marks_expiry_queue_backlog_as_lifecycle_control() {
4261        let metrics = Metrics::new();
4262        metrics.record_scanner_lifecycle_expiry_state(ScannerLifecycleExpiryStateUpdate {
4263            queued: 2,
4264            active: 1,
4265            ..Default::default()
4266        });
4267
4268        let report = metrics.report().await;
4269
4270        let lifecycle = report
4271            .maintenance_control
4272            .sources
4273            .iter()
4274            .find(|source| source.source == ScannerWorkSource::Lifecycle.as_str())
4275            .expect("lifecycle source control should be visible");
4276        assert_eq!(report.maintenance_control.primary_control, "active_source");
4277        assert_eq!(lifecycle.state, "active");
4278        assert_eq!(lifecycle.reason, "expiry_queue_backlog");
4279        assert_eq!(lifecycle.backlog, 3);
4280    }
4281
4282    #[tokio::test]
4283    async fn report_marks_running_transition_compensation_as_lifecycle_backlog() {
4284        let metrics = Metrics::new();
4285        metrics.record_scanner_lifecycle_transition_state(ScannerLifecycleTransitionStateUpdate {
4286            compensation_running: 1,
4287            ..Default::default()
4288        });
4289
4290        let report = metrics.report().await;
4291
4292        let lifecycle = report
4293            .maintenance_control
4294            .sources
4295            .iter()
4296            .find(|source| source.source == ScannerWorkSource::Lifecycle.as_str())
4297            .expect("lifecycle source control should be visible");
4298        assert_eq!(report.maintenance_control.primary_control, "active_source");
4299        assert_eq!(lifecycle.state, "active");
4300        assert_eq!(lifecycle.reason, "transition_compensation_backlog");
4301        assert_eq!(lifecycle.backlog, 1);
4302    }
4303
4304    #[tokio::test]
4305    async fn report_marks_pending_transition_compensation_as_lifecycle_backlog() {
4306        let metrics = Metrics::new();
4307        metrics.record_scanner_lifecycle_transition_state(ScannerLifecycleTransitionStateUpdate {
4308            compensation_pending: 2,
4309            ..Default::default()
4310        });
4311
4312        let report = metrics.report().await;
4313
4314        let lifecycle = report
4315            .maintenance_control
4316            .sources
4317            .iter()
4318            .find(|source| source.source == ScannerWorkSource::Lifecycle.as_str())
4319            .expect("lifecycle source control should be visible");
4320        assert_eq!(report.maintenance_control.primary_control, "active_source");
4321        assert_eq!(lifecycle.state, "active");
4322        assert_eq!(lifecycle.reason, "transition_compensation_backlog");
4323        assert_eq!(lifecycle.backlog, 2);
4324    }
4325
4326    #[tokio::test]
4327    async fn report_uses_last_cycle_source_work_for_maintenance_control_between_cycles() {
4328        let metrics = Metrics::new();
4329        let start = metrics.start_scan_cycle_work();
4330        metrics.record_scanner_source_missed(ScannerWorkSource::Lifecycle, 2);
4331
4332        metrics.finish_scan_cycle_work(start);
4333        let report = metrics.report().await;
4334
4335        assert!(report.current_cycle_source_work.is_empty());
4336        let lifecycle = report
4337            .maintenance_control
4338            .sources
4339            .iter()
4340            .find(|source| source.source == ScannerWorkSource::Lifecycle.as_str())
4341            .expect("lifecycle source control should be visible");
4342        assert_eq!(report.maintenance_control.primary_control, "blocked_source");
4343        assert_eq!(lifecycle.state, "blocked");
4344        assert_eq!(lifecycle.reason, "missed_work");
4345        assert_eq!(lifecycle.current_missed, 2);
4346        assert_eq!(lifecycle.backlog, 2);
4347    }
4348
4349    #[tokio::test]
4350    async fn scanner_metrics_update_source_work() {
4351        let metrics = Metrics::new();
4352        metrics.record_source_work_for_metric(Metric::ScanObject, 3);
4353        metrics.record_source_work_for_metric(Metric::ScanFolder, 2);
4354        metrics.record_source_work_for_metric(Metric::SaveUsage, 1);
4355        metrics.record_source_work_for_metric(Metric::CheckReplication, 4);
4356        metrics.record_source_work_for_metric(Metric::HealCheck, 5);
4357        metrics.record_source_work_for_metric(Metric::HealAbandonedObject, 6);
4358
4359        let report = metrics.report().await;
4360
4361        let usage = report
4362            .source_work
4363            .iter()
4364            .find(|work| work.source == ScannerWorkSource::Usage.as_str())
4365            .expect("usage source work should be visible");
4366        assert_eq!(usage.checked, 5);
4367        assert_eq!(usage.executed, 1);
4368
4369        let bucket_replication = report
4370            .source_work
4371            .iter()
4372            .find(|work| work.source == ScannerWorkSource::BucketReplication.as_str())
4373            .expect("bucket replication source work should be visible");
4374        assert_eq!(bucket_replication.checked, 4);
4375
4376        let heal = report
4377            .source_work
4378            .iter()
4379            .find(|work| work.source == ScannerWorkSource::Heal.as_str())
4380            .expect("heal source work should be visible");
4381        assert_eq!(heal.checked, 5);
4382        assert_eq!(heal.executed, 0);
4383    }
4384
4385    #[tokio::test]
4386    async fn report_includes_scanner_replication_repair_work() {
4387        let metrics = Metrics::new();
4388
4389        metrics.record_scanner_replication_repair_work(
4390            ScannerReplicationRepairKind::BucketObject,
4391            ScannerSourceWorkUpdate::queued(2),
4392        );
4393        metrics.record_scanner_replication_repair_work(
4394            ScannerReplicationRepairKind::BucketDeleteMarker,
4395            ScannerSourceWorkUpdate::missed(1),
4396        );
4397        metrics.record_scanner_replication_repair_work(
4398            ScannerReplicationRepairKind::BucketVersionPurge,
4399            ScannerSourceWorkUpdate {
4400                skipped: 3,
4401                ..Default::default()
4402            },
4403        );
4404        metrics.record_scanner_replication_repair_work(
4405            ScannerReplicationRepairKind::BucketExistingObject,
4406            ScannerSourceWorkUpdate::queued(4),
4407        );
4408        metrics.record_scanner_replication_repair_work(
4409            ScannerReplicationRepairKind::SitePassiveRequeue,
4410            ScannerSourceWorkUpdate::missed(5),
4411        );
4412        metrics.record_scanner_replication_repair_work(
4413            ScannerReplicationRepairKind::SiteActiveResync,
4414            ScannerSourceWorkUpdate::queued(6),
4415        );
4416
4417        let report = metrics.report().await;
4418        let object = report
4419            .replication_repair
4420            .iter()
4421            .find(|repair| repair.source == "bucket_replication" && repair.kind == "object")
4422            .expect("bucket object repair work should be visible");
4423        assert_eq!(object.scanner_role, "repair_admission");
4424        assert_eq!(object.execution_owner, "bucket_replication_queue");
4425        assert_eq!(object.queued, 2);
4426
4427        let delete_marker = report
4428            .replication_repair
4429            .iter()
4430            .find(|repair| repair.source == "bucket_replication" && repair.kind == "delete_marker")
4431            .expect("bucket delete-marker repair work should be visible");
4432        assert_eq!(delete_marker.scanner_role, "repair_admission");
4433        assert_eq!(delete_marker.execution_owner, "bucket_replication_queue");
4434        assert_eq!(delete_marker.missed, 1);
4435
4436        let version_purge = report
4437            .replication_repair
4438            .iter()
4439            .find(|repair| repair.source == "bucket_replication" && repair.kind == "version_purge")
4440            .expect("bucket version purge repair work should be visible");
4441        assert_eq!(version_purge.scanner_role, "repair_admission");
4442        assert_eq!(version_purge.execution_owner, "bucket_replication_queue");
4443        assert_eq!(version_purge.skipped, 3);
4444
4445        let existing_object = report
4446            .replication_repair
4447            .iter()
4448            .find(|repair| repair.source == "bucket_replication" && repair.kind == "existing_object")
4449            .expect("bucket existing object repair work should be visible");
4450        assert_eq!(existing_object.scanner_role, "repair_admission");
4451        assert_eq!(existing_object.execution_owner, "bucket_replication_queue");
4452        assert_eq!(existing_object.queued, 4);
4453
4454        let passive_requeue = report
4455            .replication_repair
4456            .iter()
4457            .find(|repair| repair.source == "site_replication" && repair.kind == "passive_requeue")
4458            .expect("site passive requeue boundary work should be visible");
4459        assert_eq!(passive_requeue.scanner_role, "boundary_signal");
4460        assert_eq!(passive_requeue.execution_owner, "site_replication_runtime");
4461        assert_eq!(passive_requeue.missed, 5);
4462
4463        let active_resync = report
4464            .replication_repair
4465            .iter()
4466            .find(|repair| repair.source == "site_replication" && repair.kind == "active_resync")
4467            .expect("site active resync boundary work should be visible");
4468        assert_eq!(active_resync.scanner_role, "boundary_signal");
4469        assert_eq!(active_resync.execution_owner, "site_replication_runtime");
4470        assert_eq!(active_resync.queued, 6);
4471    }
4472
4473    #[test]
4474    fn scanner_replication_repair_kind_sources_are_stable() {
4475        let bucket_kinds = [
4476            ScannerReplicationRepairKind::BucketObject,
4477            ScannerReplicationRepairKind::BucketDeleteMarker,
4478            ScannerReplicationRepairKind::BucketVersionPurge,
4479            ScannerReplicationRepairKind::BucketExistingObject,
4480        ];
4481        for kind in bucket_kinds {
4482            assert_eq!(kind.source(), ScannerWorkSource::BucketReplication);
4483            assert_eq!(kind.scanner_role(), ScannerReplicationRepairRole::RepairAdmission);
4484            assert_eq!(kind.execution_owner(), ScannerReplicationExecutionOwner::BucketReplicationQueue);
4485        }
4486
4487        assert_eq!(
4488            ScannerReplicationRepairKind::SitePassiveRequeue.source(),
4489            ScannerWorkSource::SiteReplication
4490        );
4491        assert_eq!(
4492            ScannerReplicationRepairKind::SiteActiveResync.source(),
4493            ScannerWorkSource::SiteReplication
4494        );
4495        assert_eq!(
4496            ScannerReplicationRepairKind::SitePassiveRequeue.scanner_role(),
4497            ScannerReplicationRepairRole::BoundarySignal
4498        );
4499        assert_eq!(
4500            ScannerReplicationRepairKind::SiteActiveResync.scanner_role(),
4501            ScannerReplicationRepairRole::BoundarySignal
4502        );
4503        assert_eq!(
4504            ScannerReplicationRepairKind::SitePassiveRequeue.execution_owner(),
4505            ScannerReplicationExecutionOwner::SiteReplicationRuntime
4506        );
4507        assert_eq!(
4508            ScannerReplicationRepairKind::SiteActiveResync.execution_owner(),
4509            ScannerReplicationExecutionOwner::SiteReplicationRuntime
4510        );
4511        assert_eq!(ScannerReplicationRepairKind::SitePassiveRequeue.as_str(), "passive_requeue");
4512        assert_eq!(ScannerReplicationRepairKind::SiteActiveResync.as_str(), "active_resync");
4513    }
4514
4515    #[tokio::test]
4516    async fn report_includes_scan_cycle_source_work() {
4517        let metrics = Metrics::new();
4518        metrics.record_scanner_source_work(
4519            ScannerWorkSource::Usage,
4520            ScannerSourceWorkUpdate {
4521                checked: 10,
4522                executed: 1,
4523                ..Default::default()
4524            },
4525        );
4526
4527        let start = metrics.start_scan_cycle_work();
4528        metrics.record_scanner_source_work(
4529            ScannerWorkSource::Usage,
4530            ScannerSourceWorkUpdate {
4531                checked: 3,
4532                executed: 2,
4533                ..Default::default()
4534            },
4535        );
4536        metrics.record_scanner_source_work(
4537            ScannerWorkSource::Lifecycle,
4538            ScannerSourceWorkUpdate {
4539                queued: 1,
4540                executed: 1,
4541                missed: 2,
4542                ..Default::default()
4543            },
4544        );
4545
4546        let report = metrics.report().await;
4547        let usage = report
4548            .current_cycle_source_work
4549            .iter()
4550            .find(|work| work.source == ScannerWorkSource::Usage.as_str())
4551            .expect("current cycle usage source work should be visible");
4552        assert_eq!(usage.checked, 3);
4553        assert_eq!(usage.executed, 2);
4554
4555        let lifecycle = report
4556            .current_cycle_source_work
4557            .iter()
4558            .find(|work| work.source == ScannerWorkSource::Lifecycle.as_str())
4559            .expect("current cycle lifecycle source work should be visible");
4560        assert_eq!(lifecycle.queued, 1);
4561        assert_eq!(lifecycle.executed, 1);
4562        assert_eq!(lifecycle.missed, 2);
4563
4564        metrics.finish_scan_cycle_work(start);
4565        let report = metrics.report().await;
4566        assert!(report.current_cycle_source_work.is_empty());
4567
4568        let usage = report
4569            .last_cycle_source_work
4570            .iter()
4571            .find(|work| work.source == ScannerWorkSource::Usage.as_str())
4572            .expect("last cycle usage source work should be visible");
4573        assert_eq!(usage.checked, 3);
4574        assert_eq!(usage.executed, 2);
4575
4576        let lifecycle = report
4577            .last_cycle_source_work
4578            .iter()
4579            .find(|work| work.source == ScannerWorkSource::Lifecycle.as_str())
4580            .expect("last cycle lifecycle source work should be visible");
4581        assert_eq!(lifecycle.queued, 1);
4582        assert_eq!(lifecycle.executed, 1);
4583        assert_eq!(lifecycle.missed, 2);
4584    }
4585
4586    #[tokio::test]
4587    async fn report_preserves_current_cycle_started_time() {
4588        let previous_init_time = *crate::init_time::GLOBAL_INIT_TIME.read().await;
4589        let init_time = Utc::now() - chrono::Duration::hours(1);
4590        let cycle_started = Utc::now();
4591        *crate::init_time::GLOBAL_INIT_TIME.write().await = Some(init_time);
4592
4593        let metrics = Metrics::new();
4594        metrics
4595            .set_cycle(Some(CurrentCycle {
4596                current: 7,
4597                started: cycle_started,
4598                ..Default::default()
4599            }))
4600            .await;
4601
4602        let report = metrics.report().await;
4603        *crate::init_time::GLOBAL_INIT_TIME.write().await = previous_init_time;
4604
4605        assert_eq!(report.current_started, chrono_to_jiff_timestamp(cycle_started));
4606    }
4607
4608    #[tokio::test]
4609    async fn report_includes_current_scan_mode() {
4610        let metrics = Metrics::new();
4611        metrics.set_current_scan_mode(HealScanMode::Deep);
4612
4613        let report = metrics.report().await;
4614
4615        assert_eq!(report.current_scan_mode, HealScanMode::Deep.as_str());
4616    }
4617
4618    #[tokio::test]
4619    async fn report_includes_scanner_leader_liveness() {
4620        let metrics = Metrics::new();
4621
4622        metrics.record_scanner_leader_liveness("contended", false, "lock busy").await;
4623        metrics.record_scanner_cycle_end_time();
4624        let report = metrics.report().await;
4625
4626        assert_eq!(report.leader_lock_state, "contended");
4627        assert!(!report.leader_lock_held_by_this_process);
4628        assert_eq!(report.leader_lock_last_error, "lock busy");
4629        assert!(report.leader_lock_last_update_unix_secs > 0);
4630        assert!(report.last_cycle_end_unix_secs > 0);
4631    }
4632
4633    #[tokio::test]
4634    async fn report_includes_last_scan_cycle_result() {
4635        let metrics = Metrics::new();
4636        metrics.record_scan_cycle_complete(false, Duration::from_millis(1500));
4637
4638        let report = metrics.report().await;
4639
4640        assert_eq!(report.last_cycle_result, SCAN_CYCLE_RESULT_ERROR_LABEL);
4641        assert_eq!(report.last_cycle_result_code, SCAN_CYCLE_RESULT_ERROR as u64);
4642        assert_eq!(report.last_cycle_duration_seconds, 1.5);
4643        assert_eq!(report.failed_cycles, 1);
4644    }
4645
4646    #[tokio::test]
4647    async fn report_tracks_superseded_cycle_without_failed_increment() {
4648        let metrics = Metrics::new();
4649        metrics.record_scan_cycle_superseded(Duration::from_millis(750));
4650
4651        let report = metrics.report().await;
4652
4653        assert_eq!(report.last_cycle_result, SCAN_CYCLE_RESULT_SUPERSEDED_LABEL);
4654        assert_eq!(report.last_cycle_result_code, u64::from(SCAN_CYCLE_RESULT_SUPERSEDED));
4655        assert_eq!(report.last_cycle_duration_seconds, 0.75);
4656        assert_eq!(report.failed_cycles, 0);
4657        assert_eq!(report.superseded_cycles, 1);
4658        assert_eq!(report.partial_cycles, 0);
4659    }
4660
4661    #[tokio::test]
4662    async fn report_tracks_deferred_cycle_without_failed_increment() {
4663        let metrics = Metrics::new();
4664        metrics.record_scan_cycle_deferred(Duration::from_millis(250));
4665
4666        let report = metrics.report().await;
4667
4668        assert_eq!(report.last_cycle_result, SCAN_CYCLE_RESULT_DEFERRED_LABEL);
4669        assert_eq!(report.last_cycle_result_code, u64::from(SCAN_CYCLE_RESULT_DEFERRED));
4670        assert_eq!(report.last_cycle_duration_seconds, 0.25);
4671        assert_eq!(report.failed_cycles, 0);
4672        assert_eq!(report.superseded_cycles, 0);
4673        assert_eq!(report.partial_cycles, 0);
4674    }
4675
4676    #[tokio::test]
4677    async fn report_tracks_successful_scan_cycle_without_failed_increment() {
4678        let metrics = Metrics::new();
4679        metrics.record_scan_cycle_partial_with_source(
4680            Duration::from_millis(500),
4681            ScanCyclePartialReason::Runtime,
4682            Some(ScannerWorkSource::Heal),
4683        );
4684        metrics.record_scan_cycle_complete(true, Duration::from_secs(2));
4685
4686        let report = metrics.report().await;
4687
4688        assert_eq!(report.last_cycle_result, SCAN_CYCLE_RESULT_SUCCESS_LABEL);
4689        assert_eq!(report.last_cycle_result_code, SCAN_CYCLE_RESULT_SUCCESS as u64);
4690        assert_eq!(report.last_cycle_partial_reason, SCAN_CYCLE_RESULT_UNKNOWN_LABEL);
4691        assert_eq!(report.last_cycle_partial_reason_code, ScanCyclePartialReason::Unknown as u8 as u64);
4692        assert_eq!(report.last_cycle_partial_source, SCAN_CYCLE_RESULT_UNKNOWN_LABEL);
4693        assert_eq!(report.last_cycle_partial_source_code, 0);
4694        assert_eq!(report.last_cycle_duration_seconds, 2.0);
4695        assert_eq!(report.failed_cycles, 0);
4696        assert_eq!(report.partial_cycles, 1);
4697    }
4698
4699    #[tokio::test]
4700    async fn report_tracks_partial_scan_cycle_without_failed_increment() {
4701        let metrics = Metrics::new();
4702        metrics.record_scan_cycle_partial_with_source(
4703            Duration::from_millis(2500),
4704            ScanCyclePartialReason::Directories,
4705            Some(ScannerWorkSource::Usage),
4706        );
4707
4708        let report = metrics.report().await;
4709
4710        assert_eq!(report.last_cycle_result, SCAN_CYCLE_RESULT_PARTIAL_LABEL);
4711        assert_eq!(report.last_cycle_result_code, SCAN_CYCLE_RESULT_PARTIAL as u64);
4712        assert_eq!(report.last_cycle_partial_reason, "directories");
4713        assert_eq!(report.last_cycle_partial_reason_code, ScanCyclePartialReason::Directories as u8 as u64);
4714        assert_eq!(report.last_cycle_partial_source, ScannerWorkSource::Usage.as_str());
4715        assert_eq!(report.last_cycle_partial_source_code, 1);
4716        assert_eq!(report.last_cycle_duration_seconds, 2.5);
4717        assert_eq!(report.failed_cycles, 0);
4718        assert_eq!(report.partial_cycles, 1);
4719        assert_eq!(report.partial_cycles_directories, 1);
4720        assert_eq!(report.partial_cycles_runtime, 0);
4721        assert_eq!(report.partial_cycles_objects, 0);
4722        assert_eq!(report.partial_cycles_unknown, 0);
4723        let usage = report
4724            .partial_cycles_by_source
4725            .iter()
4726            .find(|source| source.source == ScannerWorkSource::Usage.as_str())
4727            .expect("usage partial source count should be visible");
4728        assert_eq!(usage.cycles, 1);
4729    }
4730
4731    #[tokio::test]
4732    async fn report_includes_last_scan_cycle_work() {
4733        let metrics = Metrics::new();
4734        metrics.record_scan_cycle_work(ScanCycleWorkSnapshot {
4735            objects_scanned: 11,
4736            directories_scanned: 7,
4737            bucket_drive_scans: 3,
4738            bucket_drive_failures: 2,
4739            yield_events: 5,
4740            yield_duration_millis: 250,
4741            throttle_sleep_events: 7,
4742            throttle_sleep_duration_millis: 500,
4743            ilm_actions: 13,
4744            heal_objects: 2,
4745            replication_checks: 4,
4746            usage_saves: 6,
4747            ..Default::default()
4748        });
4749
4750        let report = metrics.report().await;
4751
4752        assert_eq!(report.last_cycle_objects_scanned, 11);
4753        assert_eq!(report.last_cycle_directories_scanned, 7);
4754        assert_eq!(report.last_cycle_bucket_drive_scans, 3);
4755        assert_eq!(report.last_cycle_bucket_drive_failures, 2);
4756        assert_eq!(report.last_cycle_yield_events, 5);
4757        assert_eq!(report.last_cycle_yield_duration_seconds, 0.25);
4758        assert_eq!(report.last_cycle_throttle_sleep_events, 7);
4759        assert_eq!(report.last_cycle_throttle_sleep_duration_seconds, 0.5);
4760        assert_eq!(report.last_cycle_ilm_actions, 13);
4761        assert_eq!(report.last_cycle_heal_objects, 2);
4762        assert_eq!(report.last_cycle_replication_checks, 4);
4763        assert_eq!(report.last_cycle_usage_saves, 6);
4764    }
4765
4766    #[tokio::test]
4767    async fn report_includes_bucket_drive_scan_starts() {
4768        let metrics = Metrics::new();
4769        metrics.record_scan_bucket_drive_start(ScannerWorkSource::Usage, "bucket-a", "/mnt/data/1");
4770        metrics.record_scan_bucket_drive_failure();
4771
4772        let report = metrics.report().await;
4773
4774        assert_eq!(report.life_time_ops.get("scan_bucket_drive_start"), Some(&1));
4775        assert_eq!(report.life_time_ops.get("scan_bucket_drive_failure"), Some(&1));
4776    }
4777
4778    #[tokio::test]
4779    async fn active_bucket_drive_snapshot_is_structured_and_retired_on_end() {
4780        let metrics = Metrics::new();
4781        metrics.record_scan_bucket_drive_start(ScannerWorkSource::Usage, "bucket-a", "/mnt/data/1");
4782        metrics.record_scan_bucket_drive_start(ScannerWorkSource::Usage, "bucket-a", "/mnt/data/1");
4783        let active = metrics.scanner_runtime_details_report().active_bucket_drive_scans;
4784        assert_eq!(active.len(), 1);
4785        assert_eq!(active[0].source, ScannerWorkSource::Usage.as_str());
4786        assert_eq!(active[0].bucket, "bucket-a");
4787        assert_eq!(active[0].drive, "/mnt/data/1");
4788        assert_eq!(active[0].count, 2);
4789
4790        metrics.record_scan_bucket_drive_end(ScannerWorkSource::Usage, "bucket-a", "/mnt/data/1");
4791        assert_eq!(metrics.scanner_runtime_details_report().active_bucket_drive_scans[0].count, 1);
4792        metrics.record_scan_bucket_drive_end(ScannerWorkSource::Usage, "bucket-a", "/mnt/data/1");
4793        assert!(metrics.scanner_runtime_details_report().active_bucket_drive_scans.is_empty());
4794
4795        metrics.record_scan_bucket_drive_start(ScannerWorkSource::Usage, "", "/mnt/data/1");
4796        assert!(metrics.scanner_runtime_details_report().active_bucket_drive_scans.is_empty());
4797    }
4798
4799    #[tokio::test]
4800    async fn report_includes_structured_bucket_drive_results() {
4801        let metrics = Metrics::new();
4802        metrics.record_scanner_bucket_drive_result("photos", "/data1", "success");
4803
4804        let cycle_start = metrics.start_scan_cycle_work();
4805        metrics.record_scanner_bucket_drive_result("photos", "/data1", "partial");
4806
4807        let active_report = metrics.scanner_runtime_details_report();
4808        assert_eq!(
4809            active_report.current_cycle_bucket_drive_results,
4810            vec![ScannerBucketDriveResultSnapshot {
4811                bucket: "photos".to_string(),
4812                drive: "/data1".to_string(),
4813                result: "partial".to_string(),
4814                count: 1,
4815            }]
4816        );
4817
4818        metrics.finish_scan_cycle_work(cycle_start);
4819
4820        let report = metrics.scanner_runtime_details_report();
4821
4822        assert_eq!(
4823            report.bucket_drive_results,
4824            vec![
4825                ScannerBucketDriveResultSnapshot {
4826                    bucket: "photos".to_string(),
4827                    drive: "/data1".to_string(),
4828                    result: "partial".to_string(),
4829                    count: 1,
4830                },
4831                ScannerBucketDriveResultSnapshot {
4832                    bucket: "photos".to_string(),
4833                    drive: "/data1".to_string(),
4834                    result: "success".to_string(),
4835                    count: 1,
4836                },
4837            ]
4838        );
4839        assert!(report.current_cycle_bucket_drive_results.is_empty());
4840        assert_eq!(
4841            report.last_cycle_bucket_drive_results,
4842            vec![ScannerBucketDriveResultSnapshot {
4843                bucket: "photos".to_string(),
4844                drive: "/data1".to_string(),
4845                result: "partial".to_string(),
4846                count: 1,
4847            }]
4848        );
4849    }
4850
4851    #[tokio::test]
4852    async fn scanner_bucket_drive_results_are_bounded() {
4853        let metrics = Metrics::new();
4854        for index in 0..MAX_SCANNER_BUCKET_DRIVE_RESULT_KEYS {
4855            metrics.record_scanner_bucket_drive_result(&format!("bucket-{index}"), "/data1", "success");
4856        }
4857        metrics.record_scanner_bucket_drive_result("overflow", "/data1", "success");
4858
4859        let report = metrics.scanner_runtime_details_report();
4860
4861        assert_eq!(report.bucket_drive_results.len(), MAX_SCANNER_BUCKET_DRIVE_RESULT_KEYS);
4862        assert!(
4863            report
4864                .bucket_drive_results
4865                .iter()
4866                .any(|snapshot| snapshot.bucket == "overflow")
4867        );
4868        assert!(
4869            report
4870                .bucket_drive_results
4871                .iter()
4872                .all(|snapshot| snapshot.bucket != "bucket-0")
4873        );
4874    }
4875
4876    #[tokio::test]
4877    async fn scanner_bucket_drive_result_eviction_keeps_recent_keys() {
4878        let metrics = Metrics::new();
4879        for index in 0..MAX_SCANNER_BUCKET_DRIVE_RESULT_KEYS {
4880            metrics.record_scanner_bucket_drive_result(&format!("bucket-{index}"), "/data1", "success");
4881        }
4882        metrics.record_scanner_bucket_drive_result("bucket-0", "/data1", "success");
4883        metrics.record_scanner_bucket_drive_result("overflow", "/data1", "success");
4884
4885        let report = metrics.scanner_runtime_details_report();
4886
4887        assert_eq!(report.bucket_drive_results.len(), MAX_SCANNER_BUCKET_DRIVE_RESULT_KEYS);
4888        assert!(
4889            report
4890                .bucket_drive_results
4891                .iter()
4892                .any(|snapshot| snapshot.bucket == "bucket-0" && snapshot.count == 2)
4893        );
4894        assert!(
4895            report
4896                .bucket_drive_results
4897                .iter()
4898                .all(|snapshot| snapshot.bucket != "bucket-1")
4899        );
4900    }
4901
4902    #[tokio::test]
4903    async fn scanner_bucket_drive_result_eviction_survives_full_refresh() {
4904        let metrics = Metrics::new();
4905        for index in 0..MAX_SCANNER_BUCKET_DRIVE_RESULT_KEYS {
4906            metrics.record_scanner_bucket_drive_result(&format!("bucket-{index}"), "/data1", "success");
4907        }
4908        for index in 0..MAX_SCANNER_BUCKET_DRIVE_RESULT_KEYS {
4909            metrics.record_scanner_bucket_drive_result(&format!("bucket-{index}"), "/data1", "success");
4910        }
4911        metrics.record_scanner_bucket_drive_result("overflow", "/data1", "success");
4912
4913        let report = metrics.scanner_runtime_details_report();
4914
4915        assert_eq!(report.bucket_drive_results.len(), MAX_SCANNER_BUCKET_DRIVE_RESULT_KEYS);
4916        assert!(
4917            report
4918                .bucket_drive_results
4919                .iter()
4920                .any(|snapshot| snapshot.bucket == "overflow")
4921        );
4922        assert!(
4923            report
4924                .bucket_drive_results
4925                .iter()
4926                .all(|snapshot| snapshot.bucket != "bucket-0")
4927        );
4928    }
4929
4930    #[tokio::test]
4931    async fn report_includes_usage_freshness_status() {
4932        let metrics = Metrics::new();
4933        metrics.record_scanner_dirty_usage_pending(2);
4934        metrics.record_scanner_dirty_usage_cycle_snapshot(1);
4935        metrics.record_scanner_dirty_usage_cycle_clear(1, 1);
4936        metrics.record_scanner_usage_save_result(ScannerUsageSaveResult::Success);
4937        metrics.record_scanner_usage_deferred("data_movement");
4938
4939        let report = metrics.report().await;
4940
4941        assert_eq!(report.usage_freshness.dirty_pending_buckets, 1);
4942        assert!(report.usage_freshness.last_dirty_mark_unix_secs > 0);
4943        assert!(report.usage_freshness.last_dirty_clear_unix_secs > 0);
4944        assert_eq!(report.usage_freshness.last_cycle_dirty_buckets, 1);
4945        assert_eq!(report.usage_freshness.last_cycle_cleared_dirty_buckets, 1);
4946        assert!(report.usage_freshness.last_usage_save_unix_secs > 0);
4947        assert_eq!(report.usage_freshness.last_usage_save_result, "success");
4948        assert_eq!(report.usage_freshness.last_usage_save_result_code, 1);
4949        assert!(report.usage_freshness.deferred_pending);
4950        assert_eq!(report.usage_freshness.deferred_total, 1);
4951        assert!(report.usage_freshness.last_deferred_unix_secs > 0);
4952        assert_eq!(report.usage_freshness.last_deferred_reason, "data_movement");
4953
4954        metrics.record_scanner_usage_durable_success();
4955        let report = metrics.report().await;
4956        assert!(!report.usage_freshness.deferred_pending);
4957        assert_eq!(report.usage_freshness.deferred_total, 1);
4958        assert!(report.usage_freshness.last_durable_success_unix_secs > 0);
4959        assert_eq!(report.usage_freshness.last_publication_state, "success");
4960
4961        metrics.record_scanner_usage_publication("no_update", "no_update");
4962        let report = metrics.report().await;
4963        assert_eq!(report.usage_freshness.last_publication_state, "no_update");
4964        assert_eq!(report.usage_freshness.last_publication_reason, "no_update");
4965    }
4966
4967    #[tokio::test]
4968    async fn dirty_usage_cycle_snapshot_resets_stale_cleared_count() {
4969        let metrics = Metrics::new();
4970        metrics.record_scanner_dirty_usage_cycle_snapshot(2);
4971        metrics.record_scanner_dirty_usage_cycle_clear(2, 0);
4972        metrics.record_scanner_dirty_usage_cycle_snapshot(1);
4973
4974        let report = metrics.report().await;
4975
4976        assert_eq!(report.usage_freshness.last_cycle_dirty_buckets, 1);
4977        assert_eq!(report.usage_freshness.last_cycle_cleared_dirty_buckets, 0);
4978    }
4979
4980    #[tokio::test]
4981    async fn report_includes_active_scan_cycle_work() {
4982        let metrics = Metrics::new();
4983        metrics.operations[Metric::ScanObject as usize].store(10, Ordering::Relaxed);
4984        metrics.operations[Metric::ScanFolder as usize].store(5, Ordering::Relaxed);
4985        metrics.operations[Metric::ScanBucketDrive as usize].store(1, Ordering::Relaxed);
4986        metrics.operations[Metric::ScanBucketDriveFailure as usize].store(1, Ordering::Relaxed);
4987        metrics.operations[Metric::Yield as usize].store(2, Ordering::Relaxed);
4988        metrics.operations[Metric::ThrottleSleep as usize].store(3, Ordering::Relaxed);
4989        metrics.operations[Metric::Ilm as usize].store(6, Ordering::Relaxed);
4990        metrics.record_scanner_ilm_action(6);
4991        metrics.operations[Metric::HealAbandonedObject as usize].store(4, Ordering::Relaxed);
4992        metrics.operations[Metric::CheckReplication as usize].store(5, Ordering::Relaxed);
4993        metrics.operations[Metric::SaveUsage as usize].store(3, Ordering::Relaxed);
4994        metrics.scanner_yield_duration_millis.store(100, Ordering::Relaxed);
4995        metrics.scanner_throttle_sleep_duration_millis.store(200, Ordering::Relaxed);
4996
4997        let start = metrics.start_scan_cycle_work();
4998        metrics.operations[Metric::ScanObject as usize].store(17, Ordering::Relaxed);
4999        metrics.operations[Metric::ScanFolder as usize].store(8, Ordering::Relaxed);
5000        metrics.operations[Metric::ScanBucketDrive as usize].store(3, Ordering::Relaxed);
5001        metrics.operations[Metric::ScanBucketDriveFailure as usize].store(4, Ordering::Relaxed);
5002        metrics.operations[Metric::Ilm as usize].store(15, Ordering::Relaxed);
5003        metrics.record_scanner_ilm_action(9);
5004        metrics.operations[Metric::HealAbandonedObject as usize].store(7, Ordering::Relaxed);
5005        metrics.operations[Metric::CheckReplication as usize].store(11, Ordering::Relaxed);
5006        metrics.operations[Metric::SaveUsage as usize].store(5, Ordering::Relaxed);
5007        metrics.record_scanner_yield(Duration::from_millis(150));
5008        metrics.record_scanner_throttle_sleep(Duration::from_millis(250));
5009
5010        let report = metrics.report().await;
5011
5012        assert!(report.current_cycle_active);
5013        assert_eq!(report.current_cycle, 0);
5014        assert_eq!(report.current_cycle_objects_scanned, 7);
5015        assert_eq!(report.current_cycle_directories_scanned, 3);
5016        assert_eq!(report.current_cycle_bucket_drive_scans, 2);
5017        assert_eq!(report.current_cycle_bucket_drive_failures, 3);
5018        assert_eq!(report.current_cycle_yield_events, 1);
5019        assert_eq!(report.current_cycle_yield_duration_seconds, 0.15);
5020        assert_eq!(report.current_cycle_throttle_sleep_events, 1);
5021        assert_eq!(report.current_cycle_throttle_sleep_duration_seconds, 0.25);
5022        assert_eq!(report.current_cycle_ilm_actions, 9);
5023        assert_eq!(report.current_cycle_heal_objects, 3);
5024        assert_eq!(report.current_cycle_replication_checks, 6);
5025        assert_eq!(report.current_cycle_usage_saves, 2);
5026
5027        metrics.finish_scan_cycle_work(start);
5028        let report = metrics.report().await;
5029
5030        assert!(!report.current_cycle_active);
5031        assert_eq!(report.current_cycle, 0);
5032        assert_eq!(report.current_cycle_objects_scanned, 0);
5033        assert_eq!(report.current_cycle_directories_scanned, 0);
5034        assert_eq!(report.current_cycle_bucket_drive_scans, 0);
5035        assert_eq!(report.current_cycle_bucket_drive_failures, 0);
5036        assert_eq!(report.current_cycle_yield_events, 0);
5037        assert_eq!(report.current_cycle_yield_duration_seconds, 0.0);
5038        assert_eq!(report.current_cycle_throttle_sleep_events, 0);
5039        assert_eq!(report.current_cycle_throttle_sleep_duration_seconds, 0.0);
5040        assert_eq!(report.current_cycle_ilm_actions, 0);
5041        assert_eq!(report.current_cycle_heal_objects, 0);
5042        assert_eq!(report.current_cycle_replication_checks, 0);
5043        assert_eq!(report.current_cycle_usage_saves, 0);
5044        assert_eq!(report.last_cycle_objects_scanned, 7);
5045        assert_eq!(report.last_cycle_directories_scanned, 3);
5046        assert_eq!(report.last_cycle_bucket_drive_scans, 2);
5047        assert_eq!(report.last_cycle_bucket_drive_failures, 3);
5048        assert_eq!(report.last_cycle_yield_events, 1);
5049        assert_eq!(report.last_cycle_yield_duration_seconds, 0.15);
5050        assert_eq!(report.last_cycle_throttle_sleep_events, 1);
5051        assert_eq!(report.last_cycle_throttle_sleep_duration_seconds, 0.25);
5052        assert_eq!(report.last_cycle_ilm_actions, 9);
5053        assert_eq!(report.last_cycle_heal_objects, 3);
5054        assert_eq!(report.last_cycle_replication_checks, 6);
5055        assert_eq!(report.last_cycle_usage_saves, 2);
5056    }
5057
5058    #[tokio::test]
5059    async fn scan_cycle_activity_and_cycle_state_publish_together() {
5060        let metrics = Metrics::new();
5061        let cycle_started = Utc::now() - chrono::Duration::seconds(5);
5062        let active_cycle = CurrentCycle {
5063            current: 12,
5064            next: 13,
5065            started: cycle_started,
5066            ..Default::default()
5067        };
5068
5069        let cycle_state = metrics.cycle_info.read().await;
5070        let mut start_transition = Box::pin(metrics.start_scan_cycle_work_with_cycle(active_cycle));
5071        let waker = std::task::Waker::noop();
5072        let mut context = std::task::Context::from_waker(waker);
5073        assert!(start_transition.as_mut().poll(&mut context).is_pending());
5074        assert!(!metrics.current_scan_cycle_work_active.load(Ordering::Acquire));
5075        drop(cycle_state);
5076
5077        let start = start_transition.await;
5078        let active = metrics.report().await;
5079        assert!(active.current_cycle_active);
5080        assert_eq!(active.current_cycle, 12);
5081        assert_eq!(active.current_started, chrono_to_jiff_timestamp(cycle_started));
5082
5083        let idle_cycle = CurrentCycle {
5084            current: 0,
5085            next: 13,
5086            started: cycle_started,
5087            ..Default::default()
5088        };
5089        let cycle_state = metrics.cycle_info.read().await;
5090        let mut finish_transition = Box::pin(metrics.finish_scan_cycle_work_with_cycle(start, idle_cycle));
5091        assert!(finish_transition.as_mut().poll(&mut context).is_pending());
5092        assert!(metrics.current_scan_cycle_work_active.load(Ordering::Acquire));
5093        drop(cycle_state);
5094
5095        finish_transition.await;
5096        let idle = metrics.report().await;
5097        assert!(!idle.current_cycle_active);
5098        assert_eq!(idle.current_cycle, 0);
5099    }
5100
5101    #[tokio::test]
5102    async fn report_keeps_cycle_identity_and_work_in_one_snapshot() {
5103        let metrics = Metrics::new();
5104        let cycle_ten = CurrentCycle {
5105            current: 10,
5106            next: 11,
5107            started: Utc::now() - chrono::Duration::seconds(10),
5108            ..Default::default()
5109        };
5110        let cycle_ten_start = metrics.start_scan_cycle_work_with_cycle(cycle_ten.clone()).await;
5111        metrics.operations[Metric::ScanObject as usize].store(1, Ordering::Relaxed);
5112        metrics.record_scanner_bucket_drive_result("cycle-ten", "/data1", "partial");
5113
5114        let paths = metrics.current_paths.write().await;
5115        let mut report = Box::pin(metrics.report_with_runtime_details());
5116        let waker = std::task::Waker::noop();
5117        let mut context = std::task::Context::from_waker(waker);
5118        assert!(report.as_mut().poll(&mut context).is_pending());
5119
5120        metrics
5121            .finish_scan_cycle_work_with_cycle(cycle_ten_start, CurrentCycle { current: 0, ..cycle_ten })
5122            .await;
5123        let cycle_eleven_start = metrics
5124            .start_scan_cycle_work_with_cycle(CurrentCycle {
5125                current: 11,
5126                next: 12,
5127                started: Utc::now(),
5128                ..Default::default()
5129            })
5130            .await;
5131        metrics.operations[Metric::ScanObject as usize].store(101, Ordering::Relaxed);
5132        metrics.record_scanner_bucket_drive_result("cycle-eleven", "/data1", "partial");
5133
5134        drop(paths);
5135        let (snapshot, runtime_details) = report.await;
5136
5137        assert_eq!(snapshot.current_cycle, 10);
5138        assert_eq!(snapshot.current_cycle_objects_scanned, 1);
5139        assert_eq!(
5140            runtime_details.current_cycle_bucket_drive_results,
5141            vec![ScannerBucketDriveResultSnapshot {
5142                bucket: "cycle-ten".to_string(),
5143                drive: "/data1".to_string(),
5144                result: "partial".to_string(),
5145                count: 1,
5146            }]
5147        );
5148
5149        metrics
5150            .finish_scan_cycle_work_with_cycle(cycle_eleven_start, CurrentCycle::default())
5151            .await;
5152    }
5153
5154    #[tokio::test]
5155    async fn scanner_cycle_ilm_actions_ignore_global_ilm_work() {
5156        let metrics = Metrics::new();
5157        metrics.operations[Metric::Ilm as usize].store(4, Ordering::Relaxed);
5158
5159        let start = metrics.start_scan_cycle_work();
5160        metrics.operations[Metric::Ilm as usize].store(11, Ordering::Relaxed);
5161
5162        let report = metrics.report().await;
5163
5164        assert_eq!(report.current_cycle_ilm_actions, 0);
5165
5166        metrics.record_scanner_ilm_action(3);
5167        let report = metrics.report().await;
5168
5169        assert_eq!(report.current_cycle_ilm_actions, 3);
5170
5171        metrics.finish_scan_cycle_work(start);
5172        let report = metrics.report().await;
5173
5174        assert_eq!(report.last_cycle_ilm_actions, 3);
5175    }
5176
5177    #[test]
5178    fn record_scanner_yield_tracks_count_and_duration() {
5179        let metrics = Metrics::new();
5180
5181        metrics.record_scanner_yield(Duration::from_millis(42));
5182
5183        assert_eq!(metrics.lifetime(Metric::Yield), 1);
5184        assert_eq!(metrics.scanner_yield_duration_millis.load(Ordering::Relaxed), 42);
5185        assert_eq!(metrics.last_minute(Metric::Yield).n, 1);
5186        assert_eq!(metrics.lifetime(Metric::ThrottleSleep), 0);
5187    }
5188
5189    #[test]
5190    fn record_scanner_throttle_sleep_tracks_count_and_duration() {
5191        let metrics = Metrics::new();
5192
5193        metrics.record_scanner_throttle_sleep(Duration::from_millis(42));
5194
5195        assert_eq!(metrics.lifetime(Metric::ThrottleSleep), 1);
5196        assert_eq!(metrics.scanner_throttle_sleep_duration_millis.load(Ordering::Relaxed), 42);
5197        assert_eq!(metrics.last_minute(Metric::ThrottleSleep).n, 1);
5198        assert_eq!(metrics.lifetime(Metric::Yield), 0);
5199    }
5200
5201    #[tokio::test]
5202    async fn report_includes_scanner_throttle_config() {
5203        let metrics = Metrics::new();
5204
5205        metrics.record_scanner_throttle_config(true, 2.5, Duration::from_millis(1500), 128);
5206        let report = metrics.report().await;
5207
5208        assert!(report.throttle_idle_mode_enabled);
5209        assert_eq!(report.throttle_sleep_factor, 2.5);
5210        assert_eq!(report.throttle_max_sleep_seconds, 1.5);
5211        assert_eq!(report.yield_every_n_objects, 128);
5212    }
5213
5214    #[tokio::test]
5215    async fn scanner_throttle_config_rejects_invalid_sleep_factor() {
5216        let metrics = Metrics::new();
5217
5218        metrics.record_scanner_throttle_config(false, f64::NAN, Duration::ZERO, 0);
5219        let report = metrics.report().await;
5220
5221        assert!(!report.throttle_idle_mode_enabled);
5222        assert_eq!(report.throttle_sleep_factor, 0.0);
5223        assert_eq!(report.throttle_max_sleep_seconds, 0.0);
5224        assert_eq!(report.yield_every_n_objects, 0);
5225    }
5226
5227    #[tokio::test]
5228    async fn report_includes_scanner_cycle_config() {
5229        let metrics = Metrics::new();
5230
5231        metrics.record_scanner_cycle_config(
5232            Duration::from_secs(3600),
5233            Some(Duration::from_secs(86400)),
5234            Some(Duration::from_secs(1800)),
5235            Some(1_000_000),
5236            Some(100_000),
5237        );
5238        let report = metrics.report().await;
5239
5240        assert_eq!(report.cycle_interval_seconds, 3600.0);
5241        assert_eq!(report.cycle_max_duration_seconds, 1800.0);
5242        assert_eq!(report.cycle_max_objects, 1_000_000);
5243        assert_eq!(report.cycle_max_directories, 100_000);
5244        assert!(report.bitrot_cycle_enabled);
5245        assert_eq!(report.bitrot_cycle_seconds, 86400.0);
5246
5247        metrics.record_scanner_cycle_config(Duration::from_secs(60), None, None, None, None);
5248        let report = metrics.report().await;
5249
5250        assert_eq!(report.cycle_interval_seconds, 60.0);
5251        assert_eq!(report.cycle_max_duration_seconds, 0.0);
5252        assert_eq!(report.cycle_max_objects, 0);
5253        assert_eq!(report.cycle_max_directories, 0);
5254        assert!(!report.bitrot_cycle_enabled);
5255        assert_eq!(report.bitrot_cycle_seconds, 0.0);
5256    }
5257
5258    #[tokio::test]
5259    async fn scanner_cycle_timeout_metrics_reset_for_a_new_cycle() {
5260        let metrics = Metrics::new();
5261        metrics.record_scanner_cycle_timeout(true, Duration::from_secs(17));
5262        let timed_out = metrics.report().await;
5263        assert_eq!(timed_out.cycle_timeout_total, 1);
5264        assert_eq!(timed_out.cycle_last_progress_age, 17);
5265        assert!(timed_out.leader_lease_without_progress);
5266
5267        metrics.record_scanner_cycle_config(Duration::from_secs(60), None, Some(Duration::from_secs(1)), None, None);
5268        let current = metrics.report().await;
5269        assert_eq!(current.cycle_timeout_total, 1);
5270        assert_eq!(current.cycle_last_progress_age, 0);
5271        assert!(!current.leader_lease_without_progress);
5272    }
5273}