Skip to main content

rustfs_ecstore/store/
mod.rs

1// Copyright 2024 RustFS Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15#![allow(clippy::map_entry)]
16
17use crate::bucket::lifecycle::bucket_lifecycle_audit::LcEventSrc;
18use crate::bucket::lifecycle::bucket_lifecycle_ops::{
19    enqueue_immediate_expiry, enqueue_transition_immediate, init_background_expiry,
20};
21use crate::bucket::metadata_sys;
22use crate::bucket::utils::check_abort_multipart_args;
23use crate::bucket::utils::check_complete_multipart_args;
24use crate::bucket::utils::check_copy_obj_args;
25use crate::bucket::utils::check_del_obj_args;
26use crate::bucket::utils::check_get_obj_args;
27use crate::bucket::utils::check_list_multipart_args;
28use crate::bucket::utils::check_list_parts_args;
29use crate::bucket::utils::check_new_multipart_args;
30use crate::bucket::utils::check_object_args;
31use crate::bucket::utils::check_put_object_args;
32use crate::bucket::utils::check_put_object_part_args;
33use crate::bucket::utils::{check_valid_bucket_name, check_valid_bucket_name_strict, is_meta_bucketname};
34use crate::cluster::rpc::{RemoteClient, S3PeerSys};
35use crate::config::storageclass;
36use crate::core::pools::{DecommissionCanceler, PoolMeta, PoolMetaWriteState};
37use crate::disk::endpoint::{Endpoint, EndpointType};
38use crate::disk::{DiskAPI, DiskInfo, DiskInfoOptions};
39use crate::error::{Error, Result};
40use crate::error::{
41    StorageError, is_err_bucket_exists, is_err_invalid_upload_id, is_err_object_not_found, is_err_read_quorum,
42    is_err_strict_volume_not_found, is_err_version_not_found, to_object_err,
43};
44use crate::runtime::global::DISK_RESERVE_FRACTION;
45use crate::runtime::instance::InstanceContext;
46use crate::runtime::sources as runtime_sources;
47use crate::services::rebalance::{RebalStatus, RebalanceMeta, is_rebalance_conflicting_with_decommission};
48use crate::storage_api_contracts::{
49    bucket::{BucketInfo, BucketOperations, BucketOptions, DeleteBucketOptions, MakeBucketOptions},
50    list::{StorageListObjectVersionsInfo, StorageListObjectsV2Info, StorageObjectInfoOrErr, StorageWalkOptions},
51    multipart::{CompletePart, ListMultipartsInfo, ListPartsInfo, MultipartInfo, MultipartUploadResult, PartInfo},
52    object::{DeletedObject, ObjectToDelete},
53    range::HTTPRangeSpec,
54};
55use crate::store::init_format::{check_disk_fatal_errs, ec_drives_no_config};
56use crate::{
57    bucket::{lifecycle::bucket_lifecycle_ops::TransitionState, metadata::BucketMetadata},
58    core::sets::Sets,
59    disk::{BUCKET_META_PREFIX, DiskOption, DiskStore, RUSTFS_META_BUCKET},
60    layout::endpoints::EndpointServerPools,
61    object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader, ScannerPublicationCommitScope},
62};
63use futures::future::join_all;
64use http::HeaderMap;
65use lazy_static::lazy_static;
66use rand::RngExt as _;
67use rustfs_config::server_config::Config;
68use rustfs_filemeta::{FileInfo, FileMeta};
69use rustfs_heal_contracts::heal_channel::{HealItemType, HealOpts};
70use rustfs_lock::{LocalClient, LockClient, NamespaceLockWrapper};
71use rustfs_madmin::heal_commands::HealResultItem;
72use rustfs_utils::path::{decode_dir_object, encode_dir_object, path_join_buf};
73use s3s::dto::{BucketVersioningStatus, ObjectLockConfiguration, ObjectLockEnabled, VersioningConfiguration};
74use std::net::SocketAddr;
75use std::process::exit;
76use std::{collections::HashMap, sync::Arc, time::Duration};
77use time::OffsetDateTime;
78use tokio::select;
79use tokio::sync::{Mutex, RwLock};
80use tokio::time::sleep;
81use tokio_util::sync::CancellationToken;
82use tracing::{debug, error, info, instrument, warn};
83use uuid::Uuid;
84
85type ListObjectsV2Info = StorageListObjectsV2Info<ObjectInfo>;
86type ListObjectVersionsInfo = StorageListObjectVersionsInfo<ObjectInfo>;
87type ObjectInfoOrErr = StorageObjectInfoOrErr<ObjectInfo, Error>;
88type WalkOptions = StorageWalkOptions<fn(&FileInfo) -> bool>;
89
90pub const SCANNER_PUBLICATION_LEASE_TTL_MS: u64 = 60_000;
91pub(crate) const BUCKET_DELETE_XLMETA_DIAGNOSTIC_MAX_BYTES: u64 = 1024 * 1024;
92pub(crate) const BUCKET_DELETE_DIAGNOSTIC_MAX_ENTRIES: usize = 4_096;
93pub(crate) const BUCKET_DELETE_DIAGNOSTIC_MAX_ELAPSED: Duration = Duration::from_millis(100);
94
95#[derive(Debug)]
96pub(crate) struct BucketDeleteDiagnosticBudget {
97    deadline: Option<tokio::time::Instant>,
98    max_elapsed: Duration,
99    entries_remaining: usize,
100    #[cfg(test)]
101    first_io_delay: Option<(Duration, Arc<std::sync::atomic::AtomicBool>)>,
102}
103
104impl BucketDeleteDiagnosticBudget {
105    pub(crate) fn new() -> Self {
106        Self::with_limits(BUCKET_DELETE_DIAGNOSTIC_MAX_ENTRIES, BUCKET_DELETE_DIAGNOSTIC_MAX_ELAPSED)
107    }
108
109    fn with_limits(entries: usize, elapsed: Duration) -> Self {
110        Self {
111            deadline: None,
112            max_elapsed: elapsed,
113            entries_remaining: entries,
114            #[cfg(test)]
115            first_io_delay: None,
116        }
117    }
118
119    #[cfg(test)]
120    fn with_first_io_delay(mut self, delay: Duration, started: Arc<std::sync::atomic::AtomicBool>) -> Self {
121        self.first_io_delay = Some((delay, started));
122        self
123    }
124
125    fn deadline(&mut self) -> tokio::time::Instant {
126        let max_elapsed = self.max_elapsed;
127        *self.deadline.get_or_insert_with(|| tokio::time::Instant::now() + max_elapsed)
128    }
129
130    fn claim_entry(&mut self) -> bool {
131        if self.entries_remaining == 0 {
132            return false;
133        }
134        let deadline = self.deadline();
135        if tokio::time::Instant::now() >= deadline {
136            return false;
137        }
138        self.entries_remaining -= 1;
139        true
140    }
141
142    async fn run_io<T, F>(&mut self, future: F) -> std::io::Result<Option<T>>
143    where
144        F: std::future::Future<Output = std::io::Result<T>>,
145    {
146        let deadline = self.deadline();
147        if tokio::time::Instant::now() >= deadline {
148            return Ok(None);
149        }
150        #[cfg(test)]
151        let first_io_delay = self.first_io_delay.take();
152        #[cfg(test)]
153        let timeout_result = tokio::time::timeout_at(deadline, async move {
154            if let Some((delay, started)) = first_io_delay {
155                started.store(true, std::sync::atomic::Ordering::SeqCst);
156                tokio::time::sleep(delay).await;
157            }
158            future.await
159        })
160        .await;
161        #[cfg(not(test))]
162        let timeout_result = tokio::time::timeout_at(deadline, future).await;
163        match timeout_result {
164            Ok(result) => result.map(Some),
165            Err(_) => Ok(None),
166        }
167    }
168}
169
170#[derive(Debug, Default, Clone, PartialEq, Eq)]
171pub(crate) struct BucketMetadataLessResidue {
172    pub(crate) xlmeta_found: bool,
173    pub(crate) xlmeta_blocker: Option<BucketDeleteBlockerKind>,
174    pub(crate) files: usize,
175    pub(crate) uuid_data_dirs: usize,
176    pub(crate) entries_scanned: usize,
177    pub(crate) diagnostic_bytes_read: u64,
178    pub(crate) diagnostic_truncated: bool,
179    pub(crate) sample: Option<String>,
180}
181
182#[derive(Debug, Clone, Copy, PartialEq, Eq)]
183pub(crate) enum BucketDeleteBlockerKind {
184    VisibleVersion,
185    TierFreeVersion,
186    UnknownXlMeta,
187    OrphanDirectory,
188    DiagnosticBudgetExceeded,
189}
190
191impl BucketDeleteBlockerKind {
192    pub(crate) const fn as_str(self) -> &'static str {
193        match self {
194            Self::VisibleVersion => "visible_version",
195            Self::TierFreeVersion => "tier_free_version",
196            Self::UnknownXlMeta => "unknown_xlmeta",
197            Self::OrphanDirectory => "orphan_directory",
198            Self::DiagnosticBudgetExceeded => "diagnostic_budget_exceeded",
199        }
200    }
201
202    /// Whether the blocking residue is something the caller can still see and
203    /// remove through the S3 API.
204    ///
205    /// A live version or a tier free-version is ordinary: the bucket really is
206    /// not empty, the client can list and delete what is left, and the 409 it
207    /// receives is a complete answer.
208    ///
209    /// The remaining kinds are not. They are on-disk state that no S3 request
210    /// can reach: the caller has drained every version the API will show and
211    /// `DeleteBucket` still refuses, with no way to find out why. That is a
212    /// server-side integrity problem, and it is the reason this classification
213    /// exists — see [`bucket_delete_blocker_level`].
214    pub(crate) const fn is_client_visible(self) -> bool {
215        matches!(self, Self::VisibleVersion | Self::TierFreeVersion)
216    }
217}
218
219impl BucketMetadataLessResidue {
220    pub(crate) fn has_residue_without_xlmeta(&self) -> bool {
221        !self.xlmeta_found && (self.files > 0 || self.diagnostic_truncated)
222    }
223
224    pub(crate) fn describe(&self) -> String {
225        let sample = self.sample.as_deref().unwrap_or("<none>");
226        format!(
227            "metadata-less on-disk residue remains after empty-bucket verification: files={}, uuid_data_dirs={}, entries_scanned={}, diagnostic_bytes_read={}, diagnostic_truncated={}, sample={sample}",
228            self.files, self.uuid_data_dirs, self.entries_scanned, self.diagnostic_bytes_read, self.diagnostic_truncated,
229        )
230    }
231}
232
233/// Check if a directory contains any xl.meta files (indicating actual S3 objects)
234/// This is used to determine if a bucket is empty for deletion purposes.
235pub(crate) async fn has_xlmeta_files(path: &std::path::Path) -> std::io::Result<bool> {
236    use crate::disk::STORAGE_FORMAT_FILE;
237    use tokio::fs;
238
239    let mut stack = vec![path.to_path_buf()];
240
241    while let Some(current_path) = stack.pop() {
242        let mut entries = match fs::read_dir(&current_path).await {
243            Ok(entries) => entries,
244            Err(err) if err.kind() == std::io::ErrorKind::NotFound => continue,
245            Err(err) => return Err(err),
246        };
247
248        while let Some(entry) = entries.next_entry().await? {
249            let file_name = entry.file_name();
250            let file_name_str = file_name.to_string_lossy();
251
252            // Check if this is an xl.meta file
253            if file_name_str == STORAGE_FORMAT_FILE {
254                return Ok(true);
255            }
256
257            // If it's a directory, add to stack for further exploration
258            if entry.file_type().await?.is_dir() {
259                stack.push(entry.path());
260            }
261        }
262    }
263
264    Ok(false)
265}
266
267#[cfg(test)]
268pub(crate) async fn scan_metadata_less_residue(path: &std::path::Path) -> std::io::Result<BucketMetadataLessResidue> {
269    let mut budget = BucketDeleteDiagnosticBudget::new();
270    scan_metadata_less_residue_with_budget(path, &mut budget).await
271}
272
273async fn scan_metadata_less_residue_with_budget(
274    path: &std::path::Path,
275    budget: &mut BucketDeleteDiagnosticBudget,
276) -> std::io::Result<BucketMetadataLessResidue> {
277    use crate::disk::STORAGE_FORMAT_FILE;
278    use tokio::fs;
279    use tokio::io::AsyncReadExt as _;
280
281    let mut scan = BucketMetadataLessResidue::default();
282    let mut stack = vec![path.to_path_buf()];
283
284    let mark_budget_exhausted = |scan: &mut BucketMetadataLessResidue| {
285        scan.diagnostic_truncated = true;
286        scan.sample.get_or_insert_with(|| "<diagnostic-budget-exceeded>".to_string());
287    };
288
289    while let Some(current_path) = stack.pop() {
290        let mut entries = match budget.run_io(fs::read_dir(&current_path)).await {
291            Ok(Some(entries)) => entries,
292            Ok(None) => {
293                mark_budget_exhausted(&mut scan);
294                return Ok(scan);
295            }
296            Err(err) if err.kind() == std::io::ErrorKind::NotFound => continue,
297            Err(err) => return Err(err),
298        };
299
300        loop {
301            let entry = match budget.run_io(entries.next_entry()).await? {
302                Some(Some(entry)) => entry,
303                Some(None) => break,
304                None => {
305                    mark_budget_exhausted(&mut scan);
306                    return Ok(scan);
307                }
308            };
309            if !budget.claim_entry() {
310                mark_budget_exhausted(&mut scan);
311                return Ok(scan);
312            }
313            scan.entries_scanned = scan.entries_scanned.saturating_add(1);
314            let Some(file_type) = budget.run_io(entry.file_type()).await? else {
315                mark_budget_exhausted(&mut scan);
316                return Ok(scan);
317            };
318            let file_name = entry.file_name();
319            let file_name_str = file_name.to_string_lossy();
320
321            if file_name_str == STORAGE_FORMAT_FILE {
322                scan.xlmeta_found = true;
323                if scan.xlmeta_blocker.is_none() {
324                    let entry_path = entry.path();
325                    let Some(metadata) = budget.run_io(fs::metadata(&entry_path)).await? else {
326                        mark_budget_exhausted(&mut scan);
327                        scan.xlmeta_blocker = Some(BucketDeleteBlockerKind::DiagnosticBudgetExceeded);
328                        return Ok(scan);
329                    };
330                    scan.xlmeta_blocker = Some(if metadata.len() > BUCKET_DELETE_XLMETA_DIAGNOSTIC_MAX_BYTES {
331                        BucketDeleteBlockerKind::UnknownXlMeta
332                    } else {
333                        match budget.run_io(fs::File::open(&entry_path)).await {
334                            Ok(Some(file)) => {
335                                let mut data = Vec::new();
336                                let read = budget
337                                    .run_io(file.take(BUCKET_DELETE_XLMETA_DIAGNOSTIC_MAX_BYTES).read_to_end(&mut data))
338                                    .await;
339                                scan.diagnostic_bytes_read = data.len() as u64;
340                                match read {
341                                    Ok(Some(_)) => match FileMeta::load(&data) {
342                                        Ok(meta)
343                                            if !meta.versions.is_empty()
344                                                && meta.versions.iter().all(|version| version.header.free_version()) =>
345                                        {
346                                            BucketDeleteBlockerKind::TierFreeVersion
347                                        }
348                                        Ok(meta) if !meta.versions.is_empty() => BucketDeleteBlockerKind::VisibleVersion,
349                                        Ok(_) | Err(_) => BucketDeleteBlockerKind::UnknownXlMeta,
350                                    },
351                                    Ok(None) => {
352                                        mark_budget_exhausted(&mut scan);
353                                        BucketDeleteBlockerKind::DiagnosticBudgetExceeded
354                                    }
355                                    Err(_) => BucketDeleteBlockerKind::UnknownXlMeta,
356                                }
357                            }
358                            Ok(None) => {
359                                mark_budget_exhausted(&mut scan);
360                                BucketDeleteBlockerKind::DiagnosticBudgetExceeded
361                            }
362                            Err(_) => BucketDeleteBlockerKind::UnknownXlMeta,
363                        }
364                    });
365                    let sample = entry_path
366                        .strip_prefix(path)
367                        .unwrap_or(entry_path.as_path())
368                        .to_string_lossy()
369                        .replace(std::path::MAIN_SEPARATOR, "/");
370                    scan.sample = Some(sample);
371                }
372                return Ok(scan);
373            }
374
375            if file_type.is_dir() {
376                if Uuid::parse_str(&file_name_str).is_ok() {
377                    scan.uuid_data_dirs = scan.uuid_data_dirs.saturating_add(1);
378                }
379                stack.push(entry.path());
380            } else {
381                scan.files = scan.files.saturating_add(1);
382                if scan.sample.is_none() {
383                    let entry_path = entry.path();
384                    let sample = entry_path
385                        .strip_prefix(path)
386                        .unwrap_or(entry_path.as_path())
387                        .to_string_lossy()
388                        .replace(std::path::MAIN_SEPARATOR, "/");
389                    scan.sample = Some(sample);
390                }
391            }
392        }
393    }
394
395    Ok(scan)
396}
397
398async fn enqueue_transition_after_write(result: Result<ObjectInfo>, src: LcEventSrc) -> Result<ObjectInfo> {
399    match result {
400        Ok(oi) => {
401            if should_enqueue_transition_immediately(&oi) {
402                enqueue_transition_immediate(&oi, src.clone()).await;
403                enqueue_immediate_expiry(&oi, src).await;
404            }
405            Ok(oi)
406        }
407        Err(err) => Err(err),
408    }
409}
410
411fn should_enqueue_transition_immediately(oi: &ObjectInfo) -> bool {
412    !is_meta_bucketname(&oi.bucket)
413}
414
415const MAX_UPLOADS_LIST: usize = 10000;
416
417mod bucket;
418mod bucket_fence;
419pub(crate) use bucket::await_bucket_namespace_operation;
420pub use bucket_fence::BucketIncarnationFenceGuard;
421mod heal;
422pub(crate) use heal::bucket_heal_scope;
423pub use heal::{HealObjectAbsenceProof, HealObjectStorageResult};
424mod heal_walk;
425pub use heal_walk::HealWalkVersion;
426mod init;
427pub(crate) mod init_format;
428pub(crate) mod list_objects;
429mod multipart;
430mod object;
431#[cfg(feature = "test-util")]
432pub use object::DeleteAfterObjectLockSnapshotBarrier;
433pub(crate) use object::{
434    DecommissionFixedReadAnchor, ObjectLockDiagGuard, RemoteTuplePublicationCommitGuard, RemoteTuplePublicationFence,
435    SourceCleanupMutationFence, tiered_data_movement_source_matches,
436};
437pub use object::{
438    PrepareSelectObjectSnapshotError, PreparedGetObjectReader, SelectObjectSnapshot, SelectObjectSnapshotReadError,
439    SnapshotConsistencyError,
440};
441mod peer;
442mod rebalance;
443pub(crate) mod utils;
444
445use peer::init_local_peer;
446pub use peer::{
447    BootstrapLocalTarget, all_local_disk, all_local_disk_path, find_local_disk_by_ref, get_disk_infos, init_local_disks,
448    init_local_disks_with_instance_ctx, init_lock_clients, prewarm_local_disk_id_map,
449    prewarm_local_disk_id_map_with_instance_ctx,
450};
451
452pub struct ECStore {
453    pub id: Uuid,
454    // pub disks: Vec<DiskStore>,
455    pub disk_map: HashMap<usize, Vec<Option<DiskStore>>>,
456    pub pools: Vec<Arc<Sets>>,
457    pub peer_sys: S3PeerSys,
458    // pub local_disks: Vec<DiskStore>,
459    pub pool_meta: RwLock<PoolMeta>,
460    pub rebalance_meta: RwLock<Option<RebalanceMeta>>,
461    pub decommission_cancelers: RwLock<Vec<Option<DecommissionCanceler>>>,
462    /// Serializes rebalance/decommission start transitions.
463    ///
464    /// Lock order: acquire `start_gate` before `pool_meta`, `rebalance_meta`,
465    /// or `decommission_cancelers`. The guarded sections may perform bounded
466    /// async metadata work so check/init/start cannot race across operations.
467    pub(crate) start_gate: Mutex<()>,
468    /// Serializes full-document pool metadata saves and retains a fail-closed
469    /// write block after startup observes an unreadable replica.
470    ///
471    /// Lock order: acquire `pool_meta_save_gate`, then the distributed
472    /// `pool.bin` fence, then clone `pool_meta` under a short read lock.
473    pub(crate) pool_meta_save_gate: Mutex<PoolMetaWriteState>,
474    /// Per-instance runtime state (Phase 5, backlog#939).
475    ///
476    /// Carries this instance's identity/runtime out of the process globals so
477    /// multiple instances can coexist without cross-contamination. `new`
478    /// adopts the process bootstrap context (never mints a fresh one) so that
479    /// startup writes and post-construction reads share one cell — single
480    /// instance behavior is unchanged.
481    pub(crate) ctx: Arc<InstanceContext>,
482    /// Memoizes bucket-incarnation validation under continuous lifecycle
483    /// read-lock coverage (see [`bucket_fence`]).
484    pub(crate) bucket_fence_registry: Arc<bucket_fence::BucketFenceRegistry>,
485}
486
487const METRIC_SCANNER_DATA_MOVEMENT_PAUSED: &str = "rustfs_scanner_data_movement_paused";
488const METRIC_SCANNER_DATA_MOVEMENT_PAUSE_DURATION_SECONDS: &str = "rustfs_scanner_data_movement_pause_duration_seconds";
489const METRIC_SCANNER_DATA_MOVEMENT_BACKLOG_WORK_ITEMS: &str = "rustfs_scanner_data_movement_backlog_work_items";
490const SCANNER_DATA_MOVEMENT_PAUSE_POLICY: &str = "global_pause";
491
492#[derive(Clone, Copy, Debug, PartialEq, Eq, serde::Serialize)]
493#[serde(rename_all = "snake_case")]
494pub enum ScannerDataMovementPauseReason {
495    OperationEpochExhausted,
496    MovementGenerationExhausted,
497    DecommissionActive,
498    DecommissionFailed,
499    DecommissionCanceled,
500    RebalanceActive,
501}
502
503#[derive(Clone, Debug, PartialEq, Eq, serde::Serialize)]
504pub struct ScannerDataMovementPauseStatus {
505    pub paused: bool,
506    pub policy: &'static str,
507    pub reasons: Vec<ScannerDataMovementPauseReason>,
508    pub started_at_unix_secs: u64,
509    pub duration_seconds: u64,
510    pub operation_epoch: u64,
511    pub movement_generation: u64,
512    pub movement_backlog_work_items: u64,
513    pub movement_backlog_estimated: bool,
514}
515
516impl Default for ScannerDataMovementPauseStatus {
517    fn default() -> Self {
518        Self {
519            paused: false,
520            policy: SCANNER_DATA_MOVEMENT_PAUSE_POLICY,
521            reasons: Vec::new(),
522            started_at_unix_secs: 0,
523            duration_seconds: 0,
524            operation_epoch: 0,
525            movement_generation: 0,
526            movement_backlog_work_items: 0,
527            movement_backlog_estimated: false,
528        }
529    }
530}
531
532#[derive(Clone, Debug, PartialEq, Eq, serde::Serialize)]
533pub struct PoolMetaWriteGateStatus {
534    pub writes_ready: bool,
535    pub check_timed_out: bool,
536    pub write_blocked: bool,
537    pub transaction_aborted: bool,
538    pub pool_meta_absent: bool,
539    pub identity_initialized: Option<bool>,
540    pub identity_needs_repair: bool,
541    pub cluster_epoch: Option<u64>,
542    pub reason: Option<&'static str>,
543    pub phase: Option<&'static str>,
544    pub since_unix_secs: Option<i64>,
545}
546
547impl Default for PoolMetaWriteGateStatus {
548    fn default() -> Self {
549        Self {
550            writes_ready: true,
551            check_timed_out: false,
552            write_blocked: false,
553            transaction_aborted: false,
554            pool_meta_absent: false,
555            identity_initialized: None,
556            identity_needs_repair: false,
557            cluster_epoch: None,
558            reason: None,
559            phase: None,
560            since_unix_secs: None,
561        }
562    }
563}
564
565fn offset_unix_seconds(value: OffsetDateTime) -> u64 {
566    u64::try_from(value.unix_timestamp()).unwrap_or(0)
567}
568
569fn earliest_timestamp(current: Option<OffsetDateTime>, candidate: Option<OffsetDateTime>) -> Option<OffsetDateTime> {
570    match (current, candidate) {
571        (Some(current), Some(candidate)) => Some(current.min(candidate)),
572        (Some(current), None) => Some(current),
573        (None, candidate) => candidate,
574    }
575}
576
577fn usize_to_u64(value: usize) -> u64 {
578    u64::try_from(value).unwrap_or(u64::MAX)
579}
580
581fn metric_u64(value: u64) -> f64 {
582    f64::from(u32::try_from(value).unwrap_or(u32::MAX))
583}
584
585pub(crate) fn scanner_data_movement_timestamp_generation(value: OffsetDateTime) -> u64 {
586    let timestamp = value.unix_timestamp_nanos();
587    if timestamp <= 0 {
588        0
589    } else {
590        u64::try_from(timestamp).unwrap_or(u64::MAX)
591    }
592}
593
594fn valid_scanner_data_movement_timestamp_generation(value: OffsetDateTime) -> Option<u64> {
595    let generation = scanner_data_movement_timestamp_generation(value);
596    (generation != 0 && generation != u64::MAX).then_some(generation)
597}
598
599fn durable_scanner_data_movement_generation(pool_meta: &PoolMeta, rebalance_meta: Option<&RebalanceMeta>) -> u64 {
600    let mut generation = 0;
601    for pool in pool_meta.pools.iter().filter(|pool| pool.decommission.is_some()) {
602        let Some(pool_generation) = valid_scanner_data_movement_timestamp_generation(pool.last_update) else {
603            return u64::MAX;
604        };
605        generation = generation.max(pool_generation);
606    }
607
608    for movement_timestamp in rebalance_meta.into_iter().flat_map(|meta| {
609        meta.stopped_at.into_iter().chain(
610            meta.pool_stats
611                .iter()
612                .flat_map(|pool| [pool.info.start_time, pool.info.end_time])
613                .flatten(),
614        )
615    }) {
616        let Some(rebalance_generation) = valid_scanner_data_movement_timestamp_generation(movement_timestamp) else {
617            return u64::MAX;
618        };
619        generation = generation.max(rebalance_generation);
620    }
621
622    if generation == 0
623        && rebalance_meta.is_some_and(|meta| !meta.id.is_empty() || !meta.pool_stats.is_empty() || meta.stopped_at.is_some())
624    {
625        u64::MAX
626    } else {
627        generation
628    }
629}
630
631#[derive(Clone, Copy)]
632struct ScannerDataMovementSequenceState {
633    operation_epoch: u64,
634    operation_epoch_exhausted: bool,
635    movement_generation: u64,
636    movement_generation_exhausted: bool,
637}
638
639fn resolve_scanner_data_movement_pause_status(
640    pool_meta: &PoolMeta,
641    rebalance_meta: Option<&RebalanceMeta>,
642    decommission_worker_active: bool,
643    sequence: ScannerDataMovementSequenceState,
644    now: OffsetDateTime,
645) -> ScannerDataMovementPauseStatus {
646    let mut decommission_active = decommission_worker_active;
647    let mut decommission_failed = false;
648    let mut decommission_canceled = false;
649    let mut rebalance_active = false;
650    let mut started_at = None;
651    let mut movement_backlog_work_items = 0_u64;
652
653    for pool in &pool_meta.pools {
654        let Some(info) = pool.decommission.as_ref() else {
655            continue;
656        };
657        let active = info.has_decommission_state() && !info.complete && !info.failed && !info.canceled;
658        let failed = !info.queued && info.failed;
659        let canceled = !info.queued && info.canceled;
660        if !(active || failed || canceled) {
661            continue;
662        }
663
664        decommission_active |= active;
665        decommission_failed |= failed;
666        decommission_canceled |= canceled;
667        started_at = earliest_timestamp(started_at, info.start_time.or(Some(pool.last_update)));
668        let queued = usize_to_u64(info.queued_buckets.len());
669        let current_bucket = if info.bucket.is_empty() { 0 } else { 1 };
670        movement_backlog_work_items = movement_backlog_work_items.saturating_add(queued.max(current_bucket));
671    }
672
673    if let Some(rebalance_meta) = rebalance_meta {
674        for pool in &rebalance_meta.pool_stats {
675            let active = (pool.participating && pool.info.status == RebalStatus::Started) || pool.info.stopping;
676            if !active {
677                continue;
678            }
679            rebalance_active = true;
680            started_at = earliest_timestamp(started_at, pool.info.start_time);
681            movement_backlog_work_items = movement_backlog_work_items.saturating_add(usize_to_u64(pool.buckets.len()));
682        }
683    }
684
685    let mut reasons = Vec::with_capacity(6);
686    if sequence.operation_epoch_exhausted {
687        reasons.push(ScannerDataMovementPauseReason::OperationEpochExhausted);
688    }
689    if sequence.movement_generation_exhausted {
690        reasons.push(ScannerDataMovementPauseReason::MovementGenerationExhausted);
691    }
692    if decommission_active {
693        reasons.push(ScannerDataMovementPauseReason::DecommissionActive);
694    }
695    if decommission_failed {
696        reasons.push(ScannerDataMovementPauseReason::DecommissionFailed);
697    }
698    if decommission_canceled {
699        reasons.push(ScannerDataMovementPauseReason::DecommissionCanceled);
700    }
701    if rebalance_active {
702        reasons.push(ScannerDataMovementPauseReason::RebalanceActive);
703    }
704    let started_at_unix_secs = started_at.map(offset_unix_seconds).unwrap_or(0);
705    let duration_seconds = started_at
706        .and_then(|started_at| u64::try_from((now - started_at).whole_seconds()).ok())
707        .unwrap_or(0);
708    let paused = !reasons.is_empty();
709
710    ScannerDataMovementPauseStatus {
711        paused,
712        policy: SCANNER_DATA_MOVEMENT_PAUSE_POLICY,
713        reasons,
714        started_at_unix_secs,
715        duration_seconds,
716        operation_epoch: sequence.operation_epoch,
717        movement_generation: sequence.movement_generation,
718        movement_backlog_work_items,
719        movement_backlog_estimated: paused,
720    }
721}
722
723fn record_scanner_data_movement_pause_status(status: &ScannerDataMovementPauseStatus) {
724    metrics::gauge!(METRIC_SCANNER_DATA_MOVEMENT_PAUSED).set(if status.paused { 1.0 } else { 0.0 });
725    metrics::gauge!(METRIC_SCANNER_DATA_MOVEMENT_PAUSE_DURATION_SECONDS).set(metric_u64(status.duration_seconds));
726    metrics::gauge!(METRIC_SCANNER_DATA_MOVEMENT_BACKLOG_WORK_ITEMS).set(metric_u64(status.movement_backlog_work_items));
727}
728
729impl std::fmt::Debug for ECStore {
730    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
731        let disk_slot_count: usize = self.disk_map.values().map(Vec::len).sum();
732
733        f.debug_struct("ECStore")
734            .field("id", &self.id)
735            .field("disk_map_pool_count", &self.disk_map.len())
736            .field("disk_slot_count", &disk_slot_count)
737            .field("pool_count", &self.pools.len())
738            .finish_non_exhaustive()
739    }
740}
741
742/// Phase 2: Accessor methods for config globals
743/// These delegate to the process-global statics. No local state — the globals
744/// remain the single source of truth until the migration is complete.
745impl ECStore {
746    /// Every erasure set across all pools, pool-major order.
747    ///
748    /// Read-only queries that must consult each set's own copy of a
749    /// per-bucket object (e.g. the scanner's `.usage-cache.bin`) iterate
750    /// this instead of the hash-routed store path, which would always land
751    /// on one set (rustfs/backlog#1872).
752    pub fn all_set_disks(&self) -> Vec<Arc<crate::set_disk::SetDisks>> {
753        self.pools.iter().flat_map(|pool| pool.disk_set.iter().cloned()).collect()
754    }
755
756    /// Erasure sets that may receive scanner pause-backlog replicas.
757    ///
758    /// An actively decommissioning or already decommissioned source pool is
759    /// excluded so an operational record acknowledged during movement always
760    /// has a copy on storage that remains in the cluster. The record is kept
761    /// separate from pool and rebalance metadata.
762    pub async fn scanner_pause_backlog_writable_set_disks(&self) -> Vec<Arc<crate::set_disk::SetDisks>> {
763        let pool_meta = self.pool_meta.read().await;
764        self.pools
765            .iter()
766            .enumerate()
767            .filter(|(pool_index, _)| pool_meta.scanner_pause_backlog_pool_writable(*pool_index))
768            .flat_map(|(_, pool)| pool.disk_set.iter().cloned())
769            .collect()
770    }
771
772    /// Get server configuration (delegates to global)
773    pub fn get_server_config(&self) -> Option<Config> {
774        runtime_sources::server_config()
775    }
776
777    /// Set server configuration (delegates to global)
778    pub fn set_server_config(&self, cfg: Config) {
779        runtime_sources::set_server_config(cfg);
780    }
781
782    /// Get storage class configuration (delegates to global)
783    pub fn get_storage_class(&self) -> Option<crate::config::storageclass::Config> {
784        runtime_sources::storage_class_config()
785    }
786
787    /// Set storage class configuration (delegates to global)
788    pub fn set_storage_class(&self, cfg: crate::config::storageclass::Config) {
789        runtime_sources::set_storage_class_config(cfg);
790    }
791}
792
793/// Phase 3: Accessor methods for service globals
794/// These provide a unified API through ECStore for accessing cross-cutting
795/// service singletons. The globals remain the source of truth.
796impl ECStore {
797    /// Get the notification system
798    pub fn notification_system(&self) -> Option<std::sync::Arc<crate::services::notification_sys::NotificationSys>> {
799        runtime_sources::notification_sys()
800    }
801
802    /// Get the bucket metadata system
803    pub fn bucket_metadata_sys(&self) -> Option<Arc<tokio::sync::RwLock<crate::bucket::metadata_sys::BucketMetadataSys>>> {
804        runtime_sources::bucket_metadata_sys()
805    }
806
807    /// Get the global endpoints
808    pub fn endpoints(&self) -> EndpointServerPools {
809        runtime_sources::endpoint_pools().unwrap_or_else(|| Vec::new().into())
810    }
811
812    /// Get this store instance's endpoint topology without consulting process globals.
813    pub fn instance_endpoints(&self) -> Option<EndpointServerPools> {
814        self.ctx.endpoints()
815    }
816
817    /// Get the global region
818    pub fn region(&self) -> Option<s3s::region::Region> {
819        runtime_sources::region()
820    }
821
822    /// Get the tier config manager
823    pub fn tier_config_mgr(&self) -> Arc<tokio::sync::RwLock<crate::services::tier::tier::TierConfigMgr>> {
824        self.ctx.tier_config_mgr()
825    }
826
827    /// Get the server configuration
828    pub fn server_config(&self) -> Option<Config> {
829        runtime_sources::server_config()
830    }
831
832    /// Get the storage class configuration
833    pub fn storage_class(&self) -> Option<crate::config::storageclass::Config> {
834        runtime_sources::storage_class_config()
835    }
836}
837
838/// Phase 4: Server address accessors
839/// These provide a unified API through ECStore for accessing server-level
840/// configuration globals. The globals remain the source of truth.
841impl ECStore {
842    /// Get the server port
843    pub fn port(&self) -> u16 {
844        runtime_sources::rustfs_port()
845    }
846
847    /// Get the server host
848    pub async fn host(&self) -> String {
849        runtime_sources::rustfs_host().await
850    }
851
852    /// Get the server address (host:port)
853    pub async fn addr(&self) -> String {
854        runtime_sources::rustfs_addr().await
855    }
856}
857
858/// Phase 5: Per-instance erasure setup accessors (backlog#939)
859///
860/// These read this instance's own [`InstanceContext`] rather than a process
861/// global, so two instances carrying different contexts stay isolated. The
862/// legacy free-function facade (`runtime::global::is_erasure` etc.) forwards to
863/// the current instance's context, preserving single-instance behavior.
864impl ECStore {
865    /// Whether this instance uses erasure coding (single-node or distributed).
866    pub async fn setup_is_erasure(&self) -> bool {
867        self.ctx.is_erasure().await
868    }
869
870    /// Whether this instance uses distributed erasure coding.
871    pub async fn setup_is_dist_erasure(&self) -> bool {
872        self.ctx.is_dist_erasure().await
873    }
874
875    /// Whether this instance uses single-drive erasure coding.
876    pub async fn setup_is_erasure_sd(&self) -> bool {
877        self.ctx.is_erasure_sd().await
878    }
879
880    pub fn scanner_namespace_mutation_generation(&self) -> u64 {
881        list_objects::scanner_namespace_mutation_generation().saturating_add(self.ctx.namespace_commit_generation())
882    }
883
884    pub async fn scanner_data_movement_active(&self) -> bool {
885        let (decommission, rebalance) = tokio::join!(self.is_decommission_running(), self.is_rebalance_started());
886        decommission || rebalance
887    }
888
889    /// Return the storage-owned movement state and generation as one
890    /// authenticated activity snapshot. The read lock is acquired before
891    /// the state locks (cancelers, pool metadata, then rebalance metadata),
892    /// matching the transition writer order and preventing a terminal state
893    /// from being reported with the preceding generation.
894    pub async fn scanner_data_movement_activity(&self) -> (bool, bool, u64) {
895        let operation_gate = self.ctx.data_movement_operation_gate();
896        let _operation_guard = operation_gate.read_owned().await;
897        let (active, blocked) = self.scanner_data_movement_snapshot_locked().await;
898        let blocked =
899            blocked || self.ctx.data_movement_operation_epoch_exhausted() || self.ctx.data_movement_generation_exhausted();
900        self.ctx.set_scanner_publication_state(blocked);
901        (active, blocked, self.ctx.data_movement_generation())
902    }
903
904    pub fn scanner_data_movement_generation(&self) -> u64 {
905        self.ctx.data_movement_generation()
906    }
907
908    pub fn scanner_data_movement_generation_exhausted(&self) -> bool {
909        self.ctx.data_movement_generation_exhausted()
910    }
911
912    pub fn scanner_data_movement_changed(&self) -> std::sync::Arc<tokio::sync::Notify> {
913        self.ctx.data_movement_generation_notify()
914    }
915
916    /// Returns whether scanner metadata may still be hidden by a local
917    /// data-movement state. Terminal failed/canceled decommission entries
918    /// remain suspended until an operator clears or retries them, so they are
919    /// a publication barrier even after the worker has stopped. Active PUT
920    /// rename fanouts also defer publication, including post-ACK tails.
921    pub async fn scanner_data_usage_publication_blocked(&self) -> bool {
922        let operation_gate = self.ctx.data_movement_operation_gate();
923        let _operation_guard = operation_gate.read_owned().await;
924        self.scanner_data_usage_publication_snapshot_blocked().await || self.ctx.namespace_commits_pending()
925    }
926
927    pub async fn scanner_data_movement_pause_status(&self) -> ScannerDataMovementPauseStatus {
928        let operation_gate = self.ctx.data_movement_operation_gate();
929        let _operation_guard = operation_gate.read_owned().await;
930        self.scanner_data_movement_pause_snapshot().await
931    }
932
933    async fn scanner_data_usage_publication_snapshot_blocked(&self) -> bool {
934        self.scanner_data_movement_pause_snapshot().await.paused
935    }
936
937    async fn scanner_data_movement_snapshot_locked(&self) -> (bool, bool) {
938        let decommission_cancelers = self.decommission_cancelers.read().await;
939        let decommission_active = decommission_cancelers
940            .iter()
941            .any(|canceler| canceler.as_ref().is_some_and(DecommissionCanceler::is_active));
942        let pool_meta = self.pool_meta.read().await;
943        let decommission_active = decommission_active
944            || pool_meta.pools.iter().any(|pool| {
945                pool.decommission
946                    .as_ref()
947                    .is_some_and(|info| info.has_decommission_state() && !info.complete && !info.failed && !info.canceled)
948            });
949        let decommission_terminal = pool_meta.pools.iter().any(|pool| {
950            pool.decommission
951                .as_ref()
952                .is_some_and(|info| !info.queued && (info.failed || info.canceled))
953        });
954        let rebalance_meta = self.rebalance_meta.read().await;
955        let rebalance_active = rebalance_meta
956            .as_ref()
957            .is_some_and(is_rebalance_conflicting_with_decommission);
958        self.ctx
959            .observe_durable_data_movement_generation(durable_scanner_data_movement_generation(
960                &pool_meta,
961                rebalance_meta.as_ref(),
962            ));
963
964        let blocked = decommission_active || decommission_terminal || rebalance_active;
965        (decommission_active || rebalance_active, blocked)
966    }
967
968    async fn scanner_data_movement_pause_snapshot(&self) -> ScannerDataMovementPauseStatus {
969        let decommission_active = {
970            let decommission_cancelers = self.decommission_cancelers.read().await;
971            decommission_cancelers
972                .iter()
973                .any(|canceler| canceler.as_ref().is_some_and(DecommissionCanceler::is_active))
974        };
975        let pool_meta = self.pool_meta.read().await.clone();
976        let rebalance_meta = self.rebalance_meta.read().await.clone();
977        self.ctx
978            .observe_durable_data_movement_generation(durable_scanner_data_movement_generation(
979                &pool_meta,
980                rebalance_meta.as_ref(),
981            ));
982        let status = resolve_scanner_data_movement_pause_status(
983            &pool_meta,
984            rebalance_meta.as_ref(),
985            decommission_active,
986            ScannerDataMovementSequenceState {
987                operation_epoch: self.ctx.data_movement_operation_epoch(),
988                operation_epoch_exhausted: self.ctx.data_movement_operation_epoch_exhausted(),
989                movement_generation: self.ctx.data_movement_generation(),
990                movement_generation_exhausted: self.ctx.data_movement_generation_exhausted(),
991            },
992            OffsetDateTime::now_utc(),
993        );
994        self.ctx.set_scanner_publication_state(status.paused);
995        record_scanner_data_movement_pause_status(&status);
996        status
997    }
998
999    #[cfg(test)]
1000    pub(crate) async fn scanner_data_movement_pause_snapshot_for_test(&self) -> ScannerDataMovementPauseStatus {
1001        self.scanner_data_movement_pause_snapshot().await
1002    }
1003
1004    /// Admit one short data-usage publication commit under the same
1005    /// per-instance gate used by decommission side effects and transitions.
1006    /// The epoch is sampled while the read guard is held, so a transition
1007    /// cannot cross this admission without waiting for the commit to finish.
1008    pub async fn scanner_data_usage_publication_read_guard(&self) -> (tokio::sync::OwnedRwLockReadGuard<()>, u64) {
1009        let operation_gate = self.ctx.data_movement_operation_gate();
1010        let operation_guard = operation_gate.read_owned().await;
1011        let epoch = self.ctx.data_movement_operation_epoch();
1012        (operation_guard, epoch)
1013    }
1014
1015    /// Acquire the movement gate and inspect the movement owner once. The
1016    /// state inspection is performed after acquiring the read guard so a
1017    /// transition cannot update its durable state between the check and the
1018    /// publication commit.
1019    pub async fn scanner_data_usage_publication_admission_guard(&self) -> Option<(tokio::sync::OwnedRwLockReadGuard<()>, u64)> {
1020        let operation_gate = self.ctx.data_movement_operation_gate();
1021        let operation_guard = operation_gate.read_owned().await;
1022        if self.ctx.data_movement_operation_epoch_exhausted() || self.ctx.data_movement_generation_exhausted() {
1023            return None;
1024        }
1025        if self.scanner_data_usage_publication_snapshot_blocked().await {
1026            return None;
1027        }
1028
1029        Some((operation_guard, self.ctx.data_movement_operation_epoch()))
1030    }
1031
1032    /// Acquire a storage-owned scanner publication scope. Unlike the legacy
1033    /// admission helper, the movement permit is owned by the returned scope
1034    /// and therefore survives cancellation of the scanner coordinator while
1035    /// the actual metadata mutation drains.
1036    pub async fn scanner_data_usage_publication_commit_scope(
1037        &self,
1038        expected_movement_epoch: u64,
1039        safe_deadline: tokio::time::Instant,
1040        remote_lease_tokens: Vec<Uuid>,
1041    ) -> Option<ScannerPublicationCommitScope> {
1042        let (movement_permit, epoch) = self.scanner_data_usage_publication_admission_guard().await?;
1043        if epoch != expected_movement_epoch {
1044            return None;
1045        }
1046        Some(ScannerPublicationCommitScope::new_storage_owned(
1047            epoch,
1048            safe_deadline,
1049            remote_lease_tokens,
1050            movement_permit,
1051        ))
1052    }
1053
1054    pub async fn scanner_data_usage_publication_commit_scope_with_release_flag(
1055        &self,
1056        expected_movement_epoch: u64,
1057        safe_deadline: tokio::time::Instant,
1058        remote_lease_tokens: Vec<Uuid>,
1059        lease_release_safe: Arc<std::sync::atomic::AtomicBool>,
1060    ) -> Option<ScannerPublicationCommitScope> {
1061        let (movement_permit, epoch) = self.scanner_data_usage_publication_admission_guard().await?;
1062        if epoch != expected_movement_epoch {
1063            return None;
1064        }
1065        Some(ScannerPublicationCommitScope::new_storage_owned_with_release_flag(
1066            epoch,
1067            safe_deadline,
1068            remote_lease_tokens,
1069            movement_permit,
1070            lease_release_safe,
1071        ))
1072    }
1073
1074    /// Capture the current publication epoch without holding the movement
1075    /// gate across backend I/O. Callers must re-admit the same epoch before a
1076    /// mutation commits.
1077    pub(crate) async fn scanner_data_usage_publication_epoch(&self) -> Option<u64> {
1078        let (operation_guard, epoch) = self.scanner_data_usage_publication_admission_guard().await?;
1079        drop(operation_guard);
1080        Some(epoch)
1081    }
1082
1083    /// Acquire a storage-owned read admission for a coordinator's final
1084    /// scanner publication.  The guard remains in the context's lease table,
1085    /// so a local movement writer cannot pass the peer while its authoritative
1086    /// PUT is in flight.
1087    pub async fn acquire_scanner_publication_lease(
1088        &self,
1089        expected_generation: u64,
1090        ttl: std::time::Duration,
1091    ) -> Result<(Uuid, u64)> {
1092        if ttl != crate::runtime::instance::SCANNER_PUBLICATION_LEASE_TTL {
1093            return Err(Error::other("scanner publication lease TTL is not supported"));
1094        }
1095
1096        // Bind the original activity generation across the asynchronous checks;
1097        // a completed namespace commit must never refresh an existing proof.
1098        let namespace_generation = self.scanner_namespace_mutation_generation();
1099        let operation_gate = self.ctx.data_movement_operation_gate();
1100        let operation_guard = operation_gate.read_owned().await;
1101        if self.ctx.data_movement_generation_exhausted()
1102            || self.ctx.data_movement_operation_epoch_exhausted()
1103            || self.ctx.data_movement_generation() != expected_generation
1104            || namespace_generation == u64::MAX
1105        {
1106            return Err(Error::other("scanner publication lease generation is stale"));
1107        }
1108        if self.scanner_data_movement_snapshot_locked().await.1 || self.ctx.namespace_commits_pending() {
1109            return Err(Error::other("scanner publication lease is blocked by data movement"));
1110        }
1111
1112        if self.scanner_namespace_mutation_generation() != namespace_generation {
1113            return Err(Error::other("scanner publication lease generation is stale"));
1114        }
1115
1116        let token = Uuid::new_v4();
1117        let expires_at = tokio::time::Instant::now() + ttl;
1118        if !self
1119            .ctx
1120            .install_scanner_publication_lease(token, expires_at, expected_generation, namespace_generation, operation_guard)
1121            .await
1122        {
1123            return Err(Error::other("scanner publication lease capacity is exhausted"));
1124        }
1125
1126        let context = Arc::clone(&self.ctx);
1127        tokio::spawn(async move {
1128            tokio::time::sleep_until(expires_at).await;
1129            context.expire_scanner_publication_lease(token, expires_at).await;
1130        });
1131        Ok((token, expected_generation))
1132    }
1133
1134    pub async fn release_scanner_publication_lease(&self, token: Uuid) -> bool {
1135        self.ctx.remove_scanner_publication_lease(token).await
1136    }
1137
1138    /// Revalidate a previously acquired remote publication lease immediately
1139    /// before the coordinator's final metadata write.  The operation read
1140    /// guard makes the movement snapshot and token lookup one storage-owned
1141    /// admission; a restarted context has no old token and therefore fails
1142    /// closed even if its generation counter has returned to zero.
1143    pub async fn validate_scanner_publication_lease(&self, token: Uuid, expected_generation: u64) -> Result<()> {
1144        let _operation_guard = self.acquire_scanner_publication_lease_guard(token).await?;
1145        if self.ctx.data_movement_generation_exhausted()
1146            || self.ctx.data_movement_operation_epoch_exhausted()
1147            || self.ctx.data_movement_generation() != expected_generation
1148        {
1149            return Err(Error::other("scanner publication lease generation is stale"));
1150        }
1151        if self.scanner_data_movement_snapshot_locked().await.1 || self.ctx.namespace_commits_pending() {
1152            return Err(Error::other("scanner publication lease is blocked by data movement"));
1153        }
1154        let Some((lease_generation, lease_namespace_generation)) = self.ctx.scanner_publication_lease_generations(token).await
1155        else {
1156            return Err(Error::other("scanner publication lease is unknown or expired"));
1157        };
1158        let namespace_generation = self.scanner_namespace_mutation_generation();
1159        if lease_generation != self.ctx.data_movement_generation()
1160            || lease_namespace_generation != namespace_generation
1161            || namespace_generation == u64::MAX
1162        {
1163            return Err(Error::other("scanner publication lease generation is stale"));
1164        }
1165        Ok(())
1166    }
1167
1168    /// Acquire the target-side read guard bound to a previously granted lease.
1169    /// The guard is returned to the RPC handler and must remain alive through
1170    /// the complete rename/write operation.  A lease token is process-owned;
1171    /// restart, expiry, generation changes, or blocked movement all reject it
1172    /// before the target disk is touched.
1173    pub async fn acquire_scanner_publication_lease_guard(&self, token: Uuid) -> Result<tokio::sync::OwnedRwLockReadGuard<()>> {
1174        let operation_gate = self.ctx.data_movement_operation_gate();
1175        let operation_guard = operation_gate.read_owned().await;
1176        if self.ctx.data_movement_generation_exhausted() || self.ctx.data_movement_operation_epoch_exhausted() {
1177            return Err(Error::other("scanner publication lease generation is exhausted"));
1178        }
1179        if self.scanner_data_movement_snapshot_locked().await.1 || self.ctx.namespace_commits_pending() {
1180            return Err(Error::other("scanner publication lease is blocked by data movement"));
1181        }
1182        let Some((lease_generation, lease_namespace_generation)) = self.ctx.scanner_publication_lease_generations(token).await
1183        else {
1184            return Err(Error::other("scanner publication lease is unknown or expired"));
1185        };
1186        let namespace_generation = self.scanner_namespace_mutation_generation();
1187        if lease_generation != self.ctx.data_movement_generation()
1188            || lease_namespace_generation != namespace_generation
1189            || namespace_generation == u64::MAX
1190        {
1191            return Err(Error::other("scanner publication lease generation is stale"));
1192        }
1193        Ok(operation_guard)
1194    }
1195}
1196
1197// impl Clone for ECStore {
1198//     fn clone(&self) -> Self {
1199//         let pool_meta = match self.pool_meta.read() {
1200//             Ok(pool_meta) => pool_meta.clone(),
1201//             Err(_) => PoolMeta::default(),
1202//         };
1203//         Self {
1204//             id: self.id.clone(),
1205//             disk_map: self.disk_map.clone(),
1206//             pools: self.pools.clone(),
1207//             peer_sys: self.peer_sys.clone(),
1208//             pool_meta: std_RwLock::new(pool_meta),
1209//             decommission_cancelers: self.decommission_cancelers.clone(),
1210//         }
1211//     }
1212// }
1213
1214// #[derive(Debug, Default, Clone)]
1215// pub struct ListPathOptions {
1216//     pub id: String,
1217
1218//     // Bucket of the listing.
1219//     pub bucket: String,
1220
1221//     // Directory inside the bucket.
1222//     // When unset listPath will set this based on Prefix
1223//     pub base_dir: String,
1224
1225//     // Scan/return only content with prefix.
1226//     pub prefix: String,
1227
1228//     // FilterPrefix will return only results with this prefix when scanning.
1229//     // Should never contain a slash.
1230//     // Prefix should still be set.
1231//     pub filter_prefix: String,
1232
1233//     // Marker to resume listing.
1234//     // The response will be the first entry >= this object name.
1235//     pub marker: String,
1236
1237//     // Limit the number of results.
1238//     pub limit: i32,
1239// }
1240
1241#[async_trait::async_trait]
1242impl crate::storage_api_contracts::object::ObjectIO for ECStore {
1243    type Error = Error;
1244    type RangeSpec = HTTPRangeSpec;
1245    type HeaderMap = HeaderMap;
1246    type ObjectOptions = ObjectOptions;
1247    type ObjectInfo = ObjectInfo;
1248    type GetObjectReader = GetObjectReader;
1249    type PutObjectReader = PutObjReader;
1250
1251    #[instrument(level = "debug", skip(self, h))]
1252    async fn get_object_reader(
1253        &self,
1254        bucket: &str,
1255        object: &str,
1256        range: Option<HTTPRangeSpec>,
1257        h: HeaderMap,
1258        opts: &ObjectOptions,
1259    ) -> Result<GetObjectReader> {
1260        self.handle_get_object_reader(bucket, object, range, h, opts).await
1261    }
1262    #[instrument(level = "debug", skip(self, data))]
1263    async fn put_object(&self, bucket: &str, object: &str, data: &mut PutObjReader, opts: &ObjectOptions) -> Result<ObjectInfo> {
1264        self.put_object_with_old_current_size(bucket, object, data, opts)
1265            .await
1266            .map(|(object_info, _)| object_info)
1267    }
1268}
1269
1270impl ECStore {
1271    /// `put_object` plus the rename_data old-size backfill
1272    /// (rustfs/backlog#1009); see `SetDisks::put_object_with_old_current_size`.
1273    /// Post-write hooks (immediate ILM transition enqueue, list-cache
1274    /// invalidation) match the plain `put_object` path exactly.
1275    #[instrument(level = "debug", skip(self, data))]
1276    pub async fn put_object_with_old_current_size(
1277        &self,
1278        bucket: &str,
1279        object: &str,
1280        data: &mut PutObjReader,
1281        opts: &ObjectOptions,
1282    ) -> Result<(ObjectInfo, Option<crate::disk::OldCurrentSize>)> {
1283        let result = match self.handle_put_object(bucket, object, data, opts).await {
1284            Ok((object_info, old_current_size)) => enqueue_transition_after_write(Ok(object_info), LcEventSrc::S3PutObject)
1285                .await
1286                .map(|object_info| (object_info, old_current_size)),
1287            Err(err) => Err(err),
1288        };
1289        if result.is_ok() {
1290            list_objects::observe_list_objects_mutation(self, bucket).await;
1291        }
1292        result
1293    }
1294}
1295
1296lazy_static! {
1297    static ref ENABLED_OBJECT_LOCK_CONFIG: ObjectLockConfiguration = ObjectLockConfiguration {
1298        object_lock_enabled: Some(ObjectLockEnabled::from_static(ObjectLockEnabled::ENABLED)),
1299        ..Default::default()
1300    };
1301    static ref ENABLED_VERSIONING_CONFIG: VersioningConfiguration = VersioningConfiguration {
1302        status: Some(BucketVersioningStatus::from_static(BucketVersioningStatus::ENABLED)),
1303        ..Default::default()
1304    };
1305}
1306
1307#[async_trait::async_trait]
1308impl BucketOperations for ECStore {
1309    type Error = Error;
1310
1311    #[instrument(skip(self))]
1312    async fn make_bucket(&self, bucket: &str, opts: &MakeBucketOptions) -> Result<()> {
1313        Box::pin(self.handle_make_bucket(bucket, opts)).await
1314    }
1315
1316    #[instrument(skip(self))]
1317    async fn get_bucket_info(&self, bucket: &str, opts: &BucketOptions) -> Result<BucketInfo> {
1318        self.handle_get_bucket_info(bucket, opts).await
1319    }
1320    #[instrument(skip(self))]
1321    async fn list_bucket(&self, opts: &BucketOptions) -> Result<Vec<BucketInfo>> {
1322        self.handle_list_bucket(opts).await
1323    }
1324    #[instrument(skip(self))]
1325    async fn delete_bucket(&self, bucket: &str, opts: &DeleteBucketOptions) -> Result<()> {
1326        Box::pin(self.handle_delete_bucket(bucket, opts)).await
1327    }
1328}
1329
1330#[async_trait::async_trait]
1331impl crate::storage_api_contracts::object::ObjectOperations for ECStore {
1332    type Error = Error;
1333    type ObjectInfo = ObjectInfo;
1334    type ObjectOptions = ObjectOptions;
1335    type FileInfo = FileInfo;
1336    type ObjectToDelete = ObjectToDelete;
1337    type DeletedObject = DeletedObject;
1338
1339    async fn get_object_info(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<ObjectInfo> {
1340        self.handle_get_object_info(bucket, object, opts).await
1341    }
1342
1343    async fn verify_object_integrity(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<()> {
1344        self.handle_verify_object_integrity(bucket, object, opts).await
1345    }
1346
1347    #[instrument(skip(self))]
1348    async fn copy_object(
1349        &self,
1350        src_bucket: &str,
1351        src_object: &str,
1352        dst_bucket: &str,
1353        dst_object: &str,
1354        src_info: &mut ObjectInfo,
1355        src_opts: &ObjectOptions,
1356        dst_opts: &ObjectOptions,
1357    ) -> Result<ObjectInfo> {
1358        let result = enqueue_transition_after_write(
1359            self.handle_copy_object(src_bucket, src_object, dst_bucket, dst_object, src_info, src_opts, dst_opts)
1360                .await,
1361            LcEventSrc::S3CopyObject,
1362        )
1363        .await;
1364        if result.is_ok() {
1365            list_objects::observe_list_objects_mutation(self, dst_bucket).await;
1366        }
1367        result
1368    }
1369
1370    #[instrument(skip(self))]
1371    async fn delete_object_version(&self, bucket: &str, object: &str, fi: &FileInfo, force_del_marker: bool) -> Result<()> {
1372        let result = self.handle_delete_object_version(bucket, object, fi, force_del_marker).await;
1373        if result.is_ok() {
1374            list_objects::observe_list_objects_mutation(self, bucket).await;
1375        }
1376        result
1377    }
1378
1379    #[instrument(skip(self))]
1380    async fn delete_object(&self, bucket: &str, object: &str, opts: ObjectOptions) -> Result<ObjectInfo> {
1381        let result = self.handle_delete_object(bucket, object, opts).await;
1382        if result.is_ok() {
1383            list_objects::observe_list_objects_mutation(self, bucket).await;
1384        }
1385        result
1386    }
1387
1388    #[instrument(skip(self, objects, opts))]
1389    async fn delete_objects(
1390        &self,
1391        bucket: &str,
1392        objects: Vec<ObjectToDelete>,
1393        opts: ObjectOptions,
1394    ) -> (Vec<DeletedObject>, Vec<Option<Error>>) {
1395        let result = self.handle_delete_objects(bucket, objects, opts).await;
1396        let success_count = result.1.iter().filter(|err| err.is_none()).count();
1397        if success_count > 0 {
1398            list_objects::observe_list_objects_mutations(self, bucket, success_count).await;
1399        }
1400        result
1401    }
1402
1403    #[instrument(skip(self))]
1404    async fn put_object_metadata(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<ObjectInfo> {
1405        self.handle_put_object_metadata(bucket, object, opts).await
1406    }
1407    #[instrument(skip(self))]
1408    async fn get_object_tags(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<String> {
1409        self.handle_get_object_tags(bucket, object, opts).await
1410    }
1411
1412    #[instrument(level = "debug", skip(self))]
1413    async fn put_object_tags(&self, bucket: &str, object: &str, tags: &str, opts: &ObjectOptions) -> Result<ObjectInfo> {
1414        self.handle_put_object_tags(bucket, object, tags, opts).await
1415    }
1416
1417    #[instrument(skip(self))]
1418    async fn delete_object_tags(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<ObjectInfo> {
1419        self.handle_delete_object_tags(bucket, object, opts).await
1420    }
1421
1422    #[instrument(skip(self))]
1423    async fn add_partial(&self, bucket: &str, object: &str, version_id: &str) -> Result<()> {
1424        self.handle_add_partial(bucket, object, version_id).await
1425    }
1426    #[instrument(skip(self))]
1427    async fn transition_object(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<()> {
1428        self.handle_transition_object(bucket, object, opts).await
1429    }
1430
1431    #[instrument(skip(self))]
1432    async fn restore_transitioned_object(self: Arc<Self>, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<()> {
1433        self.handle_restore_transitioned_object(bucket, object, opts).await
1434    }
1435}
1436
1437#[async_trait::async_trait]
1438impl crate::storage_api_contracts::list::ListOperations for ECStore {
1439    type Error = Error;
1440    type ListObjectsV2Info = ListObjectsV2Info;
1441    type ListObjectVersionsInfo = ListObjectVersionsInfo;
1442    type ObjectInfoOrErr = ObjectInfoOrErr;
1443    type WalkOptions = WalkOptions;
1444    type WalkCancellation = CancellationToken;
1445    type WalkResultSender = tokio::sync::mpsc::Sender<ObjectInfoOrErr>;
1446
1447    // @continuation_token marker
1448    // @start_after as marker when continuation_token empty
1449    // @delimiter default="/", empty when recursive
1450    // @max_keys limit
1451    #[instrument(level = "trace", skip(self))]
1452    async fn list_objects_v2(
1453        self: Arc<Self>,
1454        bucket: &str,
1455        prefix: &str,
1456        continuation_token: Option<String>,
1457        delimiter: Option<String>,
1458        max_keys: i32,
1459        fetch_owner: bool,
1460        start_after: Option<String>,
1461        incl_deleted: bool,
1462    ) -> Result<ListObjectsV2Info> {
1463        self.inner_list_objects_v2(
1464            bucket,
1465            prefix,
1466            continuation_token,
1467            delimiter,
1468            max_keys,
1469            fetch_owner,
1470            start_after,
1471            incl_deleted,
1472        )
1473        .await
1474    }
1475
1476    #[instrument(skip(self))]
1477    async fn list_object_versions(
1478        self: Arc<Self>,
1479        bucket: &str,
1480        prefix: &str,
1481        marker: Option<String>,
1482        version_marker: Option<String>,
1483        delimiter: Option<String>,
1484        max_keys: i32,
1485    ) -> Result<ListObjectVersionsInfo> {
1486        self.inner_list_object_versions(bucket, prefix, marker, version_marker, delimiter, max_keys)
1487            .await
1488    }
1489
1490    async fn walk(
1491        self: Arc<Self>,
1492        rx: CancellationToken,
1493        bucket: &str,
1494        prefix: &str,
1495        result: tokio::sync::mpsc::Sender<ObjectInfoOrErr>,
1496        opts: WalkOptions,
1497    ) -> Result<()> {
1498        self.walk_internal(rx, bucket, prefix, result, opts).await
1499    }
1500}
1501
1502#[async_trait::async_trait]
1503impl crate::storage_api_contracts::multipart::MultipartOperations for ECStore {
1504    type Error = Error;
1505    type ObjectInfo = ObjectInfo;
1506    type ObjectOptions = ObjectOptions;
1507    type PutObjectReader = PutObjReader;
1508    type CompletePart = CompletePart;
1509    type ListMultipartsInfo = ListMultipartsInfo;
1510    type MultipartUploadResult = MultipartUploadResult;
1511    type PartInfo = PartInfo;
1512    type MultipartInfo = MultipartInfo;
1513    type ListPartsInfo = ListPartsInfo;
1514
1515    #[instrument(skip(self))]
1516    async fn list_multipart_uploads(
1517        &self,
1518        bucket: &str,
1519        prefix: &str,
1520        key_marker: Option<String>,
1521        upload_id_marker: Option<String>,
1522        delimiter: Option<String>,
1523        max_uploads: usize,
1524    ) -> Result<ListMultipartsInfo> {
1525        self.handle_list_multipart_uploads(
1526            bucket,
1527            multipart::MultipartUploadListRequest {
1528                prefix: prefix.to_string(),
1529                key_marker,
1530                upload_id_marker,
1531                delimiter,
1532                max_uploads,
1533                expected_incarnation_id: None,
1534            },
1535        )
1536        .await
1537    }
1538
1539    #[instrument(skip(self))]
1540    async fn new_multipart_upload(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<MultipartUploadResult> {
1541        self.handle_new_multipart_upload(bucket, object, opts).await
1542    }
1543
1544    #[instrument(skip(self))]
1545    async fn copy_object_part(
1546        &self,
1547        src_bucket: &str,
1548        src_object: &str,
1549        _dst_bucket: &str,
1550        _dst_object: &str,
1551        _upload_id: &str,
1552        _part_id: usize,
1553        _start_offset: i64,
1554        _length: i64,
1555        _src_info: &ObjectInfo,
1556        _src_opts: &ObjectOptions,
1557        _dst_opts: &ObjectOptions,
1558    ) -> Result<()> {
1559        self.handle_copy_object_part(
1560            src_bucket,
1561            src_object,
1562            _dst_bucket,
1563            _dst_object,
1564            _upload_id,
1565            _part_id,
1566            _start_offset,
1567            _length,
1568            _src_info,
1569            _src_opts,
1570            _dst_opts,
1571        )
1572        .await
1573    }
1574    #[instrument(skip(self, data))]
1575    async fn put_object_part(
1576        &self,
1577        bucket: &str,
1578        object: &str,
1579        upload_id: &str,
1580        part_id: usize,
1581        data: &mut PutObjReader,
1582        opts: &ObjectOptions,
1583    ) -> Result<PartInfo> {
1584        self.handle_put_object_part(bucket, object, upload_id, part_id, data, opts)
1585            .await
1586    }
1587
1588    #[instrument(skip(self))]
1589    async fn get_multipart_info(
1590        &self,
1591        bucket: &str,
1592        object: &str,
1593        upload_id: &str,
1594        opts: &ObjectOptions,
1595    ) -> Result<MultipartInfo> {
1596        self.handle_get_multipart_info(bucket, object, upload_id, opts).await
1597    }
1598
1599    #[instrument(skip(self))]
1600    async fn list_object_parts(
1601        &self,
1602        bucket: &str,
1603        object: &str,
1604        upload_id: &str,
1605        part_number_marker: Option<usize>,
1606        max_parts: usize,
1607        opts: &ObjectOptions,
1608    ) -> Result<ListPartsInfo> {
1609        self.handle_list_object_parts(bucket, object, upload_id, part_number_marker, max_parts, opts)
1610            .await
1611    }
1612
1613    #[instrument(skip(self))]
1614    async fn abort_multipart_upload(&self, bucket: &str, object: &str, upload_id: &str, opts: &ObjectOptions) -> Result<()> {
1615        self.handle_abort_multipart_upload(bucket, object, upload_id, opts).await
1616    }
1617
1618    #[instrument(skip(self))]
1619    async fn complete_multipart_upload(
1620        self: Arc<Self>,
1621        bucket: &str,
1622        object: &str,
1623        upload_id: &str,
1624        uploaded_parts: Vec<CompletePart>,
1625        opts: &ObjectOptions,
1626    ) -> Result<ObjectInfo> {
1627        let result = enqueue_transition_after_write(
1628            self.clone()
1629                .handle_complete_multipart_upload(bucket, object, upload_id, uploaded_parts, opts)
1630                .await,
1631            LcEventSrc::S3CompleteMultipartUpload,
1632        )
1633        .await;
1634        if result.is_ok() {
1635            list_objects::observe_list_objects_mutation(self.as_ref(), bucket).await;
1636        }
1637        result
1638    }
1639}
1640
1641#[async_trait::async_trait]
1642impl crate::storage_api_contracts::heal::HealOperations for ECStore {
1643    type Error = Error;
1644    type HealResultItem = HealResultItem;
1645    type HealOptions = HealOpts;
1646
1647    #[instrument(skip(self))]
1648    async fn heal_format(&self, dry_run: bool) -> Result<(HealResultItem, Option<Error>)> {
1649        self.handle_heal_format(dry_run).await
1650    }
1651
1652    #[instrument(skip(self))]
1653    async fn heal_bucket(&self, bucket: &str, opts: &HealOpts) -> Result<HealResultItem> {
1654        self.handle_heal_bucket(bucket, opts).await
1655    }
1656    #[instrument(level = "trace", skip(self, opts), fields(bucket = %bucket, object = %object, version_id = %version_id))]
1657    async fn heal_object(
1658        &self,
1659        bucket: &str,
1660        object: &str,
1661        version_id: &str,
1662        opts: &HealOpts,
1663    ) -> Result<(HealResultItem, Option<Error>)> {
1664        self.handle_heal_object(bucket, object, version_id, opts).await
1665    }
1666
1667    #[instrument(skip(self))]
1668    async fn get_pool_and_set(&self, id: &str) -> Result<(Option<usize>, Option<usize>, Option<usize>)> {
1669        self.handle_get_pool_and_set(id).await
1670    }
1671
1672    #[instrument(skip(self))]
1673    async fn check_abandoned_parts(&self, bucket: &str, object: &str, opts: &HealOpts) -> Result<()> {
1674        self.handle_check_abandoned_parts(bucket, object, opts).await
1675    }
1676}
1677
1678#[async_trait::async_trait]
1679impl crate::storage_api_contracts::namespace::NamespaceLocking for ECStore {
1680    type Error = Error;
1681    type NamespaceLock = NamespaceLockWrapper;
1682
1683    async fn new_ns_lock(&self, bucket: &str, object: &str) -> Result<NamespaceLockWrapper> {
1684        self.handle_new_ns_lock(bucket, object).await
1685    }
1686}
1687
1688#[async_trait::async_trait]
1689impl crate::storage_api_contracts::admin::StorageAdminApi for ECStore {
1690    type BackendInfo = rustfs_madmin::BackendInfo;
1691    type StorageInfo = rustfs_madmin::StorageInfo;
1692    type Disk = DiskStore;
1693    type Error = Error;
1694
1695    #[instrument(skip(self))]
1696    async fn backend_info(&self) -> Self::BackendInfo {
1697        self.handle_backend_info().await
1698    }
1699
1700    #[instrument(skip(self))]
1701    async fn storage_info(&self) -> Self::StorageInfo {
1702        self.handle_storage_info().await
1703    }
1704
1705    #[instrument(skip(self))]
1706    async fn local_storage_info(&self) -> Self::StorageInfo {
1707        self.handle_local_storage_info().await
1708    }
1709
1710    #[instrument(skip(self))]
1711    async fn disk_set_inventory(
1712        &self,
1713        selector: crate::storage_api_contracts::admin::DiskSetSelector,
1714    ) -> Result<Vec<Option<Self::Disk>>> {
1715        self.handle_get_disks(selector.pool_idx, selector.set_idx).await
1716    }
1717
1718    #[instrument(skip(self))]
1719    fn set_drive_counts(&self) -> Vec<usize> {
1720        self.handle_set_drive_counts()
1721    }
1722}
1723
1724#[cfg(test)]
1725mod tests {
1726    use super::*;
1727    use crate::core::pools::{PoolDecommissionInfo, PoolSpaceInfo, PoolStatus};
1728    use crate::layout::endpoints::{Endpoints, PoolEndpoints, SetupType};
1729    use crate::object_api::ObjectOptions;
1730    use crate::runtime::global::reset_local_disk_test_state;
1731    use crate::runtime::sources::{clear_local_disk_id_map_for_test, local_disk_path_by_id};
1732    use crate::store::init_format::{connect_load_init_formats, init_disks};
1733    use serial_test::serial;
1734    use tempfile::TempDir;
1735
1736    #[test]
1737    fn g_d2_008_default_versioning_config_keeps_persisted_bytes() {
1738        let bytes = crate::bucket::utils::serialize::<VersioningConfiguration>(&ENABLED_VERSIONING_CONFIG)
1739            .expect("the default Versioning configuration must serialize");
1740        assert_eq!(bytes, b"<VersioningConfiguration><Status>Enabled</Status></VersioningConfiguration>");
1741    }
1742
1743    #[test]
1744    fn g_d2_009_default_object_lock_config_keeps_persisted_bytes() {
1745        let bytes = crate::bucket::utils::serialize::<ObjectLockConfiguration>(&ENABLED_OBJECT_LOCK_CONFIG)
1746            .expect("the default Object Lock configuration must serialize");
1747        assert_eq!(
1748            bytes,
1749            b"<ObjectLockConfiguration><ObjectLockEnabled>Enabled</ObjectLockEnabled></ObjectLockConfiguration>"
1750        );
1751    }
1752
1753    #[tokio::test]
1754    async fn test_get_disk_infos() {
1755        let disks = vec![None, None]; // Empty disks for testing
1756        let infos = get_disk_infos(&disks).await;
1757
1758        assert_eq!(infos.len(), disks.len());
1759        // All should be None since we passed None disks
1760        assert!(infos.iter().all(|info| info.is_none()));
1761    }
1762
1763    #[test]
1764    fn ecstore_debug_is_bounded_summary() {
1765        let endpoint_pools = EndpointServerPools::default();
1766        let ctx = Arc::new(InstanceContext::new());
1767        let store = ECStore {
1768            id: uuid::Uuid::new_v4(),
1769            disk_map: [(0, vec![None, None, None, None])].into_iter().collect(),
1770            pools: Vec::new(),
1771            peer_sys: crate::cluster::rpc::S3PeerSys::new_with_instance_ctx(&endpoint_pools, ctx.clone()),
1772            pool_meta: RwLock::new(PoolMeta::default()),
1773            rebalance_meta: RwLock::new(None),
1774            decommission_cancelers: RwLock::new(Vec::new()),
1775            start_gate: Mutex::new(()),
1776            pool_meta_save_gate: Mutex::default(),
1777            ctx,
1778            bucket_fence_registry: Arc::default(),
1779        };
1780
1781        let rendered = format!("{store:?}");
1782
1783        assert!(rendered.len() < 256, "ECStore Debug should stay bounded: {rendered}");
1784        assert!(rendered.contains("disk_map_pool_count"));
1785        assert!(rendered.contains("disk_slot_count"));
1786        assert!(!rendered.contains("disk_map:"));
1787        assert!(!rendered.contains("pools:"));
1788        assert!(!rendered.contains("pool_meta"));
1789        assert!(!rendered.contains("format.json"));
1790        assert!(!rendered.contains("TimedActionSlot"));
1791        assert!(!rendered.contains("DiskHealthTracker"));
1792    }
1793
1794    #[test]
1795    fn object_options_debug_does_not_expand_tier_store_handle() {
1796        let store = build_store_with_ctx(Arc::new(InstanceContext::new()));
1797        let mut opts = ObjectOptions {
1798            version_id: Some("large-version-id".repeat(1024)),
1799            expected_current_version_id: Some("large-expected-version-id".repeat(1024)),
1800            preserve_etag: Some("large-etag".repeat(1024)),
1801            http_preconditions: Some(crate::storage_api_contracts::object::HTTPPreconditions {
1802                if_match: Some("large-if-match".repeat(1024)),
1803                if_none_match: Some("large-if-none-match".repeat(1024)),
1804                ..Default::default()
1805            }),
1806            tier_delete_journal_api: Some(store),
1807            ..Default::default()
1808        };
1809        opts.user_defined.insert("large-user-metadata".to_owned(), "x".repeat(8192));
1810        opts.eval_metadata = Some([("large-eval-metadata".to_owned(), "y".repeat(8192))].into_iter().collect());
1811        opts.transition.status = "large-transition-status".repeat(1024);
1812        opts.transition.tier = "large-transition-tier".repeat(1024);
1813        opts.lifecycle_audit_event.event.rule_id = "large-rule-id".repeat(1024);
1814        opts.lifecycle_audit_event.event.storage_class = "large-storage-class".repeat(1024);
1815
1816        let rendered = format!("{opts:?}");
1817
1818        assert!(rendered.len() < 4096, "ObjectOptions Debug should stay bounded: {rendered}");
1819        assert!(rendered.contains("tier_delete_journal_api: true"));
1820        assert!(rendered.contains("user_defined_count: 1"));
1821        assert!(rendered.contains("eval_metadata_count: Some(1)"));
1822        assert!(!rendered.contains("ECStore {"));
1823        assert!(!rendered.contains("disk_map"));
1824        assert!(!rendered.contains("large-version-id"));
1825        assert!(!rendered.contains("large-expected-version-id"));
1826        assert!(!rendered.contains("large-etag"));
1827        assert!(!rendered.contains("large-if-match"));
1828        assert!(!rendered.contains("large-transition"));
1829        assert!(!rendered.contains("large-rule-id"));
1830        assert!(!rendered.contains("large-storage-class"));
1831        assert!(!rendered.contains("large-user-metadata"));
1832        assert!(!rendered.contains("large-eval-metadata"));
1833        assert!(!rendered.contains("format.json"));
1834        assert!(!rendered.contains("TimedActionSlot"));
1835        assert!(!rendered.contains("DiskHealthTracker"));
1836    }
1837
1838    // Build a minimal ECStore carrying an explicit instance context. Empty
1839    // pools/disks are sufficient: the Phase 5 accessors read only `self.ctx`.
1840    pub(super) fn build_store_with_ctx(ctx: Arc<InstanceContext>) -> Arc<ECStore> {
1841        let endpoint_pools = EndpointServerPools::default();
1842        Arc::new(ECStore {
1843            id: uuid::Uuid::new_v4(),
1844            disk_map: std::collections::HashMap::new(),
1845            pools: Vec::new(),
1846            peer_sys: crate::cluster::rpc::S3PeerSys::new_with_instance_ctx(&endpoint_pools, ctx.clone()),
1847            pool_meta: RwLock::new(PoolMeta::default()),
1848            rebalance_meta: RwLock::new(None),
1849            decommission_cancelers: RwLock::new(Vec::new()),
1850            start_gate: Mutex::new(()),
1851            pool_meta_save_gate: Mutex::default(),
1852            ctx,
1853            bucket_fence_registry: Arc::default(),
1854        })
1855    }
1856
1857    fn scanner_sequence_state(operation_epoch: u64, movement_generation: u64) -> ScannerDataMovementSequenceState {
1858        ScannerDataMovementSequenceState {
1859            operation_epoch,
1860            operation_epoch_exhausted: false,
1861            movement_generation,
1862            movement_generation_exhausted: false,
1863        }
1864    }
1865
1866    #[test]
1867    fn scanner_pause_status_derives_restart_stable_decommission_fields() {
1868        let started_at = OffsetDateTime::from_unix_timestamp(1_000).expect("fixed timestamp should be valid");
1869        let now = OffsetDateTime::from_unix_timestamp(1_090).expect("fixed timestamp should be valid");
1870        let pool_meta = PoolMeta {
1871            pools: vec![PoolStatus {
1872                id: 0,
1873                cmd_line: "pool-0".to_string(),
1874                last_update: started_at,
1875                decommission: Some(PoolDecommissionInfo {
1876                    start_time: Some(started_at),
1877                    queued_buckets: vec!["bucket-a".to_string(), "bucket-b".to_string()],
1878                    bucket: "bucket-a".to_string(),
1879                    ..Default::default()
1880                }),
1881            }],
1882            ..Default::default()
1883        };
1884
1885        let status = resolve_scanner_data_movement_pause_status(&pool_meta, None, false, scanner_sequence_state(7, 11), now);
1886
1887        assert!(status.paused);
1888        assert_eq!(status.policy, "global_pause");
1889        assert_eq!(status.reasons, vec![ScannerDataMovementPauseReason::DecommissionActive]);
1890        assert_eq!(status.started_at_unix_secs, 1_000);
1891        assert_eq!(status.duration_seconds, 90);
1892        assert_eq!(status.operation_epoch, 7);
1893        assert_eq!(status.movement_generation, 11);
1894        assert_eq!(status.movement_backlog_work_items, 2);
1895        assert!(status.movement_backlog_estimated);
1896    }
1897
1898    #[test]
1899    fn completed_decommission_restores_durable_movement_generation() {
1900        let completed_at = OffsetDateTime::from_unix_timestamp(1_100).expect("fixed timestamp should be valid");
1901        let pool_meta = PoolMeta {
1902            pools: vec![PoolStatus {
1903                id: 0,
1904                cmd_line: "pool-0".to_string(),
1905                last_update: completed_at,
1906                decommission: Some(PoolDecommissionInfo {
1907                    complete: true,
1908                    ..Default::default()
1909                }),
1910            }],
1911            ..Default::default()
1912        };
1913        let durable_generation = durable_scanner_data_movement_generation(&pool_meta, None);
1914        let ctx = InstanceContext::new();
1915
1916        ctx.observe_durable_data_movement_generation(durable_generation);
1917
1918        assert_eq!(durable_generation, 1_100_000_000_000);
1919        assert_eq!(ctx.data_movement_generation(), durable_generation);
1920    }
1921
1922    #[tokio::test]
1923    async fn cleared_decommission_restores_durable_movement_generation_after_restart() {
1924        let mut pool_meta = PoolMeta {
1925            pools: vec![PoolStatus {
1926                id: 0,
1927                cmd_line: "pool-0".to_string(),
1928                last_update: OffsetDateTime::UNIX_EPOCH,
1929                decommission: Some(PoolDecommissionInfo {
1930                    failed: true,
1931                    ..Default::default()
1932                }),
1933            }],
1934            ..Default::default()
1935        };
1936        assert!(pool_meta.clear_decommission(0).expect("failed decommission should clear"));
1937        assert!(
1938            pool_meta.pools[0]
1939                .decommission
1940                .as_ref()
1941                .is_some_and(|info| !info.has_decommission_state())
1942        );
1943        let durable_generation = durable_scanner_data_movement_generation(&pool_meta, None);
1944        let restarted = build_store_with_ctx(Arc::new(InstanceContext::new()));
1945        *restarted.pool_meta.write().await = pool_meta;
1946
1947        let status = restarted.scanner_data_movement_pause_status().await;
1948
1949        assert_ne!(durable_generation, 0);
1950        assert!(!status.paused);
1951        assert_eq!(status.movement_generation, durable_generation);
1952        assert_eq!(restarted.scanner_data_movement_generation(), durable_generation);
1953    }
1954
1955    #[tokio::test]
1956    async fn same_tick_cleared_decommission_tombstones_advance_durable_movement_generation() {
1957        let same_tick = OffsetDateTime::from_unix_timestamp(1_100).expect("fixed timestamp should be valid");
1958        let mut pool_meta = PoolMeta {
1959            pools: vec![
1960                PoolStatus {
1961                    id: 0,
1962                    cmd_line: "pool-0".to_string(),
1963                    last_update: same_tick,
1964                    decommission: Some(PoolDecommissionInfo {
1965                        failed: true,
1966                        ..Default::default()
1967                    }),
1968                },
1969                PoolStatus {
1970                    id: 1,
1971                    cmd_line: "pool-1".to_string(),
1972                    last_update: same_tick,
1973                    decommission: Some(PoolDecommissionInfo {
1974                        canceled: true,
1975                        ..Default::default()
1976                    }),
1977                },
1978            ],
1979            ..Default::default()
1980        };
1981
1982        assert!(
1983            pool_meta
1984                .clear_decommission_at_for_test(0, same_tick, None)
1985                .expect("first terminal decommission should clear")
1986        );
1987        let first_generation = durable_scanner_data_movement_generation(&pool_meta, None);
1988        assert_eq!(
1989            first_generation,
1990            scanner_data_movement_timestamp_generation(same_tick + time::Duration::nanoseconds(1))
1991        );
1992
1993        assert!(
1994            pool_meta
1995                .clear_decommission_at_for_test(1, same_tick, None)
1996                .expect("second terminal decommission should clear")
1997        );
1998        let second_generation = durable_scanner_data_movement_generation(&pool_meta, None);
1999        assert_eq!(
2000            second_generation,
2001            scanner_data_movement_timestamp_generation(same_tick + time::Duration::nanoseconds(2))
2002        );
2003        assert!(second_generation > first_generation);
2004
2005        let restarted = build_store_with_ctx(Arc::new(InstanceContext::new()));
2006        *restarted.pool_meta.write().await = pool_meta;
2007        let status = restarted.scanner_data_movement_pause_status().await;
2008
2009        assert!(!status.paused);
2010        assert_eq!(status.movement_generation, second_generation);
2011        assert_eq!(restarted.scanner_data_movement_generation(), second_generation);
2012    }
2013
2014    #[tokio::test]
2015    async fn terminal_decommission_transitions_advance_durable_generation_across_same_or_earlier_clocks() {
2016        let same_tick = OffsetDateTime::from_unix_timestamp(1_200).expect("fixed timestamp should be valid");
2017        let earlier_tick = same_tick - time::Duration::nanoseconds(10);
2018        let rebalance_floor = same_tick + time::Duration::nanoseconds(5);
2019        let rebalance = RebalanceMeta {
2020            stopped_at: Some(rebalance_floor),
2021            id: "completed-rebalance".to_string(),
2022            ..Default::default()
2023        };
2024        let active_decommission = |id| PoolStatus {
2025            id,
2026            cmd_line: format!("pool-{id}"),
2027            last_update: same_tick,
2028            decommission: Some(PoolDecommissionInfo {
2029                start_time: Some(same_tick),
2030                ..Default::default()
2031            }),
2032        };
2033        let mut pool_meta = PoolMeta {
2034            pools: vec![active_decommission(0), active_decommission(1), active_decommission(2)],
2035            ..Default::default()
2036        };
2037
2038        assert!(pool_meta.decommission_complete_at_for_test(0, same_tick, Some(&rebalance)));
2039        assert_eq!(pool_meta.pools[0].last_update, rebalance_floor + time::Duration::nanoseconds(1));
2040
2041        assert!(pool_meta.decommission_cancel_at_for_test(1, same_tick, Some(&rebalance)));
2042        assert_eq!(pool_meta.pools[1].last_update, rebalance_floor + time::Duration::nanoseconds(2));
2043
2044        assert!(pool_meta.decommission_failed_at_for_test(2, earlier_tick, Some(&rebalance)));
2045        assert_eq!(pool_meta.pools[2].last_update, rebalance_floor + time::Duration::nanoseconds(3));
2046        let durable_generation = durable_scanner_data_movement_generation(&pool_meta, Some(&rebalance));
2047        assert_eq!(
2048            durable_generation,
2049            scanner_data_movement_timestamp_generation(rebalance_floor + time::Duration::nanoseconds(3))
2050        );
2051
2052        let restarted = build_store_with_ctx(Arc::new(InstanceContext::new()));
2053        *restarted.pool_meta.write().await = pool_meta;
2054        *restarted.rebalance_meta.write().await = Some(rebalance);
2055        let status = restarted.scanner_data_movement_pause_status().await;
2056
2057        assert_eq!(status.movement_generation, durable_generation);
2058        assert_eq!(restarted.scanner_data_movement_generation(), durable_generation);
2059        assert_eq!(
2060            status.reasons,
2061            vec![
2062                ScannerDataMovementPauseReason::DecommissionFailed,
2063                ScannerDataMovementPauseReason::DecommissionCanceled
2064            ]
2065        );
2066    }
2067
2068    #[tokio::test]
2069    async fn decommission_start_after_clear_advances_durable_generation_across_clock_rollback_after_restart() {
2070        let same_tick = OffsetDateTime::from_unix_timestamp(1_250).expect("fixed timestamp should be valid");
2071        let earlier_tick = same_tick - time::Duration::nanoseconds(10);
2072        let rebalance_floor = same_tick + time::Duration::nanoseconds(5);
2073        let rebalance = RebalanceMeta {
2074            stopped_at: Some(rebalance_floor),
2075            id: "completed-rebalance".to_string(),
2076            ..Default::default()
2077        };
2078        let mut pool_meta = PoolMeta {
2079            pools: vec![PoolStatus {
2080                id: 0,
2081                cmd_line: "pool-0".to_string(),
2082                last_update: same_tick,
2083                decommission: Some(PoolDecommissionInfo {
2084                    failed: true,
2085                    ..Default::default()
2086                }),
2087            }],
2088            ..Default::default()
2089        };
2090
2091        assert!(
2092            pool_meta
2093                .clear_decommission_at_for_test(0, same_tick, Some(&rebalance))
2094                .expect("failed decommission should clear")
2095        );
2096        let cleared_at = rebalance_floor + time::Duration::nanoseconds(1);
2097        assert_eq!(pool_meta.pools[0].last_update, cleared_at);
2098
2099        pool_meta
2100            .decommission_at_for_test(
2101                0,
2102                PoolSpaceInfo {
2103                    total: 200,
2104                    free: 50,
2105                    used: 150,
2106                },
2107                earlier_tick,
2108                Some(&rebalance),
2109            )
2110            .expect("decommission restart after clear should be allowed");
2111        let started_at = cleared_at + time::Duration::nanoseconds(1);
2112        assert_eq!(pool_meta.pools[0].last_update, started_at);
2113        assert_eq!(
2114            pool_meta.pools[0].decommission.as_ref().and_then(|info| info.start_time),
2115            Some(started_at)
2116        );
2117
2118        assert!(pool_meta.decommission_complete_at_for_test(0, earlier_tick, Some(&rebalance)));
2119        let completed_at = started_at + time::Duration::nanoseconds(1);
2120        assert_eq!(pool_meta.pools[0].last_update, completed_at);
2121        let durable_generation = durable_scanner_data_movement_generation(&pool_meta, Some(&rebalance));
2122        assert_eq!(durable_generation, scanner_data_movement_timestamp_generation(completed_at));
2123
2124        let restarted = build_store_with_ctx(Arc::new(InstanceContext::new()));
2125        *restarted.pool_meta.write().await = pool_meta;
2126        *restarted.rebalance_meta.write().await = Some(rebalance);
2127        let status = restarted.scanner_data_movement_pause_status().await;
2128
2129        assert!(!status.paused);
2130        assert_eq!(status.movement_generation, durable_generation);
2131        assert_eq!(restarted.scanner_data_movement_generation(), durable_generation);
2132    }
2133
2134    #[tokio::test]
2135    async fn decommission_terminal_reload_failure_advances_durable_generation_across_clock_rollback_after_restart() {
2136        let terminal_at = OffsetDateTime::from_unix_timestamp(1_280).expect("fixed timestamp should be valid");
2137        let earlier_tick = terminal_at - time::Duration::nanoseconds(10);
2138        let rebalance_floor = terminal_at + time::Duration::nanoseconds(5);
2139        let rebalance = RebalanceMeta {
2140            stopped_at: Some(rebalance_floor),
2141            id: "completed-rebalance".to_string(),
2142            ..Default::default()
2143        };
2144        let mut pool_meta = PoolMeta {
2145            pools: vec![PoolStatus {
2146                id: 0,
2147                cmd_line: "pool-0".to_string(),
2148                last_update: terminal_at,
2149                decommission: Some(PoolDecommissionInfo {
2150                    start_time: Some(terminal_at),
2151                    complete: true,
2152                    ..Default::default()
2153                }),
2154            }],
2155            ..Default::default()
2156        };
2157
2158        assert!(
2159            pool_meta
2160                .record_decommission_terminal_reload_failure_at_for_test(
2161                    0,
2162                    "complete_decommission",
2163                    "peer reload failed".to_string(),
2164                    earlier_tick,
2165                    Some(&rebalance),
2166                )
2167                .expect("reload failure should be recorded")
2168        );
2169        let reload_failure_at = rebalance_floor + time::Duration::nanoseconds(1);
2170        assert_eq!(pool_meta.pools[0].last_update, reload_failure_at);
2171        let info = pool_meta.pools[0]
2172            .decommission
2173            .as_ref()
2174            .expect("decommission metadata should exist");
2175        assert_eq!(info.terminal_reload_attempt_at, Some(reload_failure_at));
2176        assert_eq!(
2177            info.terminal_reload_failures,
2178            vec!["complete_decommission: peer reload failed".to_string()]
2179        );
2180        let durable_generation = durable_scanner_data_movement_generation(&pool_meta, Some(&rebalance));
2181        assert_eq!(durable_generation, scanner_data_movement_timestamp_generation(reload_failure_at));
2182
2183        let restarted = build_store_with_ctx(Arc::new(InstanceContext::new()));
2184        *restarted.pool_meta.write().await = pool_meta;
2185        *restarted.rebalance_meta.write().await = Some(rebalance);
2186        let status = restarted.scanner_data_movement_pause_status().await;
2187
2188        assert!(!status.paused);
2189        assert_eq!(status.movement_generation, durable_generation);
2190        assert_eq!(restarted.scanner_data_movement_generation(), durable_generation);
2191    }
2192
2193    #[tokio::test]
2194    async fn rebalance_transitions_advance_durable_generation_across_same_or_earlier_clocks_after_restart() {
2195        let same_tick = OffsetDateTime::from_unix_timestamp(1_300).expect("fixed timestamp should be valid");
2196        let earlier_tick = same_tick - time::Duration::nanoseconds(10);
2197        let decommission_floor = same_tick + time::Duration::nanoseconds(5);
2198        let store = build_store_with_ctx(Arc::new(InstanceContext::new()));
2199        *store.pool_meta.write().await = PoolMeta {
2200            pools: vec![PoolStatus {
2201                id: 0,
2202                cmd_line: "pool-0".to_string(),
2203                last_update: decommission_floor,
2204                decommission: Some(PoolDecommissionInfo {
2205                    complete: true,
2206                    ..Default::default()
2207                }),
2208            }],
2209            ..Default::default()
2210        };
2211
2212        let started_at = store.next_scanner_data_movement_update(same_tick).await;
2213        assert_eq!(started_at, decommission_floor + time::Duration::nanoseconds(1));
2214        *store.rebalance_meta.write().await = Some(RebalanceMeta {
2215            id: "rebalance-generation".to_string(),
2216            pool_stats: vec![crate::services::rebalance::RebalanceStats {
2217                participating: true,
2218                info: crate::services::rebalance::RebalanceInfo {
2219                    start_time: Some(started_at),
2220                    status: RebalStatus::Started,
2221                    ..Default::default()
2222                },
2223                ..Default::default()
2224            }],
2225            ..Default::default()
2226        });
2227
2228        let completed_at = store.next_scanner_data_movement_update(same_tick).await;
2229        assert_eq!(completed_at, decommission_floor + time::Duration::nanoseconds(2));
2230        {
2231            let mut rebalance_meta = store.rebalance_meta.write().await;
2232            let meta = rebalance_meta.as_mut().expect("rebalance metadata should be present");
2233            meta.pool_stats[0].info.status = RebalStatus::Completed;
2234            meta.pool_stats[0].info.end_time = Some(completed_at);
2235        }
2236
2237        let stopped_at = store.next_scanner_data_movement_update(earlier_tick).await;
2238        assert_eq!(stopped_at, decommission_floor + time::Duration::nanoseconds(3));
2239        {
2240            let mut rebalance_meta = store.rebalance_meta.write().await;
2241            let meta = rebalance_meta.as_mut().expect("rebalance metadata should be present");
2242            meta.stopped_at = Some(stopped_at);
2243        }
2244        let pool_meta = store.pool_meta.read().await.clone();
2245        let rebalance_meta = store.rebalance_meta.read().await.clone();
2246        let durable_generation = durable_scanner_data_movement_generation(&pool_meta, rebalance_meta.as_ref());
2247        assert_eq!(
2248            durable_generation,
2249            scanner_data_movement_timestamp_generation(decommission_floor + time::Duration::nanoseconds(3))
2250        );
2251
2252        let restarted = build_store_with_ctx(Arc::new(InstanceContext::new()));
2253        *restarted.pool_meta.write().await = pool_meta;
2254        *restarted.rebalance_meta.write().await = rebalance_meta;
2255        let status = restarted.scanner_data_movement_pause_status().await;
2256
2257        assert!(!status.paused);
2258        assert_eq!(status.movement_generation, durable_generation);
2259        assert_eq!(restarted.scanner_data_movement_generation(), durable_generation);
2260    }
2261
2262    #[test]
2263    fn malformed_durable_movement_timestamp_exhausts_generation_fail_closed() {
2264        let pool_meta = PoolMeta {
2265            pools: vec![PoolStatus {
2266                id: 0,
2267                cmd_line: "pool-0".to_string(),
2268                last_update: OffsetDateTime::UNIX_EPOCH,
2269                decommission: Some(PoolDecommissionInfo {
2270                    complete: true,
2271                    ..Default::default()
2272                }),
2273            }],
2274            ..Default::default()
2275        };
2276
2277        assert_eq!(durable_scanner_data_movement_generation(&pool_meta, None), u64::MAX);
2278        let exhausted_generation =
2279            OffsetDateTime::from_unix_timestamp(253_402_300_799).expect("the largest RFC 3339 timestamp should be valid");
2280        assert_eq!(scanner_data_movement_timestamp_generation(exhausted_generation), u64::MAX);
2281    }
2282
2283    #[test]
2284    fn malformed_durable_movement_timestamp_is_not_masked_by_valid_rebalance_generation() {
2285        let valid_rebalance_at = OffsetDateTime::from_unix_timestamp(2_400).expect("fixed timestamp should be valid");
2286        let pool_meta = PoolMeta {
2287            pools: vec![PoolStatus {
2288                id: 0,
2289                cmd_line: "pool-0".to_string(),
2290                last_update: OffsetDateTime::UNIX_EPOCH,
2291                decommission: Some(PoolDecommissionInfo {
2292                    complete: true,
2293                    ..Default::default()
2294                }),
2295            }],
2296            ..Default::default()
2297        };
2298        let rebalance_meta = RebalanceMeta {
2299            id: "completed-rebalance".to_string(),
2300            stopped_at: Some(valid_rebalance_at),
2301            pool_stats: vec![crate::services::rebalance::RebalanceStats {
2302                participating: true,
2303                info: crate::services::rebalance::RebalanceInfo {
2304                    start_time: Some(valid_rebalance_at - time::Duration::nanoseconds(1)),
2305                    end_time: Some(valid_rebalance_at),
2306                    status: RebalStatus::Completed,
2307                    ..Default::default()
2308                },
2309                ..Default::default()
2310            }],
2311            ..Default::default()
2312        };
2313
2314        assert_eq!(durable_scanner_data_movement_generation(&pool_meta, Some(&rebalance_meta)), u64::MAX);
2315    }
2316
2317    #[test]
2318    fn durable_movement_generation_without_records_is_zero() {
2319        assert_eq!(durable_scanner_data_movement_generation(&PoolMeta::default(), None), 0);
2320        assert_eq!(
2321            durable_scanner_data_movement_generation(&PoolMeta::default(), Some(&RebalanceMeta::default())),
2322            0
2323        );
2324    }
2325
2326    #[test]
2327    fn scanner_pause_status_distinguishes_terminal_rebalance_epoch_and_idle() {
2328        let last_update = OffsetDateTime::from_unix_timestamp(2_000).expect("fixed timestamp should be valid");
2329        let now = OffsetDateTime::from_unix_timestamp(2_030).expect("fixed timestamp should be valid");
2330        let failed = PoolMeta {
2331            pools: vec![PoolStatus {
2332                id: 0,
2333                cmd_line: "pool-0".to_string(),
2334                last_update,
2335                decommission: Some(PoolDecommissionInfo {
2336                    failed: true,
2337                    ..Default::default()
2338                }),
2339            }],
2340            ..Default::default()
2341        };
2342        let failed_status = resolve_scanner_data_movement_pause_status(&failed, None, false, scanner_sequence_state(3, 12), now);
2343        assert_eq!(failed_status.reasons, vec![ScannerDataMovementPauseReason::DecommissionFailed]);
2344        assert_eq!(failed_status.started_at_unix_secs, 2_000);
2345        assert_eq!(failed_status.duration_seconds, 30);
2346
2347        let rebalance = RebalanceMeta {
2348            pool_stats: vec![crate::services::rebalance::RebalanceStats {
2349                buckets: vec!["bucket-a".to_string(), "bucket-b".to_string()],
2350                participating: true,
2351                info: crate::services::rebalance::RebalanceInfo {
2352                    start_time: Some(last_update),
2353                    status: RebalStatus::Started,
2354                    ..Default::default()
2355                },
2356                ..Default::default()
2357            }],
2358            ..Default::default()
2359        };
2360        let rebalance_status = resolve_scanner_data_movement_pause_status(
2361            &PoolMeta::default(),
2362            Some(&rebalance),
2363            false,
2364            scanner_sequence_state(4, 13),
2365            now,
2366        );
2367        assert_eq!(rebalance_status.reasons, vec![ScannerDataMovementPauseReason::RebalanceActive]);
2368        assert_eq!(rebalance_status.movement_backlog_work_items, 2);
2369
2370        let exhausted = resolve_scanner_data_movement_pause_status(
2371            &PoolMeta::default(),
2372            None,
2373            false,
2374            ScannerDataMovementSequenceState {
2375                operation_epoch: u64::MAX,
2376                operation_epoch_exhausted: true,
2377                movement_generation: 14,
2378                movement_generation_exhausted: false,
2379            },
2380            now,
2381        );
2382        assert_eq!(exhausted.reasons, vec![ScannerDataMovementPauseReason::OperationEpochExhausted]);
2383        assert_eq!(exhausted.started_at_unix_secs, 0);
2384
2385        let generation_exhausted = resolve_scanner_data_movement_pause_status(
2386            &PoolMeta::default(),
2387            None,
2388            false,
2389            ScannerDataMovementSequenceState {
2390                operation_epoch: 5,
2391                operation_epoch_exhausted: false,
2392                movement_generation: u64::MAX,
2393                movement_generation_exhausted: true,
2394            },
2395            now,
2396        );
2397        assert_eq!(
2398            generation_exhausted.reasons,
2399            vec![ScannerDataMovementPauseReason::MovementGenerationExhausted]
2400        );
2401
2402        let idle =
2403            resolve_scanner_data_movement_pause_status(&PoolMeta::default(), None, false, scanner_sequence_state(5, 15), now);
2404        assert!(!idle.paused);
2405        assert!(idle.reasons.is_empty());
2406        assert!(!idle.movement_backlog_estimated);
2407    }
2408
2409    #[tokio::test]
2410    async fn scanner_data_usage_publication_blocks_active_and_unqueued_terminal_decommission() {
2411        let store = build_store_with_ctx(Arc::new(InstanceContext::new()));
2412        let cases = [
2413            (
2414                "active",
2415                PoolDecommissionInfo {
2416                    start_time: Some(OffsetDateTime::now_utc()),
2417                    ..Default::default()
2418                },
2419                true,
2420            ),
2421            (
2422                "failed",
2423                PoolDecommissionInfo {
2424                    failed: true,
2425                    ..Default::default()
2426                },
2427                true,
2428            ),
2429            (
2430                "canceled",
2431                PoolDecommissionInfo {
2432                    canceled: true,
2433                    ..Default::default()
2434                },
2435                true,
2436            ),
2437            (
2438                "queued_failed",
2439                PoolDecommissionInfo {
2440                    failed: true,
2441                    queued: true,
2442                    ..Default::default()
2443                },
2444                false,
2445            ),
2446            (
2447                "complete",
2448                PoolDecommissionInfo {
2449                    complete: true,
2450                    ..Default::default()
2451                },
2452                false,
2453            ),
2454            ("idle", PoolDecommissionInfo::default(), false),
2455        ];
2456
2457        for (name, decommission, expected) in cases {
2458            *store.pool_meta.write().await = PoolMeta {
2459                pools: vec![PoolStatus {
2460                    id: 0,
2461                    cmd_line: format!("scanner-publication-{name}"),
2462                    last_update: OffsetDateTime::now_utc(),
2463                    decommission: Some(decommission),
2464                }],
2465                ..Default::default()
2466            };
2467            assert_eq!(
2468                store.scanner_data_usage_publication_blocked().await,
2469                expected,
2470                "unexpected scanner publication barrier state for {name}"
2471            );
2472        }
2473    }
2474
2475    #[tokio::test]
2476    async fn scanner_data_usage_publication_admission_is_fenced_and_epoch_monotonic() {
2477        let store = build_store_with_ctx(Arc::new(InstanceContext::new()));
2478        let operation_gate = store.ctx.data_movement_operation_gate();
2479        let movement_guard = operation_gate.write().await;
2480        let pending = {
2481            let store = store.clone();
2482            tokio::spawn(async move { store.scanner_data_usage_publication_admission_guard().await })
2483        };
2484        tokio::task::yield_now().await;
2485        assert!(!pending.is_finished(), "publication admission must wait for a movement writer");
2486        drop(movement_guard);
2487
2488        let (_, epoch) = pending
2489            .await
2490            .expect("publication admission task should not panic")
2491            .expect("idle store should admit publication");
2492        assert_eq!(epoch, 0);
2493        assert_eq!(store.ctx.advance_data_movement_operation_epoch(), 1);
2494        let (_, next_epoch) = store
2495            .scanner_data_usage_publication_admission_guard()
2496            .await
2497            .expect("idle store should admit the next publication");
2498        assert_eq!(next_epoch, 1);
2499        assert_eq!(store.scanner_data_movement_generation(), 1);
2500    }
2501
2502    #[tokio::test(start_paused = true)]
2503    async fn scanner_publication_lease_blocks_movement_writer_until_release_or_expiry() {
2504        let store = build_store_with_ctx(Arc::new(InstanceContext::new()));
2505        let (token, generation) = store
2506            .acquire_scanner_publication_lease(0, crate::runtime::instance::SCANNER_PUBLICATION_LEASE_TTL)
2507            .await
2508            .expect("an idle store should grant a publication lease");
2509        assert_eq!(generation, 0);
2510
2511        let gate = store.ctx.data_movement_operation_gate();
2512        let (writer_started, writer_started_rx) = tokio::sync::oneshot::channel();
2513        let movement_writer = tokio::spawn(async move {
2514            let _ = writer_started.send(());
2515            gate.write_owned().await
2516        });
2517        writer_started_rx
2518            .await
2519            .expect("movement writer should reach the gate before waiting");
2520        assert!(
2521            !movement_writer.is_finished(),
2522            "a movement writer must wait while the remote lease owns the read guard"
2523        );
2524
2525        assert!(store.release_scanner_publication_lease(token).await);
2526        tokio::time::timeout(Duration::from_secs(1), movement_writer)
2527            .await
2528            .expect("movement writer should proceed after lease release")
2529            .expect("movement writer task should not panic");
2530
2531        let (expiring_token, _) = store
2532            .acquire_scanner_publication_lease(0, crate::runtime::instance::SCANNER_PUBLICATION_LEASE_TTL)
2533            .await
2534            .expect("the store should grant a second publication lease");
2535        let expiry_gate = store.ctx.data_movement_operation_gate();
2536        let (expiry_started, expiry_started_rx) = tokio::sync::oneshot::channel();
2537        let mut expiry_writer = tokio::spawn(async move {
2538            let _ = expiry_started.send(());
2539            expiry_gate.write_owned().await
2540        });
2541        expiry_started_rx
2542            .await
2543            .expect("expiry writer should reach the gate before waiting");
2544        assert!(!expiry_writer.is_finished());
2545        tokio::time::advance(crate::runtime::instance::SCANNER_PUBLICATION_LEASE_TTL + Duration::from_millis(1)).await;
2546        tokio::task::yield_now().await;
2547        tokio::time::timeout(Duration::from_secs(1), &mut expiry_writer)
2548            .await
2549            .expect("movement writer should proceed after lease expiry")
2550            .expect("expiry writer task should not panic");
2551        assert!(
2552            store.validate_scanner_publication_lease(expiring_token, 0).await.is_err(),
2553            "an expired lease must not validate after its read guard is released"
2554        );
2555        assert!(!store.release_scanner_publication_lease(expiring_token).await);
2556    }
2557
2558    #[tokio::test]
2559    async fn scanner_publication_lease_rejects_a_new_movement_generation() {
2560        let store = build_store_with_ctx(Arc::new(InstanceContext::new()));
2561        let (token, generation) = store
2562            .acquire_scanner_publication_lease(0, crate::runtime::instance::SCANNER_PUBLICATION_LEASE_TTL)
2563            .await
2564            .expect("an idle store should grant a publication lease");
2565
2566        assert_eq!(store.ctx.advance_data_movement_generation(), Some(1));
2567        assert!(
2568            store.validate_scanner_publication_lease(token, generation).await.is_err(),
2569            "a lease from the prior movement generation must fail closed"
2570        );
2571        assert!(store.release_scanner_publication_lease(token).await);
2572    }
2573
2574    #[tokio::test]
2575    async fn scanner_publication_lease_rejects_stale_generation_before_install() {
2576        let store = build_store_with_ctx(Arc::new(InstanceContext::new()));
2577        let error = store
2578            .acquire_scanner_publication_lease(1, crate::runtime::instance::SCANNER_PUBLICATION_LEASE_TTL)
2579            .await
2580            .expect_err("a stale movement generation must not acquire a lease");
2581        assert!(error.to_string().contains("generation is stale"));
2582    }
2583
2584    #[tokio::test]
2585    async fn scanner_publication_lease_rejects_namespace_change_during_admission() {
2586        let ctx = Arc::new(InstanceContext::new());
2587        let store = build_store_with_ctx(ctx.clone());
2588        let snapshot_blocker = store.rebalance_meta.write().await;
2589        let mut acquire =
2590            Box::pin(store.acquire_scanner_publication_lease(0, crate::runtime::instance::SCANNER_PUBLICATION_LEASE_TTL));
2591        assert!(futures::poll!(&mut acquire).is_pending(), "acquisition reaches the blocked snapshot");
2592        assert!(ctx.data_movement_operation_gate().try_write_owned().is_err());
2593        drop(ctx.begin_namespace_commit());
2594        assert_eq!(ctx.namespace_commit_generation(), 2);
2595        assert!(!ctx.namespace_commits_pending());
2596        drop(snapshot_blocker);
2597        let error = tokio::time::timeout(Duration::from_secs(1), acquire)
2598            .await
2599            .expect("snapshot admission must finish")
2600            .expect_err("acquisition must preserve its original namespace generation");
2601        assert_eq!(error.to_string(), "Io error: scanner publication lease generation is stale");
2602        assert!(ctx.data_movement_operation_gate().try_write_owned().is_ok(), "no lease was installed");
2603    }
2604
2605    #[tokio::test]
2606    async fn scanner_publication_lease_rechecks_namespace_after_validate_snapshot() {
2607        let ctx = Arc::new(InstanceContext::new());
2608        let store = build_store_with_ctx(ctx.clone());
2609        let (token, generation) = store
2610            .acquire_scanner_publication_lease(0, crate::runtime::instance::SCANNER_PUBLICATION_LEASE_TTL)
2611            .await
2612            .expect("current lease");
2613        let snapshot_blocker = store.rebalance_meta.write().await;
2614        let mut validate = Box::pin(store.validate_scanner_publication_lease(token, generation));
2615        assert!(futures::poll!(&mut validate).is_pending(), "target guard queues its first snapshot read");
2616        let mut next_writer = Box::pin(store.rebalance_meta.write());
2617        assert!(futures::poll!(&mut next_writer).is_pending());
2618        drop(snapshot_blocker);
2619        // Fair lock order admits the first read, then this queued writer, then
2620        // validate's second snapshot. The first target check has already passed.
2621        assert!(futures::poll!(&mut validate).is_pending(), "validate reaches its second snapshot");
2622        let next_writer = next_writer.await;
2623        drop(ctx.begin_namespace_commit());
2624        assert_eq!(ctx.namespace_commit_generation(), 2);
2625        assert!(!ctx.namespace_commits_pending());
2626        drop(next_writer);
2627        let error = tokio::time::timeout(Duration::from_secs(1), validate)
2628            .await
2629            .expect("validation must finish without nesting movement read locks")
2630            .expect_err("the final snapshot must reject a completed namespace commit");
2631        assert_eq!(error.to_string(), "Io error: scanner publication lease generation is stale");
2632        assert!(
2633            ctx.data_movement_operation_gate().try_write_owned().is_err(),
2634            "stale lookup retains the lease permit"
2635        );
2636        assert!(store.release_scanner_publication_lease(token).await);
2637        assert!(ctx.data_movement_operation_gate().try_write_owned().is_ok());
2638    }
2639
2640    #[tokio::test]
2641    async fn scanner_publication_lease_rejects_namespace_generation_exhaustion() {
2642        let ctx = Arc::new(InstanceContext::new());
2643        let store = build_store_with_ctx(ctx.clone());
2644        let (token, generation) = store
2645            .acquire_scanner_publication_lease(0, crate::runtime::instance::SCANNER_PUBLICATION_LEASE_TTL)
2646            .await
2647            .expect("current lease");
2648        ctx.set_namespace_commit_generation_for_test(u64::MAX);
2649        assert_eq!(store.scanner_namespace_mutation_generation(), u64::MAX);
2650        let acquire = store
2651            .acquire_scanner_publication_lease(generation, crate::runtime::instance::SCANNER_PUBLICATION_LEASE_TTL)
2652            .await
2653            .expect_err("exhausted namespace cannot become a new lease baseline");
2654        assert_eq!(acquire.to_string(), "Io error: scanner publication lease generation is stale");
2655        let validate = store.validate_scanner_publication_lease(token, generation).await;
2656        let target = store.acquire_scanner_publication_lease_guard(token).await;
2657        assert!(validate.is_err() && target.is_err(), "exhaustion rejects both old-token entrances");
2658        assert!(
2659            ctx.data_movement_operation_gate().try_write_owned().is_err(),
2660            "rejection must retain the old permit"
2661        );
2662        assert!(store.release_scanner_publication_lease(token).await);
2663        assert!(ctx.data_movement_operation_gate().try_write_owned().is_ok());
2664    }
2665
2666    #[tokio::test(start_paused = true)]
2667    async fn scanner_publication_commit_scope_owns_permit_until_terminal_drain() {
2668        let store = build_store_with_ctx(Arc::new(InstanceContext::new()));
2669        let scope = store
2670            .scanner_data_usage_publication_commit_scope(
2671                0,
2672                tokio::time::Instant::now() + Duration::from_secs(30),
2673                vec![Uuid::new_v4()],
2674            )
2675            .await
2676            .expect("idle storage should grant a publication scope");
2677        assert_eq!(scope.state(), crate::object_api::ScannerPublicationCommitState::Admitted);
2678        assert_eq!(scope.remote_lease_tokens().len(), 1);
2679
2680        let gate = store.ctx.data_movement_operation_gate();
2681        let writer = tokio::spawn(async move { gate.write_owned().await });
2682        tokio::task::yield_now().await;
2683        assert!(!writer.is_finished(), "the scope must own its movement permit after the caller returns");
2684
2685        scope.cancel();
2686        assert!(scope.mark_aborted_before_commit());
2687        assert_eq!(
2688            scope.wait_for_completion().await,
2689            crate::object_api::ScannerPublicationCommitState::AbortedBeforeCommit
2690        );
2691        assert!(scope.release_movement_permit().await);
2692        tokio::time::timeout(Duration::from_secs(1), writer)
2693            .await
2694            .expect("movement writer should proceed after the scope drains")
2695            .expect("movement writer task should not panic");
2696    }
2697
2698    #[tokio::test(start_paused = true)]
2699    async fn scanner_publication_commit_scope_rejects_late_start_and_keeps_indeterminate_permit() {
2700        let store = build_store_with_ctx(Arc::new(InstanceContext::new()));
2701        let scope = store
2702            .scanner_data_usage_publication_commit_scope(0, tokio::time::Instant::now() + Duration::from_secs(1), Vec::new())
2703            .await
2704            .expect("idle storage should grant a publication scope");
2705        tokio::time::advance(Duration::from_secs(1)).await;
2706        assert_eq!(
2707            scope.try_begin(),
2708            Err(crate::object_api::ScannerPublicationCommitStartError::DeadlineExceeded)
2709        );
2710        assert!(
2711            !scope.release_movement_permit().await,
2712            "an admitted scope is not safe to release before owner resolution"
2713        );
2714        assert!(scope.mark_aborted_before_commit());
2715        assert!(scope.release_movement_permit().await);
2716
2717        let scope = store
2718            .scanner_data_usage_publication_commit_scope(0, tokio::time::Instant::now() + Duration::from_secs(30), Vec::new())
2719            .await
2720            .expect("a second idle publication scope should be granted");
2721        scope.try_begin().expect("scope should enter the mutation state");
2722        scope.cancel();
2723        assert!(scope.mark_indeterminate());
2724        assert_eq!(
2725            scope.wait_for_completion().await,
2726            crate::object_api::ScannerPublicationCommitState::Indeterminate
2727        );
2728        assert!(!scope.release_movement_permit().await, "indeterminate mutation must retain the permit");
2729    }
2730
2731    #[tokio::test]
2732    async fn scanner_publication_scope_guard_classifies_early_returns_conservatively() {
2733        let store = build_store_with_ctx(Arc::new(InstanceContext::new()));
2734        let permit = store.ctx.data_movement_operation_gate().read_owned().await;
2735        let scope = ScannerPublicationCommitScope::new_storage_owned(
2736            0,
2737            tokio::time::Instant::now() + Duration::from_secs(30),
2738            Vec::new(),
2739            permit,
2740        );
2741        {
2742            let _guard = crate::object_api::ScannerPublicationCommitScopeGuard::new(scope.clone());
2743        }
2744        assert_eq!(scope.state(), crate::object_api::ScannerPublicationCommitState::AbortedBeforeCommit);
2745        assert!(scope.release_movement_permit().await);
2746
2747        let permit = store.ctx.data_movement_operation_gate().read_owned().await;
2748        let scope = ScannerPublicationCommitScope::new_storage_owned(
2749            0,
2750            tokio::time::Instant::now() + Duration::from_secs(30),
2751            Vec::new(),
2752            permit,
2753        );
2754        scope.try_begin().expect("scope should enter the mutation state");
2755        {
2756            let _guard = crate::object_api::ScannerPublicationCommitScopeGuard::new(scope.clone());
2757        }
2758        assert_eq!(scope.state(), crate::object_api::ScannerPublicationCommitState::Indeterminate);
2759        assert!(!scope.release_movement_permit().await);
2760    }
2761
2762    #[tokio::test]
2763    async fn scanner_target_guard_keeps_movement_writer_fenced_after_lease_release() {
2764        let store = build_store_with_ctx(Arc::new(InstanceContext::new()));
2765        let (token, _) = store
2766            .acquire_scanner_publication_lease(0, crate::runtime::instance::SCANNER_PUBLICATION_LEASE_TTL)
2767            .await
2768            .expect("an idle store should grant a publication lease");
2769        let target_guard = store
2770            .acquire_scanner_publication_lease_guard(token)
2771            .await
2772            .expect("the target-side rename should acquire its short guard");
2773        assert!(store.release_scanner_publication_lease(token).await);
2774
2775        let movement_gate = store.ctx.data_movement_operation_gate();
2776        let movement_writer = tokio::spawn(async move { movement_gate.write_owned().await });
2777        tokio::task::yield_now().await;
2778        assert!(!movement_writer.is_finished(), "the target guard must span the rename operation");
2779        drop(target_guard);
2780        tokio::time::timeout(std::time::Duration::from_secs(1), movement_writer)
2781            .await
2782            .expect("movement writer should proceed after the target rename guard is dropped")
2783            .expect("movement writer task should not panic");
2784    }
2785
2786    #[tokio::test]
2787    async fn scanner_publication_lease_rejects_restart_aba_token() {
2788        let first_store = build_store_with_ctx(Arc::new(InstanceContext::new()));
2789        let (token, generation) = first_store
2790            .acquire_scanner_publication_lease(0, crate::runtime::instance::SCANNER_PUBLICATION_LEASE_TTL)
2791            .await
2792            .expect("the initial instance should grant a publication lease");
2793        first_store
2794            .validate_scanner_publication_lease(token, generation)
2795            .await
2796            .expect("the current instance should validate its own live token");
2797        first_store
2798            .acquire_scanner_publication_lease_guard(token)
2799            .await
2800            .expect("the current instance should admit the target-side rename");
2801
2802        // A restarted storage instance starts its local generation at zero,
2803        // but its process-owned lease table is empty.  The old token must not
2804        // pass validation just because the numeric generation matches again.
2805        let restarted_store = build_store_with_ctx(Arc::new(InstanceContext::new()));
2806        let error = restarted_store
2807            .validate_scanner_publication_lease(token, generation)
2808            .await
2809            .expect_err("a token from a prior instance must fail closed after restart");
2810        assert!(error.to_string().contains("unknown or expired"));
2811        let error = restarted_store
2812            .acquire_scanner_publication_lease_guard(token)
2813            .await
2814            .expect_err("a restarted instance must reject the target-side rename token");
2815        assert!(error.to_string().contains("unknown or expired"));
2816    }
2817
2818    #[tokio::test]
2819    async fn movement_generation_notifies_waiters_and_fails_closed_at_maximum() {
2820        let store = build_store_with_ctx(Arc::new(InstanceContext::new()));
2821        let notify = store.scanner_data_movement_changed();
2822        let notified = notify.notified();
2823        tokio::pin!(notified);
2824        notified.as_mut().enable();
2825
2826        assert_eq!(store.ctx.advance_data_movement_operation_epoch(), 1);
2827        tokio::time::timeout(std::time::Duration::from_secs(1), notified)
2828            .await
2829            .expect("movement transition should wake scanner waiters");
2830        assert_eq!(store.scanner_data_movement_generation(), 1);
2831
2832        store.ctx.set_data_movement_generation_for_test(u64::MAX - 1);
2833        assert_eq!(store.ctx.advance_data_movement_generation(), Some(u64::MAX));
2834        assert!(store.scanner_data_movement_generation_exhausted());
2835        assert_eq!(store.ctx.advance_data_movement_generation(), None);
2836        assert!(
2837            store.scanner_data_usage_publication_admission_guard().await.is_none(),
2838            "generation exhaustion must close publication admission"
2839        );
2840    }
2841
2842    #[tokio::test]
2843    async fn scanner_data_usage_publication_epoch_releases_gate_before_backend_io() {
2844        let store = build_store_with_ctx(Arc::new(InstanceContext::new()));
2845        let epoch = store
2846            .scanner_data_usage_publication_epoch()
2847            .await
2848            .expect("idle store should expose a publication epoch");
2849        assert_eq!(epoch, 0);
2850
2851        let operation_gate = store.ctx.data_movement_operation_gate();
2852        let _movement_guard = tokio::time::timeout(Duration::from_secs(1), operation_gate.write())
2853            .await
2854            .expect("epoch capture must not hold the movement gate across backend I/O");
2855    }
2856
2857    #[tokio::test]
2858    async fn scanner_data_usage_publication_admission_blocks_active_rebalance_snapshot() {
2859        let store = build_store_with_ctx(Arc::new(InstanceContext::new()));
2860        *store.rebalance_meta.write().await = Some(RebalanceMeta {
2861            pool_stats: vec![crate::services::rebalance::RebalanceStats {
2862                participating: true,
2863                info: crate::services::rebalance::RebalanceInfo {
2864                    status: crate::services::rebalance::RebalStatus::Started,
2865                    ..Default::default()
2866                },
2867                ..Default::default()
2868            }],
2869            ..Default::default()
2870        });
2871
2872        assert!(
2873            store.scanner_data_usage_publication_admission_guard().await.is_none(),
2874            "active rebalance must fail closed at the storage-owned admission boundary"
2875        );
2876    }
2877
2878    #[tokio::test]
2879    async fn scanner_publication_epoch_exhaustion_fails_closed_after_max() {
2880        let store = build_store_with_ctx(Arc::new(InstanceContext::new()));
2881        store.ctx.set_data_movement_operation_epoch_for_test(u64::MAX - 1);
2882
2883        assert_eq!(store.ctx.advance_data_movement_operation_epoch(), u64::MAX);
2884        assert!(store.ctx.data_movement_operation_epoch_exhausted());
2885        assert!(
2886            store.scanner_data_usage_publication_admission_guard().await.is_none(),
2887            "publication must fail closed at the reserved terminal epoch"
2888        );
2889
2890        assert_eq!(store.ctx.advance_data_movement_operation_epoch(), u64::MAX);
2891        assert!(store.ctx.data_movement_operation_epoch_exhausted());
2892        assert!(store.scanner_data_usage_publication_blocked().await);
2893    }
2894
2895    // The object graph is the isolation carrier: two ECStore instances holding
2896    // distinct contexts report independent erasure state through their real
2897    // `&self` accessors — no cross-contamination.
2898    #[tokio::test]
2899    async fn instance_context_carrier_isolates_two_stores() {
2900        let ctx_a = Arc::new(InstanceContext::new());
2901        let ctx_b = Arc::new(InstanceContext::new());
2902        ctx_a.update_erasure_type(SetupType::DistErasure).await;
2903        ctx_b.update_erasure_type(SetupType::ErasureSD).await;
2904        ctx_a.set_endpoints(EndpointServerPools::from(vec![PoolEndpoints {
2905            legacy: false,
2906            set_count: 1,
2907            drives_per_set: 1,
2908            endpoints: Endpoints::default(),
2909            cmd_line: "instance-a".to_string(),
2910            platform: String::new(),
2911        }]));
2912        ctx_b.set_endpoints(EndpointServerPools::from(vec![PoolEndpoints {
2913            legacy: true,
2914            set_count: 2,
2915            drives_per_set: 2,
2916            endpoints: Endpoints::default(),
2917            cmd_line: "instance-b".to_string(),
2918            platform: String::new(),
2919        }]));
2920
2921        let store_a = build_store_with_ctx(ctx_a);
2922        let store_b = build_store_with_ctx(ctx_b);
2923
2924        // store_a: distributed erasure (implies is_erasure), not single-drive.
2925        assert!(store_a.setup_is_erasure().await);
2926        assert!(store_a.setup_is_dist_erasure().await);
2927        assert!(!store_a.setup_is_erasure_sd().await);
2928
2929        // store_b: single-drive erasure only.
2930        assert!(store_b.setup_is_erasure_sd().await);
2931        assert!(!store_b.setup_is_erasure().await);
2932        assert!(!store_b.setup_is_dist_erasure().await);
2933        let endpoints_a = store_a.instance_endpoints().expect("instance A endpoints");
2934        let endpoints_b = store_b.instance_endpoints().expect("instance B endpoints");
2935        assert_eq!(endpoints_a.as_ref()[0].set_count, 1);
2936        assert_eq!(endpoints_b.as_ref()[0].set_count, 2);
2937        assert!(!endpoints_a.as_ref()[0].legacy);
2938        assert!(endpoints_b.as_ref()[0].legacy);
2939    }
2940
2941    // The production/test constructors ADOPT the process bootstrap context
2942    // (same Arc), so a startup write recorded before the store existed is
2943    // visible through the store afterward — single-instance behavior preserved.
2944    #[tokio::test]
2945    async fn store_adopts_bootstrap_context() {
2946        let store = build_store_with_ctx(crate::runtime::instance::bootstrap_ctx());
2947        assert!(
2948            Arc::ptr_eq(&store.ctx, &crate::runtime::instance::bootstrap_ctx()),
2949            "store built via adoption must share the bootstrap context Arc"
2950        );
2951    }
2952
2953    #[tokio::test]
2954    async fn test_has_space_for() {
2955        let disk_infos = vec![None, None]; // No actual disk info
2956
2957        let result = crate::layout::pool_space::has_space_for(&disk_infos, 1024).await;
2958        // Should fail due to no valid disk info
2959        assert!(result.is_err());
2960    }
2961
2962    #[tokio::test]
2963    async fn test_find_local_disk() {
2964        let result = peer::find_local_disk("/nonexistent/path").await;
2965        assert!(result.is_none(), "Should return None for nonexistent path");
2966    }
2967
2968    #[tokio::test]
2969    #[serial]
2970    async fn test_find_local_disk_by_ref_backfills_uuid_map() {
2971        reset_local_disk_test_state().await;
2972
2973        let temp_dir = TempDir::new().expect("create temp dir for local disk ref test");
2974        let disk_paths = (0..4)
2975            .map(|idx| temp_dir.path().join(format!("disk{}", idx + 1)))
2976            .collect::<Vec<_>>();
2977        for disk_path in &disk_paths {
2978            std::fs::create_dir_all(disk_path).expect("create disk path");
2979        }
2980
2981        let mut endpoints = Vec::new();
2982        for (idx, disk_path) in disk_paths.iter().enumerate() {
2983            let mut endpoint = Endpoint::try_from(disk_path.to_str().expect("disk path to str")).expect("endpoint");
2984            endpoint.set_pool_index(0);
2985            endpoint.set_set_index(0);
2986            endpoint.set_disk_index(idx);
2987            endpoints.push(endpoint);
2988        }
2989
2990        let endpoint_pools = EndpointServerPools(vec![PoolEndpoints {
2991            legacy: false,
2992            set_count: 1,
2993            drives_per_set: 4,
2994            endpoints: Endpoints::from(endpoints),
2995            cmd_line: "find-local-disk-by-ref-test".to_string(),
2996            platform: "test".to_string(),
2997        }]);
2998
2999        init_local_disks(endpoint_pools.clone()).await.expect("init local disks");
3000
3001        let (mut disks, errs) = init_disks(
3002            &endpoint_pools.as_ref().first().expect("pool endpoints").endpoints,
3003            &DiskOption {
3004                cleanup: true,
3005                health_check: false,
3006            },
3007        )
3008        .await;
3009
3010        assert!(errs.iter().all(|err| err.is_none()), "disk init should succeed: {errs:?}");
3011        connect_load_init_formats(true, &mut disks, 1, 4, None)
3012            .await
3013            .expect("initialize format metadata");
3014
3015        clear_local_disk_id_map_for_test().await;
3016
3017        let local_disks = all_local_disk().await;
3018        let first_disk = local_disks.first().expect("local disk exists");
3019        let disk_id = first_disk
3020            .get_disk_id()
3021            .await
3022            .expect("get disk id should succeed")
3023            .expect("disk id should exist");
3024
3025        let found = find_local_disk_by_ref(&disk_id.to_string()).await;
3026        assert!(found.is_some(), "disk lookup by id should backfill cache");
3027        assert_eq!(local_disk_path_by_id(&disk_id).await, Some(first_disk.endpoint().to_string()));
3028
3029        reset_local_disk_test_state().await;
3030    }
3031
3032    #[tokio::test]
3033    async fn test_all_local_disk_path() {
3034        let paths = all_local_disk_path().await;
3035        // Should return empty or some paths depending on global state
3036        assert!(paths.is_empty() || !paths.is_empty());
3037    }
3038
3039    #[tokio::test]
3040    async fn test_all_local_disk() {
3041        let disks = all_local_disk().await;
3042        // Should return empty or some disks depending on global state
3043        assert!(disks.is_empty() || !disks.is_empty());
3044    }
3045
3046    #[test]
3047    fn test_should_not_enqueue_transition_for_internal_metadata_bucket() {
3048        let oi = ObjectInfo {
3049            bucket: RUSTFS_META_BUCKET.to_string(),
3050            name: format!("{BUCKET_META_PREFIX}/bucket/.metadata.bin"),
3051            ..Default::default()
3052        };
3053
3054        assert!(!should_enqueue_transition_immediately(&oi));
3055    }
3056}