Skip to main content

rustfs_ecstore/store/
mod.rs

1// Copyright 2024 RustFS Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15#![allow(clippy::map_entry)]
16
17use crate::bucket::lifecycle::bucket_lifecycle_audit::LcEventSrc;
18use crate::bucket::lifecycle::bucket_lifecycle_ops::{
19    enqueue_immediate_expiry, enqueue_transition_immediate, init_background_expiry,
20};
21use crate::bucket::metadata_sys;
22use crate::bucket::utils::check_abort_multipart_args;
23use crate::bucket::utils::check_complete_multipart_args;
24use crate::bucket::utils::check_copy_obj_args;
25use crate::bucket::utils::check_del_obj_args;
26use crate::bucket::utils::check_get_obj_args;
27use crate::bucket::utils::check_list_multipart_args;
28use crate::bucket::utils::check_list_parts_args;
29use crate::bucket::utils::check_new_multipart_args;
30use crate::bucket::utils::check_object_args;
31use crate::bucket::utils::check_put_object_args;
32use crate::bucket::utils::check_put_object_part_args;
33use crate::bucket::utils::{check_valid_bucket_name, check_valid_bucket_name_strict, is_meta_bucketname};
34use crate::cluster::rpc::{RemoteClient, S3PeerSys};
35use crate::config::storageclass;
36use crate::core::pools::{DecommissionCanceler, PoolMeta, PoolMetaWriteState};
37use crate::disk::endpoint::{Endpoint, EndpointType};
38use crate::disk::{DiskAPI, DiskInfo, DiskInfoOptions};
39use crate::error::{Error, Result};
40use crate::error::{
41    StorageError, is_err_bucket_exists, is_err_invalid_upload_id, is_err_object_not_found, is_err_read_quorum,
42    is_err_strict_volume_not_found, is_err_version_not_found, to_object_err,
43};
44use crate::runtime::global::DISK_RESERVE_FRACTION;
45use crate::runtime::instance::InstanceContext;
46use crate::runtime::sources as runtime_sources;
47use crate::services::rebalance::{RebalStatus, RebalanceMeta, is_rebalance_conflicting_with_decommission};
48use crate::storage_api_contracts::{
49    bucket::{BucketInfo, BucketOperations, BucketOptions, DeleteBucketOptions, MakeBucketOptions},
50    list::{StorageListObjectVersionsInfo, StorageListObjectsV2Info, StorageObjectInfoOrErr, StorageWalkOptions},
51    multipart::{CompletePart, ListMultipartsInfo, ListPartsInfo, MultipartInfo, MultipartUploadResult, PartInfo},
52    object::{DeletedObject, ObjectToDelete},
53    range::HTTPRangeSpec,
54};
55use crate::store::init_format::{check_disk_fatal_errs, ec_drives_no_config};
56use crate::{
57    bucket::{lifecycle::bucket_lifecycle_ops::TransitionState, metadata::BucketMetadata},
58    core::sets::Sets,
59    disk::{BUCKET_META_PREFIX, DiskOption, DiskStore, RUSTFS_META_BUCKET},
60    layout::endpoints::EndpointServerPools,
61    object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader, ScannerPublicationCommitScope},
62};
63use futures::future::join_all;
64use http::HeaderMap;
65use lazy_static::lazy_static;
66use rand::RngExt as _;
67use rustfs_config::server_config::Config;
68use rustfs_filemeta::{FileInfo, FileMeta};
69use rustfs_heal_contracts::heal_channel::{HealItemType, HealOpts};
70use rustfs_lock::{LocalClient, LockClient, NamespaceLockWrapper};
71use rustfs_madmin::heal_commands::HealResultItem;
72use rustfs_utils::path::{decode_dir_object, encode_dir_object, path_join_buf};
73use s3s::dto::{BucketVersioningStatus, ObjectLockConfiguration, ObjectLockEnabled, VersioningConfiguration};
74use std::net::SocketAddr;
75use std::process::exit;
76use std::{collections::HashMap, sync::Arc, time::Duration};
77use time::OffsetDateTime;
78use tokio::select;
79use tokio::sync::{Mutex, RwLock};
80use tokio::time::sleep;
81use tokio_util::sync::CancellationToken;
82use tracing::{debug, error, info, instrument, warn};
83use uuid::Uuid;
84
85type ListObjectsV2Info = StorageListObjectsV2Info<ObjectInfo>;
86type ListObjectVersionsInfo = StorageListObjectVersionsInfo<ObjectInfo>;
87type ObjectInfoOrErr = StorageObjectInfoOrErr<ObjectInfo, Error>;
88type WalkOptions = StorageWalkOptions<fn(&FileInfo) -> bool>;
89
90pub const SCANNER_PUBLICATION_LEASE_TTL_MS: u64 = 60_000;
91pub(crate) const BUCKET_DELETE_XLMETA_DIAGNOSTIC_MAX_BYTES: u64 = 1024 * 1024;
92pub(crate) const BUCKET_DELETE_DIAGNOSTIC_MAX_ENTRIES: usize = 4_096;
93pub(crate) const BUCKET_DELETE_DIAGNOSTIC_MAX_ELAPSED: Duration = Duration::from_millis(100);
94
95#[derive(Debug)]
96pub(crate) struct BucketDeleteDiagnosticBudget {
97    deadline: Option<tokio::time::Instant>,
98    max_elapsed: Duration,
99    entries_remaining: usize,
100    #[cfg(test)]
101    first_io_delay: Option<(Duration, Arc<std::sync::atomic::AtomicBool>)>,
102}
103
104impl BucketDeleteDiagnosticBudget {
105    pub(crate) fn new() -> Self {
106        Self::with_limits(BUCKET_DELETE_DIAGNOSTIC_MAX_ENTRIES, BUCKET_DELETE_DIAGNOSTIC_MAX_ELAPSED)
107    }
108
109    fn with_limits(entries: usize, elapsed: Duration) -> Self {
110        Self {
111            deadline: None,
112            max_elapsed: elapsed,
113            entries_remaining: entries,
114            #[cfg(test)]
115            first_io_delay: None,
116        }
117    }
118
119    #[cfg(test)]
120    fn with_first_io_delay(mut self, delay: Duration, started: Arc<std::sync::atomic::AtomicBool>) -> Self {
121        self.first_io_delay = Some((delay, started));
122        self
123    }
124
125    fn deadline(&mut self) -> tokio::time::Instant {
126        let max_elapsed = self.max_elapsed;
127        *self.deadline.get_or_insert_with(|| tokio::time::Instant::now() + max_elapsed)
128    }
129
130    fn claim_entry(&mut self) -> bool {
131        if self.entries_remaining == 0 {
132            return false;
133        }
134        let deadline = self.deadline();
135        if tokio::time::Instant::now() >= deadline {
136            return false;
137        }
138        self.entries_remaining -= 1;
139        true
140    }
141
142    async fn run_io<T, F>(&mut self, future: F) -> std::io::Result<Option<T>>
143    where
144        F: std::future::Future<Output = std::io::Result<T>>,
145    {
146        let deadline = self.deadline();
147        if tokio::time::Instant::now() >= deadline {
148            return Ok(None);
149        }
150        #[cfg(test)]
151        let first_io_delay = self.first_io_delay.take();
152        #[cfg(test)]
153        let timeout_result = tokio::time::timeout_at(deadline, async move {
154            if let Some((delay, started)) = first_io_delay {
155                started.store(true, std::sync::atomic::Ordering::SeqCst);
156                tokio::time::sleep(delay).await;
157            }
158            future.await
159        })
160        .await;
161        #[cfg(not(test))]
162        let timeout_result = tokio::time::timeout_at(deadline, future).await;
163        match timeout_result {
164            Ok(result) => result.map(Some),
165            Err(_) => Ok(None),
166        }
167    }
168}
169
170#[derive(Debug, Default, Clone, PartialEq, Eq)]
171pub(crate) struct BucketMetadataLessResidue {
172    pub(crate) xlmeta_found: bool,
173    pub(crate) xlmeta_blocker: Option<BucketDeleteBlockerKind>,
174    pub(crate) files: usize,
175    pub(crate) uuid_data_dirs: usize,
176    pub(crate) entries_scanned: usize,
177    pub(crate) diagnostic_bytes_read: u64,
178    pub(crate) diagnostic_truncated: bool,
179    pub(crate) sample: Option<String>,
180}
181
182#[derive(Debug, Clone, Copy, PartialEq, Eq)]
183pub(crate) enum BucketDeleteBlockerKind {
184    VisibleVersion,
185    TierFreeVersion,
186    UnknownXlMeta,
187    OrphanDirectory,
188    DiagnosticBudgetExceeded,
189}
190
191impl BucketDeleteBlockerKind {
192    pub(crate) const fn as_str(self) -> &'static str {
193        match self {
194            Self::VisibleVersion => "visible_version",
195            Self::TierFreeVersion => "tier_free_version",
196            Self::UnknownXlMeta => "unknown_xlmeta",
197            Self::OrphanDirectory => "orphan_directory",
198            Self::DiagnosticBudgetExceeded => "diagnostic_budget_exceeded",
199        }
200    }
201
202    /// Whether the blocking residue is something the caller can still see and
203    /// remove through the S3 API.
204    ///
205    /// A live version or a tier free-version is ordinary: the bucket really is
206    /// not empty, the client can list and delete what is left, and the 409 it
207    /// receives is a complete answer.
208    ///
209    /// The remaining kinds are not. They are on-disk state that no S3 request
210    /// can reach: the caller has drained every version the API will show and
211    /// `DeleteBucket` still refuses, with no way to find out why. That is a
212    /// server-side integrity problem, and it is the reason this classification
213    /// exists — see [`bucket_delete_blocker_level`].
214    pub(crate) const fn is_client_visible(self) -> bool {
215        matches!(self, Self::VisibleVersion | Self::TierFreeVersion)
216    }
217}
218
219impl BucketMetadataLessResidue {
220    pub(crate) fn has_residue_without_xlmeta(&self) -> bool {
221        !self.xlmeta_found && (self.files > 0 || self.diagnostic_truncated)
222    }
223
224    pub(crate) fn describe(&self) -> String {
225        let sample = self.sample.as_deref().unwrap_or("<none>");
226        format!(
227            "metadata-less on-disk residue remains after empty-bucket verification: files={}, uuid_data_dirs={}, entries_scanned={}, diagnostic_bytes_read={}, diagnostic_truncated={}, sample={sample}",
228            self.files, self.uuid_data_dirs, self.entries_scanned, self.diagnostic_bytes_read, self.diagnostic_truncated,
229        )
230    }
231}
232
233/// Check if a directory contains any xl.meta files (indicating actual S3 objects)
234/// This is used to determine if a bucket is empty for deletion purposes.
235pub(crate) async fn has_xlmeta_files(path: &std::path::Path) -> std::io::Result<bool> {
236    use crate::disk::STORAGE_FORMAT_FILE;
237    use tokio::fs;
238
239    let mut stack = vec![path.to_path_buf()];
240
241    while let Some(current_path) = stack.pop() {
242        let mut entries = match fs::read_dir(&current_path).await {
243            Ok(entries) => entries,
244            Err(err) if err.kind() == std::io::ErrorKind::NotFound => continue,
245            Err(err) => return Err(err),
246        };
247
248        while let Some(entry) = entries.next_entry().await? {
249            let file_name = entry.file_name();
250            let file_name_str = file_name.to_string_lossy();
251
252            // Check if this is an xl.meta file
253            if file_name_str == STORAGE_FORMAT_FILE {
254                return Ok(true);
255            }
256
257            // If it's a directory, add to stack for further exploration
258            if entry.file_type().await?.is_dir() {
259                stack.push(entry.path());
260            }
261        }
262    }
263
264    Ok(false)
265}
266
267#[cfg(test)]
268pub(crate) async fn scan_metadata_less_residue(path: &std::path::Path) -> std::io::Result<BucketMetadataLessResidue> {
269    let mut budget = BucketDeleteDiagnosticBudget::new();
270    scan_metadata_less_residue_with_budget(path, &mut budget).await
271}
272
273async fn scan_metadata_less_residue_with_budget(
274    path: &std::path::Path,
275    budget: &mut BucketDeleteDiagnosticBudget,
276) -> std::io::Result<BucketMetadataLessResidue> {
277    use crate::disk::STORAGE_FORMAT_FILE;
278    use tokio::fs;
279    use tokio::io::AsyncReadExt as _;
280
281    let mut scan = BucketMetadataLessResidue::default();
282    let mut stack = vec![path.to_path_buf()];
283
284    let mark_budget_exhausted = |scan: &mut BucketMetadataLessResidue| {
285        scan.diagnostic_truncated = true;
286        scan.sample.get_or_insert_with(|| "<diagnostic-budget-exceeded>".to_string());
287    };
288
289    while let Some(current_path) = stack.pop() {
290        let mut entries = match budget.run_io(fs::read_dir(&current_path)).await {
291            Ok(Some(entries)) => entries,
292            Ok(None) => {
293                mark_budget_exhausted(&mut scan);
294                return Ok(scan);
295            }
296            Err(err) if err.kind() == std::io::ErrorKind::NotFound => continue,
297            Err(err) => return Err(err),
298        };
299
300        loop {
301            let entry = match budget.run_io(entries.next_entry()).await? {
302                Some(Some(entry)) => entry,
303                Some(None) => break,
304                None => {
305                    mark_budget_exhausted(&mut scan);
306                    return Ok(scan);
307                }
308            };
309            if !budget.claim_entry() {
310                mark_budget_exhausted(&mut scan);
311                return Ok(scan);
312            }
313            scan.entries_scanned = scan.entries_scanned.saturating_add(1);
314            let Some(file_type) = budget.run_io(entry.file_type()).await? else {
315                mark_budget_exhausted(&mut scan);
316                return Ok(scan);
317            };
318            let file_name = entry.file_name();
319            let file_name_str = file_name.to_string_lossy();
320
321            if file_name_str == STORAGE_FORMAT_FILE {
322                scan.xlmeta_found = true;
323                if scan.xlmeta_blocker.is_none() {
324                    let entry_path = entry.path();
325                    let Some(metadata) = budget.run_io(fs::metadata(&entry_path)).await? else {
326                        mark_budget_exhausted(&mut scan);
327                        scan.xlmeta_blocker = Some(BucketDeleteBlockerKind::DiagnosticBudgetExceeded);
328                        return Ok(scan);
329                    };
330                    scan.xlmeta_blocker = Some(if metadata.len() > BUCKET_DELETE_XLMETA_DIAGNOSTIC_MAX_BYTES {
331                        BucketDeleteBlockerKind::UnknownXlMeta
332                    } else {
333                        match budget.run_io(fs::File::open(&entry_path)).await {
334                            Ok(Some(file)) => {
335                                let mut data = Vec::new();
336                                let read = budget
337                                    .run_io(file.take(BUCKET_DELETE_XLMETA_DIAGNOSTIC_MAX_BYTES).read_to_end(&mut data))
338                                    .await;
339                                scan.diagnostic_bytes_read = data.len() as u64;
340                                match read {
341                                    Ok(Some(_)) => match FileMeta::load(&data) {
342                                        Ok(meta)
343                                            if !meta.versions.is_empty()
344                                                && meta.versions.iter().all(|version| version.header.free_version()) =>
345                                        {
346                                            BucketDeleteBlockerKind::TierFreeVersion
347                                        }
348                                        Ok(meta) if !meta.versions.is_empty() => BucketDeleteBlockerKind::VisibleVersion,
349                                        Ok(_) | Err(_) => BucketDeleteBlockerKind::UnknownXlMeta,
350                                    },
351                                    Ok(None) => {
352                                        mark_budget_exhausted(&mut scan);
353                                        BucketDeleteBlockerKind::DiagnosticBudgetExceeded
354                                    }
355                                    Err(_) => BucketDeleteBlockerKind::UnknownXlMeta,
356                                }
357                            }
358                            Ok(None) => {
359                                mark_budget_exhausted(&mut scan);
360                                BucketDeleteBlockerKind::DiagnosticBudgetExceeded
361                            }
362                            Err(_) => BucketDeleteBlockerKind::UnknownXlMeta,
363                        }
364                    });
365                    let sample = entry_path
366                        .strip_prefix(path)
367                        .unwrap_or(entry_path.as_path())
368                        .to_string_lossy()
369                        .replace(std::path::MAIN_SEPARATOR, "/");
370                    scan.sample = Some(sample);
371                }
372                return Ok(scan);
373            }
374
375            if file_type.is_dir() {
376                if Uuid::parse_str(&file_name_str).is_ok() {
377                    scan.uuid_data_dirs = scan.uuid_data_dirs.saturating_add(1);
378                }
379                stack.push(entry.path());
380            } else {
381                scan.files = scan.files.saturating_add(1);
382                if scan.sample.is_none() {
383                    let entry_path = entry.path();
384                    let sample = entry_path
385                        .strip_prefix(path)
386                        .unwrap_or(entry_path.as_path())
387                        .to_string_lossy()
388                        .replace(std::path::MAIN_SEPARATOR, "/");
389                    scan.sample = Some(sample);
390                }
391            }
392        }
393    }
394
395    Ok(scan)
396}
397
398async fn enqueue_transition_after_write(
399    store: &ECStore,
400    result: Result<ObjectInfo>,
401    src: LcEventSrc,
402    opts: &ObjectOptions,
403) -> Result<ObjectInfo> {
404    match result {
405        Ok(oi) => {
406            // A retried completion did not publish a new object. Immediate
407            // transition/expiry already ran for the original commit; the scanner
408            // still sees the object if that commit never reached this hook.
409            if !oi.multipart_completion_replayed && should_enqueue_transition_immediately(&oi) {
410                enqueue_transition_immediate(&oi, src.clone()).await;
411                if let Ok(api) = metadata_sys::object_store_in(&store.ctx).await {
412                    enqueue_immediate_expiry(api, &oi, src, opts).await;
413                }
414            }
415            Ok(oi)
416        }
417        Err(err) => Err(err),
418    }
419}
420
421fn should_enqueue_transition_immediately(oi: &ObjectInfo) -> bool {
422    !is_meta_bucketname(&oi.bucket)
423}
424
425const MAX_UPLOADS_LIST: usize = 10000;
426
427mod bucket;
428mod bucket_fence;
429pub(crate) use bucket::await_bucket_namespace_operation;
430pub use bucket_fence::BucketIncarnationFenceGuard;
431mod heal;
432pub use heal::{HealObjectAbsenceProof, HealObjectStorageResult};
433pub(crate) use heal::{bucket_heal_scope, bucket_heal_scope_for_object};
434mod heal_walk;
435pub use heal_walk::HealWalkVersion;
436mod init;
437pub(crate) mod init_format;
438pub(crate) mod list_objects;
439mod multipart;
440mod object;
441#[cfg(feature = "test-util")]
442pub use object::DeleteAfterObjectLockSnapshotBarrier;
443pub(crate) use object::{
444    DecommissionFixedReadAnchor, ObjectLockDiagGuard, RemoteTuplePublicationCommitGuard, RemoteTuplePublicationFence,
445    SourceCleanupMutationFence, tiered_data_movement_source_matches,
446};
447pub use object::{
448    PrepareSelectObjectSnapshotError, PreparedGetObjectReader, SelectObjectSnapshot, SelectObjectSnapshotReadError,
449    SnapshotConsistencyError,
450};
451mod peer;
452mod rebalance;
453pub(crate) mod utils;
454
455use peer::init_local_peer;
456pub use peer::{
457    BootstrapLocalTarget, all_local_disk, all_local_disk_path, find_local_disk_by_ref, get_disk_infos, init_local_disks,
458    init_local_disks_with_instance_ctx, init_lock_clients, prewarm_local_disk_id_map,
459    prewarm_local_disk_id_map_with_instance_ctx,
460};
461
462pub struct ECStore {
463    pub id: Uuid,
464    // pub disks: Vec<DiskStore>,
465    pub disk_map: HashMap<usize, Vec<Option<DiskStore>>>,
466    pub pools: Vec<Arc<Sets>>,
467    pub peer_sys: S3PeerSys,
468    // pub local_disks: Vec<DiskStore>,
469    pub pool_meta: RwLock<PoolMeta>,
470    pub rebalance_meta: RwLock<Option<RebalanceMeta>>,
471    pub decommission_cancelers: RwLock<Vec<Option<DecommissionCanceler>>>,
472    /// Serializes rebalance/decommission start transitions.
473    ///
474    /// Lock order: acquire `start_gate` before `pool_meta`, `rebalance_meta`,
475    /// or `decommission_cancelers`. The guarded sections may perform bounded
476    /// async metadata work so check/init/start cannot race across operations.
477    pub(crate) start_gate: Mutex<()>,
478    /// Serializes full-document pool metadata saves and retains a fail-closed
479    /// write block after startup observes an unreadable replica.
480    ///
481    /// Lock order: acquire `pool_meta_save_gate`, then the distributed
482    /// `pool.bin` fence, then clone `pool_meta` under a short read lock.
483    pub(crate) pool_meta_save_gate: Mutex<PoolMetaWriteState>,
484    /// Per-instance runtime state (Phase 5, backlog#939).
485    ///
486    /// Carries this instance's identity/runtime out of the process globals so
487    /// multiple instances can coexist without cross-contamination. `new`
488    /// adopts the process bootstrap context (never mints a fresh one) so that
489    /// startup writes and post-construction reads share one cell — single
490    /// instance behavior is unchanged.
491    pub(crate) ctx: Arc<InstanceContext>,
492    /// Memoizes bucket-incarnation validation under continuous lifecycle
493    /// read-lock coverage (see [`bucket_fence`]).
494    pub(crate) bucket_fence_registry: Arc<bucket_fence::BucketFenceRegistry>,
495}
496
497const METRIC_SCANNER_DATA_MOVEMENT_PAUSED: &str = "rustfs_scanner_data_movement_paused";
498const METRIC_SCANNER_DATA_MOVEMENT_PAUSE_DURATION_SECONDS: &str = "rustfs_scanner_data_movement_pause_duration_seconds";
499const METRIC_SCANNER_DATA_MOVEMENT_BACKLOG_WORK_ITEMS: &str = "rustfs_scanner_data_movement_backlog_work_items";
500const SCANNER_DATA_MOVEMENT_PAUSE_POLICY: &str = "global_pause";
501
502#[derive(Clone, Copy, Debug, PartialEq, Eq, serde::Serialize)]
503#[serde(rename_all = "snake_case")]
504pub enum ScannerDataMovementPauseReason {
505    OperationEpochExhausted,
506    MovementGenerationExhausted,
507    DecommissionActive,
508    DecommissionFailed,
509    DecommissionCanceled,
510    RebalanceActive,
511}
512
513#[derive(Clone, Debug, PartialEq, Eq, serde::Serialize)]
514pub struct ScannerDataMovementPauseStatus {
515    pub paused: bool,
516    pub policy: &'static str,
517    pub reasons: Vec<ScannerDataMovementPauseReason>,
518    pub started_at_unix_secs: u64,
519    pub duration_seconds: u64,
520    pub operation_epoch: u64,
521    pub movement_generation: u64,
522    pub movement_backlog_work_items: u64,
523    pub movement_backlog_estimated: bool,
524}
525
526impl Default for ScannerDataMovementPauseStatus {
527    fn default() -> Self {
528        Self {
529            paused: false,
530            policy: SCANNER_DATA_MOVEMENT_PAUSE_POLICY,
531            reasons: Vec::new(),
532            started_at_unix_secs: 0,
533            duration_seconds: 0,
534            operation_epoch: 0,
535            movement_generation: 0,
536            movement_backlog_work_items: 0,
537            movement_backlog_estimated: false,
538        }
539    }
540}
541
542#[derive(Clone, Debug, PartialEq, Eq, serde::Serialize)]
543pub struct PoolMetaWriteGateStatus {
544    pub writes_ready: bool,
545    pub check_timed_out: bool,
546    pub write_blocked: bool,
547    pub transaction_aborted: bool,
548    pub pool_meta_absent: bool,
549    pub identity_initialized: Option<bool>,
550    pub identity_needs_repair: bool,
551    pub cluster_epoch: Option<u64>,
552    pub reason: Option<&'static str>,
553    pub phase: Option<&'static str>,
554    pub since_unix_secs: Option<i64>,
555}
556
557impl Default for PoolMetaWriteGateStatus {
558    fn default() -> Self {
559        Self {
560            writes_ready: true,
561            check_timed_out: false,
562            write_blocked: false,
563            transaction_aborted: false,
564            pool_meta_absent: false,
565            identity_initialized: None,
566            identity_needs_repair: false,
567            cluster_epoch: None,
568            reason: None,
569            phase: None,
570            since_unix_secs: None,
571        }
572    }
573}
574
575fn offset_unix_seconds(value: OffsetDateTime) -> u64 {
576    u64::try_from(value.unix_timestamp()).unwrap_or(0)
577}
578
579fn earliest_timestamp(current: Option<OffsetDateTime>, candidate: Option<OffsetDateTime>) -> Option<OffsetDateTime> {
580    match (current, candidate) {
581        (Some(current), Some(candidate)) => Some(current.min(candidate)),
582        (Some(current), None) => Some(current),
583        (None, candidate) => candidate,
584    }
585}
586
587fn usize_to_u64(value: usize) -> u64 {
588    u64::try_from(value).unwrap_or(u64::MAX)
589}
590
591fn metric_u64(value: u64) -> f64 {
592    f64::from(u32::try_from(value).unwrap_or(u32::MAX))
593}
594
595pub(crate) fn scanner_data_movement_timestamp_generation(value: OffsetDateTime) -> u64 {
596    let timestamp = value.unix_timestamp_nanos();
597    if timestamp <= 0 {
598        0
599    } else {
600        u64::try_from(timestamp).unwrap_or(u64::MAX)
601    }
602}
603
604fn valid_scanner_data_movement_timestamp_generation(value: OffsetDateTime) -> Option<u64> {
605    let generation = scanner_data_movement_timestamp_generation(value);
606    (generation != 0 && generation != u64::MAX).then_some(generation)
607}
608
609fn durable_scanner_data_movement_generation(pool_meta: &PoolMeta, rebalance_meta: Option<&RebalanceMeta>) -> u64 {
610    let mut generation = 0;
611    for pool in pool_meta.pools.iter().filter(|pool| pool.decommission.is_some()) {
612        let Some(pool_generation) = valid_scanner_data_movement_timestamp_generation(pool.last_update) else {
613            return u64::MAX;
614        };
615        generation = generation.max(pool_generation);
616    }
617
618    for movement_timestamp in rebalance_meta.into_iter().flat_map(|meta| {
619        meta.stopped_at.into_iter().chain(
620            meta.pool_stats
621                .iter()
622                .flat_map(|pool| [pool.info.start_time, pool.info.end_time])
623                .flatten(),
624        )
625    }) {
626        let Some(rebalance_generation) = valid_scanner_data_movement_timestamp_generation(movement_timestamp) else {
627            return u64::MAX;
628        };
629        generation = generation.max(rebalance_generation);
630    }
631
632    if generation == 0
633        && rebalance_meta.is_some_and(|meta| !meta.id.is_empty() || !meta.pool_stats.is_empty() || meta.stopped_at.is_some())
634    {
635        u64::MAX
636    } else {
637        generation
638    }
639}
640
641#[derive(Clone, Copy)]
642struct ScannerDataMovementSequenceState {
643    operation_epoch: u64,
644    operation_epoch_exhausted: bool,
645    movement_generation: u64,
646    movement_generation_exhausted: bool,
647}
648
649fn resolve_scanner_data_movement_pause_status(
650    pool_meta: &PoolMeta,
651    rebalance_meta: Option<&RebalanceMeta>,
652    decommission_worker_active: bool,
653    sequence: ScannerDataMovementSequenceState,
654    now: OffsetDateTime,
655) -> ScannerDataMovementPauseStatus {
656    let mut decommission_active = decommission_worker_active;
657    let mut decommission_failed = false;
658    let mut decommission_canceled = false;
659    let mut rebalance_active = false;
660    let mut started_at = None;
661    let mut movement_backlog_work_items = 0_u64;
662
663    for pool in &pool_meta.pools {
664        let Some(info) = pool.decommission.as_ref() else {
665            continue;
666        };
667        let active = info.has_decommission_state() && !info.complete && !info.failed && !info.canceled;
668        let failed = !info.queued && info.failed;
669        let canceled = !info.queued && info.canceled;
670        if !(active || failed || canceled) {
671            continue;
672        }
673
674        decommission_active |= active;
675        decommission_failed |= failed;
676        decommission_canceled |= canceled;
677        started_at = earliest_timestamp(started_at, info.start_time.or(Some(pool.last_update)));
678        let queued = usize_to_u64(info.queued_buckets.len());
679        let current_bucket = if info.bucket.is_empty() { 0 } else { 1 };
680        movement_backlog_work_items = movement_backlog_work_items.saturating_add(queued.max(current_bucket));
681    }
682
683    if let Some(rebalance_meta) = rebalance_meta {
684        for pool in &rebalance_meta.pool_stats {
685            let active = (pool.participating && pool.info.status == RebalStatus::Started) || pool.info.stopping;
686            if !active {
687                continue;
688            }
689            rebalance_active = true;
690            started_at = earliest_timestamp(started_at, pool.info.start_time);
691            movement_backlog_work_items = movement_backlog_work_items.saturating_add(usize_to_u64(pool.buckets.len()));
692        }
693    }
694
695    let mut reasons = Vec::with_capacity(6);
696    if sequence.operation_epoch_exhausted {
697        reasons.push(ScannerDataMovementPauseReason::OperationEpochExhausted);
698    }
699    if sequence.movement_generation_exhausted {
700        reasons.push(ScannerDataMovementPauseReason::MovementGenerationExhausted);
701    }
702    if decommission_active {
703        reasons.push(ScannerDataMovementPauseReason::DecommissionActive);
704    }
705    if decommission_failed {
706        reasons.push(ScannerDataMovementPauseReason::DecommissionFailed);
707    }
708    if decommission_canceled {
709        reasons.push(ScannerDataMovementPauseReason::DecommissionCanceled);
710    }
711    if rebalance_active {
712        reasons.push(ScannerDataMovementPauseReason::RebalanceActive);
713    }
714    let started_at_unix_secs = started_at.map(offset_unix_seconds).unwrap_or(0);
715    let duration_seconds = started_at
716        .and_then(|started_at| u64::try_from((now - started_at).whole_seconds()).ok())
717        .unwrap_or(0);
718    let paused = !reasons.is_empty();
719
720    ScannerDataMovementPauseStatus {
721        paused,
722        policy: SCANNER_DATA_MOVEMENT_PAUSE_POLICY,
723        reasons,
724        started_at_unix_secs,
725        duration_seconds,
726        operation_epoch: sequence.operation_epoch,
727        movement_generation: sequence.movement_generation,
728        movement_backlog_work_items,
729        movement_backlog_estimated: paused,
730    }
731}
732
733fn record_scanner_data_movement_pause_status(status: &ScannerDataMovementPauseStatus) {
734    metrics::gauge!(METRIC_SCANNER_DATA_MOVEMENT_PAUSED).set(if status.paused { 1.0 } else { 0.0 });
735    metrics::gauge!(METRIC_SCANNER_DATA_MOVEMENT_PAUSE_DURATION_SECONDS).set(metric_u64(status.duration_seconds));
736    metrics::gauge!(METRIC_SCANNER_DATA_MOVEMENT_BACKLOG_WORK_ITEMS).set(metric_u64(status.movement_backlog_work_items));
737}
738
739impl std::fmt::Debug for ECStore {
740    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
741        let disk_slot_count: usize = self.disk_map.values().map(Vec::len).sum();
742
743        f.debug_struct("ECStore")
744            .field("id", &self.id)
745            .field("disk_map_pool_count", &self.disk_map.len())
746            .field("disk_slot_count", &disk_slot_count)
747            .field("pool_count", &self.pools.len())
748            .finish_non_exhaustive()
749    }
750}
751
752/// Phase 2: Accessor methods for config globals
753/// These delegate to the process-global statics. No local state — the globals
754/// remain the single source of truth until the migration is complete.
755impl ECStore {
756    /// Every erasure set across all pools, pool-major order.
757    ///
758    /// Read-only queries that must consult each set's own copy of a
759    /// per-bucket object (e.g. the scanner's `.usage-cache.bin`) iterate
760    /// this instead of the hash-routed store path, which would always land
761    /// on one set (rustfs/backlog#1872).
762    pub fn all_set_disks(&self) -> Vec<Arc<crate::set_disk::SetDisks>> {
763        self.pools.iter().flat_map(|pool| pool.disk_set.iter().cloned()).collect()
764    }
765
766    /// Erasure sets that may receive scanner pause-backlog replicas.
767    ///
768    /// An actively decommissioning or already decommissioned source pool is
769    /// excluded so an operational record acknowledged during movement always
770    /// has a copy on storage that remains in the cluster. The record is kept
771    /// separate from pool and rebalance metadata.
772    pub async fn scanner_pause_backlog_writable_set_disks(&self) -> Vec<Arc<crate::set_disk::SetDisks>> {
773        let pool_meta = self.pool_meta.read().await;
774        self.pools
775            .iter()
776            .enumerate()
777            .filter(|(pool_index, _)| pool_meta.scanner_pause_backlog_pool_writable(*pool_index))
778            .flat_map(|(_, pool)| pool.disk_set.iter().cloned())
779            .collect()
780    }
781
782    /// Get server configuration (delegates to global)
783    pub fn get_server_config(&self) -> Option<Config> {
784        runtime_sources::server_config()
785    }
786
787    /// Set server configuration (delegates to global)
788    pub fn set_server_config(&self, cfg: Config) {
789        runtime_sources::set_server_config(cfg);
790    }
791
792    /// Get storage class configuration (delegates to global)
793    pub fn get_storage_class(&self) -> Option<crate::config::storageclass::Config> {
794        runtime_sources::storage_class_config()
795    }
796
797    /// Set storage class configuration (delegates to global)
798    pub fn set_storage_class(&self, cfg: crate::config::storageclass::Config) {
799        runtime_sources::set_storage_class_config(cfg);
800    }
801}
802
803/// Phase 3: Accessor methods for service globals
804/// These provide a unified API through ECStore for accessing cross-cutting
805/// service singletons. The globals remain the source of truth.
806impl ECStore {
807    /// Get the notification system
808    pub fn notification_system(&self) -> Option<std::sync::Arc<crate::services::notification_sys::NotificationSys>> {
809        runtime_sources::notification_sys()
810    }
811
812    /// Get the bucket metadata system
813    pub fn bucket_metadata_sys(&self) -> Option<Arc<tokio::sync::RwLock<crate::bucket::metadata_sys::BucketMetadataSys>>> {
814        runtime_sources::bucket_metadata_sys()
815    }
816
817    /// Get the global endpoints
818    pub fn endpoints(&self) -> EndpointServerPools {
819        runtime_sources::endpoint_pools().unwrap_or_else(|| Vec::new().into())
820    }
821
822    /// Get this store instance's endpoint topology without consulting process globals.
823    pub fn instance_endpoints(&self) -> Option<EndpointServerPools> {
824        self.ctx.endpoints()
825    }
826
827    /// Get the global region
828    pub fn region(&self) -> Option<s3s::region::Region> {
829        runtime_sources::region()
830    }
831
832    /// Get the tier config manager
833    pub fn tier_config_mgr(&self) -> Arc<tokio::sync::RwLock<crate::services::tier::tier::TierConfigMgr>> {
834        self.ctx.tier_config_mgr()
835    }
836
837    /// Get the server configuration
838    pub fn server_config(&self) -> Option<Config> {
839        runtime_sources::server_config()
840    }
841
842    /// Get the storage class configuration
843    pub fn storage_class(&self) -> Option<crate::config::storageclass::Config> {
844        runtime_sources::storage_class_config()
845    }
846}
847
848/// Phase 4: Server address accessors
849/// These provide a unified API through ECStore for accessing server-level
850/// configuration globals. The globals remain the source of truth.
851impl ECStore {
852    /// Get the server port
853    pub fn port(&self) -> u16 {
854        runtime_sources::rustfs_port()
855    }
856
857    /// Get the server host
858    pub async fn host(&self) -> String {
859        runtime_sources::rustfs_host().await
860    }
861
862    /// Get the server address (host:port)
863    pub async fn addr(&self) -> String {
864        runtime_sources::rustfs_addr().await
865    }
866}
867
868/// Phase 5: Per-instance erasure setup accessors (backlog#939)
869///
870/// These read this instance's own [`InstanceContext`] rather than a process
871/// global, so two instances carrying different contexts stay isolated. The
872/// legacy free-function facade (`runtime::global::is_erasure` etc.) forwards to
873/// the current instance's context, preserving single-instance behavior.
874impl ECStore {
875    /// Whether this instance uses erasure coding (single-node or distributed).
876    pub async fn setup_is_erasure(&self) -> bool {
877        self.ctx.is_erasure().await
878    }
879
880    /// Whether this instance uses distributed erasure coding.
881    pub async fn setup_is_dist_erasure(&self) -> bool {
882        self.ctx.is_dist_erasure().await
883    }
884
885    /// Whether this instance uses single-drive erasure coding.
886    pub async fn setup_is_erasure_sd(&self) -> bool {
887        self.ctx.is_erasure_sd().await
888    }
889
890    pub fn scanner_namespace_mutation_generation(&self) -> u64 {
891        list_objects::scanner_namespace_mutation_generation().saturating_add(self.ctx.namespace_commit_generation())
892    }
893
894    pub async fn scanner_data_movement_active(&self) -> bool {
895        let (decommission, rebalance) = tokio::join!(self.is_decommission_running(), self.is_rebalance_started());
896        decommission || rebalance
897    }
898
899    /// Return the storage-owned movement state and generation as one
900    /// authenticated activity snapshot. The read lock is acquired before
901    /// the state locks (cancelers, pool metadata, then rebalance metadata),
902    /// matching the transition writer order and preventing a terminal state
903    /// from being reported with the preceding generation.
904    pub async fn scanner_data_movement_activity(&self) -> (bool, bool, u64) {
905        let operation_gate = self.ctx.data_movement_operation_gate();
906        let _operation_guard = operation_gate.read_owned().await;
907        let (active, blocked) = self.scanner_data_movement_snapshot_locked().await;
908        let blocked =
909            blocked || self.ctx.data_movement_operation_epoch_exhausted() || self.ctx.data_movement_generation_exhausted();
910        self.ctx.set_scanner_publication_state(blocked);
911        (active, blocked, self.ctx.data_movement_generation())
912    }
913
914    pub fn scanner_data_movement_generation(&self) -> u64 {
915        self.ctx.data_movement_generation()
916    }
917
918    pub fn scanner_data_movement_generation_exhausted(&self) -> bool {
919        self.ctx.data_movement_generation_exhausted()
920    }
921
922    pub fn scanner_data_movement_changed(&self) -> std::sync::Arc<tokio::sync::Notify> {
923        self.ctx.data_movement_generation_notify()
924    }
925
926    /// Returns whether scanner metadata may still be hidden by a local
927    /// data-movement state. Terminal failed/canceled decommission entries
928    /// remain suspended until an operator clears or retries them, so they are
929    /// a publication barrier even after the worker has stopped. Active PUT
930    /// rename fanouts also defer publication, including post-ACK tails.
931    pub async fn scanner_data_usage_publication_blocked(&self) -> bool {
932        let operation_gate = self.ctx.data_movement_operation_gate();
933        let _operation_guard = operation_gate.read_owned().await;
934        self.scanner_data_usage_publication_snapshot_blocked().await || self.ctx.namespace_commits_pending()
935    }
936
937    pub async fn scanner_data_movement_pause_status(&self) -> ScannerDataMovementPauseStatus {
938        let operation_gate = self.ctx.data_movement_operation_gate();
939        let _operation_guard = operation_gate.read_owned().await;
940        self.scanner_data_movement_pause_snapshot().await
941    }
942
943    async fn scanner_data_usage_publication_snapshot_blocked(&self) -> bool {
944        self.scanner_data_movement_pause_snapshot().await.paused
945    }
946
947    async fn scanner_data_movement_snapshot_locked(&self) -> (bool, bool) {
948        let decommission_cancelers = self.decommission_cancelers.read().await;
949        let decommission_active = decommission_cancelers
950            .iter()
951            .any(|canceler| canceler.as_ref().is_some_and(DecommissionCanceler::is_active));
952        let pool_meta = self.pool_meta.read().await;
953        let decommission_active = decommission_active
954            || pool_meta.pools.iter().any(|pool| {
955                pool.decommission
956                    .as_ref()
957                    .is_some_and(|info| info.has_decommission_state() && !info.complete && !info.failed && !info.canceled)
958            });
959        let decommission_terminal = pool_meta.pools.iter().any(|pool| {
960            pool.decommission
961                .as_ref()
962                .is_some_and(|info| !info.queued && (info.failed || info.canceled))
963        });
964        let rebalance_meta = self.rebalance_meta.read().await;
965        let rebalance_active = rebalance_meta
966            .as_ref()
967            .is_some_and(is_rebalance_conflicting_with_decommission);
968        self.ctx
969            .observe_durable_data_movement_generation(durable_scanner_data_movement_generation(
970                &pool_meta,
971                rebalance_meta.as_ref(),
972            ));
973
974        let blocked = decommission_active || decommission_terminal || rebalance_active;
975        (decommission_active || rebalance_active, blocked)
976    }
977
978    async fn scanner_data_movement_pause_snapshot(&self) -> ScannerDataMovementPauseStatus {
979        let decommission_active = {
980            let decommission_cancelers = self.decommission_cancelers.read().await;
981            decommission_cancelers
982                .iter()
983                .any(|canceler| canceler.as_ref().is_some_and(DecommissionCanceler::is_active))
984        };
985        let pool_meta = self.pool_meta.read().await.clone();
986        let rebalance_meta = self.rebalance_meta.read().await.clone();
987        self.ctx
988            .observe_durable_data_movement_generation(durable_scanner_data_movement_generation(
989                &pool_meta,
990                rebalance_meta.as_ref(),
991            ));
992        let status = resolve_scanner_data_movement_pause_status(
993            &pool_meta,
994            rebalance_meta.as_ref(),
995            decommission_active,
996            ScannerDataMovementSequenceState {
997                operation_epoch: self.ctx.data_movement_operation_epoch(),
998                operation_epoch_exhausted: self.ctx.data_movement_operation_epoch_exhausted(),
999                movement_generation: self.ctx.data_movement_generation(),
1000                movement_generation_exhausted: self.ctx.data_movement_generation_exhausted(),
1001            },
1002            OffsetDateTime::now_utc(),
1003        );
1004        self.ctx.set_scanner_publication_state(status.paused);
1005        record_scanner_data_movement_pause_status(&status);
1006        status
1007    }
1008
1009    #[cfg(test)]
1010    pub(crate) async fn scanner_data_movement_pause_snapshot_for_test(&self) -> ScannerDataMovementPauseStatus {
1011        self.scanner_data_movement_pause_snapshot().await
1012    }
1013
1014    /// Admit one short data-usage publication commit under the same
1015    /// per-instance gate used by decommission side effects and transitions.
1016    /// The epoch is sampled while the read guard is held, so a transition
1017    /// cannot cross this admission without waiting for the commit to finish.
1018    pub async fn scanner_data_usage_publication_read_guard(&self) -> (tokio::sync::OwnedRwLockReadGuard<()>, u64) {
1019        let operation_gate = self.ctx.data_movement_operation_gate();
1020        let operation_guard = operation_gate.read_owned().await;
1021        let epoch = self.ctx.data_movement_operation_epoch();
1022        (operation_guard, epoch)
1023    }
1024
1025    /// Acquire the movement gate and inspect the movement owner once. The
1026    /// state inspection is performed after acquiring the read guard so a
1027    /// transition cannot update its durable state between the check and the
1028    /// publication commit.
1029    pub async fn scanner_data_usage_publication_admission_guard(&self) -> Option<(tokio::sync::OwnedRwLockReadGuard<()>, u64)> {
1030        let operation_gate = self.ctx.data_movement_operation_gate();
1031        let operation_guard = operation_gate.read_owned().await;
1032        if self.ctx.data_movement_operation_epoch_exhausted() || self.ctx.data_movement_generation_exhausted() {
1033            return None;
1034        }
1035        if self.scanner_data_usage_publication_snapshot_blocked().await {
1036            return None;
1037        }
1038
1039        Some((operation_guard, self.ctx.data_movement_operation_epoch()))
1040    }
1041
1042    /// Acquire a storage-owned scanner publication scope. Unlike the legacy
1043    /// admission helper, the movement permit is owned by the returned scope
1044    /// and therefore survives cancellation of the scanner coordinator while
1045    /// the actual metadata mutation drains.
1046    pub async fn scanner_data_usage_publication_commit_scope(
1047        &self,
1048        expected_movement_epoch: u64,
1049        safe_deadline: tokio::time::Instant,
1050        remote_lease_tokens: Vec<Uuid>,
1051    ) -> Option<ScannerPublicationCommitScope> {
1052        let (movement_permit, epoch) = self.scanner_data_usage_publication_admission_guard().await?;
1053        if epoch != expected_movement_epoch {
1054            return None;
1055        }
1056        Some(ScannerPublicationCommitScope::new_storage_owned(
1057            epoch,
1058            safe_deadline,
1059            remote_lease_tokens,
1060            movement_permit,
1061        ))
1062    }
1063
1064    pub async fn scanner_data_usage_publication_commit_scope_with_release_flag(
1065        &self,
1066        expected_movement_epoch: u64,
1067        safe_deadline: tokio::time::Instant,
1068        remote_lease_tokens: Vec<Uuid>,
1069        lease_release_safe: Arc<std::sync::atomic::AtomicBool>,
1070    ) -> Option<ScannerPublicationCommitScope> {
1071        let (movement_permit, epoch) = self.scanner_data_usage_publication_admission_guard().await?;
1072        if epoch != expected_movement_epoch {
1073            return None;
1074        }
1075        Some(ScannerPublicationCommitScope::new_storage_owned_with_release_flag(
1076            epoch,
1077            safe_deadline,
1078            remote_lease_tokens,
1079            movement_permit,
1080            lease_release_safe,
1081        ))
1082    }
1083
1084    /// Capture the current publication epoch without holding the movement
1085    /// gate across backend I/O. Callers must re-admit the same epoch before a
1086    /// mutation commits.
1087    pub(crate) async fn scanner_data_usage_publication_epoch(&self) -> Option<u64> {
1088        let (operation_guard, epoch) = self.scanner_data_usage_publication_admission_guard().await?;
1089        drop(operation_guard);
1090        Some(epoch)
1091    }
1092
1093    /// Acquire a storage-owned read admission for a coordinator's final
1094    /// scanner publication.  The guard remains in the context's lease table,
1095    /// so a local movement writer cannot pass the peer while its authoritative
1096    /// PUT is in flight.
1097    pub async fn acquire_scanner_publication_lease(
1098        &self,
1099        expected_generation: u64,
1100        ttl: std::time::Duration,
1101    ) -> Result<(Uuid, u64)> {
1102        if ttl != crate::runtime::instance::SCANNER_PUBLICATION_LEASE_TTL {
1103            return Err(Error::other("scanner publication lease TTL is not supported"));
1104        }
1105
1106        // Bind the original activity generation across the asynchronous checks;
1107        // a completed namespace commit must never refresh an existing proof.
1108        let namespace_generation = self.scanner_namespace_mutation_generation();
1109        let operation_gate = self.ctx.data_movement_operation_gate();
1110        let operation_guard = operation_gate.read_owned().await;
1111        if self.ctx.data_movement_generation_exhausted()
1112            || self.ctx.data_movement_operation_epoch_exhausted()
1113            || self.ctx.data_movement_generation() != expected_generation
1114            || namespace_generation == u64::MAX
1115        {
1116            return Err(Error::other("scanner publication lease generation is stale"));
1117        }
1118        if self.scanner_data_movement_snapshot_locked().await.1 || self.ctx.namespace_commits_pending() {
1119            return Err(Error::other("scanner publication lease is blocked by data movement"));
1120        }
1121
1122        if self.scanner_namespace_mutation_generation() != namespace_generation {
1123            return Err(Error::other("scanner publication lease generation is stale"));
1124        }
1125
1126        let token = Uuid::new_v4();
1127        let expires_at = tokio::time::Instant::now() + ttl;
1128        if !self
1129            .ctx
1130            .install_scanner_publication_lease(token, expires_at, expected_generation, namespace_generation, operation_guard)
1131            .await
1132        {
1133            return Err(Error::other("scanner publication lease capacity is exhausted"));
1134        }
1135
1136        let context = Arc::clone(&self.ctx);
1137        tokio::spawn(async move {
1138            tokio::time::sleep_until(expires_at).await;
1139            context.expire_scanner_publication_lease(token, expires_at).await;
1140        });
1141        Ok((token, expected_generation))
1142    }
1143
1144    pub async fn release_scanner_publication_lease(&self, token: Uuid) -> bool {
1145        self.ctx.remove_scanner_publication_lease(token).await
1146    }
1147
1148    /// Revalidate a previously acquired remote publication lease immediately
1149    /// before the coordinator's final metadata write.  The operation read
1150    /// guard makes the movement snapshot and token lookup one storage-owned
1151    /// admission; a restarted context has no old token and therefore fails
1152    /// closed even if its generation counter has returned to zero.
1153    pub async fn validate_scanner_publication_lease(&self, token: Uuid, expected_generation: u64) -> Result<()> {
1154        let _operation_guard = self.acquire_scanner_publication_lease_guard(token).await?;
1155        if self.ctx.data_movement_generation_exhausted()
1156            || self.ctx.data_movement_operation_epoch_exhausted()
1157            || self.ctx.data_movement_generation() != expected_generation
1158        {
1159            return Err(Error::other("scanner publication lease generation is stale"));
1160        }
1161        if self.scanner_data_movement_snapshot_locked().await.1 || self.ctx.namespace_commits_pending() {
1162            return Err(Error::other("scanner publication lease is blocked by data movement"));
1163        }
1164        let Some((lease_generation, lease_namespace_generation)) = self.ctx.scanner_publication_lease_generations(token).await
1165        else {
1166            return Err(Error::other("scanner publication lease is unknown or expired"));
1167        };
1168        let namespace_generation = self.scanner_namespace_mutation_generation();
1169        if lease_generation != self.ctx.data_movement_generation()
1170            || lease_namespace_generation != namespace_generation
1171            || namespace_generation == u64::MAX
1172        {
1173            return Err(Error::other("scanner publication lease generation is stale"));
1174        }
1175        Ok(())
1176    }
1177
1178    /// Acquire the target-side read guard bound to a previously granted lease.
1179    /// The guard is returned to the RPC handler and must remain alive through
1180    /// the complete rename/write operation.  A lease token is process-owned;
1181    /// restart, expiry, generation changes, or blocked movement all reject it
1182    /// before the target disk is touched.
1183    pub async fn acquire_scanner_publication_lease_guard(&self, token: Uuid) -> Result<tokio::sync::OwnedRwLockReadGuard<()>> {
1184        let operation_gate = self.ctx.data_movement_operation_gate();
1185        let operation_guard = operation_gate.read_owned().await;
1186        if self.ctx.data_movement_generation_exhausted() || self.ctx.data_movement_operation_epoch_exhausted() {
1187            return Err(Error::other("scanner publication lease generation is exhausted"));
1188        }
1189        if self.scanner_data_movement_snapshot_locked().await.1 || self.ctx.namespace_commits_pending() {
1190            return Err(Error::other("scanner publication lease is blocked by data movement"));
1191        }
1192        let Some((lease_generation, lease_namespace_generation)) = self.ctx.scanner_publication_lease_generations(token).await
1193        else {
1194            return Err(Error::other("scanner publication lease is unknown or expired"));
1195        };
1196        let namespace_generation = self.scanner_namespace_mutation_generation();
1197        if lease_generation != self.ctx.data_movement_generation()
1198            || lease_namespace_generation != namespace_generation
1199            || namespace_generation == u64::MAX
1200        {
1201            return Err(Error::other("scanner publication lease generation is stale"));
1202        }
1203        Ok(operation_guard)
1204    }
1205}
1206
1207// impl Clone for ECStore {
1208//     fn clone(&self) -> Self {
1209//         let pool_meta = match self.pool_meta.read() {
1210//             Ok(pool_meta) => pool_meta.clone(),
1211//             Err(_) => PoolMeta::default(),
1212//         };
1213//         Self {
1214//             id: self.id.clone(),
1215//             disk_map: self.disk_map.clone(),
1216//             pools: self.pools.clone(),
1217//             peer_sys: self.peer_sys.clone(),
1218//             pool_meta: std_RwLock::new(pool_meta),
1219//             decommission_cancelers: self.decommission_cancelers.clone(),
1220//         }
1221//     }
1222// }
1223
1224// #[derive(Debug, Default, Clone)]
1225// pub struct ListPathOptions {
1226//     pub id: String,
1227
1228//     // Bucket of the listing.
1229//     pub bucket: String,
1230
1231//     // Directory inside the bucket.
1232//     // When unset listPath will set this based on Prefix
1233//     pub base_dir: String,
1234
1235//     // Scan/return only content with prefix.
1236//     pub prefix: String,
1237
1238//     // FilterPrefix will return only results with this prefix when scanning.
1239//     // Should never contain a slash.
1240//     // Prefix should still be set.
1241//     pub filter_prefix: String,
1242
1243//     // Marker to resume listing.
1244//     // The response will be the first entry >= this object name.
1245//     pub marker: String,
1246
1247//     // Limit the number of results.
1248//     pub limit: i32,
1249// }
1250
1251#[async_trait::async_trait]
1252impl crate::storage_api_contracts::object::ObjectIO for ECStore {
1253    type Error = Error;
1254    type RangeSpec = HTTPRangeSpec;
1255    type HeaderMap = HeaderMap;
1256    type ObjectOptions = ObjectOptions;
1257    type ObjectInfo = ObjectInfo;
1258    type GetObjectReader = GetObjectReader;
1259    type PutObjectReader = PutObjReader;
1260
1261    #[instrument(level = "debug", skip(self, h))]
1262    async fn get_object_reader(
1263        &self,
1264        bucket: &str,
1265        object: &str,
1266        range: Option<HTTPRangeSpec>,
1267        h: HeaderMap,
1268        opts: &ObjectOptions,
1269    ) -> Result<GetObjectReader> {
1270        self.handle_get_object_reader(bucket, object, range, h, opts).await
1271    }
1272    #[instrument(level = "debug", skip(self, data))]
1273    async fn put_object(&self, bucket: &str, object: &str, data: &mut PutObjReader, opts: &ObjectOptions) -> Result<ObjectInfo> {
1274        self.put_object_with_old_current_size(bucket, object, data, opts)
1275            .await
1276            .map(|(object_info, _)| object_info)
1277    }
1278}
1279
1280impl ECStore {
1281    /// `put_object` plus the rename_data old-size backfill
1282    /// (rustfs/backlog#1009); see `SetDisks::put_object_with_old_current_size`.
1283    /// Post-write hooks (immediate ILM transition enqueue, list-cache
1284    /// invalidation) match the plain `put_object` path exactly.
1285    #[instrument(level = "debug", skip(self, data))]
1286    pub async fn put_object_with_old_current_size(
1287        &self,
1288        bucket: &str,
1289        object: &str,
1290        data: &mut PutObjReader,
1291        opts: &ObjectOptions,
1292    ) -> Result<(ObjectInfo, Option<crate::disk::OldCurrentSize>)> {
1293        let result = match self.handle_put_object(bucket, object, data, opts).await {
1294            Ok((object_info, old_current_size)) => {
1295                enqueue_transition_after_write(self, Ok(object_info), LcEventSrc::S3PutObject, opts)
1296                    .await
1297                    .map(|object_info| (object_info, old_current_size))
1298            }
1299            Err(err) => Err(err),
1300        };
1301        if result.is_ok() {
1302            list_objects::observe_list_objects_mutation(self, bucket).await;
1303        }
1304        result
1305    }
1306}
1307
1308lazy_static! {
1309    static ref ENABLED_OBJECT_LOCK_CONFIG: ObjectLockConfiguration = ObjectLockConfiguration {
1310        object_lock_enabled: Some(ObjectLockEnabled::from_static(ObjectLockEnabled::ENABLED)),
1311        ..Default::default()
1312    };
1313    static ref ENABLED_VERSIONING_CONFIG: VersioningConfiguration = VersioningConfiguration {
1314        status: Some(BucketVersioningStatus::from_static(BucketVersioningStatus::ENABLED)),
1315        ..Default::default()
1316    };
1317}
1318
1319#[async_trait::async_trait]
1320impl BucketOperations for ECStore {
1321    type Error = Error;
1322
1323    #[instrument(skip(self))]
1324    async fn make_bucket(&self, bucket: &str, opts: &MakeBucketOptions) -> Result<()> {
1325        Box::pin(self.handle_make_bucket(bucket, opts)).await
1326    }
1327
1328    #[instrument(skip(self))]
1329    async fn get_bucket_info(&self, bucket: &str, opts: &BucketOptions) -> Result<BucketInfo> {
1330        self.handle_get_bucket_info(bucket, opts).await
1331    }
1332    #[instrument(skip(self))]
1333    async fn list_bucket(&self, opts: &BucketOptions) -> Result<Vec<BucketInfo>> {
1334        self.handle_list_bucket(opts).await
1335    }
1336    #[instrument(skip(self))]
1337    async fn delete_bucket(&self, bucket: &str, opts: &DeleteBucketOptions) -> Result<()> {
1338        Box::pin(self.handle_delete_bucket(bucket, opts)).await
1339    }
1340}
1341
1342#[async_trait::async_trait]
1343impl crate::storage_api_contracts::object::ObjectOperations for ECStore {
1344    type Error = Error;
1345    type ObjectInfo = ObjectInfo;
1346    type ObjectOptions = ObjectOptions;
1347    type FileInfo = FileInfo;
1348    type ObjectToDelete = ObjectToDelete;
1349    type DeletedObject = DeletedObject;
1350
1351    async fn get_object_info(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<ObjectInfo> {
1352        self.handle_get_object_info(bucket, object, opts).await
1353    }
1354
1355    async fn verify_object_integrity(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<()> {
1356        self.handle_verify_object_integrity(bucket, object, opts).await
1357    }
1358
1359    #[instrument(skip(self))]
1360    async fn copy_object(
1361        &self,
1362        src_bucket: &str,
1363        src_object: &str,
1364        dst_bucket: &str,
1365        dst_object: &str,
1366        src_info: &mut ObjectInfo,
1367        src_opts: &ObjectOptions,
1368        dst_opts: &ObjectOptions,
1369    ) -> Result<ObjectInfo> {
1370        let result = enqueue_transition_after_write(
1371            self,
1372            self.handle_copy_object(src_bucket, src_object, dst_bucket, dst_object, src_info, src_opts, dst_opts)
1373                .await,
1374            LcEventSrc::S3CopyObject,
1375            dst_opts,
1376        )
1377        .await;
1378        if result.is_ok() {
1379            list_objects::observe_list_objects_mutation(self, dst_bucket).await;
1380        }
1381        result
1382    }
1383
1384    #[instrument(skip(self))]
1385    async fn delete_object_version(&self, bucket: &str, object: &str, fi: &FileInfo, force_del_marker: bool) -> Result<()> {
1386        let result = self.handle_delete_object_version(bucket, object, fi, force_del_marker).await;
1387        if result.is_ok() {
1388            list_objects::observe_list_objects_mutation(self, bucket).await;
1389        }
1390        result
1391    }
1392
1393    #[instrument(skip(self))]
1394    async fn delete_object(&self, bucket: &str, object: &str, opts: ObjectOptions) -> Result<ObjectInfo> {
1395        let result = self.handle_delete_object(bucket, object, opts).await;
1396        if result.is_ok() {
1397            list_objects::observe_list_objects_mutation(self, bucket).await;
1398        }
1399        result
1400    }
1401
1402    #[instrument(skip(self, objects, opts))]
1403    async fn delete_objects(
1404        &self,
1405        bucket: &str,
1406        objects: Vec<ObjectToDelete>,
1407        opts: ObjectOptions,
1408    ) -> (Vec<DeletedObject>, Vec<Option<Error>>) {
1409        let result = self.handle_delete_objects(bucket, objects, opts).await;
1410        let success_count = result.1.iter().filter(|err| err.is_none()).count();
1411        if success_count > 0 {
1412            list_objects::observe_list_objects_mutations(self, bucket, success_count).await;
1413        }
1414        result
1415    }
1416
1417    #[instrument(skip(self))]
1418    async fn put_object_metadata(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<ObjectInfo> {
1419        self.handle_put_object_metadata(bucket, object, opts).await
1420    }
1421    #[instrument(skip(self))]
1422    async fn get_object_tags(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<String> {
1423        self.handle_get_object_tags(bucket, object, opts).await
1424    }
1425
1426    #[instrument(level = "debug", skip(self))]
1427    async fn put_object_tags(&self, bucket: &str, object: &str, tags: &str, opts: &ObjectOptions) -> Result<ObjectInfo> {
1428        self.handle_put_object_tags(bucket, object, tags, opts).await
1429    }
1430
1431    #[instrument(skip(self))]
1432    async fn delete_object_tags(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<ObjectInfo> {
1433        self.handle_delete_object_tags(bucket, object, opts).await
1434    }
1435
1436    #[instrument(skip(self))]
1437    async fn add_partial(&self, bucket: &str, object: &str, version_id: &str) -> Result<()> {
1438        self.handle_add_partial(bucket, object, version_id).await
1439    }
1440    #[instrument(skip(self))]
1441    async fn transition_object(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<()> {
1442        self.handle_transition_object(bucket, object, opts).await
1443    }
1444
1445    #[instrument(skip(self))]
1446    async fn restore_transitioned_object(self: Arc<Self>, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<()> {
1447        self.handle_restore_transitioned_object(bucket, object, opts).await
1448    }
1449}
1450
1451#[async_trait::async_trait]
1452impl crate::storage_api_contracts::list::ListOperations for ECStore {
1453    type Error = Error;
1454    type ListObjectsV2Info = ListObjectsV2Info;
1455    type ListObjectVersionsInfo = ListObjectVersionsInfo;
1456    type ObjectInfoOrErr = ObjectInfoOrErr;
1457    type WalkOptions = WalkOptions;
1458    type WalkCancellation = CancellationToken;
1459    type WalkResultSender = tokio::sync::mpsc::Sender<ObjectInfoOrErr>;
1460
1461    // @continuation_token marker
1462    // @start_after as marker when continuation_token empty
1463    // @delimiter default="/", empty when recursive
1464    // @max_keys limit
1465    #[instrument(level = "trace", skip(self))]
1466    async fn list_objects_v2(
1467        self: Arc<Self>,
1468        bucket: &str,
1469        prefix: &str,
1470        continuation_token: Option<String>,
1471        delimiter: Option<String>,
1472        max_keys: i32,
1473        fetch_owner: bool,
1474        start_after: Option<String>,
1475        incl_deleted: bool,
1476    ) -> Result<ListObjectsV2Info> {
1477        self.inner_list_objects_v2(
1478            bucket,
1479            prefix,
1480            continuation_token,
1481            delimiter,
1482            max_keys,
1483            fetch_owner,
1484            start_after,
1485            incl_deleted,
1486        )
1487        .await
1488    }
1489
1490    #[instrument(skip(self))]
1491    async fn list_object_versions(
1492        self: Arc<Self>,
1493        bucket: &str,
1494        prefix: &str,
1495        marker: Option<String>,
1496        version_marker: Option<String>,
1497        delimiter: Option<String>,
1498        max_keys: i32,
1499    ) -> Result<ListObjectVersionsInfo> {
1500        self.inner_list_object_versions(bucket, prefix, marker, version_marker, delimiter, max_keys)
1501            .await
1502    }
1503
1504    async fn walk(
1505        self: Arc<Self>,
1506        rx: CancellationToken,
1507        bucket: &str,
1508        prefix: &str,
1509        result: tokio::sync::mpsc::Sender<ObjectInfoOrErr>,
1510        opts: WalkOptions,
1511    ) -> Result<()> {
1512        self.walk_internal(rx, bucket, prefix, result, opts).await
1513    }
1514}
1515
1516#[async_trait::async_trait]
1517impl crate::storage_api_contracts::multipart::MultipartOperations for ECStore {
1518    type Error = Error;
1519    type ObjectInfo = ObjectInfo;
1520    type ObjectOptions = ObjectOptions;
1521    type PutObjectReader = PutObjReader;
1522    type CompletePart = CompletePart;
1523    type ListMultipartsInfo = ListMultipartsInfo;
1524    type MultipartUploadResult = MultipartUploadResult;
1525    type PartInfo = PartInfo;
1526    type MultipartInfo = MultipartInfo;
1527    type ListPartsInfo = ListPartsInfo;
1528
1529    #[instrument(skip(self))]
1530    async fn list_multipart_uploads(
1531        &self,
1532        bucket: &str,
1533        prefix: &str,
1534        key_marker: Option<String>,
1535        upload_id_marker: Option<String>,
1536        delimiter: Option<String>,
1537        max_uploads: usize,
1538    ) -> Result<ListMultipartsInfo> {
1539        self.handle_list_multipart_uploads(
1540            bucket,
1541            multipart::MultipartUploadListRequest {
1542                prefix: prefix.to_string(),
1543                key_marker,
1544                upload_id_marker,
1545                delimiter,
1546                max_uploads,
1547                expected_incarnation_id: None,
1548            },
1549        )
1550        .await
1551    }
1552
1553    #[instrument(skip(self))]
1554    async fn new_multipart_upload(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<MultipartUploadResult> {
1555        self.handle_new_multipart_upload(bucket, object, opts).await
1556    }
1557
1558    #[instrument(skip(self))]
1559    async fn copy_object_part(
1560        &self,
1561        src_bucket: &str,
1562        src_object: &str,
1563        _dst_bucket: &str,
1564        _dst_object: &str,
1565        _upload_id: &str,
1566        _part_id: usize,
1567        _start_offset: i64,
1568        _length: i64,
1569        _src_info: &ObjectInfo,
1570        _src_opts: &ObjectOptions,
1571        _dst_opts: &ObjectOptions,
1572    ) -> Result<()> {
1573        self.handle_copy_object_part(
1574            src_bucket,
1575            src_object,
1576            _dst_bucket,
1577            _dst_object,
1578            _upload_id,
1579            _part_id,
1580            _start_offset,
1581            _length,
1582            _src_info,
1583            _src_opts,
1584            _dst_opts,
1585        )
1586        .await
1587    }
1588    #[instrument(skip(self, data))]
1589    async fn put_object_part(
1590        &self,
1591        bucket: &str,
1592        object: &str,
1593        upload_id: &str,
1594        part_id: usize,
1595        data: &mut PutObjReader,
1596        opts: &ObjectOptions,
1597    ) -> Result<PartInfo> {
1598        self.handle_put_object_part(bucket, object, upload_id, part_id, data, opts)
1599            .await
1600    }
1601
1602    #[instrument(skip(self))]
1603    async fn get_multipart_info(
1604        &self,
1605        bucket: &str,
1606        object: &str,
1607        upload_id: &str,
1608        opts: &ObjectOptions,
1609    ) -> Result<MultipartInfo> {
1610        self.handle_get_multipart_info(bucket, object, upload_id, opts).await
1611    }
1612
1613    #[instrument(skip(self))]
1614    async fn list_object_parts(
1615        &self,
1616        bucket: &str,
1617        object: &str,
1618        upload_id: &str,
1619        part_number_marker: Option<usize>,
1620        max_parts: usize,
1621        opts: &ObjectOptions,
1622    ) -> Result<ListPartsInfo> {
1623        self.handle_list_object_parts(bucket, object, upload_id, part_number_marker, max_parts, opts)
1624            .await
1625    }
1626
1627    #[instrument(skip(self))]
1628    async fn abort_multipart_upload(&self, bucket: &str, object: &str, upload_id: &str, opts: &ObjectOptions) -> Result<()> {
1629        self.handle_abort_multipart_upload(bucket, object, upload_id, opts).await
1630    }
1631
1632    #[instrument(skip(self))]
1633    async fn complete_multipart_upload(
1634        self: Arc<Self>,
1635        bucket: &str,
1636        object: &str,
1637        upload_id: &str,
1638        uploaded_parts: Vec<CompletePart>,
1639        opts: &ObjectOptions,
1640    ) -> Result<ObjectInfo> {
1641        let result = enqueue_transition_after_write(
1642            self.as_ref(),
1643            self.clone()
1644                .handle_complete_multipart_upload(bucket, object, upload_id, uploaded_parts, opts)
1645                .await,
1646            LcEventSrc::S3CompleteMultipartUpload,
1647            opts,
1648        )
1649        .await;
1650        if result.is_ok() {
1651            list_objects::observe_list_objects_mutation(self.as_ref(), bucket).await;
1652        }
1653        result
1654    }
1655}
1656
1657#[async_trait::async_trait]
1658impl crate::storage_api_contracts::heal::HealOperations for ECStore {
1659    type Error = Error;
1660    type HealResultItem = HealResultItem;
1661    type HealOptions = HealOpts;
1662
1663    #[instrument(skip(self))]
1664    async fn heal_format(&self, dry_run: bool) -> Result<(HealResultItem, Option<Error>)> {
1665        self.handle_heal_format(dry_run).await
1666    }
1667
1668    #[instrument(skip(self))]
1669    async fn heal_bucket(&self, bucket: &str, opts: &HealOpts) -> Result<HealResultItem> {
1670        self.handle_heal_bucket(bucket, opts).await
1671    }
1672    #[instrument(level = "trace", skip(self, opts), fields(bucket = %bucket, object = %object, version_id = %version_id))]
1673    async fn heal_object(
1674        &self,
1675        bucket: &str,
1676        object: &str,
1677        version_id: &str,
1678        opts: &HealOpts,
1679    ) -> Result<(HealResultItem, Option<Error>)> {
1680        self.handle_heal_object(bucket, object, version_id, opts).await
1681    }
1682
1683    #[instrument(skip(self))]
1684    async fn get_pool_and_set(&self, id: &str) -> Result<(Option<usize>, Option<usize>, Option<usize>)> {
1685        self.handle_get_pool_and_set(id).await
1686    }
1687
1688    #[instrument(skip(self))]
1689    async fn check_abandoned_parts(&self, bucket: &str, object: &str, opts: &HealOpts) -> Result<()> {
1690        self.handle_check_abandoned_parts(bucket, object, opts).await
1691    }
1692}
1693
1694#[async_trait::async_trait]
1695impl crate::storage_api_contracts::namespace::NamespaceLocking for ECStore {
1696    type Error = Error;
1697    type NamespaceLock = NamespaceLockWrapper;
1698
1699    async fn new_ns_lock(&self, bucket: &str, object: &str) -> Result<NamespaceLockWrapper> {
1700        self.handle_new_ns_lock(bucket, object).await
1701    }
1702}
1703
1704#[async_trait::async_trait]
1705impl crate::storage_api_contracts::admin::StorageAdminApi for ECStore {
1706    type BackendInfo = rustfs_madmin::BackendInfo;
1707    type StorageInfo = rustfs_madmin::StorageInfo;
1708    type Disk = DiskStore;
1709    type Error = Error;
1710
1711    #[instrument(skip(self))]
1712    async fn backend_info(&self) -> Self::BackendInfo {
1713        self.handle_backend_info().await
1714    }
1715
1716    #[instrument(skip(self))]
1717    async fn storage_info(&self) -> Self::StorageInfo {
1718        self.handle_storage_info().await
1719    }
1720
1721    #[instrument(skip(self))]
1722    async fn local_storage_info(&self) -> Self::StorageInfo {
1723        self.handle_local_storage_info().await
1724    }
1725
1726    #[instrument(skip(self))]
1727    async fn disk_set_inventory(
1728        &self,
1729        selector: crate::storage_api_contracts::admin::DiskSetSelector,
1730    ) -> Result<Vec<Option<Self::Disk>>> {
1731        self.handle_get_disks(selector.pool_idx, selector.set_idx).await
1732    }
1733
1734    #[instrument(skip(self))]
1735    fn set_drive_counts(&self) -> Vec<usize> {
1736        self.handle_set_drive_counts()
1737    }
1738}
1739
1740#[cfg(test)]
1741mod tests {
1742    use super::*;
1743    use crate::core::pools::{PoolDecommissionInfo, PoolSpaceInfo, PoolStatus};
1744    use crate::layout::endpoints::{Endpoints, PoolEndpoints, SetupType};
1745    use crate::object_api::ObjectOptions;
1746    use crate::runtime::global::reset_local_disk_test_state;
1747    use crate::runtime::sources::{clear_local_disk_id_map_for_test, local_disk_path_by_id};
1748    use crate::store::init_format::{connect_load_init_formats, init_disks};
1749    use serial_test::serial;
1750    use tempfile::TempDir;
1751
1752    #[test]
1753    fn g_d2_008_default_versioning_config_keeps_persisted_bytes() {
1754        let bytes = crate::bucket::utils::serialize::<VersioningConfiguration>(&ENABLED_VERSIONING_CONFIG)
1755            .expect("the default Versioning configuration must serialize");
1756        assert_eq!(bytes, b"<VersioningConfiguration><Status>Enabled</Status></VersioningConfiguration>");
1757    }
1758
1759    #[test]
1760    fn g_d2_009_default_object_lock_config_keeps_persisted_bytes() {
1761        let bytes = crate::bucket::utils::serialize::<ObjectLockConfiguration>(&ENABLED_OBJECT_LOCK_CONFIG)
1762            .expect("the default Object Lock configuration must serialize");
1763        assert_eq!(
1764            bytes,
1765            b"<ObjectLockConfiguration><ObjectLockEnabled>Enabled</ObjectLockEnabled></ObjectLockConfiguration>"
1766        );
1767    }
1768
1769    #[tokio::test]
1770    async fn test_get_disk_infos() {
1771        let disks = vec![None, None]; // Empty disks for testing
1772        let infos = get_disk_infos(&disks).await;
1773
1774        assert_eq!(infos.len(), disks.len());
1775        // All should be None since we passed None disks
1776        assert!(infos.iter().all(|info| info.is_none()));
1777    }
1778
1779    #[test]
1780    fn ecstore_debug_is_bounded_summary() {
1781        let endpoint_pools = EndpointServerPools::default();
1782        let ctx = Arc::new(InstanceContext::new());
1783        let store = ECStore {
1784            id: uuid::Uuid::new_v4(),
1785            disk_map: [(0, vec![None, None, None, None])].into_iter().collect(),
1786            pools: Vec::new(),
1787            peer_sys: crate::cluster::rpc::S3PeerSys::new_with_instance_ctx(&endpoint_pools, ctx.clone()),
1788            pool_meta: RwLock::new(PoolMeta::default()),
1789            rebalance_meta: RwLock::new(None),
1790            decommission_cancelers: RwLock::new(Vec::new()),
1791            start_gate: Mutex::new(()),
1792            pool_meta_save_gate: Mutex::default(),
1793            ctx,
1794            bucket_fence_registry: Arc::default(),
1795        };
1796
1797        let rendered = format!("{store:?}");
1798
1799        assert!(rendered.len() < 256, "ECStore Debug should stay bounded: {rendered}");
1800        assert!(rendered.contains("disk_map_pool_count"));
1801        assert!(rendered.contains("disk_slot_count"));
1802        assert!(!rendered.contains("disk_map:"));
1803        assert!(!rendered.contains("pools:"));
1804        assert!(!rendered.contains("pool_meta"));
1805        assert!(!rendered.contains("format.json"));
1806        assert!(!rendered.contains("TimedActionSlot"));
1807        assert!(!rendered.contains("DiskHealthTracker"));
1808    }
1809
1810    #[test]
1811    fn object_options_debug_does_not_expand_tier_store_handle() {
1812        let store = build_store_with_ctx(Arc::new(InstanceContext::new()));
1813        let mut opts = ObjectOptions {
1814            version_id: Some("large-version-id".repeat(1024)),
1815            expected_current_version_id: Some("large-expected-version-id".repeat(1024)),
1816            preserve_etag: Some("large-etag".repeat(1024)),
1817            http_preconditions: Some(crate::storage_api_contracts::object::HTTPPreconditions {
1818                if_match: Some("large-if-match".repeat(1024)),
1819                if_none_match: Some("large-if-none-match".repeat(1024)),
1820                ..Default::default()
1821            }),
1822            tier_delete_journal_api: Some(store),
1823            ..Default::default()
1824        };
1825        opts.user_defined.insert("large-user-metadata".to_owned(), "x".repeat(8192));
1826        opts.eval_metadata = Some([("large-eval-metadata".to_owned(), "y".repeat(8192))].into_iter().collect());
1827        opts.transition.status = "large-transition-status".repeat(1024);
1828        opts.transition.tier = "large-transition-tier".repeat(1024);
1829        opts.lifecycle_audit_event.event.rule_id = "large-rule-id".repeat(1024);
1830        opts.lifecycle_audit_event.event.storage_class = "large-storage-class".repeat(1024);
1831
1832        let rendered = format!("{opts:?}");
1833
1834        assert!(rendered.len() < 4096, "ObjectOptions Debug should stay bounded: {rendered}");
1835        assert!(rendered.contains("tier_delete_journal_api: true"));
1836        assert!(rendered.contains("user_defined_count: 1"));
1837        assert!(rendered.contains("eval_metadata_count: Some(1)"));
1838        assert!(!rendered.contains("ECStore {"));
1839        assert!(!rendered.contains("disk_map"));
1840        assert!(!rendered.contains("large-version-id"));
1841        assert!(!rendered.contains("large-expected-version-id"));
1842        assert!(!rendered.contains("large-etag"));
1843        assert!(!rendered.contains("large-if-match"));
1844        assert!(!rendered.contains("large-transition"));
1845        assert!(!rendered.contains("large-rule-id"));
1846        assert!(!rendered.contains("large-storage-class"));
1847        assert!(!rendered.contains("large-user-metadata"));
1848        assert!(!rendered.contains("large-eval-metadata"));
1849        assert!(!rendered.contains("format.json"));
1850        assert!(!rendered.contains("TimedActionSlot"));
1851        assert!(!rendered.contains("DiskHealthTracker"));
1852    }
1853
1854    // Build a minimal ECStore carrying an explicit instance context. Empty
1855    // pools/disks are sufficient: the Phase 5 accessors read only `self.ctx`.
1856    pub(super) fn build_store_with_ctx(ctx: Arc<InstanceContext>) -> Arc<ECStore> {
1857        let endpoint_pools = EndpointServerPools::default();
1858        Arc::new(ECStore {
1859            id: uuid::Uuid::new_v4(),
1860            disk_map: std::collections::HashMap::new(),
1861            pools: Vec::new(),
1862            peer_sys: crate::cluster::rpc::S3PeerSys::new_with_instance_ctx(&endpoint_pools, ctx.clone()),
1863            pool_meta: RwLock::new(PoolMeta::default()),
1864            rebalance_meta: RwLock::new(None),
1865            decommission_cancelers: RwLock::new(Vec::new()),
1866            start_gate: Mutex::new(()),
1867            pool_meta_save_gate: Mutex::default(),
1868            ctx,
1869            bucket_fence_registry: Arc::default(),
1870        })
1871    }
1872
1873    fn scanner_sequence_state(operation_epoch: u64, movement_generation: u64) -> ScannerDataMovementSequenceState {
1874        ScannerDataMovementSequenceState {
1875            operation_epoch,
1876            operation_epoch_exhausted: false,
1877            movement_generation,
1878            movement_generation_exhausted: false,
1879        }
1880    }
1881
1882    #[test]
1883    fn scanner_pause_status_derives_restart_stable_decommission_fields() {
1884        let started_at = OffsetDateTime::from_unix_timestamp(1_000).expect("fixed timestamp should be valid");
1885        let now = OffsetDateTime::from_unix_timestamp(1_090).expect("fixed timestamp should be valid");
1886        let pool_meta = PoolMeta {
1887            pools: vec![PoolStatus {
1888                id: 0,
1889                cmd_line: "pool-0".to_string(),
1890                last_update: started_at,
1891                decommission: Some(PoolDecommissionInfo {
1892                    start_time: Some(started_at),
1893                    queued_buckets: vec!["bucket-a".to_string(), "bucket-b".to_string()],
1894                    bucket: "bucket-a".to_string(),
1895                    ..Default::default()
1896                }),
1897            }],
1898            ..Default::default()
1899        };
1900
1901        let status = resolve_scanner_data_movement_pause_status(&pool_meta, None, false, scanner_sequence_state(7, 11), now);
1902
1903        assert!(status.paused);
1904        assert_eq!(status.policy, "global_pause");
1905        assert_eq!(status.reasons, vec![ScannerDataMovementPauseReason::DecommissionActive]);
1906        assert_eq!(status.started_at_unix_secs, 1_000);
1907        assert_eq!(status.duration_seconds, 90);
1908        assert_eq!(status.operation_epoch, 7);
1909        assert_eq!(status.movement_generation, 11);
1910        assert_eq!(status.movement_backlog_work_items, 2);
1911        assert!(status.movement_backlog_estimated);
1912    }
1913
1914    #[test]
1915    fn completed_decommission_restores_durable_movement_generation() {
1916        let completed_at = OffsetDateTime::from_unix_timestamp(1_100).expect("fixed timestamp should be valid");
1917        let pool_meta = PoolMeta {
1918            pools: vec![PoolStatus {
1919                id: 0,
1920                cmd_line: "pool-0".to_string(),
1921                last_update: completed_at,
1922                decommission: Some(PoolDecommissionInfo {
1923                    complete: true,
1924                    ..Default::default()
1925                }),
1926            }],
1927            ..Default::default()
1928        };
1929        let durable_generation = durable_scanner_data_movement_generation(&pool_meta, None);
1930        let ctx = InstanceContext::new();
1931
1932        ctx.observe_durable_data_movement_generation(durable_generation);
1933
1934        assert_eq!(durable_generation, 1_100_000_000_000);
1935        assert_eq!(ctx.data_movement_generation(), durable_generation);
1936    }
1937
1938    #[tokio::test]
1939    async fn cleared_decommission_restores_durable_movement_generation_after_restart() {
1940        let mut pool_meta = PoolMeta {
1941            pools: vec![PoolStatus {
1942                id: 0,
1943                cmd_line: "pool-0".to_string(),
1944                last_update: OffsetDateTime::UNIX_EPOCH,
1945                decommission: Some(PoolDecommissionInfo {
1946                    failed: true,
1947                    ..Default::default()
1948                }),
1949            }],
1950            ..Default::default()
1951        };
1952        assert!(pool_meta.clear_decommission(0).expect("failed decommission should clear"));
1953        assert!(
1954            pool_meta.pools[0]
1955                .decommission
1956                .as_ref()
1957                .is_some_and(|info| !info.has_decommission_state())
1958        );
1959        let durable_generation = durable_scanner_data_movement_generation(&pool_meta, None);
1960        let restarted = build_store_with_ctx(Arc::new(InstanceContext::new()));
1961        *restarted.pool_meta.write().await = pool_meta;
1962
1963        let status = restarted.scanner_data_movement_pause_status().await;
1964
1965        assert_ne!(durable_generation, 0);
1966        assert!(!status.paused);
1967        assert_eq!(status.movement_generation, durable_generation);
1968        assert_eq!(restarted.scanner_data_movement_generation(), durable_generation);
1969    }
1970
1971    #[tokio::test]
1972    async fn same_tick_cleared_decommission_tombstones_advance_durable_movement_generation() {
1973        let same_tick = OffsetDateTime::from_unix_timestamp(1_100).expect("fixed timestamp should be valid");
1974        let mut pool_meta = PoolMeta {
1975            pools: vec![
1976                PoolStatus {
1977                    id: 0,
1978                    cmd_line: "pool-0".to_string(),
1979                    last_update: same_tick,
1980                    decommission: Some(PoolDecommissionInfo {
1981                        failed: true,
1982                        ..Default::default()
1983                    }),
1984                },
1985                PoolStatus {
1986                    id: 1,
1987                    cmd_line: "pool-1".to_string(),
1988                    last_update: same_tick,
1989                    decommission: Some(PoolDecommissionInfo {
1990                        canceled: true,
1991                        ..Default::default()
1992                    }),
1993                },
1994            ],
1995            ..Default::default()
1996        };
1997
1998        assert!(
1999            pool_meta
2000                .clear_decommission_at_for_test(0, same_tick, None)
2001                .expect("first terminal decommission should clear")
2002        );
2003        let first_generation = durable_scanner_data_movement_generation(&pool_meta, None);
2004        assert_eq!(
2005            first_generation,
2006            scanner_data_movement_timestamp_generation(same_tick + time::Duration::nanoseconds(1))
2007        );
2008
2009        assert!(
2010            pool_meta
2011                .clear_decommission_at_for_test(1, same_tick, None)
2012                .expect("second terminal decommission should clear")
2013        );
2014        let second_generation = durable_scanner_data_movement_generation(&pool_meta, None);
2015        assert_eq!(
2016            second_generation,
2017            scanner_data_movement_timestamp_generation(same_tick + time::Duration::nanoseconds(2))
2018        );
2019        assert!(second_generation > first_generation);
2020
2021        let restarted = build_store_with_ctx(Arc::new(InstanceContext::new()));
2022        *restarted.pool_meta.write().await = pool_meta;
2023        let status = restarted.scanner_data_movement_pause_status().await;
2024
2025        assert!(!status.paused);
2026        assert_eq!(status.movement_generation, second_generation);
2027        assert_eq!(restarted.scanner_data_movement_generation(), second_generation);
2028    }
2029
2030    #[tokio::test]
2031    async fn terminal_decommission_transitions_advance_durable_generation_across_same_or_earlier_clocks() {
2032        let same_tick = OffsetDateTime::from_unix_timestamp(1_200).expect("fixed timestamp should be valid");
2033        let earlier_tick = same_tick - time::Duration::nanoseconds(10);
2034        let rebalance_floor = same_tick + time::Duration::nanoseconds(5);
2035        let rebalance = RebalanceMeta {
2036            stopped_at: Some(rebalance_floor),
2037            id: "completed-rebalance".to_string(),
2038            ..Default::default()
2039        };
2040        let active_decommission = |id| PoolStatus {
2041            id,
2042            cmd_line: format!("pool-{id}"),
2043            last_update: same_tick,
2044            decommission: Some(PoolDecommissionInfo {
2045                start_time: Some(same_tick),
2046                ..Default::default()
2047            }),
2048        };
2049        let mut pool_meta = PoolMeta {
2050            pools: vec![active_decommission(0), active_decommission(1), active_decommission(2)],
2051            ..Default::default()
2052        };
2053
2054        assert!(pool_meta.decommission_complete_at_for_test(0, same_tick, Some(&rebalance)));
2055        assert_eq!(pool_meta.pools[0].last_update, rebalance_floor + time::Duration::nanoseconds(1));
2056
2057        assert!(pool_meta.decommission_cancel_at_for_test(1, same_tick, Some(&rebalance)));
2058        assert_eq!(pool_meta.pools[1].last_update, rebalance_floor + time::Duration::nanoseconds(2));
2059
2060        assert!(pool_meta.decommission_failed_at_for_test(2, earlier_tick, Some(&rebalance)));
2061        assert_eq!(pool_meta.pools[2].last_update, rebalance_floor + time::Duration::nanoseconds(3));
2062        let durable_generation = durable_scanner_data_movement_generation(&pool_meta, Some(&rebalance));
2063        assert_eq!(
2064            durable_generation,
2065            scanner_data_movement_timestamp_generation(rebalance_floor + time::Duration::nanoseconds(3))
2066        );
2067
2068        let restarted = build_store_with_ctx(Arc::new(InstanceContext::new()));
2069        *restarted.pool_meta.write().await = pool_meta;
2070        *restarted.rebalance_meta.write().await = Some(rebalance);
2071        let status = restarted.scanner_data_movement_pause_status().await;
2072
2073        assert_eq!(status.movement_generation, durable_generation);
2074        assert_eq!(restarted.scanner_data_movement_generation(), durable_generation);
2075        assert_eq!(
2076            status.reasons,
2077            vec![
2078                ScannerDataMovementPauseReason::DecommissionFailed,
2079                ScannerDataMovementPauseReason::DecommissionCanceled
2080            ]
2081        );
2082    }
2083
2084    #[tokio::test]
2085    async fn decommission_start_after_clear_advances_durable_generation_across_clock_rollback_after_restart() {
2086        let same_tick = OffsetDateTime::from_unix_timestamp(1_250).expect("fixed timestamp should be valid");
2087        let earlier_tick = same_tick - time::Duration::nanoseconds(10);
2088        let rebalance_floor = same_tick + time::Duration::nanoseconds(5);
2089        let rebalance = RebalanceMeta {
2090            stopped_at: Some(rebalance_floor),
2091            id: "completed-rebalance".to_string(),
2092            ..Default::default()
2093        };
2094        let mut pool_meta = PoolMeta {
2095            pools: vec![PoolStatus {
2096                id: 0,
2097                cmd_line: "pool-0".to_string(),
2098                last_update: same_tick,
2099                decommission: Some(PoolDecommissionInfo {
2100                    failed: true,
2101                    ..Default::default()
2102                }),
2103            }],
2104            ..Default::default()
2105        };
2106
2107        assert!(
2108            pool_meta
2109                .clear_decommission_at_for_test(0, same_tick, Some(&rebalance))
2110                .expect("failed decommission should clear")
2111        );
2112        let cleared_at = rebalance_floor + time::Duration::nanoseconds(1);
2113        assert_eq!(pool_meta.pools[0].last_update, cleared_at);
2114
2115        pool_meta
2116            .decommission_at_for_test(
2117                0,
2118                PoolSpaceInfo {
2119                    total: 200,
2120                    free: 50,
2121                    used: 150,
2122                },
2123                earlier_tick,
2124                Some(&rebalance),
2125            )
2126            .expect("decommission restart after clear should be allowed");
2127        let started_at = cleared_at + time::Duration::nanoseconds(1);
2128        assert_eq!(pool_meta.pools[0].last_update, started_at);
2129        assert_eq!(
2130            pool_meta.pools[0].decommission.as_ref().and_then(|info| info.start_time),
2131            Some(started_at)
2132        );
2133
2134        assert!(pool_meta.decommission_complete_at_for_test(0, earlier_tick, Some(&rebalance)));
2135        let completed_at = started_at + time::Duration::nanoseconds(1);
2136        assert_eq!(pool_meta.pools[0].last_update, completed_at);
2137        let durable_generation = durable_scanner_data_movement_generation(&pool_meta, Some(&rebalance));
2138        assert_eq!(durable_generation, scanner_data_movement_timestamp_generation(completed_at));
2139
2140        let restarted = build_store_with_ctx(Arc::new(InstanceContext::new()));
2141        *restarted.pool_meta.write().await = pool_meta;
2142        *restarted.rebalance_meta.write().await = Some(rebalance);
2143        let status = restarted.scanner_data_movement_pause_status().await;
2144
2145        assert!(!status.paused);
2146        assert_eq!(status.movement_generation, durable_generation);
2147        assert_eq!(restarted.scanner_data_movement_generation(), durable_generation);
2148    }
2149
2150    #[tokio::test]
2151    async fn decommission_terminal_reload_failure_advances_durable_generation_across_clock_rollback_after_restart() {
2152        let terminal_at = OffsetDateTime::from_unix_timestamp(1_280).expect("fixed timestamp should be valid");
2153        let earlier_tick = terminal_at - time::Duration::nanoseconds(10);
2154        let rebalance_floor = terminal_at + time::Duration::nanoseconds(5);
2155        let rebalance = RebalanceMeta {
2156            stopped_at: Some(rebalance_floor),
2157            id: "completed-rebalance".to_string(),
2158            ..Default::default()
2159        };
2160        let mut pool_meta = PoolMeta {
2161            pools: vec![PoolStatus {
2162                id: 0,
2163                cmd_line: "pool-0".to_string(),
2164                last_update: terminal_at,
2165                decommission: Some(PoolDecommissionInfo {
2166                    start_time: Some(terminal_at),
2167                    complete: true,
2168                    ..Default::default()
2169                }),
2170            }],
2171            ..Default::default()
2172        };
2173
2174        assert!(
2175            pool_meta
2176                .record_decommission_terminal_reload_failure_at_for_test(
2177                    0,
2178                    "complete_decommission",
2179                    "peer reload failed".to_string(),
2180                    earlier_tick,
2181                    Some(&rebalance),
2182                )
2183                .expect("reload failure should be recorded")
2184        );
2185        let reload_failure_at = rebalance_floor + time::Duration::nanoseconds(1);
2186        assert_eq!(pool_meta.pools[0].last_update, reload_failure_at);
2187        let info = pool_meta.pools[0]
2188            .decommission
2189            .as_ref()
2190            .expect("decommission metadata should exist");
2191        assert_eq!(info.terminal_reload_attempt_at, Some(reload_failure_at));
2192        assert_eq!(
2193            info.terminal_reload_failures,
2194            vec!["complete_decommission: peer reload failed".to_string()]
2195        );
2196        let durable_generation = durable_scanner_data_movement_generation(&pool_meta, Some(&rebalance));
2197        assert_eq!(durable_generation, scanner_data_movement_timestamp_generation(reload_failure_at));
2198
2199        let restarted = build_store_with_ctx(Arc::new(InstanceContext::new()));
2200        *restarted.pool_meta.write().await = pool_meta;
2201        *restarted.rebalance_meta.write().await = Some(rebalance);
2202        let status = restarted.scanner_data_movement_pause_status().await;
2203
2204        assert!(!status.paused);
2205        assert_eq!(status.movement_generation, durable_generation);
2206        assert_eq!(restarted.scanner_data_movement_generation(), durable_generation);
2207    }
2208
2209    #[tokio::test]
2210    async fn rebalance_transitions_advance_durable_generation_across_same_or_earlier_clocks_after_restart() {
2211        let same_tick = OffsetDateTime::from_unix_timestamp(1_300).expect("fixed timestamp should be valid");
2212        let earlier_tick = same_tick - time::Duration::nanoseconds(10);
2213        let decommission_floor = same_tick + time::Duration::nanoseconds(5);
2214        let store = build_store_with_ctx(Arc::new(InstanceContext::new()));
2215        *store.pool_meta.write().await = PoolMeta {
2216            pools: vec![PoolStatus {
2217                id: 0,
2218                cmd_line: "pool-0".to_string(),
2219                last_update: decommission_floor,
2220                decommission: Some(PoolDecommissionInfo {
2221                    complete: true,
2222                    ..Default::default()
2223                }),
2224            }],
2225            ..Default::default()
2226        };
2227
2228        let started_at = store.next_scanner_data_movement_update(same_tick).await;
2229        assert_eq!(started_at, decommission_floor + time::Duration::nanoseconds(1));
2230        *store.rebalance_meta.write().await = Some(RebalanceMeta {
2231            id: "rebalance-generation".to_string(),
2232            pool_stats: vec![crate::services::rebalance::RebalanceStats {
2233                participating: true,
2234                info: crate::services::rebalance::RebalanceInfo {
2235                    start_time: Some(started_at),
2236                    status: RebalStatus::Started,
2237                    ..Default::default()
2238                },
2239                ..Default::default()
2240            }],
2241            ..Default::default()
2242        });
2243
2244        let completed_at = store.next_scanner_data_movement_update(same_tick).await;
2245        assert_eq!(completed_at, decommission_floor + time::Duration::nanoseconds(2));
2246        {
2247            let mut rebalance_meta = store.rebalance_meta.write().await;
2248            let meta = rebalance_meta.as_mut().expect("rebalance metadata should be present");
2249            meta.pool_stats[0].info.status = RebalStatus::Completed;
2250            meta.pool_stats[0].info.end_time = Some(completed_at);
2251        }
2252
2253        let stopped_at = store.next_scanner_data_movement_update(earlier_tick).await;
2254        assert_eq!(stopped_at, decommission_floor + time::Duration::nanoseconds(3));
2255        {
2256            let mut rebalance_meta = store.rebalance_meta.write().await;
2257            let meta = rebalance_meta.as_mut().expect("rebalance metadata should be present");
2258            meta.stopped_at = Some(stopped_at);
2259        }
2260        let pool_meta = store.pool_meta.read().await.clone();
2261        let rebalance_meta = store.rebalance_meta.read().await.clone();
2262        let durable_generation = durable_scanner_data_movement_generation(&pool_meta, rebalance_meta.as_ref());
2263        assert_eq!(
2264            durable_generation,
2265            scanner_data_movement_timestamp_generation(decommission_floor + time::Duration::nanoseconds(3))
2266        );
2267
2268        let restarted = build_store_with_ctx(Arc::new(InstanceContext::new()));
2269        *restarted.pool_meta.write().await = pool_meta;
2270        *restarted.rebalance_meta.write().await = rebalance_meta;
2271        let status = restarted.scanner_data_movement_pause_status().await;
2272
2273        assert!(!status.paused);
2274        assert_eq!(status.movement_generation, durable_generation);
2275        assert_eq!(restarted.scanner_data_movement_generation(), durable_generation);
2276    }
2277
2278    #[test]
2279    fn malformed_durable_movement_timestamp_exhausts_generation_fail_closed() {
2280        let pool_meta = PoolMeta {
2281            pools: vec![PoolStatus {
2282                id: 0,
2283                cmd_line: "pool-0".to_string(),
2284                last_update: OffsetDateTime::UNIX_EPOCH,
2285                decommission: Some(PoolDecommissionInfo {
2286                    complete: true,
2287                    ..Default::default()
2288                }),
2289            }],
2290            ..Default::default()
2291        };
2292
2293        assert_eq!(durable_scanner_data_movement_generation(&pool_meta, None), u64::MAX);
2294        let exhausted_generation =
2295            OffsetDateTime::from_unix_timestamp(253_402_300_799).expect("the largest RFC 3339 timestamp should be valid");
2296        assert_eq!(scanner_data_movement_timestamp_generation(exhausted_generation), u64::MAX);
2297    }
2298
2299    #[test]
2300    fn malformed_durable_movement_timestamp_is_not_masked_by_valid_rebalance_generation() {
2301        let valid_rebalance_at = OffsetDateTime::from_unix_timestamp(2_400).expect("fixed timestamp should be valid");
2302        let pool_meta = PoolMeta {
2303            pools: vec![PoolStatus {
2304                id: 0,
2305                cmd_line: "pool-0".to_string(),
2306                last_update: OffsetDateTime::UNIX_EPOCH,
2307                decommission: Some(PoolDecommissionInfo {
2308                    complete: true,
2309                    ..Default::default()
2310                }),
2311            }],
2312            ..Default::default()
2313        };
2314        let rebalance_meta = RebalanceMeta {
2315            id: "completed-rebalance".to_string(),
2316            stopped_at: Some(valid_rebalance_at),
2317            pool_stats: vec![crate::services::rebalance::RebalanceStats {
2318                participating: true,
2319                info: crate::services::rebalance::RebalanceInfo {
2320                    start_time: Some(valid_rebalance_at - time::Duration::nanoseconds(1)),
2321                    end_time: Some(valid_rebalance_at),
2322                    status: RebalStatus::Completed,
2323                    ..Default::default()
2324                },
2325                ..Default::default()
2326            }],
2327            ..Default::default()
2328        };
2329
2330        assert_eq!(durable_scanner_data_movement_generation(&pool_meta, Some(&rebalance_meta)), u64::MAX);
2331    }
2332
2333    #[test]
2334    fn durable_movement_generation_without_records_is_zero() {
2335        assert_eq!(durable_scanner_data_movement_generation(&PoolMeta::default(), None), 0);
2336        assert_eq!(
2337            durable_scanner_data_movement_generation(&PoolMeta::default(), Some(&RebalanceMeta::default())),
2338            0
2339        );
2340    }
2341
2342    #[test]
2343    fn scanner_pause_status_distinguishes_terminal_rebalance_epoch_and_idle() {
2344        let last_update = OffsetDateTime::from_unix_timestamp(2_000).expect("fixed timestamp should be valid");
2345        let now = OffsetDateTime::from_unix_timestamp(2_030).expect("fixed timestamp should be valid");
2346        let failed = PoolMeta {
2347            pools: vec![PoolStatus {
2348                id: 0,
2349                cmd_line: "pool-0".to_string(),
2350                last_update,
2351                decommission: Some(PoolDecommissionInfo {
2352                    failed: true,
2353                    ..Default::default()
2354                }),
2355            }],
2356            ..Default::default()
2357        };
2358        let failed_status = resolve_scanner_data_movement_pause_status(&failed, None, false, scanner_sequence_state(3, 12), now);
2359        assert_eq!(failed_status.reasons, vec![ScannerDataMovementPauseReason::DecommissionFailed]);
2360        assert_eq!(failed_status.started_at_unix_secs, 2_000);
2361        assert_eq!(failed_status.duration_seconds, 30);
2362
2363        let rebalance = RebalanceMeta {
2364            pool_stats: vec![crate::services::rebalance::RebalanceStats {
2365                buckets: vec!["bucket-a".to_string(), "bucket-b".to_string()],
2366                participating: true,
2367                info: crate::services::rebalance::RebalanceInfo {
2368                    start_time: Some(last_update),
2369                    status: RebalStatus::Started,
2370                    ..Default::default()
2371                },
2372                ..Default::default()
2373            }],
2374            ..Default::default()
2375        };
2376        let rebalance_status = resolve_scanner_data_movement_pause_status(
2377            &PoolMeta::default(),
2378            Some(&rebalance),
2379            false,
2380            scanner_sequence_state(4, 13),
2381            now,
2382        );
2383        assert_eq!(rebalance_status.reasons, vec![ScannerDataMovementPauseReason::RebalanceActive]);
2384        assert_eq!(rebalance_status.movement_backlog_work_items, 2);
2385
2386        let exhausted = resolve_scanner_data_movement_pause_status(
2387            &PoolMeta::default(),
2388            None,
2389            false,
2390            ScannerDataMovementSequenceState {
2391                operation_epoch: u64::MAX,
2392                operation_epoch_exhausted: true,
2393                movement_generation: 14,
2394                movement_generation_exhausted: false,
2395            },
2396            now,
2397        );
2398        assert_eq!(exhausted.reasons, vec![ScannerDataMovementPauseReason::OperationEpochExhausted]);
2399        assert_eq!(exhausted.started_at_unix_secs, 0);
2400
2401        let generation_exhausted = resolve_scanner_data_movement_pause_status(
2402            &PoolMeta::default(),
2403            None,
2404            false,
2405            ScannerDataMovementSequenceState {
2406                operation_epoch: 5,
2407                operation_epoch_exhausted: false,
2408                movement_generation: u64::MAX,
2409                movement_generation_exhausted: true,
2410            },
2411            now,
2412        );
2413        assert_eq!(
2414            generation_exhausted.reasons,
2415            vec![ScannerDataMovementPauseReason::MovementGenerationExhausted]
2416        );
2417
2418        let idle =
2419            resolve_scanner_data_movement_pause_status(&PoolMeta::default(), None, false, scanner_sequence_state(5, 15), now);
2420        assert!(!idle.paused);
2421        assert!(idle.reasons.is_empty());
2422        assert!(!idle.movement_backlog_estimated);
2423    }
2424
2425    #[tokio::test]
2426    async fn scanner_data_usage_publication_blocks_active_and_unqueued_terminal_decommission() {
2427        let store = build_store_with_ctx(Arc::new(InstanceContext::new()));
2428        let cases = [
2429            (
2430                "active",
2431                PoolDecommissionInfo {
2432                    start_time: Some(OffsetDateTime::now_utc()),
2433                    ..Default::default()
2434                },
2435                true,
2436            ),
2437            (
2438                "failed",
2439                PoolDecommissionInfo {
2440                    failed: true,
2441                    ..Default::default()
2442                },
2443                true,
2444            ),
2445            (
2446                "canceled",
2447                PoolDecommissionInfo {
2448                    canceled: true,
2449                    ..Default::default()
2450                },
2451                true,
2452            ),
2453            (
2454                "queued_failed",
2455                PoolDecommissionInfo {
2456                    failed: true,
2457                    queued: true,
2458                    ..Default::default()
2459                },
2460                false,
2461            ),
2462            (
2463                "complete",
2464                PoolDecommissionInfo {
2465                    complete: true,
2466                    ..Default::default()
2467                },
2468                false,
2469            ),
2470            ("idle", PoolDecommissionInfo::default(), false),
2471        ];
2472
2473        for (name, decommission, expected) in cases {
2474            *store.pool_meta.write().await = PoolMeta {
2475                pools: vec![PoolStatus {
2476                    id: 0,
2477                    cmd_line: format!("scanner-publication-{name}"),
2478                    last_update: OffsetDateTime::now_utc(),
2479                    decommission: Some(decommission),
2480                }],
2481                ..Default::default()
2482            };
2483            assert_eq!(
2484                store.scanner_data_usage_publication_blocked().await,
2485                expected,
2486                "unexpected scanner publication barrier state for {name}"
2487            );
2488        }
2489    }
2490
2491    #[tokio::test]
2492    async fn scanner_data_usage_publication_admission_is_fenced_and_epoch_monotonic() {
2493        let store = build_store_with_ctx(Arc::new(InstanceContext::new()));
2494        let operation_gate = store.ctx.data_movement_operation_gate();
2495        let movement_guard = operation_gate.write().await;
2496        let pending = {
2497            let store = store.clone();
2498            tokio::spawn(async move { store.scanner_data_usage_publication_admission_guard().await })
2499        };
2500        tokio::task::yield_now().await;
2501        assert!(!pending.is_finished(), "publication admission must wait for a movement writer");
2502        drop(movement_guard);
2503
2504        let (_, epoch) = pending
2505            .await
2506            .expect("publication admission task should not panic")
2507            .expect("idle store should admit publication");
2508        assert_eq!(epoch, 0);
2509        assert_eq!(store.ctx.advance_data_movement_operation_epoch(), 1);
2510        let (_, next_epoch) = store
2511            .scanner_data_usage_publication_admission_guard()
2512            .await
2513            .expect("idle store should admit the next publication");
2514        assert_eq!(next_epoch, 1);
2515        assert_eq!(store.scanner_data_movement_generation(), 1);
2516    }
2517
2518    #[tokio::test(start_paused = true)]
2519    async fn scanner_publication_lease_blocks_movement_writer_until_release_or_expiry() {
2520        let store = build_store_with_ctx(Arc::new(InstanceContext::new()));
2521        let (token, generation) = store
2522            .acquire_scanner_publication_lease(0, crate::runtime::instance::SCANNER_PUBLICATION_LEASE_TTL)
2523            .await
2524            .expect("an idle store should grant a publication lease");
2525        assert_eq!(generation, 0);
2526
2527        let gate = store.ctx.data_movement_operation_gate();
2528        let (writer_started, writer_started_rx) = tokio::sync::oneshot::channel();
2529        let movement_writer = tokio::spawn(async move {
2530            let _ = writer_started.send(());
2531            gate.write_owned().await
2532        });
2533        writer_started_rx
2534            .await
2535            .expect("movement writer should reach the gate before waiting");
2536        assert!(
2537            !movement_writer.is_finished(),
2538            "a movement writer must wait while the remote lease owns the read guard"
2539        );
2540
2541        assert!(store.release_scanner_publication_lease(token).await);
2542        tokio::time::timeout(Duration::from_secs(1), movement_writer)
2543            .await
2544            .expect("movement writer should proceed after lease release")
2545            .expect("movement writer task should not panic");
2546
2547        let (expiring_token, _) = store
2548            .acquire_scanner_publication_lease(0, crate::runtime::instance::SCANNER_PUBLICATION_LEASE_TTL)
2549            .await
2550            .expect("the store should grant a second publication lease");
2551        let expiry_gate = store.ctx.data_movement_operation_gate();
2552        let (expiry_started, expiry_started_rx) = tokio::sync::oneshot::channel();
2553        let mut expiry_writer = tokio::spawn(async move {
2554            let _ = expiry_started.send(());
2555            expiry_gate.write_owned().await
2556        });
2557        expiry_started_rx
2558            .await
2559            .expect("expiry writer should reach the gate before waiting");
2560        assert!(!expiry_writer.is_finished());
2561        tokio::time::advance(crate::runtime::instance::SCANNER_PUBLICATION_LEASE_TTL + Duration::from_millis(1)).await;
2562        tokio::task::yield_now().await;
2563        tokio::time::timeout(Duration::from_secs(1), &mut expiry_writer)
2564            .await
2565            .expect("movement writer should proceed after lease expiry")
2566            .expect("expiry writer task should not panic");
2567        assert!(
2568            store.validate_scanner_publication_lease(expiring_token, 0).await.is_err(),
2569            "an expired lease must not validate after its read guard is released"
2570        );
2571        assert!(!store.release_scanner_publication_lease(expiring_token).await);
2572    }
2573
2574    #[tokio::test]
2575    async fn scanner_publication_lease_rejects_a_new_movement_generation() {
2576        let store = build_store_with_ctx(Arc::new(InstanceContext::new()));
2577        let (token, generation) = store
2578            .acquire_scanner_publication_lease(0, crate::runtime::instance::SCANNER_PUBLICATION_LEASE_TTL)
2579            .await
2580            .expect("an idle store should grant a publication lease");
2581
2582        assert_eq!(store.ctx.advance_data_movement_generation(), Some(1));
2583        assert!(
2584            store.validate_scanner_publication_lease(token, generation).await.is_err(),
2585            "a lease from the prior movement generation must fail closed"
2586        );
2587        assert!(store.release_scanner_publication_lease(token).await);
2588    }
2589
2590    #[tokio::test]
2591    async fn scanner_publication_lease_rejects_stale_generation_before_install() {
2592        let store = build_store_with_ctx(Arc::new(InstanceContext::new()));
2593        let error = store
2594            .acquire_scanner_publication_lease(1, crate::runtime::instance::SCANNER_PUBLICATION_LEASE_TTL)
2595            .await
2596            .expect_err("a stale movement generation must not acquire a lease");
2597        assert!(error.to_string().contains("generation is stale"));
2598    }
2599
2600    #[tokio::test]
2601    async fn scanner_publication_lease_rejects_namespace_change_during_admission() {
2602        let ctx = Arc::new(InstanceContext::new());
2603        let store = build_store_with_ctx(ctx.clone());
2604        let snapshot_blocker = store.rebalance_meta.write().await;
2605        let mut acquire =
2606            Box::pin(store.acquire_scanner_publication_lease(0, crate::runtime::instance::SCANNER_PUBLICATION_LEASE_TTL));
2607        assert!(futures::poll!(&mut acquire).is_pending(), "acquisition reaches the blocked snapshot");
2608        assert!(ctx.data_movement_operation_gate().try_write_owned().is_err());
2609        drop(ctx.begin_namespace_commit());
2610        assert_eq!(ctx.namespace_commit_generation(), 2);
2611        assert!(!ctx.namespace_commits_pending());
2612        drop(snapshot_blocker);
2613        let error = tokio::time::timeout(Duration::from_secs(1), acquire)
2614            .await
2615            .expect("snapshot admission must finish")
2616            .expect_err("acquisition must preserve its original namespace generation");
2617        assert_eq!(error.to_string(), "Io error: scanner publication lease generation is stale");
2618        assert!(ctx.data_movement_operation_gate().try_write_owned().is_ok(), "no lease was installed");
2619    }
2620
2621    #[tokio::test]
2622    async fn scanner_publication_lease_rechecks_namespace_after_validate_snapshot() {
2623        let ctx = Arc::new(InstanceContext::new());
2624        let store = build_store_with_ctx(ctx.clone());
2625        let (token, generation) = store
2626            .acquire_scanner_publication_lease(0, crate::runtime::instance::SCANNER_PUBLICATION_LEASE_TTL)
2627            .await
2628            .expect("current lease");
2629        let snapshot_blocker = store.rebalance_meta.write().await;
2630        let mut validate = Box::pin(store.validate_scanner_publication_lease(token, generation));
2631        assert!(futures::poll!(&mut validate).is_pending(), "target guard queues its first snapshot read");
2632        let mut next_writer = Box::pin(store.rebalance_meta.write());
2633        assert!(futures::poll!(&mut next_writer).is_pending());
2634        drop(snapshot_blocker);
2635        // Fair lock order admits the first read, then this queued writer, then
2636        // validate's second snapshot. The first target check has already passed.
2637        assert!(futures::poll!(&mut validate).is_pending(), "validate reaches its second snapshot");
2638        let next_writer = next_writer.await;
2639        drop(ctx.begin_namespace_commit());
2640        assert_eq!(ctx.namespace_commit_generation(), 2);
2641        assert!(!ctx.namespace_commits_pending());
2642        drop(next_writer);
2643        let error = tokio::time::timeout(Duration::from_secs(1), validate)
2644            .await
2645            .expect("validation must finish without nesting movement read locks")
2646            .expect_err("the final snapshot must reject a completed namespace commit");
2647        assert_eq!(error.to_string(), "Io error: scanner publication lease generation is stale");
2648        assert!(
2649            ctx.data_movement_operation_gate().try_write_owned().is_err(),
2650            "stale lookup retains the lease permit"
2651        );
2652        assert!(store.release_scanner_publication_lease(token).await);
2653        assert!(ctx.data_movement_operation_gate().try_write_owned().is_ok());
2654    }
2655
2656    #[tokio::test]
2657    async fn scanner_publication_lease_rejects_namespace_generation_exhaustion() {
2658        let ctx = Arc::new(InstanceContext::new());
2659        let store = build_store_with_ctx(ctx.clone());
2660        let (token, generation) = store
2661            .acquire_scanner_publication_lease(0, crate::runtime::instance::SCANNER_PUBLICATION_LEASE_TTL)
2662            .await
2663            .expect("current lease");
2664        ctx.set_namespace_commit_generation_for_test(u64::MAX);
2665        assert_eq!(store.scanner_namespace_mutation_generation(), u64::MAX);
2666        let acquire = store
2667            .acquire_scanner_publication_lease(generation, crate::runtime::instance::SCANNER_PUBLICATION_LEASE_TTL)
2668            .await
2669            .expect_err("exhausted namespace cannot become a new lease baseline");
2670        assert_eq!(acquire.to_string(), "Io error: scanner publication lease generation is stale");
2671        let validate = store.validate_scanner_publication_lease(token, generation).await;
2672        let target = store.acquire_scanner_publication_lease_guard(token).await;
2673        assert!(validate.is_err() && target.is_err(), "exhaustion rejects both old-token entrances");
2674        assert!(
2675            ctx.data_movement_operation_gate().try_write_owned().is_err(),
2676            "rejection must retain the old permit"
2677        );
2678        assert!(store.release_scanner_publication_lease(token).await);
2679        assert!(ctx.data_movement_operation_gate().try_write_owned().is_ok());
2680    }
2681
2682    #[tokio::test(start_paused = true)]
2683    async fn scanner_publication_commit_scope_owns_permit_until_terminal_drain() {
2684        let store = build_store_with_ctx(Arc::new(InstanceContext::new()));
2685        let scope = store
2686            .scanner_data_usage_publication_commit_scope(
2687                0,
2688                tokio::time::Instant::now() + Duration::from_secs(30),
2689                vec![Uuid::new_v4()],
2690            )
2691            .await
2692            .expect("idle storage should grant a publication scope");
2693        assert_eq!(scope.state(), crate::object_api::ScannerPublicationCommitState::Admitted);
2694        assert_eq!(scope.remote_lease_tokens().len(), 1);
2695
2696        let gate = store.ctx.data_movement_operation_gate();
2697        let writer = tokio::spawn(async move { gate.write_owned().await });
2698        tokio::task::yield_now().await;
2699        assert!(!writer.is_finished(), "the scope must own its movement permit after the caller returns");
2700
2701        scope.cancel();
2702        assert!(scope.mark_aborted_before_commit());
2703        assert_eq!(
2704            scope.wait_for_completion().await,
2705            crate::object_api::ScannerPublicationCommitState::AbortedBeforeCommit
2706        );
2707        assert!(scope.release_movement_permit().await);
2708        tokio::time::timeout(Duration::from_secs(1), writer)
2709            .await
2710            .expect("movement writer should proceed after the scope drains")
2711            .expect("movement writer task should not panic");
2712    }
2713
2714    #[tokio::test(start_paused = true)]
2715    async fn scanner_publication_commit_scope_rejects_late_start_and_keeps_indeterminate_permit() {
2716        let store = build_store_with_ctx(Arc::new(InstanceContext::new()));
2717        let scope = store
2718            .scanner_data_usage_publication_commit_scope(0, tokio::time::Instant::now() + Duration::from_secs(1), Vec::new())
2719            .await
2720            .expect("idle storage should grant a publication scope");
2721        tokio::time::advance(Duration::from_secs(1)).await;
2722        assert_eq!(
2723            scope.try_begin(),
2724            Err(crate::object_api::ScannerPublicationCommitStartError::DeadlineExceeded)
2725        );
2726        assert!(
2727            !scope.release_movement_permit().await,
2728            "an admitted scope is not safe to release before owner resolution"
2729        );
2730        assert!(scope.mark_aborted_before_commit());
2731        assert!(scope.release_movement_permit().await);
2732
2733        let scope = store
2734            .scanner_data_usage_publication_commit_scope(0, tokio::time::Instant::now() + Duration::from_secs(30), Vec::new())
2735            .await
2736            .expect("a second idle publication scope should be granted");
2737        scope.try_begin().expect("scope should enter the mutation state");
2738        scope.cancel();
2739        assert!(scope.mark_indeterminate());
2740        assert_eq!(
2741            scope.wait_for_completion().await,
2742            crate::object_api::ScannerPublicationCommitState::Indeterminate
2743        );
2744        assert!(!scope.release_movement_permit().await, "indeterminate mutation must retain the permit");
2745    }
2746
2747    #[tokio::test]
2748    async fn scanner_publication_scope_guard_classifies_early_returns_conservatively() {
2749        let store = build_store_with_ctx(Arc::new(InstanceContext::new()));
2750        let permit = store.ctx.data_movement_operation_gate().read_owned().await;
2751        let scope = ScannerPublicationCommitScope::new_storage_owned(
2752            0,
2753            tokio::time::Instant::now() + Duration::from_secs(30),
2754            Vec::new(),
2755            permit,
2756        );
2757        {
2758            let _guard = crate::object_api::ScannerPublicationCommitScopeGuard::new(scope.clone());
2759        }
2760        assert_eq!(scope.state(), crate::object_api::ScannerPublicationCommitState::AbortedBeforeCommit);
2761        assert!(scope.release_movement_permit().await);
2762
2763        let permit = store.ctx.data_movement_operation_gate().read_owned().await;
2764        let scope = ScannerPublicationCommitScope::new_storage_owned(
2765            0,
2766            tokio::time::Instant::now() + Duration::from_secs(30),
2767            Vec::new(),
2768            permit,
2769        );
2770        scope.try_begin().expect("scope should enter the mutation state");
2771        {
2772            let _guard = crate::object_api::ScannerPublicationCommitScopeGuard::new(scope.clone());
2773        }
2774        assert_eq!(scope.state(), crate::object_api::ScannerPublicationCommitState::Indeterminate);
2775        assert!(!scope.release_movement_permit().await);
2776    }
2777
2778    #[tokio::test]
2779    async fn scanner_target_guard_keeps_movement_writer_fenced_after_lease_release() {
2780        let store = build_store_with_ctx(Arc::new(InstanceContext::new()));
2781        let (token, _) = store
2782            .acquire_scanner_publication_lease(0, crate::runtime::instance::SCANNER_PUBLICATION_LEASE_TTL)
2783            .await
2784            .expect("an idle store should grant a publication lease");
2785        let target_guard = store
2786            .acquire_scanner_publication_lease_guard(token)
2787            .await
2788            .expect("the target-side rename should acquire its short guard");
2789        assert!(store.release_scanner_publication_lease(token).await);
2790
2791        let movement_gate = store.ctx.data_movement_operation_gate();
2792        let movement_writer = tokio::spawn(async move { movement_gate.write_owned().await });
2793        tokio::task::yield_now().await;
2794        assert!(!movement_writer.is_finished(), "the target guard must span the rename operation");
2795        drop(target_guard);
2796        tokio::time::timeout(std::time::Duration::from_secs(1), movement_writer)
2797            .await
2798            .expect("movement writer should proceed after the target rename guard is dropped")
2799            .expect("movement writer task should not panic");
2800    }
2801
2802    #[tokio::test]
2803    async fn scanner_publication_lease_rejects_restart_aba_token() {
2804        let first_store = build_store_with_ctx(Arc::new(InstanceContext::new()));
2805        let (token, generation) = first_store
2806            .acquire_scanner_publication_lease(0, crate::runtime::instance::SCANNER_PUBLICATION_LEASE_TTL)
2807            .await
2808            .expect("the initial instance should grant a publication lease");
2809        first_store
2810            .validate_scanner_publication_lease(token, generation)
2811            .await
2812            .expect("the current instance should validate its own live token");
2813        first_store
2814            .acquire_scanner_publication_lease_guard(token)
2815            .await
2816            .expect("the current instance should admit the target-side rename");
2817
2818        // A restarted storage instance starts its local generation at zero,
2819        // but its process-owned lease table is empty.  The old token must not
2820        // pass validation just because the numeric generation matches again.
2821        let restarted_store = build_store_with_ctx(Arc::new(InstanceContext::new()));
2822        let error = restarted_store
2823            .validate_scanner_publication_lease(token, generation)
2824            .await
2825            .expect_err("a token from a prior instance must fail closed after restart");
2826        assert!(error.to_string().contains("unknown or expired"));
2827        let error = restarted_store
2828            .acquire_scanner_publication_lease_guard(token)
2829            .await
2830            .expect_err("a restarted instance must reject the target-side rename token");
2831        assert!(error.to_string().contains("unknown or expired"));
2832    }
2833
2834    #[tokio::test]
2835    async fn movement_generation_notifies_waiters_and_fails_closed_at_maximum() {
2836        let store = build_store_with_ctx(Arc::new(InstanceContext::new()));
2837        let notify = store.scanner_data_movement_changed();
2838        let notified = notify.notified();
2839        tokio::pin!(notified);
2840        notified.as_mut().enable();
2841
2842        assert_eq!(store.ctx.advance_data_movement_operation_epoch(), 1);
2843        tokio::time::timeout(std::time::Duration::from_secs(1), notified)
2844            .await
2845            .expect("movement transition should wake scanner waiters");
2846        assert_eq!(store.scanner_data_movement_generation(), 1);
2847
2848        store.ctx.set_data_movement_generation_for_test(u64::MAX - 1);
2849        assert_eq!(store.ctx.advance_data_movement_generation(), Some(u64::MAX));
2850        assert!(store.scanner_data_movement_generation_exhausted());
2851        assert_eq!(store.ctx.advance_data_movement_generation(), None);
2852        assert!(
2853            store.scanner_data_usage_publication_admission_guard().await.is_none(),
2854            "generation exhaustion must close publication admission"
2855        );
2856    }
2857
2858    #[tokio::test]
2859    async fn scanner_data_usage_publication_epoch_releases_gate_before_backend_io() {
2860        let store = build_store_with_ctx(Arc::new(InstanceContext::new()));
2861        let epoch = store
2862            .scanner_data_usage_publication_epoch()
2863            .await
2864            .expect("idle store should expose a publication epoch");
2865        assert_eq!(epoch, 0);
2866
2867        let operation_gate = store.ctx.data_movement_operation_gate();
2868        let _movement_guard = tokio::time::timeout(Duration::from_secs(1), operation_gate.write())
2869            .await
2870            .expect("epoch capture must not hold the movement gate across backend I/O");
2871    }
2872
2873    #[tokio::test]
2874    async fn scanner_data_usage_publication_admission_blocks_active_rebalance_snapshot() {
2875        let store = build_store_with_ctx(Arc::new(InstanceContext::new()));
2876        *store.rebalance_meta.write().await = Some(RebalanceMeta {
2877            pool_stats: vec![crate::services::rebalance::RebalanceStats {
2878                participating: true,
2879                info: crate::services::rebalance::RebalanceInfo {
2880                    status: crate::services::rebalance::RebalStatus::Started,
2881                    ..Default::default()
2882                },
2883                ..Default::default()
2884            }],
2885            ..Default::default()
2886        });
2887
2888        assert!(
2889            store.scanner_data_usage_publication_admission_guard().await.is_none(),
2890            "active rebalance must fail closed at the storage-owned admission boundary"
2891        );
2892    }
2893
2894    #[tokio::test]
2895    async fn scanner_publication_epoch_exhaustion_fails_closed_after_max() {
2896        let store = build_store_with_ctx(Arc::new(InstanceContext::new()));
2897        store.ctx.set_data_movement_operation_epoch_for_test(u64::MAX - 1);
2898
2899        assert_eq!(store.ctx.advance_data_movement_operation_epoch(), u64::MAX);
2900        assert!(store.ctx.data_movement_operation_epoch_exhausted());
2901        assert!(
2902            store.scanner_data_usage_publication_admission_guard().await.is_none(),
2903            "publication must fail closed at the reserved terminal epoch"
2904        );
2905
2906        assert_eq!(store.ctx.advance_data_movement_operation_epoch(), u64::MAX);
2907        assert!(store.ctx.data_movement_operation_epoch_exhausted());
2908        assert!(store.scanner_data_usage_publication_blocked().await);
2909    }
2910
2911    // The object graph is the isolation carrier: two ECStore instances holding
2912    // distinct contexts report independent erasure state through their real
2913    // `&self` accessors — no cross-contamination.
2914    #[tokio::test]
2915    async fn instance_context_carrier_isolates_two_stores() {
2916        let ctx_a = Arc::new(InstanceContext::new());
2917        let ctx_b = Arc::new(InstanceContext::new());
2918        ctx_a.update_erasure_type(SetupType::DistErasure).await;
2919        ctx_b.update_erasure_type(SetupType::ErasureSD).await;
2920        ctx_a.set_endpoints(EndpointServerPools::from(vec![PoolEndpoints {
2921            legacy: false,
2922            set_count: 1,
2923            drives_per_set: 1,
2924            endpoints: Endpoints::default(),
2925            cmd_line: "instance-a".to_string(),
2926            platform: String::new(),
2927        }]));
2928        ctx_b.set_endpoints(EndpointServerPools::from(vec![PoolEndpoints {
2929            legacy: true,
2930            set_count: 2,
2931            drives_per_set: 2,
2932            endpoints: Endpoints::default(),
2933            cmd_line: "instance-b".to_string(),
2934            platform: String::new(),
2935        }]));
2936
2937        let store_a = build_store_with_ctx(ctx_a);
2938        let store_b = build_store_with_ctx(ctx_b);
2939
2940        // store_a: distributed erasure (implies is_erasure), not single-drive.
2941        assert!(store_a.setup_is_erasure().await);
2942        assert!(store_a.setup_is_dist_erasure().await);
2943        assert!(!store_a.setup_is_erasure_sd().await);
2944
2945        // store_b: single-drive erasure only.
2946        assert!(store_b.setup_is_erasure_sd().await);
2947        assert!(!store_b.setup_is_erasure().await);
2948        assert!(!store_b.setup_is_dist_erasure().await);
2949        let endpoints_a = store_a.instance_endpoints().expect("instance A endpoints");
2950        let endpoints_b = store_b.instance_endpoints().expect("instance B endpoints");
2951        assert_eq!(endpoints_a.as_ref()[0].set_count, 1);
2952        assert_eq!(endpoints_b.as_ref()[0].set_count, 2);
2953        assert!(!endpoints_a.as_ref()[0].legacy);
2954        assert!(endpoints_b.as_ref()[0].legacy);
2955    }
2956
2957    // The production/test constructors ADOPT the process bootstrap context
2958    // (same Arc), so a startup write recorded before the store existed is
2959    // visible through the store afterward — single-instance behavior preserved.
2960    #[tokio::test]
2961    async fn store_adopts_bootstrap_context() {
2962        let store = build_store_with_ctx(crate::runtime::instance::bootstrap_ctx());
2963        assert!(
2964            Arc::ptr_eq(&store.ctx, &crate::runtime::instance::bootstrap_ctx()),
2965            "store built via adoption must share the bootstrap context Arc"
2966        );
2967    }
2968
2969    #[tokio::test]
2970    async fn test_has_space_for() {
2971        let disk_infos = vec![None, None]; // No actual disk info
2972
2973        let result = crate::layout::pool_space::has_space_for(&disk_infos, 1024).await;
2974        // Should fail due to no valid disk info
2975        assert!(result.is_err());
2976    }
2977
2978    #[tokio::test]
2979    async fn test_find_local_disk() {
2980        let result = peer::find_local_disk("/nonexistent/path").await;
2981        assert!(result.is_none(), "Should return None for nonexistent path");
2982    }
2983
2984    #[tokio::test]
2985    #[serial]
2986    async fn test_find_local_disk_by_ref_backfills_uuid_map() {
2987        reset_local_disk_test_state().await;
2988
2989        let temp_dir = TempDir::new().expect("create temp dir for local disk ref test");
2990        let disk_paths = (0..4)
2991            .map(|idx| temp_dir.path().join(format!("disk{}", idx + 1)))
2992            .collect::<Vec<_>>();
2993        for disk_path in &disk_paths {
2994            std::fs::create_dir_all(disk_path).expect("create disk path");
2995        }
2996
2997        let mut endpoints = Vec::new();
2998        for (idx, disk_path) in disk_paths.iter().enumerate() {
2999            let mut endpoint = Endpoint::try_from(disk_path.to_str().expect("disk path to str")).expect("endpoint");
3000            endpoint.set_pool_index(0);
3001            endpoint.set_set_index(0);
3002            endpoint.set_disk_index(idx);
3003            endpoints.push(endpoint);
3004        }
3005
3006        let endpoint_pools = EndpointServerPools(vec![PoolEndpoints {
3007            legacy: false,
3008            set_count: 1,
3009            drives_per_set: 4,
3010            endpoints: Endpoints::from(endpoints),
3011            cmd_line: "find-local-disk-by-ref-test".to_string(),
3012            platform: "test".to_string(),
3013        }]);
3014
3015        init_local_disks(endpoint_pools.clone()).await.expect("init local disks");
3016
3017        let (mut disks, errs) = init_disks(
3018            &endpoint_pools.as_ref().first().expect("pool endpoints").endpoints,
3019            &DiskOption {
3020                cleanup: true,
3021                health_check: false,
3022            },
3023        )
3024        .await;
3025
3026        assert!(errs.iter().all(|err| err.is_none()), "disk init should succeed: {errs:?}");
3027        connect_load_init_formats(true, &mut disks, 1, 4, None)
3028            .await
3029            .expect("initialize format metadata");
3030
3031        clear_local_disk_id_map_for_test().await;
3032
3033        let local_disks = all_local_disk().await;
3034        let first_disk = local_disks.first().expect("local disk exists");
3035        let disk_id = first_disk
3036            .get_disk_id()
3037            .await
3038            .expect("get disk id should succeed")
3039            .expect("disk id should exist");
3040
3041        let found = find_local_disk_by_ref(&disk_id.to_string()).await;
3042        assert!(found.is_some(), "disk lookup by id should backfill cache");
3043        assert_eq!(local_disk_path_by_id(&disk_id).await, Some(first_disk.endpoint().to_string()));
3044
3045        reset_local_disk_test_state().await;
3046    }
3047
3048    #[tokio::test]
3049    async fn test_all_local_disk_path() {
3050        let paths = all_local_disk_path().await;
3051        // Should return empty or some paths depending on global state
3052        assert!(paths.is_empty() || !paths.is_empty());
3053    }
3054
3055    #[tokio::test]
3056    async fn test_all_local_disk() {
3057        let disks = all_local_disk().await;
3058        // Should return empty or some disks depending on global state
3059        assert!(disks.is_empty() || !disks.is_empty());
3060    }
3061
3062    #[test]
3063    fn test_should_not_enqueue_transition_for_internal_metadata_bucket() {
3064        let oi = ObjectInfo {
3065            bucket: RUSTFS_META_BUCKET.to_string(),
3066            name: format!("{BUCKET_META_PREFIX}/bucket/.metadata.bin"),
3067            ..Default::default()
3068        };
3069
3070        assert!(!should_enqueue_transition_immediately(&oi));
3071    }
3072}