1#![allow(clippy::map_entry)]
16
17use crate::bucket::lifecycle::bucket_lifecycle_audit::LcEventSrc;
18use crate::bucket::lifecycle::bucket_lifecycle_ops::{
19 enqueue_immediate_expiry, enqueue_transition_immediate, init_background_expiry,
20};
21use crate::bucket::metadata_sys;
22use crate::bucket::utils::check_abort_multipart_args;
23use crate::bucket::utils::check_complete_multipart_args;
24use crate::bucket::utils::check_copy_obj_args;
25use crate::bucket::utils::check_del_obj_args;
26use crate::bucket::utils::check_get_obj_args;
27use crate::bucket::utils::check_list_multipart_args;
28use crate::bucket::utils::check_list_parts_args;
29use crate::bucket::utils::check_new_multipart_args;
30use crate::bucket::utils::check_object_args;
31use crate::bucket::utils::check_put_object_args;
32use crate::bucket::utils::check_put_object_part_args;
33use crate::bucket::utils::{check_valid_bucket_name, check_valid_bucket_name_strict, is_meta_bucketname};
34use crate::cluster::rpc::{RemoteClient, S3PeerSys};
35use crate::config::storageclass;
36use crate::core::pools::{DecommissionCanceler, PoolMeta, PoolMetaWriteState};
37use crate::disk::endpoint::{Endpoint, EndpointType};
38use crate::disk::{DiskAPI, DiskInfo, DiskInfoOptions};
39use crate::error::{Error, Result};
40use crate::error::{
41 StorageError, is_err_bucket_exists, is_err_invalid_upload_id, is_err_object_not_found, is_err_read_quorum,
42 is_err_strict_volume_not_found, is_err_version_not_found, to_object_err,
43};
44use crate::runtime::global::DISK_RESERVE_FRACTION;
45use crate::runtime::instance::InstanceContext;
46use crate::runtime::sources as runtime_sources;
47use crate::services::rebalance::{RebalStatus, RebalanceMeta, is_rebalance_conflicting_with_decommission};
48use crate::storage_api_contracts::{
49 bucket::{BucketInfo, BucketOperations, BucketOptions, DeleteBucketOptions, MakeBucketOptions},
50 list::{StorageListObjectVersionsInfo, StorageListObjectsV2Info, StorageObjectInfoOrErr, StorageWalkOptions},
51 multipart::{CompletePart, ListMultipartsInfo, ListPartsInfo, MultipartInfo, MultipartUploadResult, PartInfo},
52 object::{DeletedObject, ObjectToDelete},
53 range::HTTPRangeSpec,
54};
55use crate::store::init_format::{check_disk_fatal_errs, ec_drives_no_config};
56use crate::{
57 bucket::{lifecycle::bucket_lifecycle_ops::TransitionState, metadata::BucketMetadata},
58 core::sets::Sets,
59 disk::{BUCKET_META_PREFIX, DiskOption, DiskStore, RUSTFS_META_BUCKET},
60 layout::endpoints::EndpointServerPools,
61 object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader, ScannerPublicationCommitScope},
62};
63use futures::future::join_all;
64use http::HeaderMap;
65use lazy_static::lazy_static;
66use rand::RngExt as _;
67use rustfs_config::server_config::Config;
68use rustfs_filemeta::{FileInfo, FileMeta};
69use rustfs_heal_contracts::heal_channel::{HealItemType, HealOpts};
70use rustfs_lock::{LocalClient, LockClient, NamespaceLockWrapper};
71use rustfs_madmin::heal_commands::HealResultItem;
72use rustfs_utils::path::{decode_dir_object, encode_dir_object, path_join_buf};
73use s3s::dto::{BucketVersioningStatus, ObjectLockConfiguration, ObjectLockEnabled, VersioningConfiguration};
74use std::net::SocketAddr;
75use std::process::exit;
76use std::{collections::HashMap, sync::Arc, time::Duration};
77use time::OffsetDateTime;
78use tokio::select;
79use tokio::sync::{Mutex, RwLock};
80use tokio::time::sleep;
81use tokio_util::sync::CancellationToken;
82use tracing::{debug, error, info, instrument, warn};
83use uuid::Uuid;
84
85type ListObjectsV2Info = StorageListObjectsV2Info<ObjectInfo>;
86type ListObjectVersionsInfo = StorageListObjectVersionsInfo<ObjectInfo>;
87type ObjectInfoOrErr = StorageObjectInfoOrErr<ObjectInfo, Error>;
88type WalkOptions = StorageWalkOptions<fn(&FileInfo) -> bool>;
89
90pub const SCANNER_PUBLICATION_LEASE_TTL_MS: u64 = 60_000;
91pub(crate) const BUCKET_DELETE_XLMETA_DIAGNOSTIC_MAX_BYTES: u64 = 1024 * 1024;
92pub(crate) const BUCKET_DELETE_DIAGNOSTIC_MAX_ENTRIES: usize = 4_096;
93pub(crate) const BUCKET_DELETE_DIAGNOSTIC_MAX_ELAPSED: Duration = Duration::from_millis(100);
94
95#[derive(Debug)]
96pub(crate) struct BucketDeleteDiagnosticBudget {
97 deadline: Option<tokio::time::Instant>,
98 max_elapsed: Duration,
99 entries_remaining: usize,
100 #[cfg(test)]
101 first_io_delay: Option<(Duration, Arc<std::sync::atomic::AtomicBool>)>,
102}
103
104impl BucketDeleteDiagnosticBudget {
105 pub(crate) fn new() -> Self {
106 Self::with_limits(BUCKET_DELETE_DIAGNOSTIC_MAX_ENTRIES, BUCKET_DELETE_DIAGNOSTIC_MAX_ELAPSED)
107 }
108
109 fn with_limits(entries: usize, elapsed: Duration) -> Self {
110 Self {
111 deadline: None,
112 max_elapsed: elapsed,
113 entries_remaining: entries,
114 #[cfg(test)]
115 first_io_delay: None,
116 }
117 }
118
119 #[cfg(test)]
120 fn with_first_io_delay(mut self, delay: Duration, started: Arc<std::sync::atomic::AtomicBool>) -> Self {
121 self.first_io_delay = Some((delay, started));
122 self
123 }
124
125 fn deadline(&mut self) -> tokio::time::Instant {
126 let max_elapsed = self.max_elapsed;
127 *self.deadline.get_or_insert_with(|| tokio::time::Instant::now() + max_elapsed)
128 }
129
130 fn claim_entry(&mut self) -> bool {
131 if self.entries_remaining == 0 {
132 return false;
133 }
134 let deadline = self.deadline();
135 if tokio::time::Instant::now() >= deadline {
136 return false;
137 }
138 self.entries_remaining -= 1;
139 true
140 }
141
142 async fn run_io<T, F>(&mut self, future: F) -> std::io::Result<Option<T>>
143 where
144 F: std::future::Future<Output = std::io::Result<T>>,
145 {
146 let deadline = self.deadline();
147 if tokio::time::Instant::now() >= deadline {
148 return Ok(None);
149 }
150 #[cfg(test)]
151 let first_io_delay = self.first_io_delay.take();
152 #[cfg(test)]
153 let timeout_result = tokio::time::timeout_at(deadline, async move {
154 if let Some((delay, started)) = first_io_delay {
155 started.store(true, std::sync::atomic::Ordering::SeqCst);
156 tokio::time::sleep(delay).await;
157 }
158 future.await
159 })
160 .await;
161 #[cfg(not(test))]
162 let timeout_result = tokio::time::timeout_at(deadline, future).await;
163 match timeout_result {
164 Ok(result) => result.map(Some),
165 Err(_) => Ok(None),
166 }
167 }
168}
169
170#[derive(Debug, Default, Clone, PartialEq, Eq)]
171pub(crate) struct BucketMetadataLessResidue {
172 pub(crate) xlmeta_found: bool,
173 pub(crate) xlmeta_blocker: Option<BucketDeleteBlockerKind>,
174 pub(crate) files: usize,
175 pub(crate) uuid_data_dirs: usize,
176 pub(crate) entries_scanned: usize,
177 pub(crate) diagnostic_bytes_read: u64,
178 pub(crate) diagnostic_truncated: bool,
179 pub(crate) sample: Option<String>,
180}
181
182#[derive(Debug, Clone, Copy, PartialEq, Eq)]
183pub(crate) enum BucketDeleteBlockerKind {
184 VisibleVersion,
185 TierFreeVersion,
186 UnknownXlMeta,
187 OrphanDirectory,
188 DiagnosticBudgetExceeded,
189}
190
191impl BucketDeleteBlockerKind {
192 pub(crate) const fn as_str(self) -> &'static str {
193 match self {
194 Self::VisibleVersion => "visible_version",
195 Self::TierFreeVersion => "tier_free_version",
196 Self::UnknownXlMeta => "unknown_xlmeta",
197 Self::OrphanDirectory => "orphan_directory",
198 Self::DiagnosticBudgetExceeded => "diagnostic_budget_exceeded",
199 }
200 }
201
202 pub(crate) const fn is_client_visible(self) -> bool {
215 matches!(self, Self::VisibleVersion | Self::TierFreeVersion)
216 }
217}
218
219impl BucketMetadataLessResidue {
220 pub(crate) fn has_residue_without_xlmeta(&self) -> bool {
221 !self.xlmeta_found && (self.files > 0 || self.diagnostic_truncated)
222 }
223
224 pub(crate) fn describe(&self) -> String {
225 let sample = self.sample.as_deref().unwrap_or("<none>");
226 format!(
227 "metadata-less on-disk residue remains after empty-bucket verification: files={}, uuid_data_dirs={}, entries_scanned={}, diagnostic_bytes_read={}, diagnostic_truncated={}, sample={sample}",
228 self.files, self.uuid_data_dirs, self.entries_scanned, self.diagnostic_bytes_read, self.diagnostic_truncated,
229 )
230 }
231}
232
233pub(crate) async fn has_xlmeta_files(path: &std::path::Path) -> std::io::Result<bool> {
236 use crate::disk::STORAGE_FORMAT_FILE;
237 use tokio::fs;
238
239 let mut stack = vec![path.to_path_buf()];
240
241 while let Some(current_path) = stack.pop() {
242 let mut entries = match fs::read_dir(¤t_path).await {
243 Ok(entries) => entries,
244 Err(err) if err.kind() == std::io::ErrorKind::NotFound => continue,
245 Err(err) => return Err(err),
246 };
247
248 while let Some(entry) = entries.next_entry().await? {
249 let file_name = entry.file_name();
250 let file_name_str = file_name.to_string_lossy();
251
252 if file_name_str == STORAGE_FORMAT_FILE {
254 return Ok(true);
255 }
256
257 if entry.file_type().await?.is_dir() {
259 stack.push(entry.path());
260 }
261 }
262 }
263
264 Ok(false)
265}
266
267#[cfg(test)]
268pub(crate) async fn scan_metadata_less_residue(path: &std::path::Path) -> std::io::Result<BucketMetadataLessResidue> {
269 let mut budget = BucketDeleteDiagnosticBudget::new();
270 scan_metadata_less_residue_with_budget(path, &mut budget).await
271}
272
273async fn scan_metadata_less_residue_with_budget(
274 path: &std::path::Path,
275 budget: &mut BucketDeleteDiagnosticBudget,
276) -> std::io::Result<BucketMetadataLessResidue> {
277 use crate::disk::STORAGE_FORMAT_FILE;
278 use tokio::fs;
279 use tokio::io::AsyncReadExt as _;
280
281 let mut scan = BucketMetadataLessResidue::default();
282 let mut stack = vec![path.to_path_buf()];
283
284 let mark_budget_exhausted = |scan: &mut BucketMetadataLessResidue| {
285 scan.diagnostic_truncated = true;
286 scan.sample.get_or_insert_with(|| "<diagnostic-budget-exceeded>".to_string());
287 };
288
289 while let Some(current_path) = stack.pop() {
290 let mut entries = match budget.run_io(fs::read_dir(¤t_path)).await {
291 Ok(Some(entries)) => entries,
292 Ok(None) => {
293 mark_budget_exhausted(&mut scan);
294 return Ok(scan);
295 }
296 Err(err) if err.kind() == std::io::ErrorKind::NotFound => continue,
297 Err(err) => return Err(err),
298 };
299
300 loop {
301 let entry = match budget.run_io(entries.next_entry()).await? {
302 Some(Some(entry)) => entry,
303 Some(None) => break,
304 None => {
305 mark_budget_exhausted(&mut scan);
306 return Ok(scan);
307 }
308 };
309 if !budget.claim_entry() {
310 mark_budget_exhausted(&mut scan);
311 return Ok(scan);
312 }
313 scan.entries_scanned = scan.entries_scanned.saturating_add(1);
314 let Some(file_type) = budget.run_io(entry.file_type()).await? else {
315 mark_budget_exhausted(&mut scan);
316 return Ok(scan);
317 };
318 let file_name = entry.file_name();
319 let file_name_str = file_name.to_string_lossy();
320
321 if file_name_str == STORAGE_FORMAT_FILE {
322 scan.xlmeta_found = true;
323 if scan.xlmeta_blocker.is_none() {
324 let entry_path = entry.path();
325 let Some(metadata) = budget.run_io(fs::metadata(&entry_path)).await? else {
326 mark_budget_exhausted(&mut scan);
327 scan.xlmeta_blocker = Some(BucketDeleteBlockerKind::DiagnosticBudgetExceeded);
328 return Ok(scan);
329 };
330 scan.xlmeta_blocker = Some(if metadata.len() > BUCKET_DELETE_XLMETA_DIAGNOSTIC_MAX_BYTES {
331 BucketDeleteBlockerKind::UnknownXlMeta
332 } else {
333 match budget.run_io(fs::File::open(&entry_path)).await {
334 Ok(Some(file)) => {
335 let mut data = Vec::new();
336 let read = budget
337 .run_io(file.take(BUCKET_DELETE_XLMETA_DIAGNOSTIC_MAX_BYTES).read_to_end(&mut data))
338 .await;
339 scan.diagnostic_bytes_read = data.len() as u64;
340 match read {
341 Ok(Some(_)) => match FileMeta::load(&data) {
342 Ok(meta)
343 if !meta.versions.is_empty()
344 && meta.versions.iter().all(|version| version.header.free_version()) =>
345 {
346 BucketDeleteBlockerKind::TierFreeVersion
347 }
348 Ok(meta) if !meta.versions.is_empty() => BucketDeleteBlockerKind::VisibleVersion,
349 Ok(_) | Err(_) => BucketDeleteBlockerKind::UnknownXlMeta,
350 },
351 Ok(None) => {
352 mark_budget_exhausted(&mut scan);
353 BucketDeleteBlockerKind::DiagnosticBudgetExceeded
354 }
355 Err(_) => BucketDeleteBlockerKind::UnknownXlMeta,
356 }
357 }
358 Ok(None) => {
359 mark_budget_exhausted(&mut scan);
360 BucketDeleteBlockerKind::DiagnosticBudgetExceeded
361 }
362 Err(_) => BucketDeleteBlockerKind::UnknownXlMeta,
363 }
364 });
365 let sample = entry_path
366 .strip_prefix(path)
367 .unwrap_or(entry_path.as_path())
368 .to_string_lossy()
369 .replace(std::path::MAIN_SEPARATOR, "/");
370 scan.sample = Some(sample);
371 }
372 return Ok(scan);
373 }
374
375 if file_type.is_dir() {
376 if Uuid::parse_str(&file_name_str).is_ok() {
377 scan.uuid_data_dirs = scan.uuid_data_dirs.saturating_add(1);
378 }
379 stack.push(entry.path());
380 } else {
381 scan.files = scan.files.saturating_add(1);
382 if scan.sample.is_none() {
383 let entry_path = entry.path();
384 let sample = entry_path
385 .strip_prefix(path)
386 .unwrap_or(entry_path.as_path())
387 .to_string_lossy()
388 .replace(std::path::MAIN_SEPARATOR, "/");
389 scan.sample = Some(sample);
390 }
391 }
392 }
393 }
394
395 Ok(scan)
396}
397
398async fn enqueue_transition_after_write(result: Result<ObjectInfo>, src: LcEventSrc) -> Result<ObjectInfo> {
399 match result {
400 Ok(oi) => {
401 if should_enqueue_transition_immediately(&oi) {
402 enqueue_transition_immediate(&oi, src.clone()).await;
403 enqueue_immediate_expiry(&oi, src).await;
404 }
405 Ok(oi)
406 }
407 Err(err) => Err(err),
408 }
409}
410
411fn should_enqueue_transition_immediately(oi: &ObjectInfo) -> bool {
412 !is_meta_bucketname(&oi.bucket)
413}
414
415const MAX_UPLOADS_LIST: usize = 10000;
416
417mod bucket;
418mod bucket_fence;
419pub(crate) use bucket::await_bucket_namespace_operation;
420pub use bucket_fence::BucketIncarnationFenceGuard;
421mod heal;
422pub(crate) use heal::bucket_heal_scope;
423pub use heal::{HealObjectAbsenceProof, HealObjectStorageResult};
424mod heal_walk;
425pub use heal_walk::HealWalkVersion;
426mod init;
427pub(crate) mod init_format;
428pub(crate) mod list_objects;
429mod multipart;
430mod object;
431#[cfg(feature = "test-util")]
432pub use object::DeleteAfterObjectLockSnapshotBarrier;
433pub(crate) use object::{
434 DecommissionFixedReadAnchor, ObjectLockDiagGuard, RemoteTuplePublicationCommitGuard, RemoteTuplePublicationFence,
435 SourceCleanupMutationFence, tiered_data_movement_source_matches,
436};
437pub use object::{
438 PrepareSelectObjectSnapshotError, PreparedGetObjectReader, SelectObjectSnapshot, SelectObjectSnapshotReadError,
439 SnapshotConsistencyError,
440};
441mod peer;
442mod rebalance;
443pub(crate) mod utils;
444
445use peer::init_local_peer;
446pub use peer::{
447 BootstrapLocalTarget, all_local_disk, all_local_disk_path, find_local_disk_by_ref, get_disk_infos, init_local_disks,
448 init_local_disks_with_instance_ctx, init_lock_clients, prewarm_local_disk_id_map,
449 prewarm_local_disk_id_map_with_instance_ctx,
450};
451
452pub struct ECStore {
453 pub id: Uuid,
454 pub disk_map: HashMap<usize, Vec<Option<DiskStore>>>,
456 pub pools: Vec<Arc<Sets>>,
457 pub peer_sys: S3PeerSys,
458 pub pool_meta: RwLock<PoolMeta>,
460 pub rebalance_meta: RwLock<Option<RebalanceMeta>>,
461 pub decommission_cancelers: RwLock<Vec<Option<DecommissionCanceler>>>,
462 pub(crate) start_gate: Mutex<()>,
468 pub(crate) pool_meta_save_gate: Mutex<PoolMetaWriteState>,
474 pub(crate) ctx: Arc<InstanceContext>,
482 pub(crate) bucket_fence_registry: Arc<bucket_fence::BucketFenceRegistry>,
485}
486
487const METRIC_SCANNER_DATA_MOVEMENT_PAUSED: &str = "rustfs_scanner_data_movement_paused";
488const METRIC_SCANNER_DATA_MOVEMENT_PAUSE_DURATION_SECONDS: &str = "rustfs_scanner_data_movement_pause_duration_seconds";
489const METRIC_SCANNER_DATA_MOVEMENT_BACKLOG_WORK_ITEMS: &str = "rustfs_scanner_data_movement_backlog_work_items";
490const SCANNER_DATA_MOVEMENT_PAUSE_POLICY: &str = "global_pause";
491
492#[derive(Clone, Copy, Debug, PartialEq, Eq, serde::Serialize)]
493#[serde(rename_all = "snake_case")]
494pub enum ScannerDataMovementPauseReason {
495 OperationEpochExhausted,
496 MovementGenerationExhausted,
497 DecommissionActive,
498 DecommissionFailed,
499 DecommissionCanceled,
500 RebalanceActive,
501}
502
503#[derive(Clone, Debug, PartialEq, Eq, serde::Serialize)]
504pub struct ScannerDataMovementPauseStatus {
505 pub paused: bool,
506 pub policy: &'static str,
507 pub reasons: Vec<ScannerDataMovementPauseReason>,
508 pub started_at_unix_secs: u64,
509 pub duration_seconds: u64,
510 pub operation_epoch: u64,
511 pub movement_generation: u64,
512 pub movement_backlog_work_items: u64,
513 pub movement_backlog_estimated: bool,
514}
515
516impl Default for ScannerDataMovementPauseStatus {
517 fn default() -> Self {
518 Self {
519 paused: false,
520 policy: SCANNER_DATA_MOVEMENT_PAUSE_POLICY,
521 reasons: Vec::new(),
522 started_at_unix_secs: 0,
523 duration_seconds: 0,
524 operation_epoch: 0,
525 movement_generation: 0,
526 movement_backlog_work_items: 0,
527 movement_backlog_estimated: false,
528 }
529 }
530}
531
532#[derive(Clone, Debug, PartialEq, Eq, serde::Serialize)]
533pub struct PoolMetaWriteGateStatus {
534 pub writes_ready: bool,
535 pub check_timed_out: bool,
536 pub write_blocked: bool,
537 pub transaction_aborted: bool,
538 pub pool_meta_absent: bool,
539 pub identity_initialized: Option<bool>,
540 pub identity_needs_repair: bool,
541 pub cluster_epoch: Option<u64>,
542 pub reason: Option<&'static str>,
543 pub phase: Option<&'static str>,
544 pub since_unix_secs: Option<i64>,
545}
546
547impl Default for PoolMetaWriteGateStatus {
548 fn default() -> Self {
549 Self {
550 writes_ready: true,
551 check_timed_out: false,
552 write_blocked: false,
553 transaction_aborted: false,
554 pool_meta_absent: false,
555 identity_initialized: None,
556 identity_needs_repair: false,
557 cluster_epoch: None,
558 reason: None,
559 phase: None,
560 since_unix_secs: None,
561 }
562 }
563}
564
565fn offset_unix_seconds(value: OffsetDateTime) -> u64 {
566 u64::try_from(value.unix_timestamp()).unwrap_or(0)
567}
568
569fn earliest_timestamp(current: Option<OffsetDateTime>, candidate: Option<OffsetDateTime>) -> Option<OffsetDateTime> {
570 match (current, candidate) {
571 (Some(current), Some(candidate)) => Some(current.min(candidate)),
572 (Some(current), None) => Some(current),
573 (None, candidate) => candidate,
574 }
575}
576
577fn usize_to_u64(value: usize) -> u64 {
578 u64::try_from(value).unwrap_or(u64::MAX)
579}
580
581fn metric_u64(value: u64) -> f64 {
582 f64::from(u32::try_from(value).unwrap_or(u32::MAX))
583}
584
585pub(crate) fn scanner_data_movement_timestamp_generation(value: OffsetDateTime) -> u64 {
586 let timestamp = value.unix_timestamp_nanos();
587 if timestamp <= 0 {
588 0
589 } else {
590 u64::try_from(timestamp).unwrap_or(u64::MAX)
591 }
592}
593
594fn valid_scanner_data_movement_timestamp_generation(value: OffsetDateTime) -> Option<u64> {
595 let generation = scanner_data_movement_timestamp_generation(value);
596 (generation != 0 && generation != u64::MAX).then_some(generation)
597}
598
599fn durable_scanner_data_movement_generation(pool_meta: &PoolMeta, rebalance_meta: Option<&RebalanceMeta>) -> u64 {
600 let mut generation = 0;
601 for pool in pool_meta.pools.iter().filter(|pool| pool.decommission.is_some()) {
602 let Some(pool_generation) = valid_scanner_data_movement_timestamp_generation(pool.last_update) else {
603 return u64::MAX;
604 };
605 generation = generation.max(pool_generation);
606 }
607
608 for movement_timestamp in rebalance_meta.into_iter().flat_map(|meta| {
609 meta.stopped_at.into_iter().chain(
610 meta.pool_stats
611 .iter()
612 .flat_map(|pool| [pool.info.start_time, pool.info.end_time])
613 .flatten(),
614 )
615 }) {
616 let Some(rebalance_generation) = valid_scanner_data_movement_timestamp_generation(movement_timestamp) else {
617 return u64::MAX;
618 };
619 generation = generation.max(rebalance_generation);
620 }
621
622 if generation == 0
623 && rebalance_meta.is_some_and(|meta| !meta.id.is_empty() || !meta.pool_stats.is_empty() || meta.stopped_at.is_some())
624 {
625 u64::MAX
626 } else {
627 generation
628 }
629}
630
631#[derive(Clone, Copy)]
632struct ScannerDataMovementSequenceState {
633 operation_epoch: u64,
634 operation_epoch_exhausted: bool,
635 movement_generation: u64,
636 movement_generation_exhausted: bool,
637}
638
639fn resolve_scanner_data_movement_pause_status(
640 pool_meta: &PoolMeta,
641 rebalance_meta: Option<&RebalanceMeta>,
642 decommission_worker_active: bool,
643 sequence: ScannerDataMovementSequenceState,
644 now: OffsetDateTime,
645) -> ScannerDataMovementPauseStatus {
646 let mut decommission_active = decommission_worker_active;
647 let mut decommission_failed = false;
648 let mut decommission_canceled = false;
649 let mut rebalance_active = false;
650 let mut started_at = None;
651 let mut movement_backlog_work_items = 0_u64;
652
653 for pool in &pool_meta.pools {
654 let Some(info) = pool.decommission.as_ref() else {
655 continue;
656 };
657 let active = info.has_decommission_state() && !info.complete && !info.failed && !info.canceled;
658 let failed = !info.queued && info.failed;
659 let canceled = !info.queued && info.canceled;
660 if !(active || failed || canceled) {
661 continue;
662 }
663
664 decommission_active |= active;
665 decommission_failed |= failed;
666 decommission_canceled |= canceled;
667 started_at = earliest_timestamp(started_at, info.start_time.or(Some(pool.last_update)));
668 let queued = usize_to_u64(info.queued_buckets.len());
669 let current_bucket = if info.bucket.is_empty() { 0 } else { 1 };
670 movement_backlog_work_items = movement_backlog_work_items.saturating_add(queued.max(current_bucket));
671 }
672
673 if let Some(rebalance_meta) = rebalance_meta {
674 for pool in &rebalance_meta.pool_stats {
675 let active = (pool.participating && pool.info.status == RebalStatus::Started) || pool.info.stopping;
676 if !active {
677 continue;
678 }
679 rebalance_active = true;
680 started_at = earliest_timestamp(started_at, pool.info.start_time);
681 movement_backlog_work_items = movement_backlog_work_items.saturating_add(usize_to_u64(pool.buckets.len()));
682 }
683 }
684
685 let mut reasons = Vec::with_capacity(6);
686 if sequence.operation_epoch_exhausted {
687 reasons.push(ScannerDataMovementPauseReason::OperationEpochExhausted);
688 }
689 if sequence.movement_generation_exhausted {
690 reasons.push(ScannerDataMovementPauseReason::MovementGenerationExhausted);
691 }
692 if decommission_active {
693 reasons.push(ScannerDataMovementPauseReason::DecommissionActive);
694 }
695 if decommission_failed {
696 reasons.push(ScannerDataMovementPauseReason::DecommissionFailed);
697 }
698 if decommission_canceled {
699 reasons.push(ScannerDataMovementPauseReason::DecommissionCanceled);
700 }
701 if rebalance_active {
702 reasons.push(ScannerDataMovementPauseReason::RebalanceActive);
703 }
704 let started_at_unix_secs = started_at.map(offset_unix_seconds).unwrap_or(0);
705 let duration_seconds = started_at
706 .and_then(|started_at| u64::try_from((now - started_at).whole_seconds()).ok())
707 .unwrap_or(0);
708 let paused = !reasons.is_empty();
709
710 ScannerDataMovementPauseStatus {
711 paused,
712 policy: SCANNER_DATA_MOVEMENT_PAUSE_POLICY,
713 reasons,
714 started_at_unix_secs,
715 duration_seconds,
716 operation_epoch: sequence.operation_epoch,
717 movement_generation: sequence.movement_generation,
718 movement_backlog_work_items,
719 movement_backlog_estimated: paused,
720 }
721}
722
723fn record_scanner_data_movement_pause_status(status: &ScannerDataMovementPauseStatus) {
724 metrics::gauge!(METRIC_SCANNER_DATA_MOVEMENT_PAUSED).set(if status.paused { 1.0 } else { 0.0 });
725 metrics::gauge!(METRIC_SCANNER_DATA_MOVEMENT_PAUSE_DURATION_SECONDS).set(metric_u64(status.duration_seconds));
726 metrics::gauge!(METRIC_SCANNER_DATA_MOVEMENT_BACKLOG_WORK_ITEMS).set(metric_u64(status.movement_backlog_work_items));
727}
728
729impl std::fmt::Debug for ECStore {
730 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
731 let disk_slot_count: usize = self.disk_map.values().map(Vec::len).sum();
732
733 f.debug_struct("ECStore")
734 .field("id", &self.id)
735 .field("disk_map_pool_count", &self.disk_map.len())
736 .field("disk_slot_count", &disk_slot_count)
737 .field("pool_count", &self.pools.len())
738 .finish_non_exhaustive()
739 }
740}
741
742impl ECStore {
746 pub fn all_set_disks(&self) -> Vec<Arc<crate::set_disk::SetDisks>> {
753 self.pools.iter().flat_map(|pool| pool.disk_set.iter().cloned()).collect()
754 }
755
756 pub async fn scanner_pause_backlog_writable_set_disks(&self) -> Vec<Arc<crate::set_disk::SetDisks>> {
763 let pool_meta = self.pool_meta.read().await;
764 self.pools
765 .iter()
766 .enumerate()
767 .filter(|(pool_index, _)| pool_meta.scanner_pause_backlog_pool_writable(*pool_index))
768 .flat_map(|(_, pool)| pool.disk_set.iter().cloned())
769 .collect()
770 }
771
772 pub fn get_server_config(&self) -> Option<Config> {
774 runtime_sources::server_config()
775 }
776
777 pub fn set_server_config(&self, cfg: Config) {
779 runtime_sources::set_server_config(cfg);
780 }
781
782 pub fn get_storage_class(&self) -> Option<crate::config::storageclass::Config> {
784 runtime_sources::storage_class_config()
785 }
786
787 pub fn set_storage_class(&self, cfg: crate::config::storageclass::Config) {
789 runtime_sources::set_storage_class_config(cfg);
790 }
791}
792
793impl ECStore {
797 pub fn notification_system(&self) -> Option<std::sync::Arc<crate::services::notification_sys::NotificationSys>> {
799 runtime_sources::notification_sys()
800 }
801
802 pub fn bucket_metadata_sys(&self) -> Option<Arc<tokio::sync::RwLock<crate::bucket::metadata_sys::BucketMetadataSys>>> {
804 runtime_sources::bucket_metadata_sys()
805 }
806
807 pub fn endpoints(&self) -> EndpointServerPools {
809 runtime_sources::endpoint_pools().unwrap_or_else(|| Vec::new().into())
810 }
811
812 pub fn instance_endpoints(&self) -> Option<EndpointServerPools> {
814 self.ctx.endpoints()
815 }
816
817 pub fn region(&self) -> Option<s3s::region::Region> {
819 runtime_sources::region()
820 }
821
822 pub fn tier_config_mgr(&self) -> Arc<tokio::sync::RwLock<crate::services::tier::tier::TierConfigMgr>> {
824 self.ctx.tier_config_mgr()
825 }
826
827 pub fn server_config(&self) -> Option<Config> {
829 runtime_sources::server_config()
830 }
831
832 pub fn storage_class(&self) -> Option<crate::config::storageclass::Config> {
834 runtime_sources::storage_class_config()
835 }
836}
837
838impl ECStore {
842 pub fn port(&self) -> u16 {
844 runtime_sources::rustfs_port()
845 }
846
847 pub async fn host(&self) -> String {
849 runtime_sources::rustfs_host().await
850 }
851
852 pub async fn addr(&self) -> String {
854 runtime_sources::rustfs_addr().await
855 }
856}
857
858impl ECStore {
865 pub async fn setup_is_erasure(&self) -> bool {
867 self.ctx.is_erasure().await
868 }
869
870 pub async fn setup_is_dist_erasure(&self) -> bool {
872 self.ctx.is_dist_erasure().await
873 }
874
875 pub async fn setup_is_erasure_sd(&self) -> bool {
877 self.ctx.is_erasure_sd().await
878 }
879
880 pub fn scanner_namespace_mutation_generation(&self) -> u64 {
881 list_objects::scanner_namespace_mutation_generation().saturating_add(self.ctx.namespace_commit_generation())
882 }
883
884 pub async fn scanner_data_movement_active(&self) -> bool {
885 let (decommission, rebalance) = tokio::join!(self.is_decommission_running(), self.is_rebalance_started());
886 decommission || rebalance
887 }
888
889 pub async fn scanner_data_movement_activity(&self) -> (bool, bool, u64) {
895 let operation_gate = self.ctx.data_movement_operation_gate();
896 let _operation_guard = operation_gate.read_owned().await;
897 let (active, blocked) = self.scanner_data_movement_snapshot_locked().await;
898 let blocked =
899 blocked || self.ctx.data_movement_operation_epoch_exhausted() || self.ctx.data_movement_generation_exhausted();
900 self.ctx.set_scanner_publication_state(blocked);
901 (active, blocked, self.ctx.data_movement_generation())
902 }
903
904 pub fn scanner_data_movement_generation(&self) -> u64 {
905 self.ctx.data_movement_generation()
906 }
907
908 pub fn scanner_data_movement_generation_exhausted(&self) -> bool {
909 self.ctx.data_movement_generation_exhausted()
910 }
911
912 pub fn scanner_data_movement_changed(&self) -> std::sync::Arc<tokio::sync::Notify> {
913 self.ctx.data_movement_generation_notify()
914 }
915
916 pub async fn scanner_data_usage_publication_blocked(&self) -> bool {
922 let operation_gate = self.ctx.data_movement_operation_gate();
923 let _operation_guard = operation_gate.read_owned().await;
924 self.scanner_data_usage_publication_snapshot_blocked().await || self.ctx.namespace_commits_pending()
925 }
926
927 pub async fn scanner_data_movement_pause_status(&self) -> ScannerDataMovementPauseStatus {
928 let operation_gate = self.ctx.data_movement_operation_gate();
929 let _operation_guard = operation_gate.read_owned().await;
930 self.scanner_data_movement_pause_snapshot().await
931 }
932
933 async fn scanner_data_usage_publication_snapshot_blocked(&self) -> bool {
934 self.scanner_data_movement_pause_snapshot().await.paused
935 }
936
937 async fn scanner_data_movement_snapshot_locked(&self) -> (bool, bool) {
938 let decommission_cancelers = self.decommission_cancelers.read().await;
939 let decommission_active = decommission_cancelers
940 .iter()
941 .any(|canceler| canceler.as_ref().is_some_and(DecommissionCanceler::is_active));
942 let pool_meta = self.pool_meta.read().await;
943 let decommission_active = decommission_active
944 || pool_meta.pools.iter().any(|pool| {
945 pool.decommission
946 .as_ref()
947 .is_some_and(|info| info.has_decommission_state() && !info.complete && !info.failed && !info.canceled)
948 });
949 let decommission_terminal = pool_meta.pools.iter().any(|pool| {
950 pool.decommission
951 .as_ref()
952 .is_some_and(|info| !info.queued && (info.failed || info.canceled))
953 });
954 let rebalance_meta = self.rebalance_meta.read().await;
955 let rebalance_active = rebalance_meta
956 .as_ref()
957 .is_some_and(is_rebalance_conflicting_with_decommission);
958 self.ctx
959 .observe_durable_data_movement_generation(durable_scanner_data_movement_generation(
960 &pool_meta,
961 rebalance_meta.as_ref(),
962 ));
963
964 let blocked = decommission_active || decommission_terminal || rebalance_active;
965 (decommission_active || rebalance_active, blocked)
966 }
967
968 async fn scanner_data_movement_pause_snapshot(&self) -> ScannerDataMovementPauseStatus {
969 let decommission_active = {
970 let decommission_cancelers = self.decommission_cancelers.read().await;
971 decommission_cancelers
972 .iter()
973 .any(|canceler| canceler.as_ref().is_some_and(DecommissionCanceler::is_active))
974 };
975 let pool_meta = self.pool_meta.read().await.clone();
976 let rebalance_meta = self.rebalance_meta.read().await.clone();
977 self.ctx
978 .observe_durable_data_movement_generation(durable_scanner_data_movement_generation(
979 &pool_meta,
980 rebalance_meta.as_ref(),
981 ));
982 let status = resolve_scanner_data_movement_pause_status(
983 &pool_meta,
984 rebalance_meta.as_ref(),
985 decommission_active,
986 ScannerDataMovementSequenceState {
987 operation_epoch: self.ctx.data_movement_operation_epoch(),
988 operation_epoch_exhausted: self.ctx.data_movement_operation_epoch_exhausted(),
989 movement_generation: self.ctx.data_movement_generation(),
990 movement_generation_exhausted: self.ctx.data_movement_generation_exhausted(),
991 },
992 OffsetDateTime::now_utc(),
993 );
994 self.ctx.set_scanner_publication_state(status.paused);
995 record_scanner_data_movement_pause_status(&status);
996 status
997 }
998
999 #[cfg(test)]
1000 pub(crate) async fn scanner_data_movement_pause_snapshot_for_test(&self) -> ScannerDataMovementPauseStatus {
1001 self.scanner_data_movement_pause_snapshot().await
1002 }
1003
1004 pub async fn scanner_data_usage_publication_read_guard(&self) -> (tokio::sync::OwnedRwLockReadGuard<()>, u64) {
1009 let operation_gate = self.ctx.data_movement_operation_gate();
1010 let operation_guard = operation_gate.read_owned().await;
1011 let epoch = self.ctx.data_movement_operation_epoch();
1012 (operation_guard, epoch)
1013 }
1014
1015 pub async fn scanner_data_usage_publication_admission_guard(&self) -> Option<(tokio::sync::OwnedRwLockReadGuard<()>, u64)> {
1020 let operation_gate = self.ctx.data_movement_operation_gate();
1021 let operation_guard = operation_gate.read_owned().await;
1022 if self.ctx.data_movement_operation_epoch_exhausted() || self.ctx.data_movement_generation_exhausted() {
1023 return None;
1024 }
1025 if self.scanner_data_usage_publication_snapshot_blocked().await {
1026 return None;
1027 }
1028
1029 Some((operation_guard, self.ctx.data_movement_operation_epoch()))
1030 }
1031
1032 pub async fn scanner_data_usage_publication_commit_scope(
1037 &self,
1038 expected_movement_epoch: u64,
1039 safe_deadline: tokio::time::Instant,
1040 remote_lease_tokens: Vec<Uuid>,
1041 ) -> Option<ScannerPublicationCommitScope> {
1042 let (movement_permit, epoch) = self.scanner_data_usage_publication_admission_guard().await?;
1043 if epoch != expected_movement_epoch {
1044 return None;
1045 }
1046 Some(ScannerPublicationCommitScope::new_storage_owned(
1047 epoch,
1048 safe_deadline,
1049 remote_lease_tokens,
1050 movement_permit,
1051 ))
1052 }
1053
1054 pub async fn scanner_data_usage_publication_commit_scope_with_release_flag(
1055 &self,
1056 expected_movement_epoch: u64,
1057 safe_deadline: tokio::time::Instant,
1058 remote_lease_tokens: Vec<Uuid>,
1059 lease_release_safe: Arc<std::sync::atomic::AtomicBool>,
1060 ) -> Option<ScannerPublicationCommitScope> {
1061 let (movement_permit, epoch) = self.scanner_data_usage_publication_admission_guard().await?;
1062 if epoch != expected_movement_epoch {
1063 return None;
1064 }
1065 Some(ScannerPublicationCommitScope::new_storage_owned_with_release_flag(
1066 epoch,
1067 safe_deadline,
1068 remote_lease_tokens,
1069 movement_permit,
1070 lease_release_safe,
1071 ))
1072 }
1073
1074 pub(crate) async fn scanner_data_usage_publication_epoch(&self) -> Option<u64> {
1078 let (operation_guard, epoch) = self.scanner_data_usage_publication_admission_guard().await?;
1079 drop(operation_guard);
1080 Some(epoch)
1081 }
1082
1083 pub async fn acquire_scanner_publication_lease(
1088 &self,
1089 expected_generation: u64,
1090 ttl: std::time::Duration,
1091 ) -> Result<(Uuid, u64)> {
1092 if ttl != crate::runtime::instance::SCANNER_PUBLICATION_LEASE_TTL {
1093 return Err(Error::other("scanner publication lease TTL is not supported"));
1094 }
1095
1096 let namespace_generation = self.scanner_namespace_mutation_generation();
1099 let operation_gate = self.ctx.data_movement_operation_gate();
1100 let operation_guard = operation_gate.read_owned().await;
1101 if self.ctx.data_movement_generation_exhausted()
1102 || self.ctx.data_movement_operation_epoch_exhausted()
1103 || self.ctx.data_movement_generation() != expected_generation
1104 || namespace_generation == u64::MAX
1105 {
1106 return Err(Error::other("scanner publication lease generation is stale"));
1107 }
1108 if self.scanner_data_movement_snapshot_locked().await.1 || self.ctx.namespace_commits_pending() {
1109 return Err(Error::other("scanner publication lease is blocked by data movement"));
1110 }
1111
1112 if self.scanner_namespace_mutation_generation() != namespace_generation {
1113 return Err(Error::other("scanner publication lease generation is stale"));
1114 }
1115
1116 let token = Uuid::new_v4();
1117 let expires_at = tokio::time::Instant::now() + ttl;
1118 if !self
1119 .ctx
1120 .install_scanner_publication_lease(token, expires_at, expected_generation, namespace_generation, operation_guard)
1121 .await
1122 {
1123 return Err(Error::other("scanner publication lease capacity is exhausted"));
1124 }
1125
1126 let context = Arc::clone(&self.ctx);
1127 tokio::spawn(async move {
1128 tokio::time::sleep_until(expires_at).await;
1129 context.expire_scanner_publication_lease(token, expires_at).await;
1130 });
1131 Ok((token, expected_generation))
1132 }
1133
1134 pub async fn release_scanner_publication_lease(&self, token: Uuid) -> bool {
1135 self.ctx.remove_scanner_publication_lease(token).await
1136 }
1137
1138 pub async fn validate_scanner_publication_lease(&self, token: Uuid, expected_generation: u64) -> Result<()> {
1144 let _operation_guard = self.acquire_scanner_publication_lease_guard(token).await?;
1145 if self.ctx.data_movement_generation_exhausted()
1146 || self.ctx.data_movement_operation_epoch_exhausted()
1147 || self.ctx.data_movement_generation() != expected_generation
1148 {
1149 return Err(Error::other("scanner publication lease generation is stale"));
1150 }
1151 if self.scanner_data_movement_snapshot_locked().await.1 || self.ctx.namespace_commits_pending() {
1152 return Err(Error::other("scanner publication lease is blocked by data movement"));
1153 }
1154 let Some((lease_generation, lease_namespace_generation)) = self.ctx.scanner_publication_lease_generations(token).await
1155 else {
1156 return Err(Error::other("scanner publication lease is unknown or expired"));
1157 };
1158 let namespace_generation = self.scanner_namespace_mutation_generation();
1159 if lease_generation != self.ctx.data_movement_generation()
1160 || lease_namespace_generation != namespace_generation
1161 || namespace_generation == u64::MAX
1162 {
1163 return Err(Error::other("scanner publication lease generation is stale"));
1164 }
1165 Ok(())
1166 }
1167
1168 pub async fn acquire_scanner_publication_lease_guard(&self, token: Uuid) -> Result<tokio::sync::OwnedRwLockReadGuard<()>> {
1174 let operation_gate = self.ctx.data_movement_operation_gate();
1175 let operation_guard = operation_gate.read_owned().await;
1176 if self.ctx.data_movement_generation_exhausted() || self.ctx.data_movement_operation_epoch_exhausted() {
1177 return Err(Error::other("scanner publication lease generation is exhausted"));
1178 }
1179 if self.scanner_data_movement_snapshot_locked().await.1 || self.ctx.namespace_commits_pending() {
1180 return Err(Error::other("scanner publication lease is blocked by data movement"));
1181 }
1182 let Some((lease_generation, lease_namespace_generation)) = self.ctx.scanner_publication_lease_generations(token).await
1183 else {
1184 return Err(Error::other("scanner publication lease is unknown or expired"));
1185 };
1186 let namespace_generation = self.scanner_namespace_mutation_generation();
1187 if lease_generation != self.ctx.data_movement_generation()
1188 || lease_namespace_generation != namespace_generation
1189 || namespace_generation == u64::MAX
1190 {
1191 return Err(Error::other("scanner publication lease generation is stale"));
1192 }
1193 Ok(operation_guard)
1194 }
1195}
1196
1197#[async_trait::async_trait]
1242impl crate::storage_api_contracts::object::ObjectIO for ECStore {
1243 type Error = Error;
1244 type RangeSpec = HTTPRangeSpec;
1245 type HeaderMap = HeaderMap;
1246 type ObjectOptions = ObjectOptions;
1247 type ObjectInfo = ObjectInfo;
1248 type GetObjectReader = GetObjectReader;
1249 type PutObjectReader = PutObjReader;
1250
1251 #[instrument(level = "debug", skip(self, h))]
1252 async fn get_object_reader(
1253 &self,
1254 bucket: &str,
1255 object: &str,
1256 range: Option<HTTPRangeSpec>,
1257 h: HeaderMap,
1258 opts: &ObjectOptions,
1259 ) -> Result<GetObjectReader> {
1260 self.handle_get_object_reader(bucket, object, range, h, opts).await
1261 }
1262 #[instrument(level = "debug", skip(self, data))]
1263 async fn put_object(&self, bucket: &str, object: &str, data: &mut PutObjReader, opts: &ObjectOptions) -> Result<ObjectInfo> {
1264 self.put_object_with_old_current_size(bucket, object, data, opts)
1265 .await
1266 .map(|(object_info, _)| object_info)
1267 }
1268}
1269
1270impl ECStore {
1271 #[instrument(level = "debug", skip(self, data))]
1276 pub async fn put_object_with_old_current_size(
1277 &self,
1278 bucket: &str,
1279 object: &str,
1280 data: &mut PutObjReader,
1281 opts: &ObjectOptions,
1282 ) -> Result<(ObjectInfo, Option<crate::disk::OldCurrentSize>)> {
1283 let result = match self.handle_put_object(bucket, object, data, opts).await {
1284 Ok((object_info, old_current_size)) => enqueue_transition_after_write(Ok(object_info), LcEventSrc::S3PutObject)
1285 .await
1286 .map(|object_info| (object_info, old_current_size)),
1287 Err(err) => Err(err),
1288 };
1289 if result.is_ok() {
1290 list_objects::observe_list_objects_mutation(self, bucket).await;
1291 }
1292 result
1293 }
1294}
1295
1296lazy_static! {
1297 static ref ENABLED_OBJECT_LOCK_CONFIG: ObjectLockConfiguration = ObjectLockConfiguration {
1298 object_lock_enabled: Some(ObjectLockEnabled::from_static(ObjectLockEnabled::ENABLED)),
1299 ..Default::default()
1300 };
1301 static ref ENABLED_VERSIONING_CONFIG: VersioningConfiguration = VersioningConfiguration {
1302 status: Some(BucketVersioningStatus::from_static(BucketVersioningStatus::ENABLED)),
1303 ..Default::default()
1304 };
1305}
1306
1307#[async_trait::async_trait]
1308impl BucketOperations for ECStore {
1309 type Error = Error;
1310
1311 #[instrument(skip(self))]
1312 async fn make_bucket(&self, bucket: &str, opts: &MakeBucketOptions) -> Result<()> {
1313 Box::pin(self.handle_make_bucket(bucket, opts)).await
1314 }
1315
1316 #[instrument(skip(self))]
1317 async fn get_bucket_info(&self, bucket: &str, opts: &BucketOptions) -> Result<BucketInfo> {
1318 self.handle_get_bucket_info(bucket, opts).await
1319 }
1320 #[instrument(skip(self))]
1321 async fn list_bucket(&self, opts: &BucketOptions) -> Result<Vec<BucketInfo>> {
1322 self.handle_list_bucket(opts).await
1323 }
1324 #[instrument(skip(self))]
1325 async fn delete_bucket(&self, bucket: &str, opts: &DeleteBucketOptions) -> Result<()> {
1326 Box::pin(self.handle_delete_bucket(bucket, opts)).await
1327 }
1328}
1329
1330#[async_trait::async_trait]
1331impl crate::storage_api_contracts::object::ObjectOperations for ECStore {
1332 type Error = Error;
1333 type ObjectInfo = ObjectInfo;
1334 type ObjectOptions = ObjectOptions;
1335 type FileInfo = FileInfo;
1336 type ObjectToDelete = ObjectToDelete;
1337 type DeletedObject = DeletedObject;
1338
1339 async fn get_object_info(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<ObjectInfo> {
1340 self.handle_get_object_info(bucket, object, opts).await
1341 }
1342
1343 async fn verify_object_integrity(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<()> {
1344 self.handle_verify_object_integrity(bucket, object, opts).await
1345 }
1346
1347 #[instrument(skip(self))]
1348 async fn copy_object(
1349 &self,
1350 src_bucket: &str,
1351 src_object: &str,
1352 dst_bucket: &str,
1353 dst_object: &str,
1354 src_info: &mut ObjectInfo,
1355 src_opts: &ObjectOptions,
1356 dst_opts: &ObjectOptions,
1357 ) -> Result<ObjectInfo> {
1358 let result = enqueue_transition_after_write(
1359 self.handle_copy_object(src_bucket, src_object, dst_bucket, dst_object, src_info, src_opts, dst_opts)
1360 .await,
1361 LcEventSrc::S3CopyObject,
1362 )
1363 .await;
1364 if result.is_ok() {
1365 list_objects::observe_list_objects_mutation(self, dst_bucket).await;
1366 }
1367 result
1368 }
1369
1370 #[instrument(skip(self))]
1371 async fn delete_object_version(&self, bucket: &str, object: &str, fi: &FileInfo, force_del_marker: bool) -> Result<()> {
1372 let result = self.handle_delete_object_version(bucket, object, fi, force_del_marker).await;
1373 if result.is_ok() {
1374 list_objects::observe_list_objects_mutation(self, bucket).await;
1375 }
1376 result
1377 }
1378
1379 #[instrument(skip(self))]
1380 async fn delete_object(&self, bucket: &str, object: &str, opts: ObjectOptions) -> Result<ObjectInfo> {
1381 let result = self.handle_delete_object(bucket, object, opts).await;
1382 if result.is_ok() {
1383 list_objects::observe_list_objects_mutation(self, bucket).await;
1384 }
1385 result
1386 }
1387
1388 #[instrument(skip(self, objects, opts))]
1389 async fn delete_objects(
1390 &self,
1391 bucket: &str,
1392 objects: Vec<ObjectToDelete>,
1393 opts: ObjectOptions,
1394 ) -> (Vec<DeletedObject>, Vec<Option<Error>>) {
1395 let result = self.handle_delete_objects(bucket, objects, opts).await;
1396 let success_count = result.1.iter().filter(|err| err.is_none()).count();
1397 if success_count > 0 {
1398 list_objects::observe_list_objects_mutations(self, bucket, success_count).await;
1399 }
1400 result
1401 }
1402
1403 #[instrument(skip(self))]
1404 async fn put_object_metadata(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<ObjectInfo> {
1405 self.handle_put_object_metadata(bucket, object, opts).await
1406 }
1407 #[instrument(skip(self))]
1408 async fn get_object_tags(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<String> {
1409 self.handle_get_object_tags(bucket, object, opts).await
1410 }
1411
1412 #[instrument(level = "debug", skip(self))]
1413 async fn put_object_tags(&self, bucket: &str, object: &str, tags: &str, opts: &ObjectOptions) -> Result<ObjectInfo> {
1414 self.handle_put_object_tags(bucket, object, tags, opts).await
1415 }
1416
1417 #[instrument(skip(self))]
1418 async fn delete_object_tags(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<ObjectInfo> {
1419 self.handle_delete_object_tags(bucket, object, opts).await
1420 }
1421
1422 #[instrument(skip(self))]
1423 async fn add_partial(&self, bucket: &str, object: &str, version_id: &str) -> Result<()> {
1424 self.handle_add_partial(bucket, object, version_id).await
1425 }
1426 #[instrument(skip(self))]
1427 async fn transition_object(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<()> {
1428 self.handle_transition_object(bucket, object, opts).await
1429 }
1430
1431 #[instrument(skip(self))]
1432 async fn restore_transitioned_object(self: Arc<Self>, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<()> {
1433 self.handle_restore_transitioned_object(bucket, object, opts).await
1434 }
1435}
1436
1437#[async_trait::async_trait]
1438impl crate::storage_api_contracts::list::ListOperations for ECStore {
1439 type Error = Error;
1440 type ListObjectsV2Info = ListObjectsV2Info;
1441 type ListObjectVersionsInfo = ListObjectVersionsInfo;
1442 type ObjectInfoOrErr = ObjectInfoOrErr;
1443 type WalkOptions = WalkOptions;
1444 type WalkCancellation = CancellationToken;
1445 type WalkResultSender = tokio::sync::mpsc::Sender<ObjectInfoOrErr>;
1446
1447 #[instrument(level = "trace", skip(self))]
1452 async fn list_objects_v2(
1453 self: Arc<Self>,
1454 bucket: &str,
1455 prefix: &str,
1456 continuation_token: Option<String>,
1457 delimiter: Option<String>,
1458 max_keys: i32,
1459 fetch_owner: bool,
1460 start_after: Option<String>,
1461 incl_deleted: bool,
1462 ) -> Result<ListObjectsV2Info> {
1463 self.inner_list_objects_v2(
1464 bucket,
1465 prefix,
1466 continuation_token,
1467 delimiter,
1468 max_keys,
1469 fetch_owner,
1470 start_after,
1471 incl_deleted,
1472 )
1473 .await
1474 }
1475
1476 #[instrument(skip(self))]
1477 async fn list_object_versions(
1478 self: Arc<Self>,
1479 bucket: &str,
1480 prefix: &str,
1481 marker: Option<String>,
1482 version_marker: Option<String>,
1483 delimiter: Option<String>,
1484 max_keys: i32,
1485 ) -> Result<ListObjectVersionsInfo> {
1486 self.inner_list_object_versions(bucket, prefix, marker, version_marker, delimiter, max_keys)
1487 .await
1488 }
1489
1490 async fn walk(
1491 self: Arc<Self>,
1492 rx: CancellationToken,
1493 bucket: &str,
1494 prefix: &str,
1495 result: tokio::sync::mpsc::Sender<ObjectInfoOrErr>,
1496 opts: WalkOptions,
1497 ) -> Result<()> {
1498 self.walk_internal(rx, bucket, prefix, result, opts).await
1499 }
1500}
1501
1502#[async_trait::async_trait]
1503impl crate::storage_api_contracts::multipart::MultipartOperations for ECStore {
1504 type Error = Error;
1505 type ObjectInfo = ObjectInfo;
1506 type ObjectOptions = ObjectOptions;
1507 type PutObjectReader = PutObjReader;
1508 type CompletePart = CompletePart;
1509 type ListMultipartsInfo = ListMultipartsInfo;
1510 type MultipartUploadResult = MultipartUploadResult;
1511 type PartInfo = PartInfo;
1512 type MultipartInfo = MultipartInfo;
1513 type ListPartsInfo = ListPartsInfo;
1514
1515 #[instrument(skip(self))]
1516 async fn list_multipart_uploads(
1517 &self,
1518 bucket: &str,
1519 prefix: &str,
1520 key_marker: Option<String>,
1521 upload_id_marker: Option<String>,
1522 delimiter: Option<String>,
1523 max_uploads: usize,
1524 ) -> Result<ListMultipartsInfo> {
1525 self.handle_list_multipart_uploads(
1526 bucket,
1527 multipart::MultipartUploadListRequest {
1528 prefix: prefix.to_string(),
1529 key_marker,
1530 upload_id_marker,
1531 delimiter,
1532 max_uploads,
1533 expected_incarnation_id: None,
1534 },
1535 )
1536 .await
1537 }
1538
1539 #[instrument(skip(self))]
1540 async fn new_multipart_upload(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<MultipartUploadResult> {
1541 self.handle_new_multipart_upload(bucket, object, opts).await
1542 }
1543
1544 #[instrument(skip(self))]
1545 async fn copy_object_part(
1546 &self,
1547 src_bucket: &str,
1548 src_object: &str,
1549 _dst_bucket: &str,
1550 _dst_object: &str,
1551 _upload_id: &str,
1552 _part_id: usize,
1553 _start_offset: i64,
1554 _length: i64,
1555 _src_info: &ObjectInfo,
1556 _src_opts: &ObjectOptions,
1557 _dst_opts: &ObjectOptions,
1558 ) -> Result<()> {
1559 self.handle_copy_object_part(
1560 src_bucket,
1561 src_object,
1562 _dst_bucket,
1563 _dst_object,
1564 _upload_id,
1565 _part_id,
1566 _start_offset,
1567 _length,
1568 _src_info,
1569 _src_opts,
1570 _dst_opts,
1571 )
1572 .await
1573 }
1574 #[instrument(skip(self, data))]
1575 async fn put_object_part(
1576 &self,
1577 bucket: &str,
1578 object: &str,
1579 upload_id: &str,
1580 part_id: usize,
1581 data: &mut PutObjReader,
1582 opts: &ObjectOptions,
1583 ) -> Result<PartInfo> {
1584 self.handle_put_object_part(bucket, object, upload_id, part_id, data, opts)
1585 .await
1586 }
1587
1588 #[instrument(skip(self))]
1589 async fn get_multipart_info(
1590 &self,
1591 bucket: &str,
1592 object: &str,
1593 upload_id: &str,
1594 opts: &ObjectOptions,
1595 ) -> Result<MultipartInfo> {
1596 self.handle_get_multipart_info(bucket, object, upload_id, opts).await
1597 }
1598
1599 #[instrument(skip(self))]
1600 async fn list_object_parts(
1601 &self,
1602 bucket: &str,
1603 object: &str,
1604 upload_id: &str,
1605 part_number_marker: Option<usize>,
1606 max_parts: usize,
1607 opts: &ObjectOptions,
1608 ) -> Result<ListPartsInfo> {
1609 self.handle_list_object_parts(bucket, object, upload_id, part_number_marker, max_parts, opts)
1610 .await
1611 }
1612
1613 #[instrument(skip(self))]
1614 async fn abort_multipart_upload(&self, bucket: &str, object: &str, upload_id: &str, opts: &ObjectOptions) -> Result<()> {
1615 self.handle_abort_multipart_upload(bucket, object, upload_id, opts).await
1616 }
1617
1618 #[instrument(skip(self))]
1619 async fn complete_multipart_upload(
1620 self: Arc<Self>,
1621 bucket: &str,
1622 object: &str,
1623 upload_id: &str,
1624 uploaded_parts: Vec<CompletePart>,
1625 opts: &ObjectOptions,
1626 ) -> Result<ObjectInfo> {
1627 let result = enqueue_transition_after_write(
1628 self.clone()
1629 .handle_complete_multipart_upload(bucket, object, upload_id, uploaded_parts, opts)
1630 .await,
1631 LcEventSrc::S3CompleteMultipartUpload,
1632 )
1633 .await;
1634 if result.is_ok() {
1635 list_objects::observe_list_objects_mutation(self.as_ref(), bucket).await;
1636 }
1637 result
1638 }
1639}
1640
1641#[async_trait::async_trait]
1642impl crate::storage_api_contracts::heal::HealOperations for ECStore {
1643 type Error = Error;
1644 type HealResultItem = HealResultItem;
1645 type HealOptions = HealOpts;
1646
1647 #[instrument(skip(self))]
1648 async fn heal_format(&self, dry_run: bool) -> Result<(HealResultItem, Option<Error>)> {
1649 self.handle_heal_format(dry_run).await
1650 }
1651
1652 #[instrument(skip(self))]
1653 async fn heal_bucket(&self, bucket: &str, opts: &HealOpts) -> Result<HealResultItem> {
1654 self.handle_heal_bucket(bucket, opts).await
1655 }
1656 #[instrument(level = "trace", skip(self, opts), fields(bucket = %bucket, object = %object, version_id = %version_id))]
1657 async fn heal_object(
1658 &self,
1659 bucket: &str,
1660 object: &str,
1661 version_id: &str,
1662 opts: &HealOpts,
1663 ) -> Result<(HealResultItem, Option<Error>)> {
1664 self.handle_heal_object(bucket, object, version_id, opts).await
1665 }
1666
1667 #[instrument(skip(self))]
1668 async fn get_pool_and_set(&self, id: &str) -> Result<(Option<usize>, Option<usize>, Option<usize>)> {
1669 self.handle_get_pool_and_set(id).await
1670 }
1671
1672 #[instrument(skip(self))]
1673 async fn check_abandoned_parts(&self, bucket: &str, object: &str, opts: &HealOpts) -> Result<()> {
1674 self.handle_check_abandoned_parts(bucket, object, opts).await
1675 }
1676}
1677
1678#[async_trait::async_trait]
1679impl crate::storage_api_contracts::namespace::NamespaceLocking for ECStore {
1680 type Error = Error;
1681 type NamespaceLock = NamespaceLockWrapper;
1682
1683 async fn new_ns_lock(&self, bucket: &str, object: &str) -> Result<NamespaceLockWrapper> {
1684 self.handle_new_ns_lock(bucket, object).await
1685 }
1686}
1687
1688#[async_trait::async_trait]
1689impl crate::storage_api_contracts::admin::StorageAdminApi for ECStore {
1690 type BackendInfo = rustfs_madmin::BackendInfo;
1691 type StorageInfo = rustfs_madmin::StorageInfo;
1692 type Disk = DiskStore;
1693 type Error = Error;
1694
1695 #[instrument(skip(self))]
1696 async fn backend_info(&self) -> Self::BackendInfo {
1697 self.handle_backend_info().await
1698 }
1699
1700 #[instrument(skip(self))]
1701 async fn storage_info(&self) -> Self::StorageInfo {
1702 self.handle_storage_info().await
1703 }
1704
1705 #[instrument(skip(self))]
1706 async fn local_storage_info(&self) -> Self::StorageInfo {
1707 self.handle_local_storage_info().await
1708 }
1709
1710 #[instrument(skip(self))]
1711 async fn disk_set_inventory(
1712 &self,
1713 selector: crate::storage_api_contracts::admin::DiskSetSelector,
1714 ) -> Result<Vec<Option<Self::Disk>>> {
1715 self.handle_get_disks(selector.pool_idx, selector.set_idx).await
1716 }
1717
1718 #[instrument(skip(self))]
1719 fn set_drive_counts(&self) -> Vec<usize> {
1720 self.handle_set_drive_counts()
1721 }
1722}
1723
1724#[cfg(test)]
1725mod tests {
1726 use super::*;
1727 use crate::core::pools::{PoolDecommissionInfo, PoolSpaceInfo, PoolStatus};
1728 use crate::layout::endpoints::{Endpoints, PoolEndpoints, SetupType};
1729 use crate::object_api::ObjectOptions;
1730 use crate::runtime::global::reset_local_disk_test_state;
1731 use crate::runtime::sources::{clear_local_disk_id_map_for_test, local_disk_path_by_id};
1732 use crate::store::init_format::{connect_load_init_formats, init_disks};
1733 use serial_test::serial;
1734 use tempfile::TempDir;
1735
1736 #[test]
1737 fn g_d2_008_default_versioning_config_keeps_persisted_bytes() {
1738 let bytes = crate::bucket::utils::serialize::<VersioningConfiguration>(&ENABLED_VERSIONING_CONFIG)
1739 .expect("the default Versioning configuration must serialize");
1740 assert_eq!(bytes, b"<VersioningConfiguration><Status>Enabled</Status></VersioningConfiguration>");
1741 }
1742
1743 #[test]
1744 fn g_d2_009_default_object_lock_config_keeps_persisted_bytes() {
1745 let bytes = crate::bucket::utils::serialize::<ObjectLockConfiguration>(&ENABLED_OBJECT_LOCK_CONFIG)
1746 .expect("the default Object Lock configuration must serialize");
1747 assert_eq!(
1748 bytes,
1749 b"<ObjectLockConfiguration><ObjectLockEnabled>Enabled</ObjectLockEnabled></ObjectLockConfiguration>"
1750 );
1751 }
1752
1753 #[tokio::test]
1754 async fn test_get_disk_infos() {
1755 let disks = vec![None, None]; let infos = get_disk_infos(&disks).await;
1757
1758 assert_eq!(infos.len(), disks.len());
1759 assert!(infos.iter().all(|info| info.is_none()));
1761 }
1762
1763 #[test]
1764 fn ecstore_debug_is_bounded_summary() {
1765 let endpoint_pools = EndpointServerPools::default();
1766 let ctx = Arc::new(InstanceContext::new());
1767 let store = ECStore {
1768 id: uuid::Uuid::new_v4(),
1769 disk_map: [(0, vec![None, None, None, None])].into_iter().collect(),
1770 pools: Vec::new(),
1771 peer_sys: crate::cluster::rpc::S3PeerSys::new_with_instance_ctx(&endpoint_pools, ctx.clone()),
1772 pool_meta: RwLock::new(PoolMeta::default()),
1773 rebalance_meta: RwLock::new(None),
1774 decommission_cancelers: RwLock::new(Vec::new()),
1775 start_gate: Mutex::new(()),
1776 pool_meta_save_gate: Mutex::default(),
1777 ctx,
1778 bucket_fence_registry: Arc::default(),
1779 };
1780
1781 let rendered = format!("{store:?}");
1782
1783 assert!(rendered.len() < 256, "ECStore Debug should stay bounded: {rendered}");
1784 assert!(rendered.contains("disk_map_pool_count"));
1785 assert!(rendered.contains("disk_slot_count"));
1786 assert!(!rendered.contains("disk_map:"));
1787 assert!(!rendered.contains("pools:"));
1788 assert!(!rendered.contains("pool_meta"));
1789 assert!(!rendered.contains("format.json"));
1790 assert!(!rendered.contains("TimedActionSlot"));
1791 assert!(!rendered.contains("DiskHealthTracker"));
1792 }
1793
1794 #[test]
1795 fn object_options_debug_does_not_expand_tier_store_handle() {
1796 let store = build_store_with_ctx(Arc::new(InstanceContext::new()));
1797 let mut opts = ObjectOptions {
1798 version_id: Some("large-version-id".repeat(1024)),
1799 expected_current_version_id: Some("large-expected-version-id".repeat(1024)),
1800 preserve_etag: Some("large-etag".repeat(1024)),
1801 http_preconditions: Some(crate::storage_api_contracts::object::HTTPPreconditions {
1802 if_match: Some("large-if-match".repeat(1024)),
1803 if_none_match: Some("large-if-none-match".repeat(1024)),
1804 ..Default::default()
1805 }),
1806 tier_delete_journal_api: Some(store),
1807 ..Default::default()
1808 };
1809 opts.user_defined.insert("large-user-metadata".to_owned(), "x".repeat(8192));
1810 opts.eval_metadata = Some([("large-eval-metadata".to_owned(), "y".repeat(8192))].into_iter().collect());
1811 opts.transition.status = "large-transition-status".repeat(1024);
1812 opts.transition.tier = "large-transition-tier".repeat(1024);
1813 opts.lifecycle_audit_event.event.rule_id = "large-rule-id".repeat(1024);
1814 opts.lifecycle_audit_event.event.storage_class = "large-storage-class".repeat(1024);
1815
1816 let rendered = format!("{opts:?}");
1817
1818 assert!(rendered.len() < 4096, "ObjectOptions Debug should stay bounded: {rendered}");
1819 assert!(rendered.contains("tier_delete_journal_api: true"));
1820 assert!(rendered.contains("user_defined_count: 1"));
1821 assert!(rendered.contains("eval_metadata_count: Some(1)"));
1822 assert!(!rendered.contains("ECStore {"));
1823 assert!(!rendered.contains("disk_map"));
1824 assert!(!rendered.contains("large-version-id"));
1825 assert!(!rendered.contains("large-expected-version-id"));
1826 assert!(!rendered.contains("large-etag"));
1827 assert!(!rendered.contains("large-if-match"));
1828 assert!(!rendered.contains("large-transition"));
1829 assert!(!rendered.contains("large-rule-id"));
1830 assert!(!rendered.contains("large-storage-class"));
1831 assert!(!rendered.contains("large-user-metadata"));
1832 assert!(!rendered.contains("large-eval-metadata"));
1833 assert!(!rendered.contains("format.json"));
1834 assert!(!rendered.contains("TimedActionSlot"));
1835 assert!(!rendered.contains("DiskHealthTracker"));
1836 }
1837
1838 pub(super) fn build_store_with_ctx(ctx: Arc<InstanceContext>) -> Arc<ECStore> {
1841 let endpoint_pools = EndpointServerPools::default();
1842 Arc::new(ECStore {
1843 id: uuid::Uuid::new_v4(),
1844 disk_map: std::collections::HashMap::new(),
1845 pools: Vec::new(),
1846 peer_sys: crate::cluster::rpc::S3PeerSys::new_with_instance_ctx(&endpoint_pools, ctx.clone()),
1847 pool_meta: RwLock::new(PoolMeta::default()),
1848 rebalance_meta: RwLock::new(None),
1849 decommission_cancelers: RwLock::new(Vec::new()),
1850 start_gate: Mutex::new(()),
1851 pool_meta_save_gate: Mutex::default(),
1852 ctx,
1853 bucket_fence_registry: Arc::default(),
1854 })
1855 }
1856
1857 fn scanner_sequence_state(operation_epoch: u64, movement_generation: u64) -> ScannerDataMovementSequenceState {
1858 ScannerDataMovementSequenceState {
1859 operation_epoch,
1860 operation_epoch_exhausted: false,
1861 movement_generation,
1862 movement_generation_exhausted: false,
1863 }
1864 }
1865
1866 #[test]
1867 fn scanner_pause_status_derives_restart_stable_decommission_fields() {
1868 let started_at = OffsetDateTime::from_unix_timestamp(1_000).expect("fixed timestamp should be valid");
1869 let now = OffsetDateTime::from_unix_timestamp(1_090).expect("fixed timestamp should be valid");
1870 let pool_meta = PoolMeta {
1871 pools: vec![PoolStatus {
1872 id: 0,
1873 cmd_line: "pool-0".to_string(),
1874 last_update: started_at,
1875 decommission: Some(PoolDecommissionInfo {
1876 start_time: Some(started_at),
1877 queued_buckets: vec!["bucket-a".to_string(), "bucket-b".to_string()],
1878 bucket: "bucket-a".to_string(),
1879 ..Default::default()
1880 }),
1881 }],
1882 ..Default::default()
1883 };
1884
1885 let status = resolve_scanner_data_movement_pause_status(&pool_meta, None, false, scanner_sequence_state(7, 11), now);
1886
1887 assert!(status.paused);
1888 assert_eq!(status.policy, "global_pause");
1889 assert_eq!(status.reasons, vec![ScannerDataMovementPauseReason::DecommissionActive]);
1890 assert_eq!(status.started_at_unix_secs, 1_000);
1891 assert_eq!(status.duration_seconds, 90);
1892 assert_eq!(status.operation_epoch, 7);
1893 assert_eq!(status.movement_generation, 11);
1894 assert_eq!(status.movement_backlog_work_items, 2);
1895 assert!(status.movement_backlog_estimated);
1896 }
1897
1898 #[test]
1899 fn completed_decommission_restores_durable_movement_generation() {
1900 let completed_at = OffsetDateTime::from_unix_timestamp(1_100).expect("fixed timestamp should be valid");
1901 let pool_meta = PoolMeta {
1902 pools: vec![PoolStatus {
1903 id: 0,
1904 cmd_line: "pool-0".to_string(),
1905 last_update: completed_at,
1906 decommission: Some(PoolDecommissionInfo {
1907 complete: true,
1908 ..Default::default()
1909 }),
1910 }],
1911 ..Default::default()
1912 };
1913 let durable_generation = durable_scanner_data_movement_generation(&pool_meta, None);
1914 let ctx = InstanceContext::new();
1915
1916 ctx.observe_durable_data_movement_generation(durable_generation);
1917
1918 assert_eq!(durable_generation, 1_100_000_000_000);
1919 assert_eq!(ctx.data_movement_generation(), durable_generation);
1920 }
1921
1922 #[tokio::test]
1923 async fn cleared_decommission_restores_durable_movement_generation_after_restart() {
1924 let mut pool_meta = PoolMeta {
1925 pools: vec![PoolStatus {
1926 id: 0,
1927 cmd_line: "pool-0".to_string(),
1928 last_update: OffsetDateTime::UNIX_EPOCH,
1929 decommission: Some(PoolDecommissionInfo {
1930 failed: true,
1931 ..Default::default()
1932 }),
1933 }],
1934 ..Default::default()
1935 };
1936 assert!(pool_meta.clear_decommission(0).expect("failed decommission should clear"));
1937 assert!(
1938 pool_meta.pools[0]
1939 .decommission
1940 .as_ref()
1941 .is_some_and(|info| !info.has_decommission_state())
1942 );
1943 let durable_generation = durable_scanner_data_movement_generation(&pool_meta, None);
1944 let restarted = build_store_with_ctx(Arc::new(InstanceContext::new()));
1945 *restarted.pool_meta.write().await = pool_meta;
1946
1947 let status = restarted.scanner_data_movement_pause_status().await;
1948
1949 assert_ne!(durable_generation, 0);
1950 assert!(!status.paused);
1951 assert_eq!(status.movement_generation, durable_generation);
1952 assert_eq!(restarted.scanner_data_movement_generation(), durable_generation);
1953 }
1954
1955 #[tokio::test]
1956 async fn same_tick_cleared_decommission_tombstones_advance_durable_movement_generation() {
1957 let same_tick = OffsetDateTime::from_unix_timestamp(1_100).expect("fixed timestamp should be valid");
1958 let mut pool_meta = PoolMeta {
1959 pools: vec![
1960 PoolStatus {
1961 id: 0,
1962 cmd_line: "pool-0".to_string(),
1963 last_update: same_tick,
1964 decommission: Some(PoolDecommissionInfo {
1965 failed: true,
1966 ..Default::default()
1967 }),
1968 },
1969 PoolStatus {
1970 id: 1,
1971 cmd_line: "pool-1".to_string(),
1972 last_update: same_tick,
1973 decommission: Some(PoolDecommissionInfo {
1974 canceled: true,
1975 ..Default::default()
1976 }),
1977 },
1978 ],
1979 ..Default::default()
1980 };
1981
1982 assert!(
1983 pool_meta
1984 .clear_decommission_at_for_test(0, same_tick, None)
1985 .expect("first terminal decommission should clear")
1986 );
1987 let first_generation = durable_scanner_data_movement_generation(&pool_meta, None);
1988 assert_eq!(
1989 first_generation,
1990 scanner_data_movement_timestamp_generation(same_tick + time::Duration::nanoseconds(1))
1991 );
1992
1993 assert!(
1994 pool_meta
1995 .clear_decommission_at_for_test(1, same_tick, None)
1996 .expect("second terminal decommission should clear")
1997 );
1998 let second_generation = durable_scanner_data_movement_generation(&pool_meta, None);
1999 assert_eq!(
2000 second_generation,
2001 scanner_data_movement_timestamp_generation(same_tick + time::Duration::nanoseconds(2))
2002 );
2003 assert!(second_generation > first_generation);
2004
2005 let restarted = build_store_with_ctx(Arc::new(InstanceContext::new()));
2006 *restarted.pool_meta.write().await = pool_meta;
2007 let status = restarted.scanner_data_movement_pause_status().await;
2008
2009 assert!(!status.paused);
2010 assert_eq!(status.movement_generation, second_generation);
2011 assert_eq!(restarted.scanner_data_movement_generation(), second_generation);
2012 }
2013
2014 #[tokio::test]
2015 async fn terminal_decommission_transitions_advance_durable_generation_across_same_or_earlier_clocks() {
2016 let same_tick = OffsetDateTime::from_unix_timestamp(1_200).expect("fixed timestamp should be valid");
2017 let earlier_tick = same_tick - time::Duration::nanoseconds(10);
2018 let rebalance_floor = same_tick + time::Duration::nanoseconds(5);
2019 let rebalance = RebalanceMeta {
2020 stopped_at: Some(rebalance_floor),
2021 id: "completed-rebalance".to_string(),
2022 ..Default::default()
2023 };
2024 let active_decommission = |id| PoolStatus {
2025 id,
2026 cmd_line: format!("pool-{id}"),
2027 last_update: same_tick,
2028 decommission: Some(PoolDecommissionInfo {
2029 start_time: Some(same_tick),
2030 ..Default::default()
2031 }),
2032 };
2033 let mut pool_meta = PoolMeta {
2034 pools: vec![active_decommission(0), active_decommission(1), active_decommission(2)],
2035 ..Default::default()
2036 };
2037
2038 assert!(pool_meta.decommission_complete_at_for_test(0, same_tick, Some(&rebalance)));
2039 assert_eq!(pool_meta.pools[0].last_update, rebalance_floor + time::Duration::nanoseconds(1));
2040
2041 assert!(pool_meta.decommission_cancel_at_for_test(1, same_tick, Some(&rebalance)));
2042 assert_eq!(pool_meta.pools[1].last_update, rebalance_floor + time::Duration::nanoseconds(2));
2043
2044 assert!(pool_meta.decommission_failed_at_for_test(2, earlier_tick, Some(&rebalance)));
2045 assert_eq!(pool_meta.pools[2].last_update, rebalance_floor + time::Duration::nanoseconds(3));
2046 let durable_generation = durable_scanner_data_movement_generation(&pool_meta, Some(&rebalance));
2047 assert_eq!(
2048 durable_generation,
2049 scanner_data_movement_timestamp_generation(rebalance_floor + time::Duration::nanoseconds(3))
2050 );
2051
2052 let restarted = build_store_with_ctx(Arc::new(InstanceContext::new()));
2053 *restarted.pool_meta.write().await = pool_meta;
2054 *restarted.rebalance_meta.write().await = Some(rebalance);
2055 let status = restarted.scanner_data_movement_pause_status().await;
2056
2057 assert_eq!(status.movement_generation, durable_generation);
2058 assert_eq!(restarted.scanner_data_movement_generation(), durable_generation);
2059 assert_eq!(
2060 status.reasons,
2061 vec![
2062 ScannerDataMovementPauseReason::DecommissionFailed,
2063 ScannerDataMovementPauseReason::DecommissionCanceled
2064 ]
2065 );
2066 }
2067
2068 #[tokio::test]
2069 async fn decommission_start_after_clear_advances_durable_generation_across_clock_rollback_after_restart() {
2070 let same_tick = OffsetDateTime::from_unix_timestamp(1_250).expect("fixed timestamp should be valid");
2071 let earlier_tick = same_tick - time::Duration::nanoseconds(10);
2072 let rebalance_floor = same_tick + time::Duration::nanoseconds(5);
2073 let rebalance = RebalanceMeta {
2074 stopped_at: Some(rebalance_floor),
2075 id: "completed-rebalance".to_string(),
2076 ..Default::default()
2077 };
2078 let mut pool_meta = PoolMeta {
2079 pools: vec![PoolStatus {
2080 id: 0,
2081 cmd_line: "pool-0".to_string(),
2082 last_update: same_tick,
2083 decommission: Some(PoolDecommissionInfo {
2084 failed: true,
2085 ..Default::default()
2086 }),
2087 }],
2088 ..Default::default()
2089 };
2090
2091 assert!(
2092 pool_meta
2093 .clear_decommission_at_for_test(0, same_tick, Some(&rebalance))
2094 .expect("failed decommission should clear")
2095 );
2096 let cleared_at = rebalance_floor + time::Duration::nanoseconds(1);
2097 assert_eq!(pool_meta.pools[0].last_update, cleared_at);
2098
2099 pool_meta
2100 .decommission_at_for_test(
2101 0,
2102 PoolSpaceInfo {
2103 total: 200,
2104 free: 50,
2105 used: 150,
2106 },
2107 earlier_tick,
2108 Some(&rebalance),
2109 )
2110 .expect("decommission restart after clear should be allowed");
2111 let started_at = cleared_at + time::Duration::nanoseconds(1);
2112 assert_eq!(pool_meta.pools[0].last_update, started_at);
2113 assert_eq!(
2114 pool_meta.pools[0].decommission.as_ref().and_then(|info| info.start_time),
2115 Some(started_at)
2116 );
2117
2118 assert!(pool_meta.decommission_complete_at_for_test(0, earlier_tick, Some(&rebalance)));
2119 let completed_at = started_at + time::Duration::nanoseconds(1);
2120 assert_eq!(pool_meta.pools[0].last_update, completed_at);
2121 let durable_generation = durable_scanner_data_movement_generation(&pool_meta, Some(&rebalance));
2122 assert_eq!(durable_generation, scanner_data_movement_timestamp_generation(completed_at));
2123
2124 let restarted = build_store_with_ctx(Arc::new(InstanceContext::new()));
2125 *restarted.pool_meta.write().await = pool_meta;
2126 *restarted.rebalance_meta.write().await = Some(rebalance);
2127 let status = restarted.scanner_data_movement_pause_status().await;
2128
2129 assert!(!status.paused);
2130 assert_eq!(status.movement_generation, durable_generation);
2131 assert_eq!(restarted.scanner_data_movement_generation(), durable_generation);
2132 }
2133
2134 #[tokio::test]
2135 async fn decommission_terminal_reload_failure_advances_durable_generation_across_clock_rollback_after_restart() {
2136 let terminal_at = OffsetDateTime::from_unix_timestamp(1_280).expect("fixed timestamp should be valid");
2137 let earlier_tick = terminal_at - time::Duration::nanoseconds(10);
2138 let rebalance_floor = terminal_at + time::Duration::nanoseconds(5);
2139 let rebalance = RebalanceMeta {
2140 stopped_at: Some(rebalance_floor),
2141 id: "completed-rebalance".to_string(),
2142 ..Default::default()
2143 };
2144 let mut pool_meta = PoolMeta {
2145 pools: vec![PoolStatus {
2146 id: 0,
2147 cmd_line: "pool-0".to_string(),
2148 last_update: terminal_at,
2149 decommission: Some(PoolDecommissionInfo {
2150 start_time: Some(terminal_at),
2151 complete: true,
2152 ..Default::default()
2153 }),
2154 }],
2155 ..Default::default()
2156 };
2157
2158 assert!(
2159 pool_meta
2160 .record_decommission_terminal_reload_failure_at_for_test(
2161 0,
2162 "complete_decommission",
2163 "peer reload failed".to_string(),
2164 earlier_tick,
2165 Some(&rebalance),
2166 )
2167 .expect("reload failure should be recorded")
2168 );
2169 let reload_failure_at = rebalance_floor + time::Duration::nanoseconds(1);
2170 assert_eq!(pool_meta.pools[0].last_update, reload_failure_at);
2171 let info = pool_meta.pools[0]
2172 .decommission
2173 .as_ref()
2174 .expect("decommission metadata should exist");
2175 assert_eq!(info.terminal_reload_attempt_at, Some(reload_failure_at));
2176 assert_eq!(
2177 info.terminal_reload_failures,
2178 vec!["complete_decommission: peer reload failed".to_string()]
2179 );
2180 let durable_generation = durable_scanner_data_movement_generation(&pool_meta, Some(&rebalance));
2181 assert_eq!(durable_generation, scanner_data_movement_timestamp_generation(reload_failure_at));
2182
2183 let restarted = build_store_with_ctx(Arc::new(InstanceContext::new()));
2184 *restarted.pool_meta.write().await = pool_meta;
2185 *restarted.rebalance_meta.write().await = Some(rebalance);
2186 let status = restarted.scanner_data_movement_pause_status().await;
2187
2188 assert!(!status.paused);
2189 assert_eq!(status.movement_generation, durable_generation);
2190 assert_eq!(restarted.scanner_data_movement_generation(), durable_generation);
2191 }
2192
2193 #[tokio::test]
2194 async fn rebalance_transitions_advance_durable_generation_across_same_or_earlier_clocks_after_restart() {
2195 let same_tick = OffsetDateTime::from_unix_timestamp(1_300).expect("fixed timestamp should be valid");
2196 let earlier_tick = same_tick - time::Duration::nanoseconds(10);
2197 let decommission_floor = same_tick + time::Duration::nanoseconds(5);
2198 let store = build_store_with_ctx(Arc::new(InstanceContext::new()));
2199 *store.pool_meta.write().await = PoolMeta {
2200 pools: vec![PoolStatus {
2201 id: 0,
2202 cmd_line: "pool-0".to_string(),
2203 last_update: decommission_floor,
2204 decommission: Some(PoolDecommissionInfo {
2205 complete: true,
2206 ..Default::default()
2207 }),
2208 }],
2209 ..Default::default()
2210 };
2211
2212 let started_at = store.next_scanner_data_movement_update(same_tick).await;
2213 assert_eq!(started_at, decommission_floor + time::Duration::nanoseconds(1));
2214 *store.rebalance_meta.write().await = Some(RebalanceMeta {
2215 id: "rebalance-generation".to_string(),
2216 pool_stats: vec![crate::services::rebalance::RebalanceStats {
2217 participating: true,
2218 info: crate::services::rebalance::RebalanceInfo {
2219 start_time: Some(started_at),
2220 status: RebalStatus::Started,
2221 ..Default::default()
2222 },
2223 ..Default::default()
2224 }],
2225 ..Default::default()
2226 });
2227
2228 let completed_at = store.next_scanner_data_movement_update(same_tick).await;
2229 assert_eq!(completed_at, decommission_floor + time::Duration::nanoseconds(2));
2230 {
2231 let mut rebalance_meta = store.rebalance_meta.write().await;
2232 let meta = rebalance_meta.as_mut().expect("rebalance metadata should be present");
2233 meta.pool_stats[0].info.status = RebalStatus::Completed;
2234 meta.pool_stats[0].info.end_time = Some(completed_at);
2235 }
2236
2237 let stopped_at = store.next_scanner_data_movement_update(earlier_tick).await;
2238 assert_eq!(stopped_at, decommission_floor + time::Duration::nanoseconds(3));
2239 {
2240 let mut rebalance_meta = store.rebalance_meta.write().await;
2241 let meta = rebalance_meta.as_mut().expect("rebalance metadata should be present");
2242 meta.stopped_at = Some(stopped_at);
2243 }
2244 let pool_meta = store.pool_meta.read().await.clone();
2245 let rebalance_meta = store.rebalance_meta.read().await.clone();
2246 let durable_generation = durable_scanner_data_movement_generation(&pool_meta, rebalance_meta.as_ref());
2247 assert_eq!(
2248 durable_generation,
2249 scanner_data_movement_timestamp_generation(decommission_floor + time::Duration::nanoseconds(3))
2250 );
2251
2252 let restarted = build_store_with_ctx(Arc::new(InstanceContext::new()));
2253 *restarted.pool_meta.write().await = pool_meta;
2254 *restarted.rebalance_meta.write().await = rebalance_meta;
2255 let status = restarted.scanner_data_movement_pause_status().await;
2256
2257 assert!(!status.paused);
2258 assert_eq!(status.movement_generation, durable_generation);
2259 assert_eq!(restarted.scanner_data_movement_generation(), durable_generation);
2260 }
2261
2262 #[test]
2263 fn malformed_durable_movement_timestamp_exhausts_generation_fail_closed() {
2264 let pool_meta = PoolMeta {
2265 pools: vec![PoolStatus {
2266 id: 0,
2267 cmd_line: "pool-0".to_string(),
2268 last_update: OffsetDateTime::UNIX_EPOCH,
2269 decommission: Some(PoolDecommissionInfo {
2270 complete: true,
2271 ..Default::default()
2272 }),
2273 }],
2274 ..Default::default()
2275 };
2276
2277 assert_eq!(durable_scanner_data_movement_generation(&pool_meta, None), u64::MAX);
2278 let exhausted_generation =
2279 OffsetDateTime::from_unix_timestamp(253_402_300_799).expect("the largest RFC 3339 timestamp should be valid");
2280 assert_eq!(scanner_data_movement_timestamp_generation(exhausted_generation), u64::MAX);
2281 }
2282
2283 #[test]
2284 fn malformed_durable_movement_timestamp_is_not_masked_by_valid_rebalance_generation() {
2285 let valid_rebalance_at = OffsetDateTime::from_unix_timestamp(2_400).expect("fixed timestamp should be valid");
2286 let pool_meta = PoolMeta {
2287 pools: vec![PoolStatus {
2288 id: 0,
2289 cmd_line: "pool-0".to_string(),
2290 last_update: OffsetDateTime::UNIX_EPOCH,
2291 decommission: Some(PoolDecommissionInfo {
2292 complete: true,
2293 ..Default::default()
2294 }),
2295 }],
2296 ..Default::default()
2297 };
2298 let rebalance_meta = RebalanceMeta {
2299 id: "completed-rebalance".to_string(),
2300 stopped_at: Some(valid_rebalance_at),
2301 pool_stats: vec![crate::services::rebalance::RebalanceStats {
2302 participating: true,
2303 info: crate::services::rebalance::RebalanceInfo {
2304 start_time: Some(valid_rebalance_at - time::Duration::nanoseconds(1)),
2305 end_time: Some(valid_rebalance_at),
2306 status: RebalStatus::Completed,
2307 ..Default::default()
2308 },
2309 ..Default::default()
2310 }],
2311 ..Default::default()
2312 };
2313
2314 assert_eq!(durable_scanner_data_movement_generation(&pool_meta, Some(&rebalance_meta)), u64::MAX);
2315 }
2316
2317 #[test]
2318 fn durable_movement_generation_without_records_is_zero() {
2319 assert_eq!(durable_scanner_data_movement_generation(&PoolMeta::default(), None), 0);
2320 assert_eq!(
2321 durable_scanner_data_movement_generation(&PoolMeta::default(), Some(&RebalanceMeta::default())),
2322 0
2323 );
2324 }
2325
2326 #[test]
2327 fn scanner_pause_status_distinguishes_terminal_rebalance_epoch_and_idle() {
2328 let last_update = OffsetDateTime::from_unix_timestamp(2_000).expect("fixed timestamp should be valid");
2329 let now = OffsetDateTime::from_unix_timestamp(2_030).expect("fixed timestamp should be valid");
2330 let failed = PoolMeta {
2331 pools: vec![PoolStatus {
2332 id: 0,
2333 cmd_line: "pool-0".to_string(),
2334 last_update,
2335 decommission: Some(PoolDecommissionInfo {
2336 failed: true,
2337 ..Default::default()
2338 }),
2339 }],
2340 ..Default::default()
2341 };
2342 let failed_status = resolve_scanner_data_movement_pause_status(&failed, None, false, scanner_sequence_state(3, 12), now);
2343 assert_eq!(failed_status.reasons, vec![ScannerDataMovementPauseReason::DecommissionFailed]);
2344 assert_eq!(failed_status.started_at_unix_secs, 2_000);
2345 assert_eq!(failed_status.duration_seconds, 30);
2346
2347 let rebalance = RebalanceMeta {
2348 pool_stats: vec![crate::services::rebalance::RebalanceStats {
2349 buckets: vec!["bucket-a".to_string(), "bucket-b".to_string()],
2350 participating: true,
2351 info: crate::services::rebalance::RebalanceInfo {
2352 start_time: Some(last_update),
2353 status: RebalStatus::Started,
2354 ..Default::default()
2355 },
2356 ..Default::default()
2357 }],
2358 ..Default::default()
2359 };
2360 let rebalance_status = resolve_scanner_data_movement_pause_status(
2361 &PoolMeta::default(),
2362 Some(&rebalance),
2363 false,
2364 scanner_sequence_state(4, 13),
2365 now,
2366 );
2367 assert_eq!(rebalance_status.reasons, vec![ScannerDataMovementPauseReason::RebalanceActive]);
2368 assert_eq!(rebalance_status.movement_backlog_work_items, 2);
2369
2370 let exhausted = resolve_scanner_data_movement_pause_status(
2371 &PoolMeta::default(),
2372 None,
2373 false,
2374 ScannerDataMovementSequenceState {
2375 operation_epoch: u64::MAX,
2376 operation_epoch_exhausted: true,
2377 movement_generation: 14,
2378 movement_generation_exhausted: false,
2379 },
2380 now,
2381 );
2382 assert_eq!(exhausted.reasons, vec![ScannerDataMovementPauseReason::OperationEpochExhausted]);
2383 assert_eq!(exhausted.started_at_unix_secs, 0);
2384
2385 let generation_exhausted = resolve_scanner_data_movement_pause_status(
2386 &PoolMeta::default(),
2387 None,
2388 false,
2389 ScannerDataMovementSequenceState {
2390 operation_epoch: 5,
2391 operation_epoch_exhausted: false,
2392 movement_generation: u64::MAX,
2393 movement_generation_exhausted: true,
2394 },
2395 now,
2396 );
2397 assert_eq!(
2398 generation_exhausted.reasons,
2399 vec![ScannerDataMovementPauseReason::MovementGenerationExhausted]
2400 );
2401
2402 let idle =
2403 resolve_scanner_data_movement_pause_status(&PoolMeta::default(), None, false, scanner_sequence_state(5, 15), now);
2404 assert!(!idle.paused);
2405 assert!(idle.reasons.is_empty());
2406 assert!(!idle.movement_backlog_estimated);
2407 }
2408
2409 #[tokio::test]
2410 async fn scanner_data_usage_publication_blocks_active_and_unqueued_terminal_decommission() {
2411 let store = build_store_with_ctx(Arc::new(InstanceContext::new()));
2412 let cases = [
2413 (
2414 "active",
2415 PoolDecommissionInfo {
2416 start_time: Some(OffsetDateTime::now_utc()),
2417 ..Default::default()
2418 },
2419 true,
2420 ),
2421 (
2422 "failed",
2423 PoolDecommissionInfo {
2424 failed: true,
2425 ..Default::default()
2426 },
2427 true,
2428 ),
2429 (
2430 "canceled",
2431 PoolDecommissionInfo {
2432 canceled: true,
2433 ..Default::default()
2434 },
2435 true,
2436 ),
2437 (
2438 "queued_failed",
2439 PoolDecommissionInfo {
2440 failed: true,
2441 queued: true,
2442 ..Default::default()
2443 },
2444 false,
2445 ),
2446 (
2447 "complete",
2448 PoolDecommissionInfo {
2449 complete: true,
2450 ..Default::default()
2451 },
2452 false,
2453 ),
2454 ("idle", PoolDecommissionInfo::default(), false),
2455 ];
2456
2457 for (name, decommission, expected) in cases {
2458 *store.pool_meta.write().await = PoolMeta {
2459 pools: vec![PoolStatus {
2460 id: 0,
2461 cmd_line: format!("scanner-publication-{name}"),
2462 last_update: OffsetDateTime::now_utc(),
2463 decommission: Some(decommission),
2464 }],
2465 ..Default::default()
2466 };
2467 assert_eq!(
2468 store.scanner_data_usage_publication_blocked().await,
2469 expected,
2470 "unexpected scanner publication barrier state for {name}"
2471 );
2472 }
2473 }
2474
2475 #[tokio::test]
2476 async fn scanner_data_usage_publication_admission_is_fenced_and_epoch_monotonic() {
2477 let store = build_store_with_ctx(Arc::new(InstanceContext::new()));
2478 let operation_gate = store.ctx.data_movement_operation_gate();
2479 let movement_guard = operation_gate.write().await;
2480 let pending = {
2481 let store = store.clone();
2482 tokio::spawn(async move { store.scanner_data_usage_publication_admission_guard().await })
2483 };
2484 tokio::task::yield_now().await;
2485 assert!(!pending.is_finished(), "publication admission must wait for a movement writer");
2486 drop(movement_guard);
2487
2488 let (_, epoch) = pending
2489 .await
2490 .expect("publication admission task should not panic")
2491 .expect("idle store should admit publication");
2492 assert_eq!(epoch, 0);
2493 assert_eq!(store.ctx.advance_data_movement_operation_epoch(), 1);
2494 let (_, next_epoch) = store
2495 .scanner_data_usage_publication_admission_guard()
2496 .await
2497 .expect("idle store should admit the next publication");
2498 assert_eq!(next_epoch, 1);
2499 assert_eq!(store.scanner_data_movement_generation(), 1);
2500 }
2501
2502 #[tokio::test(start_paused = true)]
2503 async fn scanner_publication_lease_blocks_movement_writer_until_release_or_expiry() {
2504 let store = build_store_with_ctx(Arc::new(InstanceContext::new()));
2505 let (token, generation) = store
2506 .acquire_scanner_publication_lease(0, crate::runtime::instance::SCANNER_PUBLICATION_LEASE_TTL)
2507 .await
2508 .expect("an idle store should grant a publication lease");
2509 assert_eq!(generation, 0);
2510
2511 let gate = store.ctx.data_movement_operation_gate();
2512 let (writer_started, writer_started_rx) = tokio::sync::oneshot::channel();
2513 let movement_writer = tokio::spawn(async move {
2514 let _ = writer_started.send(());
2515 gate.write_owned().await
2516 });
2517 writer_started_rx
2518 .await
2519 .expect("movement writer should reach the gate before waiting");
2520 assert!(
2521 !movement_writer.is_finished(),
2522 "a movement writer must wait while the remote lease owns the read guard"
2523 );
2524
2525 assert!(store.release_scanner_publication_lease(token).await);
2526 tokio::time::timeout(Duration::from_secs(1), movement_writer)
2527 .await
2528 .expect("movement writer should proceed after lease release")
2529 .expect("movement writer task should not panic");
2530
2531 let (expiring_token, _) = store
2532 .acquire_scanner_publication_lease(0, crate::runtime::instance::SCANNER_PUBLICATION_LEASE_TTL)
2533 .await
2534 .expect("the store should grant a second publication lease");
2535 let expiry_gate = store.ctx.data_movement_operation_gate();
2536 let (expiry_started, expiry_started_rx) = tokio::sync::oneshot::channel();
2537 let mut expiry_writer = tokio::spawn(async move {
2538 let _ = expiry_started.send(());
2539 expiry_gate.write_owned().await
2540 });
2541 expiry_started_rx
2542 .await
2543 .expect("expiry writer should reach the gate before waiting");
2544 assert!(!expiry_writer.is_finished());
2545 tokio::time::advance(crate::runtime::instance::SCANNER_PUBLICATION_LEASE_TTL + Duration::from_millis(1)).await;
2546 tokio::task::yield_now().await;
2547 tokio::time::timeout(Duration::from_secs(1), &mut expiry_writer)
2548 .await
2549 .expect("movement writer should proceed after lease expiry")
2550 .expect("expiry writer task should not panic");
2551 assert!(
2552 store.validate_scanner_publication_lease(expiring_token, 0).await.is_err(),
2553 "an expired lease must not validate after its read guard is released"
2554 );
2555 assert!(!store.release_scanner_publication_lease(expiring_token).await);
2556 }
2557
2558 #[tokio::test]
2559 async fn scanner_publication_lease_rejects_a_new_movement_generation() {
2560 let store = build_store_with_ctx(Arc::new(InstanceContext::new()));
2561 let (token, generation) = store
2562 .acquire_scanner_publication_lease(0, crate::runtime::instance::SCANNER_PUBLICATION_LEASE_TTL)
2563 .await
2564 .expect("an idle store should grant a publication lease");
2565
2566 assert_eq!(store.ctx.advance_data_movement_generation(), Some(1));
2567 assert!(
2568 store.validate_scanner_publication_lease(token, generation).await.is_err(),
2569 "a lease from the prior movement generation must fail closed"
2570 );
2571 assert!(store.release_scanner_publication_lease(token).await);
2572 }
2573
2574 #[tokio::test]
2575 async fn scanner_publication_lease_rejects_stale_generation_before_install() {
2576 let store = build_store_with_ctx(Arc::new(InstanceContext::new()));
2577 let error = store
2578 .acquire_scanner_publication_lease(1, crate::runtime::instance::SCANNER_PUBLICATION_LEASE_TTL)
2579 .await
2580 .expect_err("a stale movement generation must not acquire a lease");
2581 assert!(error.to_string().contains("generation is stale"));
2582 }
2583
2584 #[tokio::test]
2585 async fn scanner_publication_lease_rejects_namespace_change_during_admission() {
2586 let ctx = Arc::new(InstanceContext::new());
2587 let store = build_store_with_ctx(ctx.clone());
2588 let snapshot_blocker = store.rebalance_meta.write().await;
2589 let mut acquire =
2590 Box::pin(store.acquire_scanner_publication_lease(0, crate::runtime::instance::SCANNER_PUBLICATION_LEASE_TTL));
2591 assert!(futures::poll!(&mut acquire).is_pending(), "acquisition reaches the blocked snapshot");
2592 assert!(ctx.data_movement_operation_gate().try_write_owned().is_err());
2593 drop(ctx.begin_namespace_commit());
2594 assert_eq!(ctx.namespace_commit_generation(), 2);
2595 assert!(!ctx.namespace_commits_pending());
2596 drop(snapshot_blocker);
2597 let error = tokio::time::timeout(Duration::from_secs(1), acquire)
2598 .await
2599 .expect("snapshot admission must finish")
2600 .expect_err("acquisition must preserve its original namespace generation");
2601 assert_eq!(error.to_string(), "Io error: scanner publication lease generation is stale");
2602 assert!(ctx.data_movement_operation_gate().try_write_owned().is_ok(), "no lease was installed");
2603 }
2604
2605 #[tokio::test]
2606 async fn scanner_publication_lease_rechecks_namespace_after_validate_snapshot() {
2607 let ctx = Arc::new(InstanceContext::new());
2608 let store = build_store_with_ctx(ctx.clone());
2609 let (token, generation) = store
2610 .acquire_scanner_publication_lease(0, crate::runtime::instance::SCANNER_PUBLICATION_LEASE_TTL)
2611 .await
2612 .expect("current lease");
2613 let snapshot_blocker = store.rebalance_meta.write().await;
2614 let mut validate = Box::pin(store.validate_scanner_publication_lease(token, generation));
2615 assert!(futures::poll!(&mut validate).is_pending(), "target guard queues its first snapshot read");
2616 let mut next_writer = Box::pin(store.rebalance_meta.write());
2617 assert!(futures::poll!(&mut next_writer).is_pending());
2618 drop(snapshot_blocker);
2619 assert!(futures::poll!(&mut validate).is_pending(), "validate reaches its second snapshot");
2622 let next_writer = next_writer.await;
2623 drop(ctx.begin_namespace_commit());
2624 assert_eq!(ctx.namespace_commit_generation(), 2);
2625 assert!(!ctx.namespace_commits_pending());
2626 drop(next_writer);
2627 let error = tokio::time::timeout(Duration::from_secs(1), validate)
2628 .await
2629 .expect("validation must finish without nesting movement read locks")
2630 .expect_err("the final snapshot must reject a completed namespace commit");
2631 assert_eq!(error.to_string(), "Io error: scanner publication lease generation is stale");
2632 assert!(
2633 ctx.data_movement_operation_gate().try_write_owned().is_err(),
2634 "stale lookup retains the lease permit"
2635 );
2636 assert!(store.release_scanner_publication_lease(token).await);
2637 assert!(ctx.data_movement_operation_gate().try_write_owned().is_ok());
2638 }
2639
2640 #[tokio::test]
2641 async fn scanner_publication_lease_rejects_namespace_generation_exhaustion() {
2642 let ctx = Arc::new(InstanceContext::new());
2643 let store = build_store_with_ctx(ctx.clone());
2644 let (token, generation) = store
2645 .acquire_scanner_publication_lease(0, crate::runtime::instance::SCANNER_PUBLICATION_LEASE_TTL)
2646 .await
2647 .expect("current lease");
2648 ctx.set_namespace_commit_generation_for_test(u64::MAX);
2649 assert_eq!(store.scanner_namespace_mutation_generation(), u64::MAX);
2650 let acquire = store
2651 .acquire_scanner_publication_lease(generation, crate::runtime::instance::SCANNER_PUBLICATION_LEASE_TTL)
2652 .await
2653 .expect_err("exhausted namespace cannot become a new lease baseline");
2654 assert_eq!(acquire.to_string(), "Io error: scanner publication lease generation is stale");
2655 let validate = store.validate_scanner_publication_lease(token, generation).await;
2656 let target = store.acquire_scanner_publication_lease_guard(token).await;
2657 assert!(validate.is_err() && target.is_err(), "exhaustion rejects both old-token entrances");
2658 assert!(
2659 ctx.data_movement_operation_gate().try_write_owned().is_err(),
2660 "rejection must retain the old permit"
2661 );
2662 assert!(store.release_scanner_publication_lease(token).await);
2663 assert!(ctx.data_movement_operation_gate().try_write_owned().is_ok());
2664 }
2665
2666 #[tokio::test(start_paused = true)]
2667 async fn scanner_publication_commit_scope_owns_permit_until_terminal_drain() {
2668 let store = build_store_with_ctx(Arc::new(InstanceContext::new()));
2669 let scope = store
2670 .scanner_data_usage_publication_commit_scope(
2671 0,
2672 tokio::time::Instant::now() + Duration::from_secs(30),
2673 vec![Uuid::new_v4()],
2674 )
2675 .await
2676 .expect("idle storage should grant a publication scope");
2677 assert_eq!(scope.state(), crate::object_api::ScannerPublicationCommitState::Admitted);
2678 assert_eq!(scope.remote_lease_tokens().len(), 1);
2679
2680 let gate = store.ctx.data_movement_operation_gate();
2681 let writer = tokio::spawn(async move { gate.write_owned().await });
2682 tokio::task::yield_now().await;
2683 assert!(!writer.is_finished(), "the scope must own its movement permit after the caller returns");
2684
2685 scope.cancel();
2686 assert!(scope.mark_aborted_before_commit());
2687 assert_eq!(
2688 scope.wait_for_completion().await,
2689 crate::object_api::ScannerPublicationCommitState::AbortedBeforeCommit
2690 );
2691 assert!(scope.release_movement_permit().await);
2692 tokio::time::timeout(Duration::from_secs(1), writer)
2693 .await
2694 .expect("movement writer should proceed after the scope drains")
2695 .expect("movement writer task should not panic");
2696 }
2697
2698 #[tokio::test(start_paused = true)]
2699 async fn scanner_publication_commit_scope_rejects_late_start_and_keeps_indeterminate_permit() {
2700 let store = build_store_with_ctx(Arc::new(InstanceContext::new()));
2701 let scope = store
2702 .scanner_data_usage_publication_commit_scope(0, tokio::time::Instant::now() + Duration::from_secs(1), Vec::new())
2703 .await
2704 .expect("idle storage should grant a publication scope");
2705 tokio::time::advance(Duration::from_secs(1)).await;
2706 assert_eq!(
2707 scope.try_begin(),
2708 Err(crate::object_api::ScannerPublicationCommitStartError::DeadlineExceeded)
2709 );
2710 assert!(
2711 !scope.release_movement_permit().await,
2712 "an admitted scope is not safe to release before owner resolution"
2713 );
2714 assert!(scope.mark_aborted_before_commit());
2715 assert!(scope.release_movement_permit().await);
2716
2717 let scope = store
2718 .scanner_data_usage_publication_commit_scope(0, tokio::time::Instant::now() + Duration::from_secs(30), Vec::new())
2719 .await
2720 .expect("a second idle publication scope should be granted");
2721 scope.try_begin().expect("scope should enter the mutation state");
2722 scope.cancel();
2723 assert!(scope.mark_indeterminate());
2724 assert_eq!(
2725 scope.wait_for_completion().await,
2726 crate::object_api::ScannerPublicationCommitState::Indeterminate
2727 );
2728 assert!(!scope.release_movement_permit().await, "indeterminate mutation must retain the permit");
2729 }
2730
2731 #[tokio::test]
2732 async fn scanner_publication_scope_guard_classifies_early_returns_conservatively() {
2733 let store = build_store_with_ctx(Arc::new(InstanceContext::new()));
2734 let permit = store.ctx.data_movement_operation_gate().read_owned().await;
2735 let scope = ScannerPublicationCommitScope::new_storage_owned(
2736 0,
2737 tokio::time::Instant::now() + Duration::from_secs(30),
2738 Vec::new(),
2739 permit,
2740 );
2741 {
2742 let _guard = crate::object_api::ScannerPublicationCommitScopeGuard::new(scope.clone());
2743 }
2744 assert_eq!(scope.state(), crate::object_api::ScannerPublicationCommitState::AbortedBeforeCommit);
2745 assert!(scope.release_movement_permit().await);
2746
2747 let permit = store.ctx.data_movement_operation_gate().read_owned().await;
2748 let scope = ScannerPublicationCommitScope::new_storage_owned(
2749 0,
2750 tokio::time::Instant::now() + Duration::from_secs(30),
2751 Vec::new(),
2752 permit,
2753 );
2754 scope.try_begin().expect("scope should enter the mutation state");
2755 {
2756 let _guard = crate::object_api::ScannerPublicationCommitScopeGuard::new(scope.clone());
2757 }
2758 assert_eq!(scope.state(), crate::object_api::ScannerPublicationCommitState::Indeterminate);
2759 assert!(!scope.release_movement_permit().await);
2760 }
2761
2762 #[tokio::test]
2763 async fn scanner_target_guard_keeps_movement_writer_fenced_after_lease_release() {
2764 let store = build_store_with_ctx(Arc::new(InstanceContext::new()));
2765 let (token, _) = store
2766 .acquire_scanner_publication_lease(0, crate::runtime::instance::SCANNER_PUBLICATION_LEASE_TTL)
2767 .await
2768 .expect("an idle store should grant a publication lease");
2769 let target_guard = store
2770 .acquire_scanner_publication_lease_guard(token)
2771 .await
2772 .expect("the target-side rename should acquire its short guard");
2773 assert!(store.release_scanner_publication_lease(token).await);
2774
2775 let movement_gate = store.ctx.data_movement_operation_gate();
2776 let movement_writer = tokio::spawn(async move { movement_gate.write_owned().await });
2777 tokio::task::yield_now().await;
2778 assert!(!movement_writer.is_finished(), "the target guard must span the rename operation");
2779 drop(target_guard);
2780 tokio::time::timeout(std::time::Duration::from_secs(1), movement_writer)
2781 .await
2782 .expect("movement writer should proceed after the target rename guard is dropped")
2783 .expect("movement writer task should not panic");
2784 }
2785
2786 #[tokio::test]
2787 async fn scanner_publication_lease_rejects_restart_aba_token() {
2788 let first_store = build_store_with_ctx(Arc::new(InstanceContext::new()));
2789 let (token, generation) = first_store
2790 .acquire_scanner_publication_lease(0, crate::runtime::instance::SCANNER_PUBLICATION_LEASE_TTL)
2791 .await
2792 .expect("the initial instance should grant a publication lease");
2793 first_store
2794 .validate_scanner_publication_lease(token, generation)
2795 .await
2796 .expect("the current instance should validate its own live token");
2797 first_store
2798 .acquire_scanner_publication_lease_guard(token)
2799 .await
2800 .expect("the current instance should admit the target-side rename");
2801
2802 let restarted_store = build_store_with_ctx(Arc::new(InstanceContext::new()));
2806 let error = restarted_store
2807 .validate_scanner_publication_lease(token, generation)
2808 .await
2809 .expect_err("a token from a prior instance must fail closed after restart");
2810 assert!(error.to_string().contains("unknown or expired"));
2811 let error = restarted_store
2812 .acquire_scanner_publication_lease_guard(token)
2813 .await
2814 .expect_err("a restarted instance must reject the target-side rename token");
2815 assert!(error.to_string().contains("unknown or expired"));
2816 }
2817
2818 #[tokio::test]
2819 async fn movement_generation_notifies_waiters_and_fails_closed_at_maximum() {
2820 let store = build_store_with_ctx(Arc::new(InstanceContext::new()));
2821 let notify = store.scanner_data_movement_changed();
2822 let notified = notify.notified();
2823 tokio::pin!(notified);
2824 notified.as_mut().enable();
2825
2826 assert_eq!(store.ctx.advance_data_movement_operation_epoch(), 1);
2827 tokio::time::timeout(std::time::Duration::from_secs(1), notified)
2828 .await
2829 .expect("movement transition should wake scanner waiters");
2830 assert_eq!(store.scanner_data_movement_generation(), 1);
2831
2832 store.ctx.set_data_movement_generation_for_test(u64::MAX - 1);
2833 assert_eq!(store.ctx.advance_data_movement_generation(), Some(u64::MAX));
2834 assert!(store.scanner_data_movement_generation_exhausted());
2835 assert_eq!(store.ctx.advance_data_movement_generation(), None);
2836 assert!(
2837 store.scanner_data_usage_publication_admission_guard().await.is_none(),
2838 "generation exhaustion must close publication admission"
2839 );
2840 }
2841
2842 #[tokio::test]
2843 async fn scanner_data_usage_publication_epoch_releases_gate_before_backend_io() {
2844 let store = build_store_with_ctx(Arc::new(InstanceContext::new()));
2845 let epoch = store
2846 .scanner_data_usage_publication_epoch()
2847 .await
2848 .expect("idle store should expose a publication epoch");
2849 assert_eq!(epoch, 0);
2850
2851 let operation_gate = store.ctx.data_movement_operation_gate();
2852 let _movement_guard = tokio::time::timeout(Duration::from_secs(1), operation_gate.write())
2853 .await
2854 .expect("epoch capture must not hold the movement gate across backend I/O");
2855 }
2856
2857 #[tokio::test]
2858 async fn scanner_data_usage_publication_admission_blocks_active_rebalance_snapshot() {
2859 let store = build_store_with_ctx(Arc::new(InstanceContext::new()));
2860 *store.rebalance_meta.write().await = Some(RebalanceMeta {
2861 pool_stats: vec![crate::services::rebalance::RebalanceStats {
2862 participating: true,
2863 info: crate::services::rebalance::RebalanceInfo {
2864 status: crate::services::rebalance::RebalStatus::Started,
2865 ..Default::default()
2866 },
2867 ..Default::default()
2868 }],
2869 ..Default::default()
2870 });
2871
2872 assert!(
2873 store.scanner_data_usage_publication_admission_guard().await.is_none(),
2874 "active rebalance must fail closed at the storage-owned admission boundary"
2875 );
2876 }
2877
2878 #[tokio::test]
2879 async fn scanner_publication_epoch_exhaustion_fails_closed_after_max() {
2880 let store = build_store_with_ctx(Arc::new(InstanceContext::new()));
2881 store.ctx.set_data_movement_operation_epoch_for_test(u64::MAX - 1);
2882
2883 assert_eq!(store.ctx.advance_data_movement_operation_epoch(), u64::MAX);
2884 assert!(store.ctx.data_movement_operation_epoch_exhausted());
2885 assert!(
2886 store.scanner_data_usage_publication_admission_guard().await.is_none(),
2887 "publication must fail closed at the reserved terminal epoch"
2888 );
2889
2890 assert_eq!(store.ctx.advance_data_movement_operation_epoch(), u64::MAX);
2891 assert!(store.ctx.data_movement_operation_epoch_exhausted());
2892 assert!(store.scanner_data_usage_publication_blocked().await);
2893 }
2894
2895 #[tokio::test]
2899 async fn instance_context_carrier_isolates_two_stores() {
2900 let ctx_a = Arc::new(InstanceContext::new());
2901 let ctx_b = Arc::new(InstanceContext::new());
2902 ctx_a.update_erasure_type(SetupType::DistErasure).await;
2903 ctx_b.update_erasure_type(SetupType::ErasureSD).await;
2904 ctx_a.set_endpoints(EndpointServerPools::from(vec![PoolEndpoints {
2905 legacy: false,
2906 set_count: 1,
2907 drives_per_set: 1,
2908 endpoints: Endpoints::default(),
2909 cmd_line: "instance-a".to_string(),
2910 platform: String::new(),
2911 }]));
2912 ctx_b.set_endpoints(EndpointServerPools::from(vec![PoolEndpoints {
2913 legacy: true,
2914 set_count: 2,
2915 drives_per_set: 2,
2916 endpoints: Endpoints::default(),
2917 cmd_line: "instance-b".to_string(),
2918 platform: String::new(),
2919 }]));
2920
2921 let store_a = build_store_with_ctx(ctx_a);
2922 let store_b = build_store_with_ctx(ctx_b);
2923
2924 assert!(store_a.setup_is_erasure().await);
2926 assert!(store_a.setup_is_dist_erasure().await);
2927 assert!(!store_a.setup_is_erasure_sd().await);
2928
2929 assert!(store_b.setup_is_erasure_sd().await);
2931 assert!(!store_b.setup_is_erasure().await);
2932 assert!(!store_b.setup_is_dist_erasure().await);
2933 let endpoints_a = store_a.instance_endpoints().expect("instance A endpoints");
2934 let endpoints_b = store_b.instance_endpoints().expect("instance B endpoints");
2935 assert_eq!(endpoints_a.as_ref()[0].set_count, 1);
2936 assert_eq!(endpoints_b.as_ref()[0].set_count, 2);
2937 assert!(!endpoints_a.as_ref()[0].legacy);
2938 assert!(endpoints_b.as_ref()[0].legacy);
2939 }
2940
2941 #[tokio::test]
2945 async fn store_adopts_bootstrap_context() {
2946 let store = build_store_with_ctx(crate::runtime::instance::bootstrap_ctx());
2947 assert!(
2948 Arc::ptr_eq(&store.ctx, &crate::runtime::instance::bootstrap_ctx()),
2949 "store built via adoption must share the bootstrap context Arc"
2950 );
2951 }
2952
2953 #[tokio::test]
2954 async fn test_has_space_for() {
2955 let disk_infos = vec![None, None]; let result = crate::layout::pool_space::has_space_for(&disk_infos, 1024).await;
2958 assert!(result.is_err());
2960 }
2961
2962 #[tokio::test]
2963 async fn test_find_local_disk() {
2964 let result = peer::find_local_disk("/nonexistent/path").await;
2965 assert!(result.is_none(), "Should return None for nonexistent path");
2966 }
2967
2968 #[tokio::test]
2969 #[serial]
2970 async fn test_find_local_disk_by_ref_backfills_uuid_map() {
2971 reset_local_disk_test_state().await;
2972
2973 let temp_dir = TempDir::new().expect("create temp dir for local disk ref test");
2974 let disk_paths = (0..4)
2975 .map(|idx| temp_dir.path().join(format!("disk{}", idx + 1)))
2976 .collect::<Vec<_>>();
2977 for disk_path in &disk_paths {
2978 std::fs::create_dir_all(disk_path).expect("create disk path");
2979 }
2980
2981 let mut endpoints = Vec::new();
2982 for (idx, disk_path) in disk_paths.iter().enumerate() {
2983 let mut endpoint = Endpoint::try_from(disk_path.to_str().expect("disk path to str")).expect("endpoint");
2984 endpoint.set_pool_index(0);
2985 endpoint.set_set_index(0);
2986 endpoint.set_disk_index(idx);
2987 endpoints.push(endpoint);
2988 }
2989
2990 let endpoint_pools = EndpointServerPools(vec![PoolEndpoints {
2991 legacy: false,
2992 set_count: 1,
2993 drives_per_set: 4,
2994 endpoints: Endpoints::from(endpoints),
2995 cmd_line: "find-local-disk-by-ref-test".to_string(),
2996 platform: "test".to_string(),
2997 }]);
2998
2999 init_local_disks(endpoint_pools.clone()).await.expect("init local disks");
3000
3001 let (mut disks, errs) = init_disks(
3002 &endpoint_pools.as_ref().first().expect("pool endpoints").endpoints,
3003 &DiskOption {
3004 cleanup: true,
3005 health_check: false,
3006 },
3007 )
3008 .await;
3009
3010 assert!(errs.iter().all(|err| err.is_none()), "disk init should succeed: {errs:?}");
3011 connect_load_init_formats(true, &mut disks, 1, 4, None)
3012 .await
3013 .expect("initialize format metadata");
3014
3015 clear_local_disk_id_map_for_test().await;
3016
3017 let local_disks = all_local_disk().await;
3018 let first_disk = local_disks.first().expect("local disk exists");
3019 let disk_id = first_disk
3020 .get_disk_id()
3021 .await
3022 .expect("get disk id should succeed")
3023 .expect("disk id should exist");
3024
3025 let found = find_local_disk_by_ref(&disk_id.to_string()).await;
3026 assert!(found.is_some(), "disk lookup by id should backfill cache");
3027 assert_eq!(local_disk_path_by_id(&disk_id).await, Some(first_disk.endpoint().to_string()));
3028
3029 reset_local_disk_test_state().await;
3030 }
3031
3032 #[tokio::test]
3033 async fn test_all_local_disk_path() {
3034 let paths = all_local_disk_path().await;
3035 assert!(paths.is_empty() || !paths.is_empty());
3037 }
3038
3039 #[tokio::test]
3040 async fn test_all_local_disk() {
3041 let disks = all_local_disk().await;
3042 assert!(disks.is_empty() || !disks.is_empty());
3044 }
3045
3046 #[test]
3047 fn test_should_not_enqueue_transition_for_internal_metadata_bucket() {
3048 let oi = ObjectInfo {
3049 bucket: RUSTFS_META_BUCKET.to_string(),
3050 name: format!("{BUCKET_META_PREFIX}/bucket/.metadata.bin"),
3051 ..Default::default()
3052 };
3053
3054 assert!(!should_enqueue_transition_immediately(&oi));
3055 }
3056}