1#![allow(clippy::map_entry)]
16
17use crate::bucket::lifecycle::bucket_lifecycle_audit::LcEventSrc;
18use crate::bucket::lifecycle::bucket_lifecycle_ops::{
19 enqueue_immediate_expiry, enqueue_transition_immediate, init_background_expiry,
20};
21use crate::bucket::metadata_sys;
22use crate::bucket::utils::check_abort_multipart_args;
23use crate::bucket::utils::check_complete_multipart_args;
24use crate::bucket::utils::check_copy_obj_args;
25use crate::bucket::utils::check_del_obj_args;
26use crate::bucket::utils::check_get_obj_args;
27use crate::bucket::utils::check_list_multipart_args;
28use crate::bucket::utils::check_list_parts_args;
29use crate::bucket::utils::check_new_multipart_args;
30use crate::bucket::utils::check_object_args;
31use crate::bucket::utils::check_put_object_args;
32use crate::bucket::utils::check_put_object_part_args;
33use crate::bucket::utils::{check_valid_bucket_name, check_valid_bucket_name_strict, is_meta_bucketname};
34use crate::cluster::rpc::{RemoteClient, S3PeerSys};
35use crate::config::storageclass;
36use crate::core::pools::{DecommissionCanceler, PoolMeta, PoolMetaWriteState};
37use crate::disk::endpoint::{Endpoint, EndpointType};
38use crate::disk::{DiskAPI, DiskInfo, DiskInfoOptions};
39use crate::error::{Error, Result};
40use crate::error::{
41 StorageError, is_err_bucket_exists, is_err_invalid_upload_id, is_err_object_not_found, is_err_read_quorum,
42 is_err_strict_volume_not_found, is_err_version_not_found, to_object_err,
43};
44use crate::runtime::global::DISK_RESERVE_FRACTION;
45use crate::runtime::instance::InstanceContext;
46use crate::runtime::sources as runtime_sources;
47use crate::services::rebalance::{RebalStatus, RebalanceMeta, is_rebalance_conflicting_with_decommission};
48use crate::storage_api_contracts::{
49 bucket::{BucketInfo, BucketOperations, BucketOptions, DeleteBucketOptions, MakeBucketOptions},
50 list::{StorageListObjectVersionsInfo, StorageListObjectsV2Info, StorageObjectInfoOrErr, StorageWalkOptions},
51 multipart::{CompletePart, ListMultipartsInfo, ListPartsInfo, MultipartInfo, MultipartUploadResult, PartInfo},
52 object::{DeletedObject, ObjectToDelete},
53 range::HTTPRangeSpec,
54};
55use crate::store::init_format::{check_disk_fatal_errs, ec_drives_no_config};
56use crate::{
57 bucket::{lifecycle::bucket_lifecycle_ops::TransitionState, metadata::BucketMetadata},
58 core::sets::Sets,
59 disk::{BUCKET_META_PREFIX, DiskOption, DiskStore, RUSTFS_META_BUCKET},
60 layout::endpoints::EndpointServerPools,
61 object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader, ScannerPublicationCommitScope},
62};
63use futures::future::join_all;
64use http::HeaderMap;
65use lazy_static::lazy_static;
66use rand::RngExt as _;
67use rustfs_config::server_config::Config;
68use rustfs_filemeta::{FileInfo, FileMeta};
69use rustfs_heal_contracts::heal_channel::{HealItemType, HealOpts};
70use rustfs_lock::{LocalClient, LockClient, NamespaceLockWrapper};
71use rustfs_madmin::heal_commands::HealResultItem;
72use rustfs_utils::path::{decode_dir_object, encode_dir_object, path_join_buf};
73use s3s::dto::{BucketVersioningStatus, ObjectLockConfiguration, ObjectLockEnabled, VersioningConfiguration};
74use std::net::SocketAddr;
75use std::process::exit;
76use std::{collections::HashMap, sync::Arc, time::Duration};
77use time::OffsetDateTime;
78use tokio::select;
79use tokio::sync::{Mutex, RwLock};
80use tokio::time::sleep;
81use tokio_util::sync::CancellationToken;
82use tracing::{debug, error, info, instrument, warn};
83use uuid::Uuid;
84
85type ListObjectsV2Info = StorageListObjectsV2Info<ObjectInfo>;
86type ListObjectVersionsInfo = StorageListObjectVersionsInfo<ObjectInfo>;
87type ObjectInfoOrErr = StorageObjectInfoOrErr<ObjectInfo, Error>;
88type WalkOptions = StorageWalkOptions<fn(&FileInfo) -> bool>;
89
90pub const SCANNER_PUBLICATION_LEASE_TTL_MS: u64 = 60_000;
91pub(crate) const BUCKET_DELETE_XLMETA_DIAGNOSTIC_MAX_BYTES: u64 = 1024 * 1024;
92pub(crate) const BUCKET_DELETE_DIAGNOSTIC_MAX_ENTRIES: usize = 4_096;
93pub(crate) const BUCKET_DELETE_DIAGNOSTIC_MAX_ELAPSED: Duration = Duration::from_millis(100);
94
95#[derive(Debug)]
96pub(crate) struct BucketDeleteDiagnosticBudget {
97 deadline: Option<tokio::time::Instant>,
98 max_elapsed: Duration,
99 entries_remaining: usize,
100 #[cfg(test)]
101 first_io_delay: Option<(Duration, Arc<std::sync::atomic::AtomicBool>)>,
102}
103
104impl BucketDeleteDiagnosticBudget {
105 pub(crate) fn new() -> Self {
106 Self::with_limits(BUCKET_DELETE_DIAGNOSTIC_MAX_ENTRIES, BUCKET_DELETE_DIAGNOSTIC_MAX_ELAPSED)
107 }
108
109 fn with_limits(entries: usize, elapsed: Duration) -> Self {
110 Self {
111 deadline: None,
112 max_elapsed: elapsed,
113 entries_remaining: entries,
114 #[cfg(test)]
115 first_io_delay: None,
116 }
117 }
118
119 #[cfg(test)]
120 fn with_first_io_delay(mut self, delay: Duration, started: Arc<std::sync::atomic::AtomicBool>) -> Self {
121 self.first_io_delay = Some((delay, started));
122 self
123 }
124
125 fn deadline(&mut self) -> tokio::time::Instant {
126 let max_elapsed = self.max_elapsed;
127 *self.deadline.get_or_insert_with(|| tokio::time::Instant::now() + max_elapsed)
128 }
129
130 fn claim_entry(&mut self) -> bool {
131 if self.entries_remaining == 0 {
132 return false;
133 }
134 let deadline = self.deadline();
135 if tokio::time::Instant::now() >= deadline {
136 return false;
137 }
138 self.entries_remaining -= 1;
139 true
140 }
141
142 async fn run_io<T, F>(&mut self, future: F) -> std::io::Result<Option<T>>
143 where
144 F: std::future::Future<Output = std::io::Result<T>>,
145 {
146 let deadline = self.deadline();
147 if tokio::time::Instant::now() >= deadline {
148 return Ok(None);
149 }
150 #[cfg(test)]
151 let first_io_delay = self.first_io_delay.take();
152 #[cfg(test)]
153 let timeout_result = tokio::time::timeout_at(deadline, async move {
154 if let Some((delay, started)) = first_io_delay {
155 started.store(true, std::sync::atomic::Ordering::SeqCst);
156 tokio::time::sleep(delay).await;
157 }
158 future.await
159 })
160 .await;
161 #[cfg(not(test))]
162 let timeout_result = tokio::time::timeout_at(deadline, future).await;
163 match timeout_result {
164 Ok(result) => result.map(Some),
165 Err(_) => Ok(None),
166 }
167 }
168}
169
170#[derive(Debug, Default, Clone, PartialEq, Eq)]
171pub(crate) struct BucketMetadataLessResidue {
172 pub(crate) xlmeta_found: bool,
173 pub(crate) xlmeta_blocker: Option<BucketDeleteBlockerKind>,
174 pub(crate) files: usize,
175 pub(crate) uuid_data_dirs: usize,
176 pub(crate) entries_scanned: usize,
177 pub(crate) diagnostic_bytes_read: u64,
178 pub(crate) diagnostic_truncated: bool,
179 pub(crate) sample: Option<String>,
180}
181
182#[derive(Debug, Clone, Copy, PartialEq, Eq)]
183pub(crate) enum BucketDeleteBlockerKind {
184 VisibleVersion,
185 TierFreeVersion,
186 UnknownXlMeta,
187 OrphanDirectory,
188 DiagnosticBudgetExceeded,
189}
190
191impl BucketDeleteBlockerKind {
192 pub(crate) const fn as_str(self) -> &'static str {
193 match self {
194 Self::VisibleVersion => "visible_version",
195 Self::TierFreeVersion => "tier_free_version",
196 Self::UnknownXlMeta => "unknown_xlmeta",
197 Self::OrphanDirectory => "orphan_directory",
198 Self::DiagnosticBudgetExceeded => "diagnostic_budget_exceeded",
199 }
200 }
201
202 pub(crate) const fn is_client_visible(self) -> bool {
215 matches!(self, Self::VisibleVersion | Self::TierFreeVersion)
216 }
217}
218
219impl BucketMetadataLessResidue {
220 pub(crate) fn has_residue_without_xlmeta(&self) -> bool {
221 !self.xlmeta_found && (self.files > 0 || self.diagnostic_truncated)
222 }
223
224 pub(crate) fn describe(&self) -> String {
225 let sample = self.sample.as_deref().unwrap_or("<none>");
226 format!(
227 "metadata-less on-disk residue remains after empty-bucket verification: files={}, uuid_data_dirs={}, entries_scanned={}, diagnostic_bytes_read={}, diagnostic_truncated={}, sample={sample}",
228 self.files, self.uuid_data_dirs, self.entries_scanned, self.diagnostic_bytes_read, self.diagnostic_truncated,
229 )
230 }
231}
232
233pub(crate) async fn has_xlmeta_files(path: &std::path::Path) -> std::io::Result<bool> {
236 use crate::disk::STORAGE_FORMAT_FILE;
237 use tokio::fs;
238
239 let mut stack = vec![path.to_path_buf()];
240
241 while let Some(current_path) = stack.pop() {
242 let mut entries = match fs::read_dir(¤t_path).await {
243 Ok(entries) => entries,
244 Err(err) if err.kind() == std::io::ErrorKind::NotFound => continue,
245 Err(err) => return Err(err),
246 };
247
248 while let Some(entry) = entries.next_entry().await? {
249 let file_name = entry.file_name();
250 let file_name_str = file_name.to_string_lossy();
251
252 if file_name_str == STORAGE_FORMAT_FILE {
254 return Ok(true);
255 }
256
257 if entry.file_type().await?.is_dir() {
259 stack.push(entry.path());
260 }
261 }
262 }
263
264 Ok(false)
265}
266
267#[cfg(test)]
268pub(crate) async fn scan_metadata_less_residue(path: &std::path::Path) -> std::io::Result<BucketMetadataLessResidue> {
269 let mut budget = BucketDeleteDiagnosticBudget::new();
270 scan_metadata_less_residue_with_budget(path, &mut budget).await
271}
272
273async fn scan_metadata_less_residue_with_budget(
274 path: &std::path::Path,
275 budget: &mut BucketDeleteDiagnosticBudget,
276) -> std::io::Result<BucketMetadataLessResidue> {
277 use crate::disk::STORAGE_FORMAT_FILE;
278 use tokio::fs;
279 use tokio::io::AsyncReadExt as _;
280
281 let mut scan = BucketMetadataLessResidue::default();
282 let mut stack = vec![path.to_path_buf()];
283
284 let mark_budget_exhausted = |scan: &mut BucketMetadataLessResidue| {
285 scan.diagnostic_truncated = true;
286 scan.sample.get_or_insert_with(|| "<diagnostic-budget-exceeded>".to_string());
287 };
288
289 while let Some(current_path) = stack.pop() {
290 let mut entries = match budget.run_io(fs::read_dir(¤t_path)).await {
291 Ok(Some(entries)) => entries,
292 Ok(None) => {
293 mark_budget_exhausted(&mut scan);
294 return Ok(scan);
295 }
296 Err(err) if err.kind() == std::io::ErrorKind::NotFound => continue,
297 Err(err) => return Err(err),
298 };
299
300 loop {
301 let entry = match budget.run_io(entries.next_entry()).await? {
302 Some(Some(entry)) => entry,
303 Some(None) => break,
304 None => {
305 mark_budget_exhausted(&mut scan);
306 return Ok(scan);
307 }
308 };
309 if !budget.claim_entry() {
310 mark_budget_exhausted(&mut scan);
311 return Ok(scan);
312 }
313 scan.entries_scanned = scan.entries_scanned.saturating_add(1);
314 let Some(file_type) = budget.run_io(entry.file_type()).await? else {
315 mark_budget_exhausted(&mut scan);
316 return Ok(scan);
317 };
318 let file_name = entry.file_name();
319 let file_name_str = file_name.to_string_lossy();
320
321 if file_name_str == STORAGE_FORMAT_FILE {
322 scan.xlmeta_found = true;
323 if scan.xlmeta_blocker.is_none() {
324 let entry_path = entry.path();
325 let Some(metadata) = budget.run_io(fs::metadata(&entry_path)).await? else {
326 mark_budget_exhausted(&mut scan);
327 scan.xlmeta_blocker = Some(BucketDeleteBlockerKind::DiagnosticBudgetExceeded);
328 return Ok(scan);
329 };
330 scan.xlmeta_blocker = Some(if metadata.len() > BUCKET_DELETE_XLMETA_DIAGNOSTIC_MAX_BYTES {
331 BucketDeleteBlockerKind::UnknownXlMeta
332 } else {
333 match budget.run_io(fs::File::open(&entry_path)).await {
334 Ok(Some(file)) => {
335 let mut data = Vec::new();
336 let read = budget
337 .run_io(file.take(BUCKET_DELETE_XLMETA_DIAGNOSTIC_MAX_BYTES).read_to_end(&mut data))
338 .await;
339 scan.diagnostic_bytes_read = data.len() as u64;
340 match read {
341 Ok(Some(_)) => match FileMeta::load(&data) {
342 Ok(meta)
343 if !meta.versions.is_empty()
344 && meta.versions.iter().all(|version| version.header.free_version()) =>
345 {
346 BucketDeleteBlockerKind::TierFreeVersion
347 }
348 Ok(meta) if !meta.versions.is_empty() => BucketDeleteBlockerKind::VisibleVersion,
349 Ok(_) | Err(_) => BucketDeleteBlockerKind::UnknownXlMeta,
350 },
351 Ok(None) => {
352 mark_budget_exhausted(&mut scan);
353 BucketDeleteBlockerKind::DiagnosticBudgetExceeded
354 }
355 Err(_) => BucketDeleteBlockerKind::UnknownXlMeta,
356 }
357 }
358 Ok(None) => {
359 mark_budget_exhausted(&mut scan);
360 BucketDeleteBlockerKind::DiagnosticBudgetExceeded
361 }
362 Err(_) => BucketDeleteBlockerKind::UnknownXlMeta,
363 }
364 });
365 let sample = entry_path
366 .strip_prefix(path)
367 .unwrap_or(entry_path.as_path())
368 .to_string_lossy()
369 .replace(std::path::MAIN_SEPARATOR, "/");
370 scan.sample = Some(sample);
371 }
372 return Ok(scan);
373 }
374
375 if file_type.is_dir() {
376 if Uuid::parse_str(&file_name_str).is_ok() {
377 scan.uuid_data_dirs = scan.uuid_data_dirs.saturating_add(1);
378 }
379 stack.push(entry.path());
380 } else {
381 scan.files = scan.files.saturating_add(1);
382 if scan.sample.is_none() {
383 let entry_path = entry.path();
384 let sample = entry_path
385 .strip_prefix(path)
386 .unwrap_or(entry_path.as_path())
387 .to_string_lossy()
388 .replace(std::path::MAIN_SEPARATOR, "/");
389 scan.sample = Some(sample);
390 }
391 }
392 }
393 }
394
395 Ok(scan)
396}
397
398async fn enqueue_transition_after_write(
399 store: &ECStore,
400 result: Result<ObjectInfo>,
401 src: LcEventSrc,
402 opts: &ObjectOptions,
403) -> Result<ObjectInfo> {
404 match result {
405 Ok(oi) => {
406 if !oi.multipart_completion_replayed && should_enqueue_transition_immediately(&oi) {
410 enqueue_transition_immediate(&oi, src.clone()).await;
411 if let Ok(api) = metadata_sys::object_store_in(&store.ctx).await {
412 enqueue_immediate_expiry(api, &oi, src, opts).await;
413 }
414 }
415 Ok(oi)
416 }
417 Err(err) => Err(err),
418 }
419}
420
421fn should_enqueue_transition_immediately(oi: &ObjectInfo) -> bool {
422 !is_meta_bucketname(&oi.bucket)
423}
424
425const MAX_UPLOADS_LIST: usize = 10000;
426
427mod bucket;
428mod bucket_fence;
429pub(crate) use bucket::await_bucket_namespace_operation;
430pub use bucket_fence::BucketIncarnationFenceGuard;
431mod heal;
432pub use heal::{HealObjectAbsenceProof, HealObjectStorageResult};
433pub(crate) use heal::{bucket_heal_scope, bucket_heal_scope_for_object};
434mod heal_walk;
435pub use heal_walk::HealWalkVersion;
436mod init;
437pub(crate) mod init_format;
438pub(crate) mod list_objects;
439mod multipart;
440mod object;
441#[cfg(feature = "test-util")]
442pub use object::DeleteAfterObjectLockSnapshotBarrier;
443pub(crate) use object::{
444 DecommissionFixedReadAnchor, ObjectLockDiagGuard, RemoteTuplePublicationCommitGuard, RemoteTuplePublicationFence,
445 SourceCleanupMutationFence, tiered_data_movement_source_matches,
446};
447pub use object::{
448 PrepareSelectObjectSnapshotError, PreparedGetObjectReader, SelectObjectSnapshot, SelectObjectSnapshotReadError,
449 SnapshotConsistencyError,
450};
451mod peer;
452mod rebalance;
453pub(crate) mod utils;
454
455use peer::init_local_peer;
456pub use peer::{
457 BootstrapLocalTarget, all_local_disk, all_local_disk_path, find_local_disk_by_ref, get_disk_infos, init_local_disks,
458 init_local_disks_with_instance_ctx, init_lock_clients, prewarm_local_disk_id_map,
459 prewarm_local_disk_id_map_with_instance_ctx,
460};
461
462pub struct ECStore {
463 pub id: Uuid,
464 pub disk_map: HashMap<usize, Vec<Option<DiskStore>>>,
466 pub pools: Vec<Arc<Sets>>,
467 pub peer_sys: S3PeerSys,
468 pub pool_meta: RwLock<PoolMeta>,
470 pub rebalance_meta: RwLock<Option<RebalanceMeta>>,
471 pub decommission_cancelers: RwLock<Vec<Option<DecommissionCanceler>>>,
472 pub(crate) start_gate: Mutex<()>,
478 pub(crate) pool_meta_save_gate: Mutex<PoolMetaWriteState>,
484 pub(crate) ctx: Arc<InstanceContext>,
492 pub(crate) bucket_fence_registry: Arc<bucket_fence::BucketFenceRegistry>,
495}
496
497const METRIC_SCANNER_DATA_MOVEMENT_PAUSED: &str = "rustfs_scanner_data_movement_paused";
498const METRIC_SCANNER_DATA_MOVEMENT_PAUSE_DURATION_SECONDS: &str = "rustfs_scanner_data_movement_pause_duration_seconds";
499const METRIC_SCANNER_DATA_MOVEMENT_BACKLOG_WORK_ITEMS: &str = "rustfs_scanner_data_movement_backlog_work_items";
500const SCANNER_DATA_MOVEMENT_PAUSE_POLICY: &str = "global_pause";
501
502#[derive(Clone, Copy, Debug, PartialEq, Eq, serde::Serialize)]
503#[serde(rename_all = "snake_case")]
504pub enum ScannerDataMovementPauseReason {
505 OperationEpochExhausted,
506 MovementGenerationExhausted,
507 DecommissionActive,
508 DecommissionFailed,
509 DecommissionCanceled,
510 RebalanceActive,
511}
512
513#[derive(Clone, Debug, PartialEq, Eq, serde::Serialize)]
514pub struct ScannerDataMovementPauseStatus {
515 pub paused: bool,
516 pub policy: &'static str,
517 pub reasons: Vec<ScannerDataMovementPauseReason>,
518 pub started_at_unix_secs: u64,
519 pub duration_seconds: u64,
520 pub operation_epoch: u64,
521 pub movement_generation: u64,
522 pub movement_backlog_work_items: u64,
523 pub movement_backlog_estimated: bool,
524}
525
526impl Default for ScannerDataMovementPauseStatus {
527 fn default() -> Self {
528 Self {
529 paused: false,
530 policy: SCANNER_DATA_MOVEMENT_PAUSE_POLICY,
531 reasons: Vec::new(),
532 started_at_unix_secs: 0,
533 duration_seconds: 0,
534 operation_epoch: 0,
535 movement_generation: 0,
536 movement_backlog_work_items: 0,
537 movement_backlog_estimated: false,
538 }
539 }
540}
541
542#[derive(Clone, Debug, PartialEq, Eq, serde::Serialize)]
543pub struct PoolMetaWriteGateStatus {
544 pub writes_ready: bool,
545 pub check_timed_out: bool,
546 pub write_blocked: bool,
547 pub transaction_aborted: bool,
548 pub pool_meta_absent: bool,
549 pub identity_initialized: Option<bool>,
550 pub identity_needs_repair: bool,
551 pub cluster_epoch: Option<u64>,
552 pub reason: Option<&'static str>,
553 pub phase: Option<&'static str>,
554 pub since_unix_secs: Option<i64>,
555}
556
557impl Default for PoolMetaWriteGateStatus {
558 fn default() -> Self {
559 Self {
560 writes_ready: true,
561 check_timed_out: false,
562 write_blocked: false,
563 transaction_aborted: false,
564 pool_meta_absent: false,
565 identity_initialized: None,
566 identity_needs_repair: false,
567 cluster_epoch: None,
568 reason: None,
569 phase: None,
570 since_unix_secs: None,
571 }
572 }
573}
574
575fn offset_unix_seconds(value: OffsetDateTime) -> u64 {
576 u64::try_from(value.unix_timestamp()).unwrap_or(0)
577}
578
579fn earliest_timestamp(current: Option<OffsetDateTime>, candidate: Option<OffsetDateTime>) -> Option<OffsetDateTime> {
580 match (current, candidate) {
581 (Some(current), Some(candidate)) => Some(current.min(candidate)),
582 (Some(current), None) => Some(current),
583 (None, candidate) => candidate,
584 }
585}
586
587fn usize_to_u64(value: usize) -> u64 {
588 u64::try_from(value).unwrap_or(u64::MAX)
589}
590
591fn metric_u64(value: u64) -> f64 {
592 f64::from(u32::try_from(value).unwrap_or(u32::MAX))
593}
594
595pub(crate) fn scanner_data_movement_timestamp_generation(value: OffsetDateTime) -> u64 {
596 let timestamp = value.unix_timestamp_nanos();
597 if timestamp <= 0 {
598 0
599 } else {
600 u64::try_from(timestamp).unwrap_or(u64::MAX)
601 }
602}
603
604fn valid_scanner_data_movement_timestamp_generation(value: OffsetDateTime) -> Option<u64> {
605 let generation = scanner_data_movement_timestamp_generation(value);
606 (generation != 0 && generation != u64::MAX).then_some(generation)
607}
608
609fn durable_scanner_data_movement_generation(pool_meta: &PoolMeta, rebalance_meta: Option<&RebalanceMeta>) -> u64 {
610 let mut generation = 0;
611 for pool in pool_meta.pools.iter().filter(|pool| pool.decommission.is_some()) {
612 let Some(pool_generation) = valid_scanner_data_movement_timestamp_generation(pool.last_update) else {
613 return u64::MAX;
614 };
615 generation = generation.max(pool_generation);
616 }
617
618 for movement_timestamp in rebalance_meta.into_iter().flat_map(|meta| {
619 meta.stopped_at.into_iter().chain(
620 meta.pool_stats
621 .iter()
622 .flat_map(|pool| [pool.info.start_time, pool.info.end_time])
623 .flatten(),
624 )
625 }) {
626 let Some(rebalance_generation) = valid_scanner_data_movement_timestamp_generation(movement_timestamp) else {
627 return u64::MAX;
628 };
629 generation = generation.max(rebalance_generation);
630 }
631
632 if generation == 0
633 && rebalance_meta.is_some_and(|meta| !meta.id.is_empty() || !meta.pool_stats.is_empty() || meta.stopped_at.is_some())
634 {
635 u64::MAX
636 } else {
637 generation
638 }
639}
640
641#[derive(Clone, Copy)]
642struct ScannerDataMovementSequenceState {
643 operation_epoch: u64,
644 operation_epoch_exhausted: bool,
645 movement_generation: u64,
646 movement_generation_exhausted: bool,
647}
648
649fn resolve_scanner_data_movement_pause_status(
650 pool_meta: &PoolMeta,
651 rebalance_meta: Option<&RebalanceMeta>,
652 decommission_worker_active: bool,
653 sequence: ScannerDataMovementSequenceState,
654 now: OffsetDateTime,
655) -> ScannerDataMovementPauseStatus {
656 let mut decommission_active = decommission_worker_active;
657 let mut decommission_failed = false;
658 let mut decommission_canceled = false;
659 let mut rebalance_active = false;
660 let mut started_at = None;
661 let mut movement_backlog_work_items = 0_u64;
662
663 for pool in &pool_meta.pools {
664 let Some(info) = pool.decommission.as_ref() else {
665 continue;
666 };
667 let active = info.has_decommission_state() && !info.complete && !info.failed && !info.canceled;
668 let failed = !info.queued && info.failed;
669 let canceled = !info.queued && info.canceled;
670 if !(active || failed || canceled) {
671 continue;
672 }
673
674 decommission_active |= active;
675 decommission_failed |= failed;
676 decommission_canceled |= canceled;
677 started_at = earliest_timestamp(started_at, info.start_time.or(Some(pool.last_update)));
678 let queued = usize_to_u64(info.queued_buckets.len());
679 let current_bucket = if info.bucket.is_empty() { 0 } else { 1 };
680 movement_backlog_work_items = movement_backlog_work_items.saturating_add(queued.max(current_bucket));
681 }
682
683 if let Some(rebalance_meta) = rebalance_meta {
684 for pool in &rebalance_meta.pool_stats {
685 let active = (pool.participating && pool.info.status == RebalStatus::Started) || pool.info.stopping;
686 if !active {
687 continue;
688 }
689 rebalance_active = true;
690 started_at = earliest_timestamp(started_at, pool.info.start_time);
691 movement_backlog_work_items = movement_backlog_work_items.saturating_add(usize_to_u64(pool.buckets.len()));
692 }
693 }
694
695 let mut reasons = Vec::with_capacity(6);
696 if sequence.operation_epoch_exhausted {
697 reasons.push(ScannerDataMovementPauseReason::OperationEpochExhausted);
698 }
699 if sequence.movement_generation_exhausted {
700 reasons.push(ScannerDataMovementPauseReason::MovementGenerationExhausted);
701 }
702 if decommission_active {
703 reasons.push(ScannerDataMovementPauseReason::DecommissionActive);
704 }
705 if decommission_failed {
706 reasons.push(ScannerDataMovementPauseReason::DecommissionFailed);
707 }
708 if decommission_canceled {
709 reasons.push(ScannerDataMovementPauseReason::DecommissionCanceled);
710 }
711 if rebalance_active {
712 reasons.push(ScannerDataMovementPauseReason::RebalanceActive);
713 }
714 let started_at_unix_secs = started_at.map(offset_unix_seconds).unwrap_or(0);
715 let duration_seconds = started_at
716 .and_then(|started_at| u64::try_from((now - started_at).whole_seconds()).ok())
717 .unwrap_or(0);
718 let paused = !reasons.is_empty();
719
720 ScannerDataMovementPauseStatus {
721 paused,
722 policy: SCANNER_DATA_MOVEMENT_PAUSE_POLICY,
723 reasons,
724 started_at_unix_secs,
725 duration_seconds,
726 operation_epoch: sequence.operation_epoch,
727 movement_generation: sequence.movement_generation,
728 movement_backlog_work_items,
729 movement_backlog_estimated: paused,
730 }
731}
732
733fn record_scanner_data_movement_pause_status(status: &ScannerDataMovementPauseStatus) {
734 metrics::gauge!(METRIC_SCANNER_DATA_MOVEMENT_PAUSED).set(if status.paused { 1.0 } else { 0.0 });
735 metrics::gauge!(METRIC_SCANNER_DATA_MOVEMENT_PAUSE_DURATION_SECONDS).set(metric_u64(status.duration_seconds));
736 metrics::gauge!(METRIC_SCANNER_DATA_MOVEMENT_BACKLOG_WORK_ITEMS).set(metric_u64(status.movement_backlog_work_items));
737}
738
739impl std::fmt::Debug for ECStore {
740 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
741 let disk_slot_count: usize = self.disk_map.values().map(Vec::len).sum();
742
743 f.debug_struct("ECStore")
744 .field("id", &self.id)
745 .field("disk_map_pool_count", &self.disk_map.len())
746 .field("disk_slot_count", &disk_slot_count)
747 .field("pool_count", &self.pools.len())
748 .finish_non_exhaustive()
749 }
750}
751
752impl ECStore {
756 pub fn all_set_disks(&self) -> Vec<Arc<crate::set_disk::SetDisks>> {
763 self.pools.iter().flat_map(|pool| pool.disk_set.iter().cloned()).collect()
764 }
765
766 pub async fn scanner_pause_backlog_writable_set_disks(&self) -> Vec<Arc<crate::set_disk::SetDisks>> {
773 let pool_meta = self.pool_meta.read().await;
774 self.pools
775 .iter()
776 .enumerate()
777 .filter(|(pool_index, _)| pool_meta.scanner_pause_backlog_pool_writable(*pool_index))
778 .flat_map(|(_, pool)| pool.disk_set.iter().cloned())
779 .collect()
780 }
781
782 pub fn get_server_config(&self) -> Option<Config> {
784 runtime_sources::server_config()
785 }
786
787 pub fn set_server_config(&self, cfg: Config) {
789 runtime_sources::set_server_config(cfg);
790 }
791
792 pub fn get_storage_class(&self) -> Option<crate::config::storageclass::Config> {
794 runtime_sources::storage_class_config()
795 }
796
797 pub fn set_storage_class(&self, cfg: crate::config::storageclass::Config) {
799 runtime_sources::set_storage_class_config(cfg);
800 }
801}
802
803impl ECStore {
807 pub fn notification_system(&self) -> Option<std::sync::Arc<crate::services::notification_sys::NotificationSys>> {
809 runtime_sources::notification_sys()
810 }
811
812 pub fn bucket_metadata_sys(&self) -> Option<Arc<tokio::sync::RwLock<crate::bucket::metadata_sys::BucketMetadataSys>>> {
814 runtime_sources::bucket_metadata_sys()
815 }
816
817 pub fn endpoints(&self) -> EndpointServerPools {
819 runtime_sources::endpoint_pools().unwrap_or_else(|| Vec::new().into())
820 }
821
822 pub fn instance_endpoints(&self) -> Option<EndpointServerPools> {
824 self.ctx.endpoints()
825 }
826
827 pub fn region(&self) -> Option<s3s::region::Region> {
829 runtime_sources::region()
830 }
831
832 pub fn tier_config_mgr(&self) -> Arc<tokio::sync::RwLock<crate::services::tier::tier::TierConfigMgr>> {
834 self.ctx.tier_config_mgr()
835 }
836
837 pub fn server_config(&self) -> Option<Config> {
839 runtime_sources::server_config()
840 }
841
842 pub fn storage_class(&self) -> Option<crate::config::storageclass::Config> {
844 runtime_sources::storage_class_config()
845 }
846}
847
848impl ECStore {
852 pub fn port(&self) -> u16 {
854 runtime_sources::rustfs_port()
855 }
856
857 pub async fn host(&self) -> String {
859 runtime_sources::rustfs_host().await
860 }
861
862 pub async fn addr(&self) -> String {
864 runtime_sources::rustfs_addr().await
865 }
866}
867
868impl ECStore {
875 pub async fn setup_is_erasure(&self) -> bool {
877 self.ctx.is_erasure().await
878 }
879
880 pub async fn setup_is_dist_erasure(&self) -> bool {
882 self.ctx.is_dist_erasure().await
883 }
884
885 pub async fn setup_is_erasure_sd(&self) -> bool {
887 self.ctx.is_erasure_sd().await
888 }
889
890 pub fn scanner_namespace_mutation_generation(&self) -> u64 {
891 list_objects::scanner_namespace_mutation_generation().saturating_add(self.ctx.namespace_commit_generation())
892 }
893
894 pub async fn scanner_data_movement_active(&self) -> bool {
895 let (decommission, rebalance) = tokio::join!(self.is_decommission_running(), self.is_rebalance_started());
896 decommission || rebalance
897 }
898
899 pub async fn scanner_data_movement_activity(&self) -> (bool, bool, u64) {
905 let operation_gate = self.ctx.data_movement_operation_gate();
906 let _operation_guard = operation_gate.read_owned().await;
907 let (active, blocked) = self.scanner_data_movement_snapshot_locked().await;
908 let blocked =
909 blocked || self.ctx.data_movement_operation_epoch_exhausted() || self.ctx.data_movement_generation_exhausted();
910 self.ctx.set_scanner_publication_state(blocked);
911 (active, blocked, self.ctx.data_movement_generation())
912 }
913
914 pub fn scanner_data_movement_generation(&self) -> u64 {
915 self.ctx.data_movement_generation()
916 }
917
918 pub fn scanner_data_movement_generation_exhausted(&self) -> bool {
919 self.ctx.data_movement_generation_exhausted()
920 }
921
922 pub fn scanner_data_movement_changed(&self) -> std::sync::Arc<tokio::sync::Notify> {
923 self.ctx.data_movement_generation_notify()
924 }
925
926 pub async fn scanner_data_usage_publication_blocked(&self) -> bool {
932 let operation_gate = self.ctx.data_movement_operation_gate();
933 let _operation_guard = operation_gate.read_owned().await;
934 self.scanner_data_usage_publication_snapshot_blocked().await || self.ctx.namespace_commits_pending()
935 }
936
937 pub async fn scanner_data_movement_pause_status(&self) -> ScannerDataMovementPauseStatus {
938 let operation_gate = self.ctx.data_movement_operation_gate();
939 let _operation_guard = operation_gate.read_owned().await;
940 self.scanner_data_movement_pause_snapshot().await
941 }
942
943 async fn scanner_data_usage_publication_snapshot_blocked(&self) -> bool {
944 self.scanner_data_movement_pause_snapshot().await.paused
945 }
946
947 async fn scanner_data_movement_snapshot_locked(&self) -> (bool, bool) {
948 let decommission_cancelers = self.decommission_cancelers.read().await;
949 let decommission_active = decommission_cancelers
950 .iter()
951 .any(|canceler| canceler.as_ref().is_some_and(DecommissionCanceler::is_active));
952 let pool_meta = self.pool_meta.read().await;
953 let decommission_active = decommission_active
954 || pool_meta.pools.iter().any(|pool| {
955 pool.decommission
956 .as_ref()
957 .is_some_and(|info| info.has_decommission_state() && !info.complete && !info.failed && !info.canceled)
958 });
959 let decommission_terminal = pool_meta.pools.iter().any(|pool| {
960 pool.decommission
961 .as_ref()
962 .is_some_and(|info| !info.queued && (info.failed || info.canceled))
963 });
964 let rebalance_meta = self.rebalance_meta.read().await;
965 let rebalance_active = rebalance_meta
966 .as_ref()
967 .is_some_and(is_rebalance_conflicting_with_decommission);
968 self.ctx
969 .observe_durable_data_movement_generation(durable_scanner_data_movement_generation(
970 &pool_meta,
971 rebalance_meta.as_ref(),
972 ));
973
974 let blocked = decommission_active || decommission_terminal || rebalance_active;
975 (decommission_active || rebalance_active, blocked)
976 }
977
978 async fn scanner_data_movement_pause_snapshot(&self) -> ScannerDataMovementPauseStatus {
979 let decommission_active = {
980 let decommission_cancelers = self.decommission_cancelers.read().await;
981 decommission_cancelers
982 .iter()
983 .any(|canceler| canceler.as_ref().is_some_and(DecommissionCanceler::is_active))
984 };
985 let pool_meta = self.pool_meta.read().await.clone();
986 let rebalance_meta = self.rebalance_meta.read().await.clone();
987 self.ctx
988 .observe_durable_data_movement_generation(durable_scanner_data_movement_generation(
989 &pool_meta,
990 rebalance_meta.as_ref(),
991 ));
992 let status = resolve_scanner_data_movement_pause_status(
993 &pool_meta,
994 rebalance_meta.as_ref(),
995 decommission_active,
996 ScannerDataMovementSequenceState {
997 operation_epoch: self.ctx.data_movement_operation_epoch(),
998 operation_epoch_exhausted: self.ctx.data_movement_operation_epoch_exhausted(),
999 movement_generation: self.ctx.data_movement_generation(),
1000 movement_generation_exhausted: self.ctx.data_movement_generation_exhausted(),
1001 },
1002 OffsetDateTime::now_utc(),
1003 );
1004 self.ctx.set_scanner_publication_state(status.paused);
1005 record_scanner_data_movement_pause_status(&status);
1006 status
1007 }
1008
1009 #[cfg(test)]
1010 pub(crate) async fn scanner_data_movement_pause_snapshot_for_test(&self) -> ScannerDataMovementPauseStatus {
1011 self.scanner_data_movement_pause_snapshot().await
1012 }
1013
1014 pub async fn scanner_data_usage_publication_read_guard(&self) -> (tokio::sync::OwnedRwLockReadGuard<()>, u64) {
1019 let operation_gate = self.ctx.data_movement_operation_gate();
1020 let operation_guard = operation_gate.read_owned().await;
1021 let epoch = self.ctx.data_movement_operation_epoch();
1022 (operation_guard, epoch)
1023 }
1024
1025 pub async fn scanner_data_usage_publication_admission_guard(&self) -> Option<(tokio::sync::OwnedRwLockReadGuard<()>, u64)> {
1030 let operation_gate = self.ctx.data_movement_operation_gate();
1031 let operation_guard = operation_gate.read_owned().await;
1032 if self.ctx.data_movement_operation_epoch_exhausted() || self.ctx.data_movement_generation_exhausted() {
1033 return None;
1034 }
1035 if self.scanner_data_usage_publication_snapshot_blocked().await {
1036 return None;
1037 }
1038
1039 Some((operation_guard, self.ctx.data_movement_operation_epoch()))
1040 }
1041
1042 pub async fn scanner_data_usage_publication_commit_scope(
1047 &self,
1048 expected_movement_epoch: u64,
1049 safe_deadline: tokio::time::Instant,
1050 remote_lease_tokens: Vec<Uuid>,
1051 ) -> Option<ScannerPublicationCommitScope> {
1052 let (movement_permit, epoch) = self.scanner_data_usage_publication_admission_guard().await?;
1053 if epoch != expected_movement_epoch {
1054 return None;
1055 }
1056 Some(ScannerPublicationCommitScope::new_storage_owned(
1057 epoch,
1058 safe_deadline,
1059 remote_lease_tokens,
1060 movement_permit,
1061 ))
1062 }
1063
1064 pub async fn scanner_data_usage_publication_commit_scope_with_release_flag(
1065 &self,
1066 expected_movement_epoch: u64,
1067 safe_deadline: tokio::time::Instant,
1068 remote_lease_tokens: Vec<Uuid>,
1069 lease_release_safe: Arc<std::sync::atomic::AtomicBool>,
1070 ) -> Option<ScannerPublicationCommitScope> {
1071 let (movement_permit, epoch) = self.scanner_data_usage_publication_admission_guard().await?;
1072 if epoch != expected_movement_epoch {
1073 return None;
1074 }
1075 Some(ScannerPublicationCommitScope::new_storage_owned_with_release_flag(
1076 epoch,
1077 safe_deadline,
1078 remote_lease_tokens,
1079 movement_permit,
1080 lease_release_safe,
1081 ))
1082 }
1083
1084 pub(crate) async fn scanner_data_usage_publication_epoch(&self) -> Option<u64> {
1088 let (operation_guard, epoch) = self.scanner_data_usage_publication_admission_guard().await?;
1089 drop(operation_guard);
1090 Some(epoch)
1091 }
1092
1093 pub async fn acquire_scanner_publication_lease(
1098 &self,
1099 expected_generation: u64,
1100 ttl: std::time::Duration,
1101 ) -> Result<(Uuid, u64)> {
1102 if ttl != crate::runtime::instance::SCANNER_PUBLICATION_LEASE_TTL {
1103 return Err(Error::other("scanner publication lease TTL is not supported"));
1104 }
1105
1106 let namespace_generation = self.scanner_namespace_mutation_generation();
1109 let operation_gate = self.ctx.data_movement_operation_gate();
1110 let operation_guard = operation_gate.read_owned().await;
1111 if self.ctx.data_movement_generation_exhausted()
1112 || self.ctx.data_movement_operation_epoch_exhausted()
1113 || self.ctx.data_movement_generation() != expected_generation
1114 || namespace_generation == u64::MAX
1115 {
1116 return Err(Error::other("scanner publication lease generation is stale"));
1117 }
1118 if self.scanner_data_movement_snapshot_locked().await.1 || self.ctx.namespace_commits_pending() {
1119 return Err(Error::other("scanner publication lease is blocked by data movement"));
1120 }
1121
1122 if self.scanner_namespace_mutation_generation() != namespace_generation {
1123 return Err(Error::other("scanner publication lease generation is stale"));
1124 }
1125
1126 let token = Uuid::new_v4();
1127 let expires_at = tokio::time::Instant::now() + ttl;
1128 if !self
1129 .ctx
1130 .install_scanner_publication_lease(token, expires_at, expected_generation, namespace_generation, operation_guard)
1131 .await
1132 {
1133 return Err(Error::other("scanner publication lease capacity is exhausted"));
1134 }
1135
1136 let context = Arc::clone(&self.ctx);
1137 tokio::spawn(async move {
1138 tokio::time::sleep_until(expires_at).await;
1139 context.expire_scanner_publication_lease(token, expires_at).await;
1140 });
1141 Ok((token, expected_generation))
1142 }
1143
1144 pub async fn release_scanner_publication_lease(&self, token: Uuid) -> bool {
1145 self.ctx.remove_scanner_publication_lease(token).await
1146 }
1147
1148 pub async fn validate_scanner_publication_lease(&self, token: Uuid, expected_generation: u64) -> Result<()> {
1154 let _operation_guard = self.acquire_scanner_publication_lease_guard(token).await?;
1155 if self.ctx.data_movement_generation_exhausted()
1156 || self.ctx.data_movement_operation_epoch_exhausted()
1157 || self.ctx.data_movement_generation() != expected_generation
1158 {
1159 return Err(Error::other("scanner publication lease generation is stale"));
1160 }
1161 if self.scanner_data_movement_snapshot_locked().await.1 || self.ctx.namespace_commits_pending() {
1162 return Err(Error::other("scanner publication lease is blocked by data movement"));
1163 }
1164 let Some((lease_generation, lease_namespace_generation)) = self.ctx.scanner_publication_lease_generations(token).await
1165 else {
1166 return Err(Error::other("scanner publication lease is unknown or expired"));
1167 };
1168 let namespace_generation = self.scanner_namespace_mutation_generation();
1169 if lease_generation != self.ctx.data_movement_generation()
1170 || lease_namespace_generation != namespace_generation
1171 || namespace_generation == u64::MAX
1172 {
1173 return Err(Error::other("scanner publication lease generation is stale"));
1174 }
1175 Ok(())
1176 }
1177
1178 pub async fn acquire_scanner_publication_lease_guard(&self, token: Uuid) -> Result<tokio::sync::OwnedRwLockReadGuard<()>> {
1184 let operation_gate = self.ctx.data_movement_operation_gate();
1185 let operation_guard = operation_gate.read_owned().await;
1186 if self.ctx.data_movement_generation_exhausted() || self.ctx.data_movement_operation_epoch_exhausted() {
1187 return Err(Error::other("scanner publication lease generation is exhausted"));
1188 }
1189 if self.scanner_data_movement_snapshot_locked().await.1 || self.ctx.namespace_commits_pending() {
1190 return Err(Error::other("scanner publication lease is blocked by data movement"));
1191 }
1192 let Some((lease_generation, lease_namespace_generation)) = self.ctx.scanner_publication_lease_generations(token).await
1193 else {
1194 return Err(Error::other("scanner publication lease is unknown or expired"));
1195 };
1196 let namespace_generation = self.scanner_namespace_mutation_generation();
1197 if lease_generation != self.ctx.data_movement_generation()
1198 || lease_namespace_generation != namespace_generation
1199 || namespace_generation == u64::MAX
1200 {
1201 return Err(Error::other("scanner publication lease generation is stale"));
1202 }
1203 Ok(operation_guard)
1204 }
1205}
1206
1207#[async_trait::async_trait]
1252impl crate::storage_api_contracts::object::ObjectIO for ECStore {
1253 type Error = Error;
1254 type RangeSpec = HTTPRangeSpec;
1255 type HeaderMap = HeaderMap;
1256 type ObjectOptions = ObjectOptions;
1257 type ObjectInfo = ObjectInfo;
1258 type GetObjectReader = GetObjectReader;
1259 type PutObjectReader = PutObjReader;
1260
1261 #[instrument(level = "debug", skip(self, h))]
1262 async fn get_object_reader(
1263 &self,
1264 bucket: &str,
1265 object: &str,
1266 range: Option<HTTPRangeSpec>,
1267 h: HeaderMap,
1268 opts: &ObjectOptions,
1269 ) -> Result<GetObjectReader> {
1270 self.handle_get_object_reader(bucket, object, range, h, opts).await
1271 }
1272 #[instrument(level = "debug", skip(self, data))]
1273 async fn put_object(&self, bucket: &str, object: &str, data: &mut PutObjReader, opts: &ObjectOptions) -> Result<ObjectInfo> {
1274 self.put_object_with_old_current_size(bucket, object, data, opts)
1275 .await
1276 .map(|(object_info, _)| object_info)
1277 }
1278}
1279
1280impl ECStore {
1281 #[instrument(level = "debug", skip(self, data))]
1286 pub async fn put_object_with_old_current_size(
1287 &self,
1288 bucket: &str,
1289 object: &str,
1290 data: &mut PutObjReader,
1291 opts: &ObjectOptions,
1292 ) -> Result<(ObjectInfo, Option<crate::disk::OldCurrentSize>)> {
1293 let result = match self.handle_put_object(bucket, object, data, opts).await {
1294 Ok((object_info, old_current_size)) => {
1295 enqueue_transition_after_write(self, Ok(object_info), LcEventSrc::S3PutObject, opts)
1296 .await
1297 .map(|object_info| (object_info, old_current_size))
1298 }
1299 Err(err) => Err(err),
1300 };
1301 if result.is_ok() {
1302 list_objects::observe_list_objects_mutation(self, bucket).await;
1303 }
1304 result
1305 }
1306}
1307
1308lazy_static! {
1309 static ref ENABLED_OBJECT_LOCK_CONFIG: ObjectLockConfiguration = ObjectLockConfiguration {
1310 object_lock_enabled: Some(ObjectLockEnabled::from_static(ObjectLockEnabled::ENABLED)),
1311 ..Default::default()
1312 };
1313 static ref ENABLED_VERSIONING_CONFIG: VersioningConfiguration = VersioningConfiguration {
1314 status: Some(BucketVersioningStatus::from_static(BucketVersioningStatus::ENABLED)),
1315 ..Default::default()
1316 };
1317}
1318
1319#[async_trait::async_trait]
1320impl BucketOperations for ECStore {
1321 type Error = Error;
1322
1323 #[instrument(skip(self))]
1324 async fn make_bucket(&self, bucket: &str, opts: &MakeBucketOptions) -> Result<()> {
1325 Box::pin(self.handle_make_bucket(bucket, opts)).await
1326 }
1327
1328 #[instrument(skip(self))]
1329 async fn get_bucket_info(&self, bucket: &str, opts: &BucketOptions) -> Result<BucketInfo> {
1330 self.handle_get_bucket_info(bucket, opts).await
1331 }
1332 #[instrument(skip(self))]
1333 async fn list_bucket(&self, opts: &BucketOptions) -> Result<Vec<BucketInfo>> {
1334 self.handle_list_bucket(opts).await
1335 }
1336 #[instrument(skip(self))]
1337 async fn delete_bucket(&self, bucket: &str, opts: &DeleteBucketOptions) -> Result<()> {
1338 Box::pin(self.handle_delete_bucket(bucket, opts)).await
1339 }
1340}
1341
1342#[async_trait::async_trait]
1343impl crate::storage_api_contracts::object::ObjectOperations for ECStore {
1344 type Error = Error;
1345 type ObjectInfo = ObjectInfo;
1346 type ObjectOptions = ObjectOptions;
1347 type FileInfo = FileInfo;
1348 type ObjectToDelete = ObjectToDelete;
1349 type DeletedObject = DeletedObject;
1350
1351 async fn get_object_info(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<ObjectInfo> {
1352 self.handle_get_object_info(bucket, object, opts).await
1353 }
1354
1355 async fn verify_object_integrity(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<()> {
1356 self.handle_verify_object_integrity(bucket, object, opts).await
1357 }
1358
1359 #[instrument(skip(self))]
1360 async fn copy_object(
1361 &self,
1362 src_bucket: &str,
1363 src_object: &str,
1364 dst_bucket: &str,
1365 dst_object: &str,
1366 src_info: &mut ObjectInfo,
1367 src_opts: &ObjectOptions,
1368 dst_opts: &ObjectOptions,
1369 ) -> Result<ObjectInfo> {
1370 let result = enqueue_transition_after_write(
1371 self,
1372 self.handle_copy_object(src_bucket, src_object, dst_bucket, dst_object, src_info, src_opts, dst_opts)
1373 .await,
1374 LcEventSrc::S3CopyObject,
1375 dst_opts,
1376 )
1377 .await;
1378 if result.is_ok() {
1379 list_objects::observe_list_objects_mutation(self, dst_bucket).await;
1380 }
1381 result
1382 }
1383
1384 #[instrument(skip(self))]
1385 async fn delete_object_version(&self, bucket: &str, object: &str, fi: &FileInfo, force_del_marker: bool) -> Result<()> {
1386 let result = self.handle_delete_object_version(bucket, object, fi, force_del_marker).await;
1387 if result.is_ok() {
1388 list_objects::observe_list_objects_mutation(self, bucket).await;
1389 }
1390 result
1391 }
1392
1393 #[instrument(skip(self))]
1394 async fn delete_object(&self, bucket: &str, object: &str, opts: ObjectOptions) -> Result<ObjectInfo> {
1395 let result = self.handle_delete_object(bucket, object, opts).await;
1396 if result.is_ok() {
1397 list_objects::observe_list_objects_mutation(self, bucket).await;
1398 }
1399 result
1400 }
1401
1402 #[instrument(skip(self, objects, opts))]
1403 async fn delete_objects(
1404 &self,
1405 bucket: &str,
1406 objects: Vec<ObjectToDelete>,
1407 opts: ObjectOptions,
1408 ) -> (Vec<DeletedObject>, Vec<Option<Error>>) {
1409 let result = self.handle_delete_objects(bucket, objects, opts).await;
1410 let success_count = result.1.iter().filter(|err| err.is_none()).count();
1411 if success_count > 0 {
1412 list_objects::observe_list_objects_mutations(self, bucket, success_count).await;
1413 }
1414 result
1415 }
1416
1417 #[instrument(skip(self))]
1418 async fn put_object_metadata(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<ObjectInfo> {
1419 self.handle_put_object_metadata(bucket, object, opts).await
1420 }
1421 #[instrument(skip(self))]
1422 async fn get_object_tags(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<String> {
1423 self.handle_get_object_tags(bucket, object, opts).await
1424 }
1425
1426 #[instrument(level = "debug", skip(self))]
1427 async fn put_object_tags(&self, bucket: &str, object: &str, tags: &str, opts: &ObjectOptions) -> Result<ObjectInfo> {
1428 self.handle_put_object_tags(bucket, object, tags, opts).await
1429 }
1430
1431 #[instrument(skip(self))]
1432 async fn delete_object_tags(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<ObjectInfo> {
1433 self.handle_delete_object_tags(bucket, object, opts).await
1434 }
1435
1436 #[instrument(skip(self))]
1437 async fn add_partial(&self, bucket: &str, object: &str, version_id: &str) -> Result<()> {
1438 self.handle_add_partial(bucket, object, version_id).await
1439 }
1440 #[instrument(skip(self))]
1441 async fn transition_object(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<()> {
1442 self.handle_transition_object(bucket, object, opts).await
1443 }
1444
1445 #[instrument(skip(self))]
1446 async fn restore_transitioned_object(self: Arc<Self>, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<()> {
1447 self.handle_restore_transitioned_object(bucket, object, opts).await
1448 }
1449}
1450
1451#[async_trait::async_trait]
1452impl crate::storage_api_contracts::list::ListOperations for ECStore {
1453 type Error = Error;
1454 type ListObjectsV2Info = ListObjectsV2Info;
1455 type ListObjectVersionsInfo = ListObjectVersionsInfo;
1456 type ObjectInfoOrErr = ObjectInfoOrErr;
1457 type WalkOptions = WalkOptions;
1458 type WalkCancellation = CancellationToken;
1459 type WalkResultSender = tokio::sync::mpsc::Sender<ObjectInfoOrErr>;
1460
1461 #[instrument(level = "trace", skip(self))]
1466 async fn list_objects_v2(
1467 self: Arc<Self>,
1468 bucket: &str,
1469 prefix: &str,
1470 continuation_token: Option<String>,
1471 delimiter: Option<String>,
1472 max_keys: i32,
1473 fetch_owner: bool,
1474 start_after: Option<String>,
1475 incl_deleted: bool,
1476 ) -> Result<ListObjectsV2Info> {
1477 self.inner_list_objects_v2(
1478 bucket,
1479 prefix,
1480 continuation_token,
1481 delimiter,
1482 max_keys,
1483 fetch_owner,
1484 start_after,
1485 incl_deleted,
1486 )
1487 .await
1488 }
1489
1490 #[instrument(skip(self))]
1491 async fn list_object_versions(
1492 self: Arc<Self>,
1493 bucket: &str,
1494 prefix: &str,
1495 marker: Option<String>,
1496 version_marker: Option<String>,
1497 delimiter: Option<String>,
1498 max_keys: i32,
1499 ) -> Result<ListObjectVersionsInfo> {
1500 self.inner_list_object_versions(bucket, prefix, marker, version_marker, delimiter, max_keys)
1501 .await
1502 }
1503
1504 async fn walk(
1505 self: Arc<Self>,
1506 rx: CancellationToken,
1507 bucket: &str,
1508 prefix: &str,
1509 result: tokio::sync::mpsc::Sender<ObjectInfoOrErr>,
1510 opts: WalkOptions,
1511 ) -> Result<()> {
1512 self.walk_internal(rx, bucket, prefix, result, opts).await
1513 }
1514}
1515
1516#[async_trait::async_trait]
1517impl crate::storage_api_contracts::multipart::MultipartOperations for ECStore {
1518 type Error = Error;
1519 type ObjectInfo = ObjectInfo;
1520 type ObjectOptions = ObjectOptions;
1521 type PutObjectReader = PutObjReader;
1522 type CompletePart = CompletePart;
1523 type ListMultipartsInfo = ListMultipartsInfo;
1524 type MultipartUploadResult = MultipartUploadResult;
1525 type PartInfo = PartInfo;
1526 type MultipartInfo = MultipartInfo;
1527 type ListPartsInfo = ListPartsInfo;
1528
1529 #[instrument(skip(self))]
1530 async fn list_multipart_uploads(
1531 &self,
1532 bucket: &str,
1533 prefix: &str,
1534 key_marker: Option<String>,
1535 upload_id_marker: Option<String>,
1536 delimiter: Option<String>,
1537 max_uploads: usize,
1538 ) -> Result<ListMultipartsInfo> {
1539 self.handle_list_multipart_uploads(
1540 bucket,
1541 multipart::MultipartUploadListRequest {
1542 prefix: prefix.to_string(),
1543 key_marker,
1544 upload_id_marker,
1545 delimiter,
1546 max_uploads,
1547 expected_incarnation_id: None,
1548 },
1549 )
1550 .await
1551 }
1552
1553 #[instrument(skip(self))]
1554 async fn new_multipart_upload(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<MultipartUploadResult> {
1555 self.handle_new_multipart_upload(bucket, object, opts).await
1556 }
1557
1558 #[instrument(skip(self))]
1559 async fn copy_object_part(
1560 &self,
1561 src_bucket: &str,
1562 src_object: &str,
1563 _dst_bucket: &str,
1564 _dst_object: &str,
1565 _upload_id: &str,
1566 _part_id: usize,
1567 _start_offset: i64,
1568 _length: i64,
1569 _src_info: &ObjectInfo,
1570 _src_opts: &ObjectOptions,
1571 _dst_opts: &ObjectOptions,
1572 ) -> Result<()> {
1573 self.handle_copy_object_part(
1574 src_bucket,
1575 src_object,
1576 _dst_bucket,
1577 _dst_object,
1578 _upload_id,
1579 _part_id,
1580 _start_offset,
1581 _length,
1582 _src_info,
1583 _src_opts,
1584 _dst_opts,
1585 )
1586 .await
1587 }
1588 #[instrument(skip(self, data))]
1589 async fn put_object_part(
1590 &self,
1591 bucket: &str,
1592 object: &str,
1593 upload_id: &str,
1594 part_id: usize,
1595 data: &mut PutObjReader,
1596 opts: &ObjectOptions,
1597 ) -> Result<PartInfo> {
1598 self.handle_put_object_part(bucket, object, upload_id, part_id, data, opts)
1599 .await
1600 }
1601
1602 #[instrument(skip(self))]
1603 async fn get_multipart_info(
1604 &self,
1605 bucket: &str,
1606 object: &str,
1607 upload_id: &str,
1608 opts: &ObjectOptions,
1609 ) -> Result<MultipartInfo> {
1610 self.handle_get_multipart_info(bucket, object, upload_id, opts).await
1611 }
1612
1613 #[instrument(skip(self))]
1614 async fn list_object_parts(
1615 &self,
1616 bucket: &str,
1617 object: &str,
1618 upload_id: &str,
1619 part_number_marker: Option<usize>,
1620 max_parts: usize,
1621 opts: &ObjectOptions,
1622 ) -> Result<ListPartsInfo> {
1623 self.handle_list_object_parts(bucket, object, upload_id, part_number_marker, max_parts, opts)
1624 .await
1625 }
1626
1627 #[instrument(skip(self))]
1628 async fn abort_multipart_upload(&self, bucket: &str, object: &str, upload_id: &str, opts: &ObjectOptions) -> Result<()> {
1629 self.handle_abort_multipart_upload(bucket, object, upload_id, opts).await
1630 }
1631
1632 #[instrument(skip(self))]
1633 async fn complete_multipart_upload(
1634 self: Arc<Self>,
1635 bucket: &str,
1636 object: &str,
1637 upload_id: &str,
1638 uploaded_parts: Vec<CompletePart>,
1639 opts: &ObjectOptions,
1640 ) -> Result<ObjectInfo> {
1641 let result = enqueue_transition_after_write(
1642 self.as_ref(),
1643 self.clone()
1644 .handle_complete_multipart_upload(bucket, object, upload_id, uploaded_parts, opts)
1645 .await,
1646 LcEventSrc::S3CompleteMultipartUpload,
1647 opts,
1648 )
1649 .await;
1650 if result.is_ok() {
1651 list_objects::observe_list_objects_mutation(self.as_ref(), bucket).await;
1652 }
1653 result
1654 }
1655}
1656
1657#[async_trait::async_trait]
1658impl crate::storage_api_contracts::heal::HealOperations for ECStore {
1659 type Error = Error;
1660 type HealResultItem = HealResultItem;
1661 type HealOptions = HealOpts;
1662
1663 #[instrument(skip(self))]
1664 async fn heal_format(&self, dry_run: bool) -> Result<(HealResultItem, Option<Error>)> {
1665 self.handle_heal_format(dry_run).await
1666 }
1667
1668 #[instrument(skip(self))]
1669 async fn heal_bucket(&self, bucket: &str, opts: &HealOpts) -> Result<HealResultItem> {
1670 self.handle_heal_bucket(bucket, opts).await
1671 }
1672 #[instrument(level = "trace", skip(self, opts), fields(bucket = %bucket, object = %object, version_id = %version_id))]
1673 async fn heal_object(
1674 &self,
1675 bucket: &str,
1676 object: &str,
1677 version_id: &str,
1678 opts: &HealOpts,
1679 ) -> Result<(HealResultItem, Option<Error>)> {
1680 self.handle_heal_object(bucket, object, version_id, opts).await
1681 }
1682
1683 #[instrument(skip(self))]
1684 async fn get_pool_and_set(&self, id: &str) -> Result<(Option<usize>, Option<usize>, Option<usize>)> {
1685 self.handle_get_pool_and_set(id).await
1686 }
1687
1688 #[instrument(skip(self))]
1689 async fn check_abandoned_parts(&self, bucket: &str, object: &str, opts: &HealOpts) -> Result<()> {
1690 self.handle_check_abandoned_parts(bucket, object, opts).await
1691 }
1692}
1693
1694#[async_trait::async_trait]
1695impl crate::storage_api_contracts::namespace::NamespaceLocking for ECStore {
1696 type Error = Error;
1697 type NamespaceLock = NamespaceLockWrapper;
1698
1699 async fn new_ns_lock(&self, bucket: &str, object: &str) -> Result<NamespaceLockWrapper> {
1700 self.handle_new_ns_lock(bucket, object).await
1701 }
1702}
1703
1704#[async_trait::async_trait]
1705impl crate::storage_api_contracts::admin::StorageAdminApi for ECStore {
1706 type BackendInfo = rustfs_madmin::BackendInfo;
1707 type StorageInfo = rustfs_madmin::StorageInfo;
1708 type Disk = DiskStore;
1709 type Error = Error;
1710
1711 #[instrument(skip(self))]
1712 async fn backend_info(&self) -> Self::BackendInfo {
1713 self.handle_backend_info().await
1714 }
1715
1716 #[instrument(skip(self))]
1717 async fn storage_info(&self) -> Self::StorageInfo {
1718 self.handle_storage_info().await
1719 }
1720
1721 #[instrument(skip(self))]
1722 async fn local_storage_info(&self) -> Self::StorageInfo {
1723 self.handle_local_storage_info().await
1724 }
1725
1726 #[instrument(skip(self))]
1727 async fn disk_set_inventory(
1728 &self,
1729 selector: crate::storage_api_contracts::admin::DiskSetSelector,
1730 ) -> Result<Vec<Option<Self::Disk>>> {
1731 self.handle_get_disks(selector.pool_idx, selector.set_idx).await
1732 }
1733
1734 #[instrument(skip(self))]
1735 fn set_drive_counts(&self) -> Vec<usize> {
1736 self.handle_set_drive_counts()
1737 }
1738}
1739
1740#[cfg(test)]
1741mod tests {
1742 use super::*;
1743 use crate::core::pools::{PoolDecommissionInfo, PoolSpaceInfo, PoolStatus};
1744 use crate::layout::endpoints::{Endpoints, PoolEndpoints, SetupType};
1745 use crate::object_api::ObjectOptions;
1746 use crate::runtime::global::reset_local_disk_test_state;
1747 use crate::runtime::sources::{clear_local_disk_id_map_for_test, local_disk_path_by_id};
1748 use crate::store::init_format::{connect_load_init_formats, init_disks};
1749 use serial_test::serial;
1750 use tempfile::TempDir;
1751
1752 #[test]
1753 fn g_d2_008_default_versioning_config_keeps_persisted_bytes() {
1754 let bytes = crate::bucket::utils::serialize::<VersioningConfiguration>(&ENABLED_VERSIONING_CONFIG)
1755 .expect("the default Versioning configuration must serialize");
1756 assert_eq!(bytes, b"<VersioningConfiguration><Status>Enabled</Status></VersioningConfiguration>");
1757 }
1758
1759 #[test]
1760 fn g_d2_009_default_object_lock_config_keeps_persisted_bytes() {
1761 let bytes = crate::bucket::utils::serialize::<ObjectLockConfiguration>(&ENABLED_OBJECT_LOCK_CONFIG)
1762 .expect("the default Object Lock configuration must serialize");
1763 assert_eq!(
1764 bytes,
1765 b"<ObjectLockConfiguration><ObjectLockEnabled>Enabled</ObjectLockEnabled></ObjectLockConfiguration>"
1766 );
1767 }
1768
1769 #[tokio::test]
1770 async fn test_get_disk_infos() {
1771 let disks = vec![None, None]; let infos = get_disk_infos(&disks).await;
1773
1774 assert_eq!(infos.len(), disks.len());
1775 assert!(infos.iter().all(|info| info.is_none()));
1777 }
1778
1779 #[test]
1780 fn ecstore_debug_is_bounded_summary() {
1781 let endpoint_pools = EndpointServerPools::default();
1782 let ctx = Arc::new(InstanceContext::new());
1783 let store = ECStore {
1784 id: uuid::Uuid::new_v4(),
1785 disk_map: [(0, vec![None, None, None, None])].into_iter().collect(),
1786 pools: Vec::new(),
1787 peer_sys: crate::cluster::rpc::S3PeerSys::new_with_instance_ctx(&endpoint_pools, ctx.clone()),
1788 pool_meta: RwLock::new(PoolMeta::default()),
1789 rebalance_meta: RwLock::new(None),
1790 decommission_cancelers: RwLock::new(Vec::new()),
1791 start_gate: Mutex::new(()),
1792 pool_meta_save_gate: Mutex::default(),
1793 ctx,
1794 bucket_fence_registry: Arc::default(),
1795 };
1796
1797 let rendered = format!("{store:?}");
1798
1799 assert!(rendered.len() < 256, "ECStore Debug should stay bounded: {rendered}");
1800 assert!(rendered.contains("disk_map_pool_count"));
1801 assert!(rendered.contains("disk_slot_count"));
1802 assert!(!rendered.contains("disk_map:"));
1803 assert!(!rendered.contains("pools:"));
1804 assert!(!rendered.contains("pool_meta"));
1805 assert!(!rendered.contains("format.json"));
1806 assert!(!rendered.contains("TimedActionSlot"));
1807 assert!(!rendered.contains("DiskHealthTracker"));
1808 }
1809
1810 #[test]
1811 fn object_options_debug_does_not_expand_tier_store_handle() {
1812 let store = build_store_with_ctx(Arc::new(InstanceContext::new()));
1813 let mut opts = ObjectOptions {
1814 version_id: Some("large-version-id".repeat(1024)),
1815 expected_current_version_id: Some("large-expected-version-id".repeat(1024)),
1816 preserve_etag: Some("large-etag".repeat(1024)),
1817 http_preconditions: Some(crate::storage_api_contracts::object::HTTPPreconditions {
1818 if_match: Some("large-if-match".repeat(1024)),
1819 if_none_match: Some("large-if-none-match".repeat(1024)),
1820 ..Default::default()
1821 }),
1822 tier_delete_journal_api: Some(store),
1823 ..Default::default()
1824 };
1825 opts.user_defined.insert("large-user-metadata".to_owned(), "x".repeat(8192));
1826 opts.eval_metadata = Some([("large-eval-metadata".to_owned(), "y".repeat(8192))].into_iter().collect());
1827 opts.transition.status = "large-transition-status".repeat(1024);
1828 opts.transition.tier = "large-transition-tier".repeat(1024);
1829 opts.lifecycle_audit_event.event.rule_id = "large-rule-id".repeat(1024);
1830 opts.lifecycle_audit_event.event.storage_class = "large-storage-class".repeat(1024);
1831
1832 let rendered = format!("{opts:?}");
1833
1834 assert!(rendered.len() < 4096, "ObjectOptions Debug should stay bounded: {rendered}");
1835 assert!(rendered.contains("tier_delete_journal_api: true"));
1836 assert!(rendered.contains("user_defined_count: 1"));
1837 assert!(rendered.contains("eval_metadata_count: Some(1)"));
1838 assert!(!rendered.contains("ECStore {"));
1839 assert!(!rendered.contains("disk_map"));
1840 assert!(!rendered.contains("large-version-id"));
1841 assert!(!rendered.contains("large-expected-version-id"));
1842 assert!(!rendered.contains("large-etag"));
1843 assert!(!rendered.contains("large-if-match"));
1844 assert!(!rendered.contains("large-transition"));
1845 assert!(!rendered.contains("large-rule-id"));
1846 assert!(!rendered.contains("large-storage-class"));
1847 assert!(!rendered.contains("large-user-metadata"));
1848 assert!(!rendered.contains("large-eval-metadata"));
1849 assert!(!rendered.contains("format.json"));
1850 assert!(!rendered.contains("TimedActionSlot"));
1851 assert!(!rendered.contains("DiskHealthTracker"));
1852 }
1853
1854 pub(super) fn build_store_with_ctx(ctx: Arc<InstanceContext>) -> Arc<ECStore> {
1857 let endpoint_pools = EndpointServerPools::default();
1858 Arc::new(ECStore {
1859 id: uuid::Uuid::new_v4(),
1860 disk_map: std::collections::HashMap::new(),
1861 pools: Vec::new(),
1862 peer_sys: crate::cluster::rpc::S3PeerSys::new_with_instance_ctx(&endpoint_pools, ctx.clone()),
1863 pool_meta: RwLock::new(PoolMeta::default()),
1864 rebalance_meta: RwLock::new(None),
1865 decommission_cancelers: RwLock::new(Vec::new()),
1866 start_gate: Mutex::new(()),
1867 pool_meta_save_gate: Mutex::default(),
1868 ctx,
1869 bucket_fence_registry: Arc::default(),
1870 })
1871 }
1872
1873 fn scanner_sequence_state(operation_epoch: u64, movement_generation: u64) -> ScannerDataMovementSequenceState {
1874 ScannerDataMovementSequenceState {
1875 operation_epoch,
1876 operation_epoch_exhausted: false,
1877 movement_generation,
1878 movement_generation_exhausted: false,
1879 }
1880 }
1881
1882 #[test]
1883 fn scanner_pause_status_derives_restart_stable_decommission_fields() {
1884 let started_at = OffsetDateTime::from_unix_timestamp(1_000).expect("fixed timestamp should be valid");
1885 let now = OffsetDateTime::from_unix_timestamp(1_090).expect("fixed timestamp should be valid");
1886 let pool_meta = PoolMeta {
1887 pools: vec![PoolStatus {
1888 id: 0,
1889 cmd_line: "pool-0".to_string(),
1890 last_update: started_at,
1891 decommission: Some(PoolDecommissionInfo {
1892 start_time: Some(started_at),
1893 queued_buckets: vec!["bucket-a".to_string(), "bucket-b".to_string()],
1894 bucket: "bucket-a".to_string(),
1895 ..Default::default()
1896 }),
1897 }],
1898 ..Default::default()
1899 };
1900
1901 let status = resolve_scanner_data_movement_pause_status(&pool_meta, None, false, scanner_sequence_state(7, 11), now);
1902
1903 assert!(status.paused);
1904 assert_eq!(status.policy, "global_pause");
1905 assert_eq!(status.reasons, vec![ScannerDataMovementPauseReason::DecommissionActive]);
1906 assert_eq!(status.started_at_unix_secs, 1_000);
1907 assert_eq!(status.duration_seconds, 90);
1908 assert_eq!(status.operation_epoch, 7);
1909 assert_eq!(status.movement_generation, 11);
1910 assert_eq!(status.movement_backlog_work_items, 2);
1911 assert!(status.movement_backlog_estimated);
1912 }
1913
1914 #[test]
1915 fn completed_decommission_restores_durable_movement_generation() {
1916 let completed_at = OffsetDateTime::from_unix_timestamp(1_100).expect("fixed timestamp should be valid");
1917 let pool_meta = PoolMeta {
1918 pools: vec![PoolStatus {
1919 id: 0,
1920 cmd_line: "pool-0".to_string(),
1921 last_update: completed_at,
1922 decommission: Some(PoolDecommissionInfo {
1923 complete: true,
1924 ..Default::default()
1925 }),
1926 }],
1927 ..Default::default()
1928 };
1929 let durable_generation = durable_scanner_data_movement_generation(&pool_meta, None);
1930 let ctx = InstanceContext::new();
1931
1932 ctx.observe_durable_data_movement_generation(durable_generation);
1933
1934 assert_eq!(durable_generation, 1_100_000_000_000);
1935 assert_eq!(ctx.data_movement_generation(), durable_generation);
1936 }
1937
1938 #[tokio::test]
1939 async fn cleared_decommission_restores_durable_movement_generation_after_restart() {
1940 let mut pool_meta = PoolMeta {
1941 pools: vec![PoolStatus {
1942 id: 0,
1943 cmd_line: "pool-0".to_string(),
1944 last_update: OffsetDateTime::UNIX_EPOCH,
1945 decommission: Some(PoolDecommissionInfo {
1946 failed: true,
1947 ..Default::default()
1948 }),
1949 }],
1950 ..Default::default()
1951 };
1952 assert!(pool_meta.clear_decommission(0).expect("failed decommission should clear"));
1953 assert!(
1954 pool_meta.pools[0]
1955 .decommission
1956 .as_ref()
1957 .is_some_and(|info| !info.has_decommission_state())
1958 );
1959 let durable_generation = durable_scanner_data_movement_generation(&pool_meta, None);
1960 let restarted = build_store_with_ctx(Arc::new(InstanceContext::new()));
1961 *restarted.pool_meta.write().await = pool_meta;
1962
1963 let status = restarted.scanner_data_movement_pause_status().await;
1964
1965 assert_ne!(durable_generation, 0);
1966 assert!(!status.paused);
1967 assert_eq!(status.movement_generation, durable_generation);
1968 assert_eq!(restarted.scanner_data_movement_generation(), durable_generation);
1969 }
1970
1971 #[tokio::test]
1972 async fn same_tick_cleared_decommission_tombstones_advance_durable_movement_generation() {
1973 let same_tick = OffsetDateTime::from_unix_timestamp(1_100).expect("fixed timestamp should be valid");
1974 let mut pool_meta = PoolMeta {
1975 pools: vec![
1976 PoolStatus {
1977 id: 0,
1978 cmd_line: "pool-0".to_string(),
1979 last_update: same_tick,
1980 decommission: Some(PoolDecommissionInfo {
1981 failed: true,
1982 ..Default::default()
1983 }),
1984 },
1985 PoolStatus {
1986 id: 1,
1987 cmd_line: "pool-1".to_string(),
1988 last_update: same_tick,
1989 decommission: Some(PoolDecommissionInfo {
1990 canceled: true,
1991 ..Default::default()
1992 }),
1993 },
1994 ],
1995 ..Default::default()
1996 };
1997
1998 assert!(
1999 pool_meta
2000 .clear_decommission_at_for_test(0, same_tick, None)
2001 .expect("first terminal decommission should clear")
2002 );
2003 let first_generation = durable_scanner_data_movement_generation(&pool_meta, None);
2004 assert_eq!(
2005 first_generation,
2006 scanner_data_movement_timestamp_generation(same_tick + time::Duration::nanoseconds(1))
2007 );
2008
2009 assert!(
2010 pool_meta
2011 .clear_decommission_at_for_test(1, same_tick, None)
2012 .expect("second terminal decommission should clear")
2013 );
2014 let second_generation = durable_scanner_data_movement_generation(&pool_meta, None);
2015 assert_eq!(
2016 second_generation,
2017 scanner_data_movement_timestamp_generation(same_tick + time::Duration::nanoseconds(2))
2018 );
2019 assert!(second_generation > first_generation);
2020
2021 let restarted = build_store_with_ctx(Arc::new(InstanceContext::new()));
2022 *restarted.pool_meta.write().await = pool_meta;
2023 let status = restarted.scanner_data_movement_pause_status().await;
2024
2025 assert!(!status.paused);
2026 assert_eq!(status.movement_generation, second_generation);
2027 assert_eq!(restarted.scanner_data_movement_generation(), second_generation);
2028 }
2029
2030 #[tokio::test]
2031 async fn terminal_decommission_transitions_advance_durable_generation_across_same_or_earlier_clocks() {
2032 let same_tick = OffsetDateTime::from_unix_timestamp(1_200).expect("fixed timestamp should be valid");
2033 let earlier_tick = same_tick - time::Duration::nanoseconds(10);
2034 let rebalance_floor = same_tick + time::Duration::nanoseconds(5);
2035 let rebalance = RebalanceMeta {
2036 stopped_at: Some(rebalance_floor),
2037 id: "completed-rebalance".to_string(),
2038 ..Default::default()
2039 };
2040 let active_decommission = |id| PoolStatus {
2041 id,
2042 cmd_line: format!("pool-{id}"),
2043 last_update: same_tick,
2044 decommission: Some(PoolDecommissionInfo {
2045 start_time: Some(same_tick),
2046 ..Default::default()
2047 }),
2048 };
2049 let mut pool_meta = PoolMeta {
2050 pools: vec![active_decommission(0), active_decommission(1), active_decommission(2)],
2051 ..Default::default()
2052 };
2053
2054 assert!(pool_meta.decommission_complete_at_for_test(0, same_tick, Some(&rebalance)));
2055 assert_eq!(pool_meta.pools[0].last_update, rebalance_floor + time::Duration::nanoseconds(1));
2056
2057 assert!(pool_meta.decommission_cancel_at_for_test(1, same_tick, Some(&rebalance)));
2058 assert_eq!(pool_meta.pools[1].last_update, rebalance_floor + time::Duration::nanoseconds(2));
2059
2060 assert!(pool_meta.decommission_failed_at_for_test(2, earlier_tick, Some(&rebalance)));
2061 assert_eq!(pool_meta.pools[2].last_update, rebalance_floor + time::Duration::nanoseconds(3));
2062 let durable_generation = durable_scanner_data_movement_generation(&pool_meta, Some(&rebalance));
2063 assert_eq!(
2064 durable_generation,
2065 scanner_data_movement_timestamp_generation(rebalance_floor + time::Duration::nanoseconds(3))
2066 );
2067
2068 let restarted = build_store_with_ctx(Arc::new(InstanceContext::new()));
2069 *restarted.pool_meta.write().await = pool_meta;
2070 *restarted.rebalance_meta.write().await = Some(rebalance);
2071 let status = restarted.scanner_data_movement_pause_status().await;
2072
2073 assert_eq!(status.movement_generation, durable_generation);
2074 assert_eq!(restarted.scanner_data_movement_generation(), durable_generation);
2075 assert_eq!(
2076 status.reasons,
2077 vec![
2078 ScannerDataMovementPauseReason::DecommissionFailed,
2079 ScannerDataMovementPauseReason::DecommissionCanceled
2080 ]
2081 );
2082 }
2083
2084 #[tokio::test]
2085 async fn decommission_start_after_clear_advances_durable_generation_across_clock_rollback_after_restart() {
2086 let same_tick = OffsetDateTime::from_unix_timestamp(1_250).expect("fixed timestamp should be valid");
2087 let earlier_tick = same_tick - time::Duration::nanoseconds(10);
2088 let rebalance_floor = same_tick + time::Duration::nanoseconds(5);
2089 let rebalance = RebalanceMeta {
2090 stopped_at: Some(rebalance_floor),
2091 id: "completed-rebalance".to_string(),
2092 ..Default::default()
2093 };
2094 let mut pool_meta = PoolMeta {
2095 pools: vec![PoolStatus {
2096 id: 0,
2097 cmd_line: "pool-0".to_string(),
2098 last_update: same_tick,
2099 decommission: Some(PoolDecommissionInfo {
2100 failed: true,
2101 ..Default::default()
2102 }),
2103 }],
2104 ..Default::default()
2105 };
2106
2107 assert!(
2108 pool_meta
2109 .clear_decommission_at_for_test(0, same_tick, Some(&rebalance))
2110 .expect("failed decommission should clear")
2111 );
2112 let cleared_at = rebalance_floor + time::Duration::nanoseconds(1);
2113 assert_eq!(pool_meta.pools[0].last_update, cleared_at);
2114
2115 pool_meta
2116 .decommission_at_for_test(
2117 0,
2118 PoolSpaceInfo {
2119 total: 200,
2120 free: 50,
2121 used: 150,
2122 },
2123 earlier_tick,
2124 Some(&rebalance),
2125 )
2126 .expect("decommission restart after clear should be allowed");
2127 let started_at = cleared_at + time::Duration::nanoseconds(1);
2128 assert_eq!(pool_meta.pools[0].last_update, started_at);
2129 assert_eq!(
2130 pool_meta.pools[0].decommission.as_ref().and_then(|info| info.start_time),
2131 Some(started_at)
2132 );
2133
2134 assert!(pool_meta.decommission_complete_at_for_test(0, earlier_tick, Some(&rebalance)));
2135 let completed_at = started_at + time::Duration::nanoseconds(1);
2136 assert_eq!(pool_meta.pools[0].last_update, completed_at);
2137 let durable_generation = durable_scanner_data_movement_generation(&pool_meta, Some(&rebalance));
2138 assert_eq!(durable_generation, scanner_data_movement_timestamp_generation(completed_at));
2139
2140 let restarted = build_store_with_ctx(Arc::new(InstanceContext::new()));
2141 *restarted.pool_meta.write().await = pool_meta;
2142 *restarted.rebalance_meta.write().await = Some(rebalance);
2143 let status = restarted.scanner_data_movement_pause_status().await;
2144
2145 assert!(!status.paused);
2146 assert_eq!(status.movement_generation, durable_generation);
2147 assert_eq!(restarted.scanner_data_movement_generation(), durable_generation);
2148 }
2149
2150 #[tokio::test]
2151 async fn decommission_terminal_reload_failure_advances_durable_generation_across_clock_rollback_after_restart() {
2152 let terminal_at = OffsetDateTime::from_unix_timestamp(1_280).expect("fixed timestamp should be valid");
2153 let earlier_tick = terminal_at - time::Duration::nanoseconds(10);
2154 let rebalance_floor = terminal_at + time::Duration::nanoseconds(5);
2155 let rebalance = RebalanceMeta {
2156 stopped_at: Some(rebalance_floor),
2157 id: "completed-rebalance".to_string(),
2158 ..Default::default()
2159 };
2160 let mut pool_meta = PoolMeta {
2161 pools: vec![PoolStatus {
2162 id: 0,
2163 cmd_line: "pool-0".to_string(),
2164 last_update: terminal_at,
2165 decommission: Some(PoolDecommissionInfo {
2166 start_time: Some(terminal_at),
2167 complete: true,
2168 ..Default::default()
2169 }),
2170 }],
2171 ..Default::default()
2172 };
2173
2174 assert!(
2175 pool_meta
2176 .record_decommission_terminal_reload_failure_at_for_test(
2177 0,
2178 "complete_decommission",
2179 "peer reload failed".to_string(),
2180 earlier_tick,
2181 Some(&rebalance),
2182 )
2183 .expect("reload failure should be recorded")
2184 );
2185 let reload_failure_at = rebalance_floor + time::Duration::nanoseconds(1);
2186 assert_eq!(pool_meta.pools[0].last_update, reload_failure_at);
2187 let info = pool_meta.pools[0]
2188 .decommission
2189 .as_ref()
2190 .expect("decommission metadata should exist");
2191 assert_eq!(info.terminal_reload_attempt_at, Some(reload_failure_at));
2192 assert_eq!(
2193 info.terminal_reload_failures,
2194 vec!["complete_decommission: peer reload failed".to_string()]
2195 );
2196 let durable_generation = durable_scanner_data_movement_generation(&pool_meta, Some(&rebalance));
2197 assert_eq!(durable_generation, scanner_data_movement_timestamp_generation(reload_failure_at));
2198
2199 let restarted = build_store_with_ctx(Arc::new(InstanceContext::new()));
2200 *restarted.pool_meta.write().await = pool_meta;
2201 *restarted.rebalance_meta.write().await = Some(rebalance);
2202 let status = restarted.scanner_data_movement_pause_status().await;
2203
2204 assert!(!status.paused);
2205 assert_eq!(status.movement_generation, durable_generation);
2206 assert_eq!(restarted.scanner_data_movement_generation(), durable_generation);
2207 }
2208
2209 #[tokio::test]
2210 async fn rebalance_transitions_advance_durable_generation_across_same_or_earlier_clocks_after_restart() {
2211 let same_tick = OffsetDateTime::from_unix_timestamp(1_300).expect("fixed timestamp should be valid");
2212 let earlier_tick = same_tick - time::Duration::nanoseconds(10);
2213 let decommission_floor = same_tick + time::Duration::nanoseconds(5);
2214 let store = build_store_with_ctx(Arc::new(InstanceContext::new()));
2215 *store.pool_meta.write().await = PoolMeta {
2216 pools: vec![PoolStatus {
2217 id: 0,
2218 cmd_line: "pool-0".to_string(),
2219 last_update: decommission_floor,
2220 decommission: Some(PoolDecommissionInfo {
2221 complete: true,
2222 ..Default::default()
2223 }),
2224 }],
2225 ..Default::default()
2226 };
2227
2228 let started_at = store.next_scanner_data_movement_update(same_tick).await;
2229 assert_eq!(started_at, decommission_floor + time::Duration::nanoseconds(1));
2230 *store.rebalance_meta.write().await = Some(RebalanceMeta {
2231 id: "rebalance-generation".to_string(),
2232 pool_stats: vec![crate::services::rebalance::RebalanceStats {
2233 participating: true,
2234 info: crate::services::rebalance::RebalanceInfo {
2235 start_time: Some(started_at),
2236 status: RebalStatus::Started,
2237 ..Default::default()
2238 },
2239 ..Default::default()
2240 }],
2241 ..Default::default()
2242 });
2243
2244 let completed_at = store.next_scanner_data_movement_update(same_tick).await;
2245 assert_eq!(completed_at, decommission_floor + time::Duration::nanoseconds(2));
2246 {
2247 let mut rebalance_meta = store.rebalance_meta.write().await;
2248 let meta = rebalance_meta.as_mut().expect("rebalance metadata should be present");
2249 meta.pool_stats[0].info.status = RebalStatus::Completed;
2250 meta.pool_stats[0].info.end_time = Some(completed_at);
2251 }
2252
2253 let stopped_at = store.next_scanner_data_movement_update(earlier_tick).await;
2254 assert_eq!(stopped_at, decommission_floor + time::Duration::nanoseconds(3));
2255 {
2256 let mut rebalance_meta = store.rebalance_meta.write().await;
2257 let meta = rebalance_meta.as_mut().expect("rebalance metadata should be present");
2258 meta.stopped_at = Some(stopped_at);
2259 }
2260 let pool_meta = store.pool_meta.read().await.clone();
2261 let rebalance_meta = store.rebalance_meta.read().await.clone();
2262 let durable_generation = durable_scanner_data_movement_generation(&pool_meta, rebalance_meta.as_ref());
2263 assert_eq!(
2264 durable_generation,
2265 scanner_data_movement_timestamp_generation(decommission_floor + time::Duration::nanoseconds(3))
2266 );
2267
2268 let restarted = build_store_with_ctx(Arc::new(InstanceContext::new()));
2269 *restarted.pool_meta.write().await = pool_meta;
2270 *restarted.rebalance_meta.write().await = rebalance_meta;
2271 let status = restarted.scanner_data_movement_pause_status().await;
2272
2273 assert!(!status.paused);
2274 assert_eq!(status.movement_generation, durable_generation);
2275 assert_eq!(restarted.scanner_data_movement_generation(), durable_generation);
2276 }
2277
2278 #[test]
2279 fn malformed_durable_movement_timestamp_exhausts_generation_fail_closed() {
2280 let pool_meta = PoolMeta {
2281 pools: vec![PoolStatus {
2282 id: 0,
2283 cmd_line: "pool-0".to_string(),
2284 last_update: OffsetDateTime::UNIX_EPOCH,
2285 decommission: Some(PoolDecommissionInfo {
2286 complete: true,
2287 ..Default::default()
2288 }),
2289 }],
2290 ..Default::default()
2291 };
2292
2293 assert_eq!(durable_scanner_data_movement_generation(&pool_meta, None), u64::MAX);
2294 let exhausted_generation =
2295 OffsetDateTime::from_unix_timestamp(253_402_300_799).expect("the largest RFC 3339 timestamp should be valid");
2296 assert_eq!(scanner_data_movement_timestamp_generation(exhausted_generation), u64::MAX);
2297 }
2298
2299 #[test]
2300 fn malformed_durable_movement_timestamp_is_not_masked_by_valid_rebalance_generation() {
2301 let valid_rebalance_at = OffsetDateTime::from_unix_timestamp(2_400).expect("fixed timestamp should be valid");
2302 let pool_meta = PoolMeta {
2303 pools: vec![PoolStatus {
2304 id: 0,
2305 cmd_line: "pool-0".to_string(),
2306 last_update: OffsetDateTime::UNIX_EPOCH,
2307 decommission: Some(PoolDecommissionInfo {
2308 complete: true,
2309 ..Default::default()
2310 }),
2311 }],
2312 ..Default::default()
2313 };
2314 let rebalance_meta = RebalanceMeta {
2315 id: "completed-rebalance".to_string(),
2316 stopped_at: Some(valid_rebalance_at),
2317 pool_stats: vec![crate::services::rebalance::RebalanceStats {
2318 participating: true,
2319 info: crate::services::rebalance::RebalanceInfo {
2320 start_time: Some(valid_rebalance_at - time::Duration::nanoseconds(1)),
2321 end_time: Some(valid_rebalance_at),
2322 status: RebalStatus::Completed,
2323 ..Default::default()
2324 },
2325 ..Default::default()
2326 }],
2327 ..Default::default()
2328 };
2329
2330 assert_eq!(durable_scanner_data_movement_generation(&pool_meta, Some(&rebalance_meta)), u64::MAX);
2331 }
2332
2333 #[test]
2334 fn durable_movement_generation_without_records_is_zero() {
2335 assert_eq!(durable_scanner_data_movement_generation(&PoolMeta::default(), None), 0);
2336 assert_eq!(
2337 durable_scanner_data_movement_generation(&PoolMeta::default(), Some(&RebalanceMeta::default())),
2338 0
2339 );
2340 }
2341
2342 #[test]
2343 fn scanner_pause_status_distinguishes_terminal_rebalance_epoch_and_idle() {
2344 let last_update = OffsetDateTime::from_unix_timestamp(2_000).expect("fixed timestamp should be valid");
2345 let now = OffsetDateTime::from_unix_timestamp(2_030).expect("fixed timestamp should be valid");
2346 let failed = PoolMeta {
2347 pools: vec![PoolStatus {
2348 id: 0,
2349 cmd_line: "pool-0".to_string(),
2350 last_update,
2351 decommission: Some(PoolDecommissionInfo {
2352 failed: true,
2353 ..Default::default()
2354 }),
2355 }],
2356 ..Default::default()
2357 };
2358 let failed_status = resolve_scanner_data_movement_pause_status(&failed, None, false, scanner_sequence_state(3, 12), now);
2359 assert_eq!(failed_status.reasons, vec![ScannerDataMovementPauseReason::DecommissionFailed]);
2360 assert_eq!(failed_status.started_at_unix_secs, 2_000);
2361 assert_eq!(failed_status.duration_seconds, 30);
2362
2363 let rebalance = RebalanceMeta {
2364 pool_stats: vec![crate::services::rebalance::RebalanceStats {
2365 buckets: vec!["bucket-a".to_string(), "bucket-b".to_string()],
2366 participating: true,
2367 info: crate::services::rebalance::RebalanceInfo {
2368 start_time: Some(last_update),
2369 status: RebalStatus::Started,
2370 ..Default::default()
2371 },
2372 ..Default::default()
2373 }],
2374 ..Default::default()
2375 };
2376 let rebalance_status = resolve_scanner_data_movement_pause_status(
2377 &PoolMeta::default(),
2378 Some(&rebalance),
2379 false,
2380 scanner_sequence_state(4, 13),
2381 now,
2382 );
2383 assert_eq!(rebalance_status.reasons, vec![ScannerDataMovementPauseReason::RebalanceActive]);
2384 assert_eq!(rebalance_status.movement_backlog_work_items, 2);
2385
2386 let exhausted = resolve_scanner_data_movement_pause_status(
2387 &PoolMeta::default(),
2388 None,
2389 false,
2390 ScannerDataMovementSequenceState {
2391 operation_epoch: u64::MAX,
2392 operation_epoch_exhausted: true,
2393 movement_generation: 14,
2394 movement_generation_exhausted: false,
2395 },
2396 now,
2397 );
2398 assert_eq!(exhausted.reasons, vec![ScannerDataMovementPauseReason::OperationEpochExhausted]);
2399 assert_eq!(exhausted.started_at_unix_secs, 0);
2400
2401 let generation_exhausted = resolve_scanner_data_movement_pause_status(
2402 &PoolMeta::default(),
2403 None,
2404 false,
2405 ScannerDataMovementSequenceState {
2406 operation_epoch: 5,
2407 operation_epoch_exhausted: false,
2408 movement_generation: u64::MAX,
2409 movement_generation_exhausted: true,
2410 },
2411 now,
2412 );
2413 assert_eq!(
2414 generation_exhausted.reasons,
2415 vec![ScannerDataMovementPauseReason::MovementGenerationExhausted]
2416 );
2417
2418 let idle =
2419 resolve_scanner_data_movement_pause_status(&PoolMeta::default(), None, false, scanner_sequence_state(5, 15), now);
2420 assert!(!idle.paused);
2421 assert!(idle.reasons.is_empty());
2422 assert!(!idle.movement_backlog_estimated);
2423 }
2424
2425 #[tokio::test]
2426 async fn scanner_data_usage_publication_blocks_active_and_unqueued_terminal_decommission() {
2427 let store = build_store_with_ctx(Arc::new(InstanceContext::new()));
2428 let cases = [
2429 (
2430 "active",
2431 PoolDecommissionInfo {
2432 start_time: Some(OffsetDateTime::now_utc()),
2433 ..Default::default()
2434 },
2435 true,
2436 ),
2437 (
2438 "failed",
2439 PoolDecommissionInfo {
2440 failed: true,
2441 ..Default::default()
2442 },
2443 true,
2444 ),
2445 (
2446 "canceled",
2447 PoolDecommissionInfo {
2448 canceled: true,
2449 ..Default::default()
2450 },
2451 true,
2452 ),
2453 (
2454 "queued_failed",
2455 PoolDecommissionInfo {
2456 failed: true,
2457 queued: true,
2458 ..Default::default()
2459 },
2460 false,
2461 ),
2462 (
2463 "complete",
2464 PoolDecommissionInfo {
2465 complete: true,
2466 ..Default::default()
2467 },
2468 false,
2469 ),
2470 ("idle", PoolDecommissionInfo::default(), false),
2471 ];
2472
2473 for (name, decommission, expected) in cases {
2474 *store.pool_meta.write().await = PoolMeta {
2475 pools: vec![PoolStatus {
2476 id: 0,
2477 cmd_line: format!("scanner-publication-{name}"),
2478 last_update: OffsetDateTime::now_utc(),
2479 decommission: Some(decommission),
2480 }],
2481 ..Default::default()
2482 };
2483 assert_eq!(
2484 store.scanner_data_usage_publication_blocked().await,
2485 expected,
2486 "unexpected scanner publication barrier state for {name}"
2487 );
2488 }
2489 }
2490
2491 #[tokio::test]
2492 async fn scanner_data_usage_publication_admission_is_fenced_and_epoch_monotonic() {
2493 let store = build_store_with_ctx(Arc::new(InstanceContext::new()));
2494 let operation_gate = store.ctx.data_movement_operation_gate();
2495 let movement_guard = operation_gate.write().await;
2496 let pending = {
2497 let store = store.clone();
2498 tokio::spawn(async move { store.scanner_data_usage_publication_admission_guard().await })
2499 };
2500 tokio::task::yield_now().await;
2501 assert!(!pending.is_finished(), "publication admission must wait for a movement writer");
2502 drop(movement_guard);
2503
2504 let (_, epoch) = pending
2505 .await
2506 .expect("publication admission task should not panic")
2507 .expect("idle store should admit publication");
2508 assert_eq!(epoch, 0);
2509 assert_eq!(store.ctx.advance_data_movement_operation_epoch(), 1);
2510 let (_, next_epoch) = store
2511 .scanner_data_usage_publication_admission_guard()
2512 .await
2513 .expect("idle store should admit the next publication");
2514 assert_eq!(next_epoch, 1);
2515 assert_eq!(store.scanner_data_movement_generation(), 1);
2516 }
2517
2518 #[tokio::test(start_paused = true)]
2519 async fn scanner_publication_lease_blocks_movement_writer_until_release_or_expiry() {
2520 let store = build_store_with_ctx(Arc::new(InstanceContext::new()));
2521 let (token, generation) = store
2522 .acquire_scanner_publication_lease(0, crate::runtime::instance::SCANNER_PUBLICATION_LEASE_TTL)
2523 .await
2524 .expect("an idle store should grant a publication lease");
2525 assert_eq!(generation, 0);
2526
2527 let gate = store.ctx.data_movement_operation_gate();
2528 let (writer_started, writer_started_rx) = tokio::sync::oneshot::channel();
2529 let movement_writer = tokio::spawn(async move {
2530 let _ = writer_started.send(());
2531 gate.write_owned().await
2532 });
2533 writer_started_rx
2534 .await
2535 .expect("movement writer should reach the gate before waiting");
2536 assert!(
2537 !movement_writer.is_finished(),
2538 "a movement writer must wait while the remote lease owns the read guard"
2539 );
2540
2541 assert!(store.release_scanner_publication_lease(token).await);
2542 tokio::time::timeout(Duration::from_secs(1), movement_writer)
2543 .await
2544 .expect("movement writer should proceed after lease release")
2545 .expect("movement writer task should not panic");
2546
2547 let (expiring_token, _) = store
2548 .acquire_scanner_publication_lease(0, crate::runtime::instance::SCANNER_PUBLICATION_LEASE_TTL)
2549 .await
2550 .expect("the store should grant a second publication lease");
2551 let expiry_gate = store.ctx.data_movement_operation_gate();
2552 let (expiry_started, expiry_started_rx) = tokio::sync::oneshot::channel();
2553 let mut expiry_writer = tokio::spawn(async move {
2554 let _ = expiry_started.send(());
2555 expiry_gate.write_owned().await
2556 });
2557 expiry_started_rx
2558 .await
2559 .expect("expiry writer should reach the gate before waiting");
2560 assert!(!expiry_writer.is_finished());
2561 tokio::time::advance(crate::runtime::instance::SCANNER_PUBLICATION_LEASE_TTL + Duration::from_millis(1)).await;
2562 tokio::task::yield_now().await;
2563 tokio::time::timeout(Duration::from_secs(1), &mut expiry_writer)
2564 .await
2565 .expect("movement writer should proceed after lease expiry")
2566 .expect("expiry writer task should not panic");
2567 assert!(
2568 store.validate_scanner_publication_lease(expiring_token, 0).await.is_err(),
2569 "an expired lease must not validate after its read guard is released"
2570 );
2571 assert!(!store.release_scanner_publication_lease(expiring_token).await);
2572 }
2573
2574 #[tokio::test]
2575 async fn scanner_publication_lease_rejects_a_new_movement_generation() {
2576 let store = build_store_with_ctx(Arc::new(InstanceContext::new()));
2577 let (token, generation) = store
2578 .acquire_scanner_publication_lease(0, crate::runtime::instance::SCANNER_PUBLICATION_LEASE_TTL)
2579 .await
2580 .expect("an idle store should grant a publication lease");
2581
2582 assert_eq!(store.ctx.advance_data_movement_generation(), Some(1));
2583 assert!(
2584 store.validate_scanner_publication_lease(token, generation).await.is_err(),
2585 "a lease from the prior movement generation must fail closed"
2586 );
2587 assert!(store.release_scanner_publication_lease(token).await);
2588 }
2589
2590 #[tokio::test]
2591 async fn scanner_publication_lease_rejects_stale_generation_before_install() {
2592 let store = build_store_with_ctx(Arc::new(InstanceContext::new()));
2593 let error = store
2594 .acquire_scanner_publication_lease(1, crate::runtime::instance::SCANNER_PUBLICATION_LEASE_TTL)
2595 .await
2596 .expect_err("a stale movement generation must not acquire a lease");
2597 assert!(error.to_string().contains("generation is stale"));
2598 }
2599
2600 #[tokio::test]
2601 async fn scanner_publication_lease_rejects_namespace_change_during_admission() {
2602 let ctx = Arc::new(InstanceContext::new());
2603 let store = build_store_with_ctx(ctx.clone());
2604 let snapshot_blocker = store.rebalance_meta.write().await;
2605 let mut acquire =
2606 Box::pin(store.acquire_scanner_publication_lease(0, crate::runtime::instance::SCANNER_PUBLICATION_LEASE_TTL));
2607 assert!(futures::poll!(&mut acquire).is_pending(), "acquisition reaches the blocked snapshot");
2608 assert!(ctx.data_movement_operation_gate().try_write_owned().is_err());
2609 drop(ctx.begin_namespace_commit());
2610 assert_eq!(ctx.namespace_commit_generation(), 2);
2611 assert!(!ctx.namespace_commits_pending());
2612 drop(snapshot_blocker);
2613 let error = tokio::time::timeout(Duration::from_secs(1), acquire)
2614 .await
2615 .expect("snapshot admission must finish")
2616 .expect_err("acquisition must preserve its original namespace generation");
2617 assert_eq!(error.to_string(), "Io error: scanner publication lease generation is stale");
2618 assert!(ctx.data_movement_operation_gate().try_write_owned().is_ok(), "no lease was installed");
2619 }
2620
2621 #[tokio::test]
2622 async fn scanner_publication_lease_rechecks_namespace_after_validate_snapshot() {
2623 let ctx = Arc::new(InstanceContext::new());
2624 let store = build_store_with_ctx(ctx.clone());
2625 let (token, generation) = store
2626 .acquire_scanner_publication_lease(0, crate::runtime::instance::SCANNER_PUBLICATION_LEASE_TTL)
2627 .await
2628 .expect("current lease");
2629 let snapshot_blocker = store.rebalance_meta.write().await;
2630 let mut validate = Box::pin(store.validate_scanner_publication_lease(token, generation));
2631 assert!(futures::poll!(&mut validate).is_pending(), "target guard queues its first snapshot read");
2632 let mut next_writer = Box::pin(store.rebalance_meta.write());
2633 assert!(futures::poll!(&mut next_writer).is_pending());
2634 drop(snapshot_blocker);
2635 assert!(futures::poll!(&mut validate).is_pending(), "validate reaches its second snapshot");
2638 let next_writer = next_writer.await;
2639 drop(ctx.begin_namespace_commit());
2640 assert_eq!(ctx.namespace_commit_generation(), 2);
2641 assert!(!ctx.namespace_commits_pending());
2642 drop(next_writer);
2643 let error = tokio::time::timeout(Duration::from_secs(1), validate)
2644 .await
2645 .expect("validation must finish without nesting movement read locks")
2646 .expect_err("the final snapshot must reject a completed namespace commit");
2647 assert_eq!(error.to_string(), "Io error: scanner publication lease generation is stale");
2648 assert!(
2649 ctx.data_movement_operation_gate().try_write_owned().is_err(),
2650 "stale lookup retains the lease permit"
2651 );
2652 assert!(store.release_scanner_publication_lease(token).await);
2653 assert!(ctx.data_movement_operation_gate().try_write_owned().is_ok());
2654 }
2655
2656 #[tokio::test]
2657 async fn scanner_publication_lease_rejects_namespace_generation_exhaustion() {
2658 let ctx = Arc::new(InstanceContext::new());
2659 let store = build_store_with_ctx(ctx.clone());
2660 let (token, generation) = store
2661 .acquire_scanner_publication_lease(0, crate::runtime::instance::SCANNER_PUBLICATION_LEASE_TTL)
2662 .await
2663 .expect("current lease");
2664 ctx.set_namespace_commit_generation_for_test(u64::MAX);
2665 assert_eq!(store.scanner_namespace_mutation_generation(), u64::MAX);
2666 let acquire = store
2667 .acquire_scanner_publication_lease(generation, crate::runtime::instance::SCANNER_PUBLICATION_LEASE_TTL)
2668 .await
2669 .expect_err("exhausted namespace cannot become a new lease baseline");
2670 assert_eq!(acquire.to_string(), "Io error: scanner publication lease generation is stale");
2671 let validate = store.validate_scanner_publication_lease(token, generation).await;
2672 let target = store.acquire_scanner_publication_lease_guard(token).await;
2673 assert!(validate.is_err() && target.is_err(), "exhaustion rejects both old-token entrances");
2674 assert!(
2675 ctx.data_movement_operation_gate().try_write_owned().is_err(),
2676 "rejection must retain the old permit"
2677 );
2678 assert!(store.release_scanner_publication_lease(token).await);
2679 assert!(ctx.data_movement_operation_gate().try_write_owned().is_ok());
2680 }
2681
2682 #[tokio::test(start_paused = true)]
2683 async fn scanner_publication_commit_scope_owns_permit_until_terminal_drain() {
2684 let store = build_store_with_ctx(Arc::new(InstanceContext::new()));
2685 let scope = store
2686 .scanner_data_usage_publication_commit_scope(
2687 0,
2688 tokio::time::Instant::now() + Duration::from_secs(30),
2689 vec![Uuid::new_v4()],
2690 )
2691 .await
2692 .expect("idle storage should grant a publication scope");
2693 assert_eq!(scope.state(), crate::object_api::ScannerPublicationCommitState::Admitted);
2694 assert_eq!(scope.remote_lease_tokens().len(), 1);
2695
2696 let gate = store.ctx.data_movement_operation_gate();
2697 let writer = tokio::spawn(async move { gate.write_owned().await });
2698 tokio::task::yield_now().await;
2699 assert!(!writer.is_finished(), "the scope must own its movement permit after the caller returns");
2700
2701 scope.cancel();
2702 assert!(scope.mark_aborted_before_commit());
2703 assert_eq!(
2704 scope.wait_for_completion().await,
2705 crate::object_api::ScannerPublicationCommitState::AbortedBeforeCommit
2706 );
2707 assert!(scope.release_movement_permit().await);
2708 tokio::time::timeout(Duration::from_secs(1), writer)
2709 .await
2710 .expect("movement writer should proceed after the scope drains")
2711 .expect("movement writer task should not panic");
2712 }
2713
2714 #[tokio::test(start_paused = true)]
2715 async fn scanner_publication_commit_scope_rejects_late_start_and_keeps_indeterminate_permit() {
2716 let store = build_store_with_ctx(Arc::new(InstanceContext::new()));
2717 let scope = store
2718 .scanner_data_usage_publication_commit_scope(0, tokio::time::Instant::now() + Duration::from_secs(1), Vec::new())
2719 .await
2720 .expect("idle storage should grant a publication scope");
2721 tokio::time::advance(Duration::from_secs(1)).await;
2722 assert_eq!(
2723 scope.try_begin(),
2724 Err(crate::object_api::ScannerPublicationCommitStartError::DeadlineExceeded)
2725 );
2726 assert!(
2727 !scope.release_movement_permit().await,
2728 "an admitted scope is not safe to release before owner resolution"
2729 );
2730 assert!(scope.mark_aborted_before_commit());
2731 assert!(scope.release_movement_permit().await);
2732
2733 let scope = store
2734 .scanner_data_usage_publication_commit_scope(0, tokio::time::Instant::now() + Duration::from_secs(30), Vec::new())
2735 .await
2736 .expect("a second idle publication scope should be granted");
2737 scope.try_begin().expect("scope should enter the mutation state");
2738 scope.cancel();
2739 assert!(scope.mark_indeterminate());
2740 assert_eq!(
2741 scope.wait_for_completion().await,
2742 crate::object_api::ScannerPublicationCommitState::Indeterminate
2743 );
2744 assert!(!scope.release_movement_permit().await, "indeterminate mutation must retain the permit");
2745 }
2746
2747 #[tokio::test]
2748 async fn scanner_publication_scope_guard_classifies_early_returns_conservatively() {
2749 let store = build_store_with_ctx(Arc::new(InstanceContext::new()));
2750 let permit = store.ctx.data_movement_operation_gate().read_owned().await;
2751 let scope = ScannerPublicationCommitScope::new_storage_owned(
2752 0,
2753 tokio::time::Instant::now() + Duration::from_secs(30),
2754 Vec::new(),
2755 permit,
2756 );
2757 {
2758 let _guard = crate::object_api::ScannerPublicationCommitScopeGuard::new(scope.clone());
2759 }
2760 assert_eq!(scope.state(), crate::object_api::ScannerPublicationCommitState::AbortedBeforeCommit);
2761 assert!(scope.release_movement_permit().await);
2762
2763 let permit = store.ctx.data_movement_operation_gate().read_owned().await;
2764 let scope = ScannerPublicationCommitScope::new_storage_owned(
2765 0,
2766 tokio::time::Instant::now() + Duration::from_secs(30),
2767 Vec::new(),
2768 permit,
2769 );
2770 scope.try_begin().expect("scope should enter the mutation state");
2771 {
2772 let _guard = crate::object_api::ScannerPublicationCommitScopeGuard::new(scope.clone());
2773 }
2774 assert_eq!(scope.state(), crate::object_api::ScannerPublicationCommitState::Indeterminate);
2775 assert!(!scope.release_movement_permit().await);
2776 }
2777
2778 #[tokio::test]
2779 async fn scanner_target_guard_keeps_movement_writer_fenced_after_lease_release() {
2780 let store = build_store_with_ctx(Arc::new(InstanceContext::new()));
2781 let (token, _) = store
2782 .acquire_scanner_publication_lease(0, crate::runtime::instance::SCANNER_PUBLICATION_LEASE_TTL)
2783 .await
2784 .expect("an idle store should grant a publication lease");
2785 let target_guard = store
2786 .acquire_scanner_publication_lease_guard(token)
2787 .await
2788 .expect("the target-side rename should acquire its short guard");
2789 assert!(store.release_scanner_publication_lease(token).await);
2790
2791 let movement_gate = store.ctx.data_movement_operation_gate();
2792 let movement_writer = tokio::spawn(async move { movement_gate.write_owned().await });
2793 tokio::task::yield_now().await;
2794 assert!(!movement_writer.is_finished(), "the target guard must span the rename operation");
2795 drop(target_guard);
2796 tokio::time::timeout(std::time::Duration::from_secs(1), movement_writer)
2797 .await
2798 .expect("movement writer should proceed after the target rename guard is dropped")
2799 .expect("movement writer task should not panic");
2800 }
2801
2802 #[tokio::test]
2803 async fn scanner_publication_lease_rejects_restart_aba_token() {
2804 let first_store = build_store_with_ctx(Arc::new(InstanceContext::new()));
2805 let (token, generation) = first_store
2806 .acquire_scanner_publication_lease(0, crate::runtime::instance::SCANNER_PUBLICATION_LEASE_TTL)
2807 .await
2808 .expect("the initial instance should grant a publication lease");
2809 first_store
2810 .validate_scanner_publication_lease(token, generation)
2811 .await
2812 .expect("the current instance should validate its own live token");
2813 first_store
2814 .acquire_scanner_publication_lease_guard(token)
2815 .await
2816 .expect("the current instance should admit the target-side rename");
2817
2818 let restarted_store = build_store_with_ctx(Arc::new(InstanceContext::new()));
2822 let error = restarted_store
2823 .validate_scanner_publication_lease(token, generation)
2824 .await
2825 .expect_err("a token from a prior instance must fail closed after restart");
2826 assert!(error.to_string().contains("unknown or expired"));
2827 let error = restarted_store
2828 .acquire_scanner_publication_lease_guard(token)
2829 .await
2830 .expect_err("a restarted instance must reject the target-side rename token");
2831 assert!(error.to_string().contains("unknown or expired"));
2832 }
2833
2834 #[tokio::test]
2835 async fn movement_generation_notifies_waiters_and_fails_closed_at_maximum() {
2836 let store = build_store_with_ctx(Arc::new(InstanceContext::new()));
2837 let notify = store.scanner_data_movement_changed();
2838 let notified = notify.notified();
2839 tokio::pin!(notified);
2840 notified.as_mut().enable();
2841
2842 assert_eq!(store.ctx.advance_data_movement_operation_epoch(), 1);
2843 tokio::time::timeout(std::time::Duration::from_secs(1), notified)
2844 .await
2845 .expect("movement transition should wake scanner waiters");
2846 assert_eq!(store.scanner_data_movement_generation(), 1);
2847
2848 store.ctx.set_data_movement_generation_for_test(u64::MAX - 1);
2849 assert_eq!(store.ctx.advance_data_movement_generation(), Some(u64::MAX));
2850 assert!(store.scanner_data_movement_generation_exhausted());
2851 assert_eq!(store.ctx.advance_data_movement_generation(), None);
2852 assert!(
2853 store.scanner_data_usage_publication_admission_guard().await.is_none(),
2854 "generation exhaustion must close publication admission"
2855 );
2856 }
2857
2858 #[tokio::test]
2859 async fn scanner_data_usage_publication_epoch_releases_gate_before_backend_io() {
2860 let store = build_store_with_ctx(Arc::new(InstanceContext::new()));
2861 let epoch = store
2862 .scanner_data_usage_publication_epoch()
2863 .await
2864 .expect("idle store should expose a publication epoch");
2865 assert_eq!(epoch, 0);
2866
2867 let operation_gate = store.ctx.data_movement_operation_gate();
2868 let _movement_guard = tokio::time::timeout(Duration::from_secs(1), operation_gate.write())
2869 .await
2870 .expect("epoch capture must not hold the movement gate across backend I/O");
2871 }
2872
2873 #[tokio::test]
2874 async fn scanner_data_usage_publication_admission_blocks_active_rebalance_snapshot() {
2875 let store = build_store_with_ctx(Arc::new(InstanceContext::new()));
2876 *store.rebalance_meta.write().await = Some(RebalanceMeta {
2877 pool_stats: vec![crate::services::rebalance::RebalanceStats {
2878 participating: true,
2879 info: crate::services::rebalance::RebalanceInfo {
2880 status: crate::services::rebalance::RebalStatus::Started,
2881 ..Default::default()
2882 },
2883 ..Default::default()
2884 }],
2885 ..Default::default()
2886 });
2887
2888 assert!(
2889 store.scanner_data_usage_publication_admission_guard().await.is_none(),
2890 "active rebalance must fail closed at the storage-owned admission boundary"
2891 );
2892 }
2893
2894 #[tokio::test]
2895 async fn scanner_publication_epoch_exhaustion_fails_closed_after_max() {
2896 let store = build_store_with_ctx(Arc::new(InstanceContext::new()));
2897 store.ctx.set_data_movement_operation_epoch_for_test(u64::MAX - 1);
2898
2899 assert_eq!(store.ctx.advance_data_movement_operation_epoch(), u64::MAX);
2900 assert!(store.ctx.data_movement_operation_epoch_exhausted());
2901 assert!(
2902 store.scanner_data_usage_publication_admission_guard().await.is_none(),
2903 "publication must fail closed at the reserved terminal epoch"
2904 );
2905
2906 assert_eq!(store.ctx.advance_data_movement_operation_epoch(), u64::MAX);
2907 assert!(store.ctx.data_movement_operation_epoch_exhausted());
2908 assert!(store.scanner_data_usage_publication_blocked().await);
2909 }
2910
2911 #[tokio::test]
2915 async fn instance_context_carrier_isolates_two_stores() {
2916 let ctx_a = Arc::new(InstanceContext::new());
2917 let ctx_b = Arc::new(InstanceContext::new());
2918 ctx_a.update_erasure_type(SetupType::DistErasure).await;
2919 ctx_b.update_erasure_type(SetupType::ErasureSD).await;
2920 ctx_a.set_endpoints(EndpointServerPools::from(vec![PoolEndpoints {
2921 legacy: false,
2922 set_count: 1,
2923 drives_per_set: 1,
2924 endpoints: Endpoints::default(),
2925 cmd_line: "instance-a".to_string(),
2926 platform: String::new(),
2927 }]));
2928 ctx_b.set_endpoints(EndpointServerPools::from(vec![PoolEndpoints {
2929 legacy: true,
2930 set_count: 2,
2931 drives_per_set: 2,
2932 endpoints: Endpoints::default(),
2933 cmd_line: "instance-b".to_string(),
2934 platform: String::new(),
2935 }]));
2936
2937 let store_a = build_store_with_ctx(ctx_a);
2938 let store_b = build_store_with_ctx(ctx_b);
2939
2940 assert!(store_a.setup_is_erasure().await);
2942 assert!(store_a.setup_is_dist_erasure().await);
2943 assert!(!store_a.setup_is_erasure_sd().await);
2944
2945 assert!(store_b.setup_is_erasure_sd().await);
2947 assert!(!store_b.setup_is_erasure().await);
2948 assert!(!store_b.setup_is_dist_erasure().await);
2949 let endpoints_a = store_a.instance_endpoints().expect("instance A endpoints");
2950 let endpoints_b = store_b.instance_endpoints().expect("instance B endpoints");
2951 assert_eq!(endpoints_a.as_ref()[0].set_count, 1);
2952 assert_eq!(endpoints_b.as_ref()[0].set_count, 2);
2953 assert!(!endpoints_a.as_ref()[0].legacy);
2954 assert!(endpoints_b.as_ref()[0].legacy);
2955 }
2956
2957 #[tokio::test]
2961 async fn store_adopts_bootstrap_context() {
2962 let store = build_store_with_ctx(crate::runtime::instance::bootstrap_ctx());
2963 assert!(
2964 Arc::ptr_eq(&store.ctx, &crate::runtime::instance::bootstrap_ctx()),
2965 "store built via adoption must share the bootstrap context Arc"
2966 );
2967 }
2968
2969 #[tokio::test]
2970 async fn test_has_space_for() {
2971 let disk_infos = vec![None, None]; let result = crate::layout::pool_space::has_space_for(&disk_infos, 1024).await;
2974 assert!(result.is_err());
2976 }
2977
2978 #[tokio::test]
2979 async fn test_find_local_disk() {
2980 let result = peer::find_local_disk("/nonexistent/path").await;
2981 assert!(result.is_none(), "Should return None for nonexistent path");
2982 }
2983
2984 #[tokio::test]
2985 #[serial]
2986 async fn test_find_local_disk_by_ref_backfills_uuid_map() {
2987 reset_local_disk_test_state().await;
2988
2989 let temp_dir = TempDir::new().expect("create temp dir for local disk ref test");
2990 let disk_paths = (0..4)
2991 .map(|idx| temp_dir.path().join(format!("disk{}", idx + 1)))
2992 .collect::<Vec<_>>();
2993 for disk_path in &disk_paths {
2994 std::fs::create_dir_all(disk_path).expect("create disk path");
2995 }
2996
2997 let mut endpoints = Vec::new();
2998 for (idx, disk_path) in disk_paths.iter().enumerate() {
2999 let mut endpoint = Endpoint::try_from(disk_path.to_str().expect("disk path to str")).expect("endpoint");
3000 endpoint.set_pool_index(0);
3001 endpoint.set_set_index(0);
3002 endpoint.set_disk_index(idx);
3003 endpoints.push(endpoint);
3004 }
3005
3006 let endpoint_pools = EndpointServerPools(vec![PoolEndpoints {
3007 legacy: false,
3008 set_count: 1,
3009 drives_per_set: 4,
3010 endpoints: Endpoints::from(endpoints),
3011 cmd_line: "find-local-disk-by-ref-test".to_string(),
3012 platform: "test".to_string(),
3013 }]);
3014
3015 init_local_disks(endpoint_pools.clone()).await.expect("init local disks");
3016
3017 let (mut disks, errs) = init_disks(
3018 &endpoint_pools.as_ref().first().expect("pool endpoints").endpoints,
3019 &DiskOption {
3020 cleanup: true,
3021 health_check: false,
3022 },
3023 )
3024 .await;
3025
3026 assert!(errs.iter().all(|err| err.is_none()), "disk init should succeed: {errs:?}");
3027 connect_load_init_formats(true, &mut disks, 1, 4, None)
3028 .await
3029 .expect("initialize format metadata");
3030
3031 clear_local_disk_id_map_for_test().await;
3032
3033 let local_disks = all_local_disk().await;
3034 let first_disk = local_disks.first().expect("local disk exists");
3035 let disk_id = first_disk
3036 .get_disk_id()
3037 .await
3038 .expect("get disk id should succeed")
3039 .expect("disk id should exist");
3040
3041 let found = find_local_disk_by_ref(&disk_id.to_string()).await;
3042 assert!(found.is_some(), "disk lookup by id should backfill cache");
3043 assert_eq!(local_disk_path_by_id(&disk_id).await, Some(first_disk.endpoint().to_string()));
3044
3045 reset_local_disk_test_state().await;
3046 }
3047
3048 #[tokio::test]
3049 async fn test_all_local_disk_path() {
3050 let paths = all_local_disk_path().await;
3051 assert!(paths.is_empty() || !paths.is_empty());
3053 }
3054
3055 #[tokio::test]
3056 async fn test_all_local_disk() {
3057 let disks = all_local_disk().await;
3058 assert!(disks.is_empty() || !disks.is_empty());
3060 }
3061
3062 #[test]
3063 fn test_should_not_enqueue_transition_for_internal_metadata_bucket() {
3064 let oi = ObjectInfo {
3065 bucket: RUSTFS_META_BUCKET.to_string(),
3066 name: format!("{BUCKET_META_PREFIX}/bucket/.metadata.bin"),
3067 ..Default::default()
3068 };
3069
3070 assert!(!should_enqueue_transition_immediately(&oi));
3071 }
3072}