1use super::*;
16use crate::bucket::{
17 metadata::{BUCKET_TABLE_RESERVED_PREFIX, table_bucket_catalog_metadata_prefix},
18 utils::is_meta_bucketname,
19};
20use crate::error::is_err_bucket_not_found;
21use crate::runtime::sources as runtime_sources;
22use crate::set_disk::{BucketInfoQuorum, get_lock_acquire_timeout};
23use crate::storage_api_contracts::bucket::{BUCKET_LIFECYCLE_LOCK_OBJECT, SRBucketDeleteOp};
24use crate::storage_api_contracts::namespace::NamespaceLocking as _;
25use futures::stream::{self, StreamExt};
26use rustfs_policy::policy::BucketPolicy;
27use std::collections::BTreeMap;
28use std::future::Future;
29
30const DELETED_BUCKETS_PREFIX: &str = ".deleted";
31const SCANNER_BUCKET_LIST_SET_CONCURRENCY: usize = 4;
32const EVENT_BUCKET_DELETE_BLOCKED: &str = "bucket_delete_blocked";
33const EVENT_BUCKET_DELETE_ROLLBACK_FAILED: &str = "bucket_delete_rollback_failed";
34
35fn record_bucket_delete_blocker(bucket: &str, kind: BucketDeleteBlockerKind, residue: &BucketMetadataLessResidue) {
53 metrics::counter!("rustfs_bucket_delete_blockers_total", "kind" => kind.as_str()).increment(1);
54 if kind.is_client_visible() {
55 warn!(
56 event = EVENT_BUCKET_DELETE_BLOCKED,
57 component = "ecstore",
58 subsystem = "bucket",
59 bucket,
60 blocker = kind.as_str(),
61 files = residue.files,
62 uuid_data_dirs = residue.uuid_data_dirs,
63 entries_scanned = residue.entries_scanned,
64 diagnostic_bytes_read = residue.diagnostic_bytes_read,
65 diagnostic_truncated = residue.diagnostic_truncated,
66 sample = residue.sample.as_deref().unwrap_or("<none>"),
67 "Bucket deletion was blocked by durable local state"
68 );
69 return;
70 }
71
72 error!(
73 event = EVENT_BUCKET_DELETE_BLOCKED,
74 component = "ecstore",
75 subsystem = "bucket",
76 bucket,
77 blocker = kind.as_str(),
78 files = residue.files,
79 uuid_data_dirs = residue.uuid_data_dirs,
80 entries_scanned = residue.entries_scanned,
81 diagnostic_bytes_read = residue.diagnostic_bytes_read,
82 diagnostic_truncated = residue.diagnostic_truncated,
83 sample = residue.sample.as_deref().unwrap_or("<none>"),
84 "Bucket deletion was blocked by residue the client cannot reach through the S3 API"
85 );
86}
87
88fn bucket_list_set_concurrency(set_count: usize, max_concurrency: usize) -> usize {
89 set_count.clamp(1, max_concurrency.max(1))
90}
91
92fn should_override_created_from_metadata(created: OffsetDateTime) -> bool {
93 created != OffsetDateTime::UNIX_EPOCH
94}
95
96fn validate_table_bucket_delete_allowed(
97 bucket: &str,
98 table_bucket_enabled: bool,
99 table_catalog_metadata_exists: bool,
100) -> Result<()> {
101 if table_bucket_enabled && table_catalog_metadata_exists {
102 return Err(StorageError::BucketNotEmpty(bucket.to_string()));
103 }
104
105 Ok(())
106}
107
108async fn table_catalog_metadata_exists(ctx: &crate::runtime::instance::InstanceContext, bucket: &str) -> Result<bool> {
109 let local_disks = runtime_sources::local_disks_in(ctx).await;
110 for disk in local_disks.iter() {
111 let Some(bucket_path) = disk.get_bucket_path_for_io_if_local(bucket) else {
112 continue;
113 };
114 let catalog_path = bucket_path?.join(BUCKET_TABLE_RESERVED_PREFIX);
115 if has_xlmeta_files(&catalog_path).await? {
116 return Ok(true);
117 }
118 }
119
120 Ok(false)
121}
122
123async fn validate_table_bucket_delete_guard(ctx: &crate::runtime::instance::InstanceContext, bucket: &str) -> Result<()> {
124 let table_bucket_enabled = metadata_sys::get_in(ctx, bucket)
125 .await
126 .is_ok_and(|metadata| metadata.table_bucket_enabled());
127 if table_bucket_enabled {
128 validate_table_bucket_delete_allowed(bucket, true, table_catalog_metadata_exists(ctx, bucket).await?)?;
129 }
130
131 Ok(())
132}
133
134fn bucket_delete_metadata_cleanup_prefixes(bucket: &str) -> [String; 2] {
135 [
136 table_bucket_catalog_metadata_prefix(bucket),
137 format!("{BUCKET_META_PREFIX}/{bucket}"),
138 ]
139}
140
141fn bucket_deleted_marker_prefix(bucket: &str) -> String {
142 format!("{BUCKET_META_PREFIX}/{DELETED_BUCKETS_PREFIX}/{bucket}")
143}
144
145fn bucket_deleted_marker_volume(bucket: &str) -> String {
146 format!("{RUSTFS_META_BUCKET}/{}", bucket_deleted_marker_prefix(bucket))
147}
148
149pub(crate) async fn await_bucket_namespace_operation<T, F>(
150 guard: Option<&rustfs_lock::NamespaceLockGuard>,
151 bucket: &str,
152 operation: &'static str,
153 future: F,
154) -> Result<T>
155where
156 F: Future<Output = Result<T>>,
157{
158 let Some(guard) = guard else {
159 return future.await;
160 };
161 if guard.is_lock_lost() {
162 return Err(StorageError::other(format!(
163 "bucket namespace lock was lost before {operation}: {bucket}"
164 )));
165 }
166 tokio::select! {
167 biased;
168 _ = guard.lock_lost_notified() => Err(StorageError::other(format!(
169 "bucket namespace lock was lost during {operation}: {bucket}"
170 ))),
171 result = future => result,
172 }
173}
174
175async fn await_bucket_lifecycle_operation<T, F>(
176 lifecycle_guard: Option<&rustfs_lock::NamespaceLockGuard>,
177 namespace_guard: Option<&rustfs_lock::NamespaceLockGuard>,
178 bucket: &str,
179 operation: &'static str,
180 future: F,
181) -> Result<T>
182where
183 F: Future<Output = Result<T>>,
184{
185 await_bucket_namespace_operation(
186 lifecycle_guard,
187 bucket,
188 operation,
189 await_bucket_namespace_operation(namespace_guard, bucket, operation, future),
190 )
191 .await
192}
193
194async fn run_bucket_usage_cleanup<F>(guard: Option<&rustfs_lock::NamespaceLockGuard>, bucket: &str, future: F) -> Result<()>
195where
196 F: Future<Output = Result<()>>,
197{
198 await_bucket_namespace_operation(guard, bucket, "bucket usage cleanup", future).await
199}
200
201async fn run_physical_bucket_deletion<F>(guard: Option<&rustfs_lock::NamespaceLockGuard>, bucket: &str, future: F) -> Result<()>
202where
203 F: Future<Output = Result<()>>,
204{
205 list_objects::observe_scanner_namespace_mutations(bucket, 1);
208 await_bucket_namespace_operation(guard, bucket, "physical bucket deletion", future).await
209}
210
211async fn bucket_delete_local_blocker(
212 ctx: &crate::runtime::instance::InstanceContext,
213 bucket: &str,
214 budget: &mut BucketDeleteDiagnosticBudget,
215) -> Result<Option<StorageError>> {
216 let local_disks = runtime_sources::local_disks_in(ctx).await;
217 let mut residue = BucketMetadataLessResidue::default();
218
219 for disk in local_disks.iter() {
220 let Some(bucket_path) = disk.get_bucket_path_for_io_if_local(bucket) else {
221 continue;
222 };
223 let scan = scan_metadata_less_residue_with_budget(&bucket_path?, budget).await?;
224 if scan.xlmeta_found {
225 record_bucket_delete_blocker(bucket, scan.xlmeta_blocker.unwrap_or(BucketDeleteBlockerKind::UnknownXlMeta), &scan);
226 return Ok(Some(StorageError::BucketNotEmpty(bucket.to_string())));
227 }
228 residue.files = residue.files.saturating_add(scan.files);
229 residue.uuid_data_dirs = residue.uuid_data_dirs.saturating_add(scan.uuid_data_dirs);
230 residue.entries_scanned = residue.entries_scanned.saturating_add(scan.entries_scanned);
231 residue.diagnostic_bytes_read = residue.diagnostic_bytes_read.saturating_add(scan.diagnostic_bytes_read);
232 if residue.sample.is_none() {
233 residue.sample = scan.sample.clone();
234 }
235 if scan.diagnostic_truncated {
236 residue.diagnostic_truncated = true;
237 if residue.files == 0 {
238 return Ok(None);
241 }
242 record_bucket_delete_blocker(bucket, BucketDeleteBlockerKind::DiagnosticBudgetExceeded, &residue);
243 return Ok(Some(StorageError::BucketNotEmptyWithDetails {
244 bucket: bucket.to_string(),
245 details: residue.describe(),
246 }));
247 }
248 }
249
250 if residue.has_residue_without_xlmeta() {
251 record_bucket_delete_blocker(bucket, BucketDeleteBlockerKind::OrphanDirectory, &residue);
252 return Ok(Some(StorageError::BucketNotEmptyWithDetails {
253 bucket: bucket.to_string(),
254 details: residue.describe(),
255 }));
256 }
257
258 Ok(None)
259}
260
261impl ECStore {
262 fn bucket_sets(&self) -> impl Iterator<Item = (usize, usize, Arc<crate::set_disk::SetDisks>)> + '_ {
263 self.pools.iter().flat_map(|pool| {
264 pool.disk_set
265 .iter()
266 .map(|set| (set.pool_index, set.set_index, Arc::clone(set)))
267 })
268 }
269
270 pub async fn get_bucket_metadata(&self, bucket: &str) -> Result<Arc<BucketMetadata>> {
271 let sys = metadata_sys::require_bucket_metadata_sys_in(&self.ctx)?;
272 sys.read().await.get(bucket).await
273 }
274
275 pub async fn get_bucket_policy(&self, bucket: &str) -> Result<(BucketPolicy, OffsetDateTime)> {
276 let sys = metadata_sys::require_bucket_metadata_sys_in(&self.ctx)?;
277 sys.read().await.get_bucket_policy(bucket).await
278 }
279
280 pub async fn get_bucket_policy_raw(&self, bucket: &str) -> Result<(String, OffsetDateTime)> {
281 let sys = metadata_sys::require_bucket_metadata_sys_in(&self.ctx)?;
282 sys.read().await.get_bucket_policy_raw(bucket).await
283 }
284
285 pub async fn restricts_public_bucket_access(&self, bucket: &str) -> Result<bool> {
286 let sys = metadata_sys::require_bucket_metadata_sys_in(&self.ctx)?;
287 let (config, _) = sys.read().await.get_public_access_block_config(bucket).await?;
288 Ok(config.restrict_public_buckets.unwrap_or(false))
289 }
290
291 pub async fn update_bucket_metadata_config(&self, bucket: &str, config_file: &str, data: Vec<u8>) -> Result<OffsetDateTime> {
292 metadata_sys::update_in(&self.ctx, bucket, config_file, data).await
293 }
294
295 pub async fn bucket_incarnation_id(&self, bucket: &str) -> Result<Uuid> {
296 metadata_sys::get_cached_bucket_incarnation_id_in(&self.ctx, bucket).await
297 }
298
299 pub async fn bucket_incarnation_id_from_disk(&self, bucket: &str) -> Result<Uuid> {
300 metadata_sys::get_bucket_incarnation_id_in(&self.ctx, bucket).await
301 }
302
303 pub(crate) async fn acquire_bucket_lifecycle_read_lock(&self, bucket: &str) -> Result<rustfs_lock::NamespaceLockGuard> {
306 let lock = self.new_ns_lock(bucket, BUCKET_LIFECYCLE_LOCK_OBJECT).await?;
307 lock.get_read_lock(get_lock_acquire_timeout()).await.map_err(|err| match err {
308 rustfs_lock::error::LockError::QuorumNotReached { required, achieved } => {
309 StorageError::NamespaceLockQuorumUnavailable {
310 mode: "bucket_lifecycle_read",
311 bucket: bucket.to_string(),
312 object: BUCKET_LIFECYCLE_LOCK_OBJECT.to_string(),
313 required,
314 achieved,
315 }
316 }
317 other => StorageError::Lock(other),
318 })
319 }
320
321 pub async fn acquire_bucket_incarnation_fence(
333 &self,
334 bucket: &str,
335 expected: uuid::Uuid,
336 ) -> Result<super::BucketIncarnationFenceGuard> {
337 let inner = self.acquire_bucket_lifecycle_read_lock(bucket).await?;
338 let pieces = super::bucket_fence::FencePieces {
339 registry: self.bucket_fence_registry.clone(),
340 inner,
341 };
342 let registration = pieces.enter(bucket);
343 let current = match registration.memoized {
344 Some(current) => current,
345 None => match metadata_sys::get_bucket_incarnation_id_in(&self.ctx, bucket).await {
346 Ok(current) => {
347 if !pieces.lock_lost() {
350 pieces.memoize(bucket, current);
351 }
352 current
353 }
354 Err(err) => {
355 pieces.abandon(bucket, registration.token);
356 return Err(err);
357 }
358 },
359 };
360 if current != expected {
361 pieces.abandon(bucket, registration.token);
362 return Err(StorageError::BucketNotFound(bucket.to_string()));
363 }
364 Ok(pieces.into_guard(bucket, registration.token))
365 }
366
367 pub async fn lock_bucket_for_recursive_delete(&self, bucket: &str) -> Result<rustfs_lock::NamespaceLockGuard> {
370 if self.ctx.lock_manager().is_disabled() {
371 return Err(StorageError::InvalidArgument(
372 bucket.to_owned(),
373 String::new(),
374 "Recursive deletion requires namespace locking".to_owned(),
375 ));
376 }
377 self.acquire_bucket_lifecycle_write_lock(bucket).await
378 }
379
380 pub(crate) async fn acquire_bucket_lifecycle_write_lock(&self, bucket: &str) -> Result<rustfs_lock::NamespaceLockGuard> {
381 let lock = self.new_ns_lock(bucket, BUCKET_LIFECYCLE_LOCK_OBJECT).await?;
382 lock.get_write_lock(get_lock_acquire_timeout())
383 .await
384 .map_err(|err| match err {
385 rustfs_lock::error::LockError::QuorumNotReached { required, achieved } => {
386 StorageError::NamespaceLockQuorumUnavailable {
387 mode: "bucket_lifecycle_write",
388 bucket: bucket.to_string(),
389 object: BUCKET_LIFECYCLE_LOCK_OBJECT.to_string(),
390 required,
391 achieved,
392 }
393 }
394 other => StorageError::Lock(other),
395 })
396 }
397
398 async fn mark_bucket_deleted(&self, bucket: &str) -> Result<()> {
399 let marker_volume = bucket_deleted_marker_volume(bucket);
400
401 self.make_bucket_on_sets(
402 marker_volume.as_str(),
403 &MakeBucketOptions {
404 force_create: true,
405 ..Default::default()
406 },
407 )
408 .await
409 .map_err(|err| to_object_err(err, vec![bucket]))?;
410
411 Ok(())
412 }
413
414 async fn make_bucket_on_sets(&self, bucket: &str, opts: &MakeBucketOptions) -> Result<()> {
415 let results = futures::future::join_all(
416 self.bucket_sets()
417 .map(|(_, _, set)| async move { set.make_bucket(bucket, opts).await }),
418 )
419 .await;
420 if results.is_empty() {
421 return Err(StorageError::ErasureWriteQuorum);
422 }
423
424 let mut bucket_exists_error = None;
425 let mut first_hard_error = None;
426 for result in results {
427 let Err(err) = result else {
428 continue;
429 };
430 if matches!(err, StorageError::VolumeExists) || is_err_bucket_exists(&err) {
431 if bucket_exists_error.is_none() {
432 bucket_exists_error = Some(err);
433 }
434 } else if first_hard_error.is_none() {
435 first_hard_error = Some(err);
436 }
437 }
438
439 match first_hard_error.or(bucket_exists_error) {
440 Some(err) => Err(err),
441 None => Ok(()),
442 }
443 }
444
445 async fn delete_bucket_on_sets(&self, bucket: &str, opts: &DeleteBucketOptions) -> Result<()> {
446 let results = futures::future::join_all(
447 self.bucket_sets()
448 .map(|(_, _, set)| async move { set.delete_bucket(bucket, opts).await }),
449 )
450 .await;
451 if results.is_empty() {
452 return Err(StorageError::ErasureWriteQuorum);
453 }
454
455 let mut deleted = false;
456 let mut first_error = None;
457 for result in results {
458 match result {
459 Ok(()) => deleted = true,
460 Err(err) if is_err_strict_volume_not_found(&err) => {}
461 Err(err) if first_error.is_none() => first_error = Some(err),
462 Err(_) => {}
463 }
464 }
465
466 if let Some(delete_error) = first_error {
467 if !opts.no_recreate {
468 let rollback_opts = MakeBucketOptions {
469 force_create: true,
470 no_lock: true,
471 ..Default::default()
472 };
473 if let Err(rollback_error) = self.make_bucket_on_sets(bucket, &rollback_opts).await {
474 warn!(
475 event = EVENT_BUCKET_DELETE_ROLLBACK_FAILED,
476 component = "ecstore",
477 subsystem = "bucket",
478 bucket,
479 deletion_error = ?delete_error,
480 rollback_error = ?rollback_error,
481 "Bucket deletion rollback could not restore every bucket volume"
482 );
483 }
484 }
485 return Err(delete_error);
486 }
487
488 if deleted { Ok(()) } else { Err(StorageError::VolumeNotFound) }
489 }
490
491 async fn cleanup_deleted_bucket_metadata(
492 &self,
493 bucket: &str,
494 include_deleted_marker: bool,
495 guard: Option<&rustfs_lock::NamespaceLockGuard>,
496 ) -> Result<()> {
497 for prefix in bucket_delete_metadata_cleanup_prefixes(bucket) {
498 await_bucket_namespace_operation(
499 guard,
500 bucket,
501 "deleted bucket metadata cleanup",
502 self.delete_all(RUSTFS_META_BUCKET, prefix.as_str()),
503 )
504 .await?;
505 }
506
507 if include_deleted_marker {
508 let marker_prefix = bucket_deleted_marker_prefix(bucket);
509 await_bucket_namespace_operation(
510 guard,
511 bucket,
512 "deleted bucket marker cleanup",
513 self.delete_all(RUSTFS_META_BUCKET, marker_prefix.as_str()),
514 )
515 .await?;
516 }
517
518 await_bucket_namespace_operation(
519 guard,
520 bucket,
521 "deleted bucket metadata cache cleanup",
522 metadata_sys::remove_bucket_metadata_in(&self.ctx, bucket),
523 )
524 .await?;
525 runtime_sources::delete_bucket_monitor_entry(bucket);
526 Ok(())
527 }
528
529 async fn cleanup_bucket_usage(&self, bucket: &str, guard: Option<&rustfs_lock::NamespaceLockGuard>) -> Result<()> {
530 run_bucket_usage_cleanup(guard, bucket, async {
531 crate::data_usage::prepare_bucket_usage_for_namespace_change(bucket, guard).await?;
532 crate::data_usage::remove_bucket_usage_from_backend_with_guard_fenced(self, bucket, guard).await
533 })
534 .await
535 }
536
537 async fn cleanup_bucket_usage_best_effort(&self, bucket: &str, guard: Option<&rustfs_lock::NamespaceLockGuard>) {
538 if let Err(err) = self.cleanup_bucket_usage(bucket, guard).await {
539 warn!(
540 bucket = %bucket,
541 error = ?err,
542 "bucket data usage cleanup deferred to scanner reconciliation"
543 );
544 }
545 }
546
547 async fn rollback_failed_bucket_creation(
548 &self,
549 bucket: &str,
550 lifecycle_guard: Option<&rustfs_lock::NamespaceLockGuard>,
551 namespace_guard: Option<&rustfs_lock::NamespaceLockGuard>,
552 ) {
553 let rollback_opts = DeleteBucketOptions {
554 no_lock: true,
555 no_recreate: true,
556 ..Default::default()
557 };
558 if let Err(err) =
559 await_bucket_lifecycle_operation(lifecycle_guard, namespace_guard, bucket, "failed bucket creation rollback", async {
560 self.delete_bucket_on_sets(bucket, &rollback_opts)
561 .await
562 .map_err(|rollback_err| to_object_err(rollback_err, vec![bucket]))
563 })
564 .await
565 {
566 warn!(
567 bucket = %bucket,
568 error = ?err,
569 "failed bucket creation rollback did not remove every physical bucket volume"
570 );
571 return;
572 }
573 if let Err(err) = self.cleanup_deleted_bucket_metadata(bucket, false, namespace_guard).await {
574 warn!(
575 bucket = %bucket,
576 error = ?err,
577 "failed bucket creation rollback left internal bucket metadata"
578 );
579 }
580 }
581
582 #[instrument(skip(self))]
583 pub(super) async fn handle_make_bucket(&self, bucket: &str, opts: &MakeBucketOptions) -> Result<()> {
584 if !is_meta_bucketname(bucket)
585 && let Err(err) = check_valid_bucket_name_strict(bucket)
586 {
587 return Err(StorageError::BucketNameInvalid(err.to_string()));
588 }
589
590 let bucket_lifecycle_guard = if !opts.no_lock {
592 Some(self.acquire_bucket_lifecycle_write_lock(bucket).await?)
593 } else {
594 None
595 };
596 let metadata_transaction_guard = if !opts.no_lock && !is_meta_bucketname(bucket) {
597 Some(metadata_sys::acquire_bucket_metadata_transaction_lock_in(&self.ctx, bucket).await?)
598 } else {
599 None
600 };
601 let ns_guard = if !opts.no_lock {
602 let ns_lock = self.new_ns_lock(bucket, bucket).await?;
603 Some(
604 await_bucket_namespace_operation(
605 bucket_lifecycle_guard.as_ref(),
606 bucket,
607 "bucket namespace lock acquisition",
608 async {
609 ns_lock.get_write_lock(get_lock_acquire_timeout()).await.map_err(|e| match e {
610 rustfs_lock::error::LockError::QuorumNotReached { required, achieved } => {
611 StorageError::NamespaceLockQuorumUnavailable {
612 mode: "write",
613 bucket: bucket.to_string(),
614 object: bucket.to_string(),
615 required,
616 achieved,
617 }
618 }
619 other => StorageError::Lock(other),
620 })
621 },
622 )
623 .await?,
624 )
625 } else {
626 None
627 };
628
629 let existing_bucket_info = match self.get_bucket_info_from_sets(bucket, &BucketOptions::default()).await {
630 Ok(info) => Some(info),
631 Err(err) => {
632 if is_err_bucket_not_found(&err) {
633 None
634 } else {
635 return Err(to_object_err(err, vec![bucket]));
636 }
637 }
638 };
639 let confirmed_missing = existing_bucket_info.is_none();
640 let existing_metadata = if opts.force_create && !confirmed_missing && !is_meta_bucketname(bucket) {
641 let (mut metadata, persisted) = metadata_sys::get_config_from_disk_with_presence_in(&self.ctx, bucket).await?;
642 if !persisted {
643 metadata = BucketMetadata::new(bucket);
644 metadata.created = existing_bucket_info
645 .as_ref()
646 .and_then(|info| info.created)
647 .unwrap_or(OffsetDateTime::UNIX_EPOCH);
648 let store = metadata_sys::object_store_in(&self.ctx).await?;
654 if let Some(stored) = crate::bucket::metadata::load_bucket_incarnation(store.clone(), bucket).await?
655 && !crate::bucket::retirement::is_retired(store, bucket, stored).await?
656 {
657 metadata.bucket_incarnation_id = stored;
658 }
659 } else if !metadata.bucket_incarnation_sidecar && !metadata.bucket_incarnation_id.is_nil() {
660 return Err(Error::other(format!(
661 "bucket incarnation sidecar is missing for new-format metadata: {bucket}"
662 )));
663 } else if !metadata.bucket_incarnation_sidecar {
664 metadata.bucket_incarnation_id = Uuid::new_v4();
665 }
666 Some(metadata)
667 } else {
668 None
669 };
670
671 let mut meta = existing_metadata.unwrap_or_else(|| {
672 if confirmed_missing && !is_meta_bucketname(bucket) {
673 BucketMetadata::new_with_default_durability(bucket)
674 } else {
675 BucketMetadata::new(bucket)
676 }
677 });
678 let existing_incarnation_is_authoritative = meta.bucket_incarnation_sidecar;
679 if confirmed_missing || is_meta_bucketname(bucket) {
680 meta.set_created(opts.created_at);
681
682 if opts.versioning_enabled {
683 meta.versioning_config_xml =
684 crate::bucket::utils::serialize::<VersioningConfiguration>(&ENABLED_VERSIONING_CONFIG)?;
685 }
686 }
687
688 let lock_newly_enabled = opts.lock_enabled && !meta.lock_enabled;
695 if opts.lock_enabled {
696 meta.lock_enabled = true;
697 meta.object_lock_config_xml = crate::bucket::utils::serialize(&*crate::store::ENABLED_OBJECT_LOCK_CONFIG)?;
698 meta.versioning_config_xml = crate::bucket::utils::serialize::<VersioningConfiguration>(&ENABLED_VERSIONING_CONFIG)?;
699 }
700
701 let metadata_persisted_before_physical = confirmed_missing && !is_meta_bucketname(bucket) && opts.lock_enabled;
702 if metadata_persisted_before_physical {
703 metadata_sys::set_new_bucket_metadata_intent_in(&self.ctx, meta.clone()).await?;
704 if bucket_lifecycle_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
705 || metadata_transaction_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
706 || ns_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
707 {
708 return Err(Error::other(format!("bucket metadata creation intent lock was lost: {bucket}")));
709 }
710 }
711
712 if confirmed_missing && !is_meta_bucketname(bucket) {
713 crate::store::list_objects::observe_scanner_namespace_mutations(bucket, 1);
717 self.cleanup_bucket_usage(bucket, ns_guard.as_ref()).await?;
718 }
719
720 if let Err(err) = await_bucket_lifecycle_operation(
721 bucket_lifecycle_guard.as_ref(),
722 ns_guard.as_ref(),
723 bucket,
724 "physical bucket creation",
725 await_bucket_namespace_operation(
726 metadata_transaction_guard.as_ref(),
727 bucket,
728 "bucket creation metadata transaction",
729 async {
730 self.make_bucket_on_sets(bucket, opts)
731 .await
732 .map_err(|err| to_object_err(err, vec![bucket]))
733 },
734 ),
735 )
736 .await
737 {
738 if is_err_bucket_exists(&err)
739 && let Err(heal_err) = self
740 .handle_heal_bucket(
741 bucket,
742 &HealOpts {
743 recreate: true,
744 ..Default::default()
745 },
746 )
747 .await
748 {
749 warn!("best-effort bucket heal after BucketExists failed: {heal_err}");
750 }
751 if confirmed_missing && !is_err_bucket_exists(&err) && ns_guard.as_ref().is_none_or(|guard| !guard.is_lock_lost()) {
752 error!("make bucket failed: {err}");
753 self.rollback_failed_bucket_creation(bucket, bucket_lifecycle_guard.as_ref(), ns_guard.as_ref())
754 .await;
755 }
756 return Err(err);
757 };
758
759 let metadata_result = async {
760 if metadata_persisted_before_physical {
761 return metadata_sys::commit_bucket_metadata_in(&self.ctx, meta).await;
762 }
763 if is_meta_bucketname(bucket) {
764 metadata_sys::set_bucket_metadata_in(&self.ctx, meta).await
765 } else if existing_incarnation_is_authoritative && !lock_newly_enabled {
766 metadata_sys::commit_bucket_metadata_in(&self.ctx, meta).await
767 } else {
768 metadata_sys::set_new_bucket_metadata_in(&self.ctx, meta).await
769 }
770 }
771 .await;
772 let metadata_lock_lost = bucket_lifecycle_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
773 || metadata_transaction_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
774 || ns_guard.as_ref().is_some_and(|guard| guard.is_lock_lost());
775 if let Err(err) = metadata_result {
776 if !metadata_lock_lost {
777 self.rollback_failed_bucket_creation(bucket, bucket_lifecycle_guard.as_ref(), ns_guard.as_ref())
778 .await;
779 }
780 return Err(err);
781 }
782 if metadata_lock_lost {
783 return Err(Error::other(format!("bucket metadata initialization lock was lost: {bucket}")));
784 }
785
786 if confirmed_missing && !is_meta_bucketname(bucket) {
787 crate::store::list_objects::observe_scanner_namespace_mutations(bucket, 1);
791 }
792
793 Ok(())
794 }
795
796 pub(crate) async fn bucket_exists_for_heal(&self, bucket: &str) -> Result<bool> {
801 let results = futures::future::join_all(
802 self.bucket_sets()
803 .map(|(_, _, set)| async move { set.get_bucket_info(bucket, &BucketOptions::default()).await }),
804 )
805 .await;
806 let mut first_error = None;
807 for result in results {
808 match result {
809 Ok(_) => return Ok(true),
810 Err(err) if is_err_strict_volume_not_found(&err) => {}
811 Err(err) if first_error.is_none() => first_error = Some(err),
812 Err(_) => {}
813 }
814 }
815 match first_error {
816 Some(err) => Err(err),
817 None => Ok(false),
818 }
819 }
820
821 #[instrument(skip(self))]
822 pub(crate) async fn get_bucket_info_from_sets(&self, bucket: &str, opts: &BucketOptions) -> Result<BucketInfo> {
823 self.get_bucket_info_from_sets_with_quorum(bucket, opts, BucketInfoQuorum::Write)
824 .await
825 }
826
827 pub(crate) async fn get_bucket_info_from_sets_at_read_quorum(
828 &self,
829 bucket: &str,
830 opts: &BucketOptions,
831 ) -> Result<BucketInfo> {
832 self.get_bucket_info_from_sets_with_quorum(bucket, opts, BucketInfoQuorum::Read)
833 .await
834 }
835
836 async fn get_bucket_info_from_sets_with_quorum(
837 &self,
838 bucket: &str,
839 opts: &BucketOptions,
840 quorum: BucketInfoQuorum,
841 ) -> Result<BucketInfo> {
842 let mut scoped_results = futures::future::join_all(self.bucket_sets().map(|(pool_index, set_index, set)| async move {
849 let result = match quorum {
850 BucketInfoQuorum::Read => set.stat_bucket_with_quorum(bucket, quorum).await,
851 BucketInfoQuorum::Write => set.get_bucket_info(bucket, opts).await,
852 };
853 (pool_index, set_index, result)
854 }))
855 .await;
856 scoped_results.sort_unstable_by_key(|(pool_index, set_index, _)| (*pool_index, *set_index));
857
858 let mut first_info = None;
859 let mut first_error = None;
860 for (_, _, result) in scoped_results {
861 match result {
862 Ok(info) if first_info.is_none() => first_info = Some(info),
863 Ok(_) => {}
864 Err(err) if is_err_strict_volume_not_found(&err) => {}
865 Err(err) if first_error.is_none() => first_error = Some(err),
866 Err(_) => {}
867 }
868 }
869
870 if let Some(err) = first_error {
871 return Err(err);
872 }
873
874 first_info.ok_or(Error::VolumeNotFound)
875 }
876
877 #[instrument(skip(self))]
878 pub(super) async fn handle_get_bucket_info(&self, bucket: &str, opts: &BucketOptions) -> Result<BucketInfo> {
879 let mut info = match self.get_bucket_info_from_sets(bucket, opts).await {
880 Ok(info) => info,
881 Err(Error::ErasureWriteQuorum) => return self.get_bucket_info_at_read_quorum(bucket, opts).await,
882 Err(err) => return Err(err),
883 };
884
885 if let Ok(sys) = metadata_sys::get_in(&self.ctx, bucket).await {
886 if should_override_created_from_metadata(sys.created) {
887 info.created = Some(sys.created);
888 }
889 info.versioning = sys.versioning();
890 info.object_locking = sys.object_locking();
891 }
892
893 Ok(info)
894 }
895
896 async fn get_bucket_info_at_read_quorum(&self, bucket: &str, opts: &BucketOptions) -> Result<BucketInfo> {
897 let guard = self.acquire_bucket_lifecycle_read_lock(bucket).await?;
901 await_bucket_namespace_operation(Some(&guard), bucket, "bucket read quorum validation", async {
902 let mut info = self
903 .get_bucket_info_from_sets_with_quorum(bucket, opts, BucketInfoQuorum::Read)
904 .await?;
905 let (metadata, persisted) = metadata_sys::get_config_from_disk_with_presence_in(&self.ctx, bucket).await?;
906 if !persisted {
907 return Err(Error::ErasureReadQuorum);
911 }
912 if metadata.lock_enabled && !metadata.bucket_creation_committed {
913 return Err(Error::ErasureWriteQuorum);
914 }
915 if metadata.name != bucket {
916 return Err(Error::FileCorrupt);
917 }
918 if should_override_created_from_metadata(metadata.created) {
919 info.created = Some(metadata.created);
920 }
921 info.versioning = metadata.versioning();
922 info.object_locking = metadata.object_locking();
923 Ok(info)
924 })
925 .await
926 }
927
928 #[instrument(skip(self))]
929 pub(super) async fn handle_list_bucket(&self, opts: &BucketOptions) -> Result<Vec<BucketInfo>> {
930 Ok(self.list_bucket_from_sets(opts, usize::MAX).await?.buckets)
932 }
933
934 pub async fn list_bucket_for_scanner(&self, opts: &BucketOptions) -> Result<crate::cluster::rpc::ScannerBucketListing> {
935 self.list_bucket_from_sets(opts, SCANNER_BUCKET_LIST_SET_CONCURRENCY).await
936 }
937
938 async fn list_bucket_from_sets(
939 &self,
940 opts: &BucketOptions,
941 max_concurrency: usize,
942 ) -> Result<crate::cluster::rpc::ScannerBucketListing> {
943 let set_count = self.pools.iter().map(|pool| pool.disk_set.len()).sum();
944 let concurrency = bucket_list_set_concurrency(set_count, max_concurrency);
945 let deleted = opts.deleted;
946 let cached = opts.cached;
947 let no_metadata = opts.no_metadata;
948 let mut set_listings = stream::iter(self.bucket_sets().map(move |(pool_index, set_index, set)| {
949 let opts = BucketOptions {
950 deleted,
951 cached,
952 no_metadata,
953 };
954 async move {
955 set.list_bucket_for_scanner(&opts)
956 .await
957 .map(|(buckets, complete)| (pool_index, set_index, buckets, complete))
958 }
959 }))
960 .buffer_unordered(concurrency);
961 let mut topology_complete = set_count != 0;
962 let mut bucket_map = BTreeMap::<String, BucketInfo>::new();
963 let mut scoped_buckets = Vec::with_capacity(set_count);
964 while let Some(set_listing) = set_listings.next().await {
965 let (pool_index, set_index, buckets, set_complete) = set_listing?;
966 topology_complete &= set_complete;
967 for bucket in &buckets {
968 bucket_map.entry(bucket.name.clone()).or_insert_with(|| bucket.clone());
969 }
970 scoped_buckets.push(crate::cluster::rpc::ScannerSetBucketListing {
971 pool_index,
972 set_index,
973 buckets,
974 });
975 }
976 scoped_buckets.sort_unstable_by_key(|scope| (scope.pool_index, scope.set_index));
977 let mut listing = crate::cluster::rpc::ScannerBucketListing {
978 buckets: bucket_map.into_values().collect(),
979 set_buckets: scoped_buckets,
980 topology_complete,
981 };
982
983 if !opts.no_metadata {
984 for bucket in &mut listing.buckets {
985 if let Ok(created) = metadata_sys::created_at_in(&self.ctx, &bucket.name).await
986 && should_override_created_from_metadata(created)
987 {
988 bucket.created = Some(created);
989 }
990 }
991 let created_by_bucket = listing
992 .buckets
993 .iter()
994 .map(|bucket| (bucket.name.as_str(), bucket.created))
995 .collect::<BTreeMap<_, _>>();
996 for scope in &mut listing.set_buckets {
997 for bucket in &mut scope.buckets {
998 if let Some(created) = created_by_bucket.get(bucket.name.as_str()) {
999 bucket.created = *created;
1000 }
1001 }
1002 }
1003 }
1004
1005 Ok(listing)
1006 }
1007
1008 #[instrument(skip(self))]
1009 pub(super) async fn handle_delete_bucket(&self, bucket: &str, opts: &DeleteBucketOptions) -> Result<()> {
1010 self.handle_delete_bucket_with_diagnostic_budget(bucket, opts, BucketDeleteDiagnosticBudget::new())
1011 .await
1012 }
1013
1014 async fn handle_delete_bucket_with_diagnostic_budget(
1015 &self,
1016 bucket: &str,
1017 opts: &DeleteBucketOptions,
1018 mut diagnostic_budget: BucketDeleteDiagnosticBudget,
1019 ) -> Result<()> {
1020 if is_meta_bucketname(bucket) {
1021 return Err(StorageError::BucketNameInvalid(bucket.to_string()));
1022 }
1023
1024 if let Err(err) = check_valid_bucket_name(bucket) {
1025 return Err(StorageError::BucketNameInvalid(err.to_string()));
1026 }
1027
1028 let bucket_lifecycle_guard = if !opts.no_lock {
1029 Some(self.acquire_bucket_lifecycle_write_lock(bucket).await?)
1030 } else {
1031 None
1032 };
1033 let ns_guard = if !opts.no_lock {
1034 let ns_lock = self.new_ns_lock(bucket, bucket).await?;
1035 Some(
1036 await_bucket_namespace_operation(
1037 bucket_lifecycle_guard.as_ref(),
1038 bucket,
1039 "bucket namespace lock acquisition",
1040 async {
1041 ns_lock.get_write_lock(get_lock_acquire_timeout()).await.map_err(|e| match e {
1042 rustfs_lock::error::LockError::QuorumNotReached { required, achieved } => {
1043 StorageError::NamespaceLockQuorumUnavailable {
1044 mode: "write",
1045 bucket: bucket.to_string(),
1046 object: bucket.to_string(),
1047 required,
1048 achieved,
1049 }
1050 }
1051 other => StorageError::Lock(other),
1052 })
1053 },
1054 )
1055 .await?,
1056 )
1057 } else {
1058 None
1059 };
1060
1061 let sr_mark_delete = opts.srdelete_op == SRBucketDeleteOp::MarkDelete;
1062 let sr_purge = opts.srdelete_op == SRBucketDeleteOp::Purge;
1063 let sr_delete = sr_mark_delete || sr_purge;
1064 let mut delete_opts = opts.clone();
1065 let bucket_exists = match self.get_bucket_info_from_sets(bucket, &BucketOptions::default()).await {
1066 Ok(_) => true,
1067 Err(err) => {
1068 if is_err_strict_volume_not_found(&err) && sr_delete {
1069 false
1070 } else if is_err_strict_volume_not_found(&err) {
1071 return Err(StorageError::BucketNotFound(bucket.to_string()));
1072 } else {
1073 return Err(to_object_err(err, vec![bucket]));
1074 }
1075 }
1076 };
1077 if bucket_exists {
1078 validate_table_bucket_delete_guard(&self.ctx, bucket).await?;
1079
1080 if !opts.force {
1081 if let Some(blocker) = bucket_delete_local_blocker(&self.ctx, bucket, &mut diagnostic_budget).await? {
1082 return Err(blocker);
1083 }
1084 delete_opts.force_if_empty = true;
1085 }
1086 }
1087
1088 if sr_delete && !bucket_exists {
1089 delete_opts.force_if_empty = true;
1090 }
1091
1092 #[cfg(test)]
1093 crate::cluster::rpc::peer_s3_client::pause_after_delete_bucket_empty_scan().await;
1094
1095 if sr_mark_delete {
1096 await_bucket_lifecycle_operation(
1097 bucket_lifecycle_guard.as_ref(),
1098 ns_guard.as_ref(),
1099 bucket,
1100 "bucket delete marker creation",
1101 self.mark_bucket_deleted(bucket),
1102 )
1103 .await?;
1104 }
1105
1106 let retirement = if bucket_exists && bucket_lifecycle_guard.is_some() && !is_meta_bucketname(bucket) {
1109 if let Some(store) = metadata_sys::object_store_if_initialized_in(&self.ctx).await {
1110 crate::bucket::metadata::load_bucket_incarnation(store.clone(), bucket)
1111 .await?
1112 .map(|incarnation| (store, incarnation))
1113 } else {
1114 None
1115 }
1116 } else {
1117 None
1118 };
1119
1120 let delete_result = await_bucket_namespace_operation(
1121 bucket_lifecycle_guard.as_ref(),
1122 bucket,
1123 "physical bucket deletion",
1124 run_physical_bucket_deletion(ns_guard.as_ref(), bucket, async {
1125 self.delete_bucket_on_sets(bucket, &delete_opts)
1126 .await
1127 .map_err(|err| to_object_err(err, vec![bucket]))
1128 }),
1129 )
1130 .await;
1131 if let Err(err) = delete_result
1132 && (!sr_delete || !is_err_strict_volume_not_found(&err))
1133 {
1134 if delete_opts.force_if_empty && matches!(&err, StorageError::BucketNotEmpty(_)) {
1135 let mut diagnostic_budget = BucketDeleteDiagnosticBudget::new();
1136 if let Some(blocker) = bucket_delete_local_blocker(&self.ctx, bucket, &mut diagnostic_budget).await? {
1137 return Err(blocker);
1138 }
1139 }
1140 return Err(err);
1141 }
1142
1143 if let Some((store, incarnation)) = retirement {
1144 let mut record_opts = ObjectOptions {
1145 max_parity: true,
1146 ..Default::default()
1147 };
1148 if let Some(guard) = bucket_lifecycle_guard.as_ref() {
1149 record_opts.add_bucket_lifecycle_lock_guard(guard);
1150 }
1151 if let Some(guard) = ns_guard.as_ref() {
1152 record_opts.add_namespace_lock_guard(guard);
1153 }
1154 await_bucket_lifecycle_operation(
1155 bucket_lifecycle_guard.as_ref(),
1156 ns_guard.as_ref(),
1157 bucket,
1158 "bucket retirement publication",
1159 crate::bucket::retirement::commit_retirement(store, bucket, incarnation, &record_opts),
1160 )
1161 .await?;
1162 }
1163
1164 self.cleanup_bucket_usage_best_effort(bucket, ns_guard.as_ref()).await;
1165
1166 if let Err(err) = self
1167 .cleanup_deleted_bucket_metadata(bucket, sr_purge, ns_guard.as_ref())
1168 .await
1169 {
1170 warn!(
1171 bucket = %bucket,
1172 error = ?err,
1173 "physical bucket deletion succeeded but metadata cleanup remains pending"
1174 );
1175 }
1176 crate::store::list_objects::observe_scanner_namespace_mutations(bucket, 1);
1179 Ok(())
1180 }
1181}
1182
1183#[cfg(test)]
1184mod tests {
1185 use super::{
1186 BUCKET_DELETE_DIAGNOSTIC_MAX_ELAPSED, BUCKET_DELETE_DIAGNOSTIC_MAX_ENTRIES, BUCKET_DELETE_XLMETA_DIAGNOSTIC_MAX_BYTES,
1187 BucketDeleteBlockerKind, BucketDeleteDiagnosticBudget, BucketMetadataLessResidue, SCANNER_BUCKET_LIST_SET_CONCURRENCY,
1188 await_bucket_namespace_operation, bucket_delete_metadata_cleanup_prefixes, bucket_deleted_marker_prefix,
1189 bucket_deleted_marker_volume, bucket_list_set_concurrency, record_bucket_delete_blocker, run_bucket_usage_cleanup,
1190 run_physical_bucket_deletion, scan_metadata_less_residue, scan_metadata_less_residue_with_budget,
1191 should_override_created_from_metadata, validate_table_bucket_delete_allowed,
1192 };
1193 use crate::bucket::metadata::{BucketMetadata, table_bucket_catalog_metadata_prefix};
1194 use crate::bucket::metadata_sys;
1195 use crate::cluster::rpc::peer_s3_client::install_delete_bucket_empty_scan_barrier;
1196 use crate::disk::{BUCKET_META_PREFIX, DiskAPI, RUSTFS_META_BUCKET, STORAGE_FORMAT_FILE};
1197 use crate::error::StorageError;
1198 use crate::object_api::{ObjectOptions, PutObjReader};
1199 use crate::runtime::instance::InstanceContext;
1200 use crate::storage_api_contracts::{
1201 bucket::{BucketOperations as _, BucketOptions, DeleteBucketOptions, MakeBucketOptions, SRBucketDeleteOp},
1202 list::ListOperations as _,
1203 namespace::NamespaceLocking as _,
1204 object::{ObjectIO as _, ObjectOperations as _},
1205 };
1206 use crate::store::{ECStore, init_local_disks_with_instance_ctx};
1207 use crate::{
1208 disk::endpoint::Endpoint,
1209 layout::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints},
1210 };
1211 use rustfs_data_usage::{BucketUsageInfo, DATA_USAGE_OBJECT_NAME, DataUsageInfo};
1212 use rustfs_filemeta::{FileInfo, FileMeta, TRANSITION_COMPLETE};
1213 use rustfs_lock::{LocalClient, LockRequest, LockType, NamespaceLock, ObjectKey};
1214 use serial_test::serial;
1215 use std::path::{Path, PathBuf};
1216 use std::sync::Arc;
1217 use std::sync::atomic::{AtomicBool, Ordering};
1218 use std::time::{Duration, SystemTime};
1219 use time::OffsetDateTime;
1220 use tokio::io::AsyncReadExt;
1221 use tokio::sync::{Notify, OnceCell};
1222 use tokio_util::sync::CancellationToken;
1223 use uuid::Uuid;
1224
1225 static BUCKET_DELETE_TEST_ENV: OnceCell<(Vec<PathBuf>, Arc<ECStore>)> = OnceCell::const_new();
1226
1227 #[tokio::test(start_paused = true)]
1228 async fn bucket_delete_diagnostic_budget_starts_with_first_scan_io_and_latches_once() {
1229 let mut budget = BucketDeleteDiagnosticBudget::with_limits(8, Duration::from_millis(100));
1230 tokio::time::advance(Duration::from_secs(10)).await;
1231
1232 let first_polled = Arc::new(AtomicBool::new(false));
1233 let first_polled_for_io = first_polled.clone();
1234 let first = budget
1235 .run_io(async move {
1236 first_polled_for_io.store(true, Ordering::SeqCst);
1237 Ok::<_, std::io::Error>(7_u8)
1238 })
1239 .await
1240 .expect("the first diagnostic IO should succeed");
1241 assert_eq!(first, Some(7));
1242 assert!(first_polled.load(Ordering::SeqCst));
1243
1244 tokio::time::advance(Duration::from_millis(101)).await;
1245 let expired_polled = Arc::new(AtomicBool::new(false));
1246 let expired_polled_for_io = expired_polled.clone();
1247 let expired = budget
1248 .run_io(async move {
1249 expired_polled_for_io.store(true, Ordering::SeqCst);
1250 Ok::<_, std::io::Error>(9_u8)
1251 })
1252 .await
1253 .expect("an expired diagnostic budget should not become an IO error");
1254 assert_eq!(expired, None);
1255 assert!(
1256 !expired_polled.load(Ordering::SeqCst),
1257 "the deadline must remain latched after the first scan IO"
1258 );
1259 }
1260
1261 #[tokio::test(start_paused = true)]
1262 async fn bucket_delete_diagnostic_budget_times_out_its_first_pending_io() {
1263 let mut budget = BucketDeleteDiagnosticBudget::with_limits(8, Duration::from_millis(100));
1264 let io_polled = Arc::new(AtomicBool::new(false));
1265 let io_polled_for_future = io_polled.clone();
1266
1267 let result = budget
1268 .run_io(std::future::poll_fn(move |_cx| {
1269 io_polled_for_future.store(true, Ordering::SeqCst);
1270 std::task::Poll::<std::io::Result<()>>::Pending
1271 }))
1272 .await
1273 .expect("a diagnostic timeout should fail closed without an IO error");
1274
1275 assert_eq!(result, None);
1276 assert!(
1277 io_polled.load(Ordering::SeqCst),
1278 "the first diagnostic IO must be polled before its timeout"
1279 );
1280 }
1281
1282 #[tokio::test(start_paused = true)]
1283 async fn delayed_metadata_less_scans_still_detect_orphans_and_xlmeta() {
1284 let root = tempfile::tempdir().expect("temporary delayed-scan roots should be created");
1285 let orphan_root = root.path().join("orphan-root");
1286 let xlmeta_root = root.path().join("xlmeta-root");
1287 std::fs::create_dir_all(&orphan_root).expect("orphan root should be created");
1288 std::fs::create_dir_all(&xlmeta_root).expect("xlmeta root should be created");
1289 std::fs::write(orphan_root.join("orphan-part"), b"orphan").expect("orphan fixture should be written");
1290 std::fs::write(xlmeta_root.join(STORAGE_FORMAT_FILE), b"invalid-xlmeta").expect("xl.meta fixture should be written");
1291
1292 let mut orphan_budget = BucketDeleteDiagnosticBudget::with_limits(16, Duration::from_secs(5));
1293 let mut xlmeta_budget = BucketDeleteDiagnosticBudget::with_limits(16, Duration::from_secs(5));
1294 tokio::time::advance(Duration::from_secs(60)).await;
1295
1296 let orphan = scan_metadata_less_residue_with_budget(&orphan_root, &mut orphan_budget)
1297 .await
1298 .expect("delayed orphan scan should complete");
1299 assert!(orphan.has_residue_without_xlmeta());
1300 assert!(!orphan.diagnostic_truncated);
1301
1302 let xlmeta = scan_metadata_less_residue_with_budget(&xlmeta_root, &mut xlmeta_budget)
1303 .await
1304 .expect("delayed xl.meta scan should complete");
1305 assert!(xlmeta.xlmeta_found);
1306 assert!(!xlmeta.diagnostic_truncated);
1307 }
1308
1309 #[tokio::test(start_paused = true)]
1310 async fn bucket_namespace_operation_fails_closed_after_lease_expiry() {
1311 let ttl = Duration::from_millis(20);
1312 let lock = NamespaceLock::new("bucket-operation-test".to_string(), Arc::new(LocalClient::new()));
1313 let request = LockRequest::new(ObjectKey::new("bucket", ""), LockType::Exclusive, "test-owner")
1314 .with_acquire_timeout(Duration::from_secs(1))
1315 .with_ttl(ttl)
1316 .with_refresh_interval(ttl);
1317 let guard = lock
1318 .acquire_guard(&request)
1319 .await
1320 .expect("namespace lock acquisition should not fail")
1321 .expect("namespace lock should be acquired");
1322 tokio::time::advance(ttl + Duration::from_millis(1)).await;
1323
1324 let operation_ran = Arc::new(AtomicBool::new(false));
1325 let operation_ran_for_future = operation_ran.clone();
1326 let result = await_bucket_namespace_operation(Some(&guard), "bucket", "test operation", async move {
1327 operation_ran_for_future.store(true, Ordering::SeqCst);
1328 Ok(())
1329 })
1330 .await;
1331
1332 assert!(result.is_err(), "an expired namespace lease must fence the operation");
1333 assert!(
1334 !operation_ran.load(Ordering::SeqCst),
1335 "a fenced namespace operation must not poll its mutation future"
1336 );
1337 }
1338
1339 #[tokio::test]
1340 #[serial]
1341 async fn physical_bucket_delete_fences_scanner_before_polling_storage() {
1342 let generation_before = crate::store::list_objects::scanner_namespace_mutation_generation();
1343 let storage_polled = Arc::new(AtomicBool::new(false));
1344 let storage_polled_for_future = storage_polled.clone();
1345
1346 run_physical_bucket_deletion(None, "generation-order-bucket", async move {
1347 assert!(
1348 crate::store::list_objects::scanner_namespace_mutation_generation() > generation_before,
1349 "scanner generation must advance before physical deletion is polled"
1350 );
1351 storage_polled_for_future.store(true, Ordering::SeqCst);
1352 Ok(())
1353 })
1354 .await
1355 .expect("synthetic physical deletion should succeed");
1356
1357 assert!(storage_polled.load(Ordering::SeqCst));
1358 }
1359
1360 #[tokio::test]
1361 async fn bucket_usage_cleanup_stops_after_parent_cancellation() {
1362 let started = Arc::new(Notify::new());
1363 let started_wait = started.notified();
1364 let release = Arc::new(Notify::new());
1365 let completed = Arc::new(AtomicBool::new(false));
1366 let started_for_cleanup = started.clone();
1367 let release_for_cleanup = release.clone();
1368 let completed_for_cleanup = completed.clone();
1369 let parent = tokio::spawn(run_bucket_usage_cleanup(None, "bucket", async move {
1370 started_for_cleanup.notify_one();
1371 release_for_cleanup.notified().await;
1372 completed_for_cleanup.store(true, Ordering::SeqCst);
1373 Ok(())
1374 }));
1375 started_wait.await;
1376
1377 parent.abort();
1378 let _ = parent.await;
1379 tokio::task::yield_now().await;
1380 release.notify_waiters();
1381 tokio::task::yield_now().await;
1382 assert!(
1383 !completed.load(Ordering::SeqCst),
1384 "a cancelled cleanup future must not continue in a detached task"
1385 );
1386 }
1387
1388 #[tokio::test(start_paused = true)]
1389 #[serial]
1390 async fn bucket_namespace_operation_stops_in_flight_work_after_lock_loss() {
1391 let ttl = Duration::from_millis(20);
1392 let lock = NamespaceLock::new("bucket-operation-in-flight-loss-test".to_string(), Arc::new(LocalClient::new()));
1393 let request = LockRequest::new(ObjectKey::new("bucket", ""), LockType::Exclusive, "test-owner")
1394 .with_acquire_timeout(Duration::from_secs(1))
1395 .with_ttl(ttl)
1396 .with_refresh_interval(ttl);
1397 let guard = lock
1398 .acquire_guard(&request)
1399 .await
1400 .expect("namespace lock acquisition should not fail")
1401 .expect("namespace lock should be acquired");
1402
1403 let operation_started = Arc::new(Notify::new());
1404 let started_wait = operation_started.notified();
1405 let operation_release = Arc::new(Notify::new());
1406 let operation_completed = Arc::new(AtomicBool::new(false));
1407 let started_for_operation = operation_started.clone();
1408 let release_for_operation = operation_release.clone();
1409 let completed_for_operation = operation_completed.clone();
1410 let task = tokio::spawn(async move {
1411 await_bucket_namespace_operation(Some(&guard), "bucket", "test operation", async move {
1412 started_for_operation.notify_one();
1413 release_for_operation.notified().await;
1414 completed_for_operation.store(true, Ordering::SeqCst);
1415 Ok(())
1416 })
1417 .await
1418 });
1419 started_wait.await;
1420
1421 tokio::time::advance(ttl + Duration::from_millis(1)).await;
1422 let err = task
1423 .await
1424 .expect("operation task should join")
1425 .expect_err("an operation still running after lease loss must be fenced");
1426 assert!(err.to_string().contains("namespace lock was lost during test operation"));
1427 operation_release.notify_waiters();
1428 tokio::task::yield_now().await;
1429 assert!(
1430 !operation_completed.load(Ordering::SeqCst),
1431 "lock loss must stop the old owner before a successor can acquire the namespace"
1432 );
1433 }
1434
1435 async fn setup_bucket_delete_test_env() -> (Vec<PathBuf>, Arc<ECStore>) {
1436 BUCKET_DELETE_TEST_ENV
1437 .get_or_init(|| async {
1438 let temp_dir = std::env::temp_dir().join(format!("rustfs_bucket_delete_test_{}", Uuid::new_v4()));
1439 tokio::fs::create_dir_all(&temp_dir)
1440 .await
1441 .expect("test base directory should be created");
1442
1443 let disk_paths = (0..4)
1444 .map(|disk_idx| temp_dir.join(format!("disk{disk_idx}")))
1445 .collect::<Vec<_>>();
1446
1447 for disk_path in &disk_paths {
1448 tokio::fs::create_dir_all(disk_path)
1449 .await
1450 .expect("disk directory should be created");
1451 }
1452
1453 let mut endpoints = Vec::with_capacity(disk_paths.len());
1454 for (disk_idx, disk_path) in disk_paths.iter().enumerate() {
1455 let mut endpoint =
1456 Endpoint::try_from(disk_path.to_str().expect("disk path should be utf8")).expect("endpoint should parse");
1457 endpoint.set_pool_index(0);
1458 endpoint.set_set_index(0);
1459 endpoint.set_disk_index(disk_idx);
1460 endpoints.push(endpoint);
1461 }
1462
1463 let endpoint_pools = EndpointServerPools(vec![PoolEndpoints {
1464 legacy: false,
1465 set_count: 1,
1466 drives_per_set: 4,
1467 endpoints: Endpoints::from(endpoints),
1468 cmd_line: "bucket-delete-test".to_string(),
1469 platform: format!("OS: {} | Arch: {}", std::env::consts::OS, std::env::consts::ARCH),
1470 }]);
1471
1472 let instance_ctx = Arc::new(InstanceContext::new());
1473 init_local_disks_with_instance_ctx(&instance_ctx, endpoint_pools.clone())
1474 .await
1475 .expect("local disks should initialize");
1476 let ecstore = ECStore::new_with_instance_ctx(
1477 "127.0.0.1:0".parse().expect("test address"),
1478 endpoint_pools,
1479 CancellationToken::new(),
1480 instance_ctx,
1481 )
1482 .await
1483 .expect("ECStore should initialize");
1484 let storage_class = crate::config::storageclass::lookup_config_for_pools_without_env(
1485 &rustfs_config::server_config::KVS::new(),
1486 &[4],
1487 )
1488 .expect("bucket test storage class should match its four-disk pool");
1489 for pool in &ecstore.pools {
1490 for set in &pool.disk_set {
1491 set.set_test_storage_class_config(storage_class.clone());
1492 }
1493 }
1494
1495 metadata_sys::init_bucket_metadata_sys(ecstore.clone(), Vec::new()).await;
1496
1497 (disk_paths, ecstore)
1498 })
1499 .await
1500 .clone()
1501 }
1502
1503 async fn setup_multi_pool_bucket_test_env() -> (tempfile::TempDir, Arc<ECStore>) {
1504 setup_bucket_quorum_test_env(&[4, 4], None).await
1505 }
1506
1507 async fn setup_bucket_quorum_test_env(
1508 drives_per_pool: &[usize],
1509 standard_parity: Option<usize>,
1510 ) -> (tempfile::TempDir, Arc<ECStore>) {
1511 let temp_dir = tempfile::tempdir().expect("multi-pool bucket test directory should be created");
1512 let mut pools = Vec::new();
1513 for (pool_index, &drive_count) in drives_per_pool.iter().enumerate() {
1514 let mut endpoints = Vec::new();
1515 for disk_index in 0..drive_count {
1516 let disk_path = temp_dir.path().join(format!("pool{pool_index}-disk{disk_index}"));
1517 tokio::fs::create_dir_all(&disk_path)
1518 .await
1519 .expect("multi-pool bucket test disk should be created");
1520 let mut endpoint =
1521 Endpoint::try_from(disk_path.to_str().expect("disk path should be utf8")).expect("endpoint should parse");
1522 endpoint.set_pool_index(pool_index);
1523 endpoint.set_set_index(0);
1524 endpoint.set_disk_index(disk_index);
1525 endpoints.push(endpoint);
1526 }
1527 pools.push(PoolEndpoints {
1528 legacy: false,
1529 set_count: 1,
1530 drives_per_set: drive_count,
1531 endpoints: Endpoints::from(endpoints),
1532 cmd_line: format!("bucket-test-pool-{pool_index}"),
1533 platform: format!("OS: {} | Arch: {}", std::env::consts::OS, std::env::consts::ARCH),
1534 });
1535 }
1536
1537 let endpoint_pools = EndpointServerPools(pools);
1538 let instance_ctx = Arc::new(InstanceContext::new());
1539 instance_ctx.set_endpoints(endpoint_pools.clone());
1540 init_local_disks_with_instance_ctx(&instance_ctx, endpoint_pools.clone())
1541 .await
1542 .expect("multi-pool local disks should initialize");
1543 let ecstore = ECStore::new_with_instance_ctx(
1544 "127.0.0.1:0".parse().expect("test address"),
1545 endpoint_pools,
1546 CancellationToken::new(),
1547 instance_ctx,
1548 )
1549 .await
1550 .expect("multi-pool ECStore should initialize");
1551 let mut storage_class_kvs = rustfs_config::server_config::KVS::new();
1552 if let Some(parity) = standard_parity {
1553 storage_class_kvs.insert(crate::config::storageclass::CLASS_STANDARD.to_string(), format!("EC:{parity}"));
1554 }
1555 let storage_class = crate::config::storageclass::lookup_config_for_pools_without_env(&storage_class_kvs, drives_per_pool)
1556 .expect("storage class should match every test erasure set");
1557 for pool in &ecstore.pools {
1558 for set in &pool.disk_set {
1559 set.set_test_storage_class_config(storage_class.clone());
1560 }
1561 }
1562
1563 (temp_dir, ecstore)
1564 }
1565
1566 async fn take_set_disks_offline(
1567 ecstore: &ECStore,
1568 set: &Arc<crate::set_disk::SetDisks>,
1569 disk_indexes: &[usize],
1570 ) -> Vec<(usize, crate::disk::DiskStore)> {
1571 let offline = {
1572 let disks = set.disks.read().await;
1573 disk_indexes
1574 .iter()
1575 .map(|index| (*index, disks[*index].clone().expect("fault-injection disk should start online")))
1576 .collect::<Vec<_>>()
1577 };
1578 for (_, disk) in &offline {
1579 disk.close().await.expect("fault injection should stop per-disk monitoring");
1580 disk.force_offline_for_test();
1581 }
1582 set.connect_disks().await;
1585 {
1586 let disks = set.disks.read().await;
1587 for (index, disk) in &offline {
1588 let current = disks[*index].as_ref().expect("fault-injection slot should remain populated");
1589 assert!(Arc::ptr_eq(current, disk), "reconnect must preserve the fault-injection handle");
1590 assert_eq!(disk.runtime_state(), crate::disk::health_state::RuntimeDriveHealthState::Offline);
1591 }
1592 }
1593 let local_disk_map = ecstore.ctx.local_disk_map();
1594 let mut local_disks = local_disk_map.write().await;
1595 for (_, disk) in &offline {
1596 local_disks.insert(disk.endpoint().to_string(), None);
1597 }
1598 offline
1599 }
1600
1601 async fn restore_set_disks(
1602 ecstore: &ECStore,
1603 set: &Arc<crate::set_disk::SetDisks>,
1604 offline: Vec<(usize, crate::disk::DiskStore)>,
1605 ) {
1606 {
1607 let local_disk_map = ecstore.ctx.local_disk_map();
1608 let mut local_disks = local_disk_map.write().await;
1609 for (_, disk) in &offline {
1610 local_disks.insert(disk.endpoint().to_string(), Some(Arc::clone(disk)));
1611 }
1612 }
1613 let disks = set.disks.read().await;
1614 for (index, disk) in offline {
1615 let current = disks[index].as_ref().expect("fault-injection slot should remain populated");
1616 assert!(Arc::ptr_eq(current, &disk), "fault-injection disk handle should remain unchanged");
1617 assert_eq!(disk.runtime_state(), crate::disk::health_state::RuntimeDriveHealthState::Offline);
1618 disk.reset_health_for_store_init_retry();
1619 }
1620 }
1621
1622 #[tokio::test]
1623 async fn request_metadata_methods_fail_closed_before_instance_initialization() {
1624 let (_temp_dir, store) = setup_multi_pool_bucket_test_env().await;
1625
1626 let expected = "bucket metadata sys not initialized for this instance";
1627 let errors = [
1628 store.get_bucket_metadata("bucket").await.unwrap_err(),
1629 store.get_bucket_policy("bucket").await.unwrap_err(),
1630 store.get_bucket_policy_raw("bucket").await.unwrap_err(),
1631 store.restricts_public_bucket_access("bucket").await.unwrap_err(),
1632 store
1633 .update_bucket_metadata_config("bucket", crate::bucket::metadata::BUCKET_POLICY_CONFIG, Vec::new())
1634 .await
1635 .unwrap_err(),
1636 ];
1637 for error in errors {
1638 assert_eq!(error.to_string(), format!("Io error: {expected}"));
1639 }
1640 }
1641
1642 async fn create_bucket_with_object(ecstore: &Arc<ECStore>, bucket: &str, object: &str) {
1643 let generation_before_make = ecstore.scanner_namespace_mutation_generation();
1644 ecstore
1645 .make_bucket(bucket, &MakeBucketOptions::default())
1646 .await
1647 .expect("bucket should be created");
1648 assert_eq!(
1649 ecstore.scanner_namespace_mutation_generation(),
1650 generation_before_make.saturating_add(2),
1651 "successful bucket creation should fence scanner namespace activity before and after creation"
1652 );
1653
1654 let generation_before_put = ecstore.scanner_namespace_mutation_generation();
1655 let mut reader = PutObjReader::from_vec(b"delete bucket semantics".to_vec());
1656 ecstore
1657 .put_object(bucket, object, &mut reader, &ObjectOptions::default())
1658 .await
1659 .expect("object should be written");
1660 let lock = ecstore.pools[0].disk_set[0]
1661 .new_ns_lock(bucket, object)
1662 .await
1663 .expect("fixture namespace lock should be created");
1664 drop(
1665 lock.get_write_lock(Duration::from_secs(30))
1666 .await
1667 .expect("fixture rename tail should finish before checking its generation"),
1668 );
1669 assert_eq!(
1670 ecstore.scanner_namespace_mutation_generation(),
1671 generation_before_put.saturating_add(3),
1672 "successful object creation must observe the logical mutation and both fanout boundaries"
1673 );
1674 ecstore
1675 .get_object_info(bucket, object, &ObjectOptions::default())
1676 .await
1677 .expect("object should be readable before bucket delete");
1678 }
1679
1680 async fn any_disk_path_exists(disk_paths: &[PathBuf], relative_path: impl AsRef<Path>) -> bool {
1681 for disk_path in disk_paths {
1682 if tokio::fs::try_exists(disk_path.join(relative_path.as_ref()))
1683 .await
1684 .expect("test disk path should be stat-able")
1685 {
1686 return true;
1687 }
1688 }
1689 false
1690 }
1691
1692 async fn any_disk_has_object_metadata(disk_paths: &[PathBuf], bucket: &str) -> bool {
1693 for disk_path in disk_paths {
1694 if super::has_xlmeta_files(&disk_path.join(bucket))
1695 .await
1696 .expect("object metadata scan should succeed")
1697 {
1698 return true;
1699 }
1700 }
1701 false
1702 }
1703
1704 async fn write_metadata_less_part_on_all_disks(disk_paths: &[PathBuf], bucket: &str, object: &str) {
1705 for disk_path in disk_paths {
1706 let data_dir = Uuid::new_v4();
1707 let part_path = disk_path.join(bucket).join(object).join(data_dir.to_string()).join("part.1");
1708 tokio::fs::create_dir_all(part_path.parent().expect("part path should have a parent"))
1709 .await
1710 .expect("metadata-less data dir should be created");
1711 tokio::fs::write(part_path, b"orphan shard")
1712 .await
1713 .expect("metadata-less part should be written");
1714 }
1715 }
1716
1717 async fn write_bucket_metadata_marker(disk_paths: &[PathBuf], metadata_prefix: &str) {
1718 for disk_path in disk_paths {
1719 let marker_path = disk_path.join(metadata_prefix).join("config.json");
1720 tokio::fs::create_dir_all(marker_path.parent().expect("metadata marker path should have a parent"))
1721 .await
1722 .expect("metadata marker parent should be created");
1723 tokio::fs::write(marker_path, b"bucket metadata")
1724 .await
1725 .expect("metadata marker should be written");
1726 }
1727 }
1728
1729 #[test]
1730 fn should_not_override_when_metadata_created_is_unix_epoch() {
1731 assert!(!should_override_created_from_metadata(OffsetDateTime::UNIX_EPOCH));
1732 }
1733
1734 #[test]
1735 fn should_override_when_metadata_created_is_valid_time() {
1736 let created = OffsetDateTime::from_unix_timestamp(1704067200).expect("valid timestamp");
1737 assert!(should_override_created_from_metadata(created));
1738 }
1739
1740 #[test]
1741 fn table_bucket_delete_guard_rejects_remaining_catalog_metadata() {
1742 let err = validate_table_bucket_delete_allowed("table-bucket", true, true).unwrap_err();
1743
1744 assert!(matches!(err, StorageError::BucketNotEmpty(bucket) if bucket == "table-bucket"));
1745 assert!(validate_table_bucket_delete_allowed("table-bucket", true, false).is_ok());
1746 assert!(validate_table_bucket_delete_allowed("regular-bucket", false, true).is_ok());
1747 }
1748
1749 #[test]
1750 fn bucket_delete_metadata_cleanup_removes_internal_table_catalog_prefix() {
1751 let prefixes = bucket_delete_metadata_cleanup_prefixes("analytics");
1752
1753 assert!(prefixes.contains(&table_bucket_catalog_metadata_prefix("analytics")));
1754 assert!(prefixes.contains(&"buckets/analytics".to_string()));
1755 }
1756
1757 #[test]
1758 fn bucket_delete_marker_path_uses_internal_deleted_bucket_metadata_prefix() {
1759 assert_eq!(bucket_deleted_marker_prefix("analytics"), "buckets/.deleted/analytics");
1760 assert_eq!(
1761 bucket_deleted_marker_volume("analytics"),
1762 format!("{RUSTFS_META_BUCKET}/{BUCKET_META_PREFIX}/.deleted/analytics")
1763 );
1764 }
1765
1766 #[test]
1767 fn bucket_listing_selects_request_and_scanner_fanout() {
1768 assert_eq!(bucket_list_set_concurrency(0, SCANNER_BUCKET_LIST_SET_CONCURRENCY), 1);
1769 assert_eq!(bucket_list_set_concurrency(2, SCANNER_BUCKET_LIST_SET_CONCURRENCY), 2);
1770 assert_eq!(
1771 bucket_list_set_concurrency(100, SCANNER_BUCKET_LIST_SET_CONCURRENCY),
1772 SCANNER_BUCKET_LIST_SET_CONCURRENCY
1773 );
1774 assert_eq!(bucket_list_set_concurrency(100, usize::MAX), 100);
1775 }
1776
1777 #[tokio::test]
1778 async fn metadata_less_residue_scan_ignores_empty_dirs_and_reports_uuid_data() {
1779 let root = tempfile::tempdir().expect("temporary bucket root should be created");
1780 let bucket_path = root.path().join("bucket");
1781 tokio::fs::create_dir_all(bucket_path.join("empty/child"))
1782 .await
1783 .expect("empty directory residue should be created");
1784
1785 let empty = scan_metadata_less_residue(&bucket_path)
1786 .await
1787 .expect("empty residue scan should succeed");
1788 assert!(!empty.has_residue_without_xlmeta());
1789
1790 let data_dir = Uuid::new_v4();
1791 let part_path = bucket_path.join("object").join(data_dir.to_string()).join("part.1");
1792 tokio::fs::create_dir_all(part_path.parent().expect("part path should have a parent"))
1793 .await
1794 .expect("data dir should be created");
1795 tokio::fs::write(&part_path, b"orphan shard")
1796 .await
1797 .expect("part file should be written");
1798
1799 let residue = scan_metadata_less_residue(&bucket_path)
1800 .await
1801 .expect("metadata-less residue scan should succeed");
1802 assert!(residue.has_residue_without_xlmeta());
1803 assert_eq!(residue.files, 1);
1804 assert_eq!(residue.uuid_data_dirs, 1);
1805 let sample = residue.sample.as_deref().expect("part sample should be recorded");
1806 assert!(sample.starts_with("object/"));
1807 assert!(sample.ends_with("/part.1"));
1808 }
1809
1810 #[tokio::test]
1811 async fn metadata_less_residue_scan_shares_one_entry_budget_across_roots() {
1812 let root = tempfile::tempdir().expect("temporary diagnostic roots should be created");
1813 let first_root = root.path().join("disk-a");
1814 let second_root = root.path().join("disk-b");
1815 tokio::fs::create_dir_all(&first_root)
1816 .await
1817 .expect("first diagnostic root should be created");
1818 tokio::fs::create_dir_all(&second_root)
1819 .await
1820 .expect("second diagnostic root should be created");
1821 for index in 0..3 {
1822 std::fs::write(first_root.join(format!("first-{index}")), b"").expect("first-root fixture should be written");
1823 std::fs::write(second_root.join(format!("second-{index}")), b"").expect("second-root fixture should be written");
1824 }
1825
1826 let mut budget = BucketDeleteDiagnosticBudget::with_limits(4, Duration::from_secs(5));
1827 let first = scan_metadata_less_residue_with_budget(&first_root, &mut budget)
1828 .await
1829 .expect("first root should fit the shared budget");
1830 assert!(!first.diagnostic_truncated);
1831 let second = scan_metadata_less_residue_with_budget(&second_root, &mut budget)
1832 .await
1833 .expect("second root should stop at the remaining shared budget");
1834 assert!(second.diagnostic_truncated);
1835 assert!(first.entries_scanned + second.entries_scanned <= 4);
1836 }
1837
1838 #[tokio::test]
1839 async fn metadata_less_residue_scan_honors_an_expired_request_deadline() {
1840 let root = tempfile::tempdir().expect("temporary diagnostic root should be created");
1841 std::fs::write(root.path().join("orphan"), b"").expect("deadline fixture should be written");
1842 let mut budget = BucketDeleteDiagnosticBudget::with_limits(8, Duration::ZERO);
1843
1844 let scan = scan_metadata_less_residue_with_budget(root.path(), &mut budget)
1845 .await
1846 .expect("an expired diagnostic budget should fail closed without an IO error");
1847
1848 assert!(scan.diagnostic_truncated);
1849 assert_eq!(scan.entries_scanned, 0);
1850 assert_eq!(scan.diagnostic_bytes_read, 0);
1851 }
1852
1853 #[tokio::test]
1854 async fn metadata_less_residue_scan_stops_at_diagnostic_budget() {
1855 let root = tempfile::tempdir().expect("temporary bucket root should be created");
1856 let bucket_path = root.path().join("bucket");
1857 tokio::fs::create_dir_all(&bucket_path)
1858 .await
1859 .expect("budget fixture directory should be created");
1860 for index in 0..(BUCKET_DELETE_DIAGNOSTIC_MAX_ENTRIES + 32) {
1861 std::fs::write(bucket_path.join(format!("orphan-{index:05}")), b"").expect("budget fixture file should be created");
1862 }
1863
1864 let residue = scan_metadata_less_residue(&bucket_path)
1865 .await
1866 .expect("budgeted residue scan should fail closed without an IO error");
1867 assert!(residue.diagnostic_truncated);
1868 assert!(residue.has_residue_without_xlmeta());
1869 assert!(!residue.xlmeta_found);
1870 assert!(residue.entries_scanned <= BUCKET_DELETE_DIAGNOSTIC_MAX_ENTRIES);
1871 assert!(residue.files <= BUCKET_DELETE_DIAGNOSTIC_MAX_ENTRIES);
1872 assert_eq!(residue.diagnostic_bytes_read, 0);
1873 }
1874
1875 #[tokio::test(start_paused = true)]
1876 async fn bucket_residue_scan_distinguishes_visible_and_tier_free_xlmeta() {
1877 let (clock_guard_tx, clock_guard_rx) = std::sync::mpsc::channel::<()>();
1880 let clock_guard = tokio::task::spawn_blocking(move || clock_guard_rx.recv());
1881 let root = tempfile::tempdir().expect("temporary bucket root should be created");
1882 let bucket_path = root.path().join("bucket");
1883 let visible_path = bucket_path.join("visible").join(STORAGE_FORMAT_FILE);
1884 tokio::fs::create_dir_all(visible_path.parent().expect("visible xl.meta should have a parent"))
1885 .await
1886 .expect("visible object directory should be created");
1887 let mut visible = FileMeta::new();
1888 visible
1889 .add_version(FileInfo {
1890 version_id: Some(Uuid::new_v4()),
1891 mod_time: Some(OffsetDateTime::now_utc()),
1892 ..Default::default()
1893 })
1894 .expect("visible version should encode");
1895 tokio::fs::write(&visible_path, visible.marshal_msg().expect("visible xl.meta should marshal"))
1896 .await
1897 .expect("visible xl.meta should be written");
1898
1899 let visible_scan = scan_metadata_less_residue(&bucket_path)
1900 .await
1901 .expect("visible xl.meta scan should succeed");
1902 assert!(!visible_scan.diagnostic_truncated, "{visible_scan:?}");
1903 assert_eq!(visible_scan.xlmeta_blocker, Some(BucketDeleteBlockerKind::VisibleVersion));
1904
1905 tokio::fs::remove_dir_all(bucket_path.join("visible"))
1906 .await
1907 .expect("visible fixture should be removed");
1908 let free_path = bucket_path.join("free").join(STORAGE_FORMAT_FILE);
1909 tokio::fs::create_dir_all(free_path.parent().expect("free xl.meta should have a parent"))
1910 .await
1911 .expect("free-version object directory should be created");
1912 let source_version_id = Uuid::new_v4();
1913 let mut free = FileMeta::new();
1914 free.add_version(FileInfo {
1915 version_id: Some(source_version_id),
1916 transition_status: TRANSITION_COMPLETE.to_string(),
1917 transitioned_objname: "remote/object".to_string(),
1918 transition_version_id: Some(Uuid::new_v4()),
1919 transition_tier: "WARM".to_string(),
1920 mod_time: Some(OffsetDateTime::now_utc()),
1921 ..Default::default()
1922 })
1923 .expect("transitioned source should encode");
1924 let mut delete = FileInfo {
1925 version_id: Some(source_version_id),
1926 mod_time: Some(OffsetDateTime::now_utc()),
1927 ..Default::default()
1928 };
1929 delete.set_tier_free_version_id(&Uuid::new_v4().to_string());
1930 free.delete_version(&delete)
1931 .expect("transitioned source delete should create a free-version");
1932 tokio::fs::write(&free_path, free.marshal_msg().expect("free-version xl.meta should marshal"))
1933 .await
1934 .expect("free-version xl.meta should be written");
1935
1936 let free_scan = scan_metadata_less_residue(&bucket_path)
1937 .await
1938 .expect("free-version xl.meta scan should succeed");
1939 assert!(!free_scan.diagnostic_truncated, "{free_scan:?}");
1940 assert_eq!(free_scan.xlmeta_blocker, Some(BucketDeleteBlockerKind::TierFreeVersion));
1941
1942 tokio::fs::remove_dir_all(bucket_path.join("free"))
1943 .await
1944 .expect("free-version fixture should be removed");
1945
1946 let exact_limit_path = bucket_path.join("exact-limit").join(STORAGE_FORMAT_FILE);
1947 tokio::fs::create_dir_all(exact_limit_path.parent().expect("exact-limit xl.meta should have a parent"))
1948 .await
1949 .expect("exact-limit object directory should be created");
1950 let exact_limit = tokio::fs::File::create(&exact_limit_path)
1951 .await
1952 .expect("exact-limit xl.meta should be created");
1953 exact_limit
1954 .set_len(BUCKET_DELETE_XLMETA_DIAGNOSTIC_MAX_BYTES)
1955 .await
1956 .expect("exact-limit xl.meta should be extended without allocating its contents");
1957 let exact_limit_scan = scan_metadata_less_residue(&bucket_path)
1958 .await
1959 .expect("exact-limit xl.meta scan should remain fail closed");
1960 assert!(!exact_limit_scan.diagnostic_truncated, "{exact_limit_scan:?}");
1961 assert_eq!(exact_limit_scan.xlmeta_blocker, Some(BucketDeleteBlockerKind::UnknownXlMeta));
1962 assert_eq!(exact_limit_scan.diagnostic_bytes_read, BUCKET_DELETE_XLMETA_DIAGNOSTIC_MAX_BYTES);
1963 tokio::fs::remove_dir_all(bucket_path.join("exact-limit"))
1964 .await
1965 .expect("exact-limit fixture should be removed");
1966
1967 let oversized_path = bucket_path.join("oversized").join(STORAGE_FORMAT_FILE);
1968 tokio::fs::create_dir_all(oversized_path.parent().expect("oversized xl.meta should have a parent"))
1969 .await
1970 .expect("oversized object directory should be created");
1971 let oversized = tokio::fs::File::create(&oversized_path)
1972 .await
1973 .expect("oversized xl.meta should be created");
1974 oversized
1975 .set_len(BUCKET_DELETE_XLMETA_DIAGNOSTIC_MAX_BYTES + 1)
1976 .await
1977 .expect("oversized xl.meta should be extended without allocating its contents");
1978 let oversized_scan = scan_metadata_less_residue(&bucket_path)
1979 .await
1980 .expect("oversized xl.meta scan should remain fail closed");
1981 assert!(!oversized_scan.diagnostic_truncated, "{oversized_scan:?}");
1982 assert_eq!(oversized_scan.xlmeta_blocker, Some(BucketDeleteBlockerKind::UnknownXlMeta));
1983 assert_eq!(oversized_scan.diagnostic_bytes_read, 0);
1984 assert!(oversized_scan.diagnostic_bytes_read <= BUCKET_DELETE_XLMETA_DIAGNOSTIC_MAX_BYTES);
1985
1986 drop(clock_guard_tx);
1987 let _ = clock_guard
1988 .await
1989 .expect("classification clock guard should exit after its sender is dropped");
1990
1991 let first_io_started = Arc::new(AtomicBool::new(false));
1994 let mut budget = BucketDeleteDiagnosticBudget::new().with_first_io_delay(
1995 BUCKET_DELETE_DIAGNOSTIC_MAX_ELAPSED + Duration::from_millis(100),
1996 first_io_started.clone(),
1997 );
1998 let truncated = scan_metadata_less_residue_with_budget(&bucket_path, &mut budget)
1999 .await
2000 .expect("a diagnostic timeout should return a partial scan");
2001 assert!(first_io_started.load(Ordering::SeqCst));
2002 assert!(truncated.diagnostic_truncated, "{truncated:?}");
2003 assert_eq!(truncated.xlmeta_blocker, None);
2004 assert_eq!(truncated.entries_scanned, 0);
2005 assert_eq!(truncated.diagnostic_bytes_read, 0);
2006 }
2007
2008 #[tokio::test]
2009 #[serial]
2010 async fn bucket_namespace_reads_keep_pre_expansion_bucket_visible() {
2011 let (_temp_dir, ecstore) = setup_multi_pool_bucket_test_env().await;
2012 let bucket = format!("pre-expansion-{}", Uuid::new_v4().simple());
2013 let object = "existing-object";
2014 ecstore.pools[0].disk_set[0]
2015 .make_bucket(&bucket, &MakeBucketOptions::default())
2016 .await
2017 .expect("bucket should be created in the original pool only");
2018 let mut reader = PutObjReader::from_vec(b"object written before pool expansion".to_vec());
2019 ecstore.pools[0]
2020 .put_object(&bucket, object, &mut reader, &ObjectOptions::default())
2021 .await
2022 .expect("object should be written in the original pool only");
2023
2024 let buckets = ecstore
2025 .list_bucket(&BucketOptions::default())
2026 .await
2027 .expect("S3 bucket listing should remain available after adding an empty pool");
2028 assert!(buckets.iter().any(|entry| entry.name == bucket));
2029
2030 let info = ecstore
2031 .get_bucket_info(&bucket, &BucketOptions::default())
2032 .await
2033 .expect("bucket validation should accept a bucket present in the original pool");
2034 assert_eq!(info.name, bucket);
2035
2036 metadata_sys::init_bucket_metadata_sys(ecstore.clone(), vec![bucket.clone()]).await;
2037 let listed = ecstore
2038 .clone()
2039 .list_objects_v2(&bucket, "", None, None, 1000, false, None, false)
2040 .await
2041 .expect("ListObjectsV2 should remain available after adding an empty pool");
2042 assert!(listed.objects.iter().any(|entry| entry.name == object));
2043 ecstore.pools[1].disk_set[0]
2044 .get_bucket_info(&bucket, &BucketOptions::default())
2045 .await
2046 .expect("metadata initialization should heal the bucket volume into the expansion pool");
2047
2048 let object_info = ecstore
2049 .get_object_info(&bucket, object, &ObjectOptions::default())
2050 .await
2051 .expect("an object in the original pool should remain readable after expansion");
2052 assert_eq!(object_info.name, object);
2053 }
2054
2055 #[tokio::test]
2056 #[serial]
2057 async fn bucket_creation_accepts_exact_quorum_in_every_set() {
2058 let (_temp_dir, ecstore) = setup_multi_pool_bucket_test_env().await;
2059 let bucket = format!("exact-set-quorum-{}", Uuid::new_v4().simple());
2060 let original_set = Arc::clone(&ecstore.pools[0].disk_set[0]);
2061 let offline = take_set_disks_offline(&ecstore, &original_set, &[0]).await;
2062
2063 ecstore
2064 .make_bucket_on_sets(&bucket, &MakeBucketOptions::default())
2065 .await
2066 .expect("three of four disks should satisfy the original set quorum");
2067
2068 restore_set_disks(&ecstore, &original_set, offline).await;
2069 for pool in &ecstore.pools {
2070 pool.disk_set[0]
2071 .get_bucket_info(&bucket, &BucketOptions::default())
2072 .await
2073 .expect("every set should expose a bucket created at its exact quorum");
2074 }
2075 }
2076
2077 #[tokio::test]
2078 #[serial]
2079 async fn bucket_creation_rejects_cross_pool_quorum_subsidy() {
2080 let (_temp_dir, ecstore) = setup_multi_pool_bucket_test_env().await;
2081 metadata_sys::init_bucket_metadata_sys(ecstore.clone(), Vec::new()).await;
2082 let bucket = format!("create-set-quorum-minus-one-{}", Uuid::new_v4().simple());
2083 let original_set = Arc::clone(&ecstore.pools[0].disk_set[0]);
2084 let offline = take_set_disks_offline(&ecstore, &original_set, &[0, 1]).await;
2085
2086 let err = ecstore
2087 .make_bucket(&bucket, &MakeBucketOptions::default())
2088 .await
2089 .expect_err("a healthy expansion pool must not subsidize a failed original set");
2090 restore_set_disks(&ecstore, &original_set, offline).await;
2091 assert!(
2092 matches!(&err, StorageError::InsufficientWriteQuorum(name, object) if name == &bucket && object.is_empty()),
2093 "unexpected error: {err}"
2094 );
2095 for pool in &ecstore.pools {
2096 assert_eq!(
2097 pool.disk_set[0]
2098 .get_bucket_info(&bucket, &BucketOptions::default())
2099 .await
2100 .expect_err("failed creation must not leave an authoritative bucket"),
2101 StorageError::VolumeNotFound
2102 );
2103 }
2104 }
2105
2106 #[tokio::test]
2107 #[serial]
2108 async fn bucket_creation_prioritizes_hard_set_error_over_existing_bucket() {
2109 let (_temp_dir, ecstore) = setup_multi_pool_bucket_test_env().await;
2110 metadata_sys::init_bucket_metadata_sys(ecstore.clone(), Vec::new()).await;
2111 let bucket = format!("create-hard-error-{}", Uuid::new_v4().simple());
2112 ecstore.pools[0].disk_set[0]
2113 .make_bucket(&bucket, &MakeBucketOptions::default())
2114 .await
2115 .expect("the original pool should contain the bucket");
2116 let expansion_set = Arc::clone(&ecstore.pools[1].disk_set[0]);
2117 let offline = take_set_disks_offline(&ecstore, &expansion_set, &[0, 1]).await;
2118
2119 let err = ecstore
2120 .make_bucket(&bucket, &MakeBucketOptions::default())
2121 .await
2122 .expect_err("a hard set failure must outrank BucketExists from another set");
2123 restore_set_disks(&ecstore, &expansion_set, offline).await;
2124 assert!(
2125 matches!(&err, StorageError::InsufficientWriteQuorum(name, object) if name == &bucket && object.is_empty()),
2126 "unexpected error: {err}"
2127 );
2128 ecstore.pools[0].disk_set[0]
2129 .get_bucket_info(&bucket, &BucketOptions::default())
2130 .await
2131 .expect("failure in another set must not roll back a pre-existing bucket");
2132 }
2133
2134 #[tokio::test]
2135 #[serial]
2136 async fn bucket_delete_rejects_cross_pool_quorum_subsidy_before_mutation() {
2137 let (_temp_dir, ecstore) = setup_multi_pool_bucket_test_env().await;
2138 metadata_sys::init_bucket_metadata_sys(ecstore.clone(), Vec::new()).await;
2139 let bucket = format!("delete-set-quorum-minus-one-{}", Uuid::new_v4().simple());
2140 ecstore
2141 .make_bucket(&bucket, &MakeBucketOptions::default())
2142 .await
2143 .expect("bucket should be created in every set before deletion");
2144 let original_set = Arc::clone(&ecstore.pools[0].disk_set[0]);
2145 let offline = take_set_disks_offline(&ecstore, &original_set, &[0, 1, 2]).await;
2146
2147 let err = ecstore
2148 .delete_bucket(
2149 &bucket,
2150 &DeleteBucketOptions {
2151 force: true,
2152 ..Default::default()
2153 },
2154 )
2155 .await
2156 .expect_err("a healthy expansion pool must not subsidize a failed original set deletion");
2157 restore_set_disks(&ecstore, &original_set, offline).await;
2158 assert!(
2159 matches!(&err, StorageError::InsufficientWriteQuorum(name, object) if name == &bucket && object.is_empty()),
2160 "unexpected error: {err}"
2161 );
2162 for pool in &ecstore.pools {
2163 pool.disk_set[0]
2164 .get_bucket_info(&bucket, &BucketOptions::default())
2165 .await
2166 .expect("failed deletion preflight must preserve every bucket volume");
2167 }
2168 let (_, persisted) = metadata_sys::get_config_from_disk_with_presence_in(&ecstore.ctx, &bucket)
2169 .await
2170 .expect("failed deletion must preserve bucket metadata");
2171 assert!(persisted);
2172 }
2173
2174 #[tokio::test]
2175 #[serial]
2176 async fn bucket_delete_rolls_back_sets_after_partial_failure() {
2177 let (_temp_dir, ecstore) = setup_multi_pool_bucket_test_env().await;
2178 let bucket = format!("delete-set-rollback-{}", Uuid::new_v4().simple());
2179 ecstore
2180 .make_bucket_on_sets(&bucket, &MakeBucketOptions::default())
2181 .await
2182 .expect("bucket should be created in every set before deletion");
2183 let original_set = Arc::clone(&ecstore.pools[0].disk_set[0]);
2184 let offline = take_set_disks_offline(&ecstore, &original_set, &[0, 1]).await;
2185
2186 let err = ecstore
2187 .delete_bucket_on_sets(&bucket, &DeleteBucketOptions::default())
2188 .await
2189 .expect_err("a failed set must fail the complete bucket deletion");
2190 restore_set_disks(&ecstore, &original_set, offline).await;
2191 assert_eq!(err, StorageError::ErasureWriteQuorum);
2192 for pool in &ecstore.pools {
2193 pool.disk_set[0]
2194 .get_bucket_info(&bucket, &BucketOptions::default())
2195 .await
2196 .expect("delete rollback should restore the bucket namespace in every set");
2197 }
2198 }
2199
2200 #[tokio::test]
2201 #[serial]
2202 async fn bucket_config_update_accepts_pre_expansion_bucket() {
2203 let (_temp_dir, ecstore) = setup_multi_pool_bucket_test_env().await;
2204 metadata_sys::init_bucket_metadata_sys(ecstore.clone(), Vec::new()).await;
2205 let bucket = format!("config-pre-expansion-{}", Uuid::new_v4().simple());
2206 ecstore
2207 .make_bucket(&bucket, &MakeBucketOptions::default())
2208 .await
2209 .expect("bucket should be created with authoritative metadata");
2210 ecstore.pools[1].disk_set[0]
2211 .delete_bucket(&bucket, &DeleteBucketOptions::default())
2212 .await
2213 .expect("the expansion pool bucket volume should be removed for the regression state");
2214
2215 let policy = br#"{"Version":"2012-10-17","Statement":[]}"#.to_vec();
2216 ecstore
2217 .update_bucket_metadata_config(&bucket, crate::bucket::metadata::BUCKET_POLICY_CONFIG, policy.clone())
2218 .await
2219 .expect("config mutation should validate existence per erasure set");
2220 let (stored, _) = ecstore
2221 .get_bucket_policy_raw(&bucket)
2222 .await
2223 .expect("updated bucket policy should remain readable");
2224 assert_eq!(stored.as_bytes(), policy);
2225 }
2226
2227 #[tokio::test]
2228 #[serial]
2229 async fn bucket_metadata_init_recreates_pre_expansion_bucket_in_new_pool() {
2230 let (_temp_dir, ecstore) = setup_multi_pool_bucket_test_env().await;
2231 let bucket = format!("metadata-expansion-{}", Uuid::new_v4().simple());
2232 ecstore.pools[0].disk_set[0]
2233 .make_bucket(&bucket, &MakeBucketOptions::default())
2234 .await
2235 .expect("bucket should be created in the original pool only");
2236 assert_eq!(
2237 ecstore.pools[1].disk_set[0]
2238 .get_bucket_info(&bucket, &BucketOptions::default())
2239 .await
2240 .expect_err("new pool should initially have no bucket volume"),
2241 StorageError::VolumeNotFound
2242 );
2243
2244 metadata_sys::init_bucket_metadata_sys(ecstore.clone(), vec![bucket.clone()]).await;
2245
2246 ecstore.pools[1].disk_set[0]
2247 .get_bucket_info(&bucket, &BucketOptions::default())
2248 .await
2249 .expect("metadata initialization should recreate the bucket volume in the new pool");
2250 }
2251
2252 #[tokio::test]
2253 #[serial]
2254 async fn bucket_metadata_init_repairs_half_created_expansion_pool() {
2255 for complete_pool in 0..2 {
2258 let (temp_dir, ecstore) = setup_multi_pool_bucket_test_env().await;
2259 let bucket = format!("partial-expansion-{}", Uuid::new_v4().simple());
2260 for pool_index in 0..2 {
2261 let present_disks = if pool_index == complete_pool { 4 } else { 2 };
2262 for disk_index in 0..present_disks {
2263 tokio::fs::create_dir(
2264 temp_dir
2265 .path()
2266 .join(format!("pool{pool_index}-disk{disk_index}"))
2267 .join(&bucket),
2268 )
2269 .await
2270 .expect("fixture bucket volume should be created");
2271 }
2272 }
2273 assert_eq!(
2274 ecstore
2275 .get_bucket_info_from_sets(&bucket, &BucketOptions::default())
2276 .await
2277 .expect_err("request validation must reject a half-created expansion set"),
2278 StorageError::ErasureWriteQuorum
2279 );
2280
2281 metadata_sys::init_bucket_metadata_sys(ecstore.clone(), vec![bucket.clone()]).await;
2282
2283 for pool_index in 0..2 {
2284 for disk_index in 0..4 {
2285 assert!(
2286 temp_dir
2287 .path()
2288 .join(format!("pool{pool_index}-disk{disk_index}"))
2289 .join(&bucket)
2290 .is_dir(),
2291 "metadata initialization must heal every missing expansion volume"
2292 );
2293 }
2294 }
2295 ecstore
2296 .get_bucket_info_from_sets(&bucket, &BucketOptions::default())
2297 .await
2298 .expect("strict request validation should succeed after volume repair");
2299 }
2300 }
2301
2302 #[tokio::test]
2303 #[serial]
2304 async fn bucket_metadata_init_does_not_combine_partial_set_evidence() {
2305 let (temp_dir, ecstore) = setup_multi_pool_bucket_test_env().await;
2306 let bucket = format!("no-quorum-expansion-{}", Uuid::new_v4().simple());
2307 for pool_index in 0..2 {
2308 for disk_index in 0..2 {
2309 tokio::fs::create_dir(
2310 temp_dir
2311 .path()
2312 .join(format!("pool{pool_index}-disk{disk_index}"))
2313 .join(&bucket),
2314 )
2315 .await
2316 .expect("fixture bucket volume should be created");
2317 }
2318 }
2319 assert_eq!(
2320 ecstore
2321 .bucket_exists_for_heal(&bucket)
2322 .await
2323 .expect_err("repair must require a complete quorum within one set"),
2324 StorageError::ErasureWriteQuorum
2325 );
2326
2327 metadata_sys::init_bucket_metadata_sys(ecstore.clone(), vec![bucket.clone()]).await;
2328
2329 for pool_index in 0..2 {
2330 for disk_index in 0..4 {
2331 assert_eq!(
2332 temp_dir
2333 .path()
2334 .join(format!("pool{pool_index}-disk{disk_index}"))
2335 .join(&bucket)
2336 .is_dir(),
2337 disk_index < 2,
2338 "unproven bucket generations must not recreate missing volumes"
2339 );
2340 }
2341 }
2342 }
2343
2344 #[tokio::test]
2345 #[serial]
2346 async fn bucket_metadata_init_does_not_recreate_stale_bucket_name() {
2347 let (_temp_dir, ecstore) = setup_multi_pool_bucket_test_env().await;
2348 let bucket = format!("stale-expansion-{}", Uuid::new_v4().simple());
2349
2350 metadata_sys::init_bucket_metadata_sys(ecstore.clone(), vec![bucket.clone()]).await;
2351
2352 for pool in &ecstore.pools {
2353 let err = pool.disk_set[0]
2354 .get_bucket_info(&bucket, &BucketOptions::default())
2355 .await
2356 .expect_err("a stale startup listing must not recreate a deleted bucket");
2357 assert_eq!(err, StorageError::VolumeNotFound);
2358 }
2359 }
2360
2361 #[tokio::test]
2362 #[serial]
2363 async fn bucket_info_read_quorum_tracks_erasure_layout() {
2364 for (drive_count, parity) in [(2, 1), (3, 1), (4, 2), (5, 2), (6, 3), (8, 4), (6, 2), (12, 6)] {
2365 let (_temp_dir, store) = setup_bucket_quorum_test_env(&[drive_count], Some(parity)).await;
2366 metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
2367 let bucket = format!("read-quorum-{drive_count}-{parity}");
2368 let object = "uncached-object";
2369 let body = b"erasure read quorum must follow the persisted layout".repeat(32_768);
2370 store
2371 .make_bucket(&bucket, &MakeBucketOptions::default())
2372 .await
2373 .expect("healthy namespace should accept bucket creation");
2374 store
2375 .put_object(&bucket, object, &mut PutObjReader::from_vec(body.clone()), &ObjectOptions::default())
2376 .await
2377 .expect("healthy erasure set should accept the seed object");
2378 let set = &store.pools[0].disk_set[0];
2379 let lock = set
2380 .new_ns_lock(&bucket, object)
2381 .await
2382 .expect("seed namespace lock should resolve");
2383 drop(
2384 lock.get_write_lock(Duration::from_secs(30))
2385 .await
2386 .expect("seed physical fanout must finish before taking disks offline"),
2387 );
2388 if (drive_count, parity) == (6, 3) {
2389 let mut kvs = rustfs_config::server_config::KVS::new();
2390 kvs.insert(crate::config::storageclass::CLASS_STANDARD.to_string(), "EC:2".to_string());
2391 set.set_test_storage_class_config(
2392 crate::config::storageclass::lookup_config_for_pools_without_env(&kvs, &[drive_count])
2393 .expect("a later storage-class change must not raise old objects' read quorum"),
2394 );
2395 }
2396
2397 let offline_indexes = (0..parity).collect::<Vec<_>>();
2398 let offline = take_set_disks_offline(&store, set, &offline_indexes).await;
2399 let info = store
2400 .get_bucket_info(&bucket, &BucketOptions::default())
2401 .await
2402 .expect("bucket validation must admit the object's exact read quorum");
2403 assert_eq!(info.name, bucket);
2404
2405 let mut reader = store
2406 .get_object_reader(&bucket, object, None, Default::default(), &ObjectOptions::default())
2407 .await
2408 .expect("the persisted layout should remain readable at its exact data-shard quorum");
2409 let mut restored = Vec::new();
2410 reader
2411 .stream
2412 .read_to_end(&mut restored)
2413 .await
2414 .expect("quorum read should reconstruct the body");
2415 assert_eq!(restored, body, "layout {drive_count}/{parity} must retain exact object contents");
2416 drop(reader);
2417
2418 if drive_count - parity == drive_count / 2 {
2419 let error = store
2420 .get_bucket_info_from_sets(&bucket, &BucketOptions::default())
2421 .await
2422 .expect_err("bucket mutations must retain their majority namespace check");
2423 assert_eq!(error, StorageError::ErasureWriteQuorum);
2424 }
2425
2426 let below_quorum = take_set_disks_offline(&store, set, &[parity]).await;
2427 let read = store
2428 .get_object_reader(&bucket, object, None, Default::default(), &ObjectOptions::default())
2429 .await;
2430 match read {
2431 Ok(mut reader) => assert!(
2432 reader.stream.read_to_end(&mut Vec::new()).await.is_err(),
2433 "layout {drive_count}/{parity} must reject fewer than its data-shard quorum"
2434 ),
2435 Err(error) => assert!(
2436 matches!(error, StorageError::ErasureReadQuorum | StorageError::InsufficientReadQuorum(_, _)),
2437 "a missing shard must report read quorum loss, got {error}"
2438 ),
2439 }
2440 restore_set_disks(&store, set, below_quorum).await;
2441 restore_set_disks(&store, set, offline).await;
2442 }
2443 }
2444
2445 #[tokio::test]
2446 #[serial]
2447 async fn lazy_metadata_load_uses_read_quorum_for_non_lock_bucket() {
2448 let (_temp_dir, store) = setup_bucket_quorum_test_env(&[4], Some(2)).await;
2449 metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
2450 let bucket = format!("lazy-read-quorum-{}", Uuid::new_v4().simple());
2451 store
2452 .make_bucket(&bucket, &MakeBucketOptions::default())
2453 .await
2454 .expect("healthy namespace should accept bucket creation");
2455
2456 metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
2459 let set = &store.pools[0].disk_set[0];
2460 let offline = take_set_disks_offline(&store, set, &[0, 1]).await;
2461
2462 metadata_sys::get_on_demand_migration_config_in(&store, &bucket)
2463 .await
2464 .expect("cold metadata load should admit the exact namespace read quorum");
2465 assert!(
2466 metadata_sys::get_in(&store.ctx, &bucket).await.is_ok(),
2467 "the lazy load should publish authoritative metadata"
2468 );
2469 assert_eq!(
2470 store
2471 .get_bucket_info_from_sets(&bucket, &BucketOptions::default())
2472 .await
2473 .expect_err("bucket mutations must retain their majority namespace check"),
2474 StorageError::ErasureWriteQuorum
2475 );
2476
2477 restore_set_disks(&store, set, offline).await;
2478 }
2479
2480 #[tokio::test]
2484 #[serial]
2485 async fn cold_bucket_reads_succeed_below_write_quorum() {
2486 let (_temp_dir, store) = setup_bucket_quorum_test_env(&[4], Some(2)).await;
2487 metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
2488 let bucket = format!("cold-read-quorum-{}", Uuid::new_v4().simple());
2489 let object = "seed-object";
2490 let body = b"cold bucket reads must survive lost write quorum".to_vec();
2491 store
2492 .make_bucket(&bucket, &MakeBucketOptions::default())
2493 .await
2494 .expect("healthy namespace should accept bucket creation");
2495 store
2496 .put_object(&bucket, object, &mut PutObjReader::from_vec(body.clone()), &ObjectOptions::default())
2497 .await
2498 .expect("healthy erasure set should accept the seed object");
2499
2500 metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
2501 assert!(
2502 metadata_sys::get_in(&store.ctx, &bucket).await.is_err(),
2503 "the replacement metadata system must start without this bucket"
2504 );
2505 let set = &store.pools[0].disk_set[0];
2506 let lock = set
2507 .new_ns_lock(&bucket, object)
2508 .await
2509 .expect("seed namespace lock should resolve");
2510 drop(
2511 lock.get_write_lock(Duration::from_secs(30))
2512 .await
2513 .expect("seed physical fanout must finish before taking disks offline"),
2514 );
2515 let offline = take_set_disks_offline(&store, set, &[0, 1]).await;
2516
2517 let info = store
2518 .get_bucket_info(&bucket, &BucketOptions::default())
2519 .await
2520 .expect("HEAD bucket must succeed at read quorum before the bucket is cached");
2521 assert_eq!(info.name, bucket);
2522
2523 let mut reader = store
2524 .get_object_reader(&bucket, object, None, Default::default(), &ObjectOptions::default())
2525 .await
2526 .expect("GET must succeed for a bucket that was not loaded at startup");
2527 let mut restored = Vec::new();
2528 reader
2529 .stream
2530 .read_to_end(&mut restored)
2531 .await
2532 .expect("quorum read should reconstruct the body");
2533 assert_eq!(restored, body);
2534 drop(reader);
2535
2536 let object_info = store
2537 .get_object_info(&bucket, object, &ObjectOptions::default())
2538 .await
2539 .expect("HEAD object must succeed at read quorum");
2540 assert_eq!(object_info.name, object);
2541 assert_eq!(object_info.size, i64::try_from(body.len()).expect("body length fits i64"));
2542
2543 let listed = store
2544 .clone()
2545 .list_objects_v2(&bucket, "", None, None, 100, false, None, false)
2546 .await
2547 .expect("List must succeed at read quorum");
2548 assert!(
2549 listed.objects.iter().any(|item| item.name == object),
2550 "list should include the seed object, got {:?}",
2551 listed.objects.iter().map(|item| item.name.clone()).collect::<Vec<_>>()
2552 );
2553 assert!(
2554 metadata_sys::get_in(&store.ctx, &bucket).await.is_ok(),
2555 "the cold load should publish authoritative metadata"
2556 );
2557
2558 let write_error = store
2559 .put_object(&bucket, "rejected-object", &mut PutObjReader::from_vec(body), &ObjectOptions::default())
2560 .await
2561 .expect_err("writes must still fail without write quorum");
2562 assert!(
2563 matches!(
2564 write_error,
2565 StorageError::InsufficientWriteQuorum(_, _) | StorageError::ErasureWriteQuorum
2566 ),
2567 "writes must fail with a write-quorum error, got {write_error}"
2568 );
2569
2570 restore_set_disks(&store, set, offline).await;
2571 }
2572
2573 #[tokio::test]
2574 #[serial]
2575 async fn lazy_metadata_load_rejects_below_read_quorum() {
2576 let (_temp_dir, store) = setup_bucket_quorum_test_env(&[4], Some(2)).await;
2577 metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
2578 let bucket = format!("lazy-below-quorum-{}", Uuid::new_v4().simple());
2579 store
2580 .make_bucket(&bucket, &MakeBucketOptions::default())
2581 .await
2582 .expect("healthy namespace should accept bucket creation");
2583
2584 metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
2585 let set = &store.pools[0].disk_set[0];
2586 let offline = take_set_disks_offline(&store, set, &[0, 1, 2]).await;
2587
2588 let error = metadata_sys::get_on_demand_migration_config_in(&store, &bucket)
2589 .await
2590 .expect_err("cold metadata load must reject read quorum minus one");
2591 assert!(
2592 matches!(error, StorageError::ErasureReadQuorum | StorageError::InsufficientReadQuorum(_, _)),
2593 "below read quorum must fail with a read-quorum error, got {error}"
2594 );
2595 assert!(
2596 metadata_sys::get_in(&store.ctx, &bucket).await.is_err(),
2597 "a below-quorum load must not publish metadata"
2598 );
2599
2600 restore_set_disks(&store, set, offline).await;
2601 }
2602
2603 #[tokio::test]
2604 #[serial]
2605 async fn lazy_metadata_load_keeps_lock_creation_intent_fail_closed() {
2606 let (temp_dir, store) = setup_bucket_quorum_test_env(&[4], Some(2)).await;
2607 metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
2608 let bucket = format!("lazy-lock-intent-{}", Uuid::new_v4().simple());
2609 let mut intent = BucketMetadata::new(&bucket);
2610 intent.lock_enabled = true;
2611 metadata_sys::set_new_bucket_metadata_intent_in(&store.ctx, intent)
2612 .await
2613 .expect("Object Lock creation intent should persist metadata and its incarnation sidecar");
2614
2615 let set = &store.pools[0].disk_set[0];
2616 let offline = take_set_disks_offline(&store, set, &[0, 1]).await;
2617 assert_eq!(
2618 store
2619 .make_bucket_on_sets(&bucket, &MakeBucketOptions::default())
2620 .await
2621 .expect_err("physical creation must not commit below write quorum"),
2622 StorageError::ErasureWriteQuorum
2623 );
2624 assert!(!temp_dir.path().join("pool0-disk0").join(&bucket).exists());
2625 assert!(!temp_dir.path().join("pool0-disk1").join(&bucket).exists());
2626 assert!(temp_dir.path().join("pool0-disk2").join(&bucket).is_dir());
2627 assert!(temp_dir.path().join("pool0-disk3").join(&bucket).is_dir());
2628
2629 metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
2630 let error = metadata_sys::get_on_demand_migration_config_in(&store, &bucket)
2631 .await
2632 .expect_err("Object Lock intent must not become visible at read quorum");
2633 assert_eq!(error, StorageError::ErasureWriteQuorum);
2634 assert!(
2635 metadata_sys::get_in(&store.ctx, &bucket).await.is_err(),
2636 "the rejected Object Lock intent must not enter the metadata cache"
2637 );
2638 assert_eq!(
2639 store
2640 .get_bucket_info(&bucket, &BucketOptions::default())
2641 .await
2642 .expect_err("bucket-info read fallback must also reject a pending Object Lock intent"),
2643 StorageError::ErasureWriteQuorum
2644 );
2645
2646 restore_set_disks(&store, set, offline).await;
2647 }
2648
2649 #[tokio::test]
2650 #[serial]
2651 async fn lazy_metadata_load_admits_committed_lock_bucket_at_read_quorum() {
2652 let (_temp_dir, store) = setup_bucket_quorum_test_env(&[4], Some(2)).await;
2653 metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
2654 let bucket = format!("lazy-committed-lock-{}", Uuid::new_v4().simple());
2655 store
2656 .make_bucket(
2657 &bucket,
2658 &MakeBucketOptions {
2659 lock_enabled: true,
2660 ..Default::default()
2661 },
2662 )
2663 .await
2664 .expect("healthy namespace should accept Object Lock bucket creation");
2665 let (metadata, persisted) = metadata_sys::get_config_from_disk_with_presence_in(&store.ctx, &bucket)
2666 .await
2667 .expect("read committed Object Lock metadata");
2668 assert!(persisted && metadata.bucket_creation_committed);
2669
2670 metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
2671 let set = &store.pools[0].disk_set[0];
2672 let offline = take_set_disks_offline(&store, set, &[0, 1]).await;
2673
2674 metadata_sys::get_on_demand_migration_config_in(&store, &bucket)
2675 .await
2676 .expect("committed Object Lock metadata should load at read quorum");
2677 assert!(
2678 metadata_sys::get_in(&store.ctx, &bucket).await.is_ok(),
2679 "committed Object Lock metadata should be published"
2680 );
2681
2682 restore_set_disks(&store, set, offline).await;
2683 }
2684
2685 #[tokio::test]
2686 #[serial]
2687 async fn creation_commit_backfill_runs_under_the_config_write_fence() {
2688 let (_temp_dir, store) = setup_bucket_quorum_test_env(&[4], Some(2)).await;
2689 metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
2690 let bucket = format!("lazy-commit-backfill-{}", Uuid::new_v4().simple());
2691 let mut intent = BucketMetadata::new(&bucket);
2692 intent.lock_enabled = true;
2693 metadata_sys::set_new_bucket_metadata_intent_in(&store.ctx, intent)
2694 .await
2695 .expect("persist pending Object Lock intent");
2696 store
2697 .make_bucket_on_sets(&bucket, &MakeBucketOptions::default())
2698 .await
2699 .expect("physical creation should commit at full write quorum");
2700
2701 metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
2702 metadata_sys::get_object_lock_config_state_in(&store.ctx, &bucket)
2703 .await
2704 .expect("the fenced authority path should backfill the creation commit");
2705 let (metadata, persisted) = metadata_sys::get_config_from_disk_with_presence_in(&store.ctx, &bucket)
2706 .await
2707 .expect("read backfilled creation commit");
2708 assert!(persisted && metadata.bucket_creation_committed);
2709
2710 metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
2711 let set = &store.pools[0].disk_set[0];
2712 let offline = take_set_disks_offline(&store, set, &[0, 1]).await;
2713 metadata_sys::get_on_demand_migration_config_in(&store, &bucket)
2714 .await
2715 .expect("backfilled Object Lock bucket should load at read quorum");
2716 restore_set_disks(&store, set, offline).await;
2717 }
2718
2719 #[tokio::test]
2720 #[serial]
2721 async fn enabling_object_lock_on_existing_bucket_establishes_commitment() {
2722 let (_temp_dir, store) = setup_bucket_quorum_test_env(&[4], Some(2)).await;
2723 metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
2724 let bucket = format!("enable-lock-legacy-{}", Uuid::new_v4().simple());
2725 store
2726 .make_bucket(&bucket, &MakeBucketOptions::default())
2727 .await
2728 .expect("create a versioned non-lock bucket");
2729
2730 let mut legacy = metadata_sys::get_in(&store.ctx, &bucket)
2732 .await
2733 .expect("cached metadata")
2734 .as_ref()
2735 .clone();
2736 legacy.lock_enabled = false;
2737 legacy.object_lock_config_xml = Vec::new();
2738 legacy.object_lock_config = None;
2739 legacy.bucket_creation_committed = false;
2740 metadata_sys::set_new_bucket_metadata_intent_in(&store.ctx, legacy)
2741 .await
2742 .expect("persist legacy-shaped metadata");
2743
2744 let lock_xml = crate::bucket::utils::serialize(&*crate::store::ENABLED_OBJECT_LOCK_CONFIG)
2745 .expect("serialize Object Lock configuration");
2746 metadata_sys::update_in(&store.ctx, &bucket, "object-lock.xml", lock_xml)
2747 .await
2748 .expect("enabling Object Lock on an existing bucket must succeed");
2749
2750 assert!(
2751 matches!(
2752 metadata_sys::get_object_lock_config_state_in(&store.ctx, &bucket)
2753 .await
2754 .expect("read the freshly enabled Object Lock state"),
2755 metadata_sys::ObjectLockConfigState::Configured { .. }
2756 ),
2757 "a healthy cluster must not report ErasureWriteQuorum after enabling Object Lock"
2758 );
2759 let (metadata, persisted) = metadata_sys::get_config_from_disk_with_presence_in(&store.ctx, &bucket)
2760 .await
2761 .expect("read persisted metadata");
2762 assert!(persisted && metadata.bucket_creation_committed);
2763 }
2764
2765 #[tokio::test]
2766 #[serial]
2767 async fn creation_commit_backfill_preserves_concurrent_config_update() {
2768 let (_temp_dir, store) = setup_bucket_quorum_test_env(&[4], Some(2)).await;
2769 metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
2770 let bucket = format!("commit-backfill-pack-{}", Uuid::new_v4().simple());
2771 store
2772 .make_bucket(&bucket, &MakeBucketOptions::default())
2773 .await
2774 .expect("create bucket");
2775
2776 let mut stale = metadata_sys::get_in(&store.ctx, &bucket)
2779 .await
2780 .expect("cached metadata")
2781 .as_ref()
2782 .clone();
2783 stale.bucket_creation_committed = false;
2784 metadata_sys::set_new_bucket_metadata_intent_in(&store.ctx, stale.clone())
2785 .await
2786 .expect("persist the stale pre-upgrade snapshot");
2787
2788 let mut newer = stale.clone();
2791 newer.lock_enabled = true;
2792 newer.object_lock_config_xml = crate::bucket::utils::serialize(&*crate::store::ENABLED_OBJECT_LOCK_CONFIG)
2793 .expect("serialize Object Lock configuration");
2794 newer
2795 .save_with_store(store.clone())
2796 .await
2797 .expect("persist the newer configuration");
2798
2799 metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
2802 metadata_sys::get_object_lock_config_state_in(&store.ctx, &bucket)
2803 .await
2804 .expect("fenced commit migration");
2805
2806 let (metadata, persisted) = metadata_sys::get_config_from_disk_with_presence_in(&store.ctx, &bucket)
2807 .await
2808 .expect("read persisted metadata");
2809 assert!(persisted && metadata.bucket_creation_committed);
2810 assert!(
2811 metadata.lock_enabled && !metadata.object_lock_config_xml.is_empty(),
2812 "the acknowledged Object Lock update must survive the backfill"
2813 );
2814 }
2815
2816 #[tokio::test]
2817 #[serial]
2818 async fn creation_commit_backfill_keeps_recreated_generation() {
2819 let (_temp_dir, store) = setup_bucket_quorum_test_env(&[4], Some(2)).await;
2820 metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
2821 let bucket = format!("commit-backfill-recreate-{}", Uuid::new_v4().simple());
2822 store
2823 .make_bucket(&bucket, &MakeBucketOptions::default())
2824 .await
2825 .expect("create bucket");
2826 let old_incarnation = store.bucket_incarnation_id_from_disk(&bucket).await.expect("old incarnation");
2827
2828 let mut stale = metadata_sys::get_in(&store.ctx, &bucket)
2831 .await
2832 .expect("cached metadata")
2833 .as_ref()
2834 .clone();
2835 stale.lock_enabled = true;
2836 stale.object_lock_config_xml = crate::bucket::utils::serialize(&*crate::store::ENABLED_OBJECT_LOCK_CONFIG)
2837 .expect("serialize Object Lock configuration");
2838 stale.bucket_creation_committed = false;
2839 stale.save_with_store(store.clone()).await.expect("persist stale metadata");
2840
2841 store
2842 .delete_bucket(&bucket, &crate::storage_api_contracts::bucket::DeleteBucketOptions::default())
2843 .await
2844 .expect("delete bucket");
2845 store
2846 .make_bucket(
2847 &bucket,
2848 &MakeBucketOptions {
2849 lock_enabled: true,
2850 ..Default::default()
2851 },
2852 )
2853 .await
2854 .expect("recreate bucket with Object Lock");
2855 let new_incarnation = store.bucket_incarnation_id_from_disk(&bucket).await.expect("new incarnation");
2856 assert_ne!(old_incarnation, new_incarnation);
2857
2858 metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
2859 metadata_sys::get_object_lock_config_state_in(&store.ctx, &bucket)
2860 .await
2861 .expect("fenced backfill on the replacement generation");
2862
2863 let (metadata, persisted) = metadata_sys::get_config_from_disk_with_presence_in(&store.ctx, &bucket)
2864 .await
2865 .expect("read persisted metadata");
2866 assert!(persisted);
2867 assert_eq!(
2868 metadata.bucket_incarnation_id, new_incarnation,
2869 "the replacement generation must stay authoritative"
2870 );
2871 assert!(metadata.bucket_creation_committed);
2872 }
2873
2874 #[tokio::test]
2875 #[serial]
2876 async fn bucket_info_read_quorum_is_scoped_to_each_erasure_set() {
2877 let (_temp_dir, store) = setup_bucket_quorum_test_env(&[4, 6], None).await;
2878 metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
2879 let bucket = "read-quorum-mixed-pools";
2880 store
2881 .make_bucket(bucket, &MakeBucketOptions::default())
2882 .await
2883 .expect("healthy pools should accept bucket creation");
2884
2885 let first_set = &store.pools[0].disk_set[0];
2886 let second_set = &store.pools[1].disk_set[0];
2887 let first_offline = take_set_disks_offline(&store, first_set, &[0, 1]).await;
2888 let second_offline = take_set_disks_offline(&store, second_set, &[0, 1, 2]).await;
2889 store
2890 .get_bucket_info(bucket, &BucketOptions::default())
2891 .await
2892 .expect("each set independently satisfies its namespace read quorum");
2893
2894 for (set, extra_disk) in [(first_set, 2), (second_set, 3)] {
2895 let extra_offline = take_set_disks_offline(&store, set, &[extra_disk]).await;
2896 assert_eq!(
2897 store
2898 .get_bucket_info(bucket, &BucketOptions::default())
2899 .await
2900 .expect_err("another pool must not subsidize a set below its read quorum"),
2901 StorageError::ErasureReadQuorum
2902 );
2903 restore_set_disks(&store, set, extra_offline).await;
2904 }
2905 restore_set_disks(&store, first_set, first_offline).await;
2906 restore_set_disks(&store, second_set, second_offline).await;
2907 }
2908
2909 #[tokio::test]
2910 #[serial]
2911 async fn bucket_info_read_quorum_requires_authoritative_metadata() {
2912 for state in ["missing", "corrupt", "foreign", "incarnation"] {
2913 let (_temp_dir, store) = setup_bucket_quorum_test_env(&[4], None).await;
2914 metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
2915 let bucket = format!("read-quorum-{state}-metadata");
2916 let mut metadata = if state == "missing" {
2917 store
2918 .make_bucket_on_sets(&bucket, &MakeBucketOptions::default())
2919 .await
2920 .expect("simulate directories left before bucket metadata is published");
2921 BucketMetadata::new(&bucket)
2922 } else {
2923 store
2924 .make_bucket(&bucket, &MakeBucketOptions::default())
2925 .await
2926 .expect("healthy bucket should publish metadata");
2927 metadata_sys::get_in(&store.ctx, &bucket)
2928 .await
2929 .expect("seed metadata should be cached")
2930 .as_ref()
2931 .clone()
2932 };
2933 let path = metadata.save_file_path();
2934 match state {
2935 "corrupt" => crate::config::com::save_config(store.clone(), &path, b"corrupt".to_vec())
2936 .await
2937 .expect("persist corrupt metadata while the cached copy remains valid"),
2938 "foreign" => {
2939 metadata.name = "different-bucket".to_string();
2940 let mut encoded = vec![1, 0, 1, 0];
2941 encoded.extend(metadata.marshal_msg().expect("foreign metadata should encode"));
2942 crate::config::com::save_config(store.clone(), &path, encoded)
2943 .await
2944 .expect("persist metadata for a different bucket at the requested path");
2945 }
2946 "incarnation" => crate::bucket::metadata::save_bucket_incarnation(store.clone(), &bucket, Uuid::new_v4())
2947 .await
2948 .expect("persist a different bucket generation"),
2949 _ => {}
2950 }
2951
2952 let set = &store.pools[0].disk_set[0];
2953 let offline = take_set_disks_offline(&store, set, &[0, 1]).await;
2954 let error = store
2955 .get_bucket_info(&bucket, &BucketOptions::default())
2956 .await
2957 .expect_err("read admission must not trust residual directories or cached metadata");
2958 match state {
2959 "missing" => assert_eq!(error, StorageError::ErasureReadQuorum),
2960 "foreign" => assert_eq!(error, StorageError::FileCorrupt),
2961 "incarnation" => assert!(error.to_string().contains("sidecar does not match bucket metadata")),
2962 "corrupt" => assert!(error.to_string().contains("format invalid"), "unexpected corruption error: {error}"),
2963 _ => unreachable!(),
2964 }
2965 restore_set_disks(&store, set, offline).await;
2966 }
2967 }
2968
2969 #[tokio::test]
2970 #[serial]
2971 async fn bucket_info_read_quorum_accepts_persisted_legacy_metadata() {
2972 let (_temp_dir, store) = setup_bucket_quorum_test_env(&[4], None).await;
2973 metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
2974 let bucket = "interop";
2975 store
2976 .make_bucket_on_sets(bucket, &MakeBucketOptions::default())
2977 .await
2978 .expect("legacy bucket directories should exist");
2979 let hex = include_str!("../../tests/fixtures/minio/bucket_metadata.blob.hex")
2980 .split_whitespace()
2981 .collect::<String>();
2982 let body = (0..hex.len())
2983 .step_by(2)
2984 .map(|index| u8::from_str_radix(&hex[index..index + 2], 16).expect("pinned MinIO metadata fixture"))
2985 .collect();
2986 crate::config::com::save_config(store.clone(), &BucketMetadata::new(bucket).save_file_path(), body)
2987 .await
2988 .expect("legacy metadata should be persisted without an incarnation sidecar");
2989
2990 let set = &store.pools[0].disk_set[0];
2991 let offline = take_set_disks_offline(&store, set, &[0, 1]).await;
2992 let info = store
2993 .get_bucket_info(bucket, &BucketOptions::default())
2994 .await
2995 .expect("persisted MinIO metadata should authorize reads at the namespace read quorum");
2996 assert_eq!(info.name, bucket);
2997 assert!(info.versioning);
2998 assert!(info.object_locking);
2999 restore_set_disks(&store, set, offline).await;
3000 }
3001
3002 #[tokio::test]
3003 #[serial]
3004 async fn bucket_namespace_reads_report_missing_when_every_set_is_absent() {
3005 let (_temp_dir, ecstore) = setup_multi_pool_bucket_test_env().await;
3006 let bucket = format!("absent-{}", Uuid::new_v4().simple());
3007
3008 let buckets = ecstore
3009 .list_bucket(&BucketOptions::default())
3010 .await
3011 .expect("an empty healthy namespace should remain listable");
3012 assert!(buckets.iter().all(|entry| entry.name != bucket));
3013
3014 let err = ecstore
3015 .get_bucket_info(&bucket, &BucketOptions::default())
3016 .await
3017 .expect_err("a bucket absent from every set must remain missing");
3018 assert_eq!(err, StorageError::VolumeNotFound);
3019 }
3020
3021 #[tokio::test]
3022 #[serial]
3023 async fn bucket_namespace_reads_fail_closed_when_any_set_loses_quorum() {
3024 let (_temp_dir, ecstore) = setup_multi_pool_bucket_test_env().await;
3025 metadata_sys::init_bucket_metadata_sys(ecstore.clone(), Vec::new()).await;
3026 let bucket = format!("degraded-expansion-{}", Uuid::new_v4().simple());
3027 ecstore.pools[0].disk_set[0]
3028 .make_bucket(&bucket, &MakeBucketOptions::default())
3029 .await
3030 .expect("bucket should be created in the original pool only");
3031 ecstore.pools[1].disk_set[0].disks.write().await[0] = None;
3032 ecstore.pools[1].disk_set[0].disks.write().await[1] = None;
3033 ecstore.pools[1].disk_set[0].disks.write().await[2] = None;
3034
3035 let list_err = ecstore
3036 .list_bucket(&BucketOptions::default())
3037 .await
3038 .expect_err("listing must not hide an unavailable expansion pool");
3039 assert_eq!(list_err, StorageError::ErasureWriteQuorum);
3040
3041 let info_err = ecstore
3042 .get_bucket_info(&bucket, &BucketOptions::default())
3043 .await
3044 .expect_err("bucket validation must fail when an expansion pool is unavailable");
3045 assert_eq!(info_err, StorageError::ErasureReadQuorum);
3046 }
3047
3048 #[tokio::test]
3049 #[serial]
3050 async fn scanner_bucket_listing_unions_every_erasure_set() {
3051 let (_temp_dir, ecstore) = setup_multi_pool_bucket_test_env().await;
3052 let bucket = format!("second-pool-only-{}", Uuid::new_v4().simple());
3053 ecstore.pools[1].disk_set[0]
3054 .make_bucket(&bucket, &MakeBucketOptions::default())
3055 .await
3056 .expect("bucket should be created in the second pool only");
3057
3058 let listing = ecstore
3059 .list_bucket_for_scanner(&crate::storage_api_contracts::bucket::BucketOptions {
3060 no_metadata: true,
3061 ..Default::default()
3062 })
3063 .await
3064 .expect("scanner should enumerate every pool and set");
3065
3066 assert!(listing.topology_complete);
3067 assert!(listing.buckets.iter().any(|entry| entry.name == bucket));
3068 assert_eq!(listing.set_buckets.len(), 2);
3069 assert!(
3070 listing
3071 .set_buckets
3072 .iter()
3073 .find(|scope| scope.pool_index == 0 && scope.set_index == 0)
3074 .is_some_and(|scope| scope.buckets.is_empty())
3075 );
3076 assert!(
3077 listing
3078 .set_buckets
3079 .iter()
3080 .find(|scope| scope.pool_index == 1 && scope.set_index == 0)
3081 .is_some_and(|scope| scope.buckets.iter().any(|entry| entry.name == bucket))
3082 );
3083 }
3084
3085 #[tokio::test]
3086 async fn scanner_bucket_listing_marks_degraded_set_incomplete() {
3087 let (_temp_dir, ecstore) = setup_multi_pool_bucket_test_env().await;
3088 let bucket = format!("degraded-set-{}", Uuid::new_v4().simple());
3089 let set = &ecstore.pools[0].disk_set[0];
3090 set.make_bucket(&bucket, &MakeBucketOptions::default())
3091 .await
3092 .expect("bucket should be created before a disk is removed");
3093 set.disks.write().await[0] = None;
3094
3095 let listing = ecstore
3096 .list_bucket_for_scanner(&crate::storage_api_contracts::bucket::BucketOptions {
3097 no_metadata: true,
3098 ..Default::default()
3099 })
3100 .await
3101 .expect("a degraded set with quorum should still return candidate buckets");
3102
3103 assert!(listing.buckets.iter().any(|entry| entry.name == bucket));
3104 assert!(!listing.topology_complete);
3105 }
3106
3107 #[tokio::test]
3108 async fn scanner_bucket_listing_marks_divergent_disk_views_incomplete() {
3109 let (temp_dir, ecstore) = setup_multi_pool_bucket_test_env().await;
3110 let bucket = format!("divergent-set-{}", Uuid::new_v4().simple());
3111 ecstore.pools[0].disk_set[0]
3112 .make_bucket(&bucket, &MakeBucketOptions::default())
3113 .await
3114 .expect("bucket should be created before disk views diverge");
3115 for disk_index in 0..2 {
3116 tokio::fs::remove_dir_all(temp_dir.path().join(format!("pool0-disk{disk_index}")).join(&bucket))
3117 .await
3118 .expect("test bucket directory should be removed from a minority disk view");
3119 }
3120
3121 let listing = ecstore
3122 .list_bucket_for_scanner(&crate::storage_api_contracts::bucket::BucketOptions {
3123 no_metadata: true,
3124 ..Default::default()
3125 })
3126 .await
3127 .expect("responsive disks should still produce a scanner candidate listing");
3128
3129 assert!(listing.buckets.iter().all(|entry| entry.name != bucket));
3130 assert!(!listing.topology_complete);
3131 }
3132
3133 #[tokio::test]
3136 #[serial]
3137 async fn bucket_delete_mark_delete_removes_empty_bucket_and_keeps_deleted_marker() {
3138 let (disk_paths, ecstore) = setup_bucket_delete_test_env().await;
3139 let bucket = format!("bucket-mark-delete-{}", Uuid::new_v4().simple());
3140
3141 ecstore
3142 .make_bucket(&bucket, &MakeBucketOptions::default())
3143 .await
3144 .expect("bucket should be created");
3145 for disk_path in &disk_paths {
3146 tokio::fs::create_dir_all(disk_path.join(&bucket).join("empty-directory/nested/leaf"))
3147 .await
3148 .expect("empty directory remnant should be created");
3149 }
3150 assert!(metadata_sys::get_in(&ecstore.ctx, &bucket).await.is_ok());
3151
3152 let generation_before_delete = ecstore.scanner_namespace_mutation_generation();
3153 ecstore
3154 .delete_bucket(
3155 &bucket,
3156 &DeleteBucketOptions {
3157 srdelete_op: SRBucketDeleteOp::MarkDelete,
3158 ..Default::default()
3159 },
3160 )
3161 .await
3162 .expect("MarkDelete should remove an empty bucket");
3163 assert_eq!(
3164 ecstore.scanner_namespace_mutation_generation(),
3165 generation_before_delete.saturating_add(2),
3166 "successful bucket deletion should fence scanner namespace activity before and after deletion"
3167 );
3168
3169 assert!(
3170 !any_disk_path_exists(&disk_paths, &bucket).await,
3171 "MarkDelete should remove the bucket volume"
3172 );
3173 assert!(
3174 any_disk_path_exists(&disk_paths, bucket_deleted_marker_volume(&bucket)).await,
3175 "MarkDelete should persist the deleted-bucket marker"
3176 );
3177 assert!(
3178 metadata_sys::get_in(&ecstore.ctx, &bucket).await.is_err(),
3179 "deleted bucket metadata must be removed from the local cache"
3180 );
3181 let buckets = ecstore
3182 .list_bucket(&BucketOptions::default())
3183 .await
3184 .expect("bucket listing should succeed after MarkDelete");
3185 assert!(!buckets.iter().any(|info| info.name == bucket));
3186 ecstore
3187 .delete_all(RUSTFS_META_BUCKET, &bucket_deleted_marker_prefix(&bucket))
3188 .await
3189 .expect("deleted-bucket marker should be removed to simulate a partial failure");
3190 assert!(!any_disk_path_exists(&disk_paths, bucket_deleted_marker_volume(&bucket)).await);
3191 ecstore
3192 .delete_bucket(
3193 &bucket,
3194 &DeleteBucketOptions {
3195 srdelete_op: SRBucketDeleteOp::MarkDelete,
3196 ..Default::default()
3197 },
3198 )
3199 .await
3200 .expect("retried MarkDelete should recreate a missing tombstone");
3201 assert!(any_disk_path_exists(&disk_paths, bucket_deleted_marker_volume(&bucket)).await);
3202 }
3203
3204 #[tokio::test]
3205 #[serial]
3206 async fn bucket_delete_mark_delete_rejects_non_empty_bucket_without_force() {
3207 let (disk_paths, ecstore) = setup_bucket_delete_test_env().await;
3208 let bucket = format!("bucket-mark-delete-non-empty-{}", Uuid::new_v4().simple());
3209
3210 create_bucket_with_object(&ecstore, &bucket, "object.txt").await;
3211
3212 let err = ecstore
3213 .delete_bucket(
3214 &bucket,
3215 &DeleteBucketOptions {
3216 srdelete_op: SRBucketDeleteOp::MarkDelete,
3217 ..Default::default()
3218 },
3219 )
3220 .await
3221 .expect_err("MarkDelete should reject a non-empty bucket without force");
3222
3223 assert!(matches!(err, StorageError::BucketNotEmpty(name) if name == bucket));
3224 assert!(any_disk_has_object_metadata(&disk_paths, &bucket).await);
3225 assert!(!any_disk_path_exists(&disk_paths, bucket_deleted_marker_volume(&bucket)).await);
3226 }
3227
3228 #[tokio::test]
3229 #[serial]
3230 async fn bucket_delete_mark_delete_rejects_hidden_object_paths_without_force() {
3231 let (disk_paths, ecstore) = setup_bucket_delete_test_env().await;
3232 let bucket = format!("bucket-mark-delete-hidden-{}", Uuid::new_v4().simple());
3233
3234 create_bucket_with_object(&ecstore, &bucket, ".well-known/acme-challenge").await;
3235 let mut reader = PutObjReader::from_vec(b"delete bucket semantics".to_vec());
3236 ecstore
3237 .put_object(&bucket, ".rustfs.sys/object", &mut reader, &ObjectOptions::default())
3238 .await
3239 .expect("second hidden object should be written");
3240
3241 let err = ecstore
3242 .delete_bucket(
3243 &bucket,
3244 &DeleteBucketOptions {
3245 srdelete_op: SRBucketDeleteOp::MarkDelete,
3246 ..Default::default()
3247 },
3248 )
3249 .await
3250 .expect_err("MarkDelete should reject a hidden object path without force");
3251
3252 assert!(matches!(err, StorageError::BucketNotEmpty(name) if name == bucket));
3253 assert!(any_disk_has_object_metadata(&disk_paths, &bucket).await);
3254 }
3255
3256 #[tokio::test]
3257 #[serial]
3258 async fn bucket_delete_purge_removes_bucket_data_and_internal_metadata() {
3259 let (disk_paths, ecstore) = setup_bucket_delete_test_env().await;
3260 let bucket = format!("bucket-purge-{}", Uuid::new_v4().simple());
3261 let object = "object.txt";
3262 let metadata_prefix = format!("{RUSTFS_META_BUCKET}/{BUCKET_META_PREFIX}/{bucket}");
3263
3264 create_bucket_with_object(&ecstore, &bucket, object).await;
3265 write_bucket_metadata_marker(&disk_paths, &metadata_prefix).await;
3266 ecstore
3267 .mark_bucket_deleted(&bucket)
3268 .await
3269 .expect("deleted-bucket marker should be created");
3270 assert!(any_disk_path_exists(&disk_paths, &metadata_prefix).await);
3271 assert!(any_disk_path_exists(&disk_paths, bucket_deleted_marker_volume(&bucket)).await);
3272
3273 let generation_before_delete = ecstore.scanner_namespace_mutation_generation();
3274 ecstore
3275 .delete_bucket(
3276 &bucket,
3277 &DeleteBucketOptions {
3278 force: true,
3279 srdelete_op: SRBucketDeleteOp::Purge,
3280 ..Default::default()
3281 },
3282 )
3283 .await
3284 .expect("Purge should force-delete bucket data");
3285 assert_eq!(
3286 ecstore.scanner_namespace_mutation_generation(),
3287 generation_before_delete.saturating_add(2),
3288 "successful bucket purge should fence scanner namespace activity before and after deletion"
3289 );
3290
3291 assert!(!any_disk_path_exists(&disk_paths, &bucket).await, "Purge should remove the bucket volume");
3292 assert!(
3293 !any_disk_path_exists(&disk_paths, &metadata_prefix).await,
3294 "Purge should remove bucket metadata prefix"
3295 );
3296 assert!(
3297 !any_disk_path_exists(&disk_paths, bucket_deleted_marker_volume(&bucket)).await,
3298 "Purge should remove the deleted-bucket marker"
3299 );
3300 assert!(
3301 metadata_sys::get_in(&ecstore.ctx, &bucket).await.is_err(),
3302 "purged bucket metadata must be removed from the local cache"
3303 );
3304 write_bucket_metadata_marker(&disk_paths, &metadata_prefix).await;
3305 ecstore
3306 .mark_bucket_deleted(&bucket)
3307 .await
3308 .expect("stale deleted-bucket marker should be recreated");
3309 ecstore
3310 .delete_bucket(
3311 &bucket,
3312 &DeleteBucketOptions {
3313 force: true,
3314 srdelete_op: SRBucketDeleteOp::Purge,
3315 ..Default::default()
3316 },
3317 )
3318 .await
3319 .expect("retried Purge should remove stale metadata without a bucket volume");
3320 assert!(!any_disk_path_exists(&disk_paths, &metadata_prefix).await);
3321 assert!(!any_disk_path_exists(&disk_paths, bucket_deleted_marker_volume(&bucket)).await);
3322 }
3323
3324 #[tokio::test(flavor = "multi_thread")]
3330 #[serial]
3331 async fn force_create_enables_object_lock_on_an_existing_bucket() {
3332 let (_disk_paths, ecstore) = setup_bucket_delete_test_env().await;
3333 let bucket = format!("bucket-force-lock-{}", Uuid::new_v4().simple());
3334
3335 ecstore
3336 .make_bucket(&bucket, &MakeBucketOptions::default())
3337 .await
3338 .expect("plain bucket should be created");
3339 assert!(
3340 !metadata_sys::get_in(&ecstore.ctx, &bucket)
3341 .await
3342 .expect("metadata should load")
3343 .lock_enabled,
3344 "test setup: the bucket must start unlocked"
3345 );
3346
3347 ecstore
3348 .make_bucket(
3349 &bucket,
3350 &MakeBucketOptions {
3351 force_create: true,
3352 lock_enabled: true,
3353 ..Default::default()
3354 },
3355 )
3356 .await
3357 .expect("force create with lock_enabled should succeed on an existing bucket");
3358
3359 let meta = metadata_sys::get_in(&ecstore.ctx, &bucket)
3360 .await
3361 .expect("metadata should load after the lock enable");
3362 assert!(meta.lock_enabled, "Object Lock must be enabled on the existing bucket");
3363 assert!(
3364 !meta.object_lock_config_xml.is_empty(),
3365 "the Object Lock configuration must be persisted, not just the flag"
3366 );
3367 assert!(
3368 !meta.versioning_config_xml.is_empty(),
3369 "Object Lock requires versioning, so that must be persisted too"
3370 );
3371 }
3372
3373 #[tokio::test(flavor = "multi_thread")]
3378 #[serial]
3379 async fn issue_8003_force_create_adopts_existing_incarnation_sidecar() {
3380 let (disk_paths, ecstore) = setup_bucket_delete_test_env().await;
3381 let bucket = format!("bucket-8003-sidecar-{}", Uuid::new_v4().simple());
3382
3383 ecstore
3384 .make_bucket(&bucket, &MakeBucketOptions::default())
3385 .await
3386 .expect("bucket should be created");
3387 let stored = metadata_sys::get_in(&ecstore.ctx, &bucket)
3388 .await
3389 .expect("metadata should load")
3390 .bucket_incarnation_id;
3391 assert!(!stored.is_nil());
3392
3393 let metadata_path = format!("{BUCKET_META_PREFIX}/{bucket}/{}", crate::bucket::metadata::BUCKET_METADATA_FILE);
3394 crate::config::com::delete_config(ecstore.clone(), &metadata_path)
3395 .await
3396 .expect("simulate the interrupted migration by removing only .metadata.bin");
3397 assert!(!any_disk_path_exists(&disk_paths, format!("{RUSTFS_META_BUCKET}/{metadata_path}")).await);
3398 metadata_sys::remove_bucket_metadata_in(&ecstore.ctx, &bucket)
3399 .await
3400 .expect("drop the cached copy so the next read hits disk");
3401
3402 ecstore
3403 .make_bucket(
3404 &bucket,
3405 &MakeBucketOptions {
3406 force_create: true,
3407 ..Default::default()
3408 },
3409 )
3410 .await
3411 .expect("force create must repair the sidecar-only bucket");
3412
3413 let (repaired, persisted) = metadata_sys::get_config_from_disk_with_presence_in(&ecstore.ctx, &bucket)
3414 .await
3415 .expect("repaired metadata should load");
3416 assert!(persisted, "force create must persist .metadata.bin again");
3417 assert!(repaired.bucket_incarnation_sidecar);
3418 assert_eq!(
3419 repaired.bucket_incarnation_id, stored,
3420 "the repair must keep the incarnation the sidecar already published"
3421 );
3422 }
3423
3424 #[tokio::test(flavor = "multi_thread")]
3425 #[serial]
3426 async fn make_bucket_inherits_default_durability() {
3427 temp_env::async_with_vars([(crate::bucket::durability::ENV_NEW_BUCKET_DURABILITY_MODE, None::<&str>)], async {
3428 let (_disk_paths, ecstore) = setup_bucket_delete_test_env().await;
3429 let bucket = format!("bucket-default-durability-{}", Uuid::new_v4().simple());
3430
3431 ecstore
3432 .make_bucket(&bucket, &MakeBucketOptions::default())
3433 .await
3434 .expect("new bucket should be created");
3435
3436 let metadata = metadata_sys::get_in(&ecstore.ctx, &bucket)
3437 .await
3438 .expect("metadata should load for the new bucket");
3439 assert!(metadata.durability_config().is_none());
3440 })
3441 .await;
3442 }
3443
3444 #[tokio::test(flavor = "multi_thread")]
3445 #[serial]
3446 async fn force_create_existing_bucket_keeps_durability_override() {
3447 let (_disk_paths, ecstore) = setup_bucket_delete_test_env().await;
3448 let bucket = format!("bucket-force-durability-{}", Uuid::new_v4().simple());
3449
3450 temp_env::async_with_vars([(crate::bucket::durability::ENV_NEW_BUCKET_DURABILITY_MODE, Some("inherit"))], async {
3451 ecstore
3452 .make_bucket(&bucket, &MakeBucketOptions::default())
3453 .await
3454 .expect("plain bucket should be created without a durability override");
3455 })
3456 .await;
3457 assert!(
3458 metadata_sys::get_in(&ecstore.ctx, &bucket)
3459 .await
3460 .expect("metadata should load after initial create")
3461 .durability_config()
3462 .is_none(),
3463 "test setup: the existing bucket must start without an override"
3464 );
3465
3466 temp_env::async_with_vars([(crate::bucket::durability::ENV_NEW_BUCKET_DURABILITY_MODE, None::<&str>)], async {
3467 ecstore
3468 .make_bucket(
3469 &bucket,
3470 &MakeBucketOptions {
3471 force_create: true,
3472 lock_enabled: true,
3473 ..Default::default()
3474 },
3475 )
3476 .await
3477 .expect("force create should update existing bucket metadata");
3478 })
3479 .await;
3480
3481 let metadata = metadata_sys::get_in(&ecstore.ctx, &bucket)
3482 .await
3483 .expect("metadata should load after force create");
3484 assert!(metadata.lock_enabled, "force create sanity check: Object Lock should be enabled");
3485 assert!(
3486 metadata.durability_config().is_none(),
3487 "force create must not apply the new-bucket default to existing bucket metadata"
3488 );
3489 }
3490
3491 #[tokio::test(flavor = "multi_thread")]
3497 #[serial]
3498 async fn bucket_delete_succeeds_after_the_last_object_version_is_deleted() {
3499 let (disk_paths, ecstore) = setup_bucket_delete_test_env().await;
3500 let bucket = format!("bucket-delete-after-drain-{}", Uuid::new_v4().simple());
3501 let object = "object.txt";
3502
3503 create_bucket_with_object(&ecstore, &bucket, object).await;
3504
3505 ecstore
3506 .delete_object(&bucket, object, ObjectOptions::default())
3507 .await
3508 .expect("client delete of the only object should succeed");
3509
3510 assert!(
3511 !any_disk_has_object_metadata(&disk_paths, &bucket).await,
3512 "deleting the last version must not leave xl.meta on disk: DeleteBucket scans the raw \
3513 bucket directory, so residue here is reported to clients as BucketNotEmpty"
3514 );
3515
3516 ecstore
3517 .delete_bucket(&bucket, &DeleteBucketOptions::default())
3518 .await
3519 .expect("DeleteBucket must succeed once the client has drained the bucket");
3520 }
3521
3522 #[tokio::test(flavor = "multi_thread")]
3523 #[serial]
3524 async fn bucket_delete_defers_zero_evidence_diagnostic_timeout_to_physical_empty_check() {
3525 let (disk_paths, ecstore) = setup_bucket_delete_test_env().await;
3526 let bucket = format!("bucket-del-diag-{}", Uuid::new_v4().simple());
3527
3528 ecstore
3529 .make_bucket(&bucket, &MakeBucketOptions::default())
3530 .await
3531 .expect("empty bucket should be created");
3532 let first_io_started = Arc::new(AtomicBool::new(false));
3533 let diagnostic_budget = BucketDeleteDiagnosticBudget::new().with_first_io_delay(
3534 BUCKET_DELETE_DIAGNOSTIC_MAX_ELAPSED + Duration::from_millis(100),
3535 first_io_started.clone(),
3536 );
3537
3538 ecstore
3539 .handle_delete_bucket_with_diagnostic_budget(&bucket, &DeleteBucketOptions::default(), diagnostic_budget)
3540 .await
3541 .expect("a zero-evidence diagnostic timeout must defer to the physical empty check");
3542
3543 assert!(
3544 first_io_started.load(Ordering::SeqCst),
3545 "the regression must delay the first diagnostic read_dir"
3546 );
3547 assert!(
3548 !any_disk_path_exists(&disk_paths, &bucket).await,
3549 "the physical empty check should allow deletion of the actually empty bucket"
3550 );
3551 }
3552
3553 #[tokio::test(flavor = "multi_thread")]
3554 #[serial]
3555 async fn bucket_delete_preserves_unobserved_residue_after_diagnostic_timeout() {
3556 let (disk_paths, ecstore) = setup_bucket_delete_test_env().await;
3557 let bucket = format!("bucket-del-residue-{}", Uuid::new_v4().simple());
3558 let object = "object.txt";
3559
3560 ecstore
3561 .make_bucket(&bucket, &MakeBucketOptions::default())
3562 .await
3563 .expect("bucket should be created");
3564 write_metadata_less_part_on_all_disks(&disk_paths, &bucket, object).await;
3565 let first_io_started = Arc::new(AtomicBool::new(false));
3566 let diagnostic_budget = BucketDeleteDiagnosticBudget::new().with_first_io_delay(
3567 BUCKET_DELETE_DIAGNOSTIC_MAX_ELAPSED + Duration::from_millis(100),
3568 first_io_started.clone(),
3569 );
3570
3571 let err = ecstore
3572 .handle_delete_bucket_with_diagnostic_budget(&bucket, &DeleteBucketOptions::default(), diagnostic_budget)
3573 .await
3574 .expect_err("physical empty-bucket enforcement must reject unobserved residue");
3575 assert!(
3576 first_io_started.load(Ordering::SeqCst),
3577 "the regression must time out before observing the residue"
3578 );
3579 match &err {
3580 StorageError::BucketNotEmpty(err_bucket) | StorageError::BucketNotEmptyWithDetails { bucket: err_bucket, .. } => {
3581 assert_eq!(err_bucket, &bucket)
3582 }
3583 other => panic!("expected an S3-compatible BucketNotEmpty error, got {other:?}"),
3584 }
3585 assert!(
3586 any_disk_path_exists(&disk_paths, Path::new(&bucket).join(object)).await,
3587 "physical empty-bucket enforcement must preserve unobserved residue"
3588 );
3589
3590 ecstore
3591 .delete_bucket(
3592 &bucket,
3593 &DeleteBucketOptions {
3594 force: true,
3595 ..Default::default()
3596 },
3597 )
3598 .await
3599 .expect("explicit force should clean up the test-owned residue");
3600 }
3601
3602 #[tokio::test(flavor = "multi_thread")]
3603 #[serial]
3604 async fn bucket_delete_succeeds_after_listing_and_deleting_an_unversioned_overwrite() {
3605 let (disk_paths, ecstore) = setup_bucket_delete_test_env().await;
3606 let bucket = format!("bucket-delete-after-overwrite-{}", Uuid::new_v4().simple());
3607 let object = "object.txt";
3608
3609 ecstore
3610 .make_bucket(&bucket, &MakeBucketOptions::default())
3611 .await
3612 .expect("unversioned bucket should be created");
3613
3614 let mut first_reader = PutObjReader::from_vec(b"version A".to_vec());
3615 let first = ecstore
3616 .put_object(&bucket, object, &mut first_reader, &ObjectOptions::default())
3617 .await
3618 .expect("version A should be written");
3619 let mut second_reader = PutObjReader::from_vec(b"version B".to_vec());
3620 let second = ecstore
3621 .put_object(&bucket, object, &mut second_reader, &ObjectOptions::default())
3622 .await
3623 .expect("version B should overwrite version A");
3624 assert_ne!(first.data_dir, second.data_dir, "the overwrite must publish a new body generation");
3625
3626 let listing = ecstore
3627 .clone()
3628 .list_object_versions(&bucket, "", None, None, None, 1000)
3629 .await
3630 .expect("the overwritten object should remain listable for teardown");
3631 assert_eq!(listing.objects.len(), 1, "an unversioned overwrite should expose one current version");
3632 let current = &listing.objects[0];
3633 assert_eq!(current.name, object);
3634 assert!(current.is_latest, "the listed null version must be current");
3635 assert_eq!(current.version_id, None, "an unversioned object must be exposed as the null version");
3636 assert_eq!(
3637 current.data_dir, second.data_dir,
3638 "listing must expose version B, not the overwritten body"
3639 );
3640
3641 for version in listing.objects {
3642 let version_id = version.version_id.map(|version_id| version_id.to_string());
3643 ecstore
3644 .delete_object(
3645 &bucket,
3646 &version.name,
3647 ObjectOptions {
3648 version_id,
3649 ..Default::default()
3650 },
3651 )
3652 .await
3653 .expect("each version returned by teardown listing should be deletable");
3654 }
3655
3656 assert!(
3657 !any_disk_has_object_metadata(&disk_paths, &bucket).await,
3658 "deleting the listed null version must remove every xl.meta"
3659 );
3660 ecstore
3661 .delete_bucket(&bucket, &DeleteBucketOptions::default())
3662 .await
3663 .expect("DeleteBucket should succeed after the listed overwrite is deleted");
3664 }
3665
3666 #[tokio::test]
3667 #[serial]
3668 async fn bucket_delete_default_s3_delete_still_rejects_non_empty_bucket() {
3669 let (disk_paths, ecstore) = setup_bucket_delete_test_env().await;
3670 let bucket = format!("bucket-s3-delete-{}", Uuid::new_v4().simple());
3671 let object = "object.txt";
3672
3673 create_bucket_with_object(&ecstore, &bucket, object).await;
3674
3675 let generation_before_delete = ecstore.scanner_namespace_mutation_generation();
3676 let err = ecstore
3677 .delete_bucket(&bucket, &DeleteBucketOptions::default())
3678 .await
3679 .expect_err("default S3 DeleteBucket should reject non-empty buckets");
3680
3681 assert!(matches!(err, StorageError::BucketNotEmpty(name) if name == bucket));
3682 assert_eq!(
3683 ecstore.scanner_namespace_mutation_generation(),
3684 generation_before_delete,
3685 "failed bucket deletion must not advance scanner namespace activity"
3686 );
3687 assert!(
3688 any_disk_has_object_metadata(&disk_paths, &bucket).await,
3689 "failed default S3 DeleteBucket must keep object data"
3690 );
3691 assert!(
3692 metadata_sys::get_in(&ecstore.ctx, &bucket).await.is_ok(),
3693 "failed default S3 DeleteBucket must keep metadata cache"
3694 );
3695 }
3696
3697 #[tokio::test(flavor = "multi_thread")]
3698 #[serial]
3699 async fn bucket_delete_reports_metadata_less_residue_without_removing_it() {
3700 let (disk_paths, ecstore) = setup_bucket_delete_test_env().await;
3701 let bucket = format!("bucket-delete-orphan-datadir-{}", Uuid::new_v4().simple());
3702 let object = "object.txt";
3703
3704 ecstore
3705 .make_bucket(&bucket, &MakeBucketOptions::default())
3706 .await
3707 .expect("bucket should be created");
3708 write_metadata_less_part_on_all_disks(&disk_paths, &bucket, object).await;
3709 assert!(
3710 !any_disk_has_object_metadata(&disk_paths, &bucket).await,
3711 "test setup must reproduce a bucket with data dirs but no xl.meta"
3712 );
3713
3714 let err = ecstore
3715 .delete_bucket(&bucket, &DeleteBucketOptions::default())
3716 .await
3717 .expect_err("metadata-less data dirs must block ordinary DeleteBucket");
3718
3719 match err {
3720 StorageError::BucketNotEmptyWithDetails {
3721 bucket: err_bucket,
3722 details,
3723 } => {
3724 assert_eq!(err_bucket, bucket);
3725 assert!(
3726 details.contains("metadata-less on-disk residue"),
3727 "diagnostic should identify metadata-less residue, got: {details}"
3728 );
3729 assert!(details.contains("files="), "diagnostic should include a bounded count");
3730 assert!(details.contains("uuid_data_dirs="), "diagnostic should include data-dir count");
3731 }
3732 other => panic!("expected detailed BucketNotEmpty, got {other:?}"),
3733 }
3734 assert!(
3735 any_disk_path_exists(&disk_paths, Path::new(&bucket).join(object)).await,
3736 "ordinary DeleteBucket must preserve metadata-less data until an explicit operator action"
3737 );
3738 }
3739
3740 #[tokio::test]
3741 #[serial]
3742 async fn bucket_delete_fences_put_started_after_empty_scan() {
3743 let (_, ecstore) = setup_bucket_delete_test_env().await;
3744 let bucket = format!("bucket-delete-empty-scan-race-{}", Uuid::new_v4().simple());
3745 let object = "committed-after-empty-scan";
3746 let payload = b"object started after DeleteBucket empty scan".to_vec();
3747
3748 ecstore
3749 .make_bucket(&bucket, &MakeBucketOptions::default())
3750 .await
3751 .expect("bucket should be created");
3752
3753 let barrier = install_delete_bucket_empty_scan_barrier();
3754 let delete_store = ecstore.clone();
3755 let delete_bucket = bucket.clone();
3756 let delete = tokio::spawn(async move {
3757 delete_store
3758 .delete_bucket(&delete_bucket, &DeleteBucketOptions::default())
3759 .await
3760 });
3761 barrier.wait_until_paused().await;
3762
3763 let put_store = ecstore.clone();
3764 let put_bucket = bucket.clone();
3765 let mut put = tokio::spawn(async move {
3766 let mut put_reader = PutObjReader::from_vec(payload);
3767 put_store
3768 .put_object(&put_bucket, object, &mut put_reader, &ObjectOptions::default())
3769 .await
3770 });
3771 assert!(
3772 tokio::time::timeout(Duration::from_millis(100), &mut put).await.is_err(),
3773 "PUT must wait behind the DeleteBucket lifecycle fence"
3774 );
3775
3776 barrier.release();
3777 delete
3778 .await
3779 .expect("DeleteBucket task should join")
3780 .expect("DeleteBucket should commit while holding the lifecycle fence");
3781 let err = put
3782 .await
3783 .expect("PUT task should join")
3784 .expect_err("PUT must not recreate an object in the deleted bucket");
3785 assert!(matches!(err, StorageError::BucketNotFound(name) if name == bucket));
3786 }
3787
3788 #[tokio::test]
3789 #[serial]
3790 async fn bucket_recreation_does_not_publish_unverified_usage() {
3791 let (_, ecstore) = setup_bucket_delete_test_env().await;
3792 let bucket = format!("bucket-usage-generation-{}", Uuid::new_v4().simple());
3793 ecstore
3794 .make_bucket(&bucket, &MakeBucketOptions::default())
3795 .await
3796 .expect("bucket should be created");
3797
3798 let mut snapshot = DataUsageInfo {
3799 last_update: Some(SystemTime::now()),
3800 buckets_count: 1,
3801 ..Default::default()
3802 };
3803 snapshot.buckets_usage.insert(
3804 bucket.clone(),
3805 BucketUsageInfo {
3806 size: 42,
3807 objects_count: 1,
3808 versions_count: 1,
3809 ..Default::default()
3810 },
3811 );
3812 snapshot.usage_snapshot_complete = true;
3813 snapshot.bucket_sizes.insert(bucket.clone(), 42);
3814 snapshot.calculate_totals();
3815 crate::data_usage::store_data_usage_in_backend(snapshot, ecstore.clone())
3816 .await
3817 .expect("usage snapshot should be stored");
3818
3819 ecstore
3820 .delete_bucket(&bucket, &DeleteBucketOptions::default())
3821 .await
3822 .expect("empty bucket should be deleted");
3823 let deleted = crate::data_usage::load_data_usage_from_backend(ecstore.clone())
3824 .await
3825 .expect("usage snapshot should remain readable");
3826 assert!(!deleted.buckets_usage.contains_key(&bucket));
3827
3828 ecstore
3829 .make_bucket(&bucket, &MakeBucketOptions::default())
3830 .await
3831 .expect("same bucket name should be recreated after delete returns");
3832 crate::data_usage::record_bucket_object_write_memory(&bucket, None, 84).await;
3833
3834 let mut recreated = crate::data_usage::load_data_usage_from_backend(ecstore.clone())
3835 .await
3836 .expect("recreated bucket usage base should load");
3837 crate::data_usage::apply_bucket_usage_memory_overlay(&mut recreated).await;
3838 assert!(
3839 !recreated.buckets_usage.contains_key(&bucket),
3840 "a request-path delta without an authoritative baseline must remain unavailable"
3841 );
3842 assert_eq!(crate::data_usage::get_bucket_usage_memory(&bucket).await, None);
3843 }
3844
3845 #[tokio::test]
3846 #[serial]
3847 async fn bucket_create_removes_stale_usage_before_physical_creation() {
3848 let (_, ecstore) = setup_bucket_delete_test_env().await;
3849 let bucket = format!("bucket-create-stale-usage-{}", Uuid::new_v4().simple());
3850 let mut snapshot = DataUsageInfo {
3851 last_update: Some(SystemTime::now()),
3852 buckets_count: 1,
3853 ..Default::default()
3854 };
3855 snapshot.buckets_usage.insert(
3856 bucket.clone(),
3857 BucketUsageInfo {
3858 size: 42,
3859 objects_count: 1,
3860 versions_count: 1,
3861 ..Default::default()
3862 },
3863 );
3864 snapshot.usage_snapshot_complete = true;
3865 snapshot.bucket_sizes.insert(bucket.clone(), 42);
3866 snapshot.calculate_totals();
3867 crate::data_usage::store_data_usage_in_backend(snapshot, ecstore.clone())
3868 .await
3869 .expect("stale usage fixture should be stored");
3870
3871 ecstore
3872 .make_bucket(&bucket, &MakeBucketOptions::default())
3873 .await
3874 .expect("CreateBucket should succeed");
3875
3876 let persisted = crate::data_usage::load_data_usage_from_backend(ecstore.clone())
3877 .await
3878 .expect("usage snapshot should remain readable");
3879 assert!(
3880 !persisted.buckets_usage.contains_key(&bucket),
3881 "a newly created bucket must not inherit the predecessor generation's usage"
3882 );
3883 assert!(
3884 ecstore.get_bucket_info(&bucket, &BucketOptions::default()).await.is_ok(),
3885 "physical creation should happen after the usage fence succeeds"
3886 );
3887 }
3888
3889 #[tokio::test]
3890 #[serial]
3891 async fn failed_create_rollback_does_not_run_unfenced_usage_cleanup() {
3892 let (_, ecstore) = setup_bucket_delete_test_env().await;
3893 let bucket = format!("bucket-create-rollback-{}", Uuid::new_v4().simple());
3894 ecstore
3895 .peer_sys
3896 .make_bucket(&bucket, &MakeBucketOptions::default())
3897 .await
3898 .expect("the partial-create fixture should expose a physical bucket");
3899
3900 let mut snapshot = DataUsageInfo {
3901 last_update: Some(SystemTime::now()),
3902 buckets_count: 1,
3903 ..Default::default()
3904 };
3905 snapshot.buckets_usage.insert(
3906 bucket.clone(),
3907 BucketUsageInfo {
3908 size: 42,
3909 objects_count: 1,
3910 versions_count: 1,
3911 ..Default::default()
3912 },
3913 );
3914 snapshot.usage_snapshot_complete = true;
3915 snapshot.bucket_sizes.insert(bucket.clone(), 42);
3916 snapshot.calculate_totals();
3917 crate::data_usage::store_data_usage_in_backend(snapshot, ecstore.clone())
3918 .await
3919 .expect("the usage fixture should be stored");
3920 crate::bucket::metadata::save_bucket_incarnation(ecstore.clone(), &bucket, Uuid::new_v4())
3921 .await
3922 .expect("partial create should have an incarnation sidecar");
3923
3924 ecstore.rollback_failed_bucket_creation(&bucket, None, None).await;
3925
3926 assert!(
3927 ecstore
3928 .peer_sys
3929 .get_bucket_info(&bucket, &BucketOptions::default())
3930 .await
3931 .is_err(),
3932 "failed-create rollback should remove the partial physical bucket"
3933 );
3934 assert!(
3935 crate::bucket::metadata::load_bucket_incarnation(ecstore.clone(), &bucket)
3936 .await
3937 .expect("rollback sidecar lookup should succeed")
3938 .is_none(),
3939 "failed-create rollback must remove the orphan incarnation sidecar"
3940 );
3941 let persisted = crate::data_usage::load_data_usage_from_backend(ecstore.clone())
3942 .await
3943 .expect("the usage snapshot should remain readable");
3944 assert!(
3945 persisted.buckets_usage.contains_key(&bucket),
3946 "failed-create rollback must not start an unfenced usage cleanup"
3947 );
3948
3949 crate::data_usage::store_data_usage_in_backend(DataUsageInfo::default(), ecstore.clone())
3950 .await
3951 .expect("the rollback usage fixture should be cleared");
3952 }
3953
3954 #[tokio::test]
3955 #[serial]
3956 async fn bucket_create_fails_closed_when_usage_snapshot_cannot_be_fenced() {
3957 let (_, ecstore) = setup_bucket_delete_test_env().await;
3958 let deleted_bucket = format!("bucket-delete-corrupt-usage-{}", Uuid::new_v4().simple());
3959 ecstore
3960 .make_bucket(&deleted_bucket, &MakeBucketOptions::default())
3961 .await
3962 .expect("bucket should be created before corrupting usage");
3963
3964 let usage_path = format!("{BUCKET_META_PREFIX}/{DATA_USAGE_OBJECT_NAME}");
3965 crate::config::com::save_config(ecstore.clone(), &usage_path, b"{".to_vec())
3966 .await
3967 .expect("corrupt usage fixture should be stored");
3968
3969 ecstore
3970 .delete_bucket(&deleted_bucket, &DeleteBucketOptions::default())
3971 .await
3972 .expect("usage snapshot corruption must not block DeleteBucket");
3973 assert!(
3974 ecstore
3975 .get_bucket_info(&deleted_bucket, &crate::storage_api_contracts::bucket::BucketOptions::default())
3976 .await
3977 .is_err(),
3978 "successful DeleteBucket must remove the physical bucket"
3979 );
3980
3981 let new_bucket = format!("bucket-create-corrupt-usage-{}", Uuid::new_v4().simple());
3982 let create_err = ecstore
3983 .make_bucket(&new_bucket, &MakeBucketOptions::default())
3984 .await
3985 .expect_err("MakeBucket must fail when stale usage cannot be fenced");
3986 assert!(!create_err.to_string().is_empty(), "the usage snapshot failure should be surfaced");
3987 assert!(
3988 ecstore
3989 .get_bucket_info(&new_bucket, &crate::storage_api_contracts::bucket::BucketOptions::default())
3990 .await
3991 .is_err(),
3992 "failed MakeBucket must not expose a new physical bucket"
3993 );
3994
3995 let restored = serde_json::to_vec(&DataUsageInfo {
3996 last_update: Some(SystemTime::now()),
3997 ..Default::default()
3998 })
3999 .expect("restored usage fixture should encode");
4000 crate::config::com::save_config(ecstore, &usage_path, restored)
4001 .await
4002 .expect("usage fixture should be restored after the failure-path test");
4003 }
4004
4005 fn bucket_logs_at_default_level(emit: impl FnOnce()) -> String {
4007 use std::sync::{Arc, Mutex};
4008 use tracing_subscriber::EnvFilter;
4009 use tracing_subscriber::fmt::MakeWriter;
4010 use tracing_subscriber::layer::SubscriberExt;
4011
4012 #[derive(Clone, Default)]
4013 struct CapturedLogs {
4014 buffer: Arc<Mutex<Vec<u8>>>,
4015 }
4016 struct CapturedLogWriter {
4017 buffer: Arc<Mutex<Vec<u8>>>,
4018 }
4019 impl std::io::Write for CapturedLogWriter {
4020 fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
4021 self.buffer
4022 .lock()
4023 .expect("captured logs mutex should not be poisoned")
4024 .extend_from_slice(buf);
4025 Ok(buf.len())
4026 }
4027 fn flush(&mut self) -> std::io::Result<()> {
4028 Ok(())
4029 }
4030 }
4031 impl<'a> MakeWriter<'a> for CapturedLogs {
4032 type Writer = CapturedLogWriter;
4033 fn make_writer(&'a self) -> Self::Writer {
4034 CapturedLogWriter {
4035 buffer: Arc::clone(&self.buffer),
4036 }
4037 }
4038 }
4039
4040 let logs = CapturedLogs::default();
4041 let subscriber = tracing_subscriber::registry()
4042 .with(EnvFilter::new(rustfs_config::DEFAULT_LOG_LEVEL))
4043 .with(
4044 tracing_subscriber::fmt::layer()
4045 .with_writer(logs.clone())
4046 .with_ansi(false)
4047 .without_time(),
4048 );
4049 let _guard = tracing::subscriber::set_default(subscriber);
4050 let _callsite_pin = crate::test_tracing::pin_callsite_interest_for_test();
4051
4052 emit();
4053
4054 let buffer = logs
4055 .buffer
4056 .lock()
4057 .expect("captured logs mutex should not be poisoned")
4058 .clone();
4059 String::from_utf8(buffer).expect("captured logs should be valid UTF-8")
4060 }
4061
4062 fn residue_sample(sample: &str) -> BucketMetadataLessResidue {
4063 BucketMetadataLessResidue {
4064 files: 1,
4065 entries_scanned: 3,
4066 sample: Some(sample.to_string()),
4067 ..Default::default()
4068 }
4069 }
4070
4071 #[test]
4077 fn unreachable_residue_is_reported_at_the_default_log_level() {
4078 for kind in [
4079 BucketDeleteBlockerKind::UnknownXlMeta,
4080 BucketDeleteBlockerKind::OrphanDirectory,
4081 BucketDeleteBlockerKind::DiagnosticBudgetExceeded,
4082 ] {
4083 let logs = bucket_logs_at_default_level(|| {
4084 record_bucket_delete_blocker("drained-bucket", kind, &residue_sample("obj/8f2c/xl.meta"));
4085 });
4086
4087 assert!(logs.contains("drained-bucket"), "the bucket must be named for {kind:?}: {logs}");
4088 assert!(logs.contains(kind.as_str()), "the blocker kind must be named for {kind:?}: {logs}");
4089 assert!(
4090 logs.contains("obj/8f2c/xl.meta"),
4091 "the residue sample is the only pointer to the leftover state for {kind:?}: {logs}"
4092 );
4093 }
4094 }
4095
4096 #[test]
4100 fn client_visible_blockers_stay_below_the_default_log_level() {
4101 for kind in [
4102 BucketDeleteBlockerKind::VisibleVersion,
4103 BucketDeleteBlockerKind::TierFreeVersion,
4104 ] {
4105 let logs = bucket_logs_at_default_level(|| {
4106 record_bucket_delete_blocker("still-full-bucket", kind, &residue_sample("obj/xl.meta"));
4107 });
4108
4109 assert!(logs.is_empty(), "an ordinary non-empty bucket must not log an error for {kind:?}: {logs}");
4110 }
4111 }
4112
4113 #[test]
4114 fn blocker_kinds_split_by_whether_the_client_can_reach_the_residue() {
4115 assert!(BucketDeleteBlockerKind::VisibleVersion.is_client_visible());
4116 assert!(BucketDeleteBlockerKind::TierFreeVersion.is_client_visible());
4117 assert!(!BucketDeleteBlockerKind::UnknownXlMeta.is_client_visible());
4118 assert!(!BucketDeleteBlockerKind::OrphanDirectory.is_client_visible());
4119 assert!(!BucketDeleteBlockerKind::DiagnosticBudgetExceeded.is_client_visible());
4120 }
4121}