1use super::*;
16use crate::bucket::{
17 metadata::{BUCKET_TABLE_RESERVED_PREFIX, table_bucket_catalog_metadata_prefix},
18 utils::is_meta_bucketname,
19};
20use crate::error::is_err_bucket_not_found;
21use crate::runtime::sources as runtime_sources;
22use crate::set_disk::{BucketInfoQuorum, get_lock_acquire_timeout};
23use crate::storage_api_contracts::bucket::{BUCKET_LIFECYCLE_LOCK_OBJECT, SRBucketDeleteOp};
24use crate::storage_api_contracts::namespace::NamespaceLocking as _;
25use futures::stream::{self, StreamExt};
26use rustfs_policy::policy::BucketPolicy;
27use std::collections::BTreeMap;
28use std::future::Future;
29
30const DELETED_BUCKETS_PREFIX: &str = ".deleted";
31const SCANNER_BUCKET_LIST_SET_CONCURRENCY: usize = 4;
32const EVENT_BUCKET_DELETE_BLOCKED: &str = "bucket_delete_blocked";
33const EVENT_BUCKET_DELETE_ROLLBACK_FAILED: &str = "bucket_delete_rollback_failed";
34
35fn record_bucket_delete_blocker(bucket: &str, kind: BucketDeleteBlockerKind, residue: &BucketMetadataLessResidue) {
53 metrics::counter!("rustfs_bucket_delete_blockers_total", "kind" => kind.as_str()).increment(1);
54 if kind.is_client_visible() {
55 warn!(
56 event = EVENT_BUCKET_DELETE_BLOCKED,
57 component = "ecstore",
58 subsystem = "bucket",
59 bucket,
60 blocker = kind.as_str(),
61 files = residue.files,
62 uuid_data_dirs = residue.uuid_data_dirs,
63 entries_scanned = residue.entries_scanned,
64 diagnostic_bytes_read = residue.diagnostic_bytes_read,
65 diagnostic_truncated = residue.diagnostic_truncated,
66 sample = residue.sample.as_deref().unwrap_or("<none>"),
67 "Bucket deletion was blocked by durable local state"
68 );
69 return;
70 }
71
72 error!(
73 event = EVENT_BUCKET_DELETE_BLOCKED,
74 component = "ecstore",
75 subsystem = "bucket",
76 bucket,
77 blocker = kind.as_str(),
78 files = residue.files,
79 uuid_data_dirs = residue.uuid_data_dirs,
80 entries_scanned = residue.entries_scanned,
81 diagnostic_bytes_read = residue.diagnostic_bytes_read,
82 diagnostic_truncated = residue.diagnostic_truncated,
83 sample = residue.sample.as_deref().unwrap_or("<none>"),
84 "Bucket deletion was blocked by residue the client cannot reach through the S3 API"
85 );
86}
87
88fn bucket_list_set_concurrency(set_count: usize, max_concurrency: usize) -> usize {
89 set_count.clamp(1, max_concurrency.max(1))
90}
91
92fn should_override_created_from_metadata(created: OffsetDateTime) -> bool {
93 created != OffsetDateTime::UNIX_EPOCH
94}
95
96fn validate_table_bucket_delete_allowed(
97 bucket: &str,
98 table_bucket_enabled: bool,
99 table_catalog_metadata_exists: bool,
100) -> Result<()> {
101 if table_bucket_enabled && table_catalog_metadata_exists {
102 return Err(StorageError::BucketNotEmpty(bucket.to_string()));
103 }
104
105 Ok(())
106}
107
108async fn table_catalog_metadata_exists(ctx: &crate::runtime::instance::InstanceContext, bucket: &str) -> Result<bool> {
109 let local_disks = runtime_sources::local_disks_in(ctx).await;
110 for disk in local_disks.iter() {
111 let Some(bucket_path) = disk.get_bucket_path_for_io_if_local(bucket) else {
112 continue;
113 };
114 let catalog_path = bucket_path?.join(BUCKET_TABLE_RESERVED_PREFIX);
115 if has_xlmeta_files(&catalog_path).await? {
116 return Ok(true);
117 }
118 }
119
120 Ok(false)
121}
122
123async fn validate_table_bucket_delete_guard(ctx: &crate::runtime::instance::InstanceContext, bucket: &str) -> Result<()> {
124 let table_bucket_enabled = metadata_sys::get_in(ctx, bucket)
125 .await
126 .is_ok_and(|metadata| metadata.table_bucket_enabled());
127 if table_bucket_enabled {
128 validate_table_bucket_delete_allowed(bucket, true, table_catalog_metadata_exists(ctx, bucket).await?)?;
129 }
130
131 Ok(())
132}
133
134fn bucket_delete_metadata_cleanup_prefixes(bucket: &str) -> [String; 2] {
135 [
136 table_bucket_catalog_metadata_prefix(bucket),
137 format!("{BUCKET_META_PREFIX}/{bucket}"),
138 ]
139}
140
141fn bucket_deleted_marker_prefix(bucket: &str) -> String {
142 format!("{BUCKET_META_PREFIX}/{DELETED_BUCKETS_PREFIX}/{bucket}")
143}
144
145fn bucket_deleted_marker_volume(bucket: &str) -> String {
146 format!("{RUSTFS_META_BUCKET}/{}", bucket_deleted_marker_prefix(bucket))
147}
148
149pub(crate) async fn await_bucket_namespace_operation<T, F>(
150 guard: Option<&rustfs_lock::NamespaceLockGuard>,
151 bucket: &str,
152 operation: &'static str,
153 future: F,
154) -> Result<T>
155where
156 F: Future<Output = Result<T>>,
157{
158 let Some(guard) = guard else {
159 return future.await;
160 };
161 if guard.is_lock_lost() {
162 return Err(StorageError::other(format!(
163 "bucket namespace lock was lost before {operation}: {bucket}"
164 )));
165 }
166 tokio::select! {
167 biased;
168 _ = guard.lock_lost_notified() => Err(StorageError::other(format!(
169 "bucket namespace lock was lost during {operation}: {bucket}"
170 ))),
171 result = future => result,
172 }
173}
174
175async fn await_bucket_lifecycle_operation<T, F>(
176 lifecycle_guard: Option<&rustfs_lock::NamespaceLockGuard>,
177 namespace_guard: Option<&rustfs_lock::NamespaceLockGuard>,
178 bucket: &str,
179 operation: &'static str,
180 future: F,
181) -> Result<T>
182where
183 F: Future<Output = Result<T>>,
184{
185 await_bucket_namespace_operation(
186 lifecycle_guard,
187 bucket,
188 operation,
189 await_bucket_namespace_operation(namespace_guard, bucket, operation, future),
190 )
191 .await
192}
193
194async fn run_bucket_usage_cleanup<F>(guard: Option<&rustfs_lock::NamespaceLockGuard>, bucket: &str, future: F) -> Result<()>
195where
196 F: Future<Output = Result<()>>,
197{
198 await_bucket_namespace_operation(guard, bucket, "bucket usage cleanup", future).await
199}
200
201async fn run_physical_bucket_deletion<F>(guard: Option<&rustfs_lock::NamespaceLockGuard>, bucket: &str, future: F) -> Result<()>
202where
203 F: Future<Output = Result<()>>,
204{
205 list_objects::observe_scanner_namespace_mutations(bucket, 1);
208 await_bucket_namespace_operation(guard, bucket, "physical bucket deletion", future).await
209}
210
211async fn bucket_delete_local_blocker(
212 ctx: &crate::runtime::instance::InstanceContext,
213 bucket: &str,
214 budget: &mut BucketDeleteDiagnosticBudget,
215) -> Result<Option<StorageError>> {
216 let local_disks = runtime_sources::local_disks_in(ctx).await;
217 let mut residue = BucketMetadataLessResidue::default();
218
219 for disk in local_disks.iter() {
220 let Some(bucket_path) = disk.get_bucket_path_for_io_if_local(bucket) else {
221 continue;
222 };
223 let scan = scan_metadata_less_residue_with_budget(&bucket_path?, budget).await?;
224 if scan.xlmeta_found {
225 record_bucket_delete_blocker(bucket, scan.xlmeta_blocker.unwrap_or(BucketDeleteBlockerKind::UnknownXlMeta), &scan);
226 return Ok(Some(StorageError::BucketNotEmpty(bucket.to_string())));
227 }
228 residue.files = residue.files.saturating_add(scan.files);
229 residue.uuid_data_dirs = residue.uuid_data_dirs.saturating_add(scan.uuid_data_dirs);
230 residue.entries_scanned = residue.entries_scanned.saturating_add(scan.entries_scanned);
231 residue.diagnostic_bytes_read = residue.diagnostic_bytes_read.saturating_add(scan.diagnostic_bytes_read);
232 if residue.sample.is_none() {
233 residue.sample = scan.sample.clone();
234 }
235 if scan.diagnostic_truncated {
236 residue.diagnostic_truncated = true;
237 if residue.files == 0 {
238 return Ok(None);
241 }
242 record_bucket_delete_blocker(bucket, BucketDeleteBlockerKind::DiagnosticBudgetExceeded, &residue);
243 return Ok(Some(StorageError::BucketNotEmptyWithDetails {
244 bucket: bucket.to_string(),
245 details: residue.describe(),
246 }));
247 }
248 }
249
250 if residue.has_residue_without_xlmeta() {
251 record_bucket_delete_blocker(bucket, BucketDeleteBlockerKind::OrphanDirectory, &residue);
252 return Ok(Some(StorageError::BucketNotEmptyWithDetails {
253 bucket: bucket.to_string(),
254 details: residue.describe(),
255 }));
256 }
257
258 Ok(None)
259}
260
261impl ECStore {
262 fn bucket_sets(&self) -> impl Iterator<Item = (usize, usize, Arc<crate::set_disk::SetDisks>)> + '_ {
263 self.pools.iter().flat_map(|pool| {
264 pool.disk_set
265 .iter()
266 .map(|set| (set.pool_index, set.set_index, Arc::clone(set)))
267 })
268 }
269
270 pub async fn get_bucket_metadata(&self, bucket: &str) -> Result<Arc<BucketMetadata>> {
271 let sys = metadata_sys::require_bucket_metadata_sys_in(&self.ctx)?;
272 sys.read().await.get(bucket).await
273 }
274
275 pub async fn get_bucket_policy(&self, bucket: &str) -> Result<(BucketPolicy, OffsetDateTime)> {
276 let sys = metadata_sys::require_bucket_metadata_sys_in(&self.ctx)?;
277 sys.read().await.get_bucket_policy(bucket).await
278 }
279
280 pub async fn get_bucket_policy_raw(&self, bucket: &str) -> Result<(String, OffsetDateTime)> {
281 let sys = metadata_sys::require_bucket_metadata_sys_in(&self.ctx)?;
282 sys.read().await.get_bucket_policy_raw(bucket).await
283 }
284
285 pub async fn restricts_public_bucket_access(&self, bucket: &str) -> Result<bool> {
286 let sys = metadata_sys::require_bucket_metadata_sys_in(&self.ctx)?;
287 let (config, _) = sys.read().await.get_public_access_block_config(bucket).await?;
288 Ok(config.restrict_public_buckets.unwrap_or(false))
289 }
290
291 pub async fn update_bucket_metadata_config(&self, bucket: &str, config_file: &str, data: Vec<u8>) -> Result<OffsetDateTime> {
292 metadata_sys::update_in(&self.ctx, bucket, config_file, data).await
293 }
294
295 pub async fn bucket_incarnation_id(&self, bucket: &str) -> Result<Uuid> {
296 metadata_sys::get_cached_bucket_incarnation_id_in(&self.ctx, bucket).await
297 }
298
299 pub async fn bucket_incarnation_id_from_disk(&self, bucket: &str) -> Result<Uuid> {
300 metadata_sys::get_bucket_incarnation_id_in(&self.ctx, bucket).await
301 }
302
303 pub(crate) async fn acquire_bucket_lifecycle_read_lock(&self, bucket: &str) -> Result<rustfs_lock::NamespaceLockGuard> {
306 let lock = self.new_ns_lock(bucket, BUCKET_LIFECYCLE_LOCK_OBJECT).await?;
307 lock.get_read_lock(get_lock_acquire_timeout()).await.map_err(|err| match err {
308 rustfs_lock::error::LockError::QuorumNotReached { required, achieved } => {
309 StorageError::NamespaceLockQuorumUnavailable {
310 mode: "bucket_lifecycle_read",
311 bucket: bucket.to_string(),
312 object: BUCKET_LIFECYCLE_LOCK_OBJECT.to_string(),
313 required,
314 achieved,
315 }
316 }
317 other => StorageError::Lock(other),
318 })
319 }
320
321 pub async fn acquire_bucket_incarnation_fence(
333 &self,
334 bucket: &str,
335 expected: uuid::Uuid,
336 ) -> Result<super::BucketIncarnationFenceGuard> {
337 let inner = self.acquire_bucket_lifecycle_read_lock(bucket).await?;
338 let pieces = super::bucket_fence::FencePieces {
339 registry: self.bucket_fence_registry.clone(),
340 inner,
341 };
342 let registration = pieces.enter(bucket);
343 let current = match registration.memoized {
344 Some(current) => current,
345 None => match metadata_sys::get_bucket_incarnation_id_in(&self.ctx, bucket).await {
346 Ok(current) => {
347 if !pieces.lock_lost() {
350 pieces.memoize(bucket, current);
351 }
352 current
353 }
354 Err(err) => {
355 pieces.abandon(bucket, registration.token);
356 return Err(err);
357 }
358 },
359 };
360 if current != expected {
361 pieces.abandon(bucket, registration.token);
362 return Err(StorageError::BucketNotFound(bucket.to_string()));
363 }
364 Ok(pieces.into_guard(bucket, registration.token))
365 }
366
367 pub async fn lock_bucket_for_recursive_delete(&self, bucket: &str) -> Result<rustfs_lock::NamespaceLockGuard> {
370 if self.ctx.lock_manager().is_disabled() {
371 return Err(StorageError::InvalidArgument(
372 bucket.to_owned(),
373 String::new(),
374 "Recursive deletion requires namespace locking".to_owned(),
375 ));
376 }
377 self.acquire_bucket_lifecycle_write_lock(bucket).await
378 }
379
380 pub(crate) async fn acquire_bucket_lifecycle_write_lock(&self, bucket: &str) -> Result<rustfs_lock::NamespaceLockGuard> {
381 let lock = self.new_ns_lock(bucket, BUCKET_LIFECYCLE_LOCK_OBJECT).await?;
382 lock.get_write_lock(get_lock_acquire_timeout())
383 .await
384 .map_err(|err| match err {
385 rustfs_lock::error::LockError::QuorumNotReached { required, achieved } => {
386 StorageError::NamespaceLockQuorumUnavailable {
387 mode: "bucket_lifecycle_write",
388 bucket: bucket.to_string(),
389 object: BUCKET_LIFECYCLE_LOCK_OBJECT.to_string(),
390 required,
391 achieved,
392 }
393 }
394 other => StorageError::Lock(other),
395 })
396 }
397
398 async fn mark_bucket_deleted(&self, bucket: &str) -> Result<()> {
399 let marker_volume = bucket_deleted_marker_volume(bucket);
400
401 self.make_bucket_on_sets(
402 marker_volume.as_str(),
403 &MakeBucketOptions {
404 force_create: true,
405 ..Default::default()
406 },
407 )
408 .await
409 .map_err(|err| to_object_err(err, vec![bucket]))?;
410
411 Ok(())
412 }
413
414 async fn make_bucket_on_sets(&self, bucket: &str, opts: &MakeBucketOptions) -> Result<()> {
415 let results = futures::future::join_all(
416 self.bucket_sets()
417 .map(|(_, _, set)| async move { set.make_bucket(bucket, opts).await }),
418 )
419 .await;
420 if results.is_empty() {
421 return Err(StorageError::ErasureWriteQuorum);
422 }
423
424 let mut bucket_exists_error = None;
425 let mut first_hard_error = None;
426 for result in results {
427 let Err(err) = result else {
428 continue;
429 };
430 if matches!(err, StorageError::VolumeExists) || is_err_bucket_exists(&err) {
431 if bucket_exists_error.is_none() {
432 bucket_exists_error = Some(err);
433 }
434 } else if first_hard_error.is_none() {
435 first_hard_error = Some(err);
436 }
437 }
438
439 match first_hard_error.or(bucket_exists_error) {
440 Some(err) => Err(err),
441 None => Ok(()),
442 }
443 }
444
445 async fn delete_bucket_on_sets(&self, bucket: &str, opts: &DeleteBucketOptions) -> Result<()> {
446 let results = futures::future::join_all(
447 self.bucket_sets()
448 .map(|(_, _, set)| async move { set.delete_bucket(bucket, opts).await }),
449 )
450 .await;
451 if results.is_empty() {
452 return Err(StorageError::ErasureWriteQuorum);
453 }
454
455 let mut deleted = false;
456 let mut first_error = None;
457 for result in results {
458 match result {
459 Ok(()) => deleted = true,
460 Err(err) if is_err_strict_volume_not_found(&err) => {}
461 Err(err) if first_error.is_none() => first_error = Some(err),
462 Err(_) => {}
463 }
464 }
465
466 if let Some(delete_error) = first_error {
467 if !opts.no_recreate {
468 let rollback_opts = MakeBucketOptions {
469 force_create: true,
470 no_lock: true,
471 ..Default::default()
472 };
473 if let Err(rollback_error) = self.make_bucket_on_sets(bucket, &rollback_opts).await {
474 warn!(
475 event = EVENT_BUCKET_DELETE_ROLLBACK_FAILED,
476 component = "ecstore",
477 subsystem = "bucket",
478 bucket,
479 deletion_error = ?delete_error,
480 rollback_error = ?rollback_error,
481 "Bucket deletion rollback could not restore every bucket volume"
482 );
483 }
484 }
485 return Err(delete_error);
486 }
487
488 if deleted { Ok(()) } else { Err(StorageError::VolumeNotFound) }
489 }
490
491 async fn cleanup_deleted_bucket_metadata(
492 &self,
493 bucket: &str,
494 include_deleted_marker: bool,
495 guard: Option<&rustfs_lock::NamespaceLockGuard>,
496 ) -> Result<()> {
497 for prefix in bucket_delete_metadata_cleanup_prefixes(bucket) {
498 await_bucket_namespace_operation(
499 guard,
500 bucket,
501 "deleted bucket metadata cleanup",
502 self.delete_all(RUSTFS_META_BUCKET, prefix.as_str()),
503 )
504 .await?;
505 }
506
507 if include_deleted_marker {
508 let marker_prefix = bucket_deleted_marker_prefix(bucket);
509 await_bucket_namespace_operation(
510 guard,
511 bucket,
512 "deleted bucket marker cleanup",
513 self.delete_all(RUSTFS_META_BUCKET, marker_prefix.as_str()),
514 )
515 .await?;
516 }
517
518 await_bucket_namespace_operation(
519 guard,
520 bucket,
521 "deleted bucket metadata cache cleanup",
522 metadata_sys::remove_bucket_metadata_in(&self.ctx, bucket),
523 )
524 .await?;
525 runtime_sources::delete_bucket_monitor_entry(bucket);
526 Ok(())
527 }
528
529 async fn cleanup_bucket_usage(&self, bucket: &str, guard: Option<&rustfs_lock::NamespaceLockGuard>) -> Result<()> {
530 run_bucket_usage_cleanup(guard, bucket, async {
531 crate::data_usage::prepare_bucket_usage_for_namespace_change(bucket, guard).await?;
532 crate::data_usage::remove_bucket_usage_from_backend_with_guard_fenced(self, bucket, guard).await
533 })
534 .await
535 }
536
537 async fn cleanup_bucket_usage_best_effort(&self, bucket: &str, guard: Option<&rustfs_lock::NamespaceLockGuard>) {
538 if let Err(err) = self.cleanup_bucket_usage(bucket, guard).await {
539 warn!(
540 bucket = %bucket,
541 error = ?err,
542 "bucket data usage cleanup deferred to scanner reconciliation"
543 );
544 }
545 }
546
547 async fn rollback_failed_bucket_creation(
548 &self,
549 bucket: &str,
550 lifecycle_guard: Option<&rustfs_lock::NamespaceLockGuard>,
551 namespace_guard: Option<&rustfs_lock::NamespaceLockGuard>,
552 ) {
553 let rollback_opts = DeleteBucketOptions {
554 no_lock: true,
555 no_recreate: true,
556 ..Default::default()
557 };
558 if let Err(err) =
559 await_bucket_lifecycle_operation(lifecycle_guard, namespace_guard, bucket, "failed bucket creation rollback", async {
560 self.delete_bucket_on_sets(bucket, &rollback_opts)
561 .await
562 .map_err(|rollback_err| to_object_err(rollback_err, vec![bucket]))
563 })
564 .await
565 {
566 warn!(
567 bucket = %bucket,
568 error = ?err,
569 "failed bucket creation rollback did not remove every physical bucket volume"
570 );
571 return;
572 }
573 if let Err(err) = self.cleanup_deleted_bucket_metadata(bucket, false, namespace_guard).await {
574 warn!(
575 bucket = %bucket,
576 error = ?err,
577 "failed bucket creation rollback left internal bucket metadata"
578 );
579 }
580 }
581
582 #[instrument(skip(self))]
583 pub(super) async fn handle_make_bucket(&self, bucket: &str, opts: &MakeBucketOptions) -> Result<()> {
584 if !is_meta_bucketname(bucket)
585 && let Err(err) = check_valid_bucket_name_strict(bucket)
586 {
587 return Err(StorageError::BucketNameInvalid(err.to_string()));
588 }
589
590 let bucket_lifecycle_guard = if !opts.no_lock {
592 Some(self.acquire_bucket_lifecycle_write_lock(bucket).await?)
593 } else {
594 None
595 };
596 let metadata_transaction_guard = if !opts.no_lock && !is_meta_bucketname(bucket) {
597 Some(metadata_sys::acquire_bucket_metadata_transaction_lock_in(&self.ctx, bucket).await?)
598 } else {
599 None
600 };
601 let ns_guard = if !opts.no_lock {
602 let ns_lock = self.new_ns_lock(bucket, bucket).await?;
603 Some(
604 await_bucket_namespace_operation(
605 bucket_lifecycle_guard.as_ref(),
606 bucket,
607 "bucket namespace lock acquisition",
608 async {
609 ns_lock.get_write_lock(get_lock_acquire_timeout()).await.map_err(|e| match e {
610 rustfs_lock::error::LockError::QuorumNotReached { required, achieved } => {
611 StorageError::NamespaceLockQuorumUnavailable {
612 mode: "write",
613 bucket: bucket.to_string(),
614 object: bucket.to_string(),
615 required,
616 achieved,
617 }
618 }
619 other => StorageError::Lock(other),
620 })
621 },
622 )
623 .await?,
624 )
625 } else {
626 None
627 };
628
629 let existing_bucket_info = match self.get_bucket_info_from_sets(bucket, &BucketOptions::default()).await {
630 Ok(info) => Some(info),
631 Err(err) => {
632 if is_err_bucket_not_found(&err) {
633 None
634 } else {
635 return Err(to_object_err(err, vec![bucket]));
636 }
637 }
638 };
639 let confirmed_missing = existing_bucket_info.is_none();
640 let existing_metadata = if opts.force_create && !confirmed_missing && !is_meta_bucketname(bucket) {
641 let (mut metadata, persisted) = metadata_sys::get_config_from_disk_with_presence_in(&self.ctx, bucket).await?;
642 if !persisted {
643 metadata = BucketMetadata::new(bucket);
644 metadata.created = existing_bucket_info
645 .as_ref()
646 .and_then(|info| info.created)
647 .unwrap_or(OffsetDateTime::UNIX_EPOCH);
648 } else if !metadata.bucket_incarnation_sidecar && !metadata.bucket_incarnation_id.is_nil() {
649 return Err(Error::other(format!(
650 "bucket incarnation sidecar is missing for new-format metadata: {bucket}"
651 )));
652 } else if !metadata.bucket_incarnation_sidecar {
653 metadata.bucket_incarnation_id = Uuid::new_v4();
654 }
655 Some(metadata)
656 } else {
657 None
658 };
659
660 let mut meta = existing_metadata.unwrap_or_else(|| {
661 if confirmed_missing && !is_meta_bucketname(bucket) {
662 BucketMetadata::new_with_default_durability(bucket)
663 } else {
664 BucketMetadata::new(bucket)
665 }
666 });
667 let existing_incarnation_is_authoritative = meta.bucket_incarnation_sidecar;
668 if confirmed_missing || is_meta_bucketname(bucket) {
669 meta.set_created(opts.created_at);
670
671 if opts.versioning_enabled {
672 meta.versioning_config_xml =
673 crate::bucket::utils::serialize::<VersioningConfiguration>(&ENABLED_VERSIONING_CONFIG)?;
674 }
675 }
676
677 let lock_newly_enabled = opts.lock_enabled && !meta.lock_enabled;
684 if opts.lock_enabled {
685 meta.lock_enabled = true;
686 meta.object_lock_config_xml =
687 crate::bucket::utils::serialize::<ObjectLockConfiguration>(&ENABLED_OBJECT_LOCK_CONFIG)?;
688 meta.versioning_config_xml = crate::bucket::utils::serialize::<VersioningConfiguration>(&ENABLED_VERSIONING_CONFIG)?;
689 }
690
691 let metadata_persisted_before_physical = confirmed_missing && !is_meta_bucketname(bucket) && opts.lock_enabled;
692 if metadata_persisted_before_physical {
693 metadata_sys::set_new_bucket_metadata_in(&self.ctx, meta.clone()).await?;
694 if bucket_lifecycle_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
695 || metadata_transaction_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
696 || ns_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
697 {
698 return Err(Error::other(format!("bucket metadata creation intent lock was lost: {bucket}")));
699 }
700 }
701
702 if confirmed_missing && !is_meta_bucketname(bucket) {
703 crate::store::list_objects::observe_scanner_namespace_mutations(bucket, 1);
707 self.cleanup_bucket_usage(bucket, ns_guard.as_ref()).await?;
708 }
709
710 if let Err(err) = await_bucket_lifecycle_operation(
711 bucket_lifecycle_guard.as_ref(),
712 ns_guard.as_ref(),
713 bucket,
714 "physical bucket creation",
715 await_bucket_namespace_operation(
716 metadata_transaction_guard.as_ref(),
717 bucket,
718 "bucket creation metadata transaction",
719 async {
720 self.make_bucket_on_sets(bucket, opts)
721 .await
722 .map_err(|err| to_object_err(err, vec![bucket]))
723 },
724 ),
725 )
726 .await
727 {
728 if is_err_bucket_exists(&err)
729 && let Err(heal_err) = self
730 .handle_heal_bucket(
731 bucket,
732 &HealOpts {
733 recreate: true,
734 ..Default::default()
735 },
736 )
737 .await
738 {
739 warn!("best-effort bucket heal after BucketExists failed: {heal_err}");
740 }
741 if confirmed_missing && !is_err_bucket_exists(&err) && ns_guard.as_ref().is_none_or(|guard| !guard.is_lock_lost()) {
742 error!("make bucket failed: {err}");
743 self.rollback_failed_bucket_creation(bucket, bucket_lifecycle_guard.as_ref(), ns_guard.as_ref())
744 .await;
745 }
746 return Err(err);
747 };
748
749 let metadata_result = async {
750 if metadata_persisted_before_physical {
751 return Ok(());
752 }
753 if is_meta_bucketname(bucket) {
754 metadata_sys::set_bucket_metadata_in(&self.ctx, meta).await
755 } else if existing_incarnation_is_authoritative && !lock_newly_enabled {
756 metadata_sys::cache_bucket_metadata_in(&self.ctx, meta).await
757 } else {
758 metadata_sys::set_new_bucket_metadata_in(&self.ctx, meta).await
759 }
760 }
761 .await;
762 let metadata_lock_lost = bucket_lifecycle_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
763 || metadata_transaction_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
764 || ns_guard.as_ref().is_some_and(|guard| guard.is_lock_lost());
765 if let Err(err) = metadata_result {
766 if !metadata_lock_lost {
767 self.rollback_failed_bucket_creation(bucket, bucket_lifecycle_guard.as_ref(), ns_guard.as_ref())
768 .await;
769 }
770 return Err(err);
771 }
772 if metadata_lock_lost {
773 return Err(Error::other(format!("bucket metadata initialization lock was lost: {bucket}")));
774 }
775
776 if confirmed_missing && !is_meta_bucketname(bucket) {
777 crate::store::list_objects::observe_scanner_namespace_mutations(bucket, 1);
781 }
782
783 Ok(())
784 }
785
786 pub(crate) async fn bucket_exists_for_heal(&self, bucket: &str) -> Result<bool> {
791 let results = futures::future::join_all(
792 self.bucket_sets()
793 .map(|(_, _, set)| async move { set.get_bucket_info(bucket, &BucketOptions::default()).await }),
794 )
795 .await;
796 let mut first_error = None;
797 for result in results {
798 match result {
799 Ok(_) => return Ok(true),
800 Err(err) if is_err_strict_volume_not_found(&err) => {}
801 Err(err) if first_error.is_none() => first_error = Some(err),
802 Err(_) => {}
803 }
804 }
805 match first_error {
806 Some(err) => Err(err),
807 None => Ok(false),
808 }
809 }
810
811 #[instrument(skip(self))]
812 pub(crate) async fn get_bucket_info_from_sets(&self, bucket: &str, opts: &BucketOptions) -> Result<BucketInfo> {
813 self.get_bucket_info_from_sets_with_quorum(bucket, opts, BucketInfoQuorum::Write)
814 .await
815 }
816
817 pub(crate) async fn get_bucket_info_from_sets_at_read_quorum(
818 &self,
819 bucket: &str,
820 opts: &BucketOptions,
821 ) -> Result<BucketInfo> {
822 self.get_bucket_info_from_sets_with_quorum(bucket, opts, BucketInfoQuorum::Read)
823 .await
824 }
825
826 async fn get_bucket_info_from_sets_with_quorum(
827 &self,
828 bucket: &str,
829 opts: &BucketOptions,
830 quorum: BucketInfoQuorum,
831 ) -> Result<BucketInfo> {
832 let mut scoped_results = futures::future::join_all(self.bucket_sets().map(|(pool_index, set_index, set)| async move {
839 let result = match quorum {
840 BucketInfoQuorum::Read => set.stat_bucket_with_quorum(bucket, quorum).await,
841 BucketInfoQuorum::Write => set.get_bucket_info(bucket, opts).await,
842 };
843 (pool_index, set_index, result)
844 }))
845 .await;
846 scoped_results.sort_unstable_by_key(|(pool_index, set_index, _)| (*pool_index, *set_index));
847
848 let mut first_info = None;
849 let mut first_error = None;
850 for (_, _, result) in scoped_results {
851 match result {
852 Ok(info) if first_info.is_none() => first_info = Some(info),
853 Ok(_) => {}
854 Err(err) if is_err_strict_volume_not_found(&err) => {}
855 Err(err) if first_error.is_none() => first_error = Some(err),
856 Err(_) => {}
857 }
858 }
859
860 if let Some(err) = first_error {
861 return Err(err);
862 }
863
864 first_info.ok_or(Error::VolumeNotFound)
865 }
866
867 #[instrument(skip(self))]
868 pub(super) async fn handle_get_bucket_info(&self, bucket: &str, opts: &BucketOptions) -> Result<BucketInfo> {
869 let mut info = match self.get_bucket_info_from_sets(bucket, opts).await {
870 Ok(info) => info,
871 Err(Error::ErasureWriteQuorum) => return self.get_bucket_info_at_read_quorum(bucket, opts).await,
872 Err(err) => return Err(err),
873 };
874
875 if let Ok(sys) = metadata_sys::get_in(&self.ctx, bucket).await {
876 if should_override_created_from_metadata(sys.created) {
877 info.created = Some(sys.created);
878 }
879 info.versioning = sys.versioning();
880 info.object_locking = sys.object_locking();
881 }
882
883 Ok(info)
884 }
885
886 async fn get_bucket_info_at_read_quorum(&self, bucket: &str, opts: &BucketOptions) -> Result<BucketInfo> {
887 let guard = self.acquire_bucket_lifecycle_read_lock(bucket).await?;
891 await_bucket_namespace_operation(Some(&guard), bucket, "bucket read quorum validation", async {
892 let mut info = self
893 .get_bucket_info_from_sets_with_quorum(bucket, opts, BucketInfoQuorum::Read)
894 .await?;
895 let (metadata, persisted) = metadata_sys::get_config_from_disk_with_presence_in(&self.ctx, bucket).await?;
896 if !persisted {
897 return Err(Error::ErasureReadQuorum);
901 }
902 if metadata.name != bucket {
903 return Err(Error::FileCorrupt);
904 }
905 if should_override_created_from_metadata(metadata.created) {
906 info.created = Some(metadata.created);
907 }
908 info.versioning = metadata.versioning();
909 info.object_locking = metadata.object_locking();
910 Ok(info)
911 })
912 .await
913 }
914
915 #[instrument(skip(self))]
916 pub(super) async fn handle_list_bucket(&self, opts: &BucketOptions) -> Result<Vec<BucketInfo>> {
917 Ok(self.list_bucket_from_sets(opts, usize::MAX).await?.buckets)
919 }
920
921 pub async fn list_bucket_for_scanner(&self, opts: &BucketOptions) -> Result<crate::cluster::rpc::ScannerBucketListing> {
922 self.list_bucket_from_sets(opts, SCANNER_BUCKET_LIST_SET_CONCURRENCY).await
923 }
924
925 async fn list_bucket_from_sets(
926 &self,
927 opts: &BucketOptions,
928 max_concurrency: usize,
929 ) -> Result<crate::cluster::rpc::ScannerBucketListing> {
930 let set_count = self.pools.iter().map(|pool| pool.disk_set.len()).sum();
931 let concurrency = bucket_list_set_concurrency(set_count, max_concurrency);
932 let deleted = opts.deleted;
933 let cached = opts.cached;
934 let no_metadata = opts.no_metadata;
935 let mut set_listings = stream::iter(self.bucket_sets().map(move |(pool_index, set_index, set)| {
936 let opts = BucketOptions {
937 deleted,
938 cached,
939 no_metadata,
940 };
941 async move {
942 set.list_bucket_for_scanner(&opts)
943 .await
944 .map(|(buckets, complete)| (pool_index, set_index, buckets, complete))
945 }
946 }))
947 .buffer_unordered(concurrency);
948 let mut topology_complete = set_count != 0;
949 let mut bucket_map = BTreeMap::<String, BucketInfo>::new();
950 let mut scoped_buckets = Vec::with_capacity(set_count);
951 while let Some(set_listing) = set_listings.next().await {
952 let (pool_index, set_index, buckets, set_complete) = set_listing?;
953 topology_complete &= set_complete;
954 for bucket in &buckets {
955 bucket_map.entry(bucket.name.clone()).or_insert_with(|| bucket.clone());
956 }
957 scoped_buckets.push(crate::cluster::rpc::ScannerSetBucketListing {
958 pool_index,
959 set_index,
960 buckets,
961 });
962 }
963 scoped_buckets.sort_unstable_by_key(|scope| (scope.pool_index, scope.set_index));
964 let mut listing = crate::cluster::rpc::ScannerBucketListing {
965 buckets: bucket_map.into_values().collect(),
966 set_buckets: scoped_buckets,
967 topology_complete,
968 };
969
970 if !opts.no_metadata {
971 for bucket in &mut listing.buckets {
972 if let Ok(created) = metadata_sys::created_at_in(&self.ctx, &bucket.name).await
973 && should_override_created_from_metadata(created)
974 {
975 bucket.created = Some(created);
976 }
977 }
978 let created_by_bucket = listing
979 .buckets
980 .iter()
981 .map(|bucket| (bucket.name.as_str(), bucket.created))
982 .collect::<BTreeMap<_, _>>();
983 for scope in &mut listing.set_buckets {
984 for bucket in &mut scope.buckets {
985 if let Some(created) = created_by_bucket.get(bucket.name.as_str()) {
986 bucket.created = *created;
987 }
988 }
989 }
990 }
991
992 Ok(listing)
993 }
994
995 #[instrument(skip(self))]
996 pub(super) async fn handle_delete_bucket(&self, bucket: &str, opts: &DeleteBucketOptions) -> Result<()> {
997 self.handle_delete_bucket_with_diagnostic_budget(bucket, opts, BucketDeleteDiagnosticBudget::new())
998 .await
999 }
1000
1001 async fn handle_delete_bucket_with_diagnostic_budget(
1002 &self,
1003 bucket: &str,
1004 opts: &DeleteBucketOptions,
1005 mut diagnostic_budget: BucketDeleteDiagnosticBudget,
1006 ) -> Result<()> {
1007 if is_meta_bucketname(bucket) {
1008 return Err(StorageError::BucketNameInvalid(bucket.to_string()));
1009 }
1010
1011 if let Err(err) = check_valid_bucket_name(bucket) {
1012 return Err(StorageError::BucketNameInvalid(err.to_string()));
1013 }
1014
1015 let bucket_lifecycle_guard = if !opts.no_lock {
1016 Some(self.acquire_bucket_lifecycle_write_lock(bucket).await?)
1017 } else {
1018 None
1019 };
1020 let ns_guard = if !opts.no_lock {
1021 let ns_lock = self.new_ns_lock(bucket, bucket).await?;
1022 Some(
1023 await_bucket_namespace_operation(
1024 bucket_lifecycle_guard.as_ref(),
1025 bucket,
1026 "bucket namespace lock acquisition",
1027 async {
1028 ns_lock.get_write_lock(get_lock_acquire_timeout()).await.map_err(|e| match e {
1029 rustfs_lock::error::LockError::QuorumNotReached { required, achieved } => {
1030 StorageError::NamespaceLockQuorumUnavailable {
1031 mode: "write",
1032 bucket: bucket.to_string(),
1033 object: bucket.to_string(),
1034 required,
1035 achieved,
1036 }
1037 }
1038 other => StorageError::Lock(other),
1039 })
1040 },
1041 )
1042 .await?,
1043 )
1044 } else {
1045 None
1046 };
1047
1048 let sr_mark_delete = opts.srdelete_op == SRBucketDeleteOp::MarkDelete;
1049 let sr_purge = opts.srdelete_op == SRBucketDeleteOp::Purge;
1050 let sr_delete = sr_mark_delete || sr_purge;
1051 let mut delete_opts = opts.clone();
1052 let bucket_exists = match self.get_bucket_info_from_sets(bucket, &BucketOptions::default()).await {
1053 Ok(_) => true,
1054 Err(err) => {
1055 if is_err_strict_volume_not_found(&err) && sr_delete {
1056 false
1057 } else if is_err_strict_volume_not_found(&err) {
1058 return Err(StorageError::BucketNotFound(bucket.to_string()));
1059 } else {
1060 return Err(to_object_err(err, vec![bucket]));
1061 }
1062 }
1063 };
1064 if bucket_exists {
1065 validate_table_bucket_delete_guard(&self.ctx, bucket).await?;
1066
1067 if !opts.force {
1068 if let Some(blocker) = bucket_delete_local_blocker(&self.ctx, bucket, &mut diagnostic_budget).await? {
1069 return Err(blocker);
1070 }
1071 delete_opts.force_if_empty = true;
1072 }
1073 }
1074
1075 if sr_delete && !bucket_exists {
1076 delete_opts.force_if_empty = true;
1077 }
1078
1079 #[cfg(test)]
1080 crate::cluster::rpc::peer_s3_client::pause_after_delete_bucket_empty_scan().await;
1081
1082 if sr_mark_delete {
1083 await_bucket_lifecycle_operation(
1084 bucket_lifecycle_guard.as_ref(),
1085 ns_guard.as_ref(),
1086 bucket,
1087 "bucket delete marker creation",
1088 self.mark_bucket_deleted(bucket),
1089 )
1090 .await?;
1091 }
1092
1093 let retirement = if bucket_exists && bucket_lifecycle_guard.is_some() && !is_meta_bucketname(bucket) {
1096 if let Some(store) = metadata_sys::object_store_if_initialized_in(&self.ctx).await {
1097 crate::bucket::metadata::load_bucket_incarnation(store.clone(), bucket)
1098 .await?
1099 .map(|incarnation| (store, incarnation))
1100 } else {
1101 None
1102 }
1103 } else {
1104 None
1105 };
1106
1107 let delete_result = await_bucket_namespace_operation(
1108 bucket_lifecycle_guard.as_ref(),
1109 bucket,
1110 "physical bucket deletion",
1111 run_physical_bucket_deletion(ns_guard.as_ref(), bucket, async {
1112 self.delete_bucket_on_sets(bucket, &delete_opts)
1113 .await
1114 .map_err(|err| to_object_err(err, vec![bucket]))
1115 }),
1116 )
1117 .await;
1118 if let Err(err) = delete_result
1119 && (!sr_delete || !is_err_strict_volume_not_found(&err))
1120 {
1121 if delete_opts.force_if_empty && matches!(&err, StorageError::BucketNotEmpty(_)) {
1122 let mut diagnostic_budget = BucketDeleteDiagnosticBudget::new();
1123 if let Some(blocker) = bucket_delete_local_blocker(&self.ctx, bucket, &mut diagnostic_budget).await? {
1124 return Err(blocker);
1125 }
1126 }
1127 return Err(err);
1128 }
1129
1130 if let Some((store, incarnation)) = retirement {
1131 let mut record_opts = ObjectOptions {
1132 max_parity: true,
1133 ..Default::default()
1134 };
1135 if let Some(guard) = bucket_lifecycle_guard.as_ref() {
1136 record_opts.add_bucket_lifecycle_lock_guard(guard);
1137 }
1138 if let Some(guard) = ns_guard.as_ref() {
1139 record_opts.add_namespace_lock_guard(guard);
1140 }
1141 await_bucket_lifecycle_operation(
1142 bucket_lifecycle_guard.as_ref(),
1143 ns_guard.as_ref(),
1144 bucket,
1145 "bucket retirement publication",
1146 crate::bucket::retirement::commit_retirement(store, bucket, incarnation, &record_opts),
1147 )
1148 .await?;
1149 }
1150
1151 self.cleanup_bucket_usage_best_effort(bucket, ns_guard.as_ref()).await;
1152
1153 if let Err(err) = self
1154 .cleanup_deleted_bucket_metadata(bucket, sr_purge, ns_guard.as_ref())
1155 .await
1156 {
1157 warn!(
1158 bucket = %bucket,
1159 error = ?err,
1160 "physical bucket deletion succeeded but metadata cleanup remains pending"
1161 );
1162 }
1163 crate::store::list_objects::observe_scanner_namespace_mutations(bucket, 1);
1166 Ok(())
1167 }
1168}
1169
1170#[cfg(test)]
1171mod tests {
1172 use super::{
1173 BUCKET_DELETE_DIAGNOSTIC_MAX_ELAPSED, BUCKET_DELETE_DIAGNOSTIC_MAX_ENTRIES, BUCKET_DELETE_XLMETA_DIAGNOSTIC_MAX_BYTES,
1174 BucketDeleteBlockerKind, BucketDeleteDiagnosticBudget, BucketMetadataLessResidue, SCANNER_BUCKET_LIST_SET_CONCURRENCY,
1175 await_bucket_namespace_operation, bucket_delete_metadata_cleanup_prefixes, bucket_deleted_marker_prefix,
1176 bucket_deleted_marker_volume, bucket_list_set_concurrency, record_bucket_delete_blocker, run_bucket_usage_cleanup,
1177 run_physical_bucket_deletion, scan_metadata_less_residue, scan_metadata_less_residue_with_budget,
1178 should_override_created_from_metadata, validate_table_bucket_delete_allowed,
1179 };
1180 use crate::bucket::metadata::{BucketMetadata, table_bucket_catalog_metadata_prefix};
1181 use crate::bucket::metadata_sys;
1182 use crate::cluster::rpc::peer_s3_client::install_delete_bucket_empty_scan_barrier;
1183 use crate::disk::{BUCKET_META_PREFIX, DiskAPI, RUSTFS_META_BUCKET, STORAGE_FORMAT_FILE};
1184 use crate::error::StorageError;
1185 use crate::object_api::{ObjectOptions, PutObjReader};
1186 use crate::runtime::instance::InstanceContext;
1187 use crate::storage_api_contracts::{
1188 bucket::{BucketOperations as _, BucketOptions, DeleteBucketOptions, MakeBucketOptions, SRBucketDeleteOp},
1189 list::ListOperations as _,
1190 namespace::NamespaceLocking as _,
1191 object::{ObjectIO as _, ObjectOperations as _},
1192 };
1193 use crate::store::{ECStore, init_local_disks_with_instance_ctx};
1194 use crate::{
1195 disk::endpoint::Endpoint,
1196 layout::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints},
1197 };
1198 use rustfs_data_usage::{BucketUsageInfo, DATA_USAGE_OBJECT_NAME, DataUsageInfo};
1199 use rustfs_filemeta::{FileInfo, FileMeta, TRANSITION_COMPLETE};
1200 use rustfs_lock::{LocalClient, LockRequest, LockType, NamespaceLock, ObjectKey};
1201 use serial_test::serial;
1202 use std::path::{Path, PathBuf};
1203 use std::sync::Arc;
1204 use std::sync::atomic::{AtomicBool, Ordering};
1205 use std::time::{Duration, SystemTime};
1206 use time::OffsetDateTime;
1207 use tokio::io::AsyncReadExt;
1208 use tokio::sync::{Notify, OnceCell};
1209 use tokio_util::sync::CancellationToken;
1210 use uuid::Uuid;
1211
1212 static BUCKET_DELETE_TEST_ENV: OnceCell<(Vec<PathBuf>, Arc<ECStore>)> = OnceCell::const_new();
1213
1214 #[tokio::test(start_paused = true)]
1215 async fn bucket_delete_diagnostic_budget_starts_with_first_scan_io_and_latches_once() {
1216 let mut budget = BucketDeleteDiagnosticBudget::with_limits(8, Duration::from_millis(100));
1217 tokio::time::advance(Duration::from_secs(10)).await;
1218
1219 let first_polled = Arc::new(AtomicBool::new(false));
1220 let first_polled_for_io = first_polled.clone();
1221 let first = budget
1222 .run_io(async move {
1223 first_polled_for_io.store(true, Ordering::SeqCst);
1224 Ok::<_, std::io::Error>(7_u8)
1225 })
1226 .await
1227 .expect("the first diagnostic IO should succeed");
1228 assert_eq!(first, Some(7));
1229 assert!(first_polled.load(Ordering::SeqCst));
1230
1231 tokio::time::advance(Duration::from_millis(101)).await;
1232 let expired_polled = Arc::new(AtomicBool::new(false));
1233 let expired_polled_for_io = expired_polled.clone();
1234 let expired = budget
1235 .run_io(async move {
1236 expired_polled_for_io.store(true, Ordering::SeqCst);
1237 Ok::<_, std::io::Error>(9_u8)
1238 })
1239 .await
1240 .expect("an expired diagnostic budget should not become an IO error");
1241 assert_eq!(expired, None);
1242 assert!(
1243 !expired_polled.load(Ordering::SeqCst),
1244 "the deadline must remain latched after the first scan IO"
1245 );
1246 }
1247
1248 #[tokio::test(start_paused = true)]
1249 async fn bucket_delete_diagnostic_budget_times_out_its_first_pending_io() {
1250 let mut budget = BucketDeleteDiagnosticBudget::with_limits(8, Duration::from_millis(100));
1251 let io_polled = Arc::new(AtomicBool::new(false));
1252 let io_polled_for_future = io_polled.clone();
1253
1254 let result = budget
1255 .run_io(std::future::poll_fn(move |_cx| {
1256 io_polled_for_future.store(true, Ordering::SeqCst);
1257 std::task::Poll::<std::io::Result<()>>::Pending
1258 }))
1259 .await
1260 .expect("a diagnostic timeout should fail closed without an IO error");
1261
1262 assert_eq!(result, None);
1263 assert!(
1264 io_polled.load(Ordering::SeqCst),
1265 "the first diagnostic IO must be polled before its timeout"
1266 );
1267 }
1268
1269 #[tokio::test(start_paused = true)]
1270 async fn delayed_metadata_less_scans_still_detect_orphans_and_xlmeta() {
1271 let root = tempfile::tempdir().expect("temporary delayed-scan roots should be created");
1272 let orphan_root = root.path().join("orphan-root");
1273 let xlmeta_root = root.path().join("xlmeta-root");
1274 std::fs::create_dir_all(&orphan_root).expect("orphan root should be created");
1275 std::fs::create_dir_all(&xlmeta_root).expect("xlmeta root should be created");
1276 std::fs::write(orphan_root.join("orphan-part"), b"orphan").expect("orphan fixture should be written");
1277 std::fs::write(xlmeta_root.join(STORAGE_FORMAT_FILE), b"invalid-xlmeta").expect("xl.meta fixture should be written");
1278
1279 let mut orphan_budget = BucketDeleteDiagnosticBudget::with_limits(16, Duration::from_secs(5));
1280 let mut xlmeta_budget = BucketDeleteDiagnosticBudget::with_limits(16, Duration::from_secs(5));
1281 tokio::time::advance(Duration::from_secs(60)).await;
1282
1283 let orphan = scan_metadata_less_residue_with_budget(&orphan_root, &mut orphan_budget)
1284 .await
1285 .expect("delayed orphan scan should complete");
1286 assert!(orphan.has_residue_without_xlmeta());
1287 assert!(!orphan.diagnostic_truncated);
1288
1289 let xlmeta = scan_metadata_less_residue_with_budget(&xlmeta_root, &mut xlmeta_budget)
1290 .await
1291 .expect("delayed xl.meta scan should complete");
1292 assert!(xlmeta.xlmeta_found);
1293 assert!(!xlmeta.diagnostic_truncated);
1294 }
1295
1296 #[tokio::test(start_paused = true)]
1297 async fn bucket_namespace_operation_fails_closed_after_lease_expiry() {
1298 let ttl = Duration::from_millis(20);
1299 let lock = NamespaceLock::new("bucket-operation-test".to_string(), Arc::new(LocalClient::new()));
1300 let request = LockRequest::new(ObjectKey::new("bucket", ""), LockType::Exclusive, "test-owner")
1301 .with_acquire_timeout(Duration::from_secs(1))
1302 .with_ttl(ttl)
1303 .with_refresh_interval(ttl);
1304 let guard = lock
1305 .acquire_guard(&request)
1306 .await
1307 .expect("namespace lock acquisition should not fail")
1308 .expect("namespace lock should be acquired");
1309 tokio::time::advance(ttl + Duration::from_millis(1)).await;
1310
1311 let operation_ran = Arc::new(AtomicBool::new(false));
1312 let operation_ran_for_future = operation_ran.clone();
1313 let result = await_bucket_namespace_operation(Some(&guard), "bucket", "test operation", async move {
1314 operation_ran_for_future.store(true, Ordering::SeqCst);
1315 Ok(())
1316 })
1317 .await;
1318
1319 assert!(result.is_err(), "an expired namespace lease must fence the operation");
1320 assert!(
1321 !operation_ran.load(Ordering::SeqCst),
1322 "a fenced namespace operation must not poll its mutation future"
1323 );
1324 }
1325
1326 #[tokio::test]
1327 #[serial]
1328 async fn physical_bucket_delete_fences_scanner_before_polling_storage() {
1329 let generation_before = crate::store::list_objects::scanner_namespace_mutation_generation();
1330 let storage_polled = Arc::new(AtomicBool::new(false));
1331 let storage_polled_for_future = storage_polled.clone();
1332
1333 run_physical_bucket_deletion(None, "generation-order-bucket", async move {
1334 assert!(
1335 crate::store::list_objects::scanner_namespace_mutation_generation() > generation_before,
1336 "scanner generation must advance before physical deletion is polled"
1337 );
1338 storage_polled_for_future.store(true, Ordering::SeqCst);
1339 Ok(())
1340 })
1341 .await
1342 .expect("synthetic physical deletion should succeed");
1343
1344 assert!(storage_polled.load(Ordering::SeqCst));
1345 }
1346
1347 #[tokio::test]
1348 async fn bucket_usage_cleanup_stops_after_parent_cancellation() {
1349 let started = Arc::new(Notify::new());
1350 let started_wait = started.notified();
1351 let release = Arc::new(Notify::new());
1352 let completed = Arc::new(AtomicBool::new(false));
1353 let started_for_cleanup = started.clone();
1354 let release_for_cleanup = release.clone();
1355 let completed_for_cleanup = completed.clone();
1356 let parent = tokio::spawn(run_bucket_usage_cleanup(None, "bucket", async move {
1357 started_for_cleanup.notify_one();
1358 release_for_cleanup.notified().await;
1359 completed_for_cleanup.store(true, Ordering::SeqCst);
1360 Ok(())
1361 }));
1362 started_wait.await;
1363
1364 parent.abort();
1365 let _ = parent.await;
1366 tokio::task::yield_now().await;
1367 release.notify_waiters();
1368 tokio::task::yield_now().await;
1369 assert!(
1370 !completed.load(Ordering::SeqCst),
1371 "a cancelled cleanup future must not continue in a detached task"
1372 );
1373 }
1374
1375 #[tokio::test(start_paused = true)]
1376 #[serial]
1377 async fn bucket_namespace_operation_stops_in_flight_work_after_lock_loss() {
1378 let ttl = Duration::from_millis(20);
1379 let lock = NamespaceLock::new("bucket-operation-in-flight-loss-test".to_string(), Arc::new(LocalClient::new()));
1380 let request = LockRequest::new(ObjectKey::new("bucket", ""), LockType::Exclusive, "test-owner")
1381 .with_acquire_timeout(Duration::from_secs(1))
1382 .with_ttl(ttl)
1383 .with_refresh_interval(ttl);
1384 let guard = lock
1385 .acquire_guard(&request)
1386 .await
1387 .expect("namespace lock acquisition should not fail")
1388 .expect("namespace lock should be acquired");
1389
1390 let operation_started = Arc::new(Notify::new());
1391 let started_wait = operation_started.notified();
1392 let operation_release = Arc::new(Notify::new());
1393 let operation_completed = Arc::new(AtomicBool::new(false));
1394 let started_for_operation = operation_started.clone();
1395 let release_for_operation = operation_release.clone();
1396 let completed_for_operation = operation_completed.clone();
1397 let task = tokio::spawn(async move {
1398 await_bucket_namespace_operation(Some(&guard), "bucket", "test operation", async move {
1399 started_for_operation.notify_one();
1400 release_for_operation.notified().await;
1401 completed_for_operation.store(true, Ordering::SeqCst);
1402 Ok(())
1403 })
1404 .await
1405 });
1406 started_wait.await;
1407
1408 tokio::time::advance(ttl + Duration::from_millis(1)).await;
1409 let err = task
1410 .await
1411 .expect("operation task should join")
1412 .expect_err("an operation still running after lease loss must be fenced");
1413 assert!(err.to_string().contains("namespace lock was lost during test operation"));
1414 operation_release.notify_waiters();
1415 tokio::task::yield_now().await;
1416 assert!(
1417 !operation_completed.load(Ordering::SeqCst),
1418 "lock loss must stop the old owner before a successor can acquire the namespace"
1419 );
1420 }
1421
1422 async fn setup_bucket_delete_test_env() -> (Vec<PathBuf>, Arc<ECStore>) {
1423 BUCKET_DELETE_TEST_ENV
1424 .get_or_init(|| async {
1425 let temp_dir = std::env::temp_dir().join(format!("rustfs_bucket_delete_test_{}", Uuid::new_v4()));
1426 tokio::fs::create_dir_all(&temp_dir)
1427 .await
1428 .expect("test base directory should be created");
1429
1430 let disk_paths = (0..4)
1431 .map(|disk_idx| temp_dir.join(format!("disk{disk_idx}")))
1432 .collect::<Vec<_>>();
1433
1434 for disk_path in &disk_paths {
1435 tokio::fs::create_dir_all(disk_path)
1436 .await
1437 .expect("disk directory should be created");
1438 }
1439
1440 let mut endpoints = Vec::with_capacity(disk_paths.len());
1441 for (disk_idx, disk_path) in disk_paths.iter().enumerate() {
1442 let mut endpoint =
1443 Endpoint::try_from(disk_path.to_str().expect("disk path should be utf8")).expect("endpoint should parse");
1444 endpoint.set_pool_index(0);
1445 endpoint.set_set_index(0);
1446 endpoint.set_disk_index(disk_idx);
1447 endpoints.push(endpoint);
1448 }
1449
1450 let endpoint_pools = EndpointServerPools(vec![PoolEndpoints {
1451 legacy: false,
1452 set_count: 1,
1453 drives_per_set: 4,
1454 endpoints: Endpoints::from(endpoints),
1455 cmd_line: "bucket-delete-test".to_string(),
1456 platform: format!("OS: {} | Arch: {}", std::env::consts::OS, std::env::consts::ARCH),
1457 }]);
1458
1459 let instance_ctx = Arc::new(InstanceContext::new());
1460 init_local_disks_with_instance_ctx(&instance_ctx, endpoint_pools.clone())
1461 .await
1462 .expect("local disks should initialize");
1463 let ecstore = ECStore::new_with_instance_ctx(
1464 "127.0.0.1:0".parse().expect("test address"),
1465 endpoint_pools,
1466 CancellationToken::new(),
1467 instance_ctx,
1468 )
1469 .await
1470 .expect("ECStore should initialize");
1471 let storage_class = crate::config::storageclass::lookup_config_for_pools_without_env(
1472 &rustfs_config::server_config::KVS::new(),
1473 &[4],
1474 )
1475 .expect("bucket test storage class should match its four-disk pool");
1476 for pool in &ecstore.pools {
1477 for set in &pool.disk_set {
1478 set.set_test_storage_class_config(storage_class.clone());
1479 }
1480 }
1481
1482 metadata_sys::init_bucket_metadata_sys(ecstore.clone(), Vec::new()).await;
1483
1484 (disk_paths, ecstore)
1485 })
1486 .await
1487 .clone()
1488 }
1489
1490 async fn setup_multi_pool_bucket_test_env() -> (tempfile::TempDir, Arc<ECStore>) {
1491 setup_bucket_quorum_test_env(&[4, 4], None).await
1492 }
1493
1494 async fn setup_bucket_quorum_test_env(
1495 drives_per_pool: &[usize],
1496 standard_parity: Option<usize>,
1497 ) -> (tempfile::TempDir, Arc<ECStore>) {
1498 let temp_dir = tempfile::tempdir().expect("multi-pool bucket test directory should be created");
1499 let mut pools = Vec::new();
1500 for (pool_index, &drive_count) in drives_per_pool.iter().enumerate() {
1501 let mut endpoints = Vec::new();
1502 for disk_index in 0..drive_count {
1503 let disk_path = temp_dir.path().join(format!("pool{pool_index}-disk{disk_index}"));
1504 tokio::fs::create_dir_all(&disk_path)
1505 .await
1506 .expect("multi-pool bucket test disk should be created");
1507 let mut endpoint =
1508 Endpoint::try_from(disk_path.to_str().expect("disk path should be utf8")).expect("endpoint should parse");
1509 endpoint.set_pool_index(pool_index);
1510 endpoint.set_set_index(0);
1511 endpoint.set_disk_index(disk_index);
1512 endpoints.push(endpoint);
1513 }
1514 pools.push(PoolEndpoints {
1515 legacy: false,
1516 set_count: 1,
1517 drives_per_set: drive_count,
1518 endpoints: Endpoints::from(endpoints),
1519 cmd_line: format!("bucket-test-pool-{pool_index}"),
1520 platform: format!("OS: {} | Arch: {}", std::env::consts::OS, std::env::consts::ARCH),
1521 });
1522 }
1523
1524 let endpoint_pools = EndpointServerPools(pools);
1525 let instance_ctx = Arc::new(InstanceContext::new());
1526 instance_ctx.set_endpoints(endpoint_pools.clone());
1527 init_local_disks_with_instance_ctx(&instance_ctx, endpoint_pools.clone())
1528 .await
1529 .expect("multi-pool local disks should initialize");
1530 let ecstore = ECStore::new_with_instance_ctx(
1531 "127.0.0.1:0".parse().expect("test address"),
1532 endpoint_pools,
1533 CancellationToken::new(),
1534 instance_ctx,
1535 )
1536 .await
1537 .expect("multi-pool ECStore should initialize");
1538 let mut storage_class_kvs = rustfs_config::server_config::KVS::new();
1539 if let Some(parity) = standard_parity {
1540 storage_class_kvs.insert(crate::config::storageclass::CLASS_STANDARD.to_string(), format!("EC:{parity}"));
1541 }
1542 let storage_class = crate::config::storageclass::lookup_config_for_pools_without_env(&storage_class_kvs, drives_per_pool)
1543 .expect("storage class should match every test erasure set");
1544 for pool in &ecstore.pools {
1545 for set in &pool.disk_set {
1546 set.set_test_storage_class_config(storage_class.clone());
1547 }
1548 }
1549
1550 (temp_dir, ecstore)
1551 }
1552
1553 async fn take_set_disks_offline(
1554 ecstore: &ECStore,
1555 set: &Arc<crate::set_disk::SetDisks>,
1556 disk_indexes: &[usize],
1557 ) -> Vec<(usize, crate::disk::DiskStore)> {
1558 let offline = {
1559 let mut disks = set.disks.write().await;
1560 disk_indexes
1561 .iter()
1562 .map(|index| (*index, disks[*index].take().expect("fault-injection disk should start online")))
1563 .collect::<Vec<_>>()
1564 };
1565 let local_disk_map = ecstore.ctx.local_disk_map();
1566 let mut local_disks = local_disk_map.write().await;
1567 for (_, disk) in &offline {
1568 local_disks.insert(disk.endpoint().to_string(), None);
1569 }
1570 offline
1571 }
1572
1573 async fn restore_set_disks(
1574 ecstore: &ECStore,
1575 set: &Arc<crate::set_disk::SetDisks>,
1576 offline: Vec<(usize, crate::disk::DiskStore)>,
1577 ) {
1578 {
1579 let local_disk_map = ecstore.ctx.local_disk_map();
1580 let mut local_disks = local_disk_map.write().await;
1581 for (_, disk) in &offline {
1582 local_disks.insert(disk.endpoint().to_string(), Some(Arc::clone(disk)));
1583 }
1584 }
1585 let mut disks = set.disks.write().await;
1586 for (index, disk) in offline {
1587 assert!(disks[index].replace(disk).is_none(), "fault-injection disk slot should remain empty");
1588 }
1589 }
1590
1591 #[tokio::test]
1592 async fn request_metadata_methods_fail_closed_before_instance_initialization() {
1593 let (_temp_dir, store) = setup_multi_pool_bucket_test_env().await;
1594
1595 let expected = "bucket metadata sys not initialized for this instance";
1596 let errors = [
1597 store.get_bucket_metadata("bucket").await.unwrap_err(),
1598 store.get_bucket_policy("bucket").await.unwrap_err(),
1599 store.get_bucket_policy_raw("bucket").await.unwrap_err(),
1600 store.restricts_public_bucket_access("bucket").await.unwrap_err(),
1601 store
1602 .update_bucket_metadata_config("bucket", crate::bucket::metadata::BUCKET_POLICY_CONFIG, Vec::new())
1603 .await
1604 .unwrap_err(),
1605 ];
1606 for error in errors {
1607 assert_eq!(error.to_string(), format!("Io error: {expected}"));
1608 }
1609 }
1610
1611 async fn create_bucket_with_object(ecstore: &Arc<ECStore>, bucket: &str, object: &str) {
1612 let generation_before_make = ecstore.scanner_namespace_mutation_generation();
1613 ecstore
1614 .make_bucket(bucket, &MakeBucketOptions::default())
1615 .await
1616 .expect("bucket should be created");
1617 assert_eq!(
1618 ecstore.scanner_namespace_mutation_generation(),
1619 generation_before_make.saturating_add(2),
1620 "successful bucket creation should fence scanner namespace activity before and after creation"
1621 );
1622
1623 let generation_before_put = ecstore.scanner_namespace_mutation_generation();
1624 let mut reader = PutObjReader::from_vec(b"delete bucket semantics".to_vec());
1625 ecstore
1626 .put_object(bucket, object, &mut reader, &ObjectOptions::default())
1627 .await
1628 .expect("object should be written");
1629 let lock = ecstore.pools[0].disk_set[0]
1630 .new_ns_lock(bucket, object)
1631 .await
1632 .expect("fixture namespace lock should be created");
1633 drop(
1634 lock.get_write_lock(Duration::from_secs(30))
1635 .await
1636 .expect("fixture rename tail should finish before checking its generation"),
1637 );
1638 assert_eq!(
1639 ecstore.scanner_namespace_mutation_generation(),
1640 generation_before_put.saturating_add(3),
1641 "successful object creation must observe the logical mutation and both fanout boundaries"
1642 );
1643 ecstore
1644 .get_object_info(bucket, object, &ObjectOptions::default())
1645 .await
1646 .expect("object should be readable before bucket delete");
1647 }
1648
1649 async fn any_disk_path_exists(disk_paths: &[PathBuf], relative_path: impl AsRef<Path>) -> bool {
1650 for disk_path in disk_paths {
1651 if tokio::fs::try_exists(disk_path.join(relative_path.as_ref()))
1652 .await
1653 .expect("test disk path should be stat-able")
1654 {
1655 return true;
1656 }
1657 }
1658 false
1659 }
1660
1661 async fn any_disk_has_object_metadata(disk_paths: &[PathBuf], bucket: &str) -> bool {
1662 for disk_path in disk_paths {
1663 if super::has_xlmeta_files(&disk_path.join(bucket))
1664 .await
1665 .expect("object metadata scan should succeed")
1666 {
1667 return true;
1668 }
1669 }
1670 false
1671 }
1672
1673 async fn write_metadata_less_part_on_all_disks(disk_paths: &[PathBuf], bucket: &str, object: &str) {
1674 for disk_path in disk_paths {
1675 let data_dir = Uuid::new_v4();
1676 let part_path = disk_path.join(bucket).join(object).join(data_dir.to_string()).join("part.1");
1677 tokio::fs::create_dir_all(part_path.parent().expect("part path should have a parent"))
1678 .await
1679 .expect("metadata-less data dir should be created");
1680 tokio::fs::write(part_path, b"orphan shard")
1681 .await
1682 .expect("metadata-less part should be written");
1683 }
1684 }
1685
1686 async fn write_bucket_metadata_marker(disk_paths: &[PathBuf], metadata_prefix: &str) {
1687 for disk_path in disk_paths {
1688 let marker_path = disk_path.join(metadata_prefix).join("config.json");
1689 tokio::fs::create_dir_all(marker_path.parent().expect("metadata marker path should have a parent"))
1690 .await
1691 .expect("metadata marker parent should be created");
1692 tokio::fs::write(marker_path, b"bucket metadata")
1693 .await
1694 .expect("metadata marker should be written");
1695 }
1696 }
1697
1698 #[test]
1699 fn should_not_override_when_metadata_created_is_unix_epoch() {
1700 assert!(!should_override_created_from_metadata(OffsetDateTime::UNIX_EPOCH));
1701 }
1702
1703 #[test]
1704 fn should_override_when_metadata_created_is_valid_time() {
1705 let created = OffsetDateTime::from_unix_timestamp(1704067200).expect("valid timestamp");
1706 assert!(should_override_created_from_metadata(created));
1707 }
1708
1709 #[test]
1710 fn table_bucket_delete_guard_rejects_remaining_catalog_metadata() {
1711 let err = validate_table_bucket_delete_allowed("table-bucket", true, true).unwrap_err();
1712
1713 assert!(matches!(err, StorageError::BucketNotEmpty(bucket) if bucket == "table-bucket"));
1714 assert!(validate_table_bucket_delete_allowed("table-bucket", true, false).is_ok());
1715 assert!(validate_table_bucket_delete_allowed("regular-bucket", false, true).is_ok());
1716 }
1717
1718 #[test]
1719 fn bucket_delete_metadata_cleanup_removes_internal_table_catalog_prefix() {
1720 let prefixes = bucket_delete_metadata_cleanup_prefixes("analytics");
1721
1722 assert!(prefixes.contains(&table_bucket_catalog_metadata_prefix("analytics")));
1723 assert!(prefixes.contains(&"buckets/analytics".to_string()));
1724 }
1725
1726 #[test]
1727 fn bucket_delete_marker_path_uses_internal_deleted_bucket_metadata_prefix() {
1728 assert_eq!(bucket_deleted_marker_prefix("analytics"), "buckets/.deleted/analytics");
1729 assert_eq!(
1730 bucket_deleted_marker_volume("analytics"),
1731 format!("{RUSTFS_META_BUCKET}/{BUCKET_META_PREFIX}/.deleted/analytics")
1732 );
1733 }
1734
1735 #[test]
1736 fn bucket_listing_selects_request_and_scanner_fanout() {
1737 assert_eq!(bucket_list_set_concurrency(0, SCANNER_BUCKET_LIST_SET_CONCURRENCY), 1);
1738 assert_eq!(bucket_list_set_concurrency(2, SCANNER_BUCKET_LIST_SET_CONCURRENCY), 2);
1739 assert_eq!(
1740 bucket_list_set_concurrency(100, SCANNER_BUCKET_LIST_SET_CONCURRENCY),
1741 SCANNER_BUCKET_LIST_SET_CONCURRENCY
1742 );
1743 assert_eq!(bucket_list_set_concurrency(100, usize::MAX), 100);
1744 }
1745
1746 #[tokio::test]
1747 async fn metadata_less_residue_scan_ignores_empty_dirs_and_reports_uuid_data() {
1748 let root = tempfile::tempdir().expect("temporary bucket root should be created");
1749 let bucket_path = root.path().join("bucket");
1750 tokio::fs::create_dir_all(bucket_path.join("empty/child"))
1751 .await
1752 .expect("empty directory residue should be created");
1753
1754 let empty = scan_metadata_less_residue(&bucket_path)
1755 .await
1756 .expect("empty residue scan should succeed");
1757 assert!(!empty.has_residue_without_xlmeta());
1758
1759 let data_dir = Uuid::new_v4();
1760 let part_path = bucket_path.join("object").join(data_dir.to_string()).join("part.1");
1761 tokio::fs::create_dir_all(part_path.parent().expect("part path should have a parent"))
1762 .await
1763 .expect("data dir should be created");
1764 tokio::fs::write(&part_path, b"orphan shard")
1765 .await
1766 .expect("part file should be written");
1767
1768 let residue = scan_metadata_less_residue(&bucket_path)
1769 .await
1770 .expect("metadata-less residue scan should succeed");
1771 assert!(residue.has_residue_without_xlmeta());
1772 assert_eq!(residue.files, 1);
1773 assert_eq!(residue.uuid_data_dirs, 1);
1774 let sample = residue.sample.as_deref().expect("part sample should be recorded");
1775 assert!(sample.starts_with("object/"));
1776 assert!(sample.ends_with("/part.1"));
1777 }
1778
1779 #[tokio::test]
1780 async fn metadata_less_residue_scan_shares_one_entry_budget_across_roots() {
1781 let root = tempfile::tempdir().expect("temporary diagnostic roots should be created");
1782 let first_root = root.path().join("disk-a");
1783 let second_root = root.path().join("disk-b");
1784 tokio::fs::create_dir_all(&first_root)
1785 .await
1786 .expect("first diagnostic root should be created");
1787 tokio::fs::create_dir_all(&second_root)
1788 .await
1789 .expect("second diagnostic root should be created");
1790 for index in 0..3 {
1791 std::fs::write(first_root.join(format!("first-{index}")), b"").expect("first-root fixture should be written");
1792 std::fs::write(second_root.join(format!("second-{index}")), b"").expect("second-root fixture should be written");
1793 }
1794
1795 let mut budget = BucketDeleteDiagnosticBudget::with_limits(4, Duration::from_secs(5));
1796 let first = scan_metadata_less_residue_with_budget(&first_root, &mut budget)
1797 .await
1798 .expect("first root should fit the shared budget");
1799 assert!(!first.diagnostic_truncated);
1800 let second = scan_metadata_less_residue_with_budget(&second_root, &mut budget)
1801 .await
1802 .expect("second root should stop at the remaining shared budget");
1803 assert!(second.diagnostic_truncated);
1804 assert!(first.entries_scanned + second.entries_scanned <= 4);
1805 }
1806
1807 #[tokio::test]
1808 async fn metadata_less_residue_scan_honors_an_expired_request_deadline() {
1809 let root = tempfile::tempdir().expect("temporary diagnostic root should be created");
1810 std::fs::write(root.path().join("orphan"), b"").expect("deadline fixture should be written");
1811 let mut budget = BucketDeleteDiagnosticBudget::with_limits(8, Duration::ZERO);
1812
1813 let scan = scan_metadata_less_residue_with_budget(root.path(), &mut budget)
1814 .await
1815 .expect("an expired diagnostic budget should fail closed without an IO error");
1816
1817 assert!(scan.diagnostic_truncated);
1818 assert_eq!(scan.entries_scanned, 0);
1819 assert_eq!(scan.diagnostic_bytes_read, 0);
1820 }
1821
1822 #[tokio::test]
1823 async fn metadata_less_residue_scan_stops_at_diagnostic_budget() {
1824 let root = tempfile::tempdir().expect("temporary bucket root should be created");
1825 let bucket_path = root.path().join("bucket");
1826 tokio::fs::create_dir_all(&bucket_path)
1827 .await
1828 .expect("budget fixture directory should be created");
1829 for index in 0..(BUCKET_DELETE_DIAGNOSTIC_MAX_ENTRIES + 32) {
1830 std::fs::write(bucket_path.join(format!("orphan-{index:05}")), b"").expect("budget fixture file should be created");
1831 }
1832
1833 let residue = scan_metadata_less_residue(&bucket_path)
1834 .await
1835 .expect("budgeted residue scan should fail closed without an IO error");
1836 assert!(residue.diagnostic_truncated);
1837 assert!(residue.has_residue_without_xlmeta());
1838 assert!(!residue.xlmeta_found);
1839 assert!(residue.entries_scanned <= BUCKET_DELETE_DIAGNOSTIC_MAX_ENTRIES);
1840 assert!(residue.files <= BUCKET_DELETE_DIAGNOSTIC_MAX_ENTRIES);
1841 assert_eq!(residue.diagnostic_bytes_read, 0);
1842 }
1843
1844 #[tokio::test]
1845 async fn bucket_residue_scan_distinguishes_visible_and_tier_free_xlmeta() {
1846 let root = tempfile::tempdir().expect("temporary bucket root should be created");
1847 let bucket_path = root.path().join("bucket");
1848 let visible_path = bucket_path.join("visible").join(STORAGE_FORMAT_FILE);
1849 tokio::fs::create_dir_all(visible_path.parent().expect("visible xl.meta should have a parent"))
1850 .await
1851 .expect("visible object directory should be created");
1852 let mut visible = FileMeta::new();
1853 visible
1854 .add_version(FileInfo {
1855 version_id: Some(Uuid::new_v4()),
1856 mod_time: Some(OffsetDateTime::now_utc()),
1857 ..Default::default()
1858 })
1859 .expect("visible version should encode");
1860 tokio::fs::write(&visible_path, visible.marshal_msg().expect("visible xl.meta should marshal"))
1861 .await
1862 .expect("visible xl.meta should be written");
1863
1864 let visible_scan = scan_metadata_less_residue(&bucket_path)
1865 .await
1866 .expect("visible xl.meta scan should succeed");
1867 assert_eq!(visible_scan.xlmeta_blocker, Some(BucketDeleteBlockerKind::VisibleVersion));
1868
1869 tokio::fs::remove_dir_all(bucket_path.join("visible"))
1870 .await
1871 .expect("visible fixture should be removed");
1872 let free_path = bucket_path.join("free").join(STORAGE_FORMAT_FILE);
1873 tokio::fs::create_dir_all(free_path.parent().expect("free xl.meta should have a parent"))
1874 .await
1875 .expect("free-version object directory should be created");
1876 let source_version_id = Uuid::new_v4();
1877 let mut free = FileMeta::new();
1878 free.add_version(FileInfo {
1879 version_id: Some(source_version_id),
1880 transition_status: TRANSITION_COMPLETE.to_string(),
1881 transitioned_objname: "remote/object".to_string(),
1882 transition_version_id: Some(Uuid::new_v4()),
1883 transition_tier: "WARM".to_string(),
1884 mod_time: Some(OffsetDateTime::now_utc()),
1885 ..Default::default()
1886 })
1887 .expect("transitioned source should encode");
1888 let mut delete = FileInfo {
1889 version_id: Some(source_version_id),
1890 mod_time: Some(OffsetDateTime::now_utc()),
1891 ..Default::default()
1892 };
1893 delete.set_tier_free_version_id(&Uuid::new_v4().to_string());
1894 free.delete_version(&delete)
1895 .expect("transitioned source delete should create a free-version");
1896 tokio::fs::write(&free_path, free.marshal_msg().expect("free-version xl.meta should marshal"))
1897 .await
1898 .expect("free-version xl.meta should be written");
1899
1900 let free_scan = scan_metadata_less_residue(&bucket_path)
1901 .await
1902 .expect("free-version xl.meta scan should succeed");
1903 assert_eq!(free_scan.xlmeta_blocker, Some(BucketDeleteBlockerKind::TierFreeVersion));
1904
1905 tokio::fs::remove_dir_all(bucket_path.join("free"))
1906 .await
1907 .expect("free-version fixture should be removed");
1908
1909 let exact_limit_path = bucket_path.join("exact-limit").join(STORAGE_FORMAT_FILE);
1910 tokio::fs::create_dir_all(exact_limit_path.parent().expect("exact-limit xl.meta should have a parent"))
1911 .await
1912 .expect("exact-limit object directory should be created");
1913 let exact_limit = tokio::fs::File::create(&exact_limit_path)
1914 .await
1915 .expect("exact-limit xl.meta should be created");
1916 exact_limit
1917 .set_len(BUCKET_DELETE_XLMETA_DIAGNOSTIC_MAX_BYTES)
1918 .await
1919 .expect("exact-limit xl.meta should be extended without allocating its contents");
1920 let exact_limit_scan = scan_metadata_less_residue(&bucket_path)
1921 .await
1922 .expect("exact-limit xl.meta scan should remain fail closed");
1923 assert_eq!(exact_limit_scan.xlmeta_blocker, Some(BucketDeleteBlockerKind::UnknownXlMeta));
1924 assert_eq!(exact_limit_scan.diagnostic_bytes_read, BUCKET_DELETE_XLMETA_DIAGNOSTIC_MAX_BYTES);
1925 tokio::fs::remove_dir_all(bucket_path.join("exact-limit"))
1926 .await
1927 .expect("exact-limit fixture should be removed");
1928
1929 let oversized_path = bucket_path.join("oversized").join(STORAGE_FORMAT_FILE);
1930 tokio::fs::create_dir_all(oversized_path.parent().expect("oversized xl.meta should have a parent"))
1931 .await
1932 .expect("oversized object directory should be created");
1933 let oversized = tokio::fs::File::create(&oversized_path)
1934 .await
1935 .expect("oversized xl.meta should be created");
1936 oversized
1937 .set_len(BUCKET_DELETE_XLMETA_DIAGNOSTIC_MAX_BYTES + 1)
1938 .await
1939 .expect("oversized xl.meta should be extended without allocating its contents");
1940 let oversized_scan = scan_metadata_less_residue(&bucket_path)
1941 .await
1942 .expect("oversized xl.meta scan should remain fail closed");
1943 assert_eq!(oversized_scan.xlmeta_blocker, Some(BucketDeleteBlockerKind::UnknownXlMeta));
1944 assert_eq!(oversized_scan.diagnostic_bytes_read, 0);
1945 assert!(oversized_scan.diagnostic_bytes_read <= BUCKET_DELETE_XLMETA_DIAGNOSTIC_MAX_BYTES);
1946 }
1947
1948 #[tokio::test]
1949 #[serial]
1950 async fn bucket_namespace_reads_keep_pre_expansion_bucket_visible() {
1951 let (_temp_dir, ecstore) = setup_multi_pool_bucket_test_env().await;
1952 let bucket = format!("pre-expansion-{}", Uuid::new_v4().simple());
1953 let object = "existing-object";
1954 ecstore.pools[0].disk_set[0]
1955 .make_bucket(&bucket, &MakeBucketOptions::default())
1956 .await
1957 .expect("bucket should be created in the original pool only");
1958 let mut reader = PutObjReader::from_vec(b"object written before pool expansion".to_vec());
1959 ecstore.pools[0]
1960 .put_object(&bucket, object, &mut reader, &ObjectOptions::default())
1961 .await
1962 .expect("object should be written in the original pool only");
1963
1964 let buckets = ecstore
1965 .list_bucket(&BucketOptions::default())
1966 .await
1967 .expect("S3 bucket listing should remain available after adding an empty pool");
1968 assert!(buckets.iter().any(|entry| entry.name == bucket));
1969
1970 let info = ecstore
1971 .get_bucket_info(&bucket, &BucketOptions::default())
1972 .await
1973 .expect("bucket validation should accept a bucket present in the original pool");
1974 assert_eq!(info.name, bucket);
1975
1976 metadata_sys::init_bucket_metadata_sys(ecstore.clone(), vec![bucket.clone()]).await;
1977 let listed = ecstore
1978 .clone()
1979 .list_objects_v2(&bucket, "", None, None, 1000, false, None, false)
1980 .await
1981 .expect("ListObjectsV2 should remain available after adding an empty pool");
1982 assert!(listed.objects.iter().any(|entry| entry.name == object));
1983 ecstore.pools[1].disk_set[0]
1984 .get_bucket_info(&bucket, &BucketOptions::default())
1985 .await
1986 .expect("metadata initialization should heal the bucket volume into the expansion pool");
1987
1988 let object_info = ecstore
1989 .get_object_info(&bucket, object, &ObjectOptions::default())
1990 .await
1991 .expect("an object in the original pool should remain readable after expansion");
1992 assert_eq!(object_info.name, object);
1993 }
1994
1995 #[tokio::test]
1996 #[serial]
1997 async fn bucket_creation_accepts_exact_quorum_in_every_set() {
1998 let (_temp_dir, ecstore) = setup_multi_pool_bucket_test_env().await;
1999 let bucket = format!("exact-set-quorum-{}", Uuid::new_v4().simple());
2000 let original_set = Arc::clone(&ecstore.pools[0].disk_set[0]);
2001 let offline = take_set_disks_offline(&ecstore, &original_set, &[0]).await;
2002
2003 ecstore
2004 .make_bucket_on_sets(&bucket, &MakeBucketOptions::default())
2005 .await
2006 .expect("three of four disks should satisfy the original set quorum");
2007
2008 restore_set_disks(&ecstore, &original_set, offline).await;
2009 for pool in &ecstore.pools {
2010 pool.disk_set[0]
2011 .get_bucket_info(&bucket, &BucketOptions::default())
2012 .await
2013 .expect("every set should expose a bucket created at its exact quorum");
2014 }
2015 }
2016
2017 #[tokio::test]
2018 #[serial]
2019 async fn bucket_creation_rejects_cross_pool_quorum_subsidy() {
2020 let (_temp_dir, ecstore) = setup_multi_pool_bucket_test_env().await;
2021 metadata_sys::init_bucket_metadata_sys(ecstore.clone(), Vec::new()).await;
2022 let bucket = format!("create-set-quorum-minus-one-{}", Uuid::new_v4().simple());
2023 let original_set = Arc::clone(&ecstore.pools[0].disk_set[0]);
2024 let offline = take_set_disks_offline(&ecstore, &original_set, &[0, 1]).await;
2025
2026 let err = ecstore
2027 .make_bucket(&bucket, &MakeBucketOptions::default())
2028 .await
2029 .expect_err("a healthy expansion pool must not subsidize a failed original set");
2030 restore_set_disks(&ecstore, &original_set, offline).await;
2031 assert!(
2032 matches!(&err, StorageError::InsufficientWriteQuorum(name, object) if name == &bucket && object.is_empty()),
2033 "unexpected error: {err}"
2034 );
2035 for pool in &ecstore.pools {
2036 assert_eq!(
2037 pool.disk_set[0]
2038 .get_bucket_info(&bucket, &BucketOptions::default())
2039 .await
2040 .expect_err("failed creation must not leave an authoritative bucket"),
2041 StorageError::VolumeNotFound
2042 );
2043 }
2044 }
2045
2046 #[tokio::test]
2047 #[serial]
2048 async fn bucket_creation_prioritizes_hard_set_error_over_existing_bucket() {
2049 let (_temp_dir, ecstore) = setup_multi_pool_bucket_test_env().await;
2050 metadata_sys::init_bucket_metadata_sys(ecstore.clone(), Vec::new()).await;
2051 let bucket = format!("create-hard-error-{}", Uuid::new_v4().simple());
2052 ecstore.pools[0].disk_set[0]
2053 .make_bucket(&bucket, &MakeBucketOptions::default())
2054 .await
2055 .expect("the original pool should contain the bucket");
2056 let expansion_set = Arc::clone(&ecstore.pools[1].disk_set[0]);
2057 let offline = take_set_disks_offline(&ecstore, &expansion_set, &[0, 1]).await;
2058
2059 let err = ecstore
2060 .make_bucket(&bucket, &MakeBucketOptions::default())
2061 .await
2062 .expect_err("a hard set failure must outrank BucketExists from another set");
2063 restore_set_disks(&ecstore, &expansion_set, offline).await;
2064 assert!(
2065 matches!(&err, StorageError::InsufficientWriteQuorum(name, object) if name == &bucket && object.is_empty()),
2066 "unexpected error: {err}"
2067 );
2068 ecstore.pools[0].disk_set[0]
2069 .get_bucket_info(&bucket, &BucketOptions::default())
2070 .await
2071 .expect("failure in another set must not roll back a pre-existing bucket");
2072 }
2073
2074 #[tokio::test]
2075 #[serial]
2076 async fn bucket_delete_rejects_cross_pool_quorum_subsidy_before_mutation() {
2077 let (_temp_dir, ecstore) = setup_multi_pool_bucket_test_env().await;
2078 metadata_sys::init_bucket_metadata_sys(ecstore.clone(), Vec::new()).await;
2079 let bucket = format!("delete-set-quorum-minus-one-{}", Uuid::new_v4().simple());
2080 ecstore
2081 .make_bucket(&bucket, &MakeBucketOptions::default())
2082 .await
2083 .expect("bucket should be created in every set before deletion");
2084 let original_set = Arc::clone(&ecstore.pools[0].disk_set[0]);
2085 let offline = take_set_disks_offline(&ecstore, &original_set, &[0, 1, 2]).await;
2086
2087 let err = ecstore
2088 .delete_bucket(
2089 &bucket,
2090 &DeleteBucketOptions {
2091 force: true,
2092 ..Default::default()
2093 },
2094 )
2095 .await
2096 .expect_err("a healthy expansion pool must not subsidize a failed original set deletion");
2097 restore_set_disks(&ecstore, &original_set, offline).await;
2098 assert!(
2099 matches!(&err, StorageError::InsufficientWriteQuorum(name, object) if name == &bucket && object.is_empty()),
2100 "unexpected error: {err}"
2101 );
2102 for pool in &ecstore.pools {
2103 pool.disk_set[0]
2104 .get_bucket_info(&bucket, &BucketOptions::default())
2105 .await
2106 .expect("failed deletion preflight must preserve every bucket volume");
2107 }
2108 let (_, persisted) = metadata_sys::get_config_from_disk_with_presence_in(&ecstore.ctx, &bucket)
2109 .await
2110 .expect("failed deletion must preserve bucket metadata");
2111 assert!(persisted);
2112 }
2113
2114 #[tokio::test]
2115 #[serial]
2116 async fn bucket_delete_rolls_back_sets_after_partial_failure() {
2117 let (_temp_dir, ecstore) = setup_multi_pool_bucket_test_env().await;
2118 let bucket = format!("delete-set-rollback-{}", Uuid::new_v4().simple());
2119 ecstore
2120 .make_bucket_on_sets(&bucket, &MakeBucketOptions::default())
2121 .await
2122 .expect("bucket should be created in every set before deletion");
2123 let original_set = Arc::clone(&ecstore.pools[0].disk_set[0]);
2124 let offline = take_set_disks_offline(&ecstore, &original_set, &[0, 1]).await;
2125
2126 let err = ecstore
2127 .delete_bucket_on_sets(&bucket, &DeleteBucketOptions::default())
2128 .await
2129 .expect_err("a failed set must fail the complete bucket deletion");
2130 restore_set_disks(&ecstore, &original_set, offline).await;
2131 assert_eq!(err, StorageError::ErasureWriteQuorum);
2132 for pool in &ecstore.pools {
2133 pool.disk_set[0]
2134 .get_bucket_info(&bucket, &BucketOptions::default())
2135 .await
2136 .expect("delete rollback should restore the bucket namespace in every set");
2137 }
2138 }
2139
2140 #[tokio::test]
2141 #[serial]
2142 async fn bucket_config_update_accepts_pre_expansion_bucket() {
2143 let (_temp_dir, ecstore) = setup_multi_pool_bucket_test_env().await;
2144 metadata_sys::init_bucket_metadata_sys(ecstore.clone(), Vec::new()).await;
2145 let bucket = format!("config-pre-expansion-{}", Uuid::new_v4().simple());
2146 ecstore
2147 .make_bucket(&bucket, &MakeBucketOptions::default())
2148 .await
2149 .expect("bucket should be created with authoritative metadata");
2150 ecstore.pools[1].disk_set[0]
2151 .delete_bucket(&bucket, &DeleteBucketOptions::default())
2152 .await
2153 .expect("the expansion pool bucket volume should be removed for the regression state");
2154
2155 let policy = br#"{"Version":"2012-10-17","Statement":[]}"#.to_vec();
2156 ecstore
2157 .update_bucket_metadata_config(&bucket, crate::bucket::metadata::BUCKET_POLICY_CONFIG, policy.clone())
2158 .await
2159 .expect("config mutation should validate existence per erasure set");
2160 let (stored, _) = ecstore
2161 .get_bucket_policy_raw(&bucket)
2162 .await
2163 .expect("updated bucket policy should remain readable");
2164 assert_eq!(stored.as_bytes(), policy);
2165 }
2166
2167 #[tokio::test]
2168 #[serial]
2169 async fn bucket_metadata_init_recreates_pre_expansion_bucket_in_new_pool() {
2170 let (_temp_dir, ecstore) = setup_multi_pool_bucket_test_env().await;
2171 let bucket = format!("metadata-expansion-{}", Uuid::new_v4().simple());
2172 ecstore.pools[0].disk_set[0]
2173 .make_bucket(&bucket, &MakeBucketOptions::default())
2174 .await
2175 .expect("bucket should be created in the original pool only");
2176 assert_eq!(
2177 ecstore.pools[1].disk_set[0]
2178 .get_bucket_info(&bucket, &BucketOptions::default())
2179 .await
2180 .expect_err("new pool should initially have no bucket volume"),
2181 StorageError::VolumeNotFound
2182 );
2183
2184 metadata_sys::init_bucket_metadata_sys(ecstore.clone(), vec![bucket.clone()]).await;
2185
2186 ecstore.pools[1].disk_set[0]
2187 .get_bucket_info(&bucket, &BucketOptions::default())
2188 .await
2189 .expect("metadata initialization should recreate the bucket volume in the new pool");
2190 }
2191
2192 #[tokio::test]
2193 #[serial]
2194 async fn bucket_metadata_init_repairs_half_created_expansion_pool() {
2195 for complete_pool in 0..2 {
2198 let (temp_dir, ecstore) = setup_multi_pool_bucket_test_env().await;
2199 let bucket = format!("partial-expansion-{}", Uuid::new_v4().simple());
2200 for pool_index in 0..2 {
2201 let present_disks = if pool_index == complete_pool { 4 } else { 2 };
2202 for disk_index in 0..present_disks {
2203 tokio::fs::create_dir(
2204 temp_dir
2205 .path()
2206 .join(format!("pool{pool_index}-disk{disk_index}"))
2207 .join(&bucket),
2208 )
2209 .await
2210 .expect("fixture bucket volume should be created");
2211 }
2212 }
2213 assert_eq!(
2214 ecstore
2215 .get_bucket_info_from_sets(&bucket, &BucketOptions::default())
2216 .await
2217 .expect_err("request validation must reject a half-created expansion set"),
2218 StorageError::ErasureWriteQuorum
2219 );
2220
2221 metadata_sys::init_bucket_metadata_sys(ecstore.clone(), vec![bucket.clone()]).await;
2222
2223 for pool_index in 0..2 {
2224 for disk_index in 0..4 {
2225 assert!(
2226 temp_dir
2227 .path()
2228 .join(format!("pool{pool_index}-disk{disk_index}"))
2229 .join(&bucket)
2230 .is_dir(),
2231 "metadata initialization must heal every missing expansion volume"
2232 );
2233 }
2234 }
2235 ecstore
2236 .get_bucket_info_from_sets(&bucket, &BucketOptions::default())
2237 .await
2238 .expect("strict request validation should succeed after volume repair");
2239 }
2240 }
2241
2242 #[tokio::test]
2243 #[serial]
2244 async fn bucket_metadata_init_does_not_combine_partial_set_evidence() {
2245 let (temp_dir, ecstore) = setup_multi_pool_bucket_test_env().await;
2246 let bucket = format!("no-quorum-expansion-{}", Uuid::new_v4().simple());
2247 for pool_index in 0..2 {
2248 for disk_index in 0..2 {
2249 tokio::fs::create_dir(
2250 temp_dir
2251 .path()
2252 .join(format!("pool{pool_index}-disk{disk_index}"))
2253 .join(&bucket),
2254 )
2255 .await
2256 .expect("fixture bucket volume should be created");
2257 }
2258 }
2259 assert_eq!(
2260 ecstore
2261 .bucket_exists_for_heal(&bucket)
2262 .await
2263 .expect_err("repair must require a complete quorum within one set"),
2264 StorageError::ErasureWriteQuorum
2265 );
2266
2267 metadata_sys::init_bucket_metadata_sys(ecstore.clone(), vec![bucket.clone()]).await;
2268
2269 for pool_index in 0..2 {
2270 for disk_index in 0..4 {
2271 assert_eq!(
2272 temp_dir
2273 .path()
2274 .join(format!("pool{pool_index}-disk{disk_index}"))
2275 .join(&bucket)
2276 .is_dir(),
2277 disk_index < 2,
2278 "unproven bucket generations must not recreate missing volumes"
2279 );
2280 }
2281 }
2282 }
2283
2284 #[tokio::test]
2285 #[serial]
2286 async fn bucket_metadata_init_does_not_recreate_stale_bucket_name() {
2287 let (_temp_dir, ecstore) = setup_multi_pool_bucket_test_env().await;
2288 let bucket = format!("stale-expansion-{}", Uuid::new_v4().simple());
2289
2290 metadata_sys::init_bucket_metadata_sys(ecstore.clone(), vec![bucket.clone()]).await;
2291
2292 for pool in &ecstore.pools {
2293 let err = pool.disk_set[0]
2294 .get_bucket_info(&bucket, &BucketOptions::default())
2295 .await
2296 .expect_err("a stale startup listing must not recreate a deleted bucket");
2297 assert_eq!(err, StorageError::VolumeNotFound);
2298 }
2299 }
2300
2301 #[tokio::test]
2302 #[serial]
2303 async fn bucket_info_read_quorum_tracks_erasure_layout() {
2304 for (drive_count, parity) in [(2, 1), (3, 1), (4, 2), (5, 2), (6, 3), (8, 4), (6, 2), (12, 6)] {
2305 let (_temp_dir, store) = setup_bucket_quorum_test_env(&[drive_count], Some(parity)).await;
2306 metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
2307 let bucket = format!("read-quorum-{drive_count}-{parity}");
2308 let object = "uncached-object";
2309 let body = b"erasure read quorum must follow the persisted layout".repeat(32_768);
2310 store
2311 .make_bucket(&bucket, &MakeBucketOptions::default())
2312 .await
2313 .expect("healthy namespace should accept bucket creation");
2314 store
2315 .put_object(&bucket, object, &mut PutObjReader::from_vec(body.clone()), &ObjectOptions::default())
2316 .await
2317 .expect("healthy erasure set should accept the seed object");
2318 let set = &store.pools[0].disk_set[0];
2319 let lock = set
2320 .new_ns_lock(&bucket, object)
2321 .await
2322 .expect("seed namespace lock should resolve");
2323 drop(
2324 lock.get_write_lock(Duration::from_secs(30))
2325 .await
2326 .expect("seed physical fanout must finish before taking disks offline"),
2327 );
2328 if (drive_count, parity) == (6, 3) {
2329 let mut kvs = rustfs_config::server_config::KVS::new();
2330 kvs.insert(crate::config::storageclass::CLASS_STANDARD.to_string(), "EC:2".to_string());
2331 set.set_test_storage_class_config(
2332 crate::config::storageclass::lookup_config_for_pools_without_env(&kvs, &[drive_count])
2333 .expect("a later storage-class change must not raise old objects' read quorum"),
2334 );
2335 }
2336
2337 let offline_indexes = (0..parity).collect::<Vec<_>>();
2338 let offline = take_set_disks_offline(&store, set, &offline_indexes).await;
2339 let info = store
2340 .get_bucket_info(&bucket, &BucketOptions::default())
2341 .await
2342 .expect("bucket validation must admit the object's exact read quorum");
2343 assert_eq!(info.name, bucket);
2344
2345 let mut reader = store
2346 .get_object_reader(&bucket, object, None, Default::default(), &ObjectOptions::default())
2347 .await
2348 .expect("the persisted layout should remain readable at its exact data-shard quorum");
2349 let mut restored = Vec::new();
2350 reader
2351 .stream
2352 .read_to_end(&mut restored)
2353 .await
2354 .expect("quorum read should reconstruct the body");
2355 assert_eq!(restored, body, "layout {drive_count}/{parity} must retain exact object contents");
2356 drop(reader);
2357
2358 if drive_count - parity == drive_count / 2 {
2359 let error = store
2360 .get_bucket_info_from_sets(&bucket, &BucketOptions::default())
2361 .await
2362 .expect_err("bucket mutations must retain their majority namespace check");
2363 assert_eq!(error, StorageError::ErasureWriteQuorum);
2364 }
2365
2366 let below_quorum = take_set_disks_offline(&store, set, &[parity]).await;
2367 let read = store
2368 .get_object_reader(&bucket, object, None, Default::default(), &ObjectOptions::default())
2369 .await;
2370 match read {
2371 Ok(mut reader) => assert!(
2372 reader.stream.read_to_end(&mut Vec::new()).await.is_err(),
2373 "layout {drive_count}/{parity} must reject fewer than its data-shard quorum"
2374 ),
2375 Err(error) => assert!(
2376 matches!(error, StorageError::ErasureReadQuorum | StorageError::InsufficientReadQuorum(_, _)),
2377 "a missing shard must report read quorum loss, got {error}"
2378 ),
2379 }
2380 restore_set_disks(&store, set, below_quorum).await;
2381 restore_set_disks(&store, set, offline).await;
2382 }
2383 }
2384
2385 #[tokio::test]
2386 #[serial]
2387 async fn bucket_info_read_quorum_is_scoped_to_each_erasure_set() {
2388 let (_temp_dir, store) = setup_bucket_quorum_test_env(&[4, 6], None).await;
2389 metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
2390 let bucket = "read-quorum-mixed-pools";
2391 store
2392 .make_bucket(bucket, &MakeBucketOptions::default())
2393 .await
2394 .expect("healthy pools should accept bucket creation");
2395
2396 let first_set = &store.pools[0].disk_set[0];
2397 let second_set = &store.pools[1].disk_set[0];
2398 let first_offline = take_set_disks_offline(&store, first_set, &[0, 1]).await;
2399 let second_offline = take_set_disks_offline(&store, second_set, &[0, 1, 2]).await;
2400 store
2401 .get_bucket_info(bucket, &BucketOptions::default())
2402 .await
2403 .expect("each set independently satisfies its namespace read quorum");
2404
2405 for (set, extra_disk) in [(first_set, 2), (second_set, 3)] {
2406 let extra_offline = take_set_disks_offline(&store, set, &[extra_disk]).await;
2407 assert_eq!(
2408 store
2409 .get_bucket_info(bucket, &BucketOptions::default())
2410 .await
2411 .expect_err("another pool must not subsidize a set below its read quorum"),
2412 StorageError::ErasureReadQuorum
2413 );
2414 restore_set_disks(&store, set, extra_offline).await;
2415 }
2416 restore_set_disks(&store, first_set, first_offline).await;
2417 restore_set_disks(&store, second_set, second_offline).await;
2418 }
2419
2420 #[tokio::test]
2421 #[serial]
2422 async fn bucket_info_read_quorum_requires_authoritative_metadata() {
2423 for state in ["missing", "corrupt", "foreign", "incarnation"] {
2424 let (_temp_dir, store) = setup_bucket_quorum_test_env(&[4], None).await;
2425 metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
2426 let bucket = format!("read-quorum-{state}-metadata");
2427 let mut metadata = if state == "missing" {
2428 store
2429 .make_bucket_on_sets(&bucket, &MakeBucketOptions::default())
2430 .await
2431 .expect("simulate directories left before bucket metadata is published");
2432 BucketMetadata::new(&bucket)
2433 } else {
2434 store
2435 .make_bucket(&bucket, &MakeBucketOptions::default())
2436 .await
2437 .expect("healthy bucket should publish metadata");
2438 metadata_sys::get_in(&store.ctx, &bucket)
2439 .await
2440 .expect("seed metadata should be cached")
2441 .as_ref()
2442 .clone()
2443 };
2444 let path = metadata.save_file_path();
2445 match state {
2446 "corrupt" => crate::config::com::save_config(store.clone(), &path, b"corrupt".to_vec())
2447 .await
2448 .expect("persist corrupt metadata while the cached copy remains valid"),
2449 "foreign" => {
2450 metadata.name = "different-bucket".to_string();
2451 let mut encoded = vec![1, 0, 1, 0];
2452 encoded.extend(metadata.marshal_msg().expect("foreign metadata should encode"));
2453 crate::config::com::save_config(store.clone(), &path, encoded)
2454 .await
2455 .expect("persist metadata for a different bucket at the requested path");
2456 }
2457 "incarnation" => crate::bucket::metadata::save_bucket_incarnation(store.clone(), &bucket, Uuid::new_v4())
2458 .await
2459 .expect("persist a different bucket generation"),
2460 _ => {}
2461 }
2462
2463 let set = &store.pools[0].disk_set[0];
2464 let offline = take_set_disks_offline(&store, set, &[0, 1]).await;
2465 let error = store
2466 .get_bucket_info(&bucket, &BucketOptions::default())
2467 .await
2468 .expect_err("read admission must not trust residual directories or cached metadata");
2469 match state {
2470 "missing" => assert_eq!(error, StorageError::ErasureReadQuorum),
2471 "foreign" => assert_eq!(error, StorageError::FileCorrupt),
2472 "incarnation" => assert!(error.to_string().contains("sidecar does not match bucket metadata")),
2473 "corrupt" => assert!(error.to_string().contains("format invalid"), "unexpected corruption error: {error}"),
2474 _ => unreachable!(),
2475 }
2476 restore_set_disks(&store, set, offline).await;
2477 }
2478 }
2479
2480 #[tokio::test]
2481 #[serial]
2482 async fn bucket_info_read_quorum_accepts_persisted_legacy_metadata() {
2483 let (_temp_dir, store) = setup_bucket_quorum_test_env(&[4], None).await;
2484 metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
2485 let bucket = "interop";
2486 store
2487 .make_bucket_on_sets(bucket, &MakeBucketOptions::default())
2488 .await
2489 .expect("legacy bucket directories should exist");
2490 let hex = include_str!("../../tests/fixtures/minio/bucket_metadata.blob.hex")
2491 .split_whitespace()
2492 .collect::<String>();
2493 let body = (0..hex.len())
2494 .step_by(2)
2495 .map(|index| u8::from_str_radix(&hex[index..index + 2], 16).expect("pinned MinIO metadata fixture"))
2496 .collect();
2497 crate::config::com::save_config(store.clone(), &BucketMetadata::new(bucket).save_file_path(), body)
2498 .await
2499 .expect("legacy metadata should be persisted without an incarnation sidecar");
2500
2501 let set = &store.pools[0].disk_set[0];
2502 let offline = take_set_disks_offline(&store, set, &[0, 1]).await;
2503 let info = store
2504 .get_bucket_info(bucket, &BucketOptions::default())
2505 .await
2506 .expect("persisted MinIO metadata should authorize reads at the namespace read quorum");
2507 assert_eq!(info.name, bucket);
2508 assert!(info.versioning);
2509 assert!(info.object_locking);
2510 restore_set_disks(&store, set, offline).await;
2511 }
2512
2513 #[tokio::test]
2514 #[serial]
2515 async fn bucket_namespace_reads_report_missing_when_every_set_is_absent() {
2516 let (_temp_dir, ecstore) = setup_multi_pool_bucket_test_env().await;
2517 let bucket = format!("absent-{}", Uuid::new_v4().simple());
2518
2519 let buckets = ecstore
2520 .list_bucket(&BucketOptions::default())
2521 .await
2522 .expect("an empty healthy namespace should remain listable");
2523 assert!(buckets.iter().all(|entry| entry.name != bucket));
2524
2525 let err = ecstore
2526 .get_bucket_info(&bucket, &BucketOptions::default())
2527 .await
2528 .expect_err("a bucket absent from every set must remain missing");
2529 assert_eq!(err, StorageError::VolumeNotFound);
2530 }
2531
2532 #[tokio::test]
2533 #[serial]
2534 async fn bucket_namespace_reads_fail_closed_when_any_set_loses_quorum() {
2535 let (_temp_dir, ecstore) = setup_multi_pool_bucket_test_env().await;
2536 metadata_sys::init_bucket_metadata_sys(ecstore.clone(), Vec::new()).await;
2537 let bucket = format!("degraded-expansion-{}", Uuid::new_v4().simple());
2538 ecstore.pools[0].disk_set[0]
2539 .make_bucket(&bucket, &MakeBucketOptions::default())
2540 .await
2541 .expect("bucket should be created in the original pool only");
2542 ecstore.pools[1].disk_set[0].disks.write().await[0] = None;
2543 ecstore.pools[1].disk_set[0].disks.write().await[1] = None;
2544 ecstore.pools[1].disk_set[0].disks.write().await[2] = None;
2545
2546 let list_err = ecstore
2547 .list_bucket(&BucketOptions::default())
2548 .await
2549 .expect_err("listing must not hide an unavailable expansion pool");
2550 assert_eq!(list_err, StorageError::ErasureWriteQuorum);
2551
2552 let info_err = ecstore
2553 .get_bucket_info(&bucket, &BucketOptions::default())
2554 .await
2555 .expect_err("bucket validation must fail when an expansion pool is unavailable");
2556 assert_eq!(info_err, StorageError::ErasureReadQuorum);
2557 }
2558
2559 #[tokio::test]
2560 #[serial]
2561 async fn scanner_bucket_listing_unions_every_erasure_set() {
2562 let (_temp_dir, ecstore) = setup_multi_pool_bucket_test_env().await;
2563 let bucket = format!("second-pool-only-{}", Uuid::new_v4().simple());
2564 ecstore.pools[1].disk_set[0]
2565 .make_bucket(&bucket, &MakeBucketOptions::default())
2566 .await
2567 .expect("bucket should be created in the second pool only");
2568
2569 let listing = ecstore
2570 .list_bucket_for_scanner(&crate::storage_api_contracts::bucket::BucketOptions {
2571 no_metadata: true,
2572 ..Default::default()
2573 })
2574 .await
2575 .expect("scanner should enumerate every pool and set");
2576
2577 assert!(listing.topology_complete);
2578 assert!(listing.buckets.iter().any(|entry| entry.name == bucket));
2579 assert_eq!(listing.set_buckets.len(), 2);
2580 assert!(
2581 listing
2582 .set_buckets
2583 .iter()
2584 .find(|scope| scope.pool_index == 0 && scope.set_index == 0)
2585 .is_some_and(|scope| scope.buckets.is_empty())
2586 );
2587 assert!(
2588 listing
2589 .set_buckets
2590 .iter()
2591 .find(|scope| scope.pool_index == 1 && scope.set_index == 0)
2592 .is_some_and(|scope| scope.buckets.iter().any(|entry| entry.name == bucket))
2593 );
2594 }
2595
2596 #[tokio::test]
2597 async fn scanner_bucket_listing_marks_degraded_set_incomplete() {
2598 let (_temp_dir, ecstore) = setup_multi_pool_bucket_test_env().await;
2599 let bucket = format!("degraded-set-{}", Uuid::new_v4().simple());
2600 let set = &ecstore.pools[0].disk_set[0];
2601 set.make_bucket(&bucket, &MakeBucketOptions::default())
2602 .await
2603 .expect("bucket should be created before a disk is removed");
2604 set.disks.write().await[0] = None;
2605
2606 let listing = ecstore
2607 .list_bucket_for_scanner(&crate::storage_api_contracts::bucket::BucketOptions {
2608 no_metadata: true,
2609 ..Default::default()
2610 })
2611 .await
2612 .expect("a degraded set with quorum should still return candidate buckets");
2613
2614 assert!(listing.buckets.iter().any(|entry| entry.name == bucket));
2615 assert!(!listing.topology_complete);
2616 }
2617
2618 #[tokio::test]
2619 async fn scanner_bucket_listing_marks_divergent_disk_views_incomplete() {
2620 let (temp_dir, ecstore) = setup_multi_pool_bucket_test_env().await;
2621 let bucket = format!("divergent-set-{}", Uuid::new_v4().simple());
2622 ecstore.pools[0].disk_set[0]
2623 .make_bucket(&bucket, &MakeBucketOptions::default())
2624 .await
2625 .expect("bucket should be created before disk views diverge");
2626 for disk_index in 0..2 {
2627 tokio::fs::remove_dir_all(temp_dir.path().join(format!("pool0-disk{disk_index}")).join(&bucket))
2628 .await
2629 .expect("test bucket directory should be removed from a minority disk view");
2630 }
2631
2632 let listing = ecstore
2633 .list_bucket_for_scanner(&crate::storage_api_contracts::bucket::BucketOptions {
2634 no_metadata: true,
2635 ..Default::default()
2636 })
2637 .await
2638 .expect("responsive disks should still produce a scanner candidate listing");
2639
2640 assert!(listing.buckets.iter().all(|entry| entry.name != bucket));
2641 assert!(!listing.topology_complete);
2642 }
2643
2644 #[tokio::test]
2647 #[serial]
2648 async fn bucket_delete_mark_delete_removes_empty_bucket_and_keeps_deleted_marker() {
2649 let (disk_paths, ecstore) = setup_bucket_delete_test_env().await;
2650 let bucket = format!("bucket-mark-delete-{}", Uuid::new_v4().simple());
2651
2652 ecstore
2653 .make_bucket(&bucket, &MakeBucketOptions::default())
2654 .await
2655 .expect("bucket should be created");
2656 for disk_path in &disk_paths {
2657 tokio::fs::create_dir_all(disk_path.join(&bucket).join("empty-directory/nested/leaf"))
2658 .await
2659 .expect("empty directory remnant should be created");
2660 }
2661 assert!(metadata_sys::get_in(&ecstore.ctx, &bucket).await.is_ok());
2662
2663 let generation_before_delete = ecstore.scanner_namespace_mutation_generation();
2664 ecstore
2665 .delete_bucket(
2666 &bucket,
2667 &DeleteBucketOptions {
2668 srdelete_op: SRBucketDeleteOp::MarkDelete,
2669 ..Default::default()
2670 },
2671 )
2672 .await
2673 .expect("MarkDelete should remove an empty bucket");
2674 assert_eq!(
2675 ecstore.scanner_namespace_mutation_generation(),
2676 generation_before_delete.saturating_add(2),
2677 "successful bucket deletion should fence scanner namespace activity before and after deletion"
2678 );
2679
2680 assert!(
2681 !any_disk_path_exists(&disk_paths, &bucket).await,
2682 "MarkDelete should remove the bucket volume"
2683 );
2684 assert!(
2685 any_disk_path_exists(&disk_paths, bucket_deleted_marker_volume(&bucket)).await,
2686 "MarkDelete should persist the deleted-bucket marker"
2687 );
2688 assert!(
2689 metadata_sys::get_in(&ecstore.ctx, &bucket).await.is_err(),
2690 "deleted bucket metadata must be removed from the local cache"
2691 );
2692 let buckets = ecstore
2693 .list_bucket(&BucketOptions::default())
2694 .await
2695 .expect("bucket listing should succeed after MarkDelete");
2696 assert!(!buckets.iter().any(|info| info.name == bucket));
2697 ecstore
2698 .delete_all(RUSTFS_META_BUCKET, &bucket_deleted_marker_prefix(&bucket))
2699 .await
2700 .expect("deleted-bucket marker should be removed to simulate a partial failure");
2701 assert!(!any_disk_path_exists(&disk_paths, bucket_deleted_marker_volume(&bucket)).await);
2702 ecstore
2703 .delete_bucket(
2704 &bucket,
2705 &DeleteBucketOptions {
2706 srdelete_op: SRBucketDeleteOp::MarkDelete,
2707 ..Default::default()
2708 },
2709 )
2710 .await
2711 .expect("retried MarkDelete should recreate a missing tombstone");
2712 assert!(any_disk_path_exists(&disk_paths, bucket_deleted_marker_volume(&bucket)).await);
2713 }
2714
2715 #[tokio::test]
2716 #[serial]
2717 async fn bucket_delete_mark_delete_rejects_non_empty_bucket_without_force() {
2718 let (disk_paths, ecstore) = setup_bucket_delete_test_env().await;
2719 let bucket = format!("bucket-mark-delete-non-empty-{}", Uuid::new_v4().simple());
2720
2721 create_bucket_with_object(&ecstore, &bucket, "object.txt").await;
2722
2723 let err = ecstore
2724 .delete_bucket(
2725 &bucket,
2726 &DeleteBucketOptions {
2727 srdelete_op: SRBucketDeleteOp::MarkDelete,
2728 ..Default::default()
2729 },
2730 )
2731 .await
2732 .expect_err("MarkDelete should reject a non-empty bucket without force");
2733
2734 assert!(matches!(err, StorageError::BucketNotEmpty(name) if name == bucket));
2735 assert!(any_disk_has_object_metadata(&disk_paths, &bucket).await);
2736 assert!(!any_disk_path_exists(&disk_paths, bucket_deleted_marker_volume(&bucket)).await);
2737 }
2738
2739 #[tokio::test]
2740 #[serial]
2741 async fn bucket_delete_mark_delete_rejects_hidden_object_paths_without_force() {
2742 let (disk_paths, ecstore) = setup_bucket_delete_test_env().await;
2743 let bucket = format!("bucket-mark-delete-hidden-{}", Uuid::new_v4().simple());
2744
2745 create_bucket_with_object(&ecstore, &bucket, ".well-known/acme-challenge").await;
2746 let mut reader = PutObjReader::from_vec(b"delete bucket semantics".to_vec());
2747 ecstore
2748 .put_object(&bucket, ".rustfs.sys/object", &mut reader, &ObjectOptions::default())
2749 .await
2750 .expect("second hidden object should be written");
2751
2752 let err = ecstore
2753 .delete_bucket(
2754 &bucket,
2755 &DeleteBucketOptions {
2756 srdelete_op: SRBucketDeleteOp::MarkDelete,
2757 ..Default::default()
2758 },
2759 )
2760 .await
2761 .expect_err("MarkDelete should reject a hidden object path without force");
2762
2763 assert!(matches!(err, StorageError::BucketNotEmpty(name) if name == bucket));
2764 assert!(any_disk_has_object_metadata(&disk_paths, &bucket).await);
2765 }
2766
2767 #[tokio::test]
2768 #[serial]
2769 async fn bucket_delete_purge_removes_bucket_data_and_internal_metadata() {
2770 let (disk_paths, ecstore) = setup_bucket_delete_test_env().await;
2771 let bucket = format!("bucket-purge-{}", Uuid::new_v4().simple());
2772 let object = "object.txt";
2773 let metadata_prefix = format!("{RUSTFS_META_BUCKET}/{BUCKET_META_PREFIX}/{bucket}");
2774
2775 create_bucket_with_object(&ecstore, &bucket, object).await;
2776 write_bucket_metadata_marker(&disk_paths, &metadata_prefix).await;
2777 ecstore
2778 .mark_bucket_deleted(&bucket)
2779 .await
2780 .expect("deleted-bucket marker should be created");
2781 assert!(any_disk_path_exists(&disk_paths, &metadata_prefix).await);
2782 assert!(any_disk_path_exists(&disk_paths, bucket_deleted_marker_volume(&bucket)).await);
2783
2784 let generation_before_delete = ecstore.scanner_namespace_mutation_generation();
2785 ecstore
2786 .delete_bucket(
2787 &bucket,
2788 &DeleteBucketOptions {
2789 force: true,
2790 srdelete_op: SRBucketDeleteOp::Purge,
2791 ..Default::default()
2792 },
2793 )
2794 .await
2795 .expect("Purge should force-delete bucket data");
2796 assert_eq!(
2797 ecstore.scanner_namespace_mutation_generation(),
2798 generation_before_delete.saturating_add(2),
2799 "successful bucket purge should fence scanner namespace activity before and after deletion"
2800 );
2801
2802 assert!(!any_disk_path_exists(&disk_paths, &bucket).await, "Purge should remove the bucket volume");
2803 assert!(
2804 !any_disk_path_exists(&disk_paths, &metadata_prefix).await,
2805 "Purge should remove bucket metadata prefix"
2806 );
2807 assert!(
2808 !any_disk_path_exists(&disk_paths, bucket_deleted_marker_volume(&bucket)).await,
2809 "Purge should remove the deleted-bucket marker"
2810 );
2811 assert!(
2812 metadata_sys::get_in(&ecstore.ctx, &bucket).await.is_err(),
2813 "purged bucket metadata must be removed from the local cache"
2814 );
2815 write_bucket_metadata_marker(&disk_paths, &metadata_prefix).await;
2816 ecstore
2817 .mark_bucket_deleted(&bucket)
2818 .await
2819 .expect("stale deleted-bucket marker should be recreated");
2820 ecstore
2821 .delete_bucket(
2822 &bucket,
2823 &DeleteBucketOptions {
2824 force: true,
2825 srdelete_op: SRBucketDeleteOp::Purge,
2826 ..Default::default()
2827 },
2828 )
2829 .await
2830 .expect("retried Purge should remove stale metadata without a bucket volume");
2831 assert!(!any_disk_path_exists(&disk_paths, &metadata_prefix).await);
2832 assert!(!any_disk_path_exists(&disk_paths, bucket_deleted_marker_volume(&bucket)).await);
2833 }
2834
2835 #[tokio::test(flavor = "multi_thread")]
2841 #[serial]
2842 async fn force_create_enables_object_lock_on_an_existing_bucket() {
2843 let (_disk_paths, ecstore) = setup_bucket_delete_test_env().await;
2844 let bucket = format!("bucket-force-lock-{}", Uuid::new_v4().simple());
2845
2846 ecstore
2847 .make_bucket(&bucket, &MakeBucketOptions::default())
2848 .await
2849 .expect("plain bucket should be created");
2850 assert!(
2851 !metadata_sys::get_in(&ecstore.ctx, &bucket)
2852 .await
2853 .expect("metadata should load")
2854 .lock_enabled,
2855 "test setup: the bucket must start unlocked"
2856 );
2857
2858 ecstore
2859 .make_bucket(
2860 &bucket,
2861 &MakeBucketOptions {
2862 force_create: true,
2863 lock_enabled: true,
2864 ..Default::default()
2865 },
2866 )
2867 .await
2868 .expect("force create with lock_enabled should succeed on an existing bucket");
2869
2870 let meta = metadata_sys::get_in(&ecstore.ctx, &bucket)
2871 .await
2872 .expect("metadata should load after the lock enable");
2873 assert!(meta.lock_enabled, "Object Lock must be enabled on the existing bucket");
2874 assert!(
2875 !meta.object_lock_config_xml.is_empty(),
2876 "the Object Lock configuration must be persisted, not just the flag"
2877 );
2878 assert!(
2879 !meta.versioning_config_xml.is_empty(),
2880 "Object Lock requires versioning, so that must be persisted too"
2881 );
2882 }
2883
2884 #[tokio::test(flavor = "multi_thread")]
2885 #[serial]
2886 async fn make_bucket_seeds_new_bucket_durability_override() {
2887 temp_env::async_with_vars([(crate::bucket::durability::ENV_NEW_BUCKET_DURABILITY_MODE, None::<&str>)], async {
2888 let (_disk_paths, ecstore) = setup_bucket_delete_test_env().await;
2889 let bucket = format!("bucket-default-durability-{}", Uuid::new_v4().simple());
2890
2891 ecstore
2892 .make_bucket(&bucket, &MakeBucketOptions::default())
2893 .await
2894 .expect("new bucket should be created");
2895
2896 let metadata = metadata_sys::get_in(&ecstore.ctx, &bucket)
2897 .await
2898 .expect("metadata should load for the new bucket");
2899 assert_eq!(
2900 metadata.durability_config().and_then(|cfg| cfg.normalized_mode()).as_deref(),
2901 Some(crate::bucket::durability::BUCKET_DURABILITY_MODE_RELAXED)
2902 );
2903 })
2904 .await;
2905 }
2906
2907 #[tokio::test(flavor = "multi_thread")]
2908 #[serial]
2909 async fn force_create_existing_bucket_keeps_durability_override() {
2910 let (_disk_paths, ecstore) = setup_bucket_delete_test_env().await;
2911 let bucket = format!("bucket-force-durability-{}", Uuid::new_v4().simple());
2912
2913 temp_env::async_with_vars([(crate::bucket::durability::ENV_NEW_BUCKET_DURABILITY_MODE, Some("inherit"))], async {
2914 ecstore
2915 .make_bucket(&bucket, &MakeBucketOptions::default())
2916 .await
2917 .expect("plain bucket should be created without a durability override");
2918 })
2919 .await;
2920 assert!(
2921 metadata_sys::get_in(&ecstore.ctx, &bucket)
2922 .await
2923 .expect("metadata should load after initial create")
2924 .durability_config()
2925 .is_none(),
2926 "test setup: the existing bucket must start without an override"
2927 );
2928
2929 temp_env::async_with_vars([(crate::bucket::durability::ENV_NEW_BUCKET_DURABILITY_MODE, None::<&str>)], async {
2930 ecstore
2931 .make_bucket(
2932 &bucket,
2933 &MakeBucketOptions {
2934 force_create: true,
2935 lock_enabled: true,
2936 ..Default::default()
2937 },
2938 )
2939 .await
2940 .expect("force create should update existing bucket metadata");
2941 })
2942 .await;
2943
2944 let metadata = metadata_sys::get_in(&ecstore.ctx, &bucket)
2945 .await
2946 .expect("metadata should load after force create");
2947 assert!(metadata.lock_enabled, "force create sanity check: Object Lock should be enabled");
2948 assert!(
2949 metadata.durability_config().is_none(),
2950 "force create must not apply the new-bucket default to existing bucket metadata"
2951 );
2952 }
2953
2954 #[tokio::test(flavor = "multi_thread")]
2960 #[serial]
2961 async fn bucket_delete_succeeds_after_the_last_object_version_is_deleted() {
2962 let (disk_paths, ecstore) = setup_bucket_delete_test_env().await;
2963 let bucket = format!("bucket-delete-after-drain-{}", Uuid::new_v4().simple());
2964 let object = "object.txt";
2965
2966 create_bucket_with_object(&ecstore, &bucket, object).await;
2967
2968 ecstore
2969 .delete_object(&bucket, object, ObjectOptions::default())
2970 .await
2971 .expect("client delete of the only object should succeed");
2972
2973 assert!(
2974 !any_disk_has_object_metadata(&disk_paths, &bucket).await,
2975 "deleting the last version must not leave xl.meta on disk: DeleteBucket scans the raw \
2976 bucket directory, so residue here is reported to clients as BucketNotEmpty"
2977 );
2978
2979 ecstore
2980 .delete_bucket(&bucket, &DeleteBucketOptions::default())
2981 .await
2982 .expect("DeleteBucket must succeed once the client has drained the bucket");
2983 }
2984
2985 #[tokio::test(flavor = "multi_thread")]
2986 #[serial]
2987 async fn bucket_delete_defers_zero_evidence_diagnostic_timeout_to_physical_empty_check() {
2988 let (disk_paths, ecstore) = setup_bucket_delete_test_env().await;
2989 let bucket = format!("bucket-del-diag-{}", Uuid::new_v4().simple());
2990
2991 ecstore
2992 .make_bucket(&bucket, &MakeBucketOptions::default())
2993 .await
2994 .expect("empty bucket should be created");
2995 let first_io_started = Arc::new(AtomicBool::new(false));
2996 let diagnostic_budget = BucketDeleteDiagnosticBudget::new().with_first_io_delay(
2997 BUCKET_DELETE_DIAGNOSTIC_MAX_ELAPSED + Duration::from_millis(100),
2998 first_io_started.clone(),
2999 );
3000
3001 ecstore
3002 .handle_delete_bucket_with_diagnostic_budget(&bucket, &DeleteBucketOptions::default(), diagnostic_budget)
3003 .await
3004 .expect("a zero-evidence diagnostic timeout must defer to the physical empty check");
3005
3006 assert!(
3007 first_io_started.load(Ordering::SeqCst),
3008 "the regression must delay the first diagnostic read_dir"
3009 );
3010 assert!(
3011 !any_disk_path_exists(&disk_paths, &bucket).await,
3012 "the physical empty check should allow deletion of the actually empty bucket"
3013 );
3014 }
3015
3016 #[tokio::test(flavor = "multi_thread")]
3017 #[serial]
3018 async fn bucket_delete_preserves_unobserved_residue_after_diagnostic_timeout() {
3019 let (disk_paths, ecstore) = setup_bucket_delete_test_env().await;
3020 let bucket = format!("bucket-del-residue-{}", Uuid::new_v4().simple());
3021 let object = "object.txt";
3022
3023 ecstore
3024 .make_bucket(&bucket, &MakeBucketOptions::default())
3025 .await
3026 .expect("bucket should be created");
3027 write_metadata_less_part_on_all_disks(&disk_paths, &bucket, object).await;
3028 let first_io_started = Arc::new(AtomicBool::new(false));
3029 let diagnostic_budget = BucketDeleteDiagnosticBudget::new().with_first_io_delay(
3030 BUCKET_DELETE_DIAGNOSTIC_MAX_ELAPSED + Duration::from_millis(100),
3031 first_io_started.clone(),
3032 );
3033
3034 let err = ecstore
3035 .handle_delete_bucket_with_diagnostic_budget(&bucket, &DeleteBucketOptions::default(), diagnostic_budget)
3036 .await
3037 .expect_err("physical empty-bucket enforcement must reject unobserved residue");
3038 assert!(
3039 first_io_started.load(Ordering::SeqCst),
3040 "the regression must time out before observing the residue"
3041 );
3042 match &err {
3043 StorageError::BucketNotEmpty(err_bucket) | StorageError::BucketNotEmptyWithDetails { bucket: err_bucket, .. } => {
3044 assert_eq!(err_bucket, &bucket)
3045 }
3046 other => panic!("expected an S3-compatible BucketNotEmpty error, got {other:?}"),
3047 }
3048 assert!(
3049 any_disk_path_exists(&disk_paths, Path::new(&bucket).join(object)).await,
3050 "physical empty-bucket enforcement must preserve unobserved residue"
3051 );
3052
3053 ecstore
3054 .delete_bucket(
3055 &bucket,
3056 &DeleteBucketOptions {
3057 force: true,
3058 ..Default::default()
3059 },
3060 )
3061 .await
3062 .expect("explicit force should clean up the test-owned residue");
3063 }
3064
3065 #[tokio::test(flavor = "multi_thread")]
3066 #[serial]
3067 async fn bucket_delete_succeeds_after_listing_and_deleting_an_unversioned_overwrite() {
3068 let (disk_paths, ecstore) = setup_bucket_delete_test_env().await;
3069 let bucket = format!("bucket-delete-after-overwrite-{}", Uuid::new_v4().simple());
3070 let object = "object.txt";
3071
3072 ecstore
3073 .make_bucket(&bucket, &MakeBucketOptions::default())
3074 .await
3075 .expect("unversioned bucket should be created");
3076
3077 let mut first_reader = PutObjReader::from_vec(b"version A".to_vec());
3078 let first = ecstore
3079 .put_object(&bucket, object, &mut first_reader, &ObjectOptions::default())
3080 .await
3081 .expect("version A should be written");
3082 let mut second_reader = PutObjReader::from_vec(b"version B".to_vec());
3083 let second = ecstore
3084 .put_object(&bucket, object, &mut second_reader, &ObjectOptions::default())
3085 .await
3086 .expect("version B should overwrite version A");
3087 assert_ne!(first.data_dir, second.data_dir, "the overwrite must publish a new body generation");
3088
3089 let listing = ecstore
3090 .clone()
3091 .list_object_versions(&bucket, "", None, None, None, 1000)
3092 .await
3093 .expect("the overwritten object should remain listable for teardown");
3094 assert_eq!(listing.objects.len(), 1, "an unversioned overwrite should expose one current version");
3095 let current = &listing.objects[0];
3096 assert_eq!(current.name, object);
3097 assert!(current.is_latest, "the listed null version must be current");
3098 assert_eq!(current.version_id, None, "an unversioned object must be exposed as the null version");
3099 assert_eq!(
3100 current.data_dir, second.data_dir,
3101 "listing must expose version B, not the overwritten body"
3102 );
3103
3104 for version in listing.objects {
3105 let version_id = version.version_id.map(|version_id| version_id.to_string());
3106 ecstore
3107 .delete_object(
3108 &bucket,
3109 &version.name,
3110 ObjectOptions {
3111 version_id,
3112 ..Default::default()
3113 },
3114 )
3115 .await
3116 .expect("each version returned by teardown listing should be deletable");
3117 }
3118
3119 assert!(
3120 !any_disk_has_object_metadata(&disk_paths, &bucket).await,
3121 "deleting the listed null version must remove every xl.meta"
3122 );
3123 ecstore
3124 .delete_bucket(&bucket, &DeleteBucketOptions::default())
3125 .await
3126 .expect("DeleteBucket should succeed after the listed overwrite is deleted");
3127 }
3128
3129 #[tokio::test]
3130 #[serial]
3131 async fn bucket_delete_default_s3_delete_still_rejects_non_empty_bucket() {
3132 let (disk_paths, ecstore) = setup_bucket_delete_test_env().await;
3133 let bucket = format!("bucket-s3-delete-{}", Uuid::new_v4().simple());
3134 let object = "object.txt";
3135
3136 create_bucket_with_object(&ecstore, &bucket, object).await;
3137
3138 let generation_before_delete = ecstore.scanner_namespace_mutation_generation();
3139 let err = ecstore
3140 .delete_bucket(&bucket, &DeleteBucketOptions::default())
3141 .await
3142 .expect_err("default S3 DeleteBucket should reject non-empty buckets");
3143
3144 assert!(matches!(err, StorageError::BucketNotEmpty(name) if name == bucket));
3145 assert_eq!(
3146 ecstore.scanner_namespace_mutation_generation(),
3147 generation_before_delete,
3148 "failed bucket deletion must not advance scanner namespace activity"
3149 );
3150 assert!(
3151 any_disk_has_object_metadata(&disk_paths, &bucket).await,
3152 "failed default S3 DeleteBucket must keep object data"
3153 );
3154 assert!(
3155 metadata_sys::get_in(&ecstore.ctx, &bucket).await.is_ok(),
3156 "failed default S3 DeleteBucket must keep metadata cache"
3157 );
3158 }
3159
3160 #[tokio::test(flavor = "multi_thread")]
3161 #[serial]
3162 async fn bucket_delete_reports_metadata_less_residue_without_removing_it() {
3163 let (disk_paths, ecstore) = setup_bucket_delete_test_env().await;
3164 let bucket = format!("bucket-delete-orphan-datadir-{}", Uuid::new_v4().simple());
3165 let object = "object.txt";
3166
3167 ecstore
3168 .make_bucket(&bucket, &MakeBucketOptions::default())
3169 .await
3170 .expect("bucket should be created");
3171 write_metadata_less_part_on_all_disks(&disk_paths, &bucket, object).await;
3172 assert!(
3173 !any_disk_has_object_metadata(&disk_paths, &bucket).await,
3174 "test setup must reproduce a bucket with data dirs but no xl.meta"
3175 );
3176
3177 let err = ecstore
3178 .delete_bucket(&bucket, &DeleteBucketOptions::default())
3179 .await
3180 .expect_err("metadata-less data dirs must block ordinary DeleteBucket");
3181
3182 match err {
3183 StorageError::BucketNotEmptyWithDetails {
3184 bucket: err_bucket,
3185 details,
3186 } => {
3187 assert_eq!(err_bucket, bucket);
3188 assert!(
3189 details.contains("metadata-less on-disk residue"),
3190 "diagnostic should identify metadata-less residue, got: {details}"
3191 );
3192 assert!(details.contains("files="), "diagnostic should include a bounded count");
3193 assert!(details.contains("uuid_data_dirs="), "diagnostic should include data-dir count");
3194 }
3195 other => panic!("expected detailed BucketNotEmpty, got {other:?}"),
3196 }
3197 assert!(
3198 any_disk_path_exists(&disk_paths, Path::new(&bucket).join(object)).await,
3199 "ordinary DeleteBucket must preserve metadata-less data until an explicit operator action"
3200 );
3201 }
3202
3203 #[tokio::test]
3204 #[serial]
3205 async fn bucket_delete_fences_put_started_after_empty_scan() {
3206 let (_, ecstore) = setup_bucket_delete_test_env().await;
3207 let bucket = format!("bucket-delete-empty-scan-race-{}", Uuid::new_v4().simple());
3208 let object = "committed-after-empty-scan";
3209 let payload = b"object started after DeleteBucket empty scan".to_vec();
3210
3211 ecstore
3212 .make_bucket(&bucket, &MakeBucketOptions::default())
3213 .await
3214 .expect("bucket should be created");
3215
3216 let barrier = install_delete_bucket_empty_scan_barrier();
3217 let delete_store = ecstore.clone();
3218 let delete_bucket = bucket.clone();
3219 let delete = tokio::spawn(async move {
3220 delete_store
3221 .delete_bucket(&delete_bucket, &DeleteBucketOptions::default())
3222 .await
3223 });
3224 barrier.wait_until_paused().await;
3225
3226 let put_store = ecstore.clone();
3227 let put_bucket = bucket.clone();
3228 let mut put = tokio::spawn(async move {
3229 let mut put_reader = PutObjReader::from_vec(payload);
3230 put_store
3231 .put_object(&put_bucket, object, &mut put_reader, &ObjectOptions::default())
3232 .await
3233 });
3234 assert!(
3235 tokio::time::timeout(Duration::from_millis(100), &mut put).await.is_err(),
3236 "PUT must wait behind the DeleteBucket lifecycle fence"
3237 );
3238
3239 barrier.release();
3240 delete
3241 .await
3242 .expect("DeleteBucket task should join")
3243 .expect("DeleteBucket should commit while holding the lifecycle fence");
3244 let err = put
3245 .await
3246 .expect("PUT task should join")
3247 .expect_err("PUT must not recreate an object in the deleted bucket");
3248 assert!(matches!(err, StorageError::BucketNotFound(name) if name == bucket));
3249 }
3250
3251 #[tokio::test]
3252 #[serial]
3253 async fn bucket_recreation_does_not_publish_unverified_usage() {
3254 let (_, ecstore) = setup_bucket_delete_test_env().await;
3255 let bucket = format!("bucket-usage-generation-{}", Uuid::new_v4().simple());
3256 ecstore
3257 .make_bucket(&bucket, &MakeBucketOptions::default())
3258 .await
3259 .expect("bucket should be created");
3260
3261 let mut snapshot = DataUsageInfo {
3262 last_update: Some(SystemTime::now()),
3263 buckets_count: 1,
3264 ..Default::default()
3265 };
3266 snapshot.buckets_usage.insert(
3267 bucket.clone(),
3268 BucketUsageInfo {
3269 size: 42,
3270 objects_count: 1,
3271 versions_count: 1,
3272 ..Default::default()
3273 },
3274 );
3275 snapshot.usage_snapshot_complete = true;
3276 snapshot.bucket_sizes.insert(bucket.clone(), 42);
3277 snapshot.calculate_totals();
3278 crate::data_usage::store_data_usage_in_backend(snapshot, ecstore.clone())
3279 .await
3280 .expect("usage snapshot should be stored");
3281
3282 ecstore
3283 .delete_bucket(&bucket, &DeleteBucketOptions::default())
3284 .await
3285 .expect("empty bucket should be deleted");
3286 let deleted = crate::data_usage::load_data_usage_from_backend(ecstore.clone())
3287 .await
3288 .expect("usage snapshot should remain readable");
3289 assert!(!deleted.buckets_usage.contains_key(&bucket));
3290
3291 ecstore
3292 .make_bucket(&bucket, &MakeBucketOptions::default())
3293 .await
3294 .expect("same bucket name should be recreated after delete returns");
3295 crate::data_usage::record_bucket_object_write_memory(&bucket, None, 84).await;
3296
3297 let mut recreated = crate::data_usage::load_data_usage_from_backend(ecstore.clone())
3298 .await
3299 .expect("recreated bucket usage base should load");
3300 crate::data_usage::apply_bucket_usage_memory_overlay(&mut recreated).await;
3301 assert!(
3302 !recreated.buckets_usage.contains_key(&bucket),
3303 "a request-path delta without an authoritative baseline must remain unavailable"
3304 );
3305 assert_eq!(crate::data_usage::get_bucket_usage_memory(&bucket).await, None);
3306 }
3307
3308 #[tokio::test]
3309 #[serial]
3310 async fn bucket_create_removes_stale_usage_before_physical_creation() {
3311 let (_, ecstore) = setup_bucket_delete_test_env().await;
3312 let bucket = format!("bucket-create-stale-usage-{}", Uuid::new_v4().simple());
3313 let mut snapshot = DataUsageInfo {
3314 last_update: Some(SystemTime::now()),
3315 buckets_count: 1,
3316 ..Default::default()
3317 };
3318 snapshot.buckets_usage.insert(
3319 bucket.clone(),
3320 BucketUsageInfo {
3321 size: 42,
3322 objects_count: 1,
3323 versions_count: 1,
3324 ..Default::default()
3325 },
3326 );
3327 snapshot.usage_snapshot_complete = true;
3328 snapshot.bucket_sizes.insert(bucket.clone(), 42);
3329 snapshot.calculate_totals();
3330 crate::data_usage::store_data_usage_in_backend(snapshot, ecstore.clone())
3331 .await
3332 .expect("stale usage fixture should be stored");
3333
3334 ecstore
3335 .make_bucket(&bucket, &MakeBucketOptions::default())
3336 .await
3337 .expect("CreateBucket should succeed");
3338
3339 let persisted = crate::data_usage::load_data_usage_from_backend(ecstore.clone())
3340 .await
3341 .expect("usage snapshot should remain readable");
3342 assert!(
3343 !persisted.buckets_usage.contains_key(&bucket),
3344 "a newly created bucket must not inherit the predecessor generation's usage"
3345 );
3346 assert!(
3347 ecstore.get_bucket_info(&bucket, &BucketOptions::default()).await.is_ok(),
3348 "physical creation should happen after the usage fence succeeds"
3349 );
3350 }
3351
3352 #[tokio::test]
3353 #[serial]
3354 async fn failed_create_rollback_does_not_run_unfenced_usage_cleanup() {
3355 let (_, ecstore) = setup_bucket_delete_test_env().await;
3356 let bucket = format!("bucket-create-rollback-{}", Uuid::new_v4().simple());
3357 ecstore
3358 .peer_sys
3359 .make_bucket(&bucket, &MakeBucketOptions::default())
3360 .await
3361 .expect("the partial-create fixture should expose a physical bucket");
3362
3363 let mut snapshot = DataUsageInfo {
3364 last_update: Some(SystemTime::now()),
3365 buckets_count: 1,
3366 ..Default::default()
3367 };
3368 snapshot.buckets_usage.insert(
3369 bucket.clone(),
3370 BucketUsageInfo {
3371 size: 42,
3372 objects_count: 1,
3373 versions_count: 1,
3374 ..Default::default()
3375 },
3376 );
3377 snapshot.usage_snapshot_complete = true;
3378 snapshot.bucket_sizes.insert(bucket.clone(), 42);
3379 snapshot.calculate_totals();
3380 crate::data_usage::store_data_usage_in_backend(snapshot, ecstore.clone())
3381 .await
3382 .expect("the usage fixture should be stored");
3383 crate::bucket::metadata::save_bucket_incarnation(ecstore.clone(), &bucket, Uuid::new_v4())
3384 .await
3385 .expect("partial create should have an incarnation sidecar");
3386
3387 ecstore.rollback_failed_bucket_creation(&bucket, None, None).await;
3388
3389 assert!(
3390 ecstore
3391 .peer_sys
3392 .get_bucket_info(&bucket, &BucketOptions::default())
3393 .await
3394 .is_err(),
3395 "failed-create rollback should remove the partial physical bucket"
3396 );
3397 assert!(
3398 crate::bucket::metadata::load_bucket_incarnation(ecstore.clone(), &bucket)
3399 .await
3400 .expect("rollback sidecar lookup should succeed")
3401 .is_none(),
3402 "failed-create rollback must remove the orphan incarnation sidecar"
3403 );
3404 let persisted = crate::data_usage::load_data_usage_from_backend(ecstore.clone())
3405 .await
3406 .expect("the usage snapshot should remain readable");
3407 assert!(
3408 persisted.buckets_usage.contains_key(&bucket),
3409 "failed-create rollback must not start an unfenced usage cleanup"
3410 );
3411
3412 crate::data_usage::store_data_usage_in_backend(DataUsageInfo::default(), ecstore.clone())
3413 .await
3414 .expect("the rollback usage fixture should be cleared");
3415 }
3416
3417 #[tokio::test]
3418 #[serial]
3419 async fn bucket_create_fails_closed_when_usage_snapshot_cannot_be_fenced() {
3420 let (_, ecstore) = setup_bucket_delete_test_env().await;
3421 let deleted_bucket = format!("bucket-delete-corrupt-usage-{}", Uuid::new_v4().simple());
3422 ecstore
3423 .make_bucket(&deleted_bucket, &MakeBucketOptions::default())
3424 .await
3425 .expect("bucket should be created before corrupting usage");
3426
3427 let usage_path = format!("{BUCKET_META_PREFIX}/{DATA_USAGE_OBJECT_NAME}");
3428 crate::config::com::save_config(ecstore.clone(), &usage_path, b"{".to_vec())
3429 .await
3430 .expect("corrupt usage fixture should be stored");
3431
3432 ecstore
3433 .delete_bucket(&deleted_bucket, &DeleteBucketOptions::default())
3434 .await
3435 .expect("usage snapshot corruption must not block DeleteBucket");
3436 assert!(
3437 ecstore
3438 .get_bucket_info(&deleted_bucket, &crate::storage_api_contracts::bucket::BucketOptions::default())
3439 .await
3440 .is_err(),
3441 "successful DeleteBucket must remove the physical bucket"
3442 );
3443
3444 let new_bucket = format!("bucket-create-corrupt-usage-{}", Uuid::new_v4().simple());
3445 let create_err = ecstore
3446 .make_bucket(&new_bucket, &MakeBucketOptions::default())
3447 .await
3448 .expect_err("MakeBucket must fail when stale usage cannot be fenced");
3449 assert!(!create_err.to_string().is_empty(), "the usage snapshot failure should be surfaced");
3450 assert!(
3451 ecstore
3452 .get_bucket_info(&new_bucket, &crate::storage_api_contracts::bucket::BucketOptions::default())
3453 .await
3454 .is_err(),
3455 "failed MakeBucket must not expose a new physical bucket"
3456 );
3457
3458 let restored = serde_json::to_vec(&DataUsageInfo {
3459 last_update: Some(SystemTime::now()),
3460 ..Default::default()
3461 })
3462 .expect("restored usage fixture should encode");
3463 crate::config::com::save_config(ecstore, &usage_path, restored)
3464 .await
3465 .expect("usage fixture should be restored after the failure-path test");
3466 }
3467
3468 fn bucket_logs_at_default_level(emit: impl FnOnce()) -> String {
3470 use std::sync::{Arc, Mutex};
3471 use tracing_subscriber::EnvFilter;
3472 use tracing_subscriber::fmt::MakeWriter;
3473 use tracing_subscriber::layer::SubscriberExt;
3474
3475 #[derive(Clone, Default)]
3476 struct CapturedLogs {
3477 buffer: Arc<Mutex<Vec<u8>>>,
3478 }
3479 struct CapturedLogWriter {
3480 buffer: Arc<Mutex<Vec<u8>>>,
3481 }
3482 impl std::io::Write for CapturedLogWriter {
3483 fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
3484 self.buffer
3485 .lock()
3486 .expect("captured logs mutex should not be poisoned")
3487 .extend_from_slice(buf);
3488 Ok(buf.len())
3489 }
3490 fn flush(&mut self) -> std::io::Result<()> {
3491 Ok(())
3492 }
3493 }
3494 impl<'a> MakeWriter<'a> for CapturedLogs {
3495 type Writer = CapturedLogWriter;
3496 fn make_writer(&'a self) -> Self::Writer {
3497 CapturedLogWriter {
3498 buffer: Arc::clone(&self.buffer),
3499 }
3500 }
3501 }
3502
3503 let logs = CapturedLogs::default();
3504 let subscriber = tracing_subscriber::registry()
3505 .with(EnvFilter::new(rustfs_config::DEFAULT_LOG_LEVEL))
3506 .with(
3507 tracing_subscriber::fmt::layer()
3508 .with_writer(logs.clone())
3509 .with_ansi(false)
3510 .without_time(),
3511 );
3512 let _guard = tracing::subscriber::set_default(subscriber);
3513 let _callsite_pin = crate::test_tracing::pin_callsite_interest_for_test();
3514
3515 emit();
3516
3517 let buffer = logs
3518 .buffer
3519 .lock()
3520 .expect("captured logs mutex should not be poisoned")
3521 .clone();
3522 String::from_utf8(buffer).expect("captured logs should be valid UTF-8")
3523 }
3524
3525 fn residue_sample(sample: &str) -> BucketMetadataLessResidue {
3526 BucketMetadataLessResidue {
3527 files: 1,
3528 entries_scanned: 3,
3529 sample: Some(sample.to_string()),
3530 ..Default::default()
3531 }
3532 }
3533
3534 #[test]
3540 fn unreachable_residue_is_reported_at_the_default_log_level() {
3541 for kind in [
3542 BucketDeleteBlockerKind::UnknownXlMeta,
3543 BucketDeleteBlockerKind::OrphanDirectory,
3544 BucketDeleteBlockerKind::DiagnosticBudgetExceeded,
3545 ] {
3546 let logs = bucket_logs_at_default_level(|| {
3547 record_bucket_delete_blocker("drained-bucket", kind, &residue_sample("obj/8f2c/xl.meta"));
3548 });
3549
3550 assert!(logs.contains("drained-bucket"), "the bucket must be named for {kind:?}: {logs}");
3551 assert!(logs.contains(kind.as_str()), "the blocker kind must be named for {kind:?}: {logs}");
3552 assert!(
3553 logs.contains("obj/8f2c/xl.meta"),
3554 "the residue sample is the only pointer to the leftover state for {kind:?}: {logs}"
3555 );
3556 }
3557 }
3558
3559 #[test]
3563 fn client_visible_blockers_stay_below_the_default_log_level() {
3564 for kind in [
3565 BucketDeleteBlockerKind::VisibleVersion,
3566 BucketDeleteBlockerKind::TierFreeVersion,
3567 ] {
3568 let logs = bucket_logs_at_default_level(|| {
3569 record_bucket_delete_blocker("still-full-bucket", kind, &residue_sample("obj/xl.meta"));
3570 });
3571
3572 assert!(logs.is_empty(), "an ordinary non-empty bucket must not log an error for {kind:?}: {logs}");
3573 }
3574 }
3575
3576 #[test]
3577 fn blocker_kinds_split_by_whether_the_client_can_reach_the_residue() {
3578 assert!(BucketDeleteBlockerKind::VisibleVersion.is_client_visible());
3579 assert!(BucketDeleteBlockerKind::TierFreeVersion.is_client_visible());
3580 assert!(!BucketDeleteBlockerKind::UnknownXlMeta.is_client_visible());
3581 assert!(!BucketDeleteBlockerKind::OrphanDirectory.is_client_visible());
3582 assert!(!BucketDeleteBlockerKind::DiagnosticBudgetExceeded.is_client_visible());
3583 }
3584}