Skip to main content

rustfs_ecstore/store/
bucket.rs

1// Copyright 2024 RustFS Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use super::*;
16use crate::bucket::{
17    metadata::{BUCKET_TABLE_RESERVED_PREFIX, table_bucket_catalog_metadata_prefix},
18    utils::is_meta_bucketname,
19};
20use crate::error::is_err_bucket_not_found;
21use crate::runtime::sources as runtime_sources;
22use crate::set_disk::{BucketInfoQuorum, get_lock_acquire_timeout};
23use crate::storage_api_contracts::bucket::{BUCKET_LIFECYCLE_LOCK_OBJECT, SRBucketDeleteOp};
24use crate::storage_api_contracts::namespace::NamespaceLocking as _;
25use futures::stream::{self, StreamExt};
26use rustfs_policy::policy::BucketPolicy;
27use std::collections::BTreeMap;
28use std::future::Future;
29
30const DELETED_BUCKETS_PREFIX: &str = ".deleted";
31const SCANNER_BUCKET_LIST_SET_CONCURRENCY: usize = 4;
32const EVENT_BUCKET_DELETE_BLOCKED: &str = "bucket_delete_blocked";
33const EVENT_BUCKET_DELETE_ROLLBACK_FAILED: &str = "bucket_delete_rollback_failed";
34
35/// Record why `DeleteBucket` refused, and at a level that matches who can act
36/// on it.
37///
38/// `DeleteBucket` answers from a raw per-disk residue scan, not from a listing,
39/// so the server can refuse for a reason the client has no way to observe: the
40/// caller drained every version the S3 API will show and still gets
41/// `BucketNotEmpty`, with nothing to go on. This event is the only record of
42/// which residue blocked it and where — and it used to be emitted at `debug`,
43/// below both the `error` default log level and the `info` the CI s3-tests lane
44/// runs at, so in practice it was never written down. An intermittent
45/// `BucketNotEmpty` in that lane left a server log with no trace of the refusal
46/// at all, which is not a diagnosable state.
47///
48/// A blocker the client can still see and delete is ordinary — the 409 already
49/// says everything useful — so it stays at `warn`. Residue the client cannot
50/// reach through the API is a server-side integrity problem and is reported at
51/// `error`, which is what makes it survive a default deployment's filter.
52fn record_bucket_delete_blocker(bucket: &str, kind: BucketDeleteBlockerKind, residue: &BucketMetadataLessResidue) {
53    metrics::counter!("rustfs_bucket_delete_blockers_total", "kind" => kind.as_str()).increment(1);
54    if kind.is_client_visible() {
55        warn!(
56            event = EVENT_BUCKET_DELETE_BLOCKED,
57            component = "ecstore",
58            subsystem = "bucket",
59            bucket,
60            blocker = kind.as_str(),
61            files = residue.files,
62            uuid_data_dirs = residue.uuid_data_dirs,
63            entries_scanned = residue.entries_scanned,
64            diagnostic_bytes_read = residue.diagnostic_bytes_read,
65            diagnostic_truncated = residue.diagnostic_truncated,
66            sample = residue.sample.as_deref().unwrap_or("<none>"),
67            "Bucket deletion was blocked by durable local state"
68        );
69        return;
70    }
71
72    error!(
73        event = EVENT_BUCKET_DELETE_BLOCKED,
74        component = "ecstore",
75        subsystem = "bucket",
76        bucket,
77        blocker = kind.as_str(),
78        files = residue.files,
79        uuid_data_dirs = residue.uuid_data_dirs,
80        entries_scanned = residue.entries_scanned,
81        diagnostic_bytes_read = residue.diagnostic_bytes_read,
82        diagnostic_truncated = residue.diagnostic_truncated,
83        sample = residue.sample.as_deref().unwrap_or("<none>"),
84        "Bucket deletion was blocked by residue the client cannot reach through the S3 API"
85    );
86}
87
88fn bucket_list_set_concurrency(set_count: usize, max_concurrency: usize) -> usize {
89    set_count.clamp(1, max_concurrency.max(1))
90}
91
92fn should_override_created_from_metadata(created: OffsetDateTime) -> bool {
93    created != OffsetDateTime::UNIX_EPOCH
94}
95
96fn validate_table_bucket_delete_allowed(
97    bucket: &str,
98    table_bucket_enabled: bool,
99    table_catalog_metadata_exists: bool,
100) -> Result<()> {
101    if table_bucket_enabled && table_catalog_metadata_exists {
102        return Err(StorageError::BucketNotEmpty(bucket.to_string()));
103    }
104
105    Ok(())
106}
107
108async fn table_catalog_metadata_exists(ctx: &crate::runtime::instance::InstanceContext, bucket: &str) -> Result<bool> {
109    let local_disks = runtime_sources::local_disks_in(ctx).await;
110    for disk in local_disks.iter() {
111        let Some(bucket_path) = disk.get_bucket_path_for_io_if_local(bucket) else {
112            continue;
113        };
114        let catalog_path = bucket_path?.join(BUCKET_TABLE_RESERVED_PREFIX);
115        if has_xlmeta_files(&catalog_path).await? {
116            return Ok(true);
117        }
118    }
119
120    Ok(false)
121}
122
123async fn validate_table_bucket_delete_guard(ctx: &crate::runtime::instance::InstanceContext, bucket: &str) -> Result<()> {
124    let table_bucket_enabled = metadata_sys::get_in(ctx, bucket)
125        .await
126        .is_ok_and(|metadata| metadata.table_bucket_enabled());
127    if table_bucket_enabled {
128        validate_table_bucket_delete_allowed(bucket, true, table_catalog_metadata_exists(ctx, bucket).await?)?;
129    }
130
131    Ok(())
132}
133
134fn bucket_delete_metadata_cleanup_prefixes(bucket: &str) -> [String; 2] {
135    [
136        table_bucket_catalog_metadata_prefix(bucket),
137        format!("{BUCKET_META_PREFIX}/{bucket}"),
138    ]
139}
140
141fn bucket_deleted_marker_prefix(bucket: &str) -> String {
142    format!("{BUCKET_META_PREFIX}/{DELETED_BUCKETS_PREFIX}/{bucket}")
143}
144
145fn bucket_deleted_marker_volume(bucket: &str) -> String {
146    format!("{RUSTFS_META_BUCKET}/{}", bucket_deleted_marker_prefix(bucket))
147}
148
149pub(crate) async fn await_bucket_namespace_operation<T, F>(
150    guard: Option<&rustfs_lock::NamespaceLockGuard>,
151    bucket: &str,
152    operation: &'static str,
153    future: F,
154) -> Result<T>
155where
156    F: Future<Output = Result<T>>,
157{
158    let Some(guard) = guard else {
159        return future.await;
160    };
161    if guard.is_lock_lost() {
162        return Err(StorageError::other(format!(
163            "bucket namespace lock was lost before {operation}: {bucket}"
164        )));
165    }
166    tokio::select! {
167        biased;
168        _ = guard.lock_lost_notified() => Err(StorageError::other(format!(
169            "bucket namespace lock was lost during {operation}: {bucket}"
170        ))),
171        result = future => result,
172    }
173}
174
175async fn await_bucket_lifecycle_operation<T, F>(
176    lifecycle_guard: Option<&rustfs_lock::NamespaceLockGuard>,
177    namespace_guard: Option<&rustfs_lock::NamespaceLockGuard>,
178    bucket: &str,
179    operation: &'static str,
180    future: F,
181) -> Result<T>
182where
183    F: Future<Output = Result<T>>,
184{
185    await_bucket_namespace_operation(
186        lifecycle_guard,
187        bucket,
188        operation,
189        await_bucket_namespace_operation(namespace_guard, bucket, operation, future),
190    )
191    .await
192}
193
194async fn run_bucket_usage_cleanup<F>(guard: Option<&rustfs_lock::NamespaceLockGuard>, bucket: &str, future: F) -> Result<()>
195where
196    F: Future<Output = Result<()>>,
197{
198    await_bucket_namespace_operation(guard, bucket, "bucket usage cleanup", future).await
199}
200
201async fn run_physical_bucket_deletion<F>(guard: Option<&rustfs_lock::NamespaceLockGuard>, bucket: &str, future: F) -> Result<()>
202where
203    F: Future<Output = Result<()>>,
204{
205    // Fence before polling deletion: the physical namespace may become
206    // invisible at any await point inside the storage operation.
207    list_objects::observe_scanner_namespace_mutations(bucket, 1);
208    await_bucket_namespace_operation(guard, bucket, "physical bucket deletion", future).await
209}
210
211async fn bucket_delete_local_blocker(
212    ctx: &crate::runtime::instance::InstanceContext,
213    bucket: &str,
214    budget: &mut BucketDeleteDiagnosticBudget,
215) -> Result<Option<StorageError>> {
216    let local_disks = runtime_sources::local_disks_in(ctx).await;
217    let mut residue = BucketMetadataLessResidue::default();
218
219    for disk in local_disks.iter() {
220        let Some(bucket_path) = disk.get_bucket_path_for_io_if_local(bucket) else {
221            continue;
222        };
223        let scan = scan_metadata_less_residue_with_budget(&bucket_path?, budget).await?;
224        if scan.xlmeta_found {
225            record_bucket_delete_blocker(bucket, scan.xlmeta_blocker.unwrap_or(BucketDeleteBlockerKind::UnknownXlMeta), &scan);
226            return Ok(Some(StorageError::BucketNotEmpty(bucket.to_string())));
227        }
228        residue.files = residue.files.saturating_add(scan.files);
229        residue.uuid_data_dirs = residue.uuid_data_dirs.saturating_add(scan.uuid_data_dirs);
230        residue.entries_scanned = residue.entries_scanned.saturating_add(scan.entries_scanned);
231        residue.diagnostic_bytes_read = residue.diagnostic_bytes_read.saturating_add(scan.diagnostic_bytes_read);
232        if residue.sample.is_none() {
233            residue.sample = scan.sample.clone();
234        }
235        if scan.diagnostic_truncated {
236            residue.diagnostic_truncated = true;
237            if residue.files == 0 {
238                // A zero-evidence timeout is inconclusive. Preflight still uses
239                // non-recursive `force_if_empty`; post-failure keeps the physical error.
240                return Ok(None);
241            }
242            record_bucket_delete_blocker(bucket, BucketDeleteBlockerKind::DiagnosticBudgetExceeded, &residue);
243            return Ok(Some(StorageError::BucketNotEmptyWithDetails {
244                bucket: bucket.to_string(),
245                details: residue.describe(),
246            }));
247        }
248    }
249
250    if residue.has_residue_without_xlmeta() {
251        record_bucket_delete_blocker(bucket, BucketDeleteBlockerKind::OrphanDirectory, &residue);
252        return Ok(Some(StorageError::BucketNotEmptyWithDetails {
253            bucket: bucket.to_string(),
254            details: residue.describe(),
255        }));
256    }
257
258    Ok(None)
259}
260
261impl ECStore {
262    fn bucket_sets(&self) -> impl Iterator<Item = (usize, usize, Arc<crate::set_disk::SetDisks>)> + '_ {
263        self.pools.iter().flat_map(|pool| {
264            pool.disk_set
265                .iter()
266                .map(|set| (set.pool_index, set.set_index, Arc::clone(set)))
267        })
268    }
269
270    pub async fn get_bucket_metadata(&self, bucket: &str) -> Result<Arc<BucketMetadata>> {
271        let sys = metadata_sys::require_bucket_metadata_sys_in(&self.ctx)?;
272        sys.read().await.get(bucket).await
273    }
274
275    pub async fn get_bucket_policy(&self, bucket: &str) -> Result<(BucketPolicy, OffsetDateTime)> {
276        let sys = metadata_sys::require_bucket_metadata_sys_in(&self.ctx)?;
277        sys.read().await.get_bucket_policy(bucket).await
278    }
279
280    pub async fn get_bucket_policy_raw(&self, bucket: &str) -> Result<(String, OffsetDateTime)> {
281        let sys = metadata_sys::require_bucket_metadata_sys_in(&self.ctx)?;
282        sys.read().await.get_bucket_policy_raw(bucket).await
283    }
284
285    pub async fn restricts_public_bucket_access(&self, bucket: &str) -> Result<bool> {
286        let sys = metadata_sys::require_bucket_metadata_sys_in(&self.ctx)?;
287        let (config, _) = sys.read().await.get_public_access_block_config(bucket).await?;
288        Ok(config.restrict_public_buckets.unwrap_or(false))
289    }
290
291    pub async fn update_bucket_metadata_config(&self, bucket: &str, config_file: &str, data: Vec<u8>) -> Result<OffsetDateTime> {
292        metadata_sys::update_in(&self.ctx, bucket, config_file, data).await
293    }
294
295    pub async fn bucket_incarnation_id(&self, bucket: &str) -> Result<Uuid> {
296        metadata_sys::get_cached_bucket_incarnation_id_in(&self.ctx, bucket).await
297    }
298
299    pub async fn bucket_incarnation_id_from_disk(&self, bucket: &str) -> Result<Uuid> {
300        metadata_sys::get_bucket_incarnation_id_in(&self.ctx, bucket).await
301    }
302
303    /// The object commit path acquires this sentinel before bucket metadata and
304    /// exact-object namespace locks.
305    pub(crate) async fn acquire_bucket_lifecycle_read_lock(&self, bucket: &str) -> Result<rustfs_lock::NamespaceLockGuard> {
306        let lock = self.new_ns_lock(bucket, BUCKET_LIFECYCLE_LOCK_OBJECT).await?;
307        lock.get_read_lock(get_lock_acquire_timeout()).await.map_err(|err| match err {
308            rustfs_lock::error::LockError::QuorumNotReached { required, achieved } => {
309                StorageError::NamespaceLockQuorumUnavailable {
310                    mode: "bucket_lifecycle_read",
311                    bucket: bucket.to_string(),
312                    object: BUCKET_LIFECYCLE_LOCK_OBJECT.to_string(),
313                    required,
314                    achieved,
315                }
316            }
317            other => StorageError::Lock(other),
318        })
319    }
320
321    /// Acquire the bucket lifecycle read lock and validate the bucket
322    /// incarnation against `expected`, memoizing the validation while this
323    /// node keeps continuous read-lock coverage (see [`super::bucket_fence`]).
324    ///
325    /// Semantics are identical to the pre-existing per-PUT
326    /// `acquire_bucket_lifecycle_read_lock` + from-disk
327    /// `validate_bucket_incarnation` pair: the first PUT in a coverage window
328    /// performs exactly that authoritative disk validation; overlapping PUTs
329    /// reuse its result, which is sound because bucket deletion/recreation
330    /// requires the lifecycle WRITE lock and therefore cannot have run while
331    /// any read guard was continuously held.
332    pub async fn acquire_bucket_incarnation_fence(
333        &self,
334        bucket: &str,
335        expected: uuid::Uuid,
336    ) -> Result<super::BucketIncarnationFenceGuard> {
337        let inner = self.acquire_bucket_lifecycle_read_lock(bucket).await?;
338        let pieces = super::bucket_fence::FencePieces {
339            registry: self.bucket_fence_registry.clone(),
340            inner,
341        };
342        let registration = pieces.enter(bucket);
343        let current = match registration.memoized {
344            Some(current) => current,
345            None => match metadata_sys::get_bucket_incarnation_id_in(&self.ctx, bucket).await {
346                Ok(current) => {
347                    // Never memoize under lost coverage: a granted lifecycle
348                    // write lock could already have changed the incarnation.
349                    if !pieces.lock_lost() {
350                        pieces.memoize(bucket, current);
351                    }
352                    current
353                }
354                Err(err) => {
355                    pieces.abandon(bucket, registration.token);
356                    return Err(err);
357                }
358            },
359        };
360        if current != expected {
361            pieces.abandon(bucket, registration.token);
362            return Err(StorageError::BucketNotFound(bucket.to_string()));
363        }
364        Ok(pieces.into_guard(bucket, registration.token))
365    }
366
367    /// Hold this guard through recursive-delete authorization and mutation so
368    /// writers cannot introduce an unchecked object into the deletion scope.
369    pub async fn lock_bucket_for_recursive_delete(&self, bucket: &str) -> Result<rustfs_lock::NamespaceLockGuard> {
370        if self.ctx.lock_manager().is_disabled() {
371            return Err(StorageError::InvalidArgument(
372                bucket.to_owned(),
373                String::new(),
374                "Recursive deletion requires namespace locking".to_owned(),
375            ));
376        }
377        self.acquire_bucket_lifecycle_write_lock(bucket).await
378    }
379
380    pub(crate) async fn acquire_bucket_lifecycle_write_lock(&self, bucket: &str) -> Result<rustfs_lock::NamespaceLockGuard> {
381        let lock = self.new_ns_lock(bucket, BUCKET_LIFECYCLE_LOCK_OBJECT).await?;
382        lock.get_write_lock(get_lock_acquire_timeout())
383            .await
384            .map_err(|err| match err {
385                rustfs_lock::error::LockError::QuorumNotReached { required, achieved } => {
386                    StorageError::NamespaceLockQuorumUnavailable {
387                        mode: "bucket_lifecycle_write",
388                        bucket: bucket.to_string(),
389                        object: BUCKET_LIFECYCLE_LOCK_OBJECT.to_string(),
390                        required,
391                        achieved,
392                    }
393                }
394                other => StorageError::Lock(other),
395            })
396    }
397
398    async fn mark_bucket_deleted(&self, bucket: &str) -> Result<()> {
399        let marker_volume = bucket_deleted_marker_volume(bucket);
400
401        self.make_bucket_on_sets(
402            marker_volume.as_str(),
403            &MakeBucketOptions {
404                force_create: true,
405                ..Default::default()
406            },
407        )
408        .await
409        .map_err(|err| to_object_err(err, vec![bucket]))?;
410
411        Ok(())
412    }
413
414    async fn make_bucket_on_sets(&self, bucket: &str, opts: &MakeBucketOptions) -> Result<()> {
415        let results = futures::future::join_all(
416            self.bucket_sets()
417                .map(|(_, _, set)| async move { set.make_bucket(bucket, opts).await }),
418        )
419        .await;
420        if results.is_empty() {
421            return Err(StorageError::ErasureWriteQuorum);
422        }
423
424        let mut bucket_exists_error = None;
425        let mut first_hard_error = None;
426        for result in results {
427            let Err(err) = result else {
428                continue;
429            };
430            if matches!(err, StorageError::VolumeExists) || is_err_bucket_exists(&err) {
431                if bucket_exists_error.is_none() {
432                    bucket_exists_error = Some(err);
433                }
434            } else if first_hard_error.is_none() {
435                first_hard_error = Some(err);
436            }
437        }
438
439        match first_hard_error.or(bucket_exists_error) {
440            Some(err) => Err(err),
441            None => Ok(()),
442        }
443    }
444
445    async fn delete_bucket_on_sets(&self, bucket: &str, opts: &DeleteBucketOptions) -> Result<()> {
446        let results = futures::future::join_all(
447            self.bucket_sets()
448                .map(|(_, _, set)| async move { set.delete_bucket(bucket, opts).await }),
449        )
450        .await;
451        if results.is_empty() {
452            return Err(StorageError::ErasureWriteQuorum);
453        }
454
455        let mut deleted = false;
456        let mut first_error = None;
457        for result in results {
458            match result {
459                Ok(()) => deleted = true,
460                Err(err) if is_err_strict_volume_not_found(&err) => {}
461                Err(err) if first_error.is_none() => first_error = Some(err),
462                Err(_) => {}
463            }
464        }
465
466        if let Some(delete_error) = first_error {
467            if !opts.no_recreate {
468                let rollback_opts = MakeBucketOptions {
469                    force_create: true,
470                    no_lock: true,
471                    ..Default::default()
472                };
473                if let Err(rollback_error) = self.make_bucket_on_sets(bucket, &rollback_opts).await {
474                    warn!(
475                        event = EVENT_BUCKET_DELETE_ROLLBACK_FAILED,
476                        component = "ecstore",
477                        subsystem = "bucket",
478                        bucket,
479                        deletion_error = ?delete_error,
480                        rollback_error = ?rollback_error,
481                        "Bucket deletion rollback could not restore every bucket volume"
482                    );
483                }
484            }
485            return Err(delete_error);
486        }
487
488        if deleted { Ok(()) } else { Err(StorageError::VolumeNotFound) }
489    }
490
491    async fn cleanup_deleted_bucket_metadata(
492        &self,
493        bucket: &str,
494        include_deleted_marker: bool,
495        guard: Option<&rustfs_lock::NamespaceLockGuard>,
496    ) -> Result<()> {
497        for prefix in bucket_delete_metadata_cleanup_prefixes(bucket) {
498            await_bucket_namespace_operation(
499                guard,
500                bucket,
501                "deleted bucket metadata cleanup",
502                self.delete_all(RUSTFS_META_BUCKET, prefix.as_str()),
503            )
504            .await?;
505        }
506
507        if include_deleted_marker {
508            let marker_prefix = bucket_deleted_marker_prefix(bucket);
509            await_bucket_namespace_operation(
510                guard,
511                bucket,
512                "deleted bucket marker cleanup",
513                self.delete_all(RUSTFS_META_BUCKET, marker_prefix.as_str()),
514            )
515            .await?;
516        }
517
518        await_bucket_namespace_operation(
519            guard,
520            bucket,
521            "deleted bucket metadata cache cleanup",
522            metadata_sys::remove_bucket_metadata_in(&self.ctx, bucket),
523        )
524        .await?;
525        runtime_sources::delete_bucket_monitor_entry(bucket);
526        Ok(())
527    }
528
529    async fn cleanup_bucket_usage(&self, bucket: &str, guard: Option<&rustfs_lock::NamespaceLockGuard>) -> Result<()> {
530        run_bucket_usage_cleanup(guard, bucket, async {
531            crate::data_usage::prepare_bucket_usage_for_namespace_change(bucket, guard).await?;
532            crate::data_usage::remove_bucket_usage_from_backend_with_guard_fenced(self, bucket, guard).await
533        })
534        .await
535    }
536
537    async fn cleanup_bucket_usage_best_effort(&self, bucket: &str, guard: Option<&rustfs_lock::NamespaceLockGuard>) {
538        if let Err(err) = self.cleanup_bucket_usage(bucket, guard).await {
539            warn!(
540                bucket = %bucket,
541                error = ?err,
542                "bucket data usage cleanup deferred to scanner reconciliation"
543            );
544        }
545    }
546
547    async fn rollback_failed_bucket_creation(
548        &self,
549        bucket: &str,
550        lifecycle_guard: Option<&rustfs_lock::NamespaceLockGuard>,
551        namespace_guard: Option<&rustfs_lock::NamespaceLockGuard>,
552    ) {
553        let rollback_opts = DeleteBucketOptions {
554            no_lock: true,
555            no_recreate: true,
556            ..Default::default()
557        };
558        if let Err(err) =
559            await_bucket_lifecycle_operation(lifecycle_guard, namespace_guard, bucket, "failed bucket creation rollback", async {
560                self.delete_bucket_on_sets(bucket, &rollback_opts)
561                    .await
562                    .map_err(|rollback_err| to_object_err(rollback_err, vec![bucket]))
563            })
564            .await
565        {
566            warn!(
567                bucket = %bucket,
568                error = ?err,
569                "failed bucket creation rollback did not remove every physical bucket volume"
570            );
571            return;
572        }
573        if let Err(err) = self.cleanup_deleted_bucket_metadata(bucket, false, namespace_guard).await {
574            warn!(
575                bucket = %bucket,
576                error = ?err,
577                "failed bucket creation rollback left internal bucket metadata"
578            );
579        }
580    }
581
582    #[instrument(skip(self))]
583    pub(super) async fn handle_make_bucket(&self, bucket: &str, opts: &MakeBucketOptions) -> Result<()> {
584        if !is_meta_bucketname(bucket)
585            && let Err(err) = check_valid_bucket_name_strict(bucket)
586        {
587            return Err(StorageError::BucketNameInvalid(err.to_string()));
588        }
589
590        // Lock order: bucket lifecycle -> metadata transaction -> exact bucket namespace.
591        let bucket_lifecycle_guard = if !opts.no_lock {
592            Some(self.acquire_bucket_lifecycle_write_lock(bucket).await?)
593        } else {
594            None
595        };
596        let metadata_transaction_guard = if !opts.no_lock && !is_meta_bucketname(bucket) {
597            Some(metadata_sys::acquire_bucket_metadata_transaction_lock_in(&self.ctx, bucket).await?)
598        } else {
599            None
600        };
601        let ns_guard = if !opts.no_lock {
602            let ns_lock = self.new_ns_lock(bucket, bucket).await?;
603            Some(
604                await_bucket_namespace_operation(
605                    bucket_lifecycle_guard.as_ref(),
606                    bucket,
607                    "bucket namespace lock acquisition",
608                    async {
609                        ns_lock.get_write_lock(get_lock_acquire_timeout()).await.map_err(|e| match e {
610                            rustfs_lock::error::LockError::QuorumNotReached { required, achieved } => {
611                                StorageError::NamespaceLockQuorumUnavailable {
612                                    mode: "write",
613                                    bucket: bucket.to_string(),
614                                    object: bucket.to_string(),
615                                    required,
616                                    achieved,
617                                }
618                            }
619                            other => StorageError::Lock(other),
620                        })
621                    },
622                )
623                .await?,
624            )
625        } else {
626            None
627        };
628
629        let existing_bucket_info = match self.get_bucket_info_from_sets(bucket, &BucketOptions::default()).await {
630            Ok(info) => Some(info),
631            Err(err) => {
632                if is_err_bucket_not_found(&err) {
633                    None
634                } else {
635                    return Err(to_object_err(err, vec![bucket]));
636                }
637            }
638        };
639        let confirmed_missing = existing_bucket_info.is_none();
640        let existing_metadata = if opts.force_create && !confirmed_missing && !is_meta_bucketname(bucket) {
641            let (mut metadata, persisted) = metadata_sys::get_config_from_disk_with_presence_in(&self.ctx, bucket).await?;
642            if !persisted {
643                metadata = BucketMetadata::new(bucket);
644                metadata.created = existing_bucket_info
645                    .as_ref()
646                    .and_then(|info| info.created)
647                    .unwrap_or(OffsetDateTime::UNIX_EPOCH);
648            } else if !metadata.bucket_incarnation_sidecar && !metadata.bucket_incarnation_id.is_nil() {
649                return Err(Error::other(format!(
650                    "bucket incarnation sidecar is missing for new-format metadata: {bucket}"
651                )));
652            } else if !metadata.bucket_incarnation_sidecar {
653                metadata.bucket_incarnation_id = Uuid::new_v4();
654            }
655            Some(metadata)
656        } else {
657            None
658        };
659
660        let mut meta = existing_metadata.unwrap_or_else(|| {
661            if confirmed_missing && !is_meta_bucketname(bucket) {
662                BucketMetadata::new_with_default_durability(bucket)
663            } else {
664                BucketMetadata::new(bucket)
665            }
666        });
667        let existing_incarnation_is_authoritative = meta.bucket_incarnation_sidecar;
668        if confirmed_missing || is_meta_bucketname(bucket) {
669            meta.set_created(opts.created_at);
670
671            if opts.versioning_enabled {
672                meta.versioning_config_xml =
673                    crate::bucket::utils::serialize::<VersioningConfiguration>(&ENABLED_VERSIONING_CONFIG)?;
674            }
675        }
676
677        // Object Lock enable is one-way, and it must apply to an existing bucket
678        // too. Site replication replays make-with-versioning carrying the
679        // source's lockEnabled against a destination bucket that already exists;
680        // gating this on `confirmed_missing` returned success while leaving the
681        // replica unlocked, so replicated versions could be deleted without the
682        // retention the source enforces.
683        let lock_newly_enabled = opts.lock_enabled && !meta.lock_enabled;
684        if opts.lock_enabled {
685            meta.lock_enabled = true;
686            meta.object_lock_config_xml =
687                crate::bucket::utils::serialize::<ObjectLockConfiguration>(&ENABLED_OBJECT_LOCK_CONFIG)?;
688            meta.versioning_config_xml = crate::bucket::utils::serialize::<VersioningConfiguration>(&ENABLED_VERSIONING_CONFIG)?;
689        }
690
691        let metadata_persisted_before_physical = confirmed_missing && !is_meta_bucketname(bucket) && opts.lock_enabled;
692        if metadata_persisted_before_physical {
693            metadata_sys::set_new_bucket_metadata_in(&self.ctx, meta.clone()).await?;
694            if bucket_lifecycle_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
695                || metadata_transaction_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
696                || ns_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
697            {
698                return Err(Error::other(format!("bucket metadata creation intent lock was lost: {bucket}")));
699            }
700        }
701
702        if confirmed_missing && !is_meta_bucketname(bucket) {
703            // Fence every scanner cycle that could have observed the namespace
704            // before physical creation. Creation may become visible even when a
705            // later metadata write or namespace-lock check fails.
706            crate::store::list_objects::observe_scanner_namespace_mutations(bucket, 1);
707            self.cleanup_bucket_usage(bucket, ns_guard.as_ref()).await?;
708        }
709
710        if let Err(err) = await_bucket_lifecycle_operation(
711            bucket_lifecycle_guard.as_ref(),
712            ns_guard.as_ref(),
713            bucket,
714            "physical bucket creation",
715            await_bucket_namespace_operation(
716                metadata_transaction_guard.as_ref(),
717                bucket,
718                "bucket creation metadata transaction",
719                async {
720                    self.make_bucket_on_sets(bucket, opts)
721                        .await
722                        .map_err(|err| to_object_err(err, vec![bucket]))
723                },
724            ),
725        )
726        .await
727        {
728            if is_err_bucket_exists(&err)
729                && let Err(heal_err) = self
730                    .handle_heal_bucket(
731                        bucket,
732                        &HealOpts {
733                            recreate: true,
734                            ..Default::default()
735                        },
736                    )
737                    .await
738            {
739                warn!("best-effort bucket heal after BucketExists failed: {heal_err}");
740            }
741            if confirmed_missing && !is_err_bucket_exists(&err) && ns_guard.as_ref().is_none_or(|guard| !guard.is_lock_lost()) {
742                error!("make bucket failed: {err}");
743                self.rollback_failed_bucket_creation(bucket, bucket_lifecycle_guard.as_ref(), ns_guard.as_ref())
744                    .await;
745            }
746            return Err(err);
747        };
748
749        let metadata_result = async {
750            if metadata_persisted_before_physical {
751                return Ok(());
752            }
753            if is_meta_bucketname(bucket) {
754                metadata_sys::set_bucket_metadata_in(&self.ctx, meta).await
755            } else if existing_incarnation_is_authoritative && !lock_newly_enabled {
756                metadata_sys::cache_bucket_metadata_in(&self.ctx, meta).await
757            } else {
758                metadata_sys::set_new_bucket_metadata_in(&self.ctx, meta).await
759            }
760        }
761        .await;
762        let metadata_lock_lost = bucket_lifecycle_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
763            || metadata_transaction_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
764            || ns_guard.as_ref().is_some_and(|guard| guard.is_lock_lost());
765        if let Err(err) = metadata_result {
766            if !metadata_lock_lost {
767                self.rollback_failed_bucket_creation(bucket, bucket_lifecycle_guard.as_ref(), ns_guard.as_ref())
768                    .await;
769            }
770            return Err(err);
771        }
772        if metadata_lock_lost {
773            return Err(Error::other(format!("bucket metadata initialization lock was lost: {bucket}")));
774        }
775
776        if confirmed_missing && !is_meta_bucketname(bucket) {
777            // A scanner may have sampled the first fence before the bucket
778            // became visible. Fence again after metadata initialization so
779            // that snapshot cannot publish as complete.
780            crate::store::list_objects::observe_scanner_namespace_mutations(bucket, 1);
781        }
782
783        Ok(())
784    }
785
786    /// Prove a live bucket generation before repairing missing expansion volumes.
787    /// Unlike request validation, repair only needs one erasure set to confirm
788    /// existence; an incomplete expansion set is precisely what repair fixes.
789    /// Callers must hold the bucket namespace lock through the subsequent heal.
790    pub(crate) async fn bucket_exists_for_heal(&self, bucket: &str) -> Result<bool> {
791        let results = futures::future::join_all(
792            self.bucket_sets()
793                .map(|(_, _, set)| async move { set.get_bucket_info(bucket, &BucketOptions::default()).await }),
794        )
795        .await;
796        let mut first_error = None;
797        for result in results {
798            match result {
799                Ok(_) => return Ok(true),
800                Err(err) if is_err_strict_volume_not_found(&err) => {}
801                Err(err) if first_error.is_none() => first_error = Some(err),
802                Err(_) => {}
803            }
804        }
805        match first_error {
806            Some(err) => Err(err),
807            None => Ok(false),
808        }
809    }
810
811    #[instrument(skip(self))]
812    pub(crate) async fn get_bucket_info_from_sets(&self, bucket: &str, opts: &BucketOptions) -> Result<BucketInfo> {
813        self.get_bucket_info_from_sets_with_quorum(bucket, opts, BucketInfoQuorum::Write)
814            .await
815    }
816
817    pub(crate) async fn get_bucket_info_from_sets_at_read_quorum(
818        &self,
819        bucket: &str,
820        opts: &BucketOptions,
821    ) -> Result<BucketInfo> {
822        self.get_bucket_info_from_sets_with_quorum(bucket, opts, BucketInfoQuorum::Read)
823            .await
824    }
825
826    async fn get_bucket_info_from_sets_with_quorum(
827        &self,
828        bucket: &str,
829        opts: &BucketOptions,
830        quorum: BucketInfoQuorum,
831    ) -> Result<BucketInfo> {
832        // One host may participate in several pools after expansion. Resolve the
833        // namespace against each erasure set so disks from different pools can
834        // never be combined into one bucket quorum.
835        // Bucket validation is request-path IO. Keep the previous peer fanout's
836        // latency shape by probing every set concurrently; scanner listings use
837        // a separate bounded path below because they run continuously.
838        let mut scoped_results = futures::future::join_all(self.bucket_sets().map(|(pool_index, set_index, set)| async move {
839            let result = match quorum {
840                BucketInfoQuorum::Read => set.stat_bucket_with_quorum(bucket, quorum).await,
841                BucketInfoQuorum::Write => set.get_bucket_info(bucket, opts).await,
842            };
843            (pool_index, set_index, result)
844        }))
845        .await;
846        scoped_results.sort_unstable_by_key(|(pool_index, set_index, _)| (*pool_index, *set_index));
847
848        let mut first_info = None;
849        let mut first_error = None;
850        for (_, _, result) in scoped_results {
851            match result {
852                Ok(info) if first_info.is_none() => first_info = Some(info),
853                Ok(_) => {}
854                Err(err) if is_err_strict_volume_not_found(&err) => {}
855                Err(err) if first_error.is_none() => first_error = Some(err),
856                Err(_) => {}
857            }
858        }
859
860        if let Some(err) = first_error {
861            return Err(err);
862        }
863
864        first_info.ok_or(Error::VolumeNotFound)
865    }
866
867    #[instrument(skip(self))]
868    pub(super) async fn handle_get_bucket_info(&self, bucket: &str, opts: &BucketOptions) -> Result<BucketInfo> {
869        let mut info = match self.get_bucket_info_from_sets(bucket, opts).await {
870            Ok(info) => info,
871            Err(Error::ErasureWriteQuorum) => return self.get_bucket_info_at_read_quorum(bucket, opts).await,
872            Err(err) => return Err(err),
873        };
874
875        if let Ok(sys) = metadata_sys::get_in(&self.ctx, bucket).await {
876            if should_override_created_from_metadata(sys.created) {
877                info.created = Some(sys.created);
878            }
879            info.versioning = sys.versioning();
880            info.object_locking = sys.object_locking();
881        }
882
883        Ok(info)
884    }
885
886    async fn get_bucket_info_at_read_quorum(&self, bucket: &str, opts: &BucketOptions) -> Result<BucketInfo> {
887        // Lock order: bucket lifecycle -> internal metadata object read locks.
888        // Keep create/delete from changing the namespace while a read quorum
889        // confirms both physical presence and persisted bucket metadata.
890        let guard = self.acquire_bucket_lifecycle_read_lock(bucket).await?;
891        await_bucket_namespace_operation(Some(&guard), bucket, "bucket read quorum validation", async {
892            let mut info = self
893                .get_bucket_info_from_sets_with_quorum(bucket, opts, BucketInfoQuorum::Read)
894                .await?;
895            let (metadata, persisted) = metadata_sys::get_config_from_disk_with_presence_in(&self.ctx, bucket).await?;
896            if !persisted {
897                // A minority of directories left by failed creation is not an
898                // authoritative bucket. Never turn fabricated defaults into
899                // permission to serve degraded reads.
900                return Err(Error::ErasureReadQuorum);
901            }
902            if metadata.name != bucket {
903                return Err(Error::FileCorrupt);
904            }
905            if should_override_created_from_metadata(metadata.created) {
906                info.created = Some(metadata.created);
907            }
908            info.versioning = metadata.versioning();
909            info.object_locking = metadata.object_locking();
910            Ok(info)
911        })
912        .await
913    }
914
915    #[instrument(skip(self))]
916    pub(super) async fn handle_list_bucket(&self, opts: &BucketOptions) -> Result<Vec<BucketInfo>> {
917        // TODO(backlog): support cached bucket listing via opts.cached
918        Ok(self.list_bucket_from_sets(opts, usize::MAX).await?.buckets)
919    }
920
921    pub async fn list_bucket_for_scanner(&self, opts: &BucketOptions) -> Result<crate::cluster::rpc::ScannerBucketListing> {
922        self.list_bucket_from_sets(opts, SCANNER_BUCKET_LIST_SET_CONCURRENCY).await
923    }
924
925    async fn list_bucket_from_sets(
926        &self,
927        opts: &BucketOptions,
928        max_concurrency: usize,
929    ) -> Result<crate::cluster::rpc::ScannerBucketListing> {
930        let set_count = self.pools.iter().map(|pool| pool.disk_set.len()).sum();
931        let concurrency = bucket_list_set_concurrency(set_count, max_concurrency);
932        let deleted = opts.deleted;
933        let cached = opts.cached;
934        let no_metadata = opts.no_metadata;
935        let mut set_listings = stream::iter(self.bucket_sets().map(move |(pool_index, set_index, set)| {
936            let opts = BucketOptions {
937                deleted,
938                cached,
939                no_metadata,
940            };
941            async move {
942                set.list_bucket_for_scanner(&opts)
943                    .await
944                    .map(|(buckets, complete)| (pool_index, set_index, buckets, complete))
945            }
946        }))
947        .buffer_unordered(concurrency);
948        let mut topology_complete = set_count != 0;
949        let mut bucket_map = BTreeMap::<String, BucketInfo>::new();
950        let mut scoped_buckets = Vec::with_capacity(set_count);
951        while let Some(set_listing) = set_listings.next().await {
952            let (pool_index, set_index, buckets, set_complete) = set_listing?;
953            topology_complete &= set_complete;
954            for bucket in &buckets {
955                bucket_map.entry(bucket.name.clone()).or_insert_with(|| bucket.clone());
956            }
957            scoped_buckets.push(crate::cluster::rpc::ScannerSetBucketListing {
958                pool_index,
959                set_index,
960                buckets,
961            });
962        }
963        scoped_buckets.sort_unstable_by_key(|scope| (scope.pool_index, scope.set_index));
964        let mut listing = crate::cluster::rpc::ScannerBucketListing {
965            buckets: bucket_map.into_values().collect(),
966            set_buckets: scoped_buckets,
967            topology_complete,
968        };
969
970        if !opts.no_metadata {
971            for bucket in &mut listing.buckets {
972                if let Ok(created) = metadata_sys::created_at_in(&self.ctx, &bucket.name).await
973                    && should_override_created_from_metadata(created)
974                {
975                    bucket.created = Some(created);
976                }
977            }
978            let created_by_bucket = listing
979                .buckets
980                .iter()
981                .map(|bucket| (bucket.name.as_str(), bucket.created))
982                .collect::<BTreeMap<_, _>>();
983            for scope in &mut listing.set_buckets {
984                for bucket in &mut scope.buckets {
985                    if let Some(created) = created_by_bucket.get(bucket.name.as_str()) {
986                        bucket.created = *created;
987                    }
988                }
989            }
990        }
991
992        Ok(listing)
993    }
994
995    #[instrument(skip(self))]
996    pub(super) async fn handle_delete_bucket(&self, bucket: &str, opts: &DeleteBucketOptions) -> Result<()> {
997        self.handle_delete_bucket_with_diagnostic_budget(bucket, opts, BucketDeleteDiagnosticBudget::new())
998            .await
999    }
1000
1001    async fn handle_delete_bucket_with_diagnostic_budget(
1002        &self,
1003        bucket: &str,
1004        opts: &DeleteBucketOptions,
1005        mut diagnostic_budget: BucketDeleteDiagnosticBudget,
1006    ) -> Result<()> {
1007        if is_meta_bucketname(bucket) {
1008            return Err(StorageError::BucketNameInvalid(bucket.to_string()));
1009        }
1010
1011        if let Err(err) = check_valid_bucket_name(bucket) {
1012            return Err(StorageError::BucketNameInvalid(err.to_string()));
1013        }
1014
1015        let bucket_lifecycle_guard = if !opts.no_lock {
1016            Some(self.acquire_bucket_lifecycle_write_lock(bucket).await?)
1017        } else {
1018            None
1019        };
1020        let ns_guard = if !opts.no_lock {
1021            let ns_lock = self.new_ns_lock(bucket, bucket).await?;
1022            Some(
1023                await_bucket_namespace_operation(
1024                    bucket_lifecycle_guard.as_ref(),
1025                    bucket,
1026                    "bucket namespace lock acquisition",
1027                    async {
1028                        ns_lock.get_write_lock(get_lock_acquire_timeout()).await.map_err(|e| match e {
1029                            rustfs_lock::error::LockError::QuorumNotReached { required, achieved } => {
1030                                StorageError::NamespaceLockQuorumUnavailable {
1031                                    mode: "write",
1032                                    bucket: bucket.to_string(),
1033                                    object: bucket.to_string(),
1034                                    required,
1035                                    achieved,
1036                                }
1037                            }
1038                            other => StorageError::Lock(other),
1039                        })
1040                    },
1041                )
1042                .await?,
1043            )
1044        } else {
1045            None
1046        };
1047
1048        let sr_mark_delete = opts.srdelete_op == SRBucketDeleteOp::MarkDelete;
1049        let sr_purge = opts.srdelete_op == SRBucketDeleteOp::Purge;
1050        let sr_delete = sr_mark_delete || sr_purge;
1051        let mut delete_opts = opts.clone();
1052        let bucket_exists = match self.get_bucket_info_from_sets(bucket, &BucketOptions::default()).await {
1053            Ok(_) => true,
1054            Err(err) => {
1055                if is_err_strict_volume_not_found(&err) && sr_delete {
1056                    false
1057                } else if is_err_strict_volume_not_found(&err) {
1058                    return Err(StorageError::BucketNotFound(bucket.to_string()));
1059                } else {
1060                    return Err(to_object_err(err, vec![bucket]));
1061                }
1062            }
1063        };
1064        if bucket_exists {
1065            validate_table_bucket_delete_guard(&self.ctx, bucket).await?;
1066
1067            if !opts.force {
1068                if let Some(blocker) = bucket_delete_local_blocker(&self.ctx, bucket, &mut diagnostic_budget).await? {
1069                    return Err(blocker);
1070                }
1071                delete_opts.force_if_empty = true;
1072            }
1073        }
1074
1075        if sr_delete && !bucket_exists {
1076            delete_opts.force_if_empty = true;
1077        }
1078
1079        #[cfg(test)]
1080        crate::cluster::rpc::peer_s3_client::pause_after_delete_bucket_empty_scan().await;
1081
1082        if sr_mark_delete {
1083            await_bucket_lifecycle_operation(
1084                bucket_lifecycle_guard.as_ref(),
1085                ns_guard.as_ref(),
1086                bucket,
1087                "bucket delete marker creation",
1088                self.mark_bucket_deleted(bucket),
1089            )
1090            .await?;
1091        }
1092
1093        // Capture the authoritative old identity before its namespace disappears.
1094        // Legacy buckets without a stamp cannot produce retirement authority.
1095        let retirement = if bucket_exists && bucket_lifecycle_guard.is_some() && !is_meta_bucketname(bucket) {
1096            if let Some(store) = metadata_sys::object_store_if_initialized_in(&self.ctx).await {
1097                crate::bucket::metadata::load_bucket_incarnation(store.clone(), bucket)
1098                    .await?
1099                    .map(|incarnation| (store, incarnation))
1100            } else {
1101                None
1102            }
1103        } else {
1104            None
1105        };
1106
1107        let delete_result = await_bucket_namespace_operation(
1108            bucket_lifecycle_guard.as_ref(),
1109            bucket,
1110            "physical bucket deletion",
1111            run_physical_bucket_deletion(ns_guard.as_ref(), bucket, async {
1112                self.delete_bucket_on_sets(bucket, &delete_opts)
1113                    .await
1114                    .map_err(|err| to_object_err(err, vec![bucket]))
1115            }),
1116        )
1117        .await;
1118        if let Err(err) = delete_result
1119            && (!sr_delete || !is_err_strict_volume_not_found(&err))
1120        {
1121            if delete_opts.force_if_empty && matches!(&err, StorageError::BucketNotEmpty(_)) {
1122                let mut diagnostic_budget = BucketDeleteDiagnosticBudget::new();
1123                if let Some(blocker) = bucket_delete_local_blocker(&self.ctx, bucket, &mut diagnostic_budget).await? {
1124                    return Err(blocker);
1125                }
1126            }
1127            return Err(err);
1128        }
1129
1130        if let Some((store, incarnation)) = retirement {
1131            let mut record_opts = ObjectOptions {
1132                max_parity: true,
1133                ..Default::default()
1134            };
1135            if let Some(guard) = bucket_lifecycle_guard.as_ref() {
1136                record_opts.add_bucket_lifecycle_lock_guard(guard);
1137            }
1138            if let Some(guard) = ns_guard.as_ref() {
1139                record_opts.add_namespace_lock_guard(guard);
1140            }
1141            await_bucket_lifecycle_operation(
1142                bucket_lifecycle_guard.as_ref(),
1143                ns_guard.as_ref(),
1144                bucket,
1145                "bucket retirement publication",
1146                crate::bucket::retirement::commit_retirement(store, bucket, incarnation, &record_opts),
1147            )
1148            .await?;
1149        }
1150
1151        self.cleanup_bucket_usage_best_effort(bucket, ns_guard.as_ref()).await;
1152
1153        if let Err(err) = self
1154            .cleanup_deleted_bucket_metadata(bucket, sr_purge, ns_guard.as_ref())
1155            .await
1156        {
1157            warn!(
1158                bucket = %bucket,
1159                error = ?err,
1160                "physical bucket deletion succeeded but metadata cleanup remains pending"
1161            );
1162        }
1163        // A scanner may have sampled the first fence before the physical
1164        // namespace disappeared. The completion fence invalidates that scan.
1165        crate::store::list_objects::observe_scanner_namespace_mutations(bucket, 1);
1166        Ok(())
1167    }
1168}
1169
1170#[cfg(test)]
1171mod tests {
1172    use super::{
1173        BUCKET_DELETE_DIAGNOSTIC_MAX_ELAPSED, BUCKET_DELETE_DIAGNOSTIC_MAX_ENTRIES, BUCKET_DELETE_XLMETA_DIAGNOSTIC_MAX_BYTES,
1174        BucketDeleteBlockerKind, BucketDeleteDiagnosticBudget, BucketMetadataLessResidue, SCANNER_BUCKET_LIST_SET_CONCURRENCY,
1175        await_bucket_namespace_operation, bucket_delete_metadata_cleanup_prefixes, bucket_deleted_marker_prefix,
1176        bucket_deleted_marker_volume, bucket_list_set_concurrency, record_bucket_delete_blocker, run_bucket_usage_cleanup,
1177        run_physical_bucket_deletion, scan_metadata_less_residue, scan_metadata_less_residue_with_budget,
1178        should_override_created_from_metadata, validate_table_bucket_delete_allowed,
1179    };
1180    use crate::bucket::metadata::{BucketMetadata, table_bucket_catalog_metadata_prefix};
1181    use crate::bucket::metadata_sys;
1182    use crate::cluster::rpc::peer_s3_client::install_delete_bucket_empty_scan_barrier;
1183    use crate::disk::{BUCKET_META_PREFIX, DiskAPI, RUSTFS_META_BUCKET, STORAGE_FORMAT_FILE};
1184    use crate::error::StorageError;
1185    use crate::object_api::{ObjectOptions, PutObjReader};
1186    use crate::runtime::instance::InstanceContext;
1187    use crate::storage_api_contracts::{
1188        bucket::{BucketOperations as _, BucketOptions, DeleteBucketOptions, MakeBucketOptions, SRBucketDeleteOp},
1189        list::ListOperations as _,
1190        namespace::NamespaceLocking as _,
1191        object::{ObjectIO as _, ObjectOperations as _},
1192    };
1193    use crate::store::{ECStore, init_local_disks_with_instance_ctx};
1194    use crate::{
1195        disk::endpoint::Endpoint,
1196        layout::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints},
1197    };
1198    use rustfs_data_usage::{BucketUsageInfo, DATA_USAGE_OBJECT_NAME, DataUsageInfo};
1199    use rustfs_filemeta::{FileInfo, FileMeta, TRANSITION_COMPLETE};
1200    use rustfs_lock::{LocalClient, LockRequest, LockType, NamespaceLock, ObjectKey};
1201    use serial_test::serial;
1202    use std::path::{Path, PathBuf};
1203    use std::sync::Arc;
1204    use std::sync::atomic::{AtomicBool, Ordering};
1205    use std::time::{Duration, SystemTime};
1206    use time::OffsetDateTime;
1207    use tokio::io::AsyncReadExt;
1208    use tokio::sync::{Notify, OnceCell};
1209    use tokio_util::sync::CancellationToken;
1210    use uuid::Uuid;
1211
1212    static BUCKET_DELETE_TEST_ENV: OnceCell<(Vec<PathBuf>, Arc<ECStore>)> = OnceCell::const_new();
1213
1214    #[tokio::test(start_paused = true)]
1215    async fn bucket_delete_diagnostic_budget_starts_with_first_scan_io_and_latches_once() {
1216        let mut budget = BucketDeleteDiagnosticBudget::with_limits(8, Duration::from_millis(100));
1217        tokio::time::advance(Duration::from_secs(10)).await;
1218
1219        let first_polled = Arc::new(AtomicBool::new(false));
1220        let first_polled_for_io = first_polled.clone();
1221        let first = budget
1222            .run_io(async move {
1223                first_polled_for_io.store(true, Ordering::SeqCst);
1224                Ok::<_, std::io::Error>(7_u8)
1225            })
1226            .await
1227            .expect("the first diagnostic IO should succeed");
1228        assert_eq!(first, Some(7));
1229        assert!(first_polled.load(Ordering::SeqCst));
1230
1231        tokio::time::advance(Duration::from_millis(101)).await;
1232        let expired_polled = Arc::new(AtomicBool::new(false));
1233        let expired_polled_for_io = expired_polled.clone();
1234        let expired = budget
1235            .run_io(async move {
1236                expired_polled_for_io.store(true, Ordering::SeqCst);
1237                Ok::<_, std::io::Error>(9_u8)
1238            })
1239            .await
1240            .expect("an expired diagnostic budget should not become an IO error");
1241        assert_eq!(expired, None);
1242        assert!(
1243            !expired_polled.load(Ordering::SeqCst),
1244            "the deadline must remain latched after the first scan IO"
1245        );
1246    }
1247
1248    #[tokio::test(start_paused = true)]
1249    async fn bucket_delete_diagnostic_budget_times_out_its_first_pending_io() {
1250        let mut budget = BucketDeleteDiagnosticBudget::with_limits(8, Duration::from_millis(100));
1251        let io_polled = Arc::new(AtomicBool::new(false));
1252        let io_polled_for_future = io_polled.clone();
1253
1254        let result = budget
1255            .run_io(std::future::poll_fn(move |_cx| {
1256                io_polled_for_future.store(true, Ordering::SeqCst);
1257                std::task::Poll::<std::io::Result<()>>::Pending
1258            }))
1259            .await
1260            .expect("a diagnostic timeout should fail closed without an IO error");
1261
1262        assert_eq!(result, None);
1263        assert!(
1264            io_polled.load(Ordering::SeqCst),
1265            "the first diagnostic IO must be polled before its timeout"
1266        );
1267    }
1268
1269    #[tokio::test(start_paused = true)]
1270    async fn delayed_metadata_less_scans_still_detect_orphans_and_xlmeta() {
1271        let root = tempfile::tempdir().expect("temporary delayed-scan roots should be created");
1272        let orphan_root = root.path().join("orphan-root");
1273        let xlmeta_root = root.path().join("xlmeta-root");
1274        std::fs::create_dir_all(&orphan_root).expect("orphan root should be created");
1275        std::fs::create_dir_all(&xlmeta_root).expect("xlmeta root should be created");
1276        std::fs::write(orphan_root.join("orphan-part"), b"orphan").expect("orphan fixture should be written");
1277        std::fs::write(xlmeta_root.join(STORAGE_FORMAT_FILE), b"invalid-xlmeta").expect("xl.meta fixture should be written");
1278
1279        let mut orphan_budget = BucketDeleteDiagnosticBudget::with_limits(16, Duration::from_secs(5));
1280        let mut xlmeta_budget = BucketDeleteDiagnosticBudget::with_limits(16, Duration::from_secs(5));
1281        tokio::time::advance(Duration::from_secs(60)).await;
1282
1283        let orphan = scan_metadata_less_residue_with_budget(&orphan_root, &mut orphan_budget)
1284            .await
1285            .expect("delayed orphan scan should complete");
1286        assert!(orphan.has_residue_without_xlmeta());
1287        assert!(!orphan.diagnostic_truncated);
1288
1289        let xlmeta = scan_metadata_less_residue_with_budget(&xlmeta_root, &mut xlmeta_budget)
1290            .await
1291            .expect("delayed xl.meta scan should complete");
1292        assert!(xlmeta.xlmeta_found);
1293        assert!(!xlmeta.diagnostic_truncated);
1294    }
1295
1296    #[tokio::test(start_paused = true)]
1297    async fn bucket_namespace_operation_fails_closed_after_lease_expiry() {
1298        let ttl = Duration::from_millis(20);
1299        let lock = NamespaceLock::new("bucket-operation-test".to_string(), Arc::new(LocalClient::new()));
1300        let request = LockRequest::new(ObjectKey::new("bucket", ""), LockType::Exclusive, "test-owner")
1301            .with_acquire_timeout(Duration::from_secs(1))
1302            .with_ttl(ttl)
1303            .with_refresh_interval(ttl);
1304        let guard = lock
1305            .acquire_guard(&request)
1306            .await
1307            .expect("namespace lock acquisition should not fail")
1308            .expect("namespace lock should be acquired");
1309        tokio::time::advance(ttl + Duration::from_millis(1)).await;
1310
1311        let operation_ran = Arc::new(AtomicBool::new(false));
1312        let operation_ran_for_future = operation_ran.clone();
1313        let result = await_bucket_namespace_operation(Some(&guard), "bucket", "test operation", async move {
1314            operation_ran_for_future.store(true, Ordering::SeqCst);
1315            Ok(())
1316        })
1317        .await;
1318
1319        assert!(result.is_err(), "an expired namespace lease must fence the operation");
1320        assert!(
1321            !operation_ran.load(Ordering::SeqCst),
1322            "a fenced namespace operation must not poll its mutation future"
1323        );
1324    }
1325
1326    #[tokio::test]
1327    #[serial]
1328    async fn physical_bucket_delete_fences_scanner_before_polling_storage() {
1329        let generation_before = crate::store::list_objects::scanner_namespace_mutation_generation();
1330        let storage_polled = Arc::new(AtomicBool::new(false));
1331        let storage_polled_for_future = storage_polled.clone();
1332
1333        run_physical_bucket_deletion(None, "generation-order-bucket", async move {
1334            assert!(
1335                crate::store::list_objects::scanner_namespace_mutation_generation() > generation_before,
1336                "scanner generation must advance before physical deletion is polled"
1337            );
1338            storage_polled_for_future.store(true, Ordering::SeqCst);
1339            Ok(())
1340        })
1341        .await
1342        .expect("synthetic physical deletion should succeed");
1343
1344        assert!(storage_polled.load(Ordering::SeqCst));
1345    }
1346
1347    #[tokio::test]
1348    async fn bucket_usage_cleanup_stops_after_parent_cancellation() {
1349        let started = Arc::new(Notify::new());
1350        let started_wait = started.notified();
1351        let release = Arc::new(Notify::new());
1352        let completed = Arc::new(AtomicBool::new(false));
1353        let started_for_cleanup = started.clone();
1354        let release_for_cleanup = release.clone();
1355        let completed_for_cleanup = completed.clone();
1356        let parent = tokio::spawn(run_bucket_usage_cleanup(None, "bucket", async move {
1357            started_for_cleanup.notify_one();
1358            release_for_cleanup.notified().await;
1359            completed_for_cleanup.store(true, Ordering::SeqCst);
1360            Ok(())
1361        }));
1362        started_wait.await;
1363
1364        parent.abort();
1365        let _ = parent.await;
1366        tokio::task::yield_now().await;
1367        release.notify_waiters();
1368        tokio::task::yield_now().await;
1369        assert!(
1370            !completed.load(Ordering::SeqCst),
1371            "a cancelled cleanup future must not continue in a detached task"
1372        );
1373    }
1374
1375    #[tokio::test(start_paused = true)]
1376    #[serial]
1377    async fn bucket_namespace_operation_stops_in_flight_work_after_lock_loss() {
1378        let ttl = Duration::from_millis(20);
1379        let lock = NamespaceLock::new("bucket-operation-in-flight-loss-test".to_string(), Arc::new(LocalClient::new()));
1380        let request = LockRequest::new(ObjectKey::new("bucket", ""), LockType::Exclusive, "test-owner")
1381            .with_acquire_timeout(Duration::from_secs(1))
1382            .with_ttl(ttl)
1383            .with_refresh_interval(ttl);
1384        let guard = lock
1385            .acquire_guard(&request)
1386            .await
1387            .expect("namespace lock acquisition should not fail")
1388            .expect("namespace lock should be acquired");
1389
1390        let operation_started = Arc::new(Notify::new());
1391        let started_wait = operation_started.notified();
1392        let operation_release = Arc::new(Notify::new());
1393        let operation_completed = Arc::new(AtomicBool::new(false));
1394        let started_for_operation = operation_started.clone();
1395        let release_for_operation = operation_release.clone();
1396        let completed_for_operation = operation_completed.clone();
1397        let task = tokio::spawn(async move {
1398            await_bucket_namespace_operation(Some(&guard), "bucket", "test operation", async move {
1399                started_for_operation.notify_one();
1400                release_for_operation.notified().await;
1401                completed_for_operation.store(true, Ordering::SeqCst);
1402                Ok(())
1403            })
1404            .await
1405        });
1406        started_wait.await;
1407
1408        tokio::time::advance(ttl + Duration::from_millis(1)).await;
1409        let err = task
1410            .await
1411            .expect("operation task should join")
1412            .expect_err("an operation still running after lease loss must be fenced");
1413        assert!(err.to_string().contains("namespace lock was lost during test operation"));
1414        operation_release.notify_waiters();
1415        tokio::task::yield_now().await;
1416        assert!(
1417            !operation_completed.load(Ordering::SeqCst),
1418            "lock loss must stop the old owner before a successor can acquire the namespace"
1419        );
1420    }
1421
1422    async fn setup_bucket_delete_test_env() -> (Vec<PathBuf>, Arc<ECStore>) {
1423        BUCKET_DELETE_TEST_ENV
1424            .get_or_init(|| async {
1425                let temp_dir = std::env::temp_dir().join(format!("rustfs_bucket_delete_test_{}", Uuid::new_v4()));
1426                tokio::fs::create_dir_all(&temp_dir)
1427                    .await
1428                    .expect("test base directory should be created");
1429
1430                let disk_paths = (0..4)
1431                    .map(|disk_idx| temp_dir.join(format!("disk{disk_idx}")))
1432                    .collect::<Vec<_>>();
1433
1434                for disk_path in &disk_paths {
1435                    tokio::fs::create_dir_all(disk_path)
1436                        .await
1437                        .expect("disk directory should be created");
1438                }
1439
1440                let mut endpoints = Vec::with_capacity(disk_paths.len());
1441                for (disk_idx, disk_path) in disk_paths.iter().enumerate() {
1442                    let mut endpoint =
1443                        Endpoint::try_from(disk_path.to_str().expect("disk path should be utf8")).expect("endpoint should parse");
1444                    endpoint.set_pool_index(0);
1445                    endpoint.set_set_index(0);
1446                    endpoint.set_disk_index(disk_idx);
1447                    endpoints.push(endpoint);
1448                }
1449
1450                let endpoint_pools = EndpointServerPools(vec![PoolEndpoints {
1451                    legacy: false,
1452                    set_count: 1,
1453                    drives_per_set: 4,
1454                    endpoints: Endpoints::from(endpoints),
1455                    cmd_line: "bucket-delete-test".to_string(),
1456                    platform: format!("OS: {} | Arch: {}", std::env::consts::OS, std::env::consts::ARCH),
1457                }]);
1458
1459                let instance_ctx = Arc::new(InstanceContext::new());
1460                init_local_disks_with_instance_ctx(&instance_ctx, endpoint_pools.clone())
1461                    .await
1462                    .expect("local disks should initialize");
1463                let ecstore = ECStore::new_with_instance_ctx(
1464                    "127.0.0.1:0".parse().expect("test address"),
1465                    endpoint_pools,
1466                    CancellationToken::new(),
1467                    instance_ctx,
1468                )
1469                .await
1470                .expect("ECStore should initialize");
1471                let storage_class = crate::config::storageclass::lookup_config_for_pools_without_env(
1472                    &rustfs_config::server_config::KVS::new(),
1473                    &[4],
1474                )
1475                .expect("bucket test storage class should match its four-disk pool");
1476                for pool in &ecstore.pools {
1477                    for set in &pool.disk_set {
1478                        set.set_test_storage_class_config(storage_class.clone());
1479                    }
1480                }
1481
1482                metadata_sys::init_bucket_metadata_sys(ecstore.clone(), Vec::new()).await;
1483
1484                (disk_paths, ecstore)
1485            })
1486            .await
1487            .clone()
1488    }
1489
1490    async fn setup_multi_pool_bucket_test_env() -> (tempfile::TempDir, Arc<ECStore>) {
1491        setup_bucket_quorum_test_env(&[4, 4], None).await
1492    }
1493
1494    async fn setup_bucket_quorum_test_env(
1495        drives_per_pool: &[usize],
1496        standard_parity: Option<usize>,
1497    ) -> (tempfile::TempDir, Arc<ECStore>) {
1498        let temp_dir = tempfile::tempdir().expect("multi-pool bucket test directory should be created");
1499        let mut pools = Vec::new();
1500        for (pool_index, &drive_count) in drives_per_pool.iter().enumerate() {
1501            let mut endpoints = Vec::new();
1502            for disk_index in 0..drive_count {
1503                let disk_path = temp_dir.path().join(format!("pool{pool_index}-disk{disk_index}"));
1504                tokio::fs::create_dir_all(&disk_path)
1505                    .await
1506                    .expect("multi-pool bucket test disk should be created");
1507                let mut endpoint =
1508                    Endpoint::try_from(disk_path.to_str().expect("disk path should be utf8")).expect("endpoint should parse");
1509                endpoint.set_pool_index(pool_index);
1510                endpoint.set_set_index(0);
1511                endpoint.set_disk_index(disk_index);
1512                endpoints.push(endpoint);
1513            }
1514            pools.push(PoolEndpoints {
1515                legacy: false,
1516                set_count: 1,
1517                drives_per_set: drive_count,
1518                endpoints: Endpoints::from(endpoints),
1519                cmd_line: format!("bucket-test-pool-{pool_index}"),
1520                platform: format!("OS: {} | Arch: {}", std::env::consts::OS, std::env::consts::ARCH),
1521            });
1522        }
1523
1524        let endpoint_pools = EndpointServerPools(pools);
1525        let instance_ctx = Arc::new(InstanceContext::new());
1526        instance_ctx.set_endpoints(endpoint_pools.clone());
1527        init_local_disks_with_instance_ctx(&instance_ctx, endpoint_pools.clone())
1528            .await
1529            .expect("multi-pool local disks should initialize");
1530        let ecstore = ECStore::new_with_instance_ctx(
1531            "127.0.0.1:0".parse().expect("test address"),
1532            endpoint_pools,
1533            CancellationToken::new(),
1534            instance_ctx,
1535        )
1536        .await
1537        .expect("multi-pool ECStore should initialize");
1538        let mut storage_class_kvs = rustfs_config::server_config::KVS::new();
1539        if let Some(parity) = standard_parity {
1540            storage_class_kvs.insert(crate::config::storageclass::CLASS_STANDARD.to_string(), format!("EC:{parity}"));
1541        }
1542        let storage_class = crate::config::storageclass::lookup_config_for_pools_without_env(&storage_class_kvs, drives_per_pool)
1543            .expect("storage class should match every test erasure set");
1544        for pool in &ecstore.pools {
1545            for set in &pool.disk_set {
1546                set.set_test_storage_class_config(storage_class.clone());
1547            }
1548        }
1549
1550        (temp_dir, ecstore)
1551    }
1552
1553    async fn take_set_disks_offline(
1554        ecstore: &ECStore,
1555        set: &Arc<crate::set_disk::SetDisks>,
1556        disk_indexes: &[usize],
1557    ) -> Vec<(usize, crate::disk::DiskStore)> {
1558        let offline = {
1559            let mut disks = set.disks.write().await;
1560            disk_indexes
1561                .iter()
1562                .map(|index| (*index, disks[*index].take().expect("fault-injection disk should start online")))
1563                .collect::<Vec<_>>()
1564        };
1565        let local_disk_map = ecstore.ctx.local_disk_map();
1566        let mut local_disks = local_disk_map.write().await;
1567        for (_, disk) in &offline {
1568            local_disks.insert(disk.endpoint().to_string(), None);
1569        }
1570        offline
1571    }
1572
1573    async fn restore_set_disks(
1574        ecstore: &ECStore,
1575        set: &Arc<crate::set_disk::SetDisks>,
1576        offline: Vec<(usize, crate::disk::DiskStore)>,
1577    ) {
1578        {
1579            let local_disk_map = ecstore.ctx.local_disk_map();
1580            let mut local_disks = local_disk_map.write().await;
1581            for (_, disk) in &offline {
1582                local_disks.insert(disk.endpoint().to_string(), Some(Arc::clone(disk)));
1583            }
1584        }
1585        let mut disks = set.disks.write().await;
1586        for (index, disk) in offline {
1587            assert!(disks[index].replace(disk).is_none(), "fault-injection disk slot should remain empty");
1588        }
1589    }
1590
1591    #[tokio::test]
1592    async fn request_metadata_methods_fail_closed_before_instance_initialization() {
1593        let (_temp_dir, store) = setup_multi_pool_bucket_test_env().await;
1594
1595        let expected = "bucket metadata sys not initialized for this instance";
1596        let errors = [
1597            store.get_bucket_metadata("bucket").await.unwrap_err(),
1598            store.get_bucket_policy("bucket").await.unwrap_err(),
1599            store.get_bucket_policy_raw("bucket").await.unwrap_err(),
1600            store.restricts_public_bucket_access("bucket").await.unwrap_err(),
1601            store
1602                .update_bucket_metadata_config("bucket", crate::bucket::metadata::BUCKET_POLICY_CONFIG, Vec::new())
1603                .await
1604                .unwrap_err(),
1605        ];
1606        for error in errors {
1607            assert_eq!(error.to_string(), format!("Io error: {expected}"));
1608        }
1609    }
1610
1611    async fn create_bucket_with_object(ecstore: &Arc<ECStore>, bucket: &str, object: &str) {
1612        let generation_before_make = ecstore.scanner_namespace_mutation_generation();
1613        ecstore
1614            .make_bucket(bucket, &MakeBucketOptions::default())
1615            .await
1616            .expect("bucket should be created");
1617        assert_eq!(
1618            ecstore.scanner_namespace_mutation_generation(),
1619            generation_before_make.saturating_add(2),
1620            "successful bucket creation should fence scanner namespace activity before and after creation"
1621        );
1622
1623        let generation_before_put = ecstore.scanner_namespace_mutation_generation();
1624        let mut reader = PutObjReader::from_vec(b"delete bucket semantics".to_vec());
1625        ecstore
1626            .put_object(bucket, object, &mut reader, &ObjectOptions::default())
1627            .await
1628            .expect("object should be written");
1629        let lock = ecstore.pools[0].disk_set[0]
1630            .new_ns_lock(bucket, object)
1631            .await
1632            .expect("fixture namespace lock should be created");
1633        drop(
1634            lock.get_write_lock(Duration::from_secs(30))
1635                .await
1636                .expect("fixture rename tail should finish before checking its generation"),
1637        );
1638        assert_eq!(
1639            ecstore.scanner_namespace_mutation_generation(),
1640            generation_before_put.saturating_add(3),
1641            "successful object creation must observe the logical mutation and both fanout boundaries"
1642        );
1643        ecstore
1644            .get_object_info(bucket, object, &ObjectOptions::default())
1645            .await
1646            .expect("object should be readable before bucket delete");
1647    }
1648
1649    async fn any_disk_path_exists(disk_paths: &[PathBuf], relative_path: impl AsRef<Path>) -> bool {
1650        for disk_path in disk_paths {
1651            if tokio::fs::try_exists(disk_path.join(relative_path.as_ref()))
1652                .await
1653                .expect("test disk path should be stat-able")
1654            {
1655                return true;
1656            }
1657        }
1658        false
1659    }
1660
1661    async fn any_disk_has_object_metadata(disk_paths: &[PathBuf], bucket: &str) -> bool {
1662        for disk_path in disk_paths {
1663            if super::has_xlmeta_files(&disk_path.join(bucket))
1664                .await
1665                .expect("object metadata scan should succeed")
1666            {
1667                return true;
1668            }
1669        }
1670        false
1671    }
1672
1673    async fn write_metadata_less_part_on_all_disks(disk_paths: &[PathBuf], bucket: &str, object: &str) {
1674        for disk_path in disk_paths {
1675            let data_dir = Uuid::new_v4();
1676            let part_path = disk_path.join(bucket).join(object).join(data_dir.to_string()).join("part.1");
1677            tokio::fs::create_dir_all(part_path.parent().expect("part path should have a parent"))
1678                .await
1679                .expect("metadata-less data dir should be created");
1680            tokio::fs::write(part_path, b"orphan shard")
1681                .await
1682                .expect("metadata-less part should be written");
1683        }
1684    }
1685
1686    async fn write_bucket_metadata_marker(disk_paths: &[PathBuf], metadata_prefix: &str) {
1687        for disk_path in disk_paths {
1688            let marker_path = disk_path.join(metadata_prefix).join("config.json");
1689            tokio::fs::create_dir_all(marker_path.parent().expect("metadata marker path should have a parent"))
1690                .await
1691                .expect("metadata marker parent should be created");
1692            tokio::fs::write(marker_path, b"bucket metadata")
1693                .await
1694                .expect("metadata marker should be written");
1695        }
1696    }
1697
1698    #[test]
1699    fn should_not_override_when_metadata_created_is_unix_epoch() {
1700        assert!(!should_override_created_from_metadata(OffsetDateTime::UNIX_EPOCH));
1701    }
1702
1703    #[test]
1704    fn should_override_when_metadata_created_is_valid_time() {
1705        let created = OffsetDateTime::from_unix_timestamp(1704067200).expect("valid timestamp");
1706        assert!(should_override_created_from_metadata(created));
1707    }
1708
1709    #[test]
1710    fn table_bucket_delete_guard_rejects_remaining_catalog_metadata() {
1711        let err = validate_table_bucket_delete_allowed("table-bucket", true, true).unwrap_err();
1712
1713        assert!(matches!(err, StorageError::BucketNotEmpty(bucket) if bucket == "table-bucket"));
1714        assert!(validate_table_bucket_delete_allowed("table-bucket", true, false).is_ok());
1715        assert!(validate_table_bucket_delete_allowed("regular-bucket", false, true).is_ok());
1716    }
1717
1718    #[test]
1719    fn bucket_delete_metadata_cleanup_removes_internal_table_catalog_prefix() {
1720        let prefixes = bucket_delete_metadata_cleanup_prefixes("analytics");
1721
1722        assert!(prefixes.contains(&table_bucket_catalog_metadata_prefix("analytics")));
1723        assert!(prefixes.contains(&"buckets/analytics".to_string()));
1724    }
1725
1726    #[test]
1727    fn bucket_delete_marker_path_uses_internal_deleted_bucket_metadata_prefix() {
1728        assert_eq!(bucket_deleted_marker_prefix("analytics"), "buckets/.deleted/analytics");
1729        assert_eq!(
1730            bucket_deleted_marker_volume("analytics"),
1731            format!("{RUSTFS_META_BUCKET}/{BUCKET_META_PREFIX}/.deleted/analytics")
1732        );
1733    }
1734
1735    #[test]
1736    fn bucket_listing_selects_request_and_scanner_fanout() {
1737        assert_eq!(bucket_list_set_concurrency(0, SCANNER_BUCKET_LIST_SET_CONCURRENCY), 1);
1738        assert_eq!(bucket_list_set_concurrency(2, SCANNER_BUCKET_LIST_SET_CONCURRENCY), 2);
1739        assert_eq!(
1740            bucket_list_set_concurrency(100, SCANNER_BUCKET_LIST_SET_CONCURRENCY),
1741            SCANNER_BUCKET_LIST_SET_CONCURRENCY
1742        );
1743        assert_eq!(bucket_list_set_concurrency(100, usize::MAX), 100);
1744    }
1745
1746    #[tokio::test]
1747    async fn metadata_less_residue_scan_ignores_empty_dirs_and_reports_uuid_data() {
1748        let root = tempfile::tempdir().expect("temporary bucket root should be created");
1749        let bucket_path = root.path().join("bucket");
1750        tokio::fs::create_dir_all(bucket_path.join("empty/child"))
1751            .await
1752            .expect("empty directory residue should be created");
1753
1754        let empty = scan_metadata_less_residue(&bucket_path)
1755            .await
1756            .expect("empty residue scan should succeed");
1757        assert!(!empty.has_residue_without_xlmeta());
1758
1759        let data_dir = Uuid::new_v4();
1760        let part_path = bucket_path.join("object").join(data_dir.to_string()).join("part.1");
1761        tokio::fs::create_dir_all(part_path.parent().expect("part path should have a parent"))
1762            .await
1763            .expect("data dir should be created");
1764        tokio::fs::write(&part_path, b"orphan shard")
1765            .await
1766            .expect("part file should be written");
1767
1768        let residue = scan_metadata_less_residue(&bucket_path)
1769            .await
1770            .expect("metadata-less residue scan should succeed");
1771        assert!(residue.has_residue_without_xlmeta());
1772        assert_eq!(residue.files, 1);
1773        assert_eq!(residue.uuid_data_dirs, 1);
1774        let sample = residue.sample.as_deref().expect("part sample should be recorded");
1775        assert!(sample.starts_with("object/"));
1776        assert!(sample.ends_with("/part.1"));
1777    }
1778
1779    #[tokio::test]
1780    async fn metadata_less_residue_scan_shares_one_entry_budget_across_roots() {
1781        let root = tempfile::tempdir().expect("temporary diagnostic roots should be created");
1782        let first_root = root.path().join("disk-a");
1783        let second_root = root.path().join("disk-b");
1784        tokio::fs::create_dir_all(&first_root)
1785            .await
1786            .expect("first diagnostic root should be created");
1787        tokio::fs::create_dir_all(&second_root)
1788            .await
1789            .expect("second diagnostic root should be created");
1790        for index in 0..3 {
1791            std::fs::write(first_root.join(format!("first-{index}")), b"").expect("first-root fixture should be written");
1792            std::fs::write(second_root.join(format!("second-{index}")), b"").expect("second-root fixture should be written");
1793        }
1794
1795        let mut budget = BucketDeleteDiagnosticBudget::with_limits(4, Duration::from_secs(5));
1796        let first = scan_metadata_less_residue_with_budget(&first_root, &mut budget)
1797            .await
1798            .expect("first root should fit the shared budget");
1799        assert!(!first.diagnostic_truncated);
1800        let second = scan_metadata_less_residue_with_budget(&second_root, &mut budget)
1801            .await
1802            .expect("second root should stop at the remaining shared budget");
1803        assert!(second.diagnostic_truncated);
1804        assert!(first.entries_scanned + second.entries_scanned <= 4);
1805    }
1806
1807    #[tokio::test]
1808    async fn metadata_less_residue_scan_honors_an_expired_request_deadline() {
1809        let root = tempfile::tempdir().expect("temporary diagnostic root should be created");
1810        std::fs::write(root.path().join("orphan"), b"").expect("deadline fixture should be written");
1811        let mut budget = BucketDeleteDiagnosticBudget::with_limits(8, Duration::ZERO);
1812
1813        let scan = scan_metadata_less_residue_with_budget(root.path(), &mut budget)
1814            .await
1815            .expect("an expired diagnostic budget should fail closed without an IO error");
1816
1817        assert!(scan.diagnostic_truncated);
1818        assert_eq!(scan.entries_scanned, 0);
1819        assert_eq!(scan.diagnostic_bytes_read, 0);
1820    }
1821
1822    #[tokio::test]
1823    async fn metadata_less_residue_scan_stops_at_diagnostic_budget() {
1824        let root = tempfile::tempdir().expect("temporary bucket root should be created");
1825        let bucket_path = root.path().join("bucket");
1826        tokio::fs::create_dir_all(&bucket_path)
1827            .await
1828            .expect("budget fixture directory should be created");
1829        for index in 0..(BUCKET_DELETE_DIAGNOSTIC_MAX_ENTRIES + 32) {
1830            std::fs::write(bucket_path.join(format!("orphan-{index:05}")), b"").expect("budget fixture file should be created");
1831        }
1832
1833        let residue = scan_metadata_less_residue(&bucket_path)
1834            .await
1835            .expect("budgeted residue scan should fail closed without an IO error");
1836        assert!(residue.diagnostic_truncated);
1837        assert!(residue.has_residue_without_xlmeta());
1838        assert!(!residue.xlmeta_found);
1839        assert!(residue.entries_scanned <= BUCKET_DELETE_DIAGNOSTIC_MAX_ENTRIES);
1840        assert!(residue.files <= BUCKET_DELETE_DIAGNOSTIC_MAX_ENTRIES);
1841        assert_eq!(residue.diagnostic_bytes_read, 0);
1842    }
1843
1844    #[tokio::test]
1845    async fn bucket_residue_scan_distinguishes_visible_and_tier_free_xlmeta() {
1846        let root = tempfile::tempdir().expect("temporary bucket root should be created");
1847        let bucket_path = root.path().join("bucket");
1848        let visible_path = bucket_path.join("visible").join(STORAGE_FORMAT_FILE);
1849        tokio::fs::create_dir_all(visible_path.parent().expect("visible xl.meta should have a parent"))
1850            .await
1851            .expect("visible object directory should be created");
1852        let mut visible = FileMeta::new();
1853        visible
1854            .add_version(FileInfo {
1855                version_id: Some(Uuid::new_v4()),
1856                mod_time: Some(OffsetDateTime::now_utc()),
1857                ..Default::default()
1858            })
1859            .expect("visible version should encode");
1860        tokio::fs::write(&visible_path, visible.marshal_msg().expect("visible xl.meta should marshal"))
1861            .await
1862            .expect("visible xl.meta should be written");
1863
1864        let visible_scan = scan_metadata_less_residue(&bucket_path)
1865            .await
1866            .expect("visible xl.meta scan should succeed");
1867        assert_eq!(visible_scan.xlmeta_blocker, Some(BucketDeleteBlockerKind::VisibleVersion));
1868
1869        tokio::fs::remove_dir_all(bucket_path.join("visible"))
1870            .await
1871            .expect("visible fixture should be removed");
1872        let free_path = bucket_path.join("free").join(STORAGE_FORMAT_FILE);
1873        tokio::fs::create_dir_all(free_path.parent().expect("free xl.meta should have a parent"))
1874            .await
1875            .expect("free-version object directory should be created");
1876        let source_version_id = Uuid::new_v4();
1877        let mut free = FileMeta::new();
1878        free.add_version(FileInfo {
1879            version_id: Some(source_version_id),
1880            transition_status: TRANSITION_COMPLETE.to_string(),
1881            transitioned_objname: "remote/object".to_string(),
1882            transition_version_id: Some(Uuid::new_v4()),
1883            transition_tier: "WARM".to_string(),
1884            mod_time: Some(OffsetDateTime::now_utc()),
1885            ..Default::default()
1886        })
1887        .expect("transitioned source should encode");
1888        let mut delete = FileInfo {
1889            version_id: Some(source_version_id),
1890            mod_time: Some(OffsetDateTime::now_utc()),
1891            ..Default::default()
1892        };
1893        delete.set_tier_free_version_id(&Uuid::new_v4().to_string());
1894        free.delete_version(&delete)
1895            .expect("transitioned source delete should create a free-version");
1896        tokio::fs::write(&free_path, free.marshal_msg().expect("free-version xl.meta should marshal"))
1897            .await
1898            .expect("free-version xl.meta should be written");
1899
1900        let free_scan = scan_metadata_less_residue(&bucket_path)
1901            .await
1902            .expect("free-version xl.meta scan should succeed");
1903        assert_eq!(free_scan.xlmeta_blocker, Some(BucketDeleteBlockerKind::TierFreeVersion));
1904
1905        tokio::fs::remove_dir_all(bucket_path.join("free"))
1906            .await
1907            .expect("free-version fixture should be removed");
1908
1909        let exact_limit_path = bucket_path.join("exact-limit").join(STORAGE_FORMAT_FILE);
1910        tokio::fs::create_dir_all(exact_limit_path.parent().expect("exact-limit xl.meta should have a parent"))
1911            .await
1912            .expect("exact-limit object directory should be created");
1913        let exact_limit = tokio::fs::File::create(&exact_limit_path)
1914            .await
1915            .expect("exact-limit xl.meta should be created");
1916        exact_limit
1917            .set_len(BUCKET_DELETE_XLMETA_DIAGNOSTIC_MAX_BYTES)
1918            .await
1919            .expect("exact-limit xl.meta should be extended without allocating its contents");
1920        let exact_limit_scan = scan_metadata_less_residue(&bucket_path)
1921            .await
1922            .expect("exact-limit xl.meta scan should remain fail closed");
1923        assert_eq!(exact_limit_scan.xlmeta_blocker, Some(BucketDeleteBlockerKind::UnknownXlMeta));
1924        assert_eq!(exact_limit_scan.diagnostic_bytes_read, BUCKET_DELETE_XLMETA_DIAGNOSTIC_MAX_BYTES);
1925        tokio::fs::remove_dir_all(bucket_path.join("exact-limit"))
1926            .await
1927            .expect("exact-limit fixture should be removed");
1928
1929        let oversized_path = bucket_path.join("oversized").join(STORAGE_FORMAT_FILE);
1930        tokio::fs::create_dir_all(oversized_path.parent().expect("oversized xl.meta should have a parent"))
1931            .await
1932            .expect("oversized object directory should be created");
1933        let oversized = tokio::fs::File::create(&oversized_path)
1934            .await
1935            .expect("oversized xl.meta should be created");
1936        oversized
1937            .set_len(BUCKET_DELETE_XLMETA_DIAGNOSTIC_MAX_BYTES + 1)
1938            .await
1939            .expect("oversized xl.meta should be extended without allocating its contents");
1940        let oversized_scan = scan_metadata_less_residue(&bucket_path)
1941            .await
1942            .expect("oversized xl.meta scan should remain fail closed");
1943        assert_eq!(oversized_scan.xlmeta_blocker, Some(BucketDeleteBlockerKind::UnknownXlMeta));
1944        assert_eq!(oversized_scan.diagnostic_bytes_read, 0);
1945        assert!(oversized_scan.diagnostic_bytes_read <= BUCKET_DELETE_XLMETA_DIAGNOSTIC_MAX_BYTES);
1946    }
1947
1948    #[tokio::test]
1949    #[serial]
1950    async fn bucket_namespace_reads_keep_pre_expansion_bucket_visible() {
1951        let (_temp_dir, ecstore) = setup_multi_pool_bucket_test_env().await;
1952        let bucket = format!("pre-expansion-{}", Uuid::new_v4().simple());
1953        let object = "existing-object";
1954        ecstore.pools[0].disk_set[0]
1955            .make_bucket(&bucket, &MakeBucketOptions::default())
1956            .await
1957            .expect("bucket should be created in the original pool only");
1958        let mut reader = PutObjReader::from_vec(b"object written before pool expansion".to_vec());
1959        ecstore.pools[0]
1960            .put_object(&bucket, object, &mut reader, &ObjectOptions::default())
1961            .await
1962            .expect("object should be written in the original pool only");
1963
1964        let buckets = ecstore
1965            .list_bucket(&BucketOptions::default())
1966            .await
1967            .expect("S3 bucket listing should remain available after adding an empty pool");
1968        assert!(buckets.iter().any(|entry| entry.name == bucket));
1969
1970        let info = ecstore
1971            .get_bucket_info(&bucket, &BucketOptions::default())
1972            .await
1973            .expect("bucket validation should accept a bucket present in the original pool");
1974        assert_eq!(info.name, bucket);
1975
1976        metadata_sys::init_bucket_metadata_sys(ecstore.clone(), vec![bucket.clone()]).await;
1977        let listed = ecstore
1978            .clone()
1979            .list_objects_v2(&bucket, "", None, None, 1000, false, None, false)
1980            .await
1981            .expect("ListObjectsV2 should remain available after adding an empty pool");
1982        assert!(listed.objects.iter().any(|entry| entry.name == object));
1983        ecstore.pools[1].disk_set[0]
1984            .get_bucket_info(&bucket, &BucketOptions::default())
1985            .await
1986            .expect("metadata initialization should heal the bucket volume into the expansion pool");
1987
1988        let object_info = ecstore
1989            .get_object_info(&bucket, object, &ObjectOptions::default())
1990            .await
1991            .expect("an object in the original pool should remain readable after expansion");
1992        assert_eq!(object_info.name, object);
1993    }
1994
1995    #[tokio::test]
1996    #[serial]
1997    async fn bucket_creation_accepts_exact_quorum_in_every_set() {
1998        let (_temp_dir, ecstore) = setup_multi_pool_bucket_test_env().await;
1999        let bucket = format!("exact-set-quorum-{}", Uuid::new_v4().simple());
2000        let original_set = Arc::clone(&ecstore.pools[0].disk_set[0]);
2001        let offline = take_set_disks_offline(&ecstore, &original_set, &[0]).await;
2002
2003        ecstore
2004            .make_bucket_on_sets(&bucket, &MakeBucketOptions::default())
2005            .await
2006            .expect("three of four disks should satisfy the original set quorum");
2007
2008        restore_set_disks(&ecstore, &original_set, offline).await;
2009        for pool in &ecstore.pools {
2010            pool.disk_set[0]
2011                .get_bucket_info(&bucket, &BucketOptions::default())
2012                .await
2013                .expect("every set should expose a bucket created at its exact quorum");
2014        }
2015    }
2016
2017    #[tokio::test]
2018    #[serial]
2019    async fn bucket_creation_rejects_cross_pool_quorum_subsidy() {
2020        let (_temp_dir, ecstore) = setup_multi_pool_bucket_test_env().await;
2021        metadata_sys::init_bucket_metadata_sys(ecstore.clone(), Vec::new()).await;
2022        let bucket = format!("create-set-quorum-minus-one-{}", Uuid::new_v4().simple());
2023        let original_set = Arc::clone(&ecstore.pools[0].disk_set[0]);
2024        let offline = take_set_disks_offline(&ecstore, &original_set, &[0, 1]).await;
2025
2026        let err = ecstore
2027            .make_bucket(&bucket, &MakeBucketOptions::default())
2028            .await
2029            .expect_err("a healthy expansion pool must not subsidize a failed original set");
2030        restore_set_disks(&ecstore, &original_set, offline).await;
2031        assert!(
2032            matches!(&err, StorageError::InsufficientWriteQuorum(name, object) if name == &bucket && object.is_empty()),
2033            "unexpected error: {err}"
2034        );
2035        for pool in &ecstore.pools {
2036            assert_eq!(
2037                pool.disk_set[0]
2038                    .get_bucket_info(&bucket, &BucketOptions::default())
2039                    .await
2040                    .expect_err("failed creation must not leave an authoritative bucket"),
2041                StorageError::VolumeNotFound
2042            );
2043        }
2044    }
2045
2046    #[tokio::test]
2047    #[serial]
2048    async fn bucket_creation_prioritizes_hard_set_error_over_existing_bucket() {
2049        let (_temp_dir, ecstore) = setup_multi_pool_bucket_test_env().await;
2050        metadata_sys::init_bucket_metadata_sys(ecstore.clone(), Vec::new()).await;
2051        let bucket = format!("create-hard-error-{}", Uuid::new_v4().simple());
2052        ecstore.pools[0].disk_set[0]
2053            .make_bucket(&bucket, &MakeBucketOptions::default())
2054            .await
2055            .expect("the original pool should contain the bucket");
2056        let expansion_set = Arc::clone(&ecstore.pools[1].disk_set[0]);
2057        let offline = take_set_disks_offline(&ecstore, &expansion_set, &[0, 1]).await;
2058
2059        let err = ecstore
2060            .make_bucket(&bucket, &MakeBucketOptions::default())
2061            .await
2062            .expect_err("a hard set failure must outrank BucketExists from another set");
2063        restore_set_disks(&ecstore, &expansion_set, offline).await;
2064        assert!(
2065            matches!(&err, StorageError::InsufficientWriteQuorum(name, object) if name == &bucket && object.is_empty()),
2066            "unexpected error: {err}"
2067        );
2068        ecstore.pools[0].disk_set[0]
2069            .get_bucket_info(&bucket, &BucketOptions::default())
2070            .await
2071            .expect("failure in another set must not roll back a pre-existing bucket");
2072    }
2073
2074    #[tokio::test]
2075    #[serial]
2076    async fn bucket_delete_rejects_cross_pool_quorum_subsidy_before_mutation() {
2077        let (_temp_dir, ecstore) = setup_multi_pool_bucket_test_env().await;
2078        metadata_sys::init_bucket_metadata_sys(ecstore.clone(), Vec::new()).await;
2079        let bucket = format!("delete-set-quorum-minus-one-{}", Uuid::new_v4().simple());
2080        ecstore
2081            .make_bucket(&bucket, &MakeBucketOptions::default())
2082            .await
2083            .expect("bucket should be created in every set before deletion");
2084        let original_set = Arc::clone(&ecstore.pools[0].disk_set[0]);
2085        let offline = take_set_disks_offline(&ecstore, &original_set, &[0, 1, 2]).await;
2086
2087        let err = ecstore
2088            .delete_bucket(
2089                &bucket,
2090                &DeleteBucketOptions {
2091                    force: true,
2092                    ..Default::default()
2093                },
2094            )
2095            .await
2096            .expect_err("a healthy expansion pool must not subsidize a failed original set deletion");
2097        restore_set_disks(&ecstore, &original_set, offline).await;
2098        assert!(
2099            matches!(&err, StorageError::InsufficientWriteQuorum(name, object) if name == &bucket && object.is_empty()),
2100            "unexpected error: {err}"
2101        );
2102        for pool in &ecstore.pools {
2103            pool.disk_set[0]
2104                .get_bucket_info(&bucket, &BucketOptions::default())
2105                .await
2106                .expect("failed deletion preflight must preserve every bucket volume");
2107        }
2108        let (_, persisted) = metadata_sys::get_config_from_disk_with_presence_in(&ecstore.ctx, &bucket)
2109            .await
2110            .expect("failed deletion must preserve bucket metadata");
2111        assert!(persisted);
2112    }
2113
2114    #[tokio::test]
2115    #[serial]
2116    async fn bucket_delete_rolls_back_sets_after_partial_failure() {
2117        let (_temp_dir, ecstore) = setup_multi_pool_bucket_test_env().await;
2118        let bucket = format!("delete-set-rollback-{}", Uuid::new_v4().simple());
2119        ecstore
2120            .make_bucket_on_sets(&bucket, &MakeBucketOptions::default())
2121            .await
2122            .expect("bucket should be created in every set before deletion");
2123        let original_set = Arc::clone(&ecstore.pools[0].disk_set[0]);
2124        let offline = take_set_disks_offline(&ecstore, &original_set, &[0, 1]).await;
2125
2126        let err = ecstore
2127            .delete_bucket_on_sets(&bucket, &DeleteBucketOptions::default())
2128            .await
2129            .expect_err("a failed set must fail the complete bucket deletion");
2130        restore_set_disks(&ecstore, &original_set, offline).await;
2131        assert_eq!(err, StorageError::ErasureWriteQuorum);
2132        for pool in &ecstore.pools {
2133            pool.disk_set[0]
2134                .get_bucket_info(&bucket, &BucketOptions::default())
2135                .await
2136                .expect("delete rollback should restore the bucket namespace in every set");
2137        }
2138    }
2139
2140    #[tokio::test]
2141    #[serial]
2142    async fn bucket_config_update_accepts_pre_expansion_bucket() {
2143        let (_temp_dir, ecstore) = setup_multi_pool_bucket_test_env().await;
2144        metadata_sys::init_bucket_metadata_sys(ecstore.clone(), Vec::new()).await;
2145        let bucket = format!("config-pre-expansion-{}", Uuid::new_v4().simple());
2146        ecstore
2147            .make_bucket(&bucket, &MakeBucketOptions::default())
2148            .await
2149            .expect("bucket should be created with authoritative metadata");
2150        ecstore.pools[1].disk_set[0]
2151            .delete_bucket(&bucket, &DeleteBucketOptions::default())
2152            .await
2153            .expect("the expansion pool bucket volume should be removed for the regression state");
2154
2155        let policy = br#"{"Version":"2012-10-17","Statement":[]}"#.to_vec();
2156        ecstore
2157            .update_bucket_metadata_config(&bucket, crate::bucket::metadata::BUCKET_POLICY_CONFIG, policy.clone())
2158            .await
2159            .expect("config mutation should validate existence per erasure set");
2160        let (stored, _) = ecstore
2161            .get_bucket_policy_raw(&bucket)
2162            .await
2163            .expect("updated bucket policy should remain readable");
2164        assert_eq!(stored.as_bytes(), policy);
2165    }
2166
2167    #[tokio::test]
2168    #[serial]
2169    async fn bucket_metadata_init_recreates_pre_expansion_bucket_in_new_pool() {
2170        let (_temp_dir, ecstore) = setup_multi_pool_bucket_test_env().await;
2171        let bucket = format!("metadata-expansion-{}", Uuid::new_v4().simple());
2172        ecstore.pools[0].disk_set[0]
2173            .make_bucket(&bucket, &MakeBucketOptions::default())
2174            .await
2175            .expect("bucket should be created in the original pool only");
2176        assert_eq!(
2177            ecstore.pools[1].disk_set[0]
2178                .get_bucket_info(&bucket, &BucketOptions::default())
2179                .await
2180                .expect_err("new pool should initially have no bucket volume"),
2181            StorageError::VolumeNotFound
2182        );
2183
2184        metadata_sys::init_bucket_metadata_sys(ecstore.clone(), vec![bucket.clone()]).await;
2185
2186        ecstore.pools[1].disk_set[0]
2187            .get_bucket_info(&bucket, &BucketOptions::default())
2188            .await
2189            .expect("metadata initialization should recreate the bucket volume in the new pool");
2190    }
2191
2192    #[tokio::test]
2193    #[serial]
2194    async fn bucket_metadata_init_repairs_half_created_expansion_pool() {
2195        // Check both pool orders: an incomplete set must not hide a later
2196        // complete set, and a complete set must not weaken request validation.
2197        for complete_pool in 0..2 {
2198            let (temp_dir, ecstore) = setup_multi_pool_bucket_test_env().await;
2199            let bucket = format!("partial-expansion-{}", Uuid::new_v4().simple());
2200            for pool_index in 0..2 {
2201                let present_disks = if pool_index == complete_pool { 4 } else { 2 };
2202                for disk_index in 0..present_disks {
2203                    tokio::fs::create_dir(
2204                        temp_dir
2205                            .path()
2206                            .join(format!("pool{pool_index}-disk{disk_index}"))
2207                            .join(&bucket),
2208                    )
2209                    .await
2210                    .expect("fixture bucket volume should be created");
2211                }
2212            }
2213            assert_eq!(
2214                ecstore
2215                    .get_bucket_info_from_sets(&bucket, &BucketOptions::default())
2216                    .await
2217                    .expect_err("request validation must reject a half-created expansion set"),
2218                StorageError::ErasureWriteQuorum
2219            );
2220
2221            metadata_sys::init_bucket_metadata_sys(ecstore.clone(), vec![bucket.clone()]).await;
2222
2223            for pool_index in 0..2 {
2224                for disk_index in 0..4 {
2225                    assert!(
2226                        temp_dir
2227                            .path()
2228                            .join(format!("pool{pool_index}-disk{disk_index}"))
2229                            .join(&bucket)
2230                            .is_dir(),
2231                        "metadata initialization must heal every missing expansion volume"
2232                    );
2233                }
2234            }
2235            ecstore
2236                .get_bucket_info_from_sets(&bucket, &BucketOptions::default())
2237                .await
2238                .expect("strict request validation should succeed after volume repair");
2239        }
2240    }
2241
2242    #[tokio::test]
2243    #[serial]
2244    async fn bucket_metadata_init_does_not_combine_partial_set_evidence() {
2245        let (temp_dir, ecstore) = setup_multi_pool_bucket_test_env().await;
2246        let bucket = format!("no-quorum-expansion-{}", Uuid::new_v4().simple());
2247        for pool_index in 0..2 {
2248            for disk_index in 0..2 {
2249                tokio::fs::create_dir(
2250                    temp_dir
2251                        .path()
2252                        .join(format!("pool{pool_index}-disk{disk_index}"))
2253                        .join(&bucket),
2254                )
2255                .await
2256                .expect("fixture bucket volume should be created");
2257            }
2258        }
2259        assert_eq!(
2260            ecstore
2261                .bucket_exists_for_heal(&bucket)
2262                .await
2263                .expect_err("repair must require a complete quorum within one set"),
2264            StorageError::ErasureWriteQuorum
2265        );
2266
2267        metadata_sys::init_bucket_metadata_sys(ecstore.clone(), vec![bucket.clone()]).await;
2268
2269        for pool_index in 0..2 {
2270            for disk_index in 0..4 {
2271                assert_eq!(
2272                    temp_dir
2273                        .path()
2274                        .join(format!("pool{pool_index}-disk{disk_index}"))
2275                        .join(&bucket)
2276                        .is_dir(),
2277                    disk_index < 2,
2278                    "unproven bucket generations must not recreate missing volumes"
2279                );
2280            }
2281        }
2282    }
2283
2284    #[tokio::test]
2285    #[serial]
2286    async fn bucket_metadata_init_does_not_recreate_stale_bucket_name() {
2287        let (_temp_dir, ecstore) = setup_multi_pool_bucket_test_env().await;
2288        let bucket = format!("stale-expansion-{}", Uuid::new_v4().simple());
2289
2290        metadata_sys::init_bucket_metadata_sys(ecstore.clone(), vec![bucket.clone()]).await;
2291
2292        for pool in &ecstore.pools {
2293            let err = pool.disk_set[0]
2294                .get_bucket_info(&bucket, &BucketOptions::default())
2295                .await
2296                .expect_err("a stale startup listing must not recreate a deleted bucket");
2297            assert_eq!(err, StorageError::VolumeNotFound);
2298        }
2299    }
2300
2301    #[tokio::test]
2302    #[serial]
2303    async fn bucket_info_read_quorum_tracks_erasure_layout() {
2304        for (drive_count, parity) in [(2, 1), (3, 1), (4, 2), (5, 2), (6, 3), (8, 4), (6, 2), (12, 6)] {
2305            let (_temp_dir, store) = setup_bucket_quorum_test_env(&[drive_count], Some(parity)).await;
2306            metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
2307            let bucket = format!("read-quorum-{drive_count}-{parity}");
2308            let object = "uncached-object";
2309            let body = b"erasure read quorum must follow the persisted layout".repeat(32_768);
2310            store
2311                .make_bucket(&bucket, &MakeBucketOptions::default())
2312                .await
2313                .expect("healthy namespace should accept bucket creation");
2314            store
2315                .put_object(&bucket, object, &mut PutObjReader::from_vec(body.clone()), &ObjectOptions::default())
2316                .await
2317                .expect("healthy erasure set should accept the seed object");
2318            let set = &store.pools[0].disk_set[0];
2319            let lock = set
2320                .new_ns_lock(&bucket, object)
2321                .await
2322                .expect("seed namespace lock should resolve");
2323            drop(
2324                lock.get_write_lock(Duration::from_secs(30))
2325                    .await
2326                    .expect("seed physical fanout must finish before taking disks offline"),
2327            );
2328            if (drive_count, parity) == (6, 3) {
2329                let mut kvs = rustfs_config::server_config::KVS::new();
2330                kvs.insert(crate::config::storageclass::CLASS_STANDARD.to_string(), "EC:2".to_string());
2331                set.set_test_storage_class_config(
2332                    crate::config::storageclass::lookup_config_for_pools_without_env(&kvs, &[drive_count])
2333                        .expect("a later storage-class change must not raise old objects' read quorum"),
2334                );
2335            }
2336
2337            let offline_indexes = (0..parity).collect::<Vec<_>>();
2338            let offline = take_set_disks_offline(&store, set, &offline_indexes).await;
2339            let info = store
2340                .get_bucket_info(&bucket, &BucketOptions::default())
2341                .await
2342                .expect("bucket validation must admit the object's exact read quorum");
2343            assert_eq!(info.name, bucket);
2344
2345            let mut reader = store
2346                .get_object_reader(&bucket, object, None, Default::default(), &ObjectOptions::default())
2347                .await
2348                .expect("the persisted layout should remain readable at its exact data-shard quorum");
2349            let mut restored = Vec::new();
2350            reader
2351                .stream
2352                .read_to_end(&mut restored)
2353                .await
2354                .expect("quorum read should reconstruct the body");
2355            assert_eq!(restored, body, "layout {drive_count}/{parity} must retain exact object contents");
2356            drop(reader);
2357
2358            if drive_count - parity == drive_count / 2 {
2359                let error = store
2360                    .get_bucket_info_from_sets(&bucket, &BucketOptions::default())
2361                    .await
2362                    .expect_err("bucket mutations must retain their majority namespace check");
2363                assert_eq!(error, StorageError::ErasureWriteQuorum);
2364            }
2365
2366            let below_quorum = take_set_disks_offline(&store, set, &[parity]).await;
2367            let read = store
2368                .get_object_reader(&bucket, object, None, Default::default(), &ObjectOptions::default())
2369                .await;
2370            match read {
2371                Ok(mut reader) => assert!(
2372                    reader.stream.read_to_end(&mut Vec::new()).await.is_err(),
2373                    "layout {drive_count}/{parity} must reject fewer than its data-shard quorum"
2374                ),
2375                Err(error) => assert!(
2376                    matches!(error, StorageError::ErasureReadQuorum | StorageError::InsufficientReadQuorum(_, _)),
2377                    "a missing shard must report read quorum loss, got {error}"
2378                ),
2379            }
2380            restore_set_disks(&store, set, below_quorum).await;
2381            restore_set_disks(&store, set, offline).await;
2382        }
2383    }
2384
2385    #[tokio::test]
2386    #[serial]
2387    async fn bucket_info_read_quorum_is_scoped_to_each_erasure_set() {
2388        let (_temp_dir, store) = setup_bucket_quorum_test_env(&[4, 6], None).await;
2389        metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
2390        let bucket = "read-quorum-mixed-pools";
2391        store
2392            .make_bucket(bucket, &MakeBucketOptions::default())
2393            .await
2394            .expect("healthy pools should accept bucket creation");
2395
2396        let first_set = &store.pools[0].disk_set[0];
2397        let second_set = &store.pools[1].disk_set[0];
2398        let first_offline = take_set_disks_offline(&store, first_set, &[0, 1]).await;
2399        let second_offline = take_set_disks_offline(&store, second_set, &[0, 1, 2]).await;
2400        store
2401            .get_bucket_info(bucket, &BucketOptions::default())
2402            .await
2403            .expect("each set independently satisfies its namespace read quorum");
2404
2405        for (set, extra_disk) in [(first_set, 2), (second_set, 3)] {
2406            let extra_offline = take_set_disks_offline(&store, set, &[extra_disk]).await;
2407            assert_eq!(
2408                store
2409                    .get_bucket_info(bucket, &BucketOptions::default())
2410                    .await
2411                    .expect_err("another pool must not subsidize a set below its read quorum"),
2412                StorageError::ErasureReadQuorum
2413            );
2414            restore_set_disks(&store, set, extra_offline).await;
2415        }
2416        restore_set_disks(&store, first_set, first_offline).await;
2417        restore_set_disks(&store, second_set, second_offline).await;
2418    }
2419
2420    #[tokio::test]
2421    #[serial]
2422    async fn bucket_info_read_quorum_requires_authoritative_metadata() {
2423        for state in ["missing", "corrupt", "foreign", "incarnation"] {
2424            let (_temp_dir, store) = setup_bucket_quorum_test_env(&[4], None).await;
2425            metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
2426            let bucket = format!("read-quorum-{state}-metadata");
2427            let mut metadata = if state == "missing" {
2428                store
2429                    .make_bucket_on_sets(&bucket, &MakeBucketOptions::default())
2430                    .await
2431                    .expect("simulate directories left before bucket metadata is published");
2432                BucketMetadata::new(&bucket)
2433            } else {
2434                store
2435                    .make_bucket(&bucket, &MakeBucketOptions::default())
2436                    .await
2437                    .expect("healthy bucket should publish metadata");
2438                metadata_sys::get_in(&store.ctx, &bucket)
2439                    .await
2440                    .expect("seed metadata should be cached")
2441                    .as_ref()
2442                    .clone()
2443            };
2444            let path = metadata.save_file_path();
2445            match state {
2446                "corrupt" => crate::config::com::save_config(store.clone(), &path, b"corrupt".to_vec())
2447                    .await
2448                    .expect("persist corrupt metadata while the cached copy remains valid"),
2449                "foreign" => {
2450                    metadata.name = "different-bucket".to_string();
2451                    let mut encoded = vec![1, 0, 1, 0];
2452                    encoded.extend(metadata.marshal_msg().expect("foreign metadata should encode"));
2453                    crate::config::com::save_config(store.clone(), &path, encoded)
2454                        .await
2455                        .expect("persist metadata for a different bucket at the requested path");
2456                }
2457                "incarnation" => crate::bucket::metadata::save_bucket_incarnation(store.clone(), &bucket, Uuid::new_v4())
2458                    .await
2459                    .expect("persist a different bucket generation"),
2460                _ => {}
2461            }
2462
2463            let set = &store.pools[0].disk_set[0];
2464            let offline = take_set_disks_offline(&store, set, &[0, 1]).await;
2465            let error = store
2466                .get_bucket_info(&bucket, &BucketOptions::default())
2467                .await
2468                .expect_err("read admission must not trust residual directories or cached metadata");
2469            match state {
2470                "missing" => assert_eq!(error, StorageError::ErasureReadQuorum),
2471                "foreign" => assert_eq!(error, StorageError::FileCorrupt),
2472                "incarnation" => assert!(error.to_string().contains("sidecar does not match bucket metadata")),
2473                "corrupt" => assert!(error.to_string().contains("format invalid"), "unexpected corruption error: {error}"),
2474                _ => unreachable!(),
2475            }
2476            restore_set_disks(&store, set, offline).await;
2477        }
2478    }
2479
2480    #[tokio::test]
2481    #[serial]
2482    async fn bucket_info_read_quorum_accepts_persisted_legacy_metadata() {
2483        let (_temp_dir, store) = setup_bucket_quorum_test_env(&[4], None).await;
2484        metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
2485        let bucket = "interop";
2486        store
2487            .make_bucket_on_sets(bucket, &MakeBucketOptions::default())
2488            .await
2489            .expect("legacy bucket directories should exist");
2490        let hex = include_str!("../../tests/fixtures/minio/bucket_metadata.blob.hex")
2491            .split_whitespace()
2492            .collect::<String>();
2493        let body = (0..hex.len())
2494            .step_by(2)
2495            .map(|index| u8::from_str_radix(&hex[index..index + 2], 16).expect("pinned MinIO metadata fixture"))
2496            .collect();
2497        crate::config::com::save_config(store.clone(), &BucketMetadata::new(bucket).save_file_path(), body)
2498            .await
2499            .expect("legacy metadata should be persisted without an incarnation sidecar");
2500
2501        let set = &store.pools[0].disk_set[0];
2502        let offline = take_set_disks_offline(&store, set, &[0, 1]).await;
2503        let info = store
2504            .get_bucket_info(bucket, &BucketOptions::default())
2505            .await
2506            .expect("persisted MinIO metadata should authorize reads at the namespace read quorum");
2507        assert_eq!(info.name, bucket);
2508        assert!(info.versioning);
2509        assert!(info.object_locking);
2510        restore_set_disks(&store, set, offline).await;
2511    }
2512
2513    #[tokio::test]
2514    #[serial]
2515    async fn bucket_namespace_reads_report_missing_when_every_set_is_absent() {
2516        let (_temp_dir, ecstore) = setup_multi_pool_bucket_test_env().await;
2517        let bucket = format!("absent-{}", Uuid::new_v4().simple());
2518
2519        let buckets = ecstore
2520            .list_bucket(&BucketOptions::default())
2521            .await
2522            .expect("an empty healthy namespace should remain listable");
2523        assert!(buckets.iter().all(|entry| entry.name != bucket));
2524
2525        let err = ecstore
2526            .get_bucket_info(&bucket, &BucketOptions::default())
2527            .await
2528            .expect_err("a bucket absent from every set must remain missing");
2529        assert_eq!(err, StorageError::VolumeNotFound);
2530    }
2531
2532    #[tokio::test]
2533    #[serial]
2534    async fn bucket_namespace_reads_fail_closed_when_any_set_loses_quorum() {
2535        let (_temp_dir, ecstore) = setup_multi_pool_bucket_test_env().await;
2536        metadata_sys::init_bucket_metadata_sys(ecstore.clone(), Vec::new()).await;
2537        let bucket = format!("degraded-expansion-{}", Uuid::new_v4().simple());
2538        ecstore.pools[0].disk_set[0]
2539            .make_bucket(&bucket, &MakeBucketOptions::default())
2540            .await
2541            .expect("bucket should be created in the original pool only");
2542        ecstore.pools[1].disk_set[0].disks.write().await[0] = None;
2543        ecstore.pools[1].disk_set[0].disks.write().await[1] = None;
2544        ecstore.pools[1].disk_set[0].disks.write().await[2] = None;
2545
2546        let list_err = ecstore
2547            .list_bucket(&BucketOptions::default())
2548            .await
2549            .expect_err("listing must not hide an unavailable expansion pool");
2550        assert_eq!(list_err, StorageError::ErasureWriteQuorum);
2551
2552        let info_err = ecstore
2553            .get_bucket_info(&bucket, &BucketOptions::default())
2554            .await
2555            .expect_err("bucket validation must fail when an expansion pool is unavailable");
2556        assert_eq!(info_err, StorageError::ErasureReadQuorum);
2557    }
2558
2559    #[tokio::test]
2560    #[serial]
2561    async fn scanner_bucket_listing_unions_every_erasure_set() {
2562        let (_temp_dir, ecstore) = setup_multi_pool_bucket_test_env().await;
2563        let bucket = format!("second-pool-only-{}", Uuid::new_v4().simple());
2564        ecstore.pools[1].disk_set[0]
2565            .make_bucket(&bucket, &MakeBucketOptions::default())
2566            .await
2567            .expect("bucket should be created in the second pool only");
2568
2569        let listing = ecstore
2570            .list_bucket_for_scanner(&crate::storage_api_contracts::bucket::BucketOptions {
2571                no_metadata: true,
2572                ..Default::default()
2573            })
2574            .await
2575            .expect("scanner should enumerate every pool and set");
2576
2577        assert!(listing.topology_complete);
2578        assert!(listing.buckets.iter().any(|entry| entry.name == bucket));
2579        assert_eq!(listing.set_buckets.len(), 2);
2580        assert!(
2581            listing
2582                .set_buckets
2583                .iter()
2584                .find(|scope| scope.pool_index == 0 && scope.set_index == 0)
2585                .is_some_and(|scope| scope.buckets.is_empty())
2586        );
2587        assert!(
2588            listing
2589                .set_buckets
2590                .iter()
2591                .find(|scope| scope.pool_index == 1 && scope.set_index == 0)
2592                .is_some_and(|scope| scope.buckets.iter().any(|entry| entry.name == bucket))
2593        );
2594    }
2595
2596    #[tokio::test]
2597    async fn scanner_bucket_listing_marks_degraded_set_incomplete() {
2598        let (_temp_dir, ecstore) = setup_multi_pool_bucket_test_env().await;
2599        let bucket = format!("degraded-set-{}", Uuid::new_v4().simple());
2600        let set = &ecstore.pools[0].disk_set[0];
2601        set.make_bucket(&bucket, &MakeBucketOptions::default())
2602            .await
2603            .expect("bucket should be created before a disk is removed");
2604        set.disks.write().await[0] = None;
2605
2606        let listing = ecstore
2607            .list_bucket_for_scanner(&crate::storage_api_contracts::bucket::BucketOptions {
2608                no_metadata: true,
2609                ..Default::default()
2610            })
2611            .await
2612            .expect("a degraded set with quorum should still return candidate buckets");
2613
2614        assert!(listing.buckets.iter().any(|entry| entry.name == bucket));
2615        assert!(!listing.topology_complete);
2616    }
2617
2618    #[tokio::test]
2619    async fn scanner_bucket_listing_marks_divergent_disk_views_incomplete() {
2620        let (temp_dir, ecstore) = setup_multi_pool_bucket_test_env().await;
2621        let bucket = format!("divergent-set-{}", Uuid::new_v4().simple());
2622        ecstore.pools[0].disk_set[0]
2623            .make_bucket(&bucket, &MakeBucketOptions::default())
2624            .await
2625            .expect("bucket should be created before disk views diverge");
2626        for disk_index in 0..2 {
2627            tokio::fs::remove_dir_all(temp_dir.path().join(format!("pool0-disk{disk_index}")).join(&bucket))
2628                .await
2629                .expect("test bucket directory should be removed from a minority disk view");
2630        }
2631
2632        let listing = ecstore
2633            .list_bucket_for_scanner(&crate::storage_api_contracts::bucket::BucketOptions {
2634                no_metadata: true,
2635                ..Default::default()
2636            })
2637            .await
2638            .expect("responsive disks should still produce a scanner candidate listing");
2639
2640        assert!(listing.buckets.iter().all(|entry| entry.name != bucket));
2641        assert!(!listing.topology_complete);
2642    }
2643
2644    // These tests share one isolated instance and mutate its bucket metadata;
2645    // serialize them so their assertions cannot observe each other's operations.
2646    #[tokio::test]
2647    #[serial]
2648    async fn bucket_delete_mark_delete_removes_empty_bucket_and_keeps_deleted_marker() {
2649        let (disk_paths, ecstore) = setup_bucket_delete_test_env().await;
2650        let bucket = format!("bucket-mark-delete-{}", Uuid::new_v4().simple());
2651
2652        ecstore
2653            .make_bucket(&bucket, &MakeBucketOptions::default())
2654            .await
2655            .expect("bucket should be created");
2656        for disk_path in &disk_paths {
2657            tokio::fs::create_dir_all(disk_path.join(&bucket).join("empty-directory/nested/leaf"))
2658                .await
2659                .expect("empty directory remnant should be created");
2660        }
2661        assert!(metadata_sys::get_in(&ecstore.ctx, &bucket).await.is_ok());
2662
2663        let generation_before_delete = ecstore.scanner_namespace_mutation_generation();
2664        ecstore
2665            .delete_bucket(
2666                &bucket,
2667                &DeleteBucketOptions {
2668                    srdelete_op: SRBucketDeleteOp::MarkDelete,
2669                    ..Default::default()
2670                },
2671            )
2672            .await
2673            .expect("MarkDelete should remove an empty bucket");
2674        assert_eq!(
2675            ecstore.scanner_namespace_mutation_generation(),
2676            generation_before_delete.saturating_add(2),
2677            "successful bucket deletion should fence scanner namespace activity before and after deletion"
2678        );
2679
2680        assert!(
2681            !any_disk_path_exists(&disk_paths, &bucket).await,
2682            "MarkDelete should remove the bucket volume"
2683        );
2684        assert!(
2685            any_disk_path_exists(&disk_paths, bucket_deleted_marker_volume(&bucket)).await,
2686            "MarkDelete should persist the deleted-bucket marker"
2687        );
2688        assert!(
2689            metadata_sys::get_in(&ecstore.ctx, &bucket).await.is_err(),
2690            "deleted bucket metadata must be removed from the local cache"
2691        );
2692        let buckets = ecstore
2693            .list_bucket(&BucketOptions::default())
2694            .await
2695            .expect("bucket listing should succeed after MarkDelete");
2696        assert!(!buckets.iter().any(|info| info.name == bucket));
2697        ecstore
2698            .delete_all(RUSTFS_META_BUCKET, &bucket_deleted_marker_prefix(&bucket))
2699            .await
2700            .expect("deleted-bucket marker should be removed to simulate a partial failure");
2701        assert!(!any_disk_path_exists(&disk_paths, bucket_deleted_marker_volume(&bucket)).await);
2702        ecstore
2703            .delete_bucket(
2704                &bucket,
2705                &DeleteBucketOptions {
2706                    srdelete_op: SRBucketDeleteOp::MarkDelete,
2707                    ..Default::default()
2708                },
2709            )
2710            .await
2711            .expect("retried MarkDelete should recreate a missing tombstone");
2712        assert!(any_disk_path_exists(&disk_paths, bucket_deleted_marker_volume(&bucket)).await);
2713    }
2714
2715    #[tokio::test]
2716    #[serial]
2717    async fn bucket_delete_mark_delete_rejects_non_empty_bucket_without_force() {
2718        let (disk_paths, ecstore) = setup_bucket_delete_test_env().await;
2719        let bucket = format!("bucket-mark-delete-non-empty-{}", Uuid::new_v4().simple());
2720
2721        create_bucket_with_object(&ecstore, &bucket, "object.txt").await;
2722
2723        let err = ecstore
2724            .delete_bucket(
2725                &bucket,
2726                &DeleteBucketOptions {
2727                    srdelete_op: SRBucketDeleteOp::MarkDelete,
2728                    ..Default::default()
2729                },
2730            )
2731            .await
2732            .expect_err("MarkDelete should reject a non-empty bucket without force");
2733
2734        assert!(matches!(err, StorageError::BucketNotEmpty(name) if name == bucket));
2735        assert!(any_disk_has_object_metadata(&disk_paths, &bucket).await);
2736        assert!(!any_disk_path_exists(&disk_paths, bucket_deleted_marker_volume(&bucket)).await);
2737    }
2738
2739    #[tokio::test]
2740    #[serial]
2741    async fn bucket_delete_mark_delete_rejects_hidden_object_paths_without_force() {
2742        let (disk_paths, ecstore) = setup_bucket_delete_test_env().await;
2743        let bucket = format!("bucket-mark-delete-hidden-{}", Uuid::new_v4().simple());
2744
2745        create_bucket_with_object(&ecstore, &bucket, ".well-known/acme-challenge").await;
2746        let mut reader = PutObjReader::from_vec(b"delete bucket semantics".to_vec());
2747        ecstore
2748            .put_object(&bucket, ".rustfs.sys/object", &mut reader, &ObjectOptions::default())
2749            .await
2750            .expect("second hidden object should be written");
2751
2752        let err = ecstore
2753            .delete_bucket(
2754                &bucket,
2755                &DeleteBucketOptions {
2756                    srdelete_op: SRBucketDeleteOp::MarkDelete,
2757                    ..Default::default()
2758                },
2759            )
2760            .await
2761            .expect_err("MarkDelete should reject a hidden object path without force");
2762
2763        assert!(matches!(err, StorageError::BucketNotEmpty(name) if name == bucket));
2764        assert!(any_disk_has_object_metadata(&disk_paths, &bucket).await);
2765    }
2766
2767    #[tokio::test]
2768    #[serial]
2769    async fn bucket_delete_purge_removes_bucket_data_and_internal_metadata() {
2770        let (disk_paths, ecstore) = setup_bucket_delete_test_env().await;
2771        let bucket = format!("bucket-purge-{}", Uuid::new_v4().simple());
2772        let object = "object.txt";
2773        let metadata_prefix = format!("{RUSTFS_META_BUCKET}/{BUCKET_META_PREFIX}/{bucket}");
2774
2775        create_bucket_with_object(&ecstore, &bucket, object).await;
2776        write_bucket_metadata_marker(&disk_paths, &metadata_prefix).await;
2777        ecstore
2778            .mark_bucket_deleted(&bucket)
2779            .await
2780            .expect("deleted-bucket marker should be created");
2781        assert!(any_disk_path_exists(&disk_paths, &metadata_prefix).await);
2782        assert!(any_disk_path_exists(&disk_paths, bucket_deleted_marker_volume(&bucket)).await);
2783
2784        let generation_before_delete = ecstore.scanner_namespace_mutation_generation();
2785        ecstore
2786            .delete_bucket(
2787                &bucket,
2788                &DeleteBucketOptions {
2789                    force: true,
2790                    srdelete_op: SRBucketDeleteOp::Purge,
2791                    ..Default::default()
2792                },
2793            )
2794            .await
2795            .expect("Purge should force-delete bucket data");
2796        assert_eq!(
2797            ecstore.scanner_namespace_mutation_generation(),
2798            generation_before_delete.saturating_add(2),
2799            "successful bucket purge should fence scanner namespace activity before and after deletion"
2800        );
2801
2802        assert!(!any_disk_path_exists(&disk_paths, &bucket).await, "Purge should remove the bucket volume");
2803        assert!(
2804            !any_disk_path_exists(&disk_paths, &metadata_prefix).await,
2805            "Purge should remove bucket metadata prefix"
2806        );
2807        assert!(
2808            !any_disk_path_exists(&disk_paths, bucket_deleted_marker_volume(&bucket)).await,
2809            "Purge should remove the deleted-bucket marker"
2810        );
2811        assert!(
2812            metadata_sys::get_in(&ecstore.ctx, &bucket).await.is_err(),
2813            "purged bucket metadata must be removed from the local cache"
2814        );
2815        write_bucket_metadata_marker(&disk_paths, &metadata_prefix).await;
2816        ecstore
2817            .mark_bucket_deleted(&bucket)
2818            .await
2819            .expect("stale deleted-bucket marker should be recreated");
2820        ecstore
2821            .delete_bucket(
2822                &bucket,
2823                &DeleteBucketOptions {
2824                    force: true,
2825                    srdelete_op: SRBucketDeleteOp::Purge,
2826                    ..Default::default()
2827                },
2828            )
2829            .await
2830            .expect("retried Purge should remove stale metadata without a bucket volume");
2831        assert!(!any_disk_path_exists(&disk_paths, &metadata_prefix).await);
2832        assert!(!any_disk_path_exists(&disk_paths, bucket_deleted_marker_volume(&bucket)).await);
2833    }
2834
2835    /// Site replication replays make-with-versioning carrying the source's
2836    /// `lockEnabled` against a destination bucket that already exists. Gating the
2837    /// lock enable on `confirmed_missing` returned success while leaving the
2838    /// replica unlocked, so replicated versions could be deleted without the
2839    /// retention the source enforces.
2840    #[tokio::test(flavor = "multi_thread")]
2841    #[serial]
2842    async fn force_create_enables_object_lock_on_an_existing_bucket() {
2843        let (_disk_paths, ecstore) = setup_bucket_delete_test_env().await;
2844        let bucket = format!("bucket-force-lock-{}", Uuid::new_v4().simple());
2845
2846        ecstore
2847            .make_bucket(&bucket, &MakeBucketOptions::default())
2848            .await
2849            .expect("plain bucket should be created");
2850        assert!(
2851            !metadata_sys::get_in(&ecstore.ctx, &bucket)
2852                .await
2853                .expect("metadata should load")
2854                .lock_enabled,
2855            "test setup: the bucket must start unlocked"
2856        );
2857
2858        ecstore
2859            .make_bucket(
2860                &bucket,
2861                &MakeBucketOptions {
2862                    force_create: true,
2863                    lock_enabled: true,
2864                    ..Default::default()
2865                },
2866            )
2867            .await
2868            .expect("force create with lock_enabled should succeed on an existing bucket");
2869
2870        let meta = metadata_sys::get_in(&ecstore.ctx, &bucket)
2871            .await
2872            .expect("metadata should load after the lock enable");
2873        assert!(meta.lock_enabled, "Object Lock must be enabled on the existing bucket");
2874        assert!(
2875            !meta.object_lock_config_xml.is_empty(),
2876            "the Object Lock configuration must be persisted, not just the flag"
2877        );
2878        assert!(
2879            !meta.versioning_config_xml.is_empty(),
2880            "Object Lock requires versioning, so that must be persisted too"
2881        );
2882    }
2883
2884    #[tokio::test(flavor = "multi_thread")]
2885    #[serial]
2886    async fn make_bucket_seeds_new_bucket_durability_override() {
2887        temp_env::async_with_vars([(crate::bucket::durability::ENV_NEW_BUCKET_DURABILITY_MODE, None::<&str>)], async {
2888            let (_disk_paths, ecstore) = setup_bucket_delete_test_env().await;
2889            let bucket = format!("bucket-default-durability-{}", Uuid::new_v4().simple());
2890
2891            ecstore
2892                .make_bucket(&bucket, &MakeBucketOptions::default())
2893                .await
2894                .expect("new bucket should be created");
2895
2896            let metadata = metadata_sys::get_in(&ecstore.ctx, &bucket)
2897                .await
2898                .expect("metadata should load for the new bucket");
2899            assert_eq!(
2900                metadata.durability_config().and_then(|cfg| cfg.normalized_mode()).as_deref(),
2901                Some(crate::bucket::durability::BUCKET_DURABILITY_MODE_RELAXED)
2902            );
2903        })
2904        .await;
2905    }
2906
2907    #[tokio::test(flavor = "multi_thread")]
2908    #[serial]
2909    async fn force_create_existing_bucket_keeps_durability_override() {
2910        let (_disk_paths, ecstore) = setup_bucket_delete_test_env().await;
2911        let bucket = format!("bucket-force-durability-{}", Uuid::new_v4().simple());
2912
2913        temp_env::async_with_vars([(crate::bucket::durability::ENV_NEW_BUCKET_DURABILITY_MODE, Some("inherit"))], async {
2914            ecstore
2915                .make_bucket(&bucket, &MakeBucketOptions::default())
2916                .await
2917                .expect("plain bucket should be created without a durability override");
2918        })
2919        .await;
2920        assert!(
2921            metadata_sys::get_in(&ecstore.ctx, &bucket)
2922                .await
2923                .expect("metadata should load after initial create")
2924                .durability_config()
2925                .is_none(),
2926            "test setup: the existing bucket must start without an override"
2927        );
2928
2929        temp_env::async_with_vars([(crate::bucket::durability::ENV_NEW_BUCKET_DURABILITY_MODE, None::<&str>)], async {
2930            ecstore
2931                .make_bucket(
2932                    &bucket,
2933                    &MakeBucketOptions {
2934                        force_create: true,
2935                        lock_enabled: true,
2936                        ..Default::default()
2937                    },
2938                )
2939                .await
2940                .expect("force create should update existing bucket metadata");
2941        })
2942        .await;
2943
2944        let metadata = metadata_sys::get_in(&ecstore.ctx, &bucket)
2945            .await
2946            .expect("metadata should load after force create");
2947        assert!(metadata.lock_enabled, "force create sanity check: Object Lock should be enabled");
2948        assert!(
2949            metadata.durability_config().is_none(),
2950            "force create must not apply the new-bucket default to existing bucket metadata"
2951        );
2952    }
2953
2954    /// `DeleteBucket`'s emptiness check is a raw disk scan (`has_xlmeta_files`),
2955    /// not an S3-level listing, so "the client drained the bucket" and "the
2956    /// bucket is deletable" are two different contracts. Nothing pinned the
2957    /// second one, which is how the s3-tests lane ended up failing 219 cases on
2958    /// `nuke_prefixed_buckets` while every test body passed.
2959    #[tokio::test(flavor = "multi_thread")]
2960    #[serial]
2961    async fn bucket_delete_succeeds_after_the_last_object_version_is_deleted() {
2962        let (disk_paths, ecstore) = setup_bucket_delete_test_env().await;
2963        let bucket = format!("bucket-delete-after-drain-{}", Uuid::new_v4().simple());
2964        let object = "object.txt";
2965
2966        create_bucket_with_object(&ecstore, &bucket, object).await;
2967
2968        ecstore
2969            .delete_object(&bucket, object, ObjectOptions::default())
2970            .await
2971            .expect("client delete of the only object should succeed");
2972
2973        assert!(
2974            !any_disk_has_object_metadata(&disk_paths, &bucket).await,
2975            "deleting the last version must not leave xl.meta on disk: DeleteBucket scans the raw \
2976             bucket directory, so residue here is reported to clients as BucketNotEmpty"
2977        );
2978
2979        ecstore
2980            .delete_bucket(&bucket, &DeleteBucketOptions::default())
2981            .await
2982            .expect("DeleteBucket must succeed once the client has drained the bucket");
2983    }
2984
2985    #[tokio::test(flavor = "multi_thread")]
2986    #[serial]
2987    async fn bucket_delete_defers_zero_evidence_diagnostic_timeout_to_physical_empty_check() {
2988        let (disk_paths, ecstore) = setup_bucket_delete_test_env().await;
2989        let bucket = format!("bucket-del-diag-{}", Uuid::new_v4().simple());
2990
2991        ecstore
2992            .make_bucket(&bucket, &MakeBucketOptions::default())
2993            .await
2994            .expect("empty bucket should be created");
2995        let first_io_started = Arc::new(AtomicBool::new(false));
2996        let diagnostic_budget = BucketDeleteDiagnosticBudget::new().with_first_io_delay(
2997            BUCKET_DELETE_DIAGNOSTIC_MAX_ELAPSED + Duration::from_millis(100),
2998            first_io_started.clone(),
2999        );
3000
3001        ecstore
3002            .handle_delete_bucket_with_diagnostic_budget(&bucket, &DeleteBucketOptions::default(), diagnostic_budget)
3003            .await
3004            .expect("a zero-evidence diagnostic timeout must defer to the physical empty check");
3005
3006        assert!(
3007            first_io_started.load(Ordering::SeqCst),
3008            "the regression must delay the first diagnostic read_dir"
3009        );
3010        assert!(
3011            !any_disk_path_exists(&disk_paths, &bucket).await,
3012            "the physical empty check should allow deletion of the actually empty bucket"
3013        );
3014    }
3015
3016    #[tokio::test(flavor = "multi_thread")]
3017    #[serial]
3018    async fn bucket_delete_preserves_unobserved_residue_after_diagnostic_timeout() {
3019        let (disk_paths, ecstore) = setup_bucket_delete_test_env().await;
3020        let bucket = format!("bucket-del-residue-{}", Uuid::new_v4().simple());
3021        let object = "object.txt";
3022
3023        ecstore
3024            .make_bucket(&bucket, &MakeBucketOptions::default())
3025            .await
3026            .expect("bucket should be created");
3027        write_metadata_less_part_on_all_disks(&disk_paths, &bucket, object).await;
3028        let first_io_started = Arc::new(AtomicBool::new(false));
3029        let diagnostic_budget = BucketDeleteDiagnosticBudget::new().with_first_io_delay(
3030            BUCKET_DELETE_DIAGNOSTIC_MAX_ELAPSED + Duration::from_millis(100),
3031            first_io_started.clone(),
3032        );
3033
3034        let err = ecstore
3035            .handle_delete_bucket_with_diagnostic_budget(&bucket, &DeleteBucketOptions::default(), diagnostic_budget)
3036            .await
3037            .expect_err("physical empty-bucket enforcement must reject unobserved residue");
3038        assert!(
3039            first_io_started.load(Ordering::SeqCst),
3040            "the regression must time out before observing the residue"
3041        );
3042        match &err {
3043            StorageError::BucketNotEmpty(err_bucket) | StorageError::BucketNotEmptyWithDetails { bucket: err_bucket, .. } => {
3044                assert_eq!(err_bucket, &bucket)
3045            }
3046            other => panic!("expected an S3-compatible BucketNotEmpty error, got {other:?}"),
3047        }
3048        assert!(
3049            any_disk_path_exists(&disk_paths, Path::new(&bucket).join(object)).await,
3050            "physical empty-bucket enforcement must preserve unobserved residue"
3051        );
3052
3053        ecstore
3054            .delete_bucket(
3055                &bucket,
3056                &DeleteBucketOptions {
3057                    force: true,
3058                    ..Default::default()
3059                },
3060            )
3061            .await
3062            .expect("explicit force should clean up the test-owned residue");
3063    }
3064
3065    #[tokio::test(flavor = "multi_thread")]
3066    #[serial]
3067    async fn bucket_delete_succeeds_after_listing_and_deleting_an_unversioned_overwrite() {
3068        let (disk_paths, ecstore) = setup_bucket_delete_test_env().await;
3069        let bucket = format!("bucket-delete-after-overwrite-{}", Uuid::new_v4().simple());
3070        let object = "object.txt";
3071
3072        ecstore
3073            .make_bucket(&bucket, &MakeBucketOptions::default())
3074            .await
3075            .expect("unversioned bucket should be created");
3076
3077        let mut first_reader = PutObjReader::from_vec(b"version A".to_vec());
3078        let first = ecstore
3079            .put_object(&bucket, object, &mut first_reader, &ObjectOptions::default())
3080            .await
3081            .expect("version A should be written");
3082        let mut second_reader = PutObjReader::from_vec(b"version B".to_vec());
3083        let second = ecstore
3084            .put_object(&bucket, object, &mut second_reader, &ObjectOptions::default())
3085            .await
3086            .expect("version B should overwrite version A");
3087        assert_ne!(first.data_dir, second.data_dir, "the overwrite must publish a new body generation");
3088
3089        let listing = ecstore
3090            .clone()
3091            .list_object_versions(&bucket, "", None, None, None, 1000)
3092            .await
3093            .expect("the overwritten object should remain listable for teardown");
3094        assert_eq!(listing.objects.len(), 1, "an unversioned overwrite should expose one current version");
3095        let current = &listing.objects[0];
3096        assert_eq!(current.name, object);
3097        assert!(current.is_latest, "the listed null version must be current");
3098        assert_eq!(current.version_id, None, "an unversioned object must be exposed as the null version");
3099        assert_eq!(
3100            current.data_dir, second.data_dir,
3101            "listing must expose version B, not the overwritten body"
3102        );
3103
3104        for version in listing.objects {
3105            let version_id = version.version_id.map(|version_id| version_id.to_string());
3106            ecstore
3107                .delete_object(
3108                    &bucket,
3109                    &version.name,
3110                    ObjectOptions {
3111                        version_id,
3112                        ..Default::default()
3113                    },
3114                )
3115                .await
3116                .expect("each version returned by teardown listing should be deletable");
3117        }
3118
3119        assert!(
3120            !any_disk_has_object_metadata(&disk_paths, &bucket).await,
3121            "deleting the listed null version must remove every xl.meta"
3122        );
3123        ecstore
3124            .delete_bucket(&bucket, &DeleteBucketOptions::default())
3125            .await
3126            .expect("DeleteBucket should succeed after the listed overwrite is deleted");
3127    }
3128
3129    #[tokio::test]
3130    #[serial]
3131    async fn bucket_delete_default_s3_delete_still_rejects_non_empty_bucket() {
3132        let (disk_paths, ecstore) = setup_bucket_delete_test_env().await;
3133        let bucket = format!("bucket-s3-delete-{}", Uuid::new_v4().simple());
3134        let object = "object.txt";
3135
3136        create_bucket_with_object(&ecstore, &bucket, object).await;
3137
3138        let generation_before_delete = ecstore.scanner_namespace_mutation_generation();
3139        let err = ecstore
3140            .delete_bucket(&bucket, &DeleteBucketOptions::default())
3141            .await
3142            .expect_err("default S3 DeleteBucket should reject non-empty buckets");
3143
3144        assert!(matches!(err, StorageError::BucketNotEmpty(name) if name == bucket));
3145        assert_eq!(
3146            ecstore.scanner_namespace_mutation_generation(),
3147            generation_before_delete,
3148            "failed bucket deletion must not advance scanner namespace activity"
3149        );
3150        assert!(
3151            any_disk_has_object_metadata(&disk_paths, &bucket).await,
3152            "failed default S3 DeleteBucket must keep object data"
3153        );
3154        assert!(
3155            metadata_sys::get_in(&ecstore.ctx, &bucket).await.is_ok(),
3156            "failed default S3 DeleteBucket must keep metadata cache"
3157        );
3158    }
3159
3160    #[tokio::test(flavor = "multi_thread")]
3161    #[serial]
3162    async fn bucket_delete_reports_metadata_less_residue_without_removing_it() {
3163        let (disk_paths, ecstore) = setup_bucket_delete_test_env().await;
3164        let bucket = format!("bucket-delete-orphan-datadir-{}", Uuid::new_v4().simple());
3165        let object = "object.txt";
3166
3167        ecstore
3168            .make_bucket(&bucket, &MakeBucketOptions::default())
3169            .await
3170            .expect("bucket should be created");
3171        write_metadata_less_part_on_all_disks(&disk_paths, &bucket, object).await;
3172        assert!(
3173            !any_disk_has_object_metadata(&disk_paths, &bucket).await,
3174            "test setup must reproduce a bucket with data dirs but no xl.meta"
3175        );
3176
3177        let err = ecstore
3178            .delete_bucket(&bucket, &DeleteBucketOptions::default())
3179            .await
3180            .expect_err("metadata-less data dirs must block ordinary DeleteBucket");
3181
3182        match err {
3183            StorageError::BucketNotEmptyWithDetails {
3184                bucket: err_bucket,
3185                details,
3186            } => {
3187                assert_eq!(err_bucket, bucket);
3188                assert!(
3189                    details.contains("metadata-less on-disk residue"),
3190                    "diagnostic should identify metadata-less residue, got: {details}"
3191                );
3192                assert!(details.contains("files="), "diagnostic should include a bounded count");
3193                assert!(details.contains("uuid_data_dirs="), "diagnostic should include data-dir count");
3194            }
3195            other => panic!("expected detailed BucketNotEmpty, got {other:?}"),
3196        }
3197        assert!(
3198            any_disk_path_exists(&disk_paths, Path::new(&bucket).join(object)).await,
3199            "ordinary DeleteBucket must preserve metadata-less data until an explicit operator action"
3200        );
3201    }
3202
3203    #[tokio::test]
3204    #[serial]
3205    async fn bucket_delete_fences_put_started_after_empty_scan() {
3206        let (_, ecstore) = setup_bucket_delete_test_env().await;
3207        let bucket = format!("bucket-delete-empty-scan-race-{}", Uuid::new_v4().simple());
3208        let object = "committed-after-empty-scan";
3209        let payload = b"object started after DeleteBucket empty scan".to_vec();
3210
3211        ecstore
3212            .make_bucket(&bucket, &MakeBucketOptions::default())
3213            .await
3214            .expect("bucket should be created");
3215
3216        let barrier = install_delete_bucket_empty_scan_barrier();
3217        let delete_store = ecstore.clone();
3218        let delete_bucket = bucket.clone();
3219        let delete = tokio::spawn(async move {
3220            delete_store
3221                .delete_bucket(&delete_bucket, &DeleteBucketOptions::default())
3222                .await
3223        });
3224        barrier.wait_until_paused().await;
3225
3226        let put_store = ecstore.clone();
3227        let put_bucket = bucket.clone();
3228        let mut put = tokio::spawn(async move {
3229            let mut put_reader = PutObjReader::from_vec(payload);
3230            put_store
3231                .put_object(&put_bucket, object, &mut put_reader, &ObjectOptions::default())
3232                .await
3233        });
3234        assert!(
3235            tokio::time::timeout(Duration::from_millis(100), &mut put).await.is_err(),
3236            "PUT must wait behind the DeleteBucket lifecycle fence"
3237        );
3238
3239        barrier.release();
3240        delete
3241            .await
3242            .expect("DeleteBucket task should join")
3243            .expect("DeleteBucket should commit while holding the lifecycle fence");
3244        let err = put
3245            .await
3246            .expect("PUT task should join")
3247            .expect_err("PUT must not recreate an object in the deleted bucket");
3248        assert!(matches!(err, StorageError::BucketNotFound(name) if name == bucket));
3249    }
3250
3251    #[tokio::test]
3252    #[serial]
3253    async fn bucket_recreation_does_not_publish_unverified_usage() {
3254        let (_, ecstore) = setup_bucket_delete_test_env().await;
3255        let bucket = format!("bucket-usage-generation-{}", Uuid::new_v4().simple());
3256        ecstore
3257            .make_bucket(&bucket, &MakeBucketOptions::default())
3258            .await
3259            .expect("bucket should be created");
3260
3261        let mut snapshot = DataUsageInfo {
3262            last_update: Some(SystemTime::now()),
3263            buckets_count: 1,
3264            ..Default::default()
3265        };
3266        snapshot.buckets_usage.insert(
3267            bucket.clone(),
3268            BucketUsageInfo {
3269                size: 42,
3270                objects_count: 1,
3271                versions_count: 1,
3272                ..Default::default()
3273            },
3274        );
3275        snapshot.usage_snapshot_complete = true;
3276        snapshot.bucket_sizes.insert(bucket.clone(), 42);
3277        snapshot.calculate_totals();
3278        crate::data_usage::store_data_usage_in_backend(snapshot, ecstore.clone())
3279            .await
3280            .expect("usage snapshot should be stored");
3281
3282        ecstore
3283            .delete_bucket(&bucket, &DeleteBucketOptions::default())
3284            .await
3285            .expect("empty bucket should be deleted");
3286        let deleted = crate::data_usage::load_data_usage_from_backend(ecstore.clone())
3287            .await
3288            .expect("usage snapshot should remain readable");
3289        assert!(!deleted.buckets_usage.contains_key(&bucket));
3290
3291        ecstore
3292            .make_bucket(&bucket, &MakeBucketOptions::default())
3293            .await
3294            .expect("same bucket name should be recreated after delete returns");
3295        crate::data_usage::record_bucket_object_write_memory(&bucket, None, 84).await;
3296
3297        let mut recreated = crate::data_usage::load_data_usage_from_backend(ecstore.clone())
3298            .await
3299            .expect("recreated bucket usage base should load");
3300        crate::data_usage::apply_bucket_usage_memory_overlay(&mut recreated).await;
3301        assert!(
3302            !recreated.buckets_usage.contains_key(&bucket),
3303            "a request-path delta without an authoritative baseline must remain unavailable"
3304        );
3305        assert_eq!(crate::data_usage::get_bucket_usage_memory(&bucket).await, None);
3306    }
3307
3308    #[tokio::test]
3309    #[serial]
3310    async fn bucket_create_removes_stale_usage_before_physical_creation() {
3311        let (_, ecstore) = setup_bucket_delete_test_env().await;
3312        let bucket = format!("bucket-create-stale-usage-{}", Uuid::new_v4().simple());
3313        let mut snapshot = DataUsageInfo {
3314            last_update: Some(SystemTime::now()),
3315            buckets_count: 1,
3316            ..Default::default()
3317        };
3318        snapshot.buckets_usage.insert(
3319            bucket.clone(),
3320            BucketUsageInfo {
3321                size: 42,
3322                objects_count: 1,
3323                versions_count: 1,
3324                ..Default::default()
3325            },
3326        );
3327        snapshot.usage_snapshot_complete = true;
3328        snapshot.bucket_sizes.insert(bucket.clone(), 42);
3329        snapshot.calculate_totals();
3330        crate::data_usage::store_data_usage_in_backend(snapshot, ecstore.clone())
3331            .await
3332            .expect("stale usage fixture should be stored");
3333
3334        ecstore
3335            .make_bucket(&bucket, &MakeBucketOptions::default())
3336            .await
3337            .expect("CreateBucket should succeed");
3338
3339        let persisted = crate::data_usage::load_data_usage_from_backend(ecstore.clone())
3340            .await
3341            .expect("usage snapshot should remain readable");
3342        assert!(
3343            !persisted.buckets_usage.contains_key(&bucket),
3344            "a newly created bucket must not inherit the predecessor generation's usage"
3345        );
3346        assert!(
3347            ecstore.get_bucket_info(&bucket, &BucketOptions::default()).await.is_ok(),
3348            "physical creation should happen after the usage fence succeeds"
3349        );
3350    }
3351
3352    #[tokio::test]
3353    #[serial]
3354    async fn failed_create_rollback_does_not_run_unfenced_usage_cleanup() {
3355        let (_, ecstore) = setup_bucket_delete_test_env().await;
3356        let bucket = format!("bucket-create-rollback-{}", Uuid::new_v4().simple());
3357        ecstore
3358            .peer_sys
3359            .make_bucket(&bucket, &MakeBucketOptions::default())
3360            .await
3361            .expect("the partial-create fixture should expose a physical bucket");
3362
3363        let mut snapshot = DataUsageInfo {
3364            last_update: Some(SystemTime::now()),
3365            buckets_count: 1,
3366            ..Default::default()
3367        };
3368        snapshot.buckets_usage.insert(
3369            bucket.clone(),
3370            BucketUsageInfo {
3371                size: 42,
3372                objects_count: 1,
3373                versions_count: 1,
3374                ..Default::default()
3375            },
3376        );
3377        snapshot.usage_snapshot_complete = true;
3378        snapshot.bucket_sizes.insert(bucket.clone(), 42);
3379        snapshot.calculate_totals();
3380        crate::data_usage::store_data_usage_in_backend(snapshot, ecstore.clone())
3381            .await
3382            .expect("the usage fixture should be stored");
3383        crate::bucket::metadata::save_bucket_incarnation(ecstore.clone(), &bucket, Uuid::new_v4())
3384            .await
3385            .expect("partial create should have an incarnation sidecar");
3386
3387        ecstore.rollback_failed_bucket_creation(&bucket, None, None).await;
3388
3389        assert!(
3390            ecstore
3391                .peer_sys
3392                .get_bucket_info(&bucket, &BucketOptions::default())
3393                .await
3394                .is_err(),
3395            "failed-create rollback should remove the partial physical bucket"
3396        );
3397        assert!(
3398            crate::bucket::metadata::load_bucket_incarnation(ecstore.clone(), &bucket)
3399                .await
3400                .expect("rollback sidecar lookup should succeed")
3401                .is_none(),
3402            "failed-create rollback must remove the orphan incarnation sidecar"
3403        );
3404        let persisted = crate::data_usage::load_data_usage_from_backend(ecstore.clone())
3405            .await
3406            .expect("the usage snapshot should remain readable");
3407        assert!(
3408            persisted.buckets_usage.contains_key(&bucket),
3409            "failed-create rollback must not start an unfenced usage cleanup"
3410        );
3411
3412        crate::data_usage::store_data_usage_in_backend(DataUsageInfo::default(), ecstore.clone())
3413            .await
3414            .expect("the rollback usage fixture should be cleared");
3415    }
3416
3417    #[tokio::test]
3418    #[serial]
3419    async fn bucket_create_fails_closed_when_usage_snapshot_cannot_be_fenced() {
3420        let (_, ecstore) = setup_bucket_delete_test_env().await;
3421        let deleted_bucket = format!("bucket-delete-corrupt-usage-{}", Uuid::new_v4().simple());
3422        ecstore
3423            .make_bucket(&deleted_bucket, &MakeBucketOptions::default())
3424            .await
3425            .expect("bucket should be created before corrupting usage");
3426
3427        let usage_path = format!("{BUCKET_META_PREFIX}/{DATA_USAGE_OBJECT_NAME}");
3428        crate::config::com::save_config(ecstore.clone(), &usage_path, b"{".to_vec())
3429            .await
3430            .expect("corrupt usage fixture should be stored");
3431
3432        ecstore
3433            .delete_bucket(&deleted_bucket, &DeleteBucketOptions::default())
3434            .await
3435            .expect("usage snapshot corruption must not block DeleteBucket");
3436        assert!(
3437            ecstore
3438                .get_bucket_info(&deleted_bucket, &crate::storage_api_contracts::bucket::BucketOptions::default())
3439                .await
3440                .is_err(),
3441            "successful DeleteBucket must remove the physical bucket"
3442        );
3443
3444        let new_bucket = format!("bucket-create-corrupt-usage-{}", Uuid::new_v4().simple());
3445        let create_err = ecstore
3446            .make_bucket(&new_bucket, &MakeBucketOptions::default())
3447            .await
3448            .expect_err("MakeBucket must fail when stale usage cannot be fenced");
3449        assert!(!create_err.to_string().is_empty(), "the usage snapshot failure should be surfaced");
3450        assert!(
3451            ecstore
3452                .get_bucket_info(&new_bucket, &crate::storage_api_contracts::bucket::BucketOptions::default())
3453                .await
3454                .is_err(),
3455            "failed MakeBucket must not expose a new physical bucket"
3456        );
3457
3458        let restored = serde_json::to_vec(&DataUsageInfo {
3459            last_update: Some(SystemTime::now()),
3460            ..Default::default()
3461        })
3462        .expect("restored usage fixture should encode");
3463        crate::config::com::save_config(ecstore, &usage_path, restored)
3464            .await
3465            .expect("usage fixture should be restored after the failure-path test");
3466    }
3467
3468    /// Capture this module's log the way a stock deployment filters it.
3469    fn bucket_logs_at_default_level(emit: impl FnOnce()) -> String {
3470        use std::sync::{Arc, Mutex};
3471        use tracing_subscriber::EnvFilter;
3472        use tracing_subscriber::fmt::MakeWriter;
3473        use tracing_subscriber::layer::SubscriberExt;
3474
3475        #[derive(Clone, Default)]
3476        struct CapturedLogs {
3477            buffer: Arc<Mutex<Vec<u8>>>,
3478        }
3479        struct CapturedLogWriter {
3480            buffer: Arc<Mutex<Vec<u8>>>,
3481        }
3482        impl std::io::Write for CapturedLogWriter {
3483            fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
3484                self.buffer
3485                    .lock()
3486                    .expect("captured logs mutex should not be poisoned")
3487                    .extend_from_slice(buf);
3488                Ok(buf.len())
3489            }
3490            fn flush(&mut self) -> std::io::Result<()> {
3491                Ok(())
3492            }
3493        }
3494        impl<'a> MakeWriter<'a> for CapturedLogs {
3495            type Writer = CapturedLogWriter;
3496            fn make_writer(&'a self) -> Self::Writer {
3497                CapturedLogWriter {
3498                    buffer: Arc::clone(&self.buffer),
3499                }
3500            }
3501        }
3502
3503        let logs = CapturedLogs::default();
3504        let subscriber = tracing_subscriber::registry()
3505            .with(EnvFilter::new(rustfs_config::DEFAULT_LOG_LEVEL))
3506            .with(
3507                tracing_subscriber::fmt::layer()
3508                    .with_writer(logs.clone())
3509                    .with_ansi(false)
3510                    .without_time(),
3511            );
3512        let _guard = tracing::subscriber::set_default(subscriber);
3513        let _callsite_pin = crate::test_tracing::pin_callsite_interest_for_test();
3514
3515        emit();
3516
3517        let buffer = logs
3518            .buffer
3519            .lock()
3520            .expect("captured logs mutex should not be poisoned")
3521            .clone();
3522        String::from_utf8(buffer).expect("captured logs should be valid UTF-8")
3523    }
3524
3525    fn residue_sample(sample: &str) -> BucketMetadataLessResidue {
3526        BucketMetadataLessResidue {
3527            files: 1,
3528            entries_scanned: 3,
3529            sample: Some(sample.to_string()),
3530            ..Default::default()
3531        }
3532    }
3533
3534    /// `DeleteBucket` answers from a raw per-disk residue scan, so it can refuse
3535    /// for a reason no S3 request can observe. When that happens the server's
3536    /// own record of which residue blocked it must survive the default log
3537    /// filter — otherwise a client that drained the bucket sees `BucketNotEmpty`
3538    /// and the server log holds no trace of the refusal at all.
3539    #[test]
3540    fn unreachable_residue_is_reported_at_the_default_log_level() {
3541        for kind in [
3542            BucketDeleteBlockerKind::UnknownXlMeta,
3543            BucketDeleteBlockerKind::OrphanDirectory,
3544            BucketDeleteBlockerKind::DiagnosticBudgetExceeded,
3545        ] {
3546            let logs = bucket_logs_at_default_level(|| {
3547                record_bucket_delete_blocker("drained-bucket", kind, &residue_sample("obj/8f2c/xl.meta"));
3548            });
3549
3550            assert!(logs.contains("drained-bucket"), "the bucket must be named for {kind:?}: {logs}");
3551            assert!(logs.contains(kind.as_str()), "the blocker kind must be named for {kind:?}: {logs}");
3552            assert!(
3553                logs.contains("obj/8f2c/xl.meta"),
3554                "the residue sample is the only pointer to the leftover state for {kind:?}: {logs}"
3555            );
3556        }
3557    }
3558
3559    /// A bucket that genuinely still holds a version is an ordinary 409: the
3560    /// client can list and delete what is left, so this must not be reported as
3561    /// a server-side fault.
3562    #[test]
3563    fn client_visible_blockers_stay_below_the_default_log_level() {
3564        for kind in [
3565            BucketDeleteBlockerKind::VisibleVersion,
3566            BucketDeleteBlockerKind::TierFreeVersion,
3567        ] {
3568            let logs = bucket_logs_at_default_level(|| {
3569                record_bucket_delete_blocker("still-full-bucket", kind, &residue_sample("obj/xl.meta"));
3570            });
3571
3572            assert!(logs.is_empty(), "an ordinary non-empty bucket must not log an error for {kind:?}: {logs}");
3573        }
3574    }
3575
3576    #[test]
3577    fn blocker_kinds_split_by_whether_the_client_can_reach_the_residue() {
3578        assert!(BucketDeleteBlockerKind::VisibleVersion.is_client_visible());
3579        assert!(BucketDeleteBlockerKind::TierFreeVersion.is_client_visible());
3580        assert!(!BucketDeleteBlockerKind::UnknownXlMeta.is_client_visible());
3581        assert!(!BucketDeleteBlockerKind::OrphanDirectory.is_client_visible());
3582        assert!(!BucketDeleteBlockerKind::DiagnosticBudgetExceeded.is_client_visible());
3583    }
3584}