Skip to main content

rustfs_ecstore/store/
bucket.rs

1// Copyright 2024 RustFS Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use super::*;
16use crate::bucket::{
17    metadata::{BUCKET_TABLE_RESERVED_PREFIX, table_bucket_catalog_metadata_prefix},
18    utils::is_meta_bucketname,
19};
20use crate::error::is_err_bucket_not_found;
21use crate::runtime::sources as runtime_sources;
22use crate::set_disk::{BucketInfoQuorum, get_lock_acquire_timeout};
23use crate::storage_api_contracts::bucket::{BUCKET_LIFECYCLE_LOCK_OBJECT, SRBucketDeleteOp};
24use crate::storage_api_contracts::namespace::NamespaceLocking as _;
25use futures::stream::{self, StreamExt};
26use rustfs_policy::policy::BucketPolicy;
27use std::collections::BTreeMap;
28use std::future::Future;
29
30const DELETED_BUCKETS_PREFIX: &str = ".deleted";
31const SCANNER_BUCKET_LIST_SET_CONCURRENCY: usize = 4;
32const EVENT_BUCKET_DELETE_BLOCKED: &str = "bucket_delete_blocked";
33const EVENT_BUCKET_DELETE_ROLLBACK_FAILED: &str = "bucket_delete_rollback_failed";
34
35/// Record why `DeleteBucket` refused, and at a level that matches who can act
36/// on it.
37///
38/// `DeleteBucket` answers from a raw per-disk residue scan, not from a listing,
39/// so the server can refuse for a reason the client has no way to observe: the
40/// caller drained every version the S3 API will show and still gets
41/// `BucketNotEmpty`, with nothing to go on. This event is the only record of
42/// which residue blocked it and where — and it used to be emitted at `debug`,
43/// below both the `error` default log level and the `info` the CI s3-tests lane
44/// runs at, so in practice it was never written down. An intermittent
45/// `BucketNotEmpty` in that lane left a server log with no trace of the refusal
46/// at all, which is not a diagnosable state.
47///
48/// A blocker the client can still see and delete is ordinary — the 409 already
49/// says everything useful — so it stays at `warn`. Residue the client cannot
50/// reach through the API is a server-side integrity problem and is reported at
51/// `error`, which is what makes it survive a default deployment's filter.
52fn record_bucket_delete_blocker(bucket: &str, kind: BucketDeleteBlockerKind, residue: &BucketMetadataLessResidue) {
53    metrics::counter!("rustfs_bucket_delete_blockers_total", "kind" => kind.as_str()).increment(1);
54    if kind.is_client_visible() {
55        warn!(
56            event = EVENT_BUCKET_DELETE_BLOCKED,
57            component = "ecstore",
58            subsystem = "bucket",
59            bucket,
60            blocker = kind.as_str(),
61            files = residue.files,
62            uuid_data_dirs = residue.uuid_data_dirs,
63            entries_scanned = residue.entries_scanned,
64            diagnostic_bytes_read = residue.diagnostic_bytes_read,
65            diagnostic_truncated = residue.diagnostic_truncated,
66            sample = residue.sample.as_deref().unwrap_or("<none>"),
67            "Bucket deletion was blocked by durable local state"
68        );
69        return;
70    }
71
72    error!(
73        event = EVENT_BUCKET_DELETE_BLOCKED,
74        component = "ecstore",
75        subsystem = "bucket",
76        bucket,
77        blocker = kind.as_str(),
78        files = residue.files,
79        uuid_data_dirs = residue.uuid_data_dirs,
80        entries_scanned = residue.entries_scanned,
81        diagnostic_bytes_read = residue.diagnostic_bytes_read,
82        diagnostic_truncated = residue.diagnostic_truncated,
83        sample = residue.sample.as_deref().unwrap_or("<none>"),
84        "Bucket deletion was blocked by residue the client cannot reach through the S3 API"
85    );
86}
87
88fn bucket_list_set_concurrency(set_count: usize, max_concurrency: usize) -> usize {
89    set_count.clamp(1, max_concurrency.max(1))
90}
91
92fn should_override_created_from_metadata(created: OffsetDateTime) -> bool {
93    created != OffsetDateTime::UNIX_EPOCH
94}
95
96fn validate_table_bucket_delete_allowed(
97    bucket: &str,
98    table_bucket_enabled: bool,
99    table_catalog_metadata_exists: bool,
100) -> Result<()> {
101    if table_bucket_enabled && table_catalog_metadata_exists {
102        return Err(StorageError::BucketNotEmpty(bucket.to_string()));
103    }
104
105    Ok(())
106}
107
108async fn table_catalog_metadata_exists(ctx: &crate::runtime::instance::InstanceContext, bucket: &str) -> Result<bool> {
109    let local_disks = runtime_sources::local_disks_in(ctx).await;
110    for disk in local_disks.iter() {
111        let Some(bucket_path) = disk.get_bucket_path_for_io_if_local(bucket) else {
112            continue;
113        };
114        let catalog_path = bucket_path?.join(BUCKET_TABLE_RESERVED_PREFIX);
115        if has_xlmeta_files(&catalog_path).await? {
116            return Ok(true);
117        }
118    }
119
120    Ok(false)
121}
122
123async fn validate_table_bucket_delete_guard(ctx: &crate::runtime::instance::InstanceContext, bucket: &str) -> Result<()> {
124    let table_bucket_enabled = metadata_sys::get_in(ctx, bucket)
125        .await
126        .is_ok_and(|metadata| metadata.table_bucket_enabled());
127    if table_bucket_enabled {
128        validate_table_bucket_delete_allowed(bucket, true, table_catalog_metadata_exists(ctx, bucket).await?)?;
129    }
130
131    Ok(())
132}
133
134fn bucket_delete_metadata_cleanup_prefixes(bucket: &str) -> [String; 2] {
135    [
136        table_bucket_catalog_metadata_prefix(bucket),
137        format!("{BUCKET_META_PREFIX}/{bucket}"),
138    ]
139}
140
141fn bucket_deleted_marker_prefix(bucket: &str) -> String {
142    format!("{BUCKET_META_PREFIX}/{DELETED_BUCKETS_PREFIX}/{bucket}")
143}
144
145fn bucket_deleted_marker_volume(bucket: &str) -> String {
146    format!("{RUSTFS_META_BUCKET}/{}", bucket_deleted_marker_prefix(bucket))
147}
148
149pub(crate) async fn await_bucket_namespace_operation<T, F>(
150    guard: Option<&rustfs_lock::NamespaceLockGuard>,
151    bucket: &str,
152    operation: &'static str,
153    future: F,
154) -> Result<T>
155where
156    F: Future<Output = Result<T>>,
157{
158    let Some(guard) = guard else {
159        return future.await;
160    };
161    if guard.is_lock_lost() {
162        return Err(StorageError::other(format!(
163            "bucket namespace lock was lost before {operation}: {bucket}"
164        )));
165    }
166    tokio::select! {
167        biased;
168        _ = guard.lock_lost_notified() => Err(StorageError::other(format!(
169            "bucket namespace lock was lost during {operation}: {bucket}"
170        ))),
171        result = future => result,
172    }
173}
174
175async fn await_bucket_lifecycle_operation<T, F>(
176    lifecycle_guard: Option<&rustfs_lock::NamespaceLockGuard>,
177    namespace_guard: Option<&rustfs_lock::NamespaceLockGuard>,
178    bucket: &str,
179    operation: &'static str,
180    future: F,
181) -> Result<T>
182where
183    F: Future<Output = Result<T>>,
184{
185    await_bucket_namespace_operation(
186        lifecycle_guard,
187        bucket,
188        operation,
189        await_bucket_namespace_operation(namespace_guard, bucket, operation, future),
190    )
191    .await
192}
193
194async fn run_bucket_usage_cleanup<F>(guard: Option<&rustfs_lock::NamespaceLockGuard>, bucket: &str, future: F) -> Result<()>
195where
196    F: Future<Output = Result<()>>,
197{
198    await_bucket_namespace_operation(guard, bucket, "bucket usage cleanup", future).await
199}
200
201async fn run_physical_bucket_deletion<F>(guard: Option<&rustfs_lock::NamespaceLockGuard>, bucket: &str, future: F) -> Result<()>
202where
203    F: Future<Output = Result<()>>,
204{
205    // Fence before polling deletion: the physical namespace may become
206    // invisible at any await point inside the storage operation.
207    list_objects::observe_scanner_namespace_mutations(bucket, 1);
208    await_bucket_namespace_operation(guard, bucket, "physical bucket deletion", future).await
209}
210
211async fn bucket_delete_local_blocker(
212    ctx: &crate::runtime::instance::InstanceContext,
213    bucket: &str,
214    budget: &mut BucketDeleteDiagnosticBudget,
215) -> Result<Option<StorageError>> {
216    let local_disks = runtime_sources::local_disks_in(ctx).await;
217    let mut residue = BucketMetadataLessResidue::default();
218
219    for disk in local_disks.iter() {
220        let Some(bucket_path) = disk.get_bucket_path_for_io_if_local(bucket) else {
221            continue;
222        };
223        let scan = scan_metadata_less_residue_with_budget(&bucket_path?, budget).await?;
224        if scan.xlmeta_found {
225            record_bucket_delete_blocker(bucket, scan.xlmeta_blocker.unwrap_or(BucketDeleteBlockerKind::UnknownXlMeta), &scan);
226            return Ok(Some(StorageError::BucketNotEmpty(bucket.to_string())));
227        }
228        residue.files = residue.files.saturating_add(scan.files);
229        residue.uuid_data_dirs = residue.uuid_data_dirs.saturating_add(scan.uuid_data_dirs);
230        residue.entries_scanned = residue.entries_scanned.saturating_add(scan.entries_scanned);
231        residue.diagnostic_bytes_read = residue.diagnostic_bytes_read.saturating_add(scan.diagnostic_bytes_read);
232        if residue.sample.is_none() {
233            residue.sample = scan.sample.clone();
234        }
235        if scan.diagnostic_truncated {
236            residue.diagnostic_truncated = true;
237            if residue.files == 0 {
238                // A zero-evidence timeout is inconclusive. Preflight still uses
239                // non-recursive `force_if_empty`; post-failure keeps the physical error.
240                return Ok(None);
241            }
242            record_bucket_delete_blocker(bucket, BucketDeleteBlockerKind::DiagnosticBudgetExceeded, &residue);
243            return Ok(Some(StorageError::BucketNotEmptyWithDetails {
244                bucket: bucket.to_string(),
245                details: residue.describe(),
246            }));
247        }
248    }
249
250    if residue.has_residue_without_xlmeta() {
251        record_bucket_delete_blocker(bucket, BucketDeleteBlockerKind::OrphanDirectory, &residue);
252        return Ok(Some(StorageError::BucketNotEmptyWithDetails {
253            bucket: bucket.to_string(),
254            details: residue.describe(),
255        }));
256    }
257
258    Ok(None)
259}
260
261impl ECStore {
262    fn bucket_sets(&self) -> impl Iterator<Item = (usize, usize, Arc<crate::set_disk::SetDisks>)> + '_ {
263        self.pools.iter().flat_map(|pool| {
264            pool.disk_set
265                .iter()
266                .map(|set| (set.pool_index, set.set_index, Arc::clone(set)))
267        })
268    }
269
270    pub async fn get_bucket_metadata(&self, bucket: &str) -> Result<Arc<BucketMetadata>> {
271        let sys = metadata_sys::require_bucket_metadata_sys_in(&self.ctx)?;
272        sys.read().await.get(bucket).await
273    }
274
275    pub async fn get_bucket_policy(&self, bucket: &str) -> Result<(BucketPolicy, OffsetDateTime)> {
276        let sys = metadata_sys::require_bucket_metadata_sys_in(&self.ctx)?;
277        sys.read().await.get_bucket_policy(bucket).await
278    }
279
280    pub async fn get_bucket_policy_raw(&self, bucket: &str) -> Result<(String, OffsetDateTime)> {
281        let sys = metadata_sys::require_bucket_metadata_sys_in(&self.ctx)?;
282        sys.read().await.get_bucket_policy_raw(bucket).await
283    }
284
285    pub async fn restricts_public_bucket_access(&self, bucket: &str) -> Result<bool> {
286        let sys = metadata_sys::require_bucket_metadata_sys_in(&self.ctx)?;
287        let (config, _) = sys.read().await.get_public_access_block_config(bucket).await?;
288        Ok(config.restrict_public_buckets.unwrap_or(false))
289    }
290
291    pub async fn update_bucket_metadata_config(&self, bucket: &str, config_file: &str, data: Vec<u8>) -> Result<OffsetDateTime> {
292        metadata_sys::update_in(&self.ctx, bucket, config_file, data).await
293    }
294
295    pub async fn bucket_incarnation_id(&self, bucket: &str) -> Result<Uuid> {
296        metadata_sys::get_cached_bucket_incarnation_id_in(&self.ctx, bucket).await
297    }
298
299    pub async fn bucket_incarnation_id_from_disk(&self, bucket: &str) -> Result<Uuid> {
300        metadata_sys::get_bucket_incarnation_id_in(&self.ctx, bucket).await
301    }
302
303    /// The object commit path acquires this sentinel before bucket metadata and
304    /// exact-object namespace locks.
305    pub(crate) async fn acquire_bucket_lifecycle_read_lock(&self, bucket: &str) -> Result<rustfs_lock::NamespaceLockGuard> {
306        let lock = self.new_ns_lock(bucket, BUCKET_LIFECYCLE_LOCK_OBJECT).await?;
307        lock.get_read_lock(get_lock_acquire_timeout()).await.map_err(|err| match err {
308            rustfs_lock::error::LockError::QuorumNotReached { required, achieved } => {
309                StorageError::NamespaceLockQuorumUnavailable {
310                    mode: "bucket_lifecycle_read",
311                    bucket: bucket.to_string(),
312                    object: BUCKET_LIFECYCLE_LOCK_OBJECT.to_string(),
313                    required,
314                    achieved,
315                }
316            }
317            other => StorageError::Lock(other),
318        })
319    }
320
321    /// Acquire the bucket lifecycle read lock and validate the bucket
322    /// incarnation against `expected`, memoizing the validation while this
323    /// node keeps continuous read-lock coverage (see [`super::bucket_fence`]).
324    ///
325    /// Semantics are identical to the pre-existing per-PUT
326    /// `acquire_bucket_lifecycle_read_lock` + from-disk
327    /// `validate_bucket_incarnation` pair: the first PUT in a coverage window
328    /// performs exactly that authoritative disk validation; overlapping PUTs
329    /// reuse its result, which is sound because bucket deletion/recreation
330    /// requires the lifecycle WRITE lock and therefore cannot have run while
331    /// any read guard was continuously held.
332    pub async fn acquire_bucket_incarnation_fence(
333        &self,
334        bucket: &str,
335        expected: uuid::Uuid,
336    ) -> Result<super::BucketIncarnationFenceGuard> {
337        let inner = self.acquire_bucket_lifecycle_read_lock(bucket).await?;
338        let pieces = super::bucket_fence::FencePieces {
339            registry: self.bucket_fence_registry.clone(),
340            inner,
341        };
342        let registration = pieces.enter(bucket);
343        let current = match registration.memoized {
344            Some(current) => current,
345            None => match metadata_sys::get_bucket_incarnation_id_in(&self.ctx, bucket).await {
346                Ok(current) => {
347                    // Never memoize under lost coverage: a granted lifecycle
348                    // write lock could already have changed the incarnation.
349                    if !pieces.lock_lost() {
350                        pieces.memoize(bucket, current);
351                    }
352                    current
353                }
354                Err(err) => {
355                    pieces.abandon(bucket, registration.token);
356                    return Err(err);
357                }
358            },
359        };
360        if current != expected {
361            pieces.abandon(bucket, registration.token);
362            return Err(StorageError::BucketNotFound(bucket.to_string()));
363        }
364        Ok(pieces.into_guard(bucket, registration.token))
365    }
366
367    /// Hold this guard through recursive-delete authorization and mutation so
368    /// writers cannot introduce an unchecked object into the deletion scope.
369    pub async fn lock_bucket_for_recursive_delete(&self, bucket: &str) -> Result<rustfs_lock::NamespaceLockGuard> {
370        if self.ctx.lock_manager().is_disabled() {
371            return Err(StorageError::InvalidArgument(
372                bucket.to_owned(),
373                String::new(),
374                "Recursive deletion requires namespace locking".to_owned(),
375            ));
376        }
377        self.acquire_bucket_lifecycle_write_lock(bucket).await
378    }
379
380    pub(crate) async fn acquire_bucket_lifecycle_write_lock(&self, bucket: &str) -> Result<rustfs_lock::NamespaceLockGuard> {
381        let lock = self.new_ns_lock(bucket, BUCKET_LIFECYCLE_LOCK_OBJECT).await?;
382        lock.get_write_lock(get_lock_acquire_timeout())
383            .await
384            .map_err(|err| match err {
385                rustfs_lock::error::LockError::QuorumNotReached { required, achieved } => {
386                    StorageError::NamespaceLockQuorumUnavailable {
387                        mode: "bucket_lifecycle_write",
388                        bucket: bucket.to_string(),
389                        object: BUCKET_LIFECYCLE_LOCK_OBJECT.to_string(),
390                        required,
391                        achieved,
392                    }
393                }
394                other => StorageError::Lock(other),
395            })
396    }
397
398    async fn mark_bucket_deleted(&self, bucket: &str) -> Result<()> {
399        let marker_volume = bucket_deleted_marker_volume(bucket);
400
401        self.make_bucket_on_sets(
402            marker_volume.as_str(),
403            &MakeBucketOptions {
404                force_create: true,
405                ..Default::default()
406            },
407        )
408        .await
409        .map_err(|err| to_object_err(err, vec![bucket]))?;
410
411        Ok(())
412    }
413
414    async fn make_bucket_on_sets(&self, bucket: &str, opts: &MakeBucketOptions) -> Result<()> {
415        let results = futures::future::join_all(
416            self.bucket_sets()
417                .map(|(_, _, set)| async move { set.make_bucket(bucket, opts).await }),
418        )
419        .await;
420        if results.is_empty() {
421            return Err(StorageError::ErasureWriteQuorum);
422        }
423
424        let mut bucket_exists_error = None;
425        let mut first_hard_error = None;
426        for result in results {
427            let Err(err) = result else {
428                continue;
429            };
430            if matches!(err, StorageError::VolumeExists) || is_err_bucket_exists(&err) {
431                if bucket_exists_error.is_none() {
432                    bucket_exists_error = Some(err);
433                }
434            } else if first_hard_error.is_none() {
435                first_hard_error = Some(err);
436            }
437        }
438
439        match first_hard_error.or(bucket_exists_error) {
440            Some(err) => Err(err),
441            None => Ok(()),
442        }
443    }
444
445    async fn delete_bucket_on_sets(&self, bucket: &str, opts: &DeleteBucketOptions) -> Result<()> {
446        let results = futures::future::join_all(
447            self.bucket_sets()
448                .map(|(_, _, set)| async move { set.delete_bucket(bucket, opts).await }),
449        )
450        .await;
451        if results.is_empty() {
452            return Err(StorageError::ErasureWriteQuorum);
453        }
454
455        let mut deleted = false;
456        let mut first_error = None;
457        for result in results {
458            match result {
459                Ok(()) => deleted = true,
460                Err(err) if is_err_strict_volume_not_found(&err) => {}
461                Err(err) if first_error.is_none() => first_error = Some(err),
462                Err(_) => {}
463            }
464        }
465
466        if let Some(delete_error) = first_error {
467            if !opts.no_recreate {
468                let rollback_opts = MakeBucketOptions {
469                    force_create: true,
470                    no_lock: true,
471                    ..Default::default()
472                };
473                if let Err(rollback_error) = self.make_bucket_on_sets(bucket, &rollback_opts).await {
474                    warn!(
475                        event = EVENT_BUCKET_DELETE_ROLLBACK_FAILED,
476                        component = "ecstore",
477                        subsystem = "bucket",
478                        bucket,
479                        deletion_error = ?delete_error,
480                        rollback_error = ?rollback_error,
481                        "Bucket deletion rollback could not restore every bucket volume"
482                    );
483                }
484            }
485            return Err(delete_error);
486        }
487
488        if deleted { Ok(()) } else { Err(StorageError::VolumeNotFound) }
489    }
490
491    async fn cleanup_deleted_bucket_metadata(
492        &self,
493        bucket: &str,
494        include_deleted_marker: bool,
495        guard: Option<&rustfs_lock::NamespaceLockGuard>,
496    ) -> Result<()> {
497        for prefix in bucket_delete_metadata_cleanup_prefixes(bucket) {
498            await_bucket_namespace_operation(
499                guard,
500                bucket,
501                "deleted bucket metadata cleanup",
502                self.delete_all(RUSTFS_META_BUCKET, prefix.as_str()),
503            )
504            .await?;
505        }
506
507        if include_deleted_marker {
508            let marker_prefix = bucket_deleted_marker_prefix(bucket);
509            await_bucket_namespace_operation(
510                guard,
511                bucket,
512                "deleted bucket marker cleanup",
513                self.delete_all(RUSTFS_META_BUCKET, marker_prefix.as_str()),
514            )
515            .await?;
516        }
517
518        await_bucket_namespace_operation(
519            guard,
520            bucket,
521            "deleted bucket metadata cache cleanup",
522            metadata_sys::remove_bucket_metadata_in(&self.ctx, bucket),
523        )
524        .await?;
525        runtime_sources::delete_bucket_monitor_entry(bucket);
526        Ok(())
527    }
528
529    async fn cleanup_bucket_usage(&self, bucket: &str, guard: Option<&rustfs_lock::NamespaceLockGuard>) -> Result<()> {
530        run_bucket_usage_cleanup(guard, bucket, async {
531            crate::data_usage::prepare_bucket_usage_for_namespace_change(bucket, guard).await?;
532            crate::data_usage::remove_bucket_usage_from_backend_with_guard_fenced(self, bucket, guard).await
533        })
534        .await
535    }
536
537    async fn cleanup_bucket_usage_best_effort(&self, bucket: &str, guard: Option<&rustfs_lock::NamespaceLockGuard>) {
538        if let Err(err) = self.cleanup_bucket_usage(bucket, guard).await {
539            warn!(
540                bucket = %bucket,
541                error = ?err,
542                "bucket data usage cleanup deferred to scanner reconciliation"
543            );
544        }
545    }
546
547    async fn rollback_failed_bucket_creation(
548        &self,
549        bucket: &str,
550        lifecycle_guard: Option<&rustfs_lock::NamespaceLockGuard>,
551        namespace_guard: Option<&rustfs_lock::NamespaceLockGuard>,
552    ) {
553        let rollback_opts = DeleteBucketOptions {
554            no_lock: true,
555            no_recreate: true,
556            ..Default::default()
557        };
558        if let Err(err) =
559            await_bucket_lifecycle_operation(lifecycle_guard, namespace_guard, bucket, "failed bucket creation rollback", async {
560                self.delete_bucket_on_sets(bucket, &rollback_opts)
561                    .await
562                    .map_err(|rollback_err| to_object_err(rollback_err, vec![bucket]))
563            })
564            .await
565        {
566            warn!(
567                bucket = %bucket,
568                error = ?err,
569                "failed bucket creation rollback did not remove every physical bucket volume"
570            );
571            return;
572        }
573        if let Err(err) = self.cleanup_deleted_bucket_metadata(bucket, false, namespace_guard).await {
574            warn!(
575                bucket = %bucket,
576                error = ?err,
577                "failed bucket creation rollback left internal bucket metadata"
578            );
579        }
580    }
581
582    #[instrument(skip(self))]
583    pub(super) async fn handle_make_bucket(&self, bucket: &str, opts: &MakeBucketOptions) -> Result<()> {
584        if !is_meta_bucketname(bucket)
585            && let Err(err) = check_valid_bucket_name_strict(bucket)
586        {
587            return Err(StorageError::BucketNameInvalid(err.to_string()));
588        }
589
590        // Lock order: bucket lifecycle -> metadata transaction -> exact bucket namespace.
591        let bucket_lifecycle_guard = if !opts.no_lock {
592            Some(self.acquire_bucket_lifecycle_write_lock(bucket).await?)
593        } else {
594            None
595        };
596        let metadata_transaction_guard = if !opts.no_lock && !is_meta_bucketname(bucket) {
597            Some(metadata_sys::acquire_bucket_metadata_transaction_lock_in(&self.ctx, bucket).await?)
598        } else {
599            None
600        };
601        let ns_guard = if !opts.no_lock {
602            let ns_lock = self.new_ns_lock(bucket, bucket).await?;
603            Some(
604                await_bucket_namespace_operation(
605                    bucket_lifecycle_guard.as_ref(),
606                    bucket,
607                    "bucket namespace lock acquisition",
608                    async {
609                        ns_lock.get_write_lock(get_lock_acquire_timeout()).await.map_err(|e| match e {
610                            rustfs_lock::error::LockError::QuorumNotReached { required, achieved } => {
611                                StorageError::NamespaceLockQuorumUnavailable {
612                                    mode: "write",
613                                    bucket: bucket.to_string(),
614                                    object: bucket.to_string(),
615                                    required,
616                                    achieved,
617                                }
618                            }
619                            other => StorageError::Lock(other),
620                        })
621                    },
622                )
623                .await?,
624            )
625        } else {
626            None
627        };
628
629        let existing_bucket_info = match self.get_bucket_info_from_sets(bucket, &BucketOptions::default()).await {
630            Ok(info) => Some(info),
631            Err(err) => {
632                if is_err_bucket_not_found(&err) {
633                    None
634                } else {
635                    return Err(to_object_err(err, vec![bucket]));
636                }
637            }
638        };
639        let confirmed_missing = existing_bucket_info.is_none();
640        let existing_metadata = if opts.force_create && !confirmed_missing && !is_meta_bucketname(bucket) {
641            let (mut metadata, persisted) = metadata_sys::get_config_from_disk_with_presence_in(&self.ctx, bucket).await?;
642            if !persisted {
643                metadata = BucketMetadata::new(bucket);
644                metadata.created = existing_bucket_info
645                    .as_ref()
646                    .and_then(|info| info.created)
647                    .unwrap_or(OffsetDateTime::UNIX_EPOCH);
648                // A sidecar left by an interrupted legacy migration is the
649                // bucket's published incarnation; keep it rather than minting
650                // one that would invalidate fences taken on the stored value.
651                // A retired incarnation is residue of a deleted bucket and
652                // must not come back.
653                let store = metadata_sys::object_store_in(&self.ctx).await?;
654                if let Some(stored) = crate::bucket::metadata::load_bucket_incarnation(store.clone(), bucket).await?
655                    && !crate::bucket::retirement::is_retired(store, bucket, stored).await?
656                {
657                    metadata.bucket_incarnation_id = stored;
658                }
659            } else if !metadata.bucket_incarnation_sidecar && !metadata.bucket_incarnation_id.is_nil() {
660                return Err(Error::other(format!(
661                    "bucket incarnation sidecar is missing for new-format metadata: {bucket}"
662                )));
663            } else if !metadata.bucket_incarnation_sidecar {
664                metadata.bucket_incarnation_id = Uuid::new_v4();
665            }
666            Some(metadata)
667        } else {
668            None
669        };
670
671        let mut meta = existing_metadata.unwrap_or_else(|| {
672            if confirmed_missing && !is_meta_bucketname(bucket) {
673                BucketMetadata::new_with_default_durability(bucket)
674            } else {
675                BucketMetadata::new(bucket)
676            }
677        });
678        let existing_incarnation_is_authoritative = meta.bucket_incarnation_sidecar;
679        if confirmed_missing || is_meta_bucketname(bucket) {
680            meta.set_created(opts.created_at);
681
682            if opts.versioning_enabled {
683                meta.versioning_config_xml =
684                    crate::bucket::utils::serialize::<VersioningConfiguration>(&ENABLED_VERSIONING_CONFIG)?;
685            }
686        }
687
688        // Object Lock enable is one-way, and it must apply to an existing bucket
689        // too. Site replication replays make-with-versioning carrying the
690        // source's lockEnabled against a destination bucket that already exists;
691        // gating this on `confirmed_missing` returned success while leaving the
692        // replica unlocked, so replicated versions could be deleted without the
693        // retention the source enforces.
694        let lock_newly_enabled = opts.lock_enabled && !meta.lock_enabled;
695        if opts.lock_enabled {
696            meta.lock_enabled = true;
697            meta.object_lock_config_xml = crate::bucket::utils::serialize(&*crate::store::ENABLED_OBJECT_LOCK_CONFIG)?;
698            meta.versioning_config_xml = crate::bucket::utils::serialize::<VersioningConfiguration>(&ENABLED_VERSIONING_CONFIG)?;
699        }
700
701        let metadata_persisted_before_physical = confirmed_missing && !is_meta_bucketname(bucket) && opts.lock_enabled;
702        if metadata_persisted_before_physical {
703            metadata_sys::set_new_bucket_metadata_intent_in(&self.ctx, meta.clone()).await?;
704            if bucket_lifecycle_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
705                || metadata_transaction_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
706                || ns_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
707            {
708                return Err(Error::other(format!("bucket metadata creation intent lock was lost: {bucket}")));
709            }
710        }
711
712        if confirmed_missing && !is_meta_bucketname(bucket) {
713            // Fence every scanner cycle that could have observed the namespace
714            // before physical creation. Creation may become visible even when a
715            // later metadata write or namespace-lock check fails.
716            crate::store::list_objects::observe_scanner_namespace_mutations(bucket, 1);
717            self.cleanup_bucket_usage(bucket, ns_guard.as_ref()).await?;
718        }
719
720        if let Err(err) = await_bucket_lifecycle_operation(
721            bucket_lifecycle_guard.as_ref(),
722            ns_guard.as_ref(),
723            bucket,
724            "physical bucket creation",
725            await_bucket_namespace_operation(
726                metadata_transaction_guard.as_ref(),
727                bucket,
728                "bucket creation metadata transaction",
729                async {
730                    self.make_bucket_on_sets(bucket, opts)
731                        .await
732                        .map_err(|err| to_object_err(err, vec![bucket]))
733                },
734            ),
735        )
736        .await
737        {
738            if is_err_bucket_exists(&err)
739                && let Err(heal_err) = self
740                    .handle_heal_bucket(
741                        bucket,
742                        &HealOpts {
743                            recreate: true,
744                            ..Default::default()
745                        },
746                    )
747                    .await
748            {
749                warn!("best-effort bucket heal after BucketExists failed: {heal_err}");
750            }
751            if confirmed_missing && !is_err_bucket_exists(&err) && ns_guard.as_ref().is_none_or(|guard| !guard.is_lock_lost()) {
752                error!("make bucket failed: {err}");
753                self.rollback_failed_bucket_creation(bucket, bucket_lifecycle_guard.as_ref(), ns_guard.as_ref())
754                    .await;
755            }
756            return Err(err);
757        };
758
759        let metadata_result = async {
760            if metadata_persisted_before_physical {
761                return metadata_sys::commit_bucket_metadata_in(&self.ctx, meta).await;
762            }
763            if is_meta_bucketname(bucket) {
764                metadata_sys::set_bucket_metadata_in(&self.ctx, meta).await
765            } else if existing_incarnation_is_authoritative && !lock_newly_enabled {
766                metadata_sys::commit_bucket_metadata_in(&self.ctx, meta).await
767            } else {
768                metadata_sys::set_new_bucket_metadata_in(&self.ctx, meta).await
769            }
770        }
771        .await;
772        let metadata_lock_lost = bucket_lifecycle_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
773            || metadata_transaction_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
774            || ns_guard.as_ref().is_some_and(|guard| guard.is_lock_lost());
775        if let Err(err) = metadata_result {
776            if !metadata_lock_lost {
777                self.rollback_failed_bucket_creation(bucket, bucket_lifecycle_guard.as_ref(), ns_guard.as_ref())
778                    .await;
779            }
780            return Err(err);
781        }
782        if metadata_lock_lost {
783            return Err(Error::other(format!("bucket metadata initialization lock was lost: {bucket}")));
784        }
785
786        if confirmed_missing && !is_meta_bucketname(bucket) {
787            // A scanner may have sampled the first fence before the bucket
788            // became visible. Fence again after metadata initialization so
789            // that snapshot cannot publish as complete.
790            crate::store::list_objects::observe_scanner_namespace_mutations(bucket, 1);
791        }
792
793        Ok(())
794    }
795
796    /// Prove a live bucket generation before repairing missing expansion volumes.
797    /// Unlike request validation, repair only needs one erasure set to confirm
798    /// existence; an incomplete expansion set is precisely what repair fixes.
799    /// Callers must hold the bucket namespace lock through the subsequent heal.
800    pub(crate) async fn bucket_exists_for_heal(&self, bucket: &str) -> Result<bool> {
801        let results = futures::future::join_all(
802            self.bucket_sets()
803                .map(|(_, _, set)| async move { set.get_bucket_info(bucket, &BucketOptions::default()).await }),
804        )
805        .await;
806        let mut first_error = None;
807        for result in results {
808            match result {
809                Ok(_) => return Ok(true),
810                Err(err) if is_err_strict_volume_not_found(&err) => {}
811                Err(err) if first_error.is_none() => first_error = Some(err),
812                Err(_) => {}
813            }
814        }
815        match first_error {
816            Some(err) => Err(err),
817            None => Ok(false),
818        }
819    }
820
821    #[instrument(skip(self))]
822    pub(crate) async fn get_bucket_info_from_sets(&self, bucket: &str, opts: &BucketOptions) -> Result<BucketInfo> {
823        self.get_bucket_info_from_sets_with_quorum(bucket, opts, BucketInfoQuorum::Write)
824            .await
825    }
826
827    pub(crate) async fn get_bucket_info_from_sets_at_read_quorum(
828        &self,
829        bucket: &str,
830        opts: &BucketOptions,
831    ) -> Result<BucketInfo> {
832        self.get_bucket_info_from_sets_with_quorum(bucket, opts, BucketInfoQuorum::Read)
833            .await
834    }
835
836    async fn get_bucket_info_from_sets_with_quorum(
837        &self,
838        bucket: &str,
839        opts: &BucketOptions,
840        quorum: BucketInfoQuorum,
841    ) -> Result<BucketInfo> {
842        // One host may participate in several pools after expansion. Resolve the
843        // namespace against each erasure set so disks from different pools can
844        // never be combined into one bucket quorum.
845        // Bucket validation is request-path IO. Keep the previous peer fanout's
846        // latency shape by probing every set concurrently; scanner listings use
847        // a separate bounded path below because they run continuously.
848        let mut scoped_results = futures::future::join_all(self.bucket_sets().map(|(pool_index, set_index, set)| async move {
849            let result = match quorum {
850                BucketInfoQuorum::Read => set.stat_bucket_with_quorum(bucket, quorum).await,
851                BucketInfoQuorum::Write => set.get_bucket_info(bucket, opts).await,
852            };
853            (pool_index, set_index, result)
854        }))
855        .await;
856        scoped_results.sort_unstable_by_key(|(pool_index, set_index, _)| (*pool_index, *set_index));
857
858        let mut first_info = None;
859        let mut first_error = None;
860        for (_, _, result) in scoped_results {
861            match result {
862                Ok(info) if first_info.is_none() => first_info = Some(info),
863                Ok(_) => {}
864                Err(err) if is_err_strict_volume_not_found(&err) => {}
865                Err(err) if first_error.is_none() => first_error = Some(err),
866                Err(_) => {}
867            }
868        }
869
870        if let Some(err) = first_error {
871            return Err(err);
872        }
873
874        first_info.ok_or(Error::VolumeNotFound)
875    }
876
877    #[instrument(skip(self))]
878    pub(super) async fn handle_get_bucket_info(&self, bucket: &str, opts: &BucketOptions) -> Result<BucketInfo> {
879        let mut info = match self.get_bucket_info_from_sets(bucket, opts).await {
880            Ok(info) => info,
881            Err(Error::ErasureWriteQuorum) => return self.get_bucket_info_at_read_quorum(bucket, opts).await,
882            Err(err) => return Err(err),
883        };
884
885        if let Ok(sys) = metadata_sys::get_in(&self.ctx, bucket).await {
886            if should_override_created_from_metadata(sys.created) {
887                info.created = Some(sys.created);
888            }
889            info.versioning = sys.versioning();
890            info.object_locking = sys.object_locking();
891        }
892
893        Ok(info)
894    }
895
896    async fn get_bucket_info_at_read_quorum(&self, bucket: &str, opts: &BucketOptions) -> Result<BucketInfo> {
897        // Lock order: bucket lifecycle -> internal metadata object read locks.
898        // Keep create/delete from changing the namespace while a read quorum
899        // confirms both physical presence and persisted bucket metadata.
900        let guard = self.acquire_bucket_lifecycle_read_lock(bucket).await?;
901        await_bucket_namespace_operation(Some(&guard), bucket, "bucket read quorum validation", async {
902            let mut info = self
903                .get_bucket_info_from_sets_with_quorum(bucket, opts, BucketInfoQuorum::Read)
904                .await?;
905            let (metadata, persisted) = metadata_sys::get_config_from_disk_with_presence_in(&self.ctx, bucket).await?;
906            if !persisted {
907                // A minority of directories left by failed creation is not an
908                // authoritative bucket. Never turn fabricated defaults into
909                // permission to serve degraded reads.
910                return Err(Error::ErasureReadQuorum);
911            }
912            if metadata.lock_enabled && !metadata.bucket_creation_committed {
913                return Err(Error::ErasureWriteQuorum);
914            }
915            if metadata.name != bucket {
916                return Err(Error::FileCorrupt);
917            }
918            if should_override_created_from_metadata(metadata.created) {
919                info.created = Some(metadata.created);
920            }
921            info.versioning = metadata.versioning();
922            info.object_locking = metadata.object_locking();
923            Ok(info)
924        })
925        .await
926    }
927
928    #[instrument(skip(self))]
929    pub(super) async fn handle_list_bucket(&self, opts: &BucketOptions) -> Result<Vec<BucketInfo>> {
930        // TODO(backlog): support cached bucket listing via opts.cached
931        Ok(self.list_bucket_from_sets(opts, usize::MAX).await?.buckets)
932    }
933
934    pub async fn list_bucket_for_scanner(&self, opts: &BucketOptions) -> Result<crate::cluster::rpc::ScannerBucketListing> {
935        self.list_bucket_from_sets(opts, SCANNER_BUCKET_LIST_SET_CONCURRENCY).await
936    }
937
938    async fn list_bucket_from_sets(
939        &self,
940        opts: &BucketOptions,
941        max_concurrency: usize,
942    ) -> Result<crate::cluster::rpc::ScannerBucketListing> {
943        let set_count = self.pools.iter().map(|pool| pool.disk_set.len()).sum();
944        let concurrency = bucket_list_set_concurrency(set_count, max_concurrency);
945        let deleted = opts.deleted;
946        let cached = opts.cached;
947        let no_metadata = opts.no_metadata;
948        let mut set_listings = stream::iter(self.bucket_sets().map(move |(pool_index, set_index, set)| {
949            let opts = BucketOptions {
950                deleted,
951                cached,
952                no_metadata,
953            };
954            async move {
955                set.list_bucket_for_scanner(&opts)
956                    .await
957                    .map(|(buckets, complete)| (pool_index, set_index, buckets, complete))
958            }
959        }))
960        .buffer_unordered(concurrency);
961        let mut topology_complete = set_count != 0;
962        let mut bucket_map = BTreeMap::<String, BucketInfo>::new();
963        let mut scoped_buckets = Vec::with_capacity(set_count);
964        while let Some(set_listing) = set_listings.next().await {
965            let (pool_index, set_index, buckets, set_complete) = set_listing?;
966            topology_complete &= set_complete;
967            for bucket in &buckets {
968                bucket_map.entry(bucket.name.clone()).or_insert_with(|| bucket.clone());
969            }
970            scoped_buckets.push(crate::cluster::rpc::ScannerSetBucketListing {
971                pool_index,
972                set_index,
973                buckets,
974            });
975        }
976        scoped_buckets.sort_unstable_by_key(|scope| (scope.pool_index, scope.set_index));
977        let mut listing = crate::cluster::rpc::ScannerBucketListing {
978            buckets: bucket_map.into_values().collect(),
979            set_buckets: scoped_buckets,
980            topology_complete,
981        };
982
983        if !opts.no_metadata {
984            for bucket in &mut listing.buckets {
985                if let Ok(created) = metadata_sys::created_at_in(&self.ctx, &bucket.name).await
986                    && should_override_created_from_metadata(created)
987                {
988                    bucket.created = Some(created);
989                }
990            }
991            let created_by_bucket = listing
992                .buckets
993                .iter()
994                .map(|bucket| (bucket.name.as_str(), bucket.created))
995                .collect::<BTreeMap<_, _>>();
996            for scope in &mut listing.set_buckets {
997                for bucket in &mut scope.buckets {
998                    if let Some(created) = created_by_bucket.get(bucket.name.as_str()) {
999                        bucket.created = *created;
1000                    }
1001                }
1002            }
1003        }
1004
1005        Ok(listing)
1006    }
1007
1008    #[instrument(skip(self))]
1009    pub(super) async fn handle_delete_bucket(&self, bucket: &str, opts: &DeleteBucketOptions) -> Result<()> {
1010        self.handle_delete_bucket_with_diagnostic_budget(bucket, opts, BucketDeleteDiagnosticBudget::new())
1011            .await
1012    }
1013
1014    async fn handle_delete_bucket_with_diagnostic_budget(
1015        &self,
1016        bucket: &str,
1017        opts: &DeleteBucketOptions,
1018        mut diagnostic_budget: BucketDeleteDiagnosticBudget,
1019    ) -> Result<()> {
1020        if is_meta_bucketname(bucket) {
1021            return Err(StorageError::BucketNameInvalid(bucket.to_string()));
1022        }
1023
1024        if let Err(err) = check_valid_bucket_name(bucket) {
1025            return Err(StorageError::BucketNameInvalid(err.to_string()));
1026        }
1027
1028        let bucket_lifecycle_guard = if !opts.no_lock {
1029            Some(self.acquire_bucket_lifecycle_write_lock(bucket).await?)
1030        } else {
1031            None
1032        };
1033        let ns_guard = if !opts.no_lock {
1034            let ns_lock = self.new_ns_lock(bucket, bucket).await?;
1035            Some(
1036                await_bucket_namespace_operation(
1037                    bucket_lifecycle_guard.as_ref(),
1038                    bucket,
1039                    "bucket namespace lock acquisition",
1040                    async {
1041                        ns_lock.get_write_lock(get_lock_acquire_timeout()).await.map_err(|e| match e {
1042                            rustfs_lock::error::LockError::QuorumNotReached { required, achieved } => {
1043                                StorageError::NamespaceLockQuorumUnavailable {
1044                                    mode: "write",
1045                                    bucket: bucket.to_string(),
1046                                    object: bucket.to_string(),
1047                                    required,
1048                                    achieved,
1049                                }
1050                            }
1051                            other => StorageError::Lock(other),
1052                        })
1053                    },
1054                )
1055                .await?,
1056            )
1057        } else {
1058            None
1059        };
1060
1061        let sr_mark_delete = opts.srdelete_op == SRBucketDeleteOp::MarkDelete;
1062        let sr_purge = opts.srdelete_op == SRBucketDeleteOp::Purge;
1063        let sr_delete = sr_mark_delete || sr_purge;
1064        let mut delete_opts = opts.clone();
1065        let bucket_exists = match self.get_bucket_info_from_sets(bucket, &BucketOptions::default()).await {
1066            Ok(_) => true,
1067            Err(err) => {
1068                if is_err_strict_volume_not_found(&err) && sr_delete {
1069                    false
1070                } else if is_err_strict_volume_not_found(&err) {
1071                    return Err(StorageError::BucketNotFound(bucket.to_string()));
1072                } else {
1073                    return Err(to_object_err(err, vec![bucket]));
1074                }
1075            }
1076        };
1077        if bucket_exists {
1078            validate_table_bucket_delete_guard(&self.ctx, bucket).await?;
1079
1080            if !opts.force {
1081                if let Some(blocker) = bucket_delete_local_blocker(&self.ctx, bucket, &mut diagnostic_budget).await? {
1082                    return Err(blocker);
1083                }
1084                delete_opts.force_if_empty = true;
1085            }
1086        }
1087
1088        if sr_delete && !bucket_exists {
1089            delete_opts.force_if_empty = true;
1090        }
1091
1092        #[cfg(test)]
1093        crate::cluster::rpc::peer_s3_client::pause_after_delete_bucket_empty_scan().await;
1094
1095        if sr_mark_delete {
1096            await_bucket_lifecycle_operation(
1097                bucket_lifecycle_guard.as_ref(),
1098                ns_guard.as_ref(),
1099                bucket,
1100                "bucket delete marker creation",
1101                self.mark_bucket_deleted(bucket),
1102            )
1103            .await?;
1104        }
1105
1106        // Capture the authoritative old identity before its namespace disappears.
1107        // Legacy buckets without a stamp cannot produce retirement authority.
1108        let retirement = if bucket_exists && bucket_lifecycle_guard.is_some() && !is_meta_bucketname(bucket) {
1109            if let Some(store) = metadata_sys::object_store_if_initialized_in(&self.ctx).await {
1110                crate::bucket::metadata::load_bucket_incarnation(store.clone(), bucket)
1111                    .await?
1112                    .map(|incarnation| (store, incarnation))
1113            } else {
1114                None
1115            }
1116        } else {
1117            None
1118        };
1119
1120        let delete_result = await_bucket_namespace_operation(
1121            bucket_lifecycle_guard.as_ref(),
1122            bucket,
1123            "physical bucket deletion",
1124            run_physical_bucket_deletion(ns_guard.as_ref(), bucket, async {
1125                self.delete_bucket_on_sets(bucket, &delete_opts)
1126                    .await
1127                    .map_err(|err| to_object_err(err, vec![bucket]))
1128            }),
1129        )
1130        .await;
1131        if let Err(err) = delete_result
1132            && (!sr_delete || !is_err_strict_volume_not_found(&err))
1133        {
1134            if delete_opts.force_if_empty && matches!(&err, StorageError::BucketNotEmpty(_)) {
1135                let mut diagnostic_budget = BucketDeleteDiagnosticBudget::new();
1136                if let Some(blocker) = bucket_delete_local_blocker(&self.ctx, bucket, &mut diagnostic_budget).await? {
1137                    return Err(blocker);
1138                }
1139            }
1140            return Err(err);
1141        }
1142
1143        if let Some((store, incarnation)) = retirement {
1144            let mut record_opts = ObjectOptions {
1145                max_parity: true,
1146                ..Default::default()
1147            };
1148            if let Some(guard) = bucket_lifecycle_guard.as_ref() {
1149                record_opts.add_bucket_lifecycle_lock_guard(guard);
1150            }
1151            if let Some(guard) = ns_guard.as_ref() {
1152                record_opts.add_namespace_lock_guard(guard);
1153            }
1154            await_bucket_lifecycle_operation(
1155                bucket_lifecycle_guard.as_ref(),
1156                ns_guard.as_ref(),
1157                bucket,
1158                "bucket retirement publication",
1159                crate::bucket::retirement::commit_retirement(store, bucket, incarnation, &record_opts),
1160            )
1161            .await?;
1162        }
1163
1164        self.cleanup_bucket_usage_best_effort(bucket, ns_guard.as_ref()).await;
1165
1166        if let Err(err) = self
1167            .cleanup_deleted_bucket_metadata(bucket, sr_purge, ns_guard.as_ref())
1168            .await
1169        {
1170            warn!(
1171                bucket = %bucket,
1172                error = ?err,
1173                "physical bucket deletion succeeded but metadata cleanup remains pending"
1174            );
1175        }
1176        // A scanner may have sampled the first fence before the physical
1177        // namespace disappeared. The completion fence invalidates that scan.
1178        crate::store::list_objects::observe_scanner_namespace_mutations(bucket, 1);
1179        Ok(())
1180    }
1181}
1182
1183#[cfg(test)]
1184mod tests {
1185    use super::{
1186        BUCKET_DELETE_DIAGNOSTIC_MAX_ELAPSED, BUCKET_DELETE_DIAGNOSTIC_MAX_ENTRIES, BUCKET_DELETE_XLMETA_DIAGNOSTIC_MAX_BYTES,
1187        BucketDeleteBlockerKind, BucketDeleteDiagnosticBudget, BucketMetadataLessResidue, SCANNER_BUCKET_LIST_SET_CONCURRENCY,
1188        await_bucket_namespace_operation, bucket_delete_metadata_cleanup_prefixes, bucket_deleted_marker_prefix,
1189        bucket_deleted_marker_volume, bucket_list_set_concurrency, record_bucket_delete_blocker, run_bucket_usage_cleanup,
1190        run_physical_bucket_deletion, scan_metadata_less_residue, scan_metadata_less_residue_with_budget,
1191        should_override_created_from_metadata, validate_table_bucket_delete_allowed,
1192    };
1193    use crate::bucket::metadata::{BucketMetadata, table_bucket_catalog_metadata_prefix};
1194    use crate::bucket::metadata_sys;
1195    use crate::cluster::rpc::peer_s3_client::install_delete_bucket_empty_scan_barrier;
1196    use crate::disk::{BUCKET_META_PREFIX, DiskAPI, RUSTFS_META_BUCKET, STORAGE_FORMAT_FILE};
1197    use crate::error::StorageError;
1198    use crate::object_api::{ObjectOptions, PutObjReader};
1199    use crate::runtime::instance::InstanceContext;
1200    use crate::storage_api_contracts::{
1201        bucket::{BucketOperations as _, BucketOptions, DeleteBucketOptions, MakeBucketOptions, SRBucketDeleteOp},
1202        list::ListOperations as _,
1203        namespace::NamespaceLocking as _,
1204        object::{ObjectIO as _, ObjectOperations as _},
1205    };
1206    use crate::store::{ECStore, init_local_disks_with_instance_ctx};
1207    use crate::{
1208        disk::endpoint::Endpoint,
1209        layout::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints},
1210    };
1211    use rustfs_data_usage::{BucketUsageInfo, DATA_USAGE_OBJECT_NAME, DataUsageInfo};
1212    use rustfs_filemeta::{FileInfo, FileMeta, TRANSITION_COMPLETE};
1213    use rustfs_lock::{LocalClient, LockRequest, LockType, NamespaceLock, ObjectKey};
1214    use serial_test::serial;
1215    use std::path::{Path, PathBuf};
1216    use std::sync::Arc;
1217    use std::sync::atomic::{AtomicBool, Ordering};
1218    use std::time::{Duration, SystemTime};
1219    use time::OffsetDateTime;
1220    use tokio::io::AsyncReadExt;
1221    use tokio::sync::{Notify, OnceCell};
1222    use tokio_util::sync::CancellationToken;
1223    use uuid::Uuid;
1224
1225    static BUCKET_DELETE_TEST_ENV: OnceCell<(Vec<PathBuf>, Arc<ECStore>)> = OnceCell::const_new();
1226
1227    #[tokio::test(start_paused = true)]
1228    async fn bucket_delete_diagnostic_budget_starts_with_first_scan_io_and_latches_once() {
1229        let mut budget = BucketDeleteDiagnosticBudget::with_limits(8, Duration::from_millis(100));
1230        tokio::time::advance(Duration::from_secs(10)).await;
1231
1232        let first_polled = Arc::new(AtomicBool::new(false));
1233        let first_polled_for_io = first_polled.clone();
1234        let first = budget
1235            .run_io(async move {
1236                first_polled_for_io.store(true, Ordering::SeqCst);
1237                Ok::<_, std::io::Error>(7_u8)
1238            })
1239            .await
1240            .expect("the first diagnostic IO should succeed");
1241        assert_eq!(first, Some(7));
1242        assert!(first_polled.load(Ordering::SeqCst));
1243
1244        tokio::time::advance(Duration::from_millis(101)).await;
1245        let expired_polled = Arc::new(AtomicBool::new(false));
1246        let expired_polled_for_io = expired_polled.clone();
1247        let expired = budget
1248            .run_io(async move {
1249                expired_polled_for_io.store(true, Ordering::SeqCst);
1250                Ok::<_, std::io::Error>(9_u8)
1251            })
1252            .await
1253            .expect("an expired diagnostic budget should not become an IO error");
1254        assert_eq!(expired, None);
1255        assert!(
1256            !expired_polled.load(Ordering::SeqCst),
1257            "the deadline must remain latched after the first scan IO"
1258        );
1259    }
1260
1261    #[tokio::test(start_paused = true)]
1262    async fn bucket_delete_diagnostic_budget_times_out_its_first_pending_io() {
1263        let mut budget = BucketDeleteDiagnosticBudget::with_limits(8, Duration::from_millis(100));
1264        let io_polled = Arc::new(AtomicBool::new(false));
1265        let io_polled_for_future = io_polled.clone();
1266
1267        let result = budget
1268            .run_io(std::future::poll_fn(move |_cx| {
1269                io_polled_for_future.store(true, Ordering::SeqCst);
1270                std::task::Poll::<std::io::Result<()>>::Pending
1271            }))
1272            .await
1273            .expect("a diagnostic timeout should fail closed without an IO error");
1274
1275        assert_eq!(result, None);
1276        assert!(
1277            io_polled.load(Ordering::SeqCst),
1278            "the first diagnostic IO must be polled before its timeout"
1279        );
1280    }
1281
1282    #[tokio::test(start_paused = true)]
1283    async fn delayed_metadata_less_scans_still_detect_orphans_and_xlmeta() {
1284        let root = tempfile::tempdir().expect("temporary delayed-scan roots should be created");
1285        let orphan_root = root.path().join("orphan-root");
1286        let xlmeta_root = root.path().join("xlmeta-root");
1287        std::fs::create_dir_all(&orphan_root).expect("orphan root should be created");
1288        std::fs::create_dir_all(&xlmeta_root).expect("xlmeta root should be created");
1289        std::fs::write(orphan_root.join("orphan-part"), b"orphan").expect("orphan fixture should be written");
1290        std::fs::write(xlmeta_root.join(STORAGE_FORMAT_FILE), b"invalid-xlmeta").expect("xl.meta fixture should be written");
1291
1292        let mut orphan_budget = BucketDeleteDiagnosticBudget::with_limits(16, Duration::from_secs(5));
1293        let mut xlmeta_budget = BucketDeleteDiagnosticBudget::with_limits(16, Duration::from_secs(5));
1294        tokio::time::advance(Duration::from_secs(60)).await;
1295
1296        let orphan = scan_metadata_less_residue_with_budget(&orphan_root, &mut orphan_budget)
1297            .await
1298            .expect("delayed orphan scan should complete");
1299        assert!(orphan.has_residue_without_xlmeta());
1300        assert!(!orphan.diagnostic_truncated);
1301
1302        let xlmeta = scan_metadata_less_residue_with_budget(&xlmeta_root, &mut xlmeta_budget)
1303            .await
1304            .expect("delayed xl.meta scan should complete");
1305        assert!(xlmeta.xlmeta_found);
1306        assert!(!xlmeta.diagnostic_truncated);
1307    }
1308
1309    #[tokio::test(start_paused = true)]
1310    async fn bucket_namespace_operation_fails_closed_after_lease_expiry() {
1311        let ttl = Duration::from_millis(20);
1312        let lock = NamespaceLock::new("bucket-operation-test".to_string(), Arc::new(LocalClient::new()));
1313        let request = LockRequest::new(ObjectKey::new("bucket", ""), LockType::Exclusive, "test-owner")
1314            .with_acquire_timeout(Duration::from_secs(1))
1315            .with_ttl(ttl)
1316            .with_refresh_interval(ttl);
1317        let guard = lock
1318            .acquire_guard(&request)
1319            .await
1320            .expect("namespace lock acquisition should not fail")
1321            .expect("namespace lock should be acquired");
1322        tokio::time::advance(ttl + Duration::from_millis(1)).await;
1323
1324        let operation_ran = Arc::new(AtomicBool::new(false));
1325        let operation_ran_for_future = operation_ran.clone();
1326        let result = await_bucket_namespace_operation(Some(&guard), "bucket", "test operation", async move {
1327            operation_ran_for_future.store(true, Ordering::SeqCst);
1328            Ok(())
1329        })
1330        .await;
1331
1332        assert!(result.is_err(), "an expired namespace lease must fence the operation");
1333        assert!(
1334            !operation_ran.load(Ordering::SeqCst),
1335            "a fenced namespace operation must not poll its mutation future"
1336        );
1337    }
1338
1339    #[tokio::test]
1340    #[serial]
1341    async fn physical_bucket_delete_fences_scanner_before_polling_storage() {
1342        let generation_before = crate::store::list_objects::scanner_namespace_mutation_generation();
1343        let storage_polled = Arc::new(AtomicBool::new(false));
1344        let storage_polled_for_future = storage_polled.clone();
1345
1346        run_physical_bucket_deletion(None, "generation-order-bucket", async move {
1347            assert!(
1348                crate::store::list_objects::scanner_namespace_mutation_generation() > generation_before,
1349                "scanner generation must advance before physical deletion is polled"
1350            );
1351            storage_polled_for_future.store(true, Ordering::SeqCst);
1352            Ok(())
1353        })
1354        .await
1355        .expect("synthetic physical deletion should succeed");
1356
1357        assert!(storage_polled.load(Ordering::SeqCst));
1358    }
1359
1360    #[tokio::test]
1361    async fn bucket_usage_cleanup_stops_after_parent_cancellation() {
1362        let started = Arc::new(Notify::new());
1363        let started_wait = started.notified();
1364        let release = Arc::new(Notify::new());
1365        let completed = Arc::new(AtomicBool::new(false));
1366        let started_for_cleanup = started.clone();
1367        let release_for_cleanup = release.clone();
1368        let completed_for_cleanup = completed.clone();
1369        let parent = tokio::spawn(run_bucket_usage_cleanup(None, "bucket", async move {
1370            started_for_cleanup.notify_one();
1371            release_for_cleanup.notified().await;
1372            completed_for_cleanup.store(true, Ordering::SeqCst);
1373            Ok(())
1374        }));
1375        started_wait.await;
1376
1377        parent.abort();
1378        let _ = parent.await;
1379        tokio::task::yield_now().await;
1380        release.notify_waiters();
1381        tokio::task::yield_now().await;
1382        assert!(
1383            !completed.load(Ordering::SeqCst),
1384            "a cancelled cleanup future must not continue in a detached task"
1385        );
1386    }
1387
1388    #[tokio::test(start_paused = true)]
1389    #[serial]
1390    async fn bucket_namespace_operation_stops_in_flight_work_after_lock_loss() {
1391        let ttl = Duration::from_millis(20);
1392        let lock = NamespaceLock::new("bucket-operation-in-flight-loss-test".to_string(), Arc::new(LocalClient::new()));
1393        let request = LockRequest::new(ObjectKey::new("bucket", ""), LockType::Exclusive, "test-owner")
1394            .with_acquire_timeout(Duration::from_secs(1))
1395            .with_ttl(ttl)
1396            .with_refresh_interval(ttl);
1397        let guard = lock
1398            .acquire_guard(&request)
1399            .await
1400            .expect("namespace lock acquisition should not fail")
1401            .expect("namespace lock should be acquired");
1402
1403        let operation_started = Arc::new(Notify::new());
1404        let started_wait = operation_started.notified();
1405        let operation_release = Arc::new(Notify::new());
1406        let operation_completed = Arc::new(AtomicBool::new(false));
1407        let started_for_operation = operation_started.clone();
1408        let release_for_operation = operation_release.clone();
1409        let completed_for_operation = operation_completed.clone();
1410        let task = tokio::spawn(async move {
1411            await_bucket_namespace_operation(Some(&guard), "bucket", "test operation", async move {
1412                started_for_operation.notify_one();
1413                release_for_operation.notified().await;
1414                completed_for_operation.store(true, Ordering::SeqCst);
1415                Ok(())
1416            })
1417            .await
1418        });
1419        started_wait.await;
1420
1421        tokio::time::advance(ttl + Duration::from_millis(1)).await;
1422        let err = task
1423            .await
1424            .expect("operation task should join")
1425            .expect_err("an operation still running after lease loss must be fenced");
1426        assert!(err.to_string().contains("namespace lock was lost during test operation"));
1427        operation_release.notify_waiters();
1428        tokio::task::yield_now().await;
1429        assert!(
1430            !operation_completed.load(Ordering::SeqCst),
1431            "lock loss must stop the old owner before a successor can acquire the namespace"
1432        );
1433    }
1434
1435    async fn setup_bucket_delete_test_env() -> (Vec<PathBuf>, Arc<ECStore>) {
1436        BUCKET_DELETE_TEST_ENV
1437            .get_or_init(|| async {
1438                let temp_dir = std::env::temp_dir().join(format!("rustfs_bucket_delete_test_{}", Uuid::new_v4()));
1439                tokio::fs::create_dir_all(&temp_dir)
1440                    .await
1441                    .expect("test base directory should be created");
1442
1443                let disk_paths = (0..4)
1444                    .map(|disk_idx| temp_dir.join(format!("disk{disk_idx}")))
1445                    .collect::<Vec<_>>();
1446
1447                for disk_path in &disk_paths {
1448                    tokio::fs::create_dir_all(disk_path)
1449                        .await
1450                        .expect("disk directory should be created");
1451                }
1452
1453                let mut endpoints = Vec::with_capacity(disk_paths.len());
1454                for (disk_idx, disk_path) in disk_paths.iter().enumerate() {
1455                    let mut endpoint =
1456                        Endpoint::try_from(disk_path.to_str().expect("disk path should be utf8")).expect("endpoint should parse");
1457                    endpoint.set_pool_index(0);
1458                    endpoint.set_set_index(0);
1459                    endpoint.set_disk_index(disk_idx);
1460                    endpoints.push(endpoint);
1461                }
1462
1463                let endpoint_pools = EndpointServerPools(vec![PoolEndpoints {
1464                    legacy: false,
1465                    set_count: 1,
1466                    drives_per_set: 4,
1467                    endpoints: Endpoints::from(endpoints),
1468                    cmd_line: "bucket-delete-test".to_string(),
1469                    platform: format!("OS: {} | Arch: {}", std::env::consts::OS, std::env::consts::ARCH),
1470                }]);
1471
1472                let instance_ctx = Arc::new(InstanceContext::new());
1473                init_local_disks_with_instance_ctx(&instance_ctx, endpoint_pools.clone())
1474                    .await
1475                    .expect("local disks should initialize");
1476                let ecstore = ECStore::new_with_instance_ctx(
1477                    "127.0.0.1:0".parse().expect("test address"),
1478                    endpoint_pools,
1479                    CancellationToken::new(),
1480                    instance_ctx,
1481                )
1482                .await
1483                .expect("ECStore should initialize");
1484                let storage_class = crate::config::storageclass::lookup_config_for_pools_without_env(
1485                    &rustfs_config::server_config::KVS::new(),
1486                    &[4],
1487                )
1488                .expect("bucket test storage class should match its four-disk pool");
1489                for pool in &ecstore.pools {
1490                    for set in &pool.disk_set {
1491                        set.set_test_storage_class_config(storage_class.clone());
1492                    }
1493                }
1494
1495                metadata_sys::init_bucket_metadata_sys(ecstore.clone(), Vec::new()).await;
1496
1497                (disk_paths, ecstore)
1498            })
1499            .await
1500            .clone()
1501    }
1502
1503    async fn setup_multi_pool_bucket_test_env() -> (tempfile::TempDir, Arc<ECStore>) {
1504        setup_bucket_quorum_test_env(&[4, 4], None).await
1505    }
1506
1507    async fn setup_bucket_quorum_test_env(
1508        drives_per_pool: &[usize],
1509        standard_parity: Option<usize>,
1510    ) -> (tempfile::TempDir, Arc<ECStore>) {
1511        let temp_dir = tempfile::tempdir().expect("multi-pool bucket test directory should be created");
1512        let mut pools = Vec::new();
1513        for (pool_index, &drive_count) in drives_per_pool.iter().enumerate() {
1514            let mut endpoints = Vec::new();
1515            for disk_index in 0..drive_count {
1516                let disk_path = temp_dir.path().join(format!("pool{pool_index}-disk{disk_index}"));
1517                tokio::fs::create_dir_all(&disk_path)
1518                    .await
1519                    .expect("multi-pool bucket test disk should be created");
1520                let mut endpoint =
1521                    Endpoint::try_from(disk_path.to_str().expect("disk path should be utf8")).expect("endpoint should parse");
1522                endpoint.set_pool_index(pool_index);
1523                endpoint.set_set_index(0);
1524                endpoint.set_disk_index(disk_index);
1525                endpoints.push(endpoint);
1526            }
1527            pools.push(PoolEndpoints {
1528                legacy: false,
1529                set_count: 1,
1530                drives_per_set: drive_count,
1531                endpoints: Endpoints::from(endpoints),
1532                cmd_line: format!("bucket-test-pool-{pool_index}"),
1533                platform: format!("OS: {} | Arch: {}", std::env::consts::OS, std::env::consts::ARCH),
1534            });
1535        }
1536
1537        let endpoint_pools = EndpointServerPools(pools);
1538        let instance_ctx = Arc::new(InstanceContext::new());
1539        instance_ctx.set_endpoints(endpoint_pools.clone());
1540        init_local_disks_with_instance_ctx(&instance_ctx, endpoint_pools.clone())
1541            .await
1542            .expect("multi-pool local disks should initialize");
1543        let ecstore = ECStore::new_with_instance_ctx(
1544            "127.0.0.1:0".parse().expect("test address"),
1545            endpoint_pools,
1546            CancellationToken::new(),
1547            instance_ctx,
1548        )
1549        .await
1550        .expect("multi-pool ECStore should initialize");
1551        let mut storage_class_kvs = rustfs_config::server_config::KVS::new();
1552        if let Some(parity) = standard_parity {
1553            storage_class_kvs.insert(crate::config::storageclass::CLASS_STANDARD.to_string(), format!("EC:{parity}"));
1554        }
1555        let storage_class = crate::config::storageclass::lookup_config_for_pools_without_env(&storage_class_kvs, drives_per_pool)
1556            .expect("storage class should match every test erasure set");
1557        for pool in &ecstore.pools {
1558            for set in &pool.disk_set {
1559                set.set_test_storage_class_config(storage_class.clone());
1560            }
1561        }
1562
1563        (temp_dir, ecstore)
1564    }
1565
1566    async fn take_set_disks_offline(
1567        ecstore: &ECStore,
1568        set: &Arc<crate::set_disk::SetDisks>,
1569        disk_indexes: &[usize],
1570    ) -> Vec<(usize, crate::disk::DiskStore)> {
1571        let offline = {
1572            let disks = set.disks.read().await;
1573            disk_indexes
1574                .iter()
1575                .map(|index| (*index, disks[*index].clone().expect("fault-injection disk should start online")))
1576                .collect::<Vec<_>>()
1577        };
1578        for (_, disk) in &offline {
1579            disk.close().await.expect("fault injection should stop per-disk monitoring");
1580            disk.force_offline_for_test();
1581        }
1582        // Empty slots can be renewed from the still-present disk paths by the endpoint monitor.
1583        // Keep the original handles and prove reconnect cannot clear their injected IO failure.
1584        set.connect_disks().await;
1585        {
1586            let disks = set.disks.read().await;
1587            for (index, disk) in &offline {
1588                let current = disks[*index].as_ref().expect("fault-injection slot should remain populated");
1589                assert!(Arc::ptr_eq(current, disk), "reconnect must preserve the fault-injection handle");
1590                assert_eq!(disk.runtime_state(), crate::disk::health_state::RuntimeDriveHealthState::Offline);
1591            }
1592        }
1593        let local_disk_map = ecstore.ctx.local_disk_map();
1594        let mut local_disks = local_disk_map.write().await;
1595        for (_, disk) in &offline {
1596            local_disks.insert(disk.endpoint().to_string(), None);
1597        }
1598        offline
1599    }
1600
1601    async fn restore_set_disks(
1602        ecstore: &ECStore,
1603        set: &Arc<crate::set_disk::SetDisks>,
1604        offline: Vec<(usize, crate::disk::DiskStore)>,
1605    ) {
1606        {
1607            let local_disk_map = ecstore.ctx.local_disk_map();
1608            let mut local_disks = local_disk_map.write().await;
1609            for (_, disk) in &offline {
1610                local_disks.insert(disk.endpoint().to_string(), Some(Arc::clone(disk)));
1611            }
1612        }
1613        let disks = set.disks.read().await;
1614        for (index, disk) in offline {
1615            let current = disks[index].as_ref().expect("fault-injection slot should remain populated");
1616            assert!(Arc::ptr_eq(current, &disk), "fault-injection disk handle should remain unchanged");
1617            assert_eq!(disk.runtime_state(), crate::disk::health_state::RuntimeDriveHealthState::Offline);
1618            disk.reset_health_for_store_init_retry();
1619        }
1620    }
1621
1622    #[tokio::test]
1623    async fn request_metadata_methods_fail_closed_before_instance_initialization() {
1624        let (_temp_dir, store) = setup_multi_pool_bucket_test_env().await;
1625
1626        let expected = "bucket metadata sys not initialized for this instance";
1627        let errors = [
1628            store.get_bucket_metadata("bucket").await.unwrap_err(),
1629            store.get_bucket_policy("bucket").await.unwrap_err(),
1630            store.get_bucket_policy_raw("bucket").await.unwrap_err(),
1631            store.restricts_public_bucket_access("bucket").await.unwrap_err(),
1632            store
1633                .update_bucket_metadata_config("bucket", crate::bucket::metadata::BUCKET_POLICY_CONFIG, Vec::new())
1634                .await
1635                .unwrap_err(),
1636        ];
1637        for error in errors {
1638            assert_eq!(error.to_string(), format!("Io error: {expected}"));
1639        }
1640    }
1641
1642    async fn create_bucket_with_object(ecstore: &Arc<ECStore>, bucket: &str, object: &str) {
1643        let generation_before_make = ecstore.scanner_namespace_mutation_generation();
1644        ecstore
1645            .make_bucket(bucket, &MakeBucketOptions::default())
1646            .await
1647            .expect("bucket should be created");
1648        assert_eq!(
1649            ecstore.scanner_namespace_mutation_generation(),
1650            generation_before_make.saturating_add(2),
1651            "successful bucket creation should fence scanner namespace activity before and after creation"
1652        );
1653
1654        let generation_before_put = ecstore.scanner_namespace_mutation_generation();
1655        let mut reader = PutObjReader::from_vec(b"delete bucket semantics".to_vec());
1656        ecstore
1657            .put_object(bucket, object, &mut reader, &ObjectOptions::default())
1658            .await
1659            .expect("object should be written");
1660        let lock = ecstore.pools[0].disk_set[0]
1661            .new_ns_lock(bucket, object)
1662            .await
1663            .expect("fixture namespace lock should be created");
1664        drop(
1665            lock.get_write_lock(Duration::from_secs(30))
1666                .await
1667                .expect("fixture rename tail should finish before checking its generation"),
1668        );
1669        assert_eq!(
1670            ecstore.scanner_namespace_mutation_generation(),
1671            generation_before_put.saturating_add(3),
1672            "successful object creation must observe the logical mutation and both fanout boundaries"
1673        );
1674        ecstore
1675            .get_object_info(bucket, object, &ObjectOptions::default())
1676            .await
1677            .expect("object should be readable before bucket delete");
1678    }
1679
1680    async fn any_disk_path_exists(disk_paths: &[PathBuf], relative_path: impl AsRef<Path>) -> bool {
1681        for disk_path in disk_paths {
1682            if tokio::fs::try_exists(disk_path.join(relative_path.as_ref()))
1683                .await
1684                .expect("test disk path should be stat-able")
1685            {
1686                return true;
1687            }
1688        }
1689        false
1690    }
1691
1692    async fn any_disk_has_object_metadata(disk_paths: &[PathBuf], bucket: &str) -> bool {
1693        for disk_path in disk_paths {
1694            if super::has_xlmeta_files(&disk_path.join(bucket))
1695                .await
1696                .expect("object metadata scan should succeed")
1697            {
1698                return true;
1699            }
1700        }
1701        false
1702    }
1703
1704    async fn write_metadata_less_part_on_all_disks(disk_paths: &[PathBuf], bucket: &str, object: &str) {
1705        for disk_path in disk_paths {
1706            let data_dir = Uuid::new_v4();
1707            let part_path = disk_path.join(bucket).join(object).join(data_dir.to_string()).join("part.1");
1708            tokio::fs::create_dir_all(part_path.parent().expect("part path should have a parent"))
1709                .await
1710                .expect("metadata-less data dir should be created");
1711            tokio::fs::write(part_path, b"orphan shard")
1712                .await
1713                .expect("metadata-less part should be written");
1714        }
1715    }
1716
1717    async fn write_bucket_metadata_marker(disk_paths: &[PathBuf], metadata_prefix: &str) {
1718        for disk_path in disk_paths {
1719            let marker_path = disk_path.join(metadata_prefix).join("config.json");
1720            tokio::fs::create_dir_all(marker_path.parent().expect("metadata marker path should have a parent"))
1721                .await
1722                .expect("metadata marker parent should be created");
1723            tokio::fs::write(marker_path, b"bucket metadata")
1724                .await
1725                .expect("metadata marker should be written");
1726        }
1727    }
1728
1729    #[test]
1730    fn should_not_override_when_metadata_created_is_unix_epoch() {
1731        assert!(!should_override_created_from_metadata(OffsetDateTime::UNIX_EPOCH));
1732    }
1733
1734    #[test]
1735    fn should_override_when_metadata_created_is_valid_time() {
1736        let created = OffsetDateTime::from_unix_timestamp(1704067200).expect("valid timestamp");
1737        assert!(should_override_created_from_metadata(created));
1738    }
1739
1740    #[test]
1741    fn table_bucket_delete_guard_rejects_remaining_catalog_metadata() {
1742        let err = validate_table_bucket_delete_allowed("table-bucket", true, true).unwrap_err();
1743
1744        assert!(matches!(err, StorageError::BucketNotEmpty(bucket) if bucket == "table-bucket"));
1745        assert!(validate_table_bucket_delete_allowed("table-bucket", true, false).is_ok());
1746        assert!(validate_table_bucket_delete_allowed("regular-bucket", false, true).is_ok());
1747    }
1748
1749    #[test]
1750    fn bucket_delete_metadata_cleanup_removes_internal_table_catalog_prefix() {
1751        let prefixes = bucket_delete_metadata_cleanup_prefixes("analytics");
1752
1753        assert!(prefixes.contains(&table_bucket_catalog_metadata_prefix("analytics")));
1754        assert!(prefixes.contains(&"buckets/analytics".to_string()));
1755    }
1756
1757    #[test]
1758    fn bucket_delete_marker_path_uses_internal_deleted_bucket_metadata_prefix() {
1759        assert_eq!(bucket_deleted_marker_prefix("analytics"), "buckets/.deleted/analytics");
1760        assert_eq!(
1761            bucket_deleted_marker_volume("analytics"),
1762            format!("{RUSTFS_META_BUCKET}/{BUCKET_META_PREFIX}/.deleted/analytics")
1763        );
1764    }
1765
1766    #[test]
1767    fn bucket_listing_selects_request_and_scanner_fanout() {
1768        assert_eq!(bucket_list_set_concurrency(0, SCANNER_BUCKET_LIST_SET_CONCURRENCY), 1);
1769        assert_eq!(bucket_list_set_concurrency(2, SCANNER_BUCKET_LIST_SET_CONCURRENCY), 2);
1770        assert_eq!(
1771            bucket_list_set_concurrency(100, SCANNER_BUCKET_LIST_SET_CONCURRENCY),
1772            SCANNER_BUCKET_LIST_SET_CONCURRENCY
1773        );
1774        assert_eq!(bucket_list_set_concurrency(100, usize::MAX), 100);
1775    }
1776
1777    #[tokio::test]
1778    async fn metadata_less_residue_scan_ignores_empty_dirs_and_reports_uuid_data() {
1779        let root = tempfile::tempdir().expect("temporary bucket root should be created");
1780        let bucket_path = root.path().join("bucket");
1781        tokio::fs::create_dir_all(bucket_path.join("empty/child"))
1782            .await
1783            .expect("empty directory residue should be created");
1784
1785        let empty = scan_metadata_less_residue(&bucket_path)
1786            .await
1787            .expect("empty residue scan should succeed");
1788        assert!(!empty.has_residue_without_xlmeta());
1789
1790        let data_dir = Uuid::new_v4();
1791        let part_path = bucket_path.join("object").join(data_dir.to_string()).join("part.1");
1792        tokio::fs::create_dir_all(part_path.parent().expect("part path should have a parent"))
1793            .await
1794            .expect("data dir should be created");
1795        tokio::fs::write(&part_path, b"orphan shard")
1796            .await
1797            .expect("part file should be written");
1798
1799        let residue = scan_metadata_less_residue(&bucket_path)
1800            .await
1801            .expect("metadata-less residue scan should succeed");
1802        assert!(residue.has_residue_without_xlmeta());
1803        assert_eq!(residue.files, 1);
1804        assert_eq!(residue.uuid_data_dirs, 1);
1805        let sample = residue.sample.as_deref().expect("part sample should be recorded");
1806        assert!(sample.starts_with("object/"));
1807        assert!(sample.ends_with("/part.1"));
1808    }
1809
1810    #[tokio::test]
1811    async fn metadata_less_residue_scan_shares_one_entry_budget_across_roots() {
1812        let root = tempfile::tempdir().expect("temporary diagnostic roots should be created");
1813        let first_root = root.path().join("disk-a");
1814        let second_root = root.path().join("disk-b");
1815        tokio::fs::create_dir_all(&first_root)
1816            .await
1817            .expect("first diagnostic root should be created");
1818        tokio::fs::create_dir_all(&second_root)
1819            .await
1820            .expect("second diagnostic root should be created");
1821        for index in 0..3 {
1822            std::fs::write(first_root.join(format!("first-{index}")), b"").expect("first-root fixture should be written");
1823            std::fs::write(second_root.join(format!("second-{index}")), b"").expect("second-root fixture should be written");
1824        }
1825
1826        let mut budget = BucketDeleteDiagnosticBudget::with_limits(4, Duration::from_secs(5));
1827        let first = scan_metadata_less_residue_with_budget(&first_root, &mut budget)
1828            .await
1829            .expect("first root should fit the shared budget");
1830        assert!(!first.diagnostic_truncated);
1831        let second = scan_metadata_less_residue_with_budget(&second_root, &mut budget)
1832            .await
1833            .expect("second root should stop at the remaining shared budget");
1834        assert!(second.diagnostic_truncated);
1835        assert!(first.entries_scanned + second.entries_scanned <= 4);
1836    }
1837
1838    #[tokio::test]
1839    async fn metadata_less_residue_scan_honors_an_expired_request_deadline() {
1840        let root = tempfile::tempdir().expect("temporary diagnostic root should be created");
1841        std::fs::write(root.path().join("orphan"), b"").expect("deadline fixture should be written");
1842        let mut budget = BucketDeleteDiagnosticBudget::with_limits(8, Duration::ZERO);
1843
1844        let scan = scan_metadata_less_residue_with_budget(root.path(), &mut budget)
1845            .await
1846            .expect("an expired diagnostic budget should fail closed without an IO error");
1847
1848        assert!(scan.diagnostic_truncated);
1849        assert_eq!(scan.entries_scanned, 0);
1850        assert_eq!(scan.diagnostic_bytes_read, 0);
1851    }
1852
1853    #[tokio::test]
1854    async fn metadata_less_residue_scan_stops_at_diagnostic_budget() {
1855        let root = tempfile::tempdir().expect("temporary bucket root should be created");
1856        let bucket_path = root.path().join("bucket");
1857        tokio::fs::create_dir_all(&bucket_path)
1858            .await
1859            .expect("budget fixture directory should be created");
1860        for index in 0..(BUCKET_DELETE_DIAGNOSTIC_MAX_ENTRIES + 32) {
1861            std::fs::write(bucket_path.join(format!("orphan-{index:05}")), b"").expect("budget fixture file should be created");
1862        }
1863
1864        let residue = scan_metadata_less_residue(&bucket_path)
1865            .await
1866            .expect("budgeted residue scan should fail closed without an IO error");
1867        assert!(residue.diagnostic_truncated);
1868        assert!(residue.has_residue_without_xlmeta());
1869        assert!(!residue.xlmeta_found);
1870        assert!(residue.entries_scanned <= BUCKET_DELETE_DIAGNOSTIC_MAX_ENTRIES);
1871        assert!(residue.files <= BUCKET_DELETE_DIAGNOSTIC_MAX_ENTRIES);
1872        assert_eq!(residue.diagnostic_bytes_read, 0);
1873    }
1874
1875    #[tokio::test(start_paused = true)]
1876    async fn bucket_residue_scan_distinguishes_visible_and_tier_free_xlmeta() {
1877        // Classification keeps the production budget, but real filesystem
1878        // scheduling must not advance the clock before classification finishes.
1879        let (clock_guard_tx, clock_guard_rx) = std::sync::mpsc::channel::<()>();
1880        let clock_guard = tokio::task::spawn_blocking(move || clock_guard_rx.recv());
1881        let root = tempfile::tempdir().expect("temporary bucket root should be created");
1882        let bucket_path = root.path().join("bucket");
1883        let visible_path = bucket_path.join("visible").join(STORAGE_FORMAT_FILE);
1884        tokio::fs::create_dir_all(visible_path.parent().expect("visible xl.meta should have a parent"))
1885            .await
1886            .expect("visible object directory should be created");
1887        let mut visible = FileMeta::new();
1888        visible
1889            .add_version(FileInfo {
1890                version_id: Some(Uuid::new_v4()),
1891                mod_time: Some(OffsetDateTime::now_utc()),
1892                ..Default::default()
1893            })
1894            .expect("visible version should encode");
1895        tokio::fs::write(&visible_path, visible.marshal_msg().expect("visible xl.meta should marshal"))
1896            .await
1897            .expect("visible xl.meta should be written");
1898
1899        let visible_scan = scan_metadata_less_residue(&bucket_path)
1900            .await
1901            .expect("visible xl.meta scan should succeed");
1902        assert!(!visible_scan.diagnostic_truncated, "{visible_scan:?}");
1903        assert_eq!(visible_scan.xlmeta_blocker, Some(BucketDeleteBlockerKind::VisibleVersion));
1904
1905        tokio::fs::remove_dir_all(bucket_path.join("visible"))
1906            .await
1907            .expect("visible fixture should be removed");
1908        let free_path = bucket_path.join("free").join(STORAGE_FORMAT_FILE);
1909        tokio::fs::create_dir_all(free_path.parent().expect("free xl.meta should have a parent"))
1910            .await
1911            .expect("free-version object directory should be created");
1912        let source_version_id = Uuid::new_v4();
1913        let mut free = FileMeta::new();
1914        free.add_version(FileInfo {
1915            version_id: Some(source_version_id),
1916            transition_status: TRANSITION_COMPLETE.to_string(),
1917            transitioned_objname: "remote/object".to_string(),
1918            transition_version_id: Some(Uuid::new_v4()),
1919            transition_tier: "WARM".to_string(),
1920            mod_time: Some(OffsetDateTime::now_utc()),
1921            ..Default::default()
1922        })
1923        .expect("transitioned source should encode");
1924        let mut delete = FileInfo {
1925            version_id: Some(source_version_id),
1926            mod_time: Some(OffsetDateTime::now_utc()),
1927            ..Default::default()
1928        };
1929        delete.set_tier_free_version_id(&Uuid::new_v4().to_string());
1930        free.delete_version(&delete)
1931            .expect("transitioned source delete should create a free-version");
1932        tokio::fs::write(&free_path, free.marshal_msg().expect("free-version xl.meta should marshal"))
1933            .await
1934            .expect("free-version xl.meta should be written");
1935
1936        let free_scan = scan_metadata_less_residue(&bucket_path)
1937            .await
1938            .expect("free-version xl.meta scan should succeed");
1939        assert!(!free_scan.diagnostic_truncated, "{free_scan:?}");
1940        assert_eq!(free_scan.xlmeta_blocker, Some(BucketDeleteBlockerKind::TierFreeVersion));
1941
1942        tokio::fs::remove_dir_all(bucket_path.join("free"))
1943            .await
1944            .expect("free-version fixture should be removed");
1945
1946        let exact_limit_path = bucket_path.join("exact-limit").join(STORAGE_FORMAT_FILE);
1947        tokio::fs::create_dir_all(exact_limit_path.parent().expect("exact-limit xl.meta should have a parent"))
1948            .await
1949            .expect("exact-limit object directory should be created");
1950        let exact_limit = tokio::fs::File::create(&exact_limit_path)
1951            .await
1952            .expect("exact-limit xl.meta should be created");
1953        exact_limit
1954            .set_len(BUCKET_DELETE_XLMETA_DIAGNOSTIC_MAX_BYTES)
1955            .await
1956            .expect("exact-limit xl.meta should be extended without allocating its contents");
1957        let exact_limit_scan = scan_metadata_less_residue(&bucket_path)
1958            .await
1959            .expect("exact-limit xl.meta scan should remain fail closed");
1960        assert!(!exact_limit_scan.diagnostic_truncated, "{exact_limit_scan:?}");
1961        assert_eq!(exact_limit_scan.xlmeta_blocker, Some(BucketDeleteBlockerKind::UnknownXlMeta));
1962        assert_eq!(exact_limit_scan.diagnostic_bytes_read, BUCKET_DELETE_XLMETA_DIAGNOSTIC_MAX_BYTES);
1963        tokio::fs::remove_dir_all(bucket_path.join("exact-limit"))
1964            .await
1965            .expect("exact-limit fixture should be removed");
1966
1967        let oversized_path = bucket_path.join("oversized").join(STORAGE_FORMAT_FILE);
1968        tokio::fs::create_dir_all(oversized_path.parent().expect("oversized xl.meta should have a parent"))
1969            .await
1970            .expect("oversized object directory should be created");
1971        let oversized = tokio::fs::File::create(&oversized_path)
1972            .await
1973            .expect("oversized xl.meta should be created");
1974        oversized
1975            .set_len(BUCKET_DELETE_XLMETA_DIAGNOSTIC_MAX_BYTES + 1)
1976            .await
1977            .expect("oversized xl.meta should be extended without allocating its contents");
1978        let oversized_scan = scan_metadata_less_residue(&bucket_path)
1979            .await
1980            .expect("oversized xl.meta scan should remain fail closed");
1981        assert!(!oversized_scan.diagnostic_truncated, "{oversized_scan:?}");
1982        assert_eq!(oversized_scan.xlmeta_blocker, Some(BucketDeleteBlockerKind::UnknownXlMeta));
1983        assert_eq!(oversized_scan.diagnostic_bytes_read, 0);
1984        assert!(oversized_scan.diagnostic_bytes_read <= BUCKET_DELETE_XLMETA_DIAGNOSTIC_MAX_BYTES);
1985
1986        drop(clock_guard_tx);
1987        let _ = clock_guard
1988            .await
1989            .expect("classification clock guard should exit after its sender is dropped");
1990
1991        // An exhausted diagnostic budget may return before observing xl.meta.
1992        // It must report truncation rather than inventing a classification.
1993        let first_io_started = Arc::new(AtomicBool::new(false));
1994        let mut budget = BucketDeleteDiagnosticBudget::new().with_first_io_delay(
1995            BUCKET_DELETE_DIAGNOSTIC_MAX_ELAPSED + Duration::from_millis(100),
1996            first_io_started.clone(),
1997        );
1998        let truncated = scan_metadata_less_residue_with_budget(&bucket_path, &mut budget)
1999            .await
2000            .expect("a diagnostic timeout should return a partial scan");
2001        assert!(first_io_started.load(Ordering::SeqCst));
2002        assert!(truncated.diagnostic_truncated, "{truncated:?}");
2003        assert_eq!(truncated.xlmeta_blocker, None);
2004        assert_eq!(truncated.entries_scanned, 0);
2005        assert_eq!(truncated.diagnostic_bytes_read, 0);
2006    }
2007
2008    #[tokio::test]
2009    #[serial]
2010    async fn bucket_namespace_reads_keep_pre_expansion_bucket_visible() {
2011        let (_temp_dir, ecstore) = setup_multi_pool_bucket_test_env().await;
2012        let bucket = format!("pre-expansion-{}", Uuid::new_v4().simple());
2013        let object = "existing-object";
2014        ecstore.pools[0].disk_set[0]
2015            .make_bucket(&bucket, &MakeBucketOptions::default())
2016            .await
2017            .expect("bucket should be created in the original pool only");
2018        let mut reader = PutObjReader::from_vec(b"object written before pool expansion".to_vec());
2019        ecstore.pools[0]
2020            .put_object(&bucket, object, &mut reader, &ObjectOptions::default())
2021            .await
2022            .expect("object should be written in the original pool only");
2023
2024        let buckets = ecstore
2025            .list_bucket(&BucketOptions::default())
2026            .await
2027            .expect("S3 bucket listing should remain available after adding an empty pool");
2028        assert!(buckets.iter().any(|entry| entry.name == bucket));
2029
2030        let info = ecstore
2031            .get_bucket_info(&bucket, &BucketOptions::default())
2032            .await
2033            .expect("bucket validation should accept a bucket present in the original pool");
2034        assert_eq!(info.name, bucket);
2035
2036        metadata_sys::init_bucket_metadata_sys(ecstore.clone(), vec![bucket.clone()]).await;
2037        let listed = ecstore
2038            .clone()
2039            .list_objects_v2(&bucket, "", None, None, 1000, false, None, false)
2040            .await
2041            .expect("ListObjectsV2 should remain available after adding an empty pool");
2042        assert!(listed.objects.iter().any(|entry| entry.name == object));
2043        ecstore.pools[1].disk_set[0]
2044            .get_bucket_info(&bucket, &BucketOptions::default())
2045            .await
2046            .expect("metadata initialization should heal the bucket volume into the expansion pool");
2047
2048        let object_info = ecstore
2049            .get_object_info(&bucket, object, &ObjectOptions::default())
2050            .await
2051            .expect("an object in the original pool should remain readable after expansion");
2052        assert_eq!(object_info.name, object);
2053    }
2054
2055    #[tokio::test]
2056    #[serial]
2057    async fn bucket_creation_accepts_exact_quorum_in_every_set() {
2058        let (_temp_dir, ecstore) = setup_multi_pool_bucket_test_env().await;
2059        let bucket = format!("exact-set-quorum-{}", Uuid::new_v4().simple());
2060        let original_set = Arc::clone(&ecstore.pools[0].disk_set[0]);
2061        let offline = take_set_disks_offline(&ecstore, &original_set, &[0]).await;
2062
2063        ecstore
2064            .make_bucket_on_sets(&bucket, &MakeBucketOptions::default())
2065            .await
2066            .expect("three of four disks should satisfy the original set quorum");
2067
2068        restore_set_disks(&ecstore, &original_set, offline).await;
2069        for pool in &ecstore.pools {
2070            pool.disk_set[0]
2071                .get_bucket_info(&bucket, &BucketOptions::default())
2072                .await
2073                .expect("every set should expose a bucket created at its exact quorum");
2074        }
2075    }
2076
2077    #[tokio::test]
2078    #[serial]
2079    async fn bucket_creation_rejects_cross_pool_quorum_subsidy() {
2080        let (_temp_dir, ecstore) = setup_multi_pool_bucket_test_env().await;
2081        metadata_sys::init_bucket_metadata_sys(ecstore.clone(), Vec::new()).await;
2082        let bucket = format!("create-set-quorum-minus-one-{}", Uuid::new_v4().simple());
2083        let original_set = Arc::clone(&ecstore.pools[0].disk_set[0]);
2084        let offline = take_set_disks_offline(&ecstore, &original_set, &[0, 1]).await;
2085
2086        let err = ecstore
2087            .make_bucket(&bucket, &MakeBucketOptions::default())
2088            .await
2089            .expect_err("a healthy expansion pool must not subsidize a failed original set");
2090        restore_set_disks(&ecstore, &original_set, offline).await;
2091        assert!(
2092            matches!(&err, StorageError::InsufficientWriteQuorum(name, object) if name == &bucket && object.is_empty()),
2093            "unexpected error: {err}"
2094        );
2095        for pool in &ecstore.pools {
2096            assert_eq!(
2097                pool.disk_set[0]
2098                    .get_bucket_info(&bucket, &BucketOptions::default())
2099                    .await
2100                    .expect_err("failed creation must not leave an authoritative bucket"),
2101                StorageError::VolumeNotFound
2102            );
2103        }
2104    }
2105
2106    #[tokio::test]
2107    #[serial]
2108    async fn bucket_creation_prioritizes_hard_set_error_over_existing_bucket() {
2109        let (_temp_dir, ecstore) = setup_multi_pool_bucket_test_env().await;
2110        metadata_sys::init_bucket_metadata_sys(ecstore.clone(), Vec::new()).await;
2111        let bucket = format!("create-hard-error-{}", Uuid::new_v4().simple());
2112        ecstore.pools[0].disk_set[0]
2113            .make_bucket(&bucket, &MakeBucketOptions::default())
2114            .await
2115            .expect("the original pool should contain the bucket");
2116        let expansion_set = Arc::clone(&ecstore.pools[1].disk_set[0]);
2117        let offline = take_set_disks_offline(&ecstore, &expansion_set, &[0, 1]).await;
2118
2119        let err = ecstore
2120            .make_bucket(&bucket, &MakeBucketOptions::default())
2121            .await
2122            .expect_err("a hard set failure must outrank BucketExists from another set");
2123        restore_set_disks(&ecstore, &expansion_set, offline).await;
2124        assert!(
2125            matches!(&err, StorageError::InsufficientWriteQuorum(name, object) if name == &bucket && object.is_empty()),
2126            "unexpected error: {err}"
2127        );
2128        ecstore.pools[0].disk_set[0]
2129            .get_bucket_info(&bucket, &BucketOptions::default())
2130            .await
2131            .expect("failure in another set must not roll back a pre-existing bucket");
2132    }
2133
2134    #[tokio::test]
2135    #[serial]
2136    async fn bucket_delete_rejects_cross_pool_quorum_subsidy_before_mutation() {
2137        let (_temp_dir, ecstore) = setup_multi_pool_bucket_test_env().await;
2138        metadata_sys::init_bucket_metadata_sys(ecstore.clone(), Vec::new()).await;
2139        let bucket = format!("delete-set-quorum-minus-one-{}", Uuid::new_v4().simple());
2140        ecstore
2141            .make_bucket(&bucket, &MakeBucketOptions::default())
2142            .await
2143            .expect("bucket should be created in every set before deletion");
2144        let original_set = Arc::clone(&ecstore.pools[0].disk_set[0]);
2145        let offline = take_set_disks_offline(&ecstore, &original_set, &[0, 1, 2]).await;
2146
2147        let err = ecstore
2148            .delete_bucket(
2149                &bucket,
2150                &DeleteBucketOptions {
2151                    force: true,
2152                    ..Default::default()
2153                },
2154            )
2155            .await
2156            .expect_err("a healthy expansion pool must not subsidize a failed original set deletion");
2157        restore_set_disks(&ecstore, &original_set, offline).await;
2158        assert!(
2159            matches!(&err, StorageError::InsufficientWriteQuorum(name, object) if name == &bucket && object.is_empty()),
2160            "unexpected error: {err}"
2161        );
2162        for pool in &ecstore.pools {
2163            pool.disk_set[0]
2164                .get_bucket_info(&bucket, &BucketOptions::default())
2165                .await
2166                .expect("failed deletion preflight must preserve every bucket volume");
2167        }
2168        let (_, persisted) = metadata_sys::get_config_from_disk_with_presence_in(&ecstore.ctx, &bucket)
2169            .await
2170            .expect("failed deletion must preserve bucket metadata");
2171        assert!(persisted);
2172    }
2173
2174    #[tokio::test]
2175    #[serial]
2176    async fn bucket_delete_rolls_back_sets_after_partial_failure() {
2177        let (_temp_dir, ecstore) = setup_multi_pool_bucket_test_env().await;
2178        let bucket = format!("delete-set-rollback-{}", Uuid::new_v4().simple());
2179        ecstore
2180            .make_bucket_on_sets(&bucket, &MakeBucketOptions::default())
2181            .await
2182            .expect("bucket should be created in every set before deletion");
2183        let original_set = Arc::clone(&ecstore.pools[0].disk_set[0]);
2184        let offline = take_set_disks_offline(&ecstore, &original_set, &[0, 1]).await;
2185
2186        let err = ecstore
2187            .delete_bucket_on_sets(&bucket, &DeleteBucketOptions::default())
2188            .await
2189            .expect_err("a failed set must fail the complete bucket deletion");
2190        restore_set_disks(&ecstore, &original_set, offline).await;
2191        assert_eq!(err, StorageError::ErasureWriteQuorum);
2192        for pool in &ecstore.pools {
2193            pool.disk_set[0]
2194                .get_bucket_info(&bucket, &BucketOptions::default())
2195                .await
2196                .expect("delete rollback should restore the bucket namespace in every set");
2197        }
2198    }
2199
2200    #[tokio::test]
2201    #[serial]
2202    async fn bucket_config_update_accepts_pre_expansion_bucket() {
2203        let (_temp_dir, ecstore) = setup_multi_pool_bucket_test_env().await;
2204        metadata_sys::init_bucket_metadata_sys(ecstore.clone(), Vec::new()).await;
2205        let bucket = format!("config-pre-expansion-{}", Uuid::new_v4().simple());
2206        ecstore
2207            .make_bucket(&bucket, &MakeBucketOptions::default())
2208            .await
2209            .expect("bucket should be created with authoritative metadata");
2210        ecstore.pools[1].disk_set[0]
2211            .delete_bucket(&bucket, &DeleteBucketOptions::default())
2212            .await
2213            .expect("the expansion pool bucket volume should be removed for the regression state");
2214
2215        let policy = br#"{"Version":"2012-10-17","Statement":[]}"#.to_vec();
2216        ecstore
2217            .update_bucket_metadata_config(&bucket, crate::bucket::metadata::BUCKET_POLICY_CONFIG, policy.clone())
2218            .await
2219            .expect("config mutation should validate existence per erasure set");
2220        let (stored, _) = ecstore
2221            .get_bucket_policy_raw(&bucket)
2222            .await
2223            .expect("updated bucket policy should remain readable");
2224        assert_eq!(stored.as_bytes(), policy);
2225    }
2226
2227    #[tokio::test]
2228    #[serial]
2229    async fn bucket_metadata_init_recreates_pre_expansion_bucket_in_new_pool() {
2230        let (_temp_dir, ecstore) = setup_multi_pool_bucket_test_env().await;
2231        let bucket = format!("metadata-expansion-{}", Uuid::new_v4().simple());
2232        ecstore.pools[0].disk_set[0]
2233            .make_bucket(&bucket, &MakeBucketOptions::default())
2234            .await
2235            .expect("bucket should be created in the original pool only");
2236        assert_eq!(
2237            ecstore.pools[1].disk_set[0]
2238                .get_bucket_info(&bucket, &BucketOptions::default())
2239                .await
2240                .expect_err("new pool should initially have no bucket volume"),
2241            StorageError::VolumeNotFound
2242        );
2243
2244        metadata_sys::init_bucket_metadata_sys(ecstore.clone(), vec![bucket.clone()]).await;
2245
2246        ecstore.pools[1].disk_set[0]
2247            .get_bucket_info(&bucket, &BucketOptions::default())
2248            .await
2249            .expect("metadata initialization should recreate the bucket volume in the new pool");
2250    }
2251
2252    #[tokio::test]
2253    #[serial]
2254    async fn bucket_metadata_init_repairs_half_created_expansion_pool() {
2255        // Check both pool orders: an incomplete set must not hide a later
2256        // complete set, and a complete set must not weaken request validation.
2257        for complete_pool in 0..2 {
2258            let (temp_dir, ecstore) = setup_multi_pool_bucket_test_env().await;
2259            let bucket = format!("partial-expansion-{}", Uuid::new_v4().simple());
2260            for pool_index in 0..2 {
2261                let present_disks = if pool_index == complete_pool { 4 } else { 2 };
2262                for disk_index in 0..present_disks {
2263                    tokio::fs::create_dir(
2264                        temp_dir
2265                            .path()
2266                            .join(format!("pool{pool_index}-disk{disk_index}"))
2267                            .join(&bucket),
2268                    )
2269                    .await
2270                    .expect("fixture bucket volume should be created");
2271                }
2272            }
2273            assert_eq!(
2274                ecstore
2275                    .get_bucket_info_from_sets(&bucket, &BucketOptions::default())
2276                    .await
2277                    .expect_err("request validation must reject a half-created expansion set"),
2278                StorageError::ErasureWriteQuorum
2279            );
2280
2281            metadata_sys::init_bucket_metadata_sys(ecstore.clone(), vec![bucket.clone()]).await;
2282
2283            for pool_index in 0..2 {
2284                for disk_index in 0..4 {
2285                    assert!(
2286                        temp_dir
2287                            .path()
2288                            .join(format!("pool{pool_index}-disk{disk_index}"))
2289                            .join(&bucket)
2290                            .is_dir(),
2291                        "metadata initialization must heal every missing expansion volume"
2292                    );
2293                }
2294            }
2295            ecstore
2296                .get_bucket_info_from_sets(&bucket, &BucketOptions::default())
2297                .await
2298                .expect("strict request validation should succeed after volume repair");
2299        }
2300    }
2301
2302    #[tokio::test]
2303    #[serial]
2304    async fn bucket_metadata_init_does_not_combine_partial_set_evidence() {
2305        let (temp_dir, ecstore) = setup_multi_pool_bucket_test_env().await;
2306        let bucket = format!("no-quorum-expansion-{}", Uuid::new_v4().simple());
2307        for pool_index in 0..2 {
2308            for disk_index in 0..2 {
2309                tokio::fs::create_dir(
2310                    temp_dir
2311                        .path()
2312                        .join(format!("pool{pool_index}-disk{disk_index}"))
2313                        .join(&bucket),
2314                )
2315                .await
2316                .expect("fixture bucket volume should be created");
2317            }
2318        }
2319        assert_eq!(
2320            ecstore
2321                .bucket_exists_for_heal(&bucket)
2322                .await
2323                .expect_err("repair must require a complete quorum within one set"),
2324            StorageError::ErasureWriteQuorum
2325        );
2326
2327        metadata_sys::init_bucket_metadata_sys(ecstore.clone(), vec![bucket.clone()]).await;
2328
2329        for pool_index in 0..2 {
2330            for disk_index in 0..4 {
2331                assert_eq!(
2332                    temp_dir
2333                        .path()
2334                        .join(format!("pool{pool_index}-disk{disk_index}"))
2335                        .join(&bucket)
2336                        .is_dir(),
2337                    disk_index < 2,
2338                    "unproven bucket generations must not recreate missing volumes"
2339                );
2340            }
2341        }
2342    }
2343
2344    #[tokio::test]
2345    #[serial]
2346    async fn bucket_metadata_init_does_not_recreate_stale_bucket_name() {
2347        let (_temp_dir, ecstore) = setup_multi_pool_bucket_test_env().await;
2348        let bucket = format!("stale-expansion-{}", Uuid::new_v4().simple());
2349
2350        metadata_sys::init_bucket_metadata_sys(ecstore.clone(), vec![bucket.clone()]).await;
2351
2352        for pool in &ecstore.pools {
2353            let err = pool.disk_set[0]
2354                .get_bucket_info(&bucket, &BucketOptions::default())
2355                .await
2356                .expect_err("a stale startup listing must not recreate a deleted bucket");
2357            assert_eq!(err, StorageError::VolumeNotFound);
2358        }
2359    }
2360
2361    #[tokio::test]
2362    #[serial]
2363    async fn bucket_info_read_quorum_tracks_erasure_layout() {
2364        for (drive_count, parity) in [(2, 1), (3, 1), (4, 2), (5, 2), (6, 3), (8, 4), (6, 2), (12, 6)] {
2365            let (_temp_dir, store) = setup_bucket_quorum_test_env(&[drive_count], Some(parity)).await;
2366            metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
2367            let bucket = format!("read-quorum-{drive_count}-{parity}");
2368            let object = "uncached-object";
2369            let body = b"erasure read quorum must follow the persisted layout".repeat(32_768);
2370            store
2371                .make_bucket(&bucket, &MakeBucketOptions::default())
2372                .await
2373                .expect("healthy namespace should accept bucket creation");
2374            store
2375                .put_object(&bucket, object, &mut PutObjReader::from_vec(body.clone()), &ObjectOptions::default())
2376                .await
2377                .expect("healthy erasure set should accept the seed object");
2378            let set = &store.pools[0].disk_set[0];
2379            let lock = set
2380                .new_ns_lock(&bucket, object)
2381                .await
2382                .expect("seed namespace lock should resolve");
2383            drop(
2384                lock.get_write_lock(Duration::from_secs(30))
2385                    .await
2386                    .expect("seed physical fanout must finish before taking disks offline"),
2387            );
2388            if (drive_count, parity) == (6, 3) {
2389                let mut kvs = rustfs_config::server_config::KVS::new();
2390                kvs.insert(crate::config::storageclass::CLASS_STANDARD.to_string(), "EC:2".to_string());
2391                set.set_test_storage_class_config(
2392                    crate::config::storageclass::lookup_config_for_pools_without_env(&kvs, &[drive_count])
2393                        .expect("a later storage-class change must not raise old objects' read quorum"),
2394                );
2395            }
2396
2397            let offline_indexes = (0..parity).collect::<Vec<_>>();
2398            let offline = take_set_disks_offline(&store, set, &offline_indexes).await;
2399            let info = store
2400                .get_bucket_info(&bucket, &BucketOptions::default())
2401                .await
2402                .expect("bucket validation must admit the object's exact read quorum");
2403            assert_eq!(info.name, bucket);
2404
2405            let mut reader = store
2406                .get_object_reader(&bucket, object, None, Default::default(), &ObjectOptions::default())
2407                .await
2408                .expect("the persisted layout should remain readable at its exact data-shard quorum");
2409            let mut restored = Vec::new();
2410            reader
2411                .stream
2412                .read_to_end(&mut restored)
2413                .await
2414                .expect("quorum read should reconstruct the body");
2415            assert_eq!(restored, body, "layout {drive_count}/{parity} must retain exact object contents");
2416            drop(reader);
2417
2418            if drive_count - parity == drive_count / 2 {
2419                let error = store
2420                    .get_bucket_info_from_sets(&bucket, &BucketOptions::default())
2421                    .await
2422                    .expect_err("bucket mutations must retain their majority namespace check");
2423                assert_eq!(error, StorageError::ErasureWriteQuorum);
2424            }
2425
2426            let below_quorum = take_set_disks_offline(&store, set, &[parity]).await;
2427            let read = store
2428                .get_object_reader(&bucket, object, None, Default::default(), &ObjectOptions::default())
2429                .await;
2430            match read {
2431                Ok(mut reader) => assert!(
2432                    reader.stream.read_to_end(&mut Vec::new()).await.is_err(),
2433                    "layout {drive_count}/{parity} must reject fewer than its data-shard quorum"
2434                ),
2435                Err(error) => assert!(
2436                    matches!(error, StorageError::ErasureReadQuorum | StorageError::InsufficientReadQuorum(_, _)),
2437                    "a missing shard must report read quorum loss, got {error}"
2438                ),
2439            }
2440            restore_set_disks(&store, set, below_quorum).await;
2441            restore_set_disks(&store, set, offline).await;
2442        }
2443    }
2444
2445    #[tokio::test]
2446    #[serial]
2447    async fn lazy_metadata_load_uses_read_quorum_for_non_lock_bucket() {
2448        let (_temp_dir, store) = setup_bucket_quorum_test_env(&[4], Some(2)).await;
2449        metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
2450        let bucket = format!("lazy-read-quorum-{}", Uuid::new_v4().simple());
2451        store
2452            .make_bucket(&bucket, &MakeBucketOptions::default())
2453            .await
2454            .expect("healthy namespace should accept bucket creation");
2455
2456        // Replace the instance metadata system with a cold cache while keeping the
2457        // persisted metadata and physical bucket in place.
2458        metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
2459        let set = &store.pools[0].disk_set[0];
2460        let offline = take_set_disks_offline(&store, set, &[0, 1]).await;
2461
2462        metadata_sys::get_on_demand_migration_config_in(&store, &bucket)
2463            .await
2464            .expect("cold metadata load should admit the exact namespace read quorum");
2465        assert!(
2466            metadata_sys::get_in(&store.ctx, &bucket).await.is_ok(),
2467            "the lazy load should publish authoritative metadata"
2468        );
2469        assert_eq!(
2470            store
2471                .get_bucket_info_from_sets(&bucket, &BucketOptions::default())
2472                .await
2473                .expect_err("bucket mutations must retain their majority namespace check"),
2474            StorageError::ErasureWriteQuorum
2475        );
2476
2477        restore_set_disks(&store, set, offline).await;
2478    }
2479
2480    /// EC 2+2 with two disks offline still has object read quorum and has lost
2481    /// write quorum. A node that has not cached the bucket must still serve
2482    /// HEAD bucket, GET, HEAD object, and List; a write must fail.
2483    #[tokio::test]
2484    #[serial]
2485    async fn cold_bucket_reads_succeed_below_write_quorum() {
2486        let (_temp_dir, store) = setup_bucket_quorum_test_env(&[4], Some(2)).await;
2487        metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
2488        let bucket = format!("cold-read-quorum-{}", Uuid::new_v4().simple());
2489        let object = "seed-object";
2490        let body = b"cold bucket reads must survive lost write quorum".to_vec();
2491        store
2492            .make_bucket(&bucket, &MakeBucketOptions::default())
2493            .await
2494            .expect("healthy namespace should accept bucket creation");
2495        store
2496            .put_object(&bucket, object, &mut PutObjReader::from_vec(body.clone()), &ObjectOptions::default())
2497            .await
2498            .expect("healthy erasure set should accept the seed object");
2499
2500        metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
2501        assert!(
2502            metadata_sys::get_in(&store.ctx, &bucket).await.is_err(),
2503            "the replacement metadata system must start without this bucket"
2504        );
2505        let set = &store.pools[0].disk_set[0];
2506        let lock = set
2507            .new_ns_lock(&bucket, object)
2508            .await
2509            .expect("seed namespace lock should resolve");
2510        drop(
2511            lock.get_write_lock(Duration::from_secs(30))
2512                .await
2513                .expect("seed physical fanout must finish before taking disks offline"),
2514        );
2515        let offline = take_set_disks_offline(&store, set, &[0, 1]).await;
2516
2517        let info = store
2518            .get_bucket_info(&bucket, &BucketOptions::default())
2519            .await
2520            .expect("HEAD bucket must succeed at read quorum before the bucket is cached");
2521        assert_eq!(info.name, bucket);
2522
2523        let mut reader = store
2524            .get_object_reader(&bucket, object, None, Default::default(), &ObjectOptions::default())
2525            .await
2526            .expect("GET must succeed for a bucket that was not loaded at startup");
2527        let mut restored = Vec::new();
2528        reader
2529            .stream
2530            .read_to_end(&mut restored)
2531            .await
2532            .expect("quorum read should reconstruct the body");
2533        assert_eq!(restored, body);
2534        drop(reader);
2535
2536        let object_info = store
2537            .get_object_info(&bucket, object, &ObjectOptions::default())
2538            .await
2539            .expect("HEAD object must succeed at read quorum");
2540        assert_eq!(object_info.name, object);
2541        assert_eq!(object_info.size, i64::try_from(body.len()).expect("body length fits i64"));
2542
2543        let listed = store
2544            .clone()
2545            .list_objects_v2(&bucket, "", None, None, 100, false, None, false)
2546            .await
2547            .expect("List must succeed at read quorum");
2548        assert!(
2549            listed.objects.iter().any(|item| item.name == object),
2550            "list should include the seed object, got {:?}",
2551            listed.objects.iter().map(|item| item.name.clone()).collect::<Vec<_>>()
2552        );
2553        assert!(
2554            metadata_sys::get_in(&store.ctx, &bucket).await.is_ok(),
2555            "the cold load should publish authoritative metadata"
2556        );
2557
2558        let write_error = store
2559            .put_object(&bucket, "rejected-object", &mut PutObjReader::from_vec(body), &ObjectOptions::default())
2560            .await
2561            .expect_err("writes must still fail without write quorum");
2562        assert!(
2563            matches!(
2564                write_error,
2565                StorageError::InsufficientWriteQuorum(_, _) | StorageError::ErasureWriteQuorum
2566            ),
2567            "writes must fail with a write-quorum error, got {write_error}"
2568        );
2569
2570        restore_set_disks(&store, set, offline).await;
2571    }
2572
2573    #[tokio::test]
2574    #[serial]
2575    async fn lazy_metadata_load_rejects_below_read_quorum() {
2576        let (_temp_dir, store) = setup_bucket_quorum_test_env(&[4], Some(2)).await;
2577        metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
2578        let bucket = format!("lazy-below-quorum-{}", Uuid::new_v4().simple());
2579        store
2580            .make_bucket(&bucket, &MakeBucketOptions::default())
2581            .await
2582            .expect("healthy namespace should accept bucket creation");
2583
2584        metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
2585        let set = &store.pools[0].disk_set[0];
2586        let offline = take_set_disks_offline(&store, set, &[0, 1, 2]).await;
2587
2588        let error = metadata_sys::get_on_demand_migration_config_in(&store, &bucket)
2589            .await
2590            .expect_err("cold metadata load must reject read quorum minus one");
2591        assert!(
2592            matches!(error, StorageError::ErasureReadQuorum | StorageError::InsufficientReadQuorum(_, _)),
2593            "below read quorum must fail with a read-quorum error, got {error}"
2594        );
2595        assert!(
2596            metadata_sys::get_in(&store.ctx, &bucket).await.is_err(),
2597            "a below-quorum load must not publish metadata"
2598        );
2599
2600        restore_set_disks(&store, set, offline).await;
2601    }
2602
2603    #[tokio::test]
2604    #[serial]
2605    async fn lazy_metadata_load_keeps_lock_creation_intent_fail_closed() {
2606        let (temp_dir, store) = setup_bucket_quorum_test_env(&[4], Some(2)).await;
2607        metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
2608        let bucket = format!("lazy-lock-intent-{}", Uuid::new_v4().simple());
2609        let mut intent = BucketMetadata::new(&bucket);
2610        intent.lock_enabled = true;
2611        metadata_sys::set_new_bucket_metadata_intent_in(&store.ctx, intent)
2612            .await
2613            .expect("Object Lock creation intent should persist metadata and its incarnation sidecar");
2614
2615        let set = &store.pools[0].disk_set[0];
2616        let offline = take_set_disks_offline(&store, set, &[0, 1]).await;
2617        assert_eq!(
2618            store
2619                .make_bucket_on_sets(&bucket, &MakeBucketOptions::default())
2620                .await
2621                .expect_err("physical creation must not commit below write quorum"),
2622            StorageError::ErasureWriteQuorum
2623        );
2624        assert!(!temp_dir.path().join("pool0-disk0").join(&bucket).exists());
2625        assert!(!temp_dir.path().join("pool0-disk1").join(&bucket).exists());
2626        assert!(temp_dir.path().join("pool0-disk2").join(&bucket).is_dir());
2627        assert!(temp_dir.path().join("pool0-disk3").join(&bucket).is_dir());
2628
2629        metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
2630        let error = metadata_sys::get_on_demand_migration_config_in(&store, &bucket)
2631            .await
2632            .expect_err("Object Lock intent must not become visible at read quorum");
2633        assert_eq!(error, StorageError::ErasureWriteQuorum);
2634        assert!(
2635            metadata_sys::get_in(&store.ctx, &bucket).await.is_err(),
2636            "the rejected Object Lock intent must not enter the metadata cache"
2637        );
2638        assert_eq!(
2639            store
2640                .get_bucket_info(&bucket, &BucketOptions::default())
2641                .await
2642                .expect_err("bucket-info read fallback must also reject a pending Object Lock intent"),
2643            StorageError::ErasureWriteQuorum
2644        );
2645
2646        restore_set_disks(&store, set, offline).await;
2647    }
2648
2649    #[tokio::test]
2650    #[serial]
2651    async fn lazy_metadata_load_admits_committed_lock_bucket_at_read_quorum() {
2652        let (_temp_dir, store) = setup_bucket_quorum_test_env(&[4], Some(2)).await;
2653        metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
2654        let bucket = format!("lazy-committed-lock-{}", Uuid::new_v4().simple());
2655        store
2656            .make_bucket(
2657                &bucket,
2658                &MakeBucketOptions {
2659                    lock_enabled: true,
2660                    ..Default::default()
2661                },
2662            )
2663            .await
2664            .expect("healthy namespace should accept Object Lock bucket creation");
2665        let (metadata, persisted) = metadata_sys::get_config_from_disk_with_presence_in(&store.ctx, &bucket)
2666            .await
2667            .expect("read committed Object Lock metadata");
2668        assert!(persisted && metadata.bucket_creation_committed);
2669
2670        metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
2671        let set = &store.pools[0].disk_set[0];
2672        let offline = take_set_disks_offline(&store, set, &[0, 1]).await;
2673
2674        metadata_sys::get_on_demand_migration_config_in(&store, &bucket)
2675            .await
2676            .expect("committed Object Lock metadata should load at read quorum");
2677        assert!(
2678            metadata_sys::get_in(&store.ctx, &bucket).await.is_ok(),
2679            "committed Object Lock metadata should be published"
2680        );
2681
2682        restore_set_disks(&store, set, offline).await;
2683    }
2684
2685    #[tokio::test]
2686    #[serial]
2687    async fn creation_commit_backfill_runs_under_the_config_write_fence() {
2688        let (_temp_dir, store) = setup_bucket_quorum_test_env(&[4], Some(2)).await;
2689        metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
2690        let bucket = format!("lazy-commit-backfill-{}", Uuid::new_v4().simple());
2691        let mut intent = BucketMetadata::new(&bucket);
2692        intent.lock_enabled = true;
2693        metadata_sys::set_new_bucket_metadata_intent_in(&store.ctx, intent)
2694            .await
2695            .expect("persist pending Object Lock intent");
2696        store
2697            .make_bucket_on_sets(&bucket, &MakeBucketOptions::default())
2698            .await
2699            .expect("physical creation should commit at full write quorum");
2700
2701        metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
2702        metadata_sys::get_object_lock_config_state_in(&store.ctx, &bucket)
2703            .await
2704            .expect("the fenced authority path should backfill the creation commit");
2705        let (metadata, persisted) = metadata_sys::get_config_from_disk_with_presence_in(&store.ctx, &bucket)
2706            .await
2707            .expect("read backfilled creation commit");
2708        assert!(persisted && metadata.bucket_creation_committed);
2709
2710        metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
2711        let set = &store.pools[0].disk_set[0];
2712        let offline = take_set_disks_offline(&store, set, &[0, 1]).await;
2713        metadata_sys::get_on_demand_migration_config_in(&store, &bucket)
2714            .await
2715            .expect("backfilled Object Lock bucket should load at read quorum");
2716        restore_set_disks(&store, set, offline).await;
2717    }
2718
2719    #[tokio::test]
2720    #[serial]
2721    async fn enabling_object_lock_on_existing_bucket_establishes_commitment() {
2722        let (_temp_dir, store) = setup_bucket_quorum_test_env(&[4], Some(2)).await;
2723        metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
2724        let bucket = format!("enable-lock-legacy-{}", Uuid::new_v4().simple());
2725        store
2726            .make_bucket(&bucket, &MakeBucketOptions::default())
2727            .await
2728            .expect("create a versioned non-lock bucket");
2729
2730        // Simulate a bucket persisted by a build before the commit field existed.
2731        let mut legacy = metadata_sys::get_in(&store.ctx, &bucket)
2732            .await
2733            .expect("cached metadata")
2734            .as_ref()
2735            .clone();
2736        legacy.lock_enabled = false;
2737        legacy.object_lock_config_xml = Vec::new();
2738        legacy.object_lock_config = None;
2739        legacy.bucket_creation_committed = false;
2740        metadata_sys::set_new_bucket_metadata_intent_in(&store.ctx, legacy)
2741            .await
2742            .expect("persist legacy-shaped metadata");
2743
2744        let lock_xml = crate::bucket::utils::serialize(&*crate::store::ENABLED_OBJECT_LOCK_CONFIG)
2745            .expect("serialize Object Lock configuration");
2746        metadata_sys::update_in(&store.ctx, &bucket, "object-lock.xml", lock_xml)
2747            .await
2748            .expect("enabling Object Lock on an existing bucket must succeed");
2749
2750        assert!(
2751            matches!(
2752                metadata_sys::get_object_lock_config_state_in(&store.ctx, &bucket)
2753                    .await
2754                    .expect("read the freshly enabled Object Lock state"),
2755                metadata_sys::ObjectLockConfigState::Configured { .. }
2756            ),
2757            "a healthy cluster must not report ErasureWriteQuorum after enabling Object Lock"
2758        );
2759        let (metadata, persisted) = metadata_sys::get_config_from_disk_with_presence_in(&store.ctx, &bucket)
2760            .await
2761            .expect("read persisted metadata");
2762        assert!(persisted && metadata.bucket_creation_committed);
2763    }
2764
2765    #[tokio::test]
2766    #[serial]
2767    async fn creation_commit_backfill_preserves_concurrent_config_update() {
2768        let (_temp_dir, store) = setup_bucket_quorum_test_env(&[4], Some(2)).await;
2769        metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
2770        let bucket = format!("commit-backfill-pack-{}", Uuid::new_v4().simple());
2771        store
2772            .make_bucket(&bucket, &MakeBucketOptions::default())
2773            .await
2774            .expect("create bucket");
2775
2776        // The cached snapshot on the migrating node still predates another
2777        // node's acknowledged configuration update.
2778        let mut stale = metadata_sys::get_in(&store.ctx, &bucket)
2779            .await
2780            .expect("cached metadata")
2781            .as_ref()
2782            .clone();
2783        stale.bucket_creation_committed = false;
2784        metadata_sys::set_new_bucket_metadata_intent_in(&store.ctx, stale.clone())
2785            .await
2786            .expect("persist the stale pre-upgrade snapshot");
2787
2788        // A newer, still-uncommitted copy carrying an acknowledged Object Lock
2789        // update lands on disk after the migrating node read its snapshot.
2790        let mut newer = stale.clone();
2791        newer.lock_enabled = true;
2792        newer.object_lock_config_xml = crate::bucket::utils::serialize(&*crate::store::ENABLED_OBJECT_LOCK_CONFIG)
2793            .expect("serialize Object Lock configuration");
2794        newer
2795            .save_with_store(store.clone())
2796            .await
2797            .expect("persist the newer configuration");
2798
2799        // A cold node must re-read the newest authoritative metadata under the
2800        // fence instead of rewriting its stale snapshot.
2801        metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
2802        metadata_sys::get_object_lock_config_state_in(&store.ctx, &bucket)
2803            .await
2804            .expect("fenced commit migration");
2805
2806        let (metadata, persisted) = metadata_sys::get_config_from_disk_with_presence_in(&store.ctx, &bucket)
2807            .await
2808            .expect("read persisted metadata");
2809        assert!(persisted && metadata.bucket_creation_committed);
2810        assert!(
2811            metadata.lock_enabled && !metadata.object_lock_config_xml.is_empty(),
2812            "the acknowledged Object Lock update must survive the backfill"
2813        );
2814    }
2815
2816    #[tokio::test]
2817    #[serial]
2818    async fn creation_commit_backfill_keeps_recreated_generation() {
2819        let (_temp_dir, store) = setup_bucket_quorum_test_env(&[4], Some(2)).await;
2820        metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
2821        let bucket = format!("commit-backfill-recreate-{}", Uuid::new_v4().simple());
2822        store
2823            .make_bucket(&bucket, &MakeBucketOptions::default())
2824            .await
2825            .expect("create bucket");
2826        let old_incarnation = store.bucket_incarnation_id_from_disk(&bucket).await.expect("old incarnation");
2827
2828        // A node that still holds the pre-recreation snapshot must never let it
2829        // outlive the replacement generation.
2830        let mut stale = metadata_sys::get_in(&store.ctx, &bucket)
2831            .await
2832            .expect("cached metadata")
2833            .as_ref()
2834            .clone();
2835        stale.lock_enabled = true;
2836        stale.object_lock_config_xml = crate::bucket::utils::serialize(&*crate::store::ENABLED_OBJECT_LOCK_CONFIG)
2837            .expect("serialize Object Lock configuration");
2838        stale.bucket_creation_committed = false;
2839        stale.save_with_store(store.clone()).await.expect("persist stale metadata");
2840
2841        store
2842            .delete_bucket(&bucket, &crate::storage_api_contracts::bucket::DeleteBucketOptions::default())
2843            .await
2844            .expect("delete bucket");
2845        store
2846            .make_bucket(
2847                &bucket,
2848                &MakeBucketOptions {
2849                    lock_enabled: true,
2850                    ..Default::default()
2851                },
2852            )
2853            .await
2854            .expect("recreate bucket with Object Lock");
2855        let new_incarnation = store.bucket_incarnation_id_from_disk(&bucket).await.expect("new incarnation");
2856        assert_ne!(old_incarnation, new_incarnation);
2857
2858        metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
2859        metadata_sys::get_object_lock_config_state_in(&store.ctx, &bucket)
2860            .await
2861            .expect("fenced backfill on the replacement generation");
2862
2863        let (metadata, persisted) = metadata_sys::get_config_from_disk_with_presence_in(&store.ctx, &bucket)
2864            .await
2865            .expect("read persisted metadata");
2866        assert!(persisted);
2867        assert_eq!(
2868            metadata.bucket_incarnation_id, new_incarnation,
2869            "the replacement generation must stay authoritative"
2870        );
2871        assert!(metadata.bucket_creation_committed);
2872    }
2873
2874    #[tokio::test]
2875    #[serial]
2876    async fn bucket_info_read_quorum_is_scoped_to_each_erasure_set() {
2877        let (_temp_dir, store) = setup_bucket_quorum_test_env(&[4, 6], None).await;
2878        metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
2879        let bucket = "read-quorum-mixed-pools";
2880        store
2881            .make_bucket(bucket, &MakeBucketOptions::default())
2882            .await
2883            .expect("healthy pools should accept bucket creation");
2884
2885        let first_set = &store.pools[0].disk_set[0];
2886        let second_set = &store.pools[1].disk_set[0];
2887        let first_offline = take_set_disks_offline(&store, first_set, &[0, 1]).await;
2888        let second_offline = take_set_disks_offline(&store, second_set, &[0, 1, 2]).await;
2889        store
2890            .get_bucket_info(bucket, &BucketOptions::default())
2891            .await
2892            .expect("each set independently satisfies its namespace read quorum");
2893
2894        for (set, extra_disk) in [(first_set, 2), (second_set, 3)] {
2895            let extra_offline = take_set_disks_offline(&store, set, &[extra_disk]).await;
2896            assert_eq!(
2897                store
2898                    .get_bucket_info(bucket, &BucketOptions::default())
2899                    .await
2900                    .expect_err("another pool must not subsidize a set below its read quorum"),
2901                StorageError::ErasureReadQuorum
2902            );
2903            restore_set_disks(&store, set, extra_offline).await;
2904        }
2905        restore_set_disks(&store, first_set, first_offline).await;
2906        restore_set_disks(&store, second_set, second_offline).await;
2907    }
2908
2909    #[tokio::test]
2910    #[serial]
2911    async fn bucket_info_read_quorum_requires_authoritative_metadata() {
2912        for state in ["missing", "corrupt", "foreign", "incarnation"] {
2913            let (_temp_dir, store) = setup_bucket_quorum_test_env(&[4], None).await;
2914            metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
2915            let bucket = format!("read-quorum-{state}-metadata");
2916            let mut metadata = if state == "missing" {
2917                store
2918                    .make_bucket_on_sets(&bucket, &MakeBucketOptions::default())
2919                    .await
2920                    .expect("simulate directories left before bucket metadata is published");
2921                BucketMetadata::new(&bucket)
2922            } else {
2923                store
2924                    .make_bucket(&bucket, &MakeBucketOptions::default())
2925                    .await
2926                    .expect("healthy bucket should publish metadata");
2927                metadata_sys::get_in(&store.ctx, &bucket)
2928                    .await
2929                    .expect("seed metadata should be cached")
2930                    .as_ref()
2931                    .clone()
2932            };
2933            let path = metadata.save_file_path();
2934            match state {
2935                "corrupt" => crate::config::com::save_config(store.clone(), &path, b"corrupt".to_vec())
2936                    .await
2937                    .expect("persist corrupt metadata while the cached copy remains valid"),
2938                "foreign" => {
2939                    metadata.name = "different-bucket".to_string();
2940                    let mut encoded = vec![1, 0, 1, 0];
2941                    encoded.extend(metadata.marshal_msg().expect("foreign metadata should encode"));
2942                    crate::config::com::save_config(store.clone(), &path, encoded)
2943                        .await
2944                        .expect("persist metadata for a different bucket at the requested path");
2945                }
2946                "incarnation" => crate::bucket::metadata::save_bucket_incarnation(store.clone(), &bucket, Uuid::new_v4())
2947                    .await
2948                    .expect("persist a different bucket generation"),
2949                _ => {}
2950            }
2951
2952            let set = &store.pools[0].disk_set[0];
2953            let offline = take_set_disks_offline(&store, set, &[0, 1]).await;
2954            let error = store
2955                .get_bucket_info(&bucket, &BucketOptions::default())
2956                .await
2957                .expect_err("read admission must not trust residual directories or cached metadata");
2958            match state {
2959                "missing" => assert_eq!(error, StorageError::ErasureReadQuorum),
2960                "foreign" => assert_eq!(error, StorageError::FileCorrupt),
2961                "incarnation" => assert!(error.to_string().contains("sidecar does not match bucket metadata")),
2962                "corrupt" => assert!(error.to_string().contains("format invalid"), "unexpected corruption error: {error}"),
2963                _ => unreachable!(),
2964            }
2965            restore_set_disks(&store, set, offline).await;
2966        }
2967    }
2968
2969    #[tokio::test]
2970    #[serial]
2971    async fn bucket_info_read_quorum_accepts_persisted_legacy_metadata() {
2972        let (_temp_dir, store) = setup_bucket_quorum_test_env(&[4], None).await;
2973        metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
2974        let bucket = "interop";
2975        store
2976            .make_bucket_on_sets(bucket, &MakeBucketOptions::default())
2977            .await
2978            .expect("legacy bucket directories should exist");
2979        let hex = include_str!("../../tests/fixtures/minio/bucket_metadata.blob.hex")
2980            .split_whitespace()
2981            .collect::<String>();
2982        let body = (0..hex.len())
2983            .step_by(2)
2984            .map(|index| u8::from_str_radix(&hex[index..index + 2], 16).expect("pinned MinIO metadata fixture"))
2985            .collect();
2986        crate::config::com::save_config(store.clone(), &BucketMetadata::new(bucket).save_file_path(), body)
2987            .await
2988            .expect("legacy metadata should be persisted without an incarnation sidecar");
2989
2990        let set = &store.pools[0].disk_set[0];
2991        let offline = take_set_disks_offline(&store, set, &[0, 1]).await;
2992        let info = store
2993            .get_bucket_info(bucket, &BucketOptions::default())
2994            .await
2995            .expect("persisted MinIO metadata should authorize reads at the namespace read quorum");
2996        assert_eq!(info.name, bucket);
2997        assert!(info.versioning);
2998        assert!(info.object_locking);
2999        restore_set_disks(&store, set, offline).await;
3000    }
3001
3002    #[tokio::test]
3003    #[serial]
3004    async fn bucket_namespace_reads_report_missing_when_every_set_is_absent() {
3005        let (_temp_dir, ecstore) = setup_multi_pool_bucket_test_env().await;
3006        let bucket = format!("absent-{}", Uuid::new_v4().simple());
3007
3008        let buckets = ecstore
3009            .list_bucket(&BucketOptions::default())
3010            .await
3011            .expect("an empty healthy namespace should remain listable");
3012        assert!(buckets.iter().all(|entry| entry.name != bucket));
3013
3014        let err = ecstore
3015            .get_bucket_info(&bucket, &BucketOptions::default())
3016            .await
3017            .expect_err("a bucket absent from every set must remain missing");
3018        assert_eq!(err, StorageError::VolumeNotFound);
3019    }
3020
3021    #[tokio::test]
3022    #[serial]
3023    async fn bucket_namespace_reads_fail_closed_when_any_set_loses_quorum() {
3024        let (_temp_dir, ecstore) = setup_multi_pool_bucket_test_env().await;
3025        metadata_sys::init_bucket_metadata_sys(ecstore.clone(), Vec::new()).await;
3026        let bucket = format!("degraded-expansion-{}", Uuid::new_v4().simple());
3027        ecstore.pools[0].disk_set[0]
3028            .make_bucket(&bucket, &MakeBucketOptions::default())
3029            .await
3030            .expect("bucket should be created in the original pool only");
3031        ecstore.pools[1].disk_set[0].disks.write().await[0] = None;
3032        ecstore.pools[1].disk_set[0].disks.write().await[1] = None;
3033        ecstore.pools[1].disk_set[0].disks.write().await[2] = None;
3034
3035        let list_err = ecstore
3036            .list_bucket(&BucketOptions::default())
3037            .await
3038            .expect_err("listing must not hide an unavailable expansion pool");
3039        assert_eq!(list_err, StorageError::ErasureWriteQuorum);
3040
3041        let info_err = ecstore
3042            .get_bucket_info(&bucket, &BucketOptions::default())
3043            .await
3044            .expect_err("bucket validation must fail when an expansion pool is unavailable");
3045        assert_eq!(info_err, StorageError::ErasureReadQuorum);
3046    }
3047
3048    #[tokio::test]
3049    #[serial]
3050    async fn scanner_bucket_listing_unions_every_erasure_set() {
3051        let (_temp_dir, ecstore) = setup_multi_pool_bucket_test_env().await;
3052        let bucket = format!("second-pool-only-{}", Uuid::new_v4().simple());
3053        ecstore.pools[1].disk_set[0]
3054            .make_bucket(&bucket, &MakeBucketOptions::default())
3055            .await
3056            .expect("bucket should be created in the second pool only");
3057
3058        let listing = ecstore
3059            .list_bucket_for_scanner(&crate::storage_api_contracts::bucket::BucketOptions {
3060                no_metadata: true,
3061                ..Default::default()
3062            })
3063            .await
3064            .expect("scanner should enumerate every pool and set");
3065
3066        assert!(listing.topology_complete);
3067        assert!(listing.buckets.iter().any(|entry| entry.name == bucket));
3068        assert_eq!(listing.set_buckets.len(), 2);
3069        assert!(
3070            listing
3071                .set_buckets
3072                .iter()
3073                .find(|scope| scope.pool_index == 0 && scope.set_index == 0)
3074                .is_some_and(|scope| scope.buckets.is_empty())
3075        );
3076        assert!(
3077            listing
3078                .set_buckets
3079                .iter()
3080                .find(|scope| scope.pool_index == 1 && scope.set_index == 0)
3081                .is_some_and(|scope| scope.buckets.iter().any(|entry| entry.name == bucket))
3082        );
3083    }
3084
3085    #[tokio::test]
3086    async fn scanner_bucket_listing_marks_degraded_set_incomplete() {
3087        let (_temp_dir, ecstore) = setup_multi_pool_bucket_test_env().await;
3088        let bucket = format!("degraded-set-{}", Uuid::new_v4().simple());
3089        let set = &ecstore.pools[0].disk_set[0];
3090        set.make_bucket(&bucket, &MakeBucketOptions::default())
3091            .await
3092            .expect("bucket should be created before a disk is removed");
3093        set.disks.write().await[0] = None;
3094
3095        let listing = ecstore
3096            .list_bucket_for_scanner(&crate::storage_api_contracts::bucket::BucketOptions {
3097                no_metadata: true,
3098                ..Default::default()
3099            })
3100            .await
3101            .expect("a degraded set with quorum should still return candidate buckets");
3102
3103        assert!(listing.buckets.iter().any(|entry| entry.name == bucket));
3104        assert!(!listing.topology_complete);
3105    }
3106
3107    #[tokio::test]
3108    async fn scanner_bucket_listing_marks_divergent_disk_views_incomplete() {
3109        let (temp_dir, ecstore) = setup_multi_pool_bucket_test_env().await;
3110        let bucket = format!("divergent-set-{}", Uuid::new_v4().simple());
3111        ecstore.pools[0].disk_set[0]
3112            .make_bucket(&bucket, &MakeBucketOptions::default())
3113            .await
3114            .expect("bucket should be created before disk views diverge");
3115        for disk_index in 0..2 {
3116            tokio::fs::remove_dir_all(temp_dir.path().join(format!("pool0-disk{disk_index}")).join(&bucket))
3117                .await
3118                .expect("test bucket directory should be removed from a minority disk view");
3119        }
3120
3121        let listing = ecstore
3122            .list_bucket_for_scanner(&crate::storage_api_contracts::bucket::BucketOptions {
3123                no_metadata: true,
3124                ..Default::default()
3125            })
3126            .await
3127            .expect("responsive disks should still produce a scanner candidate listing");
3128
3129        assert!(listing.buckets.iter().all(|entry| entry.name != bucket));
3130        assert!(!listing.topology_complete);
3131    }
3132
3133    // These tests share one isolated instance and mutate its bucket metadata;
3134    // serialize them so their assertions cannot observe each other's operations.
3135    #[tokio::test]
3136    #[serial]
3137    async fn bucket_delete_mark_delete_removes_empty_bucket_and_keeps_deleted_marker() {
3138        let (disk_paths, ecstore) = setup_bucket_delete_test_env().await;
3139        let bucket = format!("bucket-mark-delete-{}", Uuid::new_v4().simple());
3140
3141        ecstore
3142            .make_bucket(&bucket, &MakeBucketOptions::default())
3143            .await
3144            .expect("bucket should be created");
3145        for disk_path in &disk_paths {
3146            tokio::fs::create_dir_all(disk_path.join(&bucket).join("empty-directory/nested/leaf"))
3147                .await
3148                .expect("empty directory remnant should be created");
3149        }
3150        assert!(metadata_sys::get_in(&ecstore.ctx, &bucket).await.is_ok());
3151
3152        let generation_before_delete = ecstore.scanner_namespace_mutation_generation();
3153        ecstore
3154            .delete_bucket(
3155                &bucket,
3156                &DeleteBucketOptions {
3157                    srdelete_op: SRBucketDeleteOp::MarkDelete,
3158                    ..Default::default()
3159                },
3160            )
3161            .await
3162            .expect("MarkDelete should remove an empty bucket");
3163        assert_eq!(
3164            ecstore.scanner_namespace_mutation_generation(),
3165            generation_before_delete.saturating_add(2),
3166            "successful bucket deletion should fence scanner namespace activity before and after deletion"
3167        );
3168
3169        assert!(
3170            !any_disk_path_exists(&disk_paths, &bucket).await,
3171            "MarkDelete should remove the bucket volume"
3172        );
3173        assert!(
3174            any_disk_path_exists(&disk_paths, bucket_deleted_marker_volume(&bucket)).await,
3175            "MarkDelete should persist the deleted-bucket marker"
3176        );
3177        assert!(
3178            metadata_sys::get_in(&ecstore.ctx, &bucket).await.is_err(),
3179            "deleted bucket metadata must be removed from the local cache"
3180        );
3181        let buckets = ecstore
3182            .list_bucket(&BucketOptions::default())
3183            .await
3184            .expect("bucket listing should succeed after MarkDelete");
3185        assert!(!buckets.iter().any(|info| info.name == bucket));
3186        ecstore
3187            .delete_all(RUSTFS_META_BUCKET, &bucket_deleted_marker_prefix(&bucket))
3188            .await
3189            .expect("deleted-bucket marker should be removed to simulate a partial failure");
3190        assert!(!any_disk_path_exists(&disk_paths, bucket_deleted_marker_volume(&bucket)).await);
3191        ecstore
3192            .delete_bucket(
3193                &bucket,
3194                &DeleteBucketOptions {
3195                    srdelete_op: SRBucketDeleteOp::MarkDelete,
3196                    ..Default::default()
3197                },
3198            )
3199            .await
3200            .expect("retried MarkDelete should recreate a missing tombstone");
3201        assert!(any_disk_path_exists(&disk_paths, bucket_deleted_marker_volume(&bucket)).await);
3202    }
3203
3204    #[tokio::test]
3205    #[serial]
3206    async fn bucket_delete_mark_delete_rejects_non_empty_bucket_without_force() {
3207        let (disk_paths, ecstore) = setup_bucket_delete_test_env().await;
3208        let bucket = format!("bucket-mark-delete-non-empty-{}", Uuid::new_v4().simple());
3209
3210        create_bucket_with_object(&ecstore, &bucket, "object.txt").await;
3211
3212        let err = ecstore
3213            .delete_bucket(
3214                &bucket,
3215                &DeleteBucketOptions {
3216                    srdelete_op: SRBucketDeleteOp::MarkDelete,
3217                    ..Default::default()
3218                },
3219            )
3220            .await
3221            .expect_err("MarkDelete should reject a non-empty bucket without force");
3222
3223        assert!(matches!(err, StorageError::BucketNotEmpty(name) if name == bucket));
3224        assert!(any_disk_has_object_metadata(&disk_paths, &bucket).await);
3225        assert!(!any_disk_path_exists(&disk_paths, bucket_deleted_marker_volume(&bucket)).await);
3226    }
3227
3228    #[tokio::test]
3229    #[serial]
3230    async fn bucket_delete_mark_delete_rejects_hidden_object_paths_without_force() {
3231        let (disk_paths, ecstore) = setup_bucket_delete_test_env().await;
3232        let bucket = format!("bucket-mark-delete-hidden-{}", Uuid::new_v4().simple());
3233
3234        create_bucket_with_object(&ecstore, &bucket, ".well-known/acme-challenge").await;
3235        let mut reader = PutObjReader::from_vec(b"delete bucket semantics".to_vec());
3236        ecstore
3237            .put_object(&bucket, ".rustfs.sys/object", &mut reader, &ObjectOptions::default())
3238            .await
3239            .expect("second hidden object should be written");
3240
3241        let err = ecstore
3242            .delete_bucket(
3243                &bucket,
3244                &DeleteBucketOptions {
3245                    srdelete_op: SRBucketDeleteOp::MarkDelete,
3246                    ..Default::default()
3247                },
3248            )
3249            .await
3250            .expect_err("MarkDelete should reject a hidden object path without force");
3251
3252        assert!(matches!(err, StorageError::BucketNotEmpty(name) if name == bucket));
3253        assert!(any_disk_has_object_metadata(&disk_paths, &bucket).await);
3254    }
3255
3256    #[tokio::test]
3257    #[serial]
3258    async fn bucket_delete_purge_removes_bucket_data_and_internal_metadata() {
3259        let (disk_paths, ecstore) = setup_bucket_delete_test_env().await;
3260        let bucket = format!("bucket-purge-{}", Uuid::new_v4().simple());
3261        let object = "object.txt";
3262        let metadata_prefix = format!("{RUSTFS_META_BUCKET}/{BUCKET_META_PREFIX}/{bucket}");
3263
3264        create_bucket_with_object(&ecstore, &bucket, object).await;
3265        write_bucket_metadata_marker(&disk_paths, &metadata_prefix).await;
3266        ecstore
3267            .mark_bucket_deleted(&bucket)
3268            .await
3269            .expect("deleted-bucket marker should be created");
3270        assert!(any_disk_path_exists(&disk_paths, &metadata_prefix).await);
3271        assert!(any_disk_path_exists(&disk_paths, bucket_deleted_marker_volume(&bucket)).await);
3272
3273        let generation_before_delete = ecstore.scanner_namespace_mutation_generation();
3274        ecstore
3275            .delete_bucket(
3276                &bucket,
3277                &DeleteBucketOptions {
3278                    force: true,
3279                    srdelete_op: SRBucketDeleteOp::Purge,
3280                    ..Default::default()
3281                },
3282            )
3283            .await
3284            .expect("Purge should force-delete bucket data");
3285        assert_eq!(
3286            ecstore.scanner_namespace_mutation_generation(),
3287            generation_before_delete.saturating_add(2),
3288            "successful bucket purge should fence scanner namespace activity before and after deletion"
3289        );
3290
3291        assert!(!any_disk_path_exists(&disk_paths, &bucket).await, "Purge should remove the bucket volume");
3292        assert!(
3293            !any_disk_path_exists(&disk_paths, &metadata_prefix).await,
3294            "Purge should remove bucket metadata prefix"
3295        );
3296        assert!(
3297            !any_disk_path_exists(&disk_paths, bucket_deleted_marker_volume(&bucket)).await,
3298            "Purge should remove the deleted-bucket marker"
3299        );
3300        assert!(
3301            metadata_sys::get_in(&ecstore.ctx, &bucket).await.is_err(),
3302            "purged bucket metadata must be removed from the local cache"
3303        );
3304        write_bucket_metadata_marker(&disk_paths, &metadata_prefix).await;
3305        ecstore
3306            .mark_bucket_deleted(&bucket)
3307            .await
3308            .expect("stale deleted-bucket marker should be recreated");
3309        ecstore
3310            .delete_bucket(
3311                &bucket,
3312                &DeleteBucketOptions {
3313                    force: true,
3314                    srdelete_op: SRBucketDeleteOp::Purge,
3315                    ..Default::default()
3316                },
3317            )
3318            .await
3319            .expect("retried Purge should remove stale metadata without a bucket volume");
3320        assert!(!any_disk_path_exists(&disk_paths, &metadata_prefix).await);
3321        assert!(!any_disk_path_exists(&disk_paths, bucket_deleted_marker_volume(&bucket)).await);
3322    }
3323
3324    /// Site replication replays make-with-versioning carrying the source's
3325    /// `lockEnabled` against a destination bucket that already exists. Gating the
3326    /// lock enable on `confirmed_missing` returned success while leaving the
3327    /// replica unlocked, so replicated versions could be deleted without the
3328    /// retention the source enforces.
3329    #[tokio::test(flavor = "multi_thread")]
3330    #[serial]
3331    async fn force_create_enables_object_lock_on_an_existing_bucket() {
3332        let (_disk_paths, ecstore) = setup_bucket_delete_test_env().await;
3333        let bucket = format!("bucket-force-lock-{}", Uuid::new_v4().simple());
3334
3335        ecstore
3336            .make_bucket(&bucket, &MakeBucketOptions::default())
3337            .await
3338            .expect("plain bucket should be created");
3339        assert!(
3340            !metadata_sys::get_in(&ecstore.ctx, &bucket)
3341                .await
3342                .expect("metadata should load")
3343                .lock_enabled,
3344            "test setup: the bucket must start unlocked"
3345        );
3346
3347        ecstore
3348            .make_bucket(
3349                &bucket,
3350                &MakeBucketOptions {
3351                    force_create: true,
3352                    lock_enabled: true,
3353                    ..Default::default()
3354                },
3355            )
3356            .await
3357            .expect("force create with lock_enabled should succeed on an existing bucket");
3358
3359        let meta = metadata_sys::get_in(&ecstore.ctx, &bucket)
3360            .await
3361            .expect("metadata should load after the lock enable");
3362        assert!(meta.lock_enabled, "Object Lock must be enabled on the existing bucket");
3363        assert!(
3364            !meta.object_lock_config_xml.is_empty(),
3365            "the Object Lock configuration must be persisted, not just the flag"
3366        );
3367        assert!(
3368            !meta.versioning_config_xml.is_empty(),
3369            "Object Lock requires versioning, so that must be persisted too"
3370        );
3371    }
3372
3373    /// rustfs/rustfs#8003: a force-create (site replication replay, admin
3374    /// import) against a bucket whose legacy migration stopped after the
3375    /// sidecar write must persist `.metadata.bin` under the stored incarnation
3376    /// rather than fail closed or replace the published identity.
3377    #[tokio::test(flavor = "multi_thread")]
3378    #[serial]
3379    async fn issue_8003_force_create_adopts_existing_incarnation_sidecar() {
3380        let (disk_paths, ecstore) = setup_bucket_delete_test_env().await;
3381        let bucket = format!("bucket-8003-sidecar-{}", Uuid::new_v4().simple());
3382
3383        ecstore
3384            .make_bucket(&bucket, &MakeBucketOptions::default())
3385            .await
3386            .expect("bucket should be created");
3387        let stored = metadata_sys::get_in(&ecstore.ctx, &bucket)
3388            .await
3389            .expect("metadata should load")
3390            .bucket_incarnation_id;
3391        assert!(!stored.is_nil());
3392
3393        let metadata_path = format!("{BUCKET_META_PREFIX}/{bucket}/{}", crate::bucket::metadata::BUCKET_METADATA_FILE);
3394        crate::config::com::delete_config(ecstore.clone(), &metadata_path)
3395            .await
3396            .expect("simulate the interrupted migration by removing only .metadata.bin");
3397        assert!(!any_disk_path_exists(&disk_paths, format!("{RUSTFS_META_BUCKET}/{metadata_path}")).await);
3398        metadata_sys::remove_bucket_metadata_in(&ecstore.ctx, &bucket)
3399            .await
3400            .expect("drop the cached copy so the next read hits disk");
3401
3402        ecstore
3403            .make_bucket(
3404                &bucket,
3405                &MakeBucketOptions {
3406                    force_create: true,
3407                    ..Default::default()
3408                },
3409            )
3410            .await
3411            .expect("force create must repair the sidecar-only bucket");
3412
3413        let (repaired, persisted) = metadata_sys::get_config_from_disk_with_presence_in(&ecstore.ctx, &bucket)
3414            .await
3415            .expect("repaired metadata should load");
3416        assert!(persisted, "force create must persist .metadata.bin again");
3417        assert!(repaired.bucket_incarnation_sidecar);
3418        assert_eq!(
3419            repaired.bucket_incarnation_id, stored,
3420            "the repair must keep the incarnation the sidecar already published"
3421        );
3422    }
3423
3424    #[tokio::test(flavor = "multi_thread")]
3425    #[serial]
3426    async fn make_bucket_inherits_default_durability() {
3427        temp_env::async_with_vars([(crate::bucket::durability::ENV_NEW_BUCKET_DURABILITY_MODE, None::<&str>)], async {
3428            let (_disk_paths, ecstore) = setup_bucket_delete_test_env().await;
3429            let bucket = format!("bucket-default-durability-{}", Uuid::new_v4().simple());
3430
3431            ecstore
3432                .make_bucket(&bucket, &MakeBucketOptions::default())
3433                .await
3434                .expect("new bucket should be created");
3435
3436            let metadata = metadata_sys::get_in(&ecstore.ctx, &bucket)
3437                .await
3438                .expect("metadata should load for the new bucket");
3439            assert!(metadata.durability_config().is_none());
3440        })
3441        .await;
3442    }
3443
3444    #[tokio::test(flavor = "multi_thread")]
3445    #[serial]
3446    async fn force_create_existing_bucket_keeps_durability_override() {
3447        let (_disk_paths, ecstore) = setup_bucket_delete_test_env().await;
3448        let bucket = format!("bucket-force-durability-{}", Uuid::new_v4().simple());
3449
3450        temp_env::async_with_vars([(crate::bucket::durability::ENV_NEW_BUCKET_DURABILITY_MODE, Some("inherit"))], async {
3451            ecstore
3452                .make_bucket(&bucket, &MakeBucketOptions::default())
3453                .await
3454                .expect("plain bucket should be created without a durability override");
3455        })
3456        .await;
3457        assert!(
3458            metadata_sys::get_in(&ecstore.ctx, &bucket)
3459                .await
3460                .expect("metadata should load after initial create")
3461                .durability_config()
3462                .is_none(),
3463            "test setup: the existing bucket must start without an override"
3464        );
3465
3466        temp_env::async_with_vars([(crate::bucket::durability::ENV_NEW_BUCKET_DURABILITY_MODE, None::<&str>)], async {
3467            ecstore
3468                .make_bucket(
3469                    &bucket,
3470                    &MakeBucketOptions {
3471                        force_create: true,
3472                        lock_enabled: true,
3473                        ..Default::default()
3474                    },
3475                )
3476                .await
3477                .expect("force create should update existing bucket metadata");
3478        })
3479        .await;
3480
3481        let metadata = metadata_sys::get_in(&ecstore.ctx, &bucket)
3482            .await
3483            .expect("metadata should load after force create");
3484        assert!(metadata.lock_enabled, "force create sanity check: Object Lock should be enabled");
3485        assert!(
3486            metadata.durability_config().is_none(),
3487            "force create must not apply the new-bucket default to existing bucket metadata"
3488        );
3489    }
3490
3491    /// `DeleteBucket`'s emptiness check is a raw disk scan (`has_xlmeta_files`),
3492    /// not an S3-level listing, so "the client drained the bucket" and "the
3493    /// bucket is deletable" are two different contracts. Nothing pinned the
3494    /// second one, which is how the s3-tests lane ended up failing 219 cases on
3495    /// `nuke_prefixed_buckets` while every test body passed.
3496    #[tokio::test(flavor = "multi_thread")]
3497    #[serial]
3498    async fn bucket_delete_succeeds_after_the_last_object_version_is_deleted() {
3499        let (disk_paths, ecstore) = setup_bucket_delete_test_env().await;
3500        let bucket = format!("bucket-delete-after-drain-{}", Uuid::new_v4().simple());
3501        let object = "object.txt";
3502
3503        create_bucket_with_object(&ecstore, &bucket, object).await;
3504
3505        ecstore
3506            .delete_object(&bucket, object, ObjectOptions::default())
3507            .await
3508            .expect("client delete of the only object should succeed");
3509
3510        assert!(
3511            !any_disk_has_object_metadata(&disk_paths, &bucket).await,
3512            "deleting the last version must not leave xl.meta on disk: DeleteBucket scans the raw \
3513             bucket directory, so residue here is reported to clients as BucketNotEmpty"
3514        );
3515
3516        ecstore
3517            .delete_bucket(&bucket, &DeleteBucketOptions::default())
3518            .await
3519            .expect("DeleteBucket must succeed once the client has drained the bucket");
3520    }
3521
3522    #[tokio::test(flavor = "multi_thread")]
3523    #[serial]
3524    async fn bucket_delete_defers_zero_evidence_diagnostic_timeout_to_physical_empty_check() {
3525        let (disk_paths, ecstore) = setup_bucket_delete_test_env().await;
3526        let bucket = format!("bucket-del-diag-{}", Uuid::new_v4().simple());
3527
3528        ecstore
3529            .make_bucket(&bucket, &MakeBucketOptions::default())
3530            .await
3531            .expect("empty bucket should be created");
3532        let first_io_started = Arc::new(AtomicBool::new(false));
3533        let diagnostic_budget = BucketDeleteDiagnosticBudget::new().with_first_io_delay(
3534            BUCKET_DELETE_DIAGNOSTIC_MAX_ELAPSED + Duration::from_millis(100),
3535            first_io_started.clone(),
3536        );
3537
3538        ecstore
3539            .handle_delete_bucket_with_diagnostic_budget(&bucket, &DeleteBucketOptions::default(), diagnostic_budget)
3540            .await
3541            .expect("a zero-evidence diagnostic timeout must defer to the physical empty check");
3542
3543        assert!(
3544            first_io_started.load(Ordering::SeqCst),
3545            "the regression must delay the first diagnostic read_dir"
3546        );
3547        assert!(
3548            !any_disk_path_exists(&disk_paths, &bucket).await,
3549            "the physical empty check should allow deletion of the actually empty bucket"
3550        );
3551    }
3552
3553    #[tokio::test(flavor = "multi_thread")]
3554    #[serial]
3555    async fn bucket_delete_preserves_unobserved_residue_after_diagnostic_timeout() {
3556        let (disk_paths, ecstore) = setup_bucket_delete_test_env().await;
3557        let bucket = format!("bucket-del-residue-{}", Uuid::new_v4().simple());
3558        let object = "object.txt";
3559
3560        ecstore
3561            .make_bucket(&bucket, &MakeBucketOptions::default())
3562            .await
3563            .expect("bucket should be created");
3564        write_metadata_less_part_on_all_disks(&disk_paths, &bucket, object).await;
3565        let first_io_started = Arc::new(AtomicBool::new(false));
3566        let diagnostic_budget = BucketDeleteDiagnosticBudget::new().with_first_io_delay(
3567            BUCKET_DELETE_DIAGNOSTIC_MAX_ELAPSED + Duration::from_millis(100),
3568            first_io_started.clone(),
3569        );
3570
3571        let err = ecstore
3572            .handle_delete_bucket_with_diagnostic_budget(&bucket, &DeleteBucketOptions::default(), diagnostic_budget)
3573            .await
3574            .expect_err("physical empty-bucket enforcement must reject unobserved residue");
3575        assert!(
3576            first_io_started.load(Ordering::SeqCst),
3577            "the regression must time out before observing the residue"
3578        );
3579        match &err {
3580            StorageError::BucketNotEmpty(err_bucket) | StorageError::BucketNotEmptyWithDetails { bucket: err_bucket, .. } => {
3581                assert_eq!(err_bucket, &bucket)
3582            }
3583            other => panic!("expected an S3-compatible BucketNotEmpty error, got {other:?}"),
3584        }
3585        assert!(
3586            any_disk_path_exists(&disk_paths, Path::new(&bucket).join(object)).await,
3587            "physical empty-bucket enforcement must preserve unobserved residue"
3588        );
3589
3590        ecstore
3591            .delete_bucket(
3592                &bucket,
3593                &DeleteBucketOptions {
3594                    force: true,
3595                    ..Default::default()
3596                },
3597            )
3598            .await
3599            .expect("explicit force should clean up the test-owned residue");
3600    }
3601
3602    #[tokio::test(flavor = "multi_thread")]
3603    #[serial]
3604    async fn bucket_delete_succeeds_after_listing_and_deleting_an_unversioned_overwrite() {
3605        let (disk_paths, ecstore) = setup_bucket_delete_test_env().await;
3606        let bucket = format!("bucket-delete-after-overwrite-{}", Uuid::new_v4().simple());
3607        let object = "object.txt";
3608
3609        ecstore
3610            .make_bucket(&bucket, &MakeBucketOptions::default())
3611            .await
3612            .expect("unversioned bucket should be created");
3613
3614        let mut first_reader = PutObjReader::from_vec(b"version A".to_vec());
3615        let first = ecstore
3616            .put_object(&bucket, object, &mut first_reader, &ObjectOptions::default())
3617            .await
3618            .expect("version A should be written");
3619        let mut second_reader = PutObjReader::from_vec(b"version B".to_vec());
3620        let second = ecstore
3621            .put_object(&bucket, object, &mut second_reader, &ObjectOptions::default())
3622            .await
3623            .expect("version B should overwrite version A");
3624        assert_ne!(first.data_dir, second.data_dir, "the overwrite must publish a new body generation");
3625
3626        let listing = ecstore
3627            .clone()
3628            .list_object_versions(&bucket, "", None, None, None, 1000)
3629            .await
3630            .expect("the overwritten object should remain listable for teardown");
3631        assert_eq!(listing.objects.len(), 1, "an unversioned overwrite should expose one current version");
3632        let current = &listing.objects[0];
3633        assert_eq!(current.name, object);
3634        assert!(current.is_latest, "the listed null version must be current");
3635        assert_eq!(current.version_id, None, "an unversioned object must be exposed as the null version");
3636        assert_eq!(
3637            current.data_dir, second.data_dir,
3638            "listing must expose version B, not the overwritten body"
3639        );
3640
3641        for version in listing.objects {
3642            let version_id = version.version_id.map(|version_id| version_id.to_string());
3643            ecstore
3644                .delete_object(
3645                    &bucket,
3646                    &version.name,
3647                    ObjectOptions {
3648                        version_id,
3649                        ..Default::default()
3650                    },
3651                )
3652                .await
3653                .expect("each version returned by teardown listing should be deletable");
3654        }
3655
3656        assert!(
3657            !any_disk_has_object_metadata(&disk_paths, &bucket).await,
3658            "deleting the listed null version must remove every xl.meta"
3659        );
3660        ecstore
3661            .delete_bucket(&bucket, &DeleteBucketOptions::default())
3662            .await
3663            .expect("DeleteBucket should succeed after the listed overwrite is deleted");
3664    }
3665
3666    #[tokio::test]
3667    #[serial]
3668    async fn bucket_delete_default_s3_delete_still_rejects_non_empty_bucket() {
3669        let (disk_paths, ecstore) = setup_bucket_delete_test_env().await;
3670        let bucket = format!("bucket-s3-delete-{}", Uuid::new_v4().simple());
3671        let object = "object.txt";
3672
3673        create_bucket_with_object(&ecstore, &bucket, object).await;
3674
3675        let generation_before_delete = ecstore.scanner_namespace_mutation_generation();
3676        let err = ecstore
3677            .delete_bucket(&bucket, &DeleteBucketOptions::default())
3678            .await
3679            .expect_err("default S3 DeleteBucket should reject non-empty buckets");
3680
3681        assert!(matches!(err, StorageError::BucketNotEmpty(name) if name == bucket));
3682        assert_eq!(
3683            ecstore.scanner_namespace_mutation_generation(),
3684            generation_before_delete,
3685            "failed bucket deletion must not advance scanner namespace activity"
3686        );
3687        assert!(
3688            any_disk_has_object_metadata(&disk_paths, &bucket).await,
3689            "failed default S3 DeleteBucket must keep object data"
3690        );
3691        assert!(
3692            metadata_sys::get_in(&ecstore.ctx, &bucket).await.is_ok(),
3693            "failed default S3 DeleteBucket must keep metadata cache"
3694        );
3695    }
3696
3697    #[tokio::test(flavor = "multi_thread")]
3698    #[serial]
3699    async fn bucket_delete_reports_metadata_less_residue_without_removing_it() {
3700        let (disk_paths, ecstore) = setup_bucket_delete_test_env().await;
3701        let bucket = format!("bucket-delete-orphan-datadir-{}", Uuid::new_v4().simple());
3702        let object = "object.txt";
3703
3704        ecstore
3705            .make_bucket(&bucket, &MakeBucketOptions::default())
3706            .await
3707            .expect("bucket should be created");
3708        write_metadata_less_part_on_all_disks(&disk_paths, &bucket, object).await;
3709        assert!(
3710            !any_disk_has_object_metadata(&disk_paths, &bucket).await,
3711            "test setup must reproduce a bucket with data dirs but no xl.meta"
3712        );
3713
3714        let err = ecstore
3715            .delete_bucket(&bucket, &DeleteBucketOptions::default())
3716            .await
3717            .expect_err("metadata-less data dirs must block ordinary DeleteBucket");
3718
3719        match err {
3720            StorageError::BucketNotEmptyWithDetails {
3721                bucket: err_bucket,
3722                details,
3723            } => {
3724                assert_eq!(err_bucket, bucket);
3725                assert!(
3726                    details.contains("metadata-less on-disk residue"),
3727                    "diagnostic should identify metadata-less residue, got: {details}"
3728                );
3729                assert!(details.contains("files="), "diagnostic should include a bounded count");
3730                assert!(details.contains("uuid_data_dirs="), "diagnostic should include data-dir count");
3731            }
3732            other => panic!("expected detailed BucketNotEmpty, got {other:?}"),
3733        }
3734        assert!(
3735            any_disk_path_exists(&disk_paths, Path::new(&bucket).join(object)).await,
3736            "ordinary DeleteBucket must preserve metadata-less data until an explicit operator action"
3737        );
3738    }
3739
3740    #[tokio::test]
3741    #[serial]
3742    async fn bucket_delete_fences_put_started_after_empty_scan() {
3743        let (_, ecstore) = setup_bucket_delete_test_env().await;
3744        let bucket = format!("bucket-delete-empty-scan-race-{}", Uuid::new_v4().simple());
3745        let object = "committed-after-empty-scan";
3746        let payload = b"object started after DeleteBucket empty scan".to_vec();
3747
3748        ecstore
3749            .make_bucket(&bucket, &MakeBucketOptions::default())
3750            .await
3751            .expect("bucket should be created");
3752
3753        let barrier = install_delete_bucket_empty_scan_barrier();
3754        let delete_store = ecstore.clone();
3755        let delete_bucket = bucket.clone();
3756        let delete = tokio::spawn(async move {
3757            delete_store
3758                .delete_bucket(&delete_bucket, &DeleteBucketOptions::default())
3759                .await
3760        });
3761        barrier.wait_until_paused().await;
3762
3763        let put_store = ecstore.clone();
3764        let put_bucket = bucket.clone();
3765        let mut put = tokio::spawn(async move {
3766            let mut put_reader = PutObjReader::from_vec(payload);
3767            put_store
3768                .put_object(&put_bucket, object, &mut put_reader, &ObjectOptions::default())
3769                .await
3770        });
3771        assert!(
3772            tokio::time::timeout(Duration::from_millis(100), &mut put).await.is_err(),
3773            "PUT must wait behind the DeleteBucket lifecycle fence"
3774        );
3775
3776        barrier.release();
3777        delete
3778            .await
3779            .expect("DeleteBucket task should join")
3780            .expect("DeleteBucket should commit while holding the lifecycle fence");
3781        let err = put
3782            .await
3783            .expect("PUT task should join")
3784            .expect_err("PUT must not recreate an object in the deleted bucket");
3785        assert!(matches!(err, StorageError::BucketNotFound(name) if name == bucket));
3786    }
3787
3788    #[tokio::test]
3789    #[serial]
3790    async fn bucket_recreation_does_not_publish_unverified_usage() {
3791        let (_, ecstore) = setup_bucket_delete_test_env().await;
3792        let bucket = format!("bucket-usage-generation-{}", Uuid::new_v4().simple());
3793        ecstore
3794            .make_bucket(&bucket, &MakeBucketOptions::default())
3795            .await
3796            .expect("bucket should be created");
3797
3798        let mut snapshot = DataUsageInfo {
3799            last_update: Some(SystemTime::now()),
3800            buckets_count: 1,
3801            ..Default::default()
3802        };
3803        snapshot.buckets_usage.insert(
3804            bucket.clone(),
3805            BucketUsageInfo {
3806                size: 42,
3807                objects_count: 1,
3808                versions_count: 1,
3809                ..Default::default()
3810            },
3811        );
3812        snapshot.usage_snapshot_complete = true;
3813        snapshot.bucket_sizes.insert(bucket.clone(), 42);
3814        snapshot.calculate_totals();
3815        crate::data_usage::store_data_usage_in_backend(snapshot, ecstore.clone())
3816            .await
3817            .expect("usage snapshot should be stored");
3818
3819        ecstore
3820            .delete_bucket(&bucket, &DeleteBucketOptions::default())
3821            .await
3822            .expect("empty bucket should be deleted");
3823        let deleted = crate::data_usage::load_data_usage_from_backend(ecstore.clone())
3824            .await
3825            .expect("usage snapshot should remain readable");
3826        assert!(!deleted.buckets_usage.contains_key(&bucket));
3827
3828        ecstore
3829            .make_bucket(&bucket, &MakeBucketOptions::default())
3830            .await
3831            .expect("same bucket name should be recreated after delete returns");
3832        crate::data_usage::record_bucket_object_write_memory(&bucket, None, 84).await;
3833
3834        let mut recreated = crate::data_usage::load_data_usage_from_backend(ecstore.clone())
3835            .await
3836            .expect("recreated bucket usage base should load");
3837        crate::data_usage::apply_bucket_usage_memory_overlay(&mut recreated).await;
3838        assert!(
3839            !recreated.buckets_usage.contains_key(&bucket),
3840            "a request-path delta without an authoritative baseline must remain unavailable"
3841        );
3842        assert_eq!(crate::data_usage::get_bucket_usage_memory(&bucket).await, None);
3843    }
3844
3845    #[tokio::test]
3846    #[serial]
3847    async fn bucket_create_removes_stale_usage_before_physical_creation() {
3848        let (_, ecstore) = setup_bucket_delete_test_env().await;
3849        let bucket = format!("bucket-create-stale-usage-{}", Uuid::new_v4().simple());
3850        let mut snapshot = DataUsageInfo {
3851            last_update: Some(SystemTime::now()),
3852            buckets_count: 1,
3853            ..Default::default()
3854        };
3855        snapshot.buckets_usage.insert(
3856            bucket.clone(),
3857            BucketUsageInfo {
3858                size: 42,
3859                objects_count: 1,
3860                versions_count: 1,
3861                ..Default::default()
3862            },
3863        );
3864        snapshot.usage_snapshot_complete = true;
3865        snapshot.bucket_sizes.insert(bucket.clone(), 42);
3866        snapshot.calculate_totals();
3867        crate::data_usage::store_data_usage_in_backend(snapshot, ecstore.clone())
3868            .await
3869            .expect("stale usage fixture should be stored");
3870
3871        ecstore
3872            .make_bucket(&bucket, &MakeBucketOptions::default())
3873            .await
3874            .expect("CreateBucket should succeed");
3875
3876        let persisted = crate::data_usage::load_data_usage_from_backend(ecstore.clone())
3877            .await
3878            .expect("usage snapshot should remain readable");
3879        assert!(
3880            !persisted.buckets_usage.contains_key(&bucket),
3881            "a newly created bucket must not inherit the predecessor generation's usage"
3882        );
3883        assert!(
3884            ecstore.get_bucket_info(&bucket, &BucketOptions::default()).await.is_ok(),
3885            "physical creation should happen after the usage fence succeeds"
3886        );
3887    }
3888
3889    #[tokio::test]
3890    #[serial]
3891    async fn failed_create_rollback_does_not_run_unfenced_usage_cleanup() {
3892        let (_, ecstore) = setup_bucket_delete_test_env().await;
3893        let bucket = format!("bucket-create-rollback-{}", Uuid::new_v4().simple());
3894        ecstore
3895            .peer_sys
3896            .make_bucket(&bucket, &MakeBucketOptions::default())
3897            .await
3898            .expect("the partial-create fixture should expose a physical bucket");
3899
3900        let mut snapshot = DataUsageInfo {
3901            last_update: Some(SystemTime::now()),
3902            buckets_count: 1,
3903            ..Default::default()
3904        };
3905        snapshot.buckets_usage.insert(
3906            bucket.clone(),
3907            BucketUsageInfo {
3908                size: 42,
3909                objects_count: 1,
3910                versions_count: 1,
3911                ..Default::default()
3912            },
3913        );
3914        snapshot.usage_snapshot_complete = true;
3915        snapshot.bucket_sizes.insert(bucket.clone(), 42);
3916        snapshot.calculate_totals();
3917        crate::data_usage::store_data_usage_in_backend(snapshot, ecstore.clone())
3918            .await
3919            .expect("the usage fixture should be stored");
3920        crate::bucket::metadata::save_bucket_incarnation(ecstore.clone(), &bucket, Uuid::new_v4())
3921            .await
3922            .expect("partial create should have an incarnation sidecar");
3923
3924        ecstore.rollback_failed_bucket_creation(&bucket, None, None).await;
3925
3926        assert!(
3927            ecstore
3928                .peer_sys
3929                .get_bucket_info(&bucket, &BucketOptions::default())
3930                .await
3931                .is_err(),
3932            "failed-create rollback should remove the partial physical bucket"
3933        );
3934        assert!(
3935            crate::bucket::metadata::load_bucket_incarnation(ecstore.clone(), &bucket)
3936                .await
3937                .expect("rollback sidecar lookup should succeed")
3938                .is_none(),
3939            "failed-create rollback must remove the orphan incarnation sidecar"
3940        );
3941        let persisted = crate::data_usage::load_data_usage_from_backend(ecstore.clone())
3942            .await
3943            .expect("the usage snapshot should remain readable");
3944        assert!(
3945            persisted.buckets_usage.contains_key(&bucket),
3946            "failed-create rollback must not start an unfenced usage cleanup"
3947        );
3948
3949        crate::data_usage::store_data_usage_in_backend(DataUsageInfo::default(), ecstore.clone())
3950            .await
3951            .expect("the rollback usage fixture should be cleared");
3952    }
3953
3954    #[tokio::test]
3955    #[serial]
3956    async fn bucket_create_fails_closed_when_usage_snapshot_cannot_be_fenced() {
3957        let (_, ecstore) = setup_bucket_delete_test_env().await;
3958        let deleted_bucket = format!("bucket-delete-corrupt-usage-{}", Uuid::new_v4().simple());
3959        ecstore
3960            .make_bucket(&deleted_bucket, &MakeBucketOptions::default())
3961            .await
3962            .expect("bucket should be created before corrupting usage");
3963
3964        let usage_path = format!("{BUCKET_META_PREFIX}/{DATA_USAGE_OBJECT_NAME}");
3965        crate::config::com::save_config(ecstore.clone(), &usage_path, b"{".to_vec())
3966            .await
3967            .expect("corrupt usage fixture should be stored");
3968
3969        ecstore
3970            .delete_bucket(&deleted_bucket, &DeleteBucketOptions::default())
3971            .await
3972            .expect("usage snapshot corruption must not block DeleteBucket");
3973        assert!(
3974            ecstore
3975                .get_bucket_info(&deleted_bucket, &crate::storage_api_contracts::bucket::BucketOptions::default())
3976                .await
3977                .is_err(),
3978            "successful DeleteBucket must remove the physical bucket"
3979        );
3980
3981        let new_bucket = format!("bucket-create-corrupt-usage-{}", Uuid::new_v4().simple());
3982        let create_err = ecstore
3983            .make_bucket(&new_bucket, &MakeBucketOptions::default())
3984            .await
3985            .expect_err("MakeBucket must fail when stale usage cannot be fenced");
3986        assert!(!create_err.to_string().is_empty(), "the usage snapshot failure should be surfaced");
3987        assert!(
3988            ecstore
3989                .get_bucket_info(&new_bucket, &crate::storage_api_contracts::bucket::BucketOptions::default())
3990                .await
3991                .is_err(),
3992            "failed MakeBucket must not expose a new physical bucket"
3993        );
3994
3995        let restored = serde_json::to_vec(&DataUsageInfo {
3996            last_update: Some(SystemTime::now()),
3997            ..Default::default()
3998        })
3999        .expect("restored usage fixture should encode");
4000        crate::config::com::save_config(ecstore, &usage_path, restored)
4001            .await
4002            .expect("usage fixture should be restored after the failure-path test");
4003    }
4004
4005    /// Capture this module's log the way a stock deployment filters it.
4006    fn bucket_logs_at_default_level(emit: impl FnOnce()) -> String {
4007        use std::sync::{Arc, Mutex};
4008        use tracing_subscriber::EnvFilter;
4009        use tracing_subscriber::fmt::MakeWriter;
4010        use tracing_subscriber::layer::SubscriberExt;
4011
4012        #[derive(Clone, Default)]
4013        struct CapturedLogs {
4014            buffer: Arc<Mutex<Vec<u8>>>,
4015        }
4016        struct CapturedLogWriter {
4017            buffer: Arc<Mutex<Vec<u8>>>,
4018        }
4019        impl std::io::Write for CapturedLogWriter {
4020            fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
4021                self.buffer
4022                    .lock()
4023                    .expect("captured logs mutex should not be poisoned")
4024                    .extend_from_slice(buf);
4025                Ok(buf.len())
4026            }
4027            fn flush(&mut self) -> std::io::Result<()> {
4028                Ok(())
4029            }
4030        }
4031        impl<'a> MakeWriter<'a> for CapturedLogs {
4032            type Writer = CapturedLogWriter;
4033            fn make_writer(&'a self) -> Self::Writer {
4034                CapturedLogWriter {
4035                    buffer: Arc::clone(&self.buffer),
4036                }
4037            }
4038        }
4039
4040        let logs = CapturedLogs::default();
4041        let subscriber = tracing_subscriber::registry()
4042            .with(EnvFilter::new(rustfs_config::DEFAULT_LOG_LEVEL))
4043            .with(
4044                tracing_subscriber::fmt::layer()
4045                    .with_writer(logs.clone())
4046                    .with_ansi(false)
4047                    .without_time(),
4048            );
4049        let _guard = tracing::subscriber::set_default(subscriber);
4050        let _callsite_pin = crate::test_tracing::pin_callsite_interest_for_test();
4051
4052        emit();
4053
4054        let buffer = logs
4055            .buffer
4056            .lock()
4057            .expect("captured logs mutex should not be poisoned")
4058            .clone();
4059        String::from_utf8(buffer).expect("captured logs should be valid UTF-8")
4060    }
4061
4062    fn residue_sample(sample: &str) -> BucketMetadataLessResidue {
4063        BucketMetadataLessResidue {
4064            files: 1,
4065            entries_scanned: 3,
4066            sample: Some(sample.to_string()),
4067            ..Default::default()
4068        }
4069    }
4070
4071    /// `DeleteBucket` answers from a raw per-disk residue scan, so it can refuse
4072    /// for a reason no S3 request can observe. When that happens the server's
4073    /// own record of which residue blocked it must survive the default log
4074    /// filter — otherwise a client that drained the bucket sees `BucketNotEmpty`
4075    /// and the server log holds no trace of the refusal at all.
4076    #[test]
4077    fn unreachable_residue_is_reported_at_the_default_log_level() {
4078        for kind in [
4079            BucketDeleteBlockerKind::UnknownXlMeta,
4080            BucketDeleteBlockerKind::OrphanDirectory,
4081            BucketDeleteBlockerKind::DiagnosticBudgetExceeded,
4082        ] {
4083            let logs = bucket_logs_at_default_level(|| {
4084                record_bucket_delete_blocker("drained-bucket", kind, &residue_sample("obj/8f2c/xl.meta"));
4085            });
4086
4087            assert!(logs.contains("drained-bucket"), "the bucket must be named for {kind:?}: {logs}");
4088            assert!(logs.contains(kind.as_str()), "the blocker kind must be named for {kind:?}: {logs}");
4089            assert!(
4090                logs.contains("obj/8f2c/xl.meta"),
4091                "the residue sample is the only pointer to the leftover state for {kind:?}: {logs}"
4092            );
4093        }
4094    }
4095
4096    /// A bucket that genuinely still holds a version is an ordinary 409: the
4097    /// client can list and delete what is left, so this must not be reported as
4098    /// a server-side fault.
4099    #[test]
4100    fn client_visible_blockers_stay_below_the_default_log_level() {
4101        for kind in [
4102            BucketDeleteBlockerKind::VisibleVersion,
4103            BucketDeleteBlockerKind::TierFreeVersion,
4104        ] {
4105            let logs = bucket_logs_at_default_level(|| {
4106                record_bucket_delete_blocker("still-full-bucket", kind, &residue_sample("obj/xl.meta"));
4107            });
4108
4109            assert!(logs.is_empty(), "an ordinary non-empty bucket must not log an error for {kind:?}: {logs}");
4110        }
4111    }
4112
4113    #[test]
4114    fn blocker_kinds_split_by_whether_the_client_can_reach_the_residue() {
4115        assert!(BucketDeleteBlockerKind::VisibleVersion.is_client_visible());
4116        assert!(BucketDeleteBlockerKind::TierFreeVersion.is_client_visible());
4117        assert!(!BucketDeleteBlockerKind::UnknownXlMeta.is_client_visible());
4118        assert!(!BucketDeleteBlockerKind::OrphanDirectory.is_client_visible());
4119        assert!(!BucketDeleteBlockerKind::DiagnosticBudgetExceeded.is_client_visible());
4120    }
4121}