use super::*;
use crate::bucket::{
metadata::{BUCKET_TABLE_RESERVED_PREFIX, table_bucket_catalog_metadata_prefix},
utils::is_meta_bucketname,
};
use crate::error::is_err_bucket_not_found;
use crate::runtime::sources as runtime_sources;
use crate::set_disk::{BucketInfoQuorum, get_lock_acquire_timeout};
use crate::storage_api_contracts::bucket::{BUCKET_LIFECYCLE_LOCK_OBJECT, SRBucketDeleteOp};
use crate::storage_api_contracts::namespace::NamespaceLocking as _;
use futures::stream::{self, StreamExt};
use rustfs_policy::policy::BucketPolicy;
use std::collections::BTreeMap;
use std::future::Future;
const DELETED_BUCKETS_PREFIX: &str = ".deleted";
const SCANNER_BUCKET_LIST_SET_CONCURRENCY: usize = 4;
const EVENT_BUCKET_DELETE_BLOCKED: &str = "bucket_delete_blocked";
const EVENT_BUCKET_DELETE_ROLLBACK_FAILED: &str = "bucket_delete_rollback_failed";
fn record_bucket_delete_blocker(bucket: &str, kind: BucketDeleteBlockerKind, residue: &BucketMetadataLessResidue) {
metrics::counter!("rustfs_bucket_delete_blockers_total", "kind" => kind.as_str()).increment(1);
if kind.is_client_visible() {
warn!(
event = EVENT_BUCKET_DELETE_BLOCKED,
component = "ecstore",
subsystem = "bucket",
bucket,
blocker = kind.as_str(),
files = residue.files,
uuid_data_dirs = residue.uuid_data_dirs,
entries_scanned = residue.entries_scanned,
diagnostic_bytes_read = residue.diagnostic_bytes_read,
diagnostic_truncated = residue.diagnostic_truncated,
sample = residue.sample.as_deref().unwrap_or("<none>"),
"Bucket deletion was blocked by durable local state"
);
return;
}
error!(
event = EVENT_BUCKET_DELETE_BLOCKED,
component = "ecstore",
subsystem = "bucket",
bucket,
blocker = kind.as_str(),
files = residue.files,
uuid_data_dirs = residue.uuid_data_dirs,
entries_scanned = residue.entries_scanned,
diagnostic_bytes_read = residue.diagnostic_bytes_read,
diagnostic_truncated = residue.diagnostic_truncated,
sample = residue.sample.as_deref().unwrap_or("<none>"),
"Bucket deletion was blocked by residue the client cannot reach through the S3 API"
);
}
fn bucket_list_set_concurrency(set_count: usize, max_concurrency: usize) -> usize {
set_count.clamp(1, max_concurrency.max(1))
}
fn should_override_created_from_metadata(created: OffsetDateTime) -> bool {
created != OffsetDateTime::UNIX_EPOCH
}
fn validate_table_bucket_delete_allowed(
bucket: &str,
table_bucket_enabled: bool,
table_catalog_metadata_exists: bool,
) -> Result<()> {
if table_bucket_enabled && table_catalog_metadata_exists {
return Err(StorageError::BucketNotEmpty(bucket.to_string()));
}
Ok(())
}
async fn table_catalog_metadata_exists(ctx: &crate::runtime::instance::InstanceContext, bucket: &str) -> Result<bool> {
let local_disks = runtime_sources::local_disks_in(ctx).await;
for disk in local_disks.iter() {
let Some(bucket_path) = disk.get_bucket_path_for_io_if_local(bucket) else {
continue;
};
let catalog_path = bucket_path?.join(BUCKET_TABLE_RESERVED_PREFIX);
if has_xlmeta_files(&catalog_path).await? {
return Ok(true);
}
}
Ok(false)
}
async fn validate_table_bucket_delete_guard(ctx: &crate::runtime::instance::InstanceContext, bucket: &str) -> Result<()> {
let table_bucket_enabled = metadata_sys::get_in(ctx, bucket)
.await
.is_ok_and(|metadata| metadata.table_bucket_enabled());
if table_bucket_enabled {
validate_table_bucket_delete_allowed(bucket, true, table_catalog_metadata_exists(ctx, bucket).await?)?;
}
Ok(())
}
fn bucket_delete_metadata_cleanup_prefixes(bucket: &str) -> [String; 2] {
[
table_bucket_catalog_metadata_prefix(bucket),
format!("{BUCKET_META_PREFIX}/{bucket}"),
]
}
fn bucket_deleted_marker_prefix(bucket: &str) -> String {
format!("{BUCKET_META_PREFIX}/{DELETED_BUCKETS_PREFIX}/{bucket}")
}
fn bucket_deleted_marker_volume(bucket: &str) -> String {
format!("{RUSTFS_META_BUCKET}/{}", bucket_deleted_marker_prefix(bucket))
}
pub(crate) async fn await_bucket_namespace_operation<T, F>(
guard: Option<&rustfs_lock::NamespaceLockGuard>,
bucket: &str,
operation: &'static str,
future: F,
) -> Result<T>
where
F: Future<Output = Result<T>>,
{
let Some(guard) = guard else {
return future.await;
};
if guard.is_lock_lost() {
return Err(StorageError::other(format!(
"bucket namespace lock was lost before {operation}: {bucket}"
)));
}
tokio::select! {
biased;
_ = guard.lock_lost_notified() => Err(StorageError::other(format!(
"bucket namespace lock was lost during {operation}: {bucket}"
))),
result = future => result,
}
}
async fn await_bucket_lifecycle_operation<T, F>(
lifecycle_guard: Option<&rustfs_lock::NamespaceLockGuard>,
namespace_guard: Option<&rustfs_lock::NamespaceLockGuard>,
bucket: &str,
operation: &'static str,
future: F,
) -> Result<T>
where
F: Future<Output = Result<T>>,
{
await_bucket_namespace_operation(
lifecycle_guard,
bucket,
operation,
await_bucket_namespace_operation(namespace_guard, bucket, operation, future),
)
.await
}
async fn run_bucket_usage_cleanup<F>(guard: Option<&rustfs_lock::NamespaceLockGuard>, bucket: &str, future: F) -> Result<()>
where
F: Future<Output = Result<()>>,
{
await_bucket_namespace_operation(guard, bucket, "bucket usage cleanup", future).await
}
async fn run_physical_bucket_deletion<F>(guard: Option<&rustfs_lock::NamespaceLockGuard>, bucket: &str, future: F) -> Result<()>
where
F: Future<Output = Result<()>>,
{
list_objects::observe_scanner_namespace_mutations(bucket, 1);
await_bucket_namespace_operation(guard, bucket, "physical bucket deletion", future).await
}
async fn bucket_delete_local_blocker(
ctx: &crate::runtime::instance::InstanceContext,
bucket: &str,
budget: &mut BucketDeleteDiagnosticBudget,
) -> Result<Option<StorageError>> {
let local_disks = runtime_sources::local_disks_in(ctx).await;
let mut residue = BucketMetadataLessResidue::default();
for disk in local_disks.iter() {
let Some(bucket_path) = disk.get_bucket_path_for_io_if_local(bucket) else {
continue;
};
let scan = scan_metadata_less_residue_with_budget(&bucket_path?, budget).await?;
if scan.xlmeta_found {
record_bucket_delete_blocker(bucket, scan.xlmeta_blocker.unwrap_or(BucketDeleteBlockerKind::UnknownXlMeta), &scan);
return Ok(Some(StorageError::BucketNotEmpty(bucket.to_string())));
}
residue.files = residue.files.saturating_add(scan.files);
residue.uuid_data_dirs = residue.uuid_data_dirs.saturating_add(scan.uuid_data_dirs);
residue.entries_scanned = residue.entries_scanned.saturating_add(scan.entries_scanned);
residue.diagnostic_bytes_read = residue.diagnostic_bytes_read.saturating_add(scan.diagnostic_bytes_read);
if residue.sample.is_none() {
residue.sample = scan.sample.clone();
}
if scan.diagnostic_truncated {
residue.diagnostic_truncated = true;
if residue.files == 0 {
return Ok(None);
}
record_bucket_delete_blocker(bucket, BucketDeleteBlockerKind::DiagnosticBudgetExceeded, &residue);
return Ok(Some(StorageError::BucketNotEmptyWithDetails {
bucket: bucket.to_string(),
details: residue.describe(),
}));
}
}
if residue.has_residue_without_xlmeta() {
record_bucket_delete_blocker(bucket, BucketDeleteBlockerKind::OrphanDirectory, &residue);
return Ok(Some(StorageError::BucketNotEmptyWithDetails {
bucket: bucket.to_string(),
details: residue.describe(),
}));
}
Ok(None)
}
impl ECStore {
fn bucket_sets(&self) -> impl Iterator<Item = (usize, usize, Arc<crate::set_disk::SetDisks>)> + '_ {
self.pools.iter().flat_map(|pool| {
pool.disk_set
.iter()
.map(|set| (set.pool_index, set.set_index, Arc::clone(set)))
})
}
pub async fn get_bucket_metadata(&self, bucket: &str) -> Result<Arc<BucketMetadata>> {
let sys = metadata_sys::require_bucket_metadata_sys_in(&self.ctx)?;
sys.read().await.get(bucket).await
}
pub async fn get_bucket_policy(&self, bucket: &str) -> Result<(BucketPolicy, OffsetDateTime)> {
let sys = metadata_sys::require_bucket_metadata_sys_in(&self.ctx)?;
sys.read().await.get_bucket_policy(bucket).await
}
pub async fn get_bucket_policy_raw(&self, bucket: &str) -> Result<(String, OffsetDateTime)> {
let sys = metadata_sys::require_bucket_metadata_sys_in(&self.ctx)?;
sys.read().await.get_bucket_policy_raw(bucket).await
}
pub async fn restricts_public_bucket_access(&self, bucket: &str) -> Result<bool> {
let sys = metadata_sys::require_bucket_metadata_sys_in(&self.ctx)?;
let (config, _) = sys.read().await.get_public_access_block_config(bucket).await?;
Ok(config.restrict_public_buckets.unwrap_or(false))
}
pub async fn update_bucket_metadata_config(&self, bucket: &str, config_file: &str, data: Vec<u8>) -> Result<OffsetDateTime> {
metadata_sys::update_in(&self.ctx, bucket, config_file, data).await
}
pub async fn bucket_incarnation_id(&self, bucket: &str) -> Result<Uuid> {
metadata_sys::get_cached_bucket_incarnation_id_in(&self.ctx, bucket).await
}
pub async fn bucket_incarnation_id_from_disk(&self, bucket: &str) -> Result<Uuid> {
metadata_sys::get_bucket_incarnation_id_in(&self.ctx, bucket).await
}
pub(crate) async fn acquire_bucket_lifecycle_read_lock(&self, bucket: &str) -> Result<rustfs_lock::NamespaceLockGuard> {
let lock = self.new_ns_lock(bucket, BUCKET_LIFECYCLE_LOCK_OBJECT).await?;
lock.get_read_lock(get_lock_acquire_timeout()).await.map_err(|err| match err {
rustfs_lock::error::LockError::QuorumNotReached { required, achieved } => {
StorageError::NamespaceLockQuorumUnavailable {
mode: "bucket_lifecycle_read",
bucket: bucket.to_string(),
object: BUCKET_LIFECYCLE_LOCK_OBJECT.to_string(),
required,
achieved,
}
}
other => StorageError::Lock(other),
})
}
pub async fn acquire_bucket_incarnation_fence(
&self,
bucket: &str,
expected: uuid::Uuid,
) -> Result<super::BucketIncarnationFenceGuard> {
let inner = self.acquire_bucket_lifecycle_read_lock(bucket).await?;
let pieces = super::bucket_fence::FencePieces {
registry: self.bucket_fence_registry.clone(),
inner,
};
let registration = pieces.enter(bucket);
let current = match registration.memoized {
Some(current) => current,
None => match metadata_sys::get_bucket_incarnation_id_in(&self.ctx, bucket).await {
Ok(current) => {
if !pieces.lock_lost() {
pieces.memoize(bucket, current);
}
current
}
Err(err) => {
pieces.abandon(bucket, registration.token);
return Err(err);
}
},
};
if current != expected {
pieces.abandon(bucket, registration.token);
return Err(StorageError::BucketNotFound(bucket.to_string()));
}
Ok(pieces.into_guard(bucket, registration.token))
}
pub async fn lock_bucket_for_recursive_delete(&self, bucket: &str) -> Result<rustfs_lock::NamespaceLockGuard> {
if self.ctx.lock_manager().is_disabled() {
return Err(StorageError::InvalidArgument(
bucket.to_owned(),
String::new(),
"Recursive deletion requires namespace locking".to_owned(),
));
}
self.acquire_bucket_lifecycle_write_lock(bucket).await
}
pub(crate) async fn acquire_bucket_lifecycle_write_lock(&self, bucket: &str) -> Result<rustfs_lock::NamespaceLockGuard> {
let lock = self.new_ns_lock(bucket, BUCKET_LIFECYCLE_LOCK_OBJECT).await?;
lock.get_write_lock(get_lock_acquire_timeout())
.await
.map_err(|err| match err {
rustfs_lock::error::LockError::QuorumNotReached { required, achieved } => {
StorageError::NamespaceLockQuorumUnavailable {
mode: "bucket_lifecycle_write",
bucket: bucket.to_string(),
object: BUCKET_LIFECYCLE_LOCK_OBJECT.to_string(),
required,
achieved,
}
}
other => StorageError::Lock(other),
})
}
async fn mark_bucket_deleted(&self, bucket: &str) -> Result<()> {
let marker_volume = bucket_deleted_marker_volume(bucket);
self.make_bucket_on_sets(
marker_volume.as_str(),
&MakeBucketOptions {
force_create: true,
..Default::default()
},
)
.await
.map_err(|err| to_object_err(err, vec![bucket]))?;
Ok(())
}
async fn make_bucket_on_sets(&self, bucket: &str, opts: &MakeBucketOptions) -> Result<()> {
let results = futures::future::join_all(
self.bucket_sets()
.map(|(_, _, set)| async move { set.make_bucket(bucket, opts).await }),
)
.await;
if results.is_empty() {
return Err(StorageError::ErasureWriteQuorum);
}
let mut bucket_exists_error = None;
let mut first_hard_error = None;
for result in results {
let Err(err) = result else {
continue;
};
if matches!(err, StorageError::VolumeExists) || is_err_bucket_exists(&err) {
if bucket_exists_error.is_none() {
bucket_exists_error = Some(err);
}
} else if first_hard_error.is_none() {
first_hard_error = Some(err);
}
}
match first_hard_error.or(bucket_exists_error) {
Some(err) => Err(err),
None => Ok(()),
}
}
async fn delete_bucket_on_sets(&self, bucket: &str, opts: &DeleteBucketOptions) -> Result<()> {
let results = futures::future::join_all(
self.bucket_sets()
.map(|(_, _, set)| async move { set.delete_bucket(bucket, opts).await }),
)
.await;
if results.is_empty() {
return Err(StorageError::ErasureWriteQuorum);
}
let mut deleted = false;
let mut first_error = None;
for result in results {
match result {
Ok(()) => deleted = true,
Err(err) if is_err_strict_volume_not_found(&err) => {}
Err(err) if first_error.is_none() => first_error = Some(err),
Err(_) => {}
}
}
if let Some(delete_error) = first_error {
if !opts.no_recreate {
let rollback_opts = MakeBucketOptions {
force_create: true,
no_lock: true,
..Default::default()
};
if let Err(rollback_error) = self.make_bucket_on_sets(bucket, &rollback_opts).await {
warn!(
event = EVENT_BUCKET_DELETE_ROLLBACK_FAILED,
component = "ecstore",
subsystem = "bucket",
bucket,
deletion_error = ?delete_error,
rollback_error = ?rollback_error,
"Bucket deletion rollback could not restore every bucket volume"
);
}
}
return Err(delete_error);
}
if deleted { Ok(()) } else { Err(StorageError::VolumeNotFound) }
}
async fn cleanup_deleted_bucket_metadata(
&self,
bucket: &str,
include_deleted_marker: bool,
guard: Option<&rustfs_lock::NamespaceLockGuard>,
) -> Result<()> {
for prefix in bucket_delete_metadata_cleanup_prefixes(bucket) {
await_bucket_namespace_operation(
guard,
bucket,
"deleted bucket metadata cleanup",
self.delete_all(RUSTFS_META_BUCKET, prefix.as_str()),
)
.await?;
}
if include_deleted_marker {
let marker_prefix = bucket_deleted_marker_prefix(bucket);
await_bucket_namespace_operation(
guard,
bucket,
"deleted bucket marker cleanup",
self.delete_all(RUSTFS_META_BUCKET, marker_prefix.as_str()),
)
.await?;
}
await_bucket_namespace_operation(
guard,
bucket,
"deleted bucket metadata cache cleanup",
metadata_sys::remove_bucket_metadata_in(&self.ctx, bucket),
)
.await?;
runtime_sources::delete_bucket_monitor_entry(bucket);
Ok(())
}
async fn cleanup_bucket_usage(&self, bucket: &str, guard: Option<&rustfs_lock::NamespaceLockGuard>) -> Result<()> {
run_bucket_usage_cleanup(guard, bucket, async {
crate::data_usage::prepare_bucket_usage_for_namespace_change(bucket, guard).await?;
crate::data_usage::remove_bucket_usage_from_backend_with_guard_fenced(self, bucket, guard).await
})
.await
}
async fn cleanup_bucket_usage_best_effort(&self, bucket: &str, guard: Option<&rustfs_lock::NamespaceLockGuard>) {
if let Err(err) = self.cleanup_bucket_usage(bucket, guard).await {
warn!(
bucket = %bucket,
error = ?err,
"bucket data usage cleanup deferred to scanner reconciliation"
);
}
}
async fn rollback_failed_bucket_creation(
&self,
bucket: &str,
lifecycle_guard: Option<&rustfs_lock::NamespaceLockGuard>,
namespace_guard: Option<&rustfs_lock::NamespaceLockGuard>,
) {
let rollback_opts = DeleteBucketOptions {
no_lock: true,
no_recreate: true,
..Default::default()
};
if let Err(err) =
await_bucket_lifecycle_operation(lifecycle_guard, namespace_guard, bucket, "failed bucket creation rollback", async {
self.delete_bucket_on_sets(bucket, &rollback_opts)
.await
.map_err(|rollback_err| to_object_err(rollback_err, vec![bucket]))
})
.await
{
warn!(
bucket = %bucket,
error = ?err,
"failed bucket creation rollback did not remove every physical bucket volume"
);
return;
}
if let Err(err) = self.cleanup_deleted_bucket_metadata(bucket, false, namespace_guard).await {
warn!(
bucket = %bucket,
error = ?err,
"failed bucket creation rollback left internal bucket metadata"
);
}
}
#[instrument(skip(self))]
pub(super) async fn handle_make_bucket(&self, bucket: &str, opts: &MakeBucketOptions) -> Result<()> {
if !is_meta_bucketname(bucket)
&& let Err(err) = check_valid_bucket_name_strict(bucket)
{
return Err(StorageError::BucketNameInvalid(err.to_string()));
}
let bucket_lifecycle_guard = if !opts.no_lock {
Some(self.acquire_bucket_lifecycle_write_lock(bucket).await?)
} else {
None
};
let metadata_transaction_guard = if !opts.no_lock && !is_meta_bucketname(bucket) {
Some(metadata_sys::acquire_bucket_metadata_transaction_lock_in(&self.ctx, bucket).await?)
} else {
None
};
let ns_guard = if !opts.no_lock {
let ns_lock = self.new_ns_lock(bucket, bucket).await?;
Some(
await_bucket_namespace_operation(
bucket_lifecycle_guard.as_ref(),
bucket,
"bucket namespace lock acquisition",
async {
ns_lock.get_write_lock(get_lock_acquire_timeout()).await.map_err(|e| match e {
rustfs_lock::error::LockError::QuorumNotReached { required, achieved } => {
StorageError::NamespaceLockQuorumUnavailable {
mode: "write",
bucket: bucket.to_string(),
object: bucket.to_string(),
required,
achieved,
}
}
other => StorageError::Lock(other),
})
},
)
.await?,
)
} else {
None
};
let existing_bucket_info = match self.get_bucket_info_from_sets(bucket, &BucketOptions::default()).await {
Ok(info) => Some(info),
Err(err) => {
if is_err_bucket_not_found(&err) {
None
} else {
return Err(to_object_err(err, vec![bucket]));
}
}
};
let confirmed_missing = existing_bucket_info.is_none();
let existing_metadata = if opts.force_create && !confirmed_missing && !is_meta_bucketname(bucket) {
let (mut metadata, persisted) = metadata_sys::get_config_from_disk_with_presence_in(&self.ctx, bucket).await?;
if !persisted {
metadata = BucketMetadata::new(bucket);
metadata.created = existing_bucket_info
.as_ref()
.and_then(|info| info.created)
.unwrap_or(OffsetDateTime::UNIX_EPOCH);
let store = metadata_sys::object_store_in(&self.ctx).await?;
if let Some(stored) = crate::bucket::metadata::load_bucket_incarnation(store.clone(), bucket).await?
&& !crate::bucket::retirement::is_retired(store, bucket, stored).await?
{
metadata.bucket_incarnation_id = stored;
}
} else if !metadata.bucket_incarnation_sidecar && !metadata.bucket_incarnation_id.is_nil() {
return Err(Error::other(format!(
"bucket incarnation sidecar is missing for new-format metadata: {bucket}"
)));
} else if !metadata.bucket_incarnation_sidecar {
metadata.bucket_incarnation_id = Uuid::new_v4();
}
Some(metadata)
} else {
None
};
let mut meta = existing_metadata.unwrap_or_else(|| {
if confirmed_missing && !is_meta_bucketname(bucket) {
BucketMetadata::new_with_default_durability(bucket)
} else {
BucketMetadata::new(bucket)
}
});
let existing_incarnation_is_authoritative = meta.bucket_incarnation_sidecar;
if confirmed_missing || is_meta_bucketname(bucket) {
meta.set_created(opts.created_at);
if opts.versioning_enabled {
meta.versioning_config_xml =
crate::bucket::utils::serialize::<VersioningConfiguration>(&ENABLED_VERSIONING_CONFIG)?;
}
}
let lock_newly_enabled = opts.lock_enabled && !meta.lock_enabled;
if opts.lock_enabled {
meta.lock_enabled = true;
meta.object_lock_config_xml = crate::bucket::utils::serialize(&*crate::store::ENABLED_OBJECT_LOCK_CONFIG)?;
meta.versioning_config_xml = crate::bucket::utils::serialize::<VersioningConfiguration>(&ENABLED_VERSIONING_CONFIG)?;
}
let metadata_persisted_before_physical = confirmed_missing && !is_meta_bucketname(bucket) && opts.lock_enabled;
if metadata_persisted_before_physical {
metadata_sys::set_new_bucket_metadata_intent_in(&self.ctx, meta.clone()).await?;
if bucket_lifecycle_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
|| metadata_transaction_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
|| ns_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
{
return Err(Error::other(format!("bucket metadata creation intent lock was lost: {bucket}")));
}
}
if confirmed_missing && !is_meta_bucketname(bucket) {
crate::store::list_objects::observe_scanner_namespace_mutations(bucket, 1);
self.cleanup_bucket_usage(bucket, ns_guard.as_ref()).await?;
}
if let Err(err) = await_bucket_lifecycle_operation(
bucket_lifecycle_guard.as_ref(),
ns_guard.as_ref(),
bucket,
"physical bucket creation",
await_bucket_namespace_operation(
metadata_transaction_guard.as_ref(),
bucket,
"bucket creation metadata transaction",
async {
self.make_bucket_on_sets(bucket, opts)
.await
.map_err(|err| to_object_err(err, vec![bucket]))
},
),
)
.await
{
if is_err_bucket_exists(&err)
&& let Err(heal_err) = self
.handle_heal_bucket(
bucket,
&HealOpts {
recreate: true,
..Default::default()
},
)
.await
{
warn!("best-effort bucket heal after BucketExists failed: {heal_err}");
}
if confirmed_missing && !is_err_bucket_exists(&err) && ns_guard.as_ref().is_none_or(|guard| !guard.is_lock_lost()) {
error!("make bucket failed: {err}");
self.rollback_failed_bucket_creation(bucket, bucket_lifecycle_guard.as_ref(), ns_guard.as_ref())
.await;
}
return Err(err);
};
let metadata_result = async {
if metadata_persisted_before_physical {
return metadata_sys::commit_bucket_metadata_in(&self.ctx, meta).await;
}
if is_meta_bucketname(bucket) {
metadata_sys::set_bucket_metadata_in(&self.ctx, meta).await
} else if existing_incarnation_is_authoritative && !lock_newly_enabled {
metadata_sys::commit_bucket_metadata_in(&self.ctx, meta).await
} else {
metadata_sys::set_new_bucket_metadata_in(&self.ctx, meta).await
}
}
.await;
let metadata_lock_lost = bucket_lifecycle_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
|| metadata_transaction_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
|| ns_guard.as_ref().is_some_and(|guard| guard.is_lock_lost());
if let Err(err) = metadata_result {
if !metadata_lock_lost {
self.rollback_failed_bucket_creation(bucket, bucket_lifecycle_guard.as_ref(), ns_guard.as_ref())
.await;
}
return Err(err);
}
if metadata_lock_lost {
return Err(Error::other(format!("bucket metadata initialization lock was lost: {bucket}")));
}
if confirmed_missing && !is_meta_bucketname(bucket) {
crate::store::list_objects::observe_scanner_namespace_mutations(bucket, 1);
}
Ok(())
}
pub(crate) async fn bucket_exists_for_heal(&self, bucket: &str) -> Result<bool> {
let results = futures::future::join_all(
self.bucket_sets()
.map(|(_, _, set)| async move { set.get_bucket_info(bucket, &BucketOptions::default()).await }),
)
.await;
let mut first_error = None;
for result in results {
match result {
Ok(_) => return Ok(true),
Err(err) if is_err_strict_volume_not_found(&err) => {}
Err(err) if first_error.is_none() => first_error = Some(err),
Err(_) => {}
}
}
match first_error {
Some(err) => Err(err),
None => Ok(false),
}
}
#[instrument(skip(self))]
pub(crate) async fn get_bucket_info_from_sets(&self, bucket: &str, opts: &BucketOptions) -> Result<BucketInfo> {
self.get_bucket_info_from_sets_with_quorum(bucket, opts, BucketInfoQuorum::Write)
.await
}
pub(crate) async fn get_bucket_info_from_sets_at_read_quorum(
&self,
bucket: &str,
opts: &BucketOptions,
) -> Result<BucketInfo> {
self.get_bucket_info_from_sets_with_quorum(bucket, opts, BucketInfoQuorum::Read)
.await
}
async fn get_bucket_info_from_sets_with_quorum(
&self,
bucket: &str,
opts: &BucketOptions,
quorum: BucketInfoQuorum,
) -> Result<BucketInfo> {
let mut scoped_results = futures::future::join_all(self.bucket_sets().map(|(pool_index, set_index, set)| async move {
let result = match quorum {
BucketInfoQuorum::Read => set.stat_bucket_with_quorum(bucket, quorum).await,
BucketInfoQuorum::Write => set.get_bucket_info(bucket, opts).await,
};
(pool_index, set_index, result)
}))
.await;
scoped_results.sort_unstable_by_key(|(pool_index, set_index, _)| (*pool_index, *set_index));
let mut first_info = None;
let mut first_error = None;
for (_, _, result) in scoped_results {
match result {
Ok(info) if first_info.is_none() => first_info = Some(info),
Ok(_) => {}
Err(err) if is_err_strict_volume_not_found(&err) => {}
Err(err) if first_error.is_none() => first_error = Some(err),
Err(_) => {}
}
}
if let Some(err) = first_error {
return Err(err);
}
first_info.ok_or(Error::VolumeNotFound)
}
#[instrument(skip(self))]
pub(super) async fn handle_get_bucket_info(&self, bucket: &str, opts: &BucketOptions) -> Result<BucketInfo> {
let mut info = match self.get_bucket_info_from_sets(bucket, opts).await {
Ok(info) => info,
Err(Error::ErasureWriteQuorum) => return self.get_bucket_info_at_read_quorum(bucket, opts).await,
Err(err) => return Err(err),
};
if let Ok(sys) = metadata_sys::get_in(&self.ctx, bucket).await {
if should_override_created_from_metadata(sys.created) {
info.created = Some(sys.created);
}
info.versioning = sys.versioning();
info.object_locking = sys.object_locking();
}
Ok(info)
}
async fn get_bucket_info_at_read_quorum(&self, bucket: &str, opts: &BucketOptions) -> Result<BucketInfo> {
let guard = self.acquire_bucket_lifecycle_read_lock(bucket).await?;
await_bucket_namespace_operation(Some(&guard), bucket, "bucket read quorum validation", async {
let mut info = self
.get_bucket_info_from_sets_with_quorum(bucket, opts, BucketInfoQuorum::Read)
.await?;
let (metadata, persisted) = metadata_sys::get_config_from_disk_with_presence_in(&self.ctx, bucket).await?;
if !persisted {
return Err(Error::ErasureReadQuorum);
}
if metadata.lock_enabled && !metadata.bucket_creation_committed {
return Err(Error::ErasureWriteQuorum);
}
if metadata.name != bucket {
return Err(Error::FileCorrupt);
}
if should_override_created_from_metadata(metadata.created) {
info.created = Some(metadata.created);
}
info.versioning = metadata.versioning();
info.object_locking = metadata.object_locking();
Ok(info)
})
.await
}
#[instrument(skip(self))]
pub(super) async fn handle_list_bucket(&self, opts: &BucketOptions) -> Result<Vec<BucketInfo>> {
Ok(self.list_bucket_from_sets(opts, usize::MAX).await?.buckets)
}
pub async fn list_bucket_for_scanner(&self, opts: &BucketOptions) -> Result<crate::cluster::rpc::ScannerBucketListing> {
self.list_bucket_from_sets(opts, SCANNER_BUCKET_LIST_SET_CONCURRENCY).await
}
async fn list_bucket_from_sets(
&self,
opts: &BucketOptions,
max_concurrency: usize,
) -> Result<crate::cluster::rpc::ScannerBucketListing> {
let set_count = self.pools.iter().map(|pool| pool.disk_set.len()).sum();
let concurrency = bucket_list_set_concurrency(set_count, max_concurrency);
let deleted = opts.deleted;
let cached = opts.cached;
let no_metadata = opts.no_metadata;
let mut set_listings = stream::iter(self.bucket_sets().map(move |(pool_index, set_index, set)| {
let opts = BucketOptions {
deleted,
cached,
no_metadata,
};
async move {
set.list_bucket_for_scanner(&opts)
.await
.map(|(buckets, complete)| (pool_index, set_index, buckets, complete))
}
}))
.buffer_unordered(concurrency);
let mut topology_complete = set_count != 0;
let mut bucket_map = BTreeMap::<String, BucketInfo>::new();
let mut scoped_buckets = Vec::with_capacity(set_count);
while let Some(set_listing) = set_listings.next().await {
let (pool_index, set_index, buckets, set_complete) = set_listing?;
topology_complete &= set_complete;
for bucket in &buckets {
bucket_map.entry(bucket.name.clone()).or_insert_with(|| bucket.clone());
}
scoped_buckets.push(crate::cluster::rpc::ScannerSetBucketListing {
pool_index,
set_index,
buckets,
});
}
scoped_buckets.sort_unstable_by_key(|scope| (scope.pool_index, scope.set_index));
let mut listing = crate::cluster::rpc::ScannerBucketListing {
buckets: bucket_map.into_values().collect(),
set_buckets: scoped_buckets,
topology_complete,
};
if !opts.no_metadata {
for bucket in &mut listing.buckets {
if let Ok(created) = metadata_sys::created_at_in(&self.ctx, &bucket.name).await
&& should_override_created_from_metadata(created)
{
bucket.created = Some(created);
}
}
let created_by_bucket = listing
.buckets
.iter()
.map(|bucket| (bucket.name.as_str(), bucket.created))
.collect::<BTreeMap<_, _>>();
for scope in &mut listing.set_buckets {
for bucket in &mut scope.buckets {
if let Some(created) = created_by_bucket.get(bucket.name.as_str()) {
bucket.created = *created;
}
}
}
}
Ok(listing)
}
#[instrument(skip(self))]
pub(super) async fn handle_delete_bucket(&self, bucket: &str, opts: &DeleteBucketOptions) -> Result<()> {
self.handle_delete_bucket_with_diagnostic_budget(bucket, opts, BucketDeleteDiagnosticBudget::new())
.await
}
async fn handle_delete_bucket_with_diagnostic_budget(
&self,
bucket: &str,
opts: &DeleteBucketOptions,
mut diagnostic_budget: BucketDeleteDiagnosticBudget,
) -> Result<()> {
if is_meta_bucketname(bucket) {
return Err(StorageError::BucketNameInvalid(bucket.to_string()));
}
if let Err(err) = check_valid_bucket_name(bucket) {
return Err(StorageError::BucketNameInvalid(err.to_string()));
}
let bucket_lifecycle_guard = if !opts.no_lock {
Some(self.acquire_bucket_lifecycle_write_lock(bucket).await?)
} else {
None
};
let ns_guard = if !opts.no_lock {
let ns_lock = self.new_ns_lock(bucket, bucket).await?;
Some(
await_bucket_namespace_operation(
bucket_lifecycle_guard.as_ref(),
bucket,
"bucket namespace lock acquisition",
async {
ns_lock.get_write_lock(get_lock_acquire_timeout()).await.map_err(|e| match e {
rustfs_lock::error::LockError::QuorumNotReached { required, achieved } => {
StorageError::NamespaceLockQuorumUnavailable {
mode: "write",
bucket: bucket.to_string(),
object: bucket.to_string(),
required,
achieved,
}
}
other => StorageError::Lock(other),
})
},
)
.await?,
)
} else {
None
};
let sr_mark_delete = opts.srdelete_op == SRBucketDeleteOp::MarkDelete;
let sr_purge = opts.srdelete_op == SRBucketDeleteOp::Purge;
let sr_delete = sr_mark_delete || sr_purge;
let mut delete_opts = opts.clone();
let bucket_exists = match self.get_bucket_info_from_sets(bucket, &BucketOptions::default()).await {
Ok(_) => true,
Err(err) => {
if is_err_strict_volume_not_found(&err) && sr_delete {
false
} else if is_err_strict_volume_not_found(&err) {
return Err(StorageError::BucketNotFound(bucket.to_string()));
} else {
return Err(to_object_err(err, vec![bucket]));
}
}
};
if bucket_exists {
validate_table_bucket_delete_guard(&self.ctx, bucket).await?;
if !opts.force {
if let Some(blocker) = bucket_delete_local_blocker(&self.ctx, bucket, &mut diagnostic_budget).await? {
return Err(blocker);
}
delete_opts.force_if_empty = true;
}
}
if sr_delete && !bucket_exists {
delete_opts.force_if_empty = true;
}
#[cfg(test)]
crate::cluster::rpc::peer_s3_client::pause_after_delete_bucket_empty_scan().await;
if sr_mark_delete {
await_bucket_lifecycle_operation(
bucket_lifecycle_guard.as_ref(),
ns_guard.as_ref(),
bucket,
"bucket delete marker creation",
self.mark_bucket_deleted(bucket),
)
.await?;
}
let retirement = if bucket_exists && bucket_lifecycle_guard.is_some() && !is_meta_bucketname(bucket) {
if let Some(store) = metadata_sys::object_store_if_initialized_in(&self.ctx).await {
crate::bucket::metadata::load_bucket_incarnation(store.clone(), bucket)
.await?
.map(|incarnation| (store, incarnation))
} else {
None
}
} else {
None
};
let delete_result = await_bucket_namespace_operation(
bucket_lifecycle_guard.as_ref(),
bucket,
"physical bucket deletion",
run_physical_bucket_deletion(ns_guard.as_ref(), bucket, async {
self.delete_bucket_on_sets(bucket, &delete_opts)
.await
.map_err(|err| to_object_err(err, vec![bucket]))
}),
)
.await;
if let Err(err) = delete_result
&& (!sr_delete || !is_err_strict_volume_not_found(&err))
{
if delete_opts.force_if_empty && matches!(&err, StorageError::BucketNotEmpty(_)) {
let mut diagnostic_budget = BucketDeleteDiagnosticBudget::new();
if let Some(blocker) = bucket_delete_local_blocker(&self.ctx, bucket, &mut diagnostic_budget).await? {
return Err(blocker);
}
}
return Err(err);
}
if let Some((store, incarnation)) = retirement {
let mut record_opts = ObjectOptions {
max_parity: true,
..Default::default()
};
if let Some(guard) = bucket_lifecycle_guard.as_ref() {
record_opts.add_bucket_lifecycle_lock_guard(guard);
}
if let Some(guard) = ns_guard.as_ref() {
record_opts.add_namespace_lock_guard(guard);
}
await_bucket_lifecycle_operation(
bucket_lifecycle_guard.as_ref(),
ns_guard.as_ref(),
bucket,
"bucket retirement publication",
crate::bucket::retirement::commit_retirement(store, bucket, incarnation, &record_opts),
)
.await?;
}
self.cleanup_bucket_usage_best_effort(bucket, ns_guard.as_ref()).await;
if let Err(err) = self
.cleanup_deleted_bucket_metadata(bucket, sr_purge, ns_guard.as_ref())
.await
{
warn!(
bucket = %bucket,
error = ?err,
"physical bucket deletion succeeded but metadata cleanup remains pending"
);
}
crate::store::list_objects::observe_scanner_namespace_mutations(bucket, 1);
Ok(())
}
}
#[cfg(test)]
mod tests {
use super::{
BUCKET_DELETE_DIAGNOSTIC_MAX_ELAPSED, BUCKET_DELETE_DIAGNOSTIC_MAX_ENTRIES, BUCKET_DELETE_XLMETA_DIAGNOSTIC_MAX_BYTES,
BucketDeleteBlockerKind, BucketDeleteDiagnosticBudget, BucketMetadataLessResidue, SCANNER_BUCKET_LIST_SET_CONCURRENCY,
await_bucket_namespace_operation, bucket_delete_metadata_cleanup_prefixes, bucket_deleted_marker_prefix,
bucket_deleted_marker_volume, bucket_list_set_concurrency, record_bucket_delete_blocker, run_bucket_usage_cleanup,
run_physical_bucket_deletion, scan_metadata_less_residue, scan_metadata_less_residue_with_budget,
should_override_created_from_metadata, validate_table_bucket_delete_allowed,
};
use crate::bucket::metadata::{BucketMetadata, table_bucket_catalog_metadata_prefix};
use crate::bucket::metadata_sys;
use crate::cluster::rpc::peer_s3_client::install_delete_bucket_empty_scan_barrier;
use crate::disk::{BUCKET_META_PREFIX, DiskAPI, RUSTFS_META_BUCKET, STORAGE_FORMAT_FILE};
use crate::error::StorageError;
use crate::object_api::{ObjectOptions, PutObjReader};
use crate::runtime::instance::InstanceContext;
use crate::storage_api_contracts::{
bucket::{BucketOperations as _, BucketOptions, DeleteBucketOptions, MakeBucketOptions, SRBucketDeleteOp},
list::ListOperations as _,
namespace::NamespaceLocking as _,
object::{ObjectIO as _, ObjectOperations as _},
};
use crate::store::{ECStore, init_local_disks_with_instance_ctx};
use crate::{
disk::endpoint::Endpoint,
layout::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints},
};
use rustfs_data_usage::{BucketUsageInfo, DATA_USAGE_OBJECT_NAME, DataUsageInfo};
use rustfs_filemeta::{FileInfo, FileMeta, TRANSITION_COMPLETE};
use rustfs_lock::{LocalClient, LockRequest, LockType, NamespaceLock, ObjectKey};
use serial_test::serial;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, Ordering};
use std::time::{Duration, SystemTime};
use time::OffsetDateTime;
use tokio::io::AsyncReadExt;
use tokio::sync::{Notify, OnceCell};
use tokio_util::sync::CancellationToken;
use uuid::Uuid;
static BUCKET_DELETE_TEST_ENV: OnceCell<(Vec<PathBuf>, Arc<ECStore>)> = OnceCell::const_new();
#[tokio::test(start_paused = true)]
async fn bucket_delete_diagnostic_budget_starts_with_first_scan_io_and_latches_once() {
let mut budget = BucketDeleteDiagnosticBudget::with_limits(8, Duration::from_millis(100));
tokio::time::advance(Duration::from_secs(10)).await;
let first_polled = Arc::new(AtomicBool::new(false));
let first_polled_for_io = first_polled.clone();
let first = budget
.run_io(async move {
first_polled_for_io.store(true, Ordering::SeqCst);
Ok::<_, std::io::Error>(7_u8)
})
.await
.expect("the first diagnostic IO should succeed");
assert_eq!(first, Some(7));
assert!(first_polled.load(Ordering::SeqCst));
tokio::time::advance(Duration::from_millis(101)).await;
let expired_polled = Arc::new(AtomicBool::new(false));
let expired_polled_for_io = expired_polled.clone();
let expired = budget
.run_io(async move {
expired_polled_for_io.store(true, Ordering::SeqCst);
Ok::<_, std::io::Error>(9_u8)
})
.await
.expect("an expired diagnostic budget should not become an IO error");
assert_eq!(expired, None);
assert!(
!expired_polled.load(Ordering::SeqCst),
"the deadline must remain latched after the first scan IO"
);
}
#[tokio::test(start_paused = true)]
async fn bucket_delete_diagnostic_budget_times_out_its_first_pending_io() {
let mut budget = BucketDeleteDiagnosticBudget::with_limits(8, Duration::from_millis(100));
let io_polled = Arc::new(AtomicBool::new(false));
let io_polled_for_future = io_polled.clone();
let result = budget
.run_io(std::future::poll_fn(move |_cx| {
io_polled_for_future.store(true, Ordering::SeqCst);
std::task::Poll::<std::io::Result<()>>::Pending
}))
.await
.expect("a diagnostic timeout should fail closed without an IO error");
assert_eq!(result, None);
assert!(
io_polled.load(Ordering::SeqCst),
"the first diagnostic IO must be polled before its timeout"
);
}
#[tokio::test(start_paused = true)]
async fn delayed_metadata_less_scans_still_detect_orphans_and_xlmeta() {
let root = tempfile::tempdir().expect("temporary delayed-scan roots should be created");
let orphan_root = root.path().join("orphan-root");
let xlmeta_root = root.path().join("xlmeta-root");
std::fs::create_dir_all(&orphan_root).expect("orphan root should be created");
std::fs::create_dir_all(&xlmeta_root).expect("xlmeta root should be created");
std::fs::write(orphan_root.join("orphan-part"), b"orphan").expect("orphan fixture should be written");
std::fs::write(xlmeta_root.join(STORAGE_FORMAT_FILE), b"invalid-xlmeta").expect("xl.meta fixture should be written");
let mut orphan_budget = BucketDeleteDiagnosticBudget::with_limits(16, Duration::from_secs(5));
let mut xlmeta_budget = BucketDeleteDiagnosticBudget::with_limits(16, Duration::from_secs(5));
tokio::time::advance(Duration::from_secs(60)).await;
let orphan = scan_metadata_less_residue_with_budget(&orphan_root, &mut orphan_budget)
.await
.expect("delayed orphan scan should complete");
assert!(orphan.has_residue_without_xlmeta());
assert!(!orphan.diagnostic_truncated);
let xlmeta = scan_metadata_less_residue_with_budget(&xlmeta_root, &mut xlmeta_budget)
.await
.expect("delayed xl.meta scan should complete");
assert!(xlmeta.xlmeta_found);
assert!(!xlmeta.diagnostic_truncated);
}
#[tokio::test(start_paused = true)]
async fn bucket_namespace_operation_fails_closed_after_lease_expiry() {
let ttl = Duration::from_millis(20);
let lock = NamespaceLock::new("bucket-operation-test".to_string(), Arc::new(LocalClient::new()));
let request = LockRequest::new(ObjectKey::new("bucket", ""), LockType::Exclusive, "test-owner")
.with_acquire_timeout(Duration::from_secs(1))
.with_ttl(ttl)
.with_refresh_interval(ttl);
let guard = lock
.acquire_guard(&request)
.await
.expect("namespace lock acquisition should not fail")
.expect("namespace lock should be acquired");
tokio::time::advance(ttl + Duration::from_millis(1)).await;
let operation_ran = Arc::new(AtomicBool::new(false));
let operation_ran_for_future = operation_ran.clone();
let result = await_bucket_namespace_operation(Some(&guard), "bucket", "test operation", async move {
operation_ran_for_future.store(true, Ordering::SeqCst);
Ok(())
})
.await;
assert!(result.is_err(), "an expired namespace lease must fence the operation");
assert!(
!operation_ran.load(Ordering::SeqCst),
"a fenced namespace operation must not poll its mutation future"
);
}
#[tokio::test]
#[serial]
async fn physical_bucket_delete_fences_scanner_before_polling_storage() {
let generation_before = crate::store::list_objects::scanner_namespace_mutation_generation();
let storage_polled = Arc::new(AtomicBool::new(false));
let storage_polled_for_future = storage_polled.clone();
run_physical_bucket_deletion(None, "generation-order-bucket", async move {
assert!(
crate::store::list_objects::scanner_namespace_mutation_generation() > generation_before,
"scanner generation must advance before physical deletion is polled"
);
storage_polled_for_future.store(true, Ordering::SeqCst);
Ok(())
})
.await
.expect("synthetic physical deletion should succeed");
assert!(storage_polled.load(Ordering::SeqCst));
}
#[tokio::test]
async fn bucket_usage_cleanup_stops_after_parent_cancellation() {
let started = Arc::new(Notify::new());
let started_wait = started.notified();
let release = Arc::new(Notify::new());
let completed = Arc::new(AtomicBool::new(false));
let started_for_cleanup = started.clone();
let release_for_cleanup = release.clone();
let completed_for_cleanup = completed.clone();
let parent = tokio::spawn(run_bucket_usage_cleanup(None, "bucket", async move {
started_for_cleanup.notify_one();
release_for_cleanup.notified().await;
completed_for_cleanup.store(true, Ordering::SeqCst);
Ok(())
}));
started_wait.await;
parent.abort();
let _ = parent.await;
tokio::task::yield_now().await;
release.notify_waiters();
tokio::task::yield_now().await;
assert!(
!completed.load(Ordering::SeqCst),
"a cancelled cleanup future must not continue in a detached task"
);
}
#[tokio::test(start_paused = true)]
#[serial]
async fn bucket_namespace_operation_stops_in_flight_work_after_lock_loss() {
let ttl = Duration::from_millis(20);
let lock = NamespaceLock::new("bucket-operation-in-flight-loss-test".to_string(), Arc::new(LocalClient::new()));
let request = LockRequest::new(ObjectKey::new("bucket", ""), LockType::Exclusive, "test-owner")
.with_acquire_timeout(Duration::from_secs(1))
.with_ttl(ttl)
.with_refresh_interval(ttl);
let guard = lock
.acquire_guard(&request)
.await
.expect("namespace lock acquisition should not fail")
.expect("namespace lock should be acquired");
let operation_started = Arc::new(Notify::new());
let started_wait = operation_started.notified();
let operation_release = Arc::new(Notify::new());
let operation_completed = Arc::new(AtomicBool::new(false));
let started_for_operation = operation_started.clone();
let release_for_operation = operation_release.clone();
let completed_for_operation = operation_completed.clone();
let task = tokio::spawn(async move {
await_bucket_namespace_operation(Some(&guard), "bucket", "test operation", async move {
started_for_operation.notify_one();
release_for_operation.notified().await;
completed_for_operation.store(true, Ordering::SeqCst);
Ok(())
})
.await
});
started_wait.await;
tokio::time::advance(ttl + Duration::from_millis(1)).await;
let err = task
.await
.expect("operation task should join")
.expect_err("an operation still running after lease loss must be fenced");
assert!(err.to_string().contains("namespace lock was lost during test operation"));
operation_release.notify_waiters();
tokio::task::yield_now().await;
assert!(
!operation_completed.load(Ordering::SeqCst),
"lock loss must stop the old owner before a successor can acquire the namespace"
);
}
async fn setup_bucket_delete_test_env() -> (Vec<PathBuf>, Arc<ECStore>) {
BUCKET_DELETE_TEST_ENV
.get_or_init(|| async {
let temp_dir = std::env::temp_dir().join(format!("rustfs_bucket_delete_test_{}", Uuid::new_v4()));
tokio::fs::create_dir_all(&temp_dir)
.await
.expect("test base directory should be created");
let disk_paths = (0..4)
.map(|disk_idx| temp_dir.join(format!("disk{disk_idx}")))
.collect::<Vec<_>>();
for disk_path in &disk_paths {
tokio::fs::create_dir_all(disk_path)
.await
.expect("disk directory should be created");
}
let mut endpoints = Vec::with_capacity(disk_paths.len());
for (disk_idx, disk_path) in disk_paths.iter().enumerate() {
let mut endpoint =
Endpoint::try_from(disk_path.to_str().expect("disk path should be utf8")).expect("endpoint should parse");
endpoint.set_pool_index(0);
endpoint.set_set_index(0);
endpoint.set_disk_index(disk_idx);
endpoints.push(endpoint);
}
let endpoint_pools = EndpointServerPools(vec![PoolEndpoints {
legacy: false,
set_count: 1,
drives_per_set: 4,
endpoints: Endpoints::from(endpoints),
cmd_line: "bucket-delete-test".to_string(),
platform: format!("OS: {} | Arch: {}", std::env::consts::OS, std::env::consts::ARCH),
}]);
let instance_ctx = Arc::new(InstanceContext::new());
init_local_disks_with_instance_ctx(&instance_ctx, endpoint_pools.clone())
.await
.expect("local disks should initialize");
let ecstore = ECStore::new_with_instance_ctx(
"127.0.0.1:0".parse().expect("test address"),
endpoint_pools,
CancellationToken::new(),
instance_ctx,
)
.await
.expect("ECStore should initialize");
let storage_class = crate::config::storageclass::lookup_config_for_pools_without_env(
&rustfs_config::server_config::KVS::new(),
&[4],
)
.expect("bucket test storage class should match its four-disk pool");
for pool in &ecstore.pools {
for set in &pool.disk_set {
set.set_test_storage_class_config(storage_class.clone());
}
}
metadata_sys::init_bucket_metadata_sys(ecstore.clone(), Vec::new()).await;
(disk_paths, ecstore)
})
.await
.clone()
}
async fn setup_multi_pool_bucket_test_env() -> (tempfile::TempDir, Arc<ECStore>) {
setup_bucket_quorum_test_env(&[4, 4], None).await
}
async fn setup_bucket_quorum_test_env(
drives_per_pool: &[usize],
standard_parity: Option<usize>,
) -> (tempfile::TempDir, Arc<ECStore>) {
let temp_dir = tempfile::tempdir().expect("multi-pool bucket test directory should be created");
let mut pools = Vec::new();
for (pool_index, &drive_count) in drives_per_pool.iter().enumerate() {
let mut endpoints = Vec::new();
for disk_index in 0..drive_count {
let disk_path = temp_dir.path().join(format!("pool{pool_index}-disk{disk_index}"));
tokio::fs::create_dir_all(&disk_path)
.await
.expect("multi-pool bucket test disk should be created");
let mut endpoint =
Endpoint::try_from(disk_path.to_str().expect("disk path should be utf8")).expect("endpoint should parse");
endpoint.set_pool_index(pool_index);
endpoint.set_set_index(0);
endpoint.set_disk_index(disk_index);
endpoints.push(endpoint);
}
pools.push(PoolEndpoints {
legacy: false,
set_count: 1,
drives_per_set: drive_count,
endpoints: Endpoints::from(endpoints),
cmd_line: format!("bucket-test-pool-{pool_index}"),
platform: format!("OS: {} | Arch: {}", std::env::consts::OS, std::env::consts::ARCH),
});
}
let endpoint_pools = EndpointServerPools(pools);
let instance_ctx = Arc::new(InstanceContext::new());
instance_ctx.set_endpoints(endpoint_pools.clone());
init_local_disks_with_instance_ctx(&instance_ctx, endpoint_pools.clone())
.await
.expect("multi-pool local disks should initialize");
let ecstore = ECStore::new_with_instance_ctx(
"127.0.0.1:0".parse().expect("test address"),
endpoint_pools,
CancellationToken::new(),
instance_ctx,
)
.await
.expect("multi-pool ECStore should initialize");
let mut storage_class_kvs = rustfs_config::server_config::KVS::new();
if let Some(parity) = standard_parity {
storage_class_kvs.insert(crate::config::storageclass::CLASS_STANDARD.to_string(), format!("EC:{parity}"));
}
let storage_class = crate::config::storageclass::lookup_config_for_pools_without_env(&storage_class_kvs, drives_per_pool)
.expect("storage class should match every test erasure set");
for pool in &ecstore.pools {
for set in &pool.disk_set {
set.set_test_storage_class_config(storage_class.clone());
}
}
(temp_dir, ecstore)
}
async fn take_set_disks_offline(
ecstore: &ECStore,
set: &Arc<crate::set_disk::SetDisks>,
disk_indexes: &[usize],
) -> Vec<(usize, crate::disk::DiskStore)> {
let offline = {
let disks = set.disks.read().await;
disk_indexes
.iter()
.map(|index| (*index, disks[*index].clone().expect("fault-injection disk should start online")))
.collect::<Vec<_>>()
};
for (_, disk) in &offline {
disk.close().await.expect("fault injection should stop per-disk monitoring");
disk.force_offline_for_test();
}
set.connect_disks().await;
{
let disks = set.disks.read().await;
for (index, disk) in &offline {
let current = disks[*index].as_ref().expect("fault-injection slot should remain populated");
assert!(Arc::ptr_eq(current, disk), "reconnect must preserve the fault-injection handle");
assert_eq!(disk.runtime_state(), crate::disk::health_state::RuntimeDriveHealthState::Offline);
}
}
let local_disk_map = ecstore.ctx.local_disk_map();
let mut local_disks = local_disk_map.write().await;
for (_, disk) in &offline {
local_disks.insert(disk.endpoint().to_string(), None);
}
offline
}
async fn restore_set_disks(
ecstore: &ECStore,
set: &Arc<crate::set_disk::SetDisks>,
offline: Vec<(usize, crate::disk::DiskStore)>,
) {
{
let local_disk_map = ecstore.ctx.local_disk_map();
let mut local_disks = local_disk_map.write().await;
for (_, disk) in &offline {
local_disks.insert(disk.endpoint().to_string(), Some(Arc::clone(disk)));
}
}
let disks = set.disks.read().await;
for (index, disk) in offline {
let current = disks[index].as_ref().expect("fault-injection slot should remain populated");
assert!(Arc::ptr_eq(current, &disk), "fault-injection disk handle should remain unchanged");
assert_eq!(disk.runtime_state(), crate::disk::health_state::RuntimeDriveHealthState::Offline);
disk.reset_health_for_store_init_retry();
}
}
#[tokio::test]
async fn request_metadata_methods_fail_closed_before_instance_initialization() {
let (_temp_dir, store) = setup_multi_pool_bucket_test_env().await;
let expected = "bucket metadata sys not initialized for this instance";
let errors = [
store.get_bucket_metadata("bucket").await.unwrap_err(),
store.get_bucket_policy("bucket").await.unwrap_err(),
store.get_bucket_policy_raw("bucket").await.unwrap_err(),
store.restricts_public_bucket_access("bucket").await.unwrap_err(),
store
.update_bucket_metadata_config("bucket", crate::bucket::metadata::BUCKET_POLICY_CONFIG, Vec::new())
.await
.unwrap_err(),
];
for error in errors {
assert_eq!(error.to_string(), format!("Io error: {expected}"));
}
}
async fn create_bucket_with_object(ecstore: &Arc<ECStore>, bucket: &str, object: &str) {
let generation_before_make = ecstore.scanner_namespace_mutation_generation();
ecstore
.make_bucket(bucket, &MakeBucketOptions::default())
.await
.expect("bucket should be created");
assert_eq!(
ecstore.scanner_namespace_mutation_generation(),
generation_before_make.saturating_add(2),
"successful bucket creation should fence scanner namespace activity before and after creation"
);
let generation_before_put = ecstore.scanner_namespace_mutation_generation();
let mut reader = PutObjReader::from_vec(b"delete bucket semantics".to_vec());
ecstore
.put_object(bucket, object, &mut reader, &ObjectOptions::default())
.await
.expect("object should be written");
let lock = ecstore.pools[0].disk_set[0]
.new_ns_lock(bucket, object)
.await
.expect("fixture namespace lock should be created");
drop(
lock.get_write_lock(Duration::from_secs(30))
.await
.expect("fixture rename tail should finish before checking its generation"),
);
assert_eq!(
ecstore.scanner_namespace_mutation_generation(),
generation_before_put.saturating_add(3),
"successful object creation must observe the logical mutation and both fanout boundaries"
);
ecstore
.get_object_info(bucket, object, &ObjectOptions::default())
.await
.expect("object should be readable before bucket delete");
}
async fn any_disk_path_exists(disk_paths: &[PathBuf], relative_path: impl AsRef<Path>) -> bool {
for disk_path in disk_paths {
if tokio::fs::try_exists(disk_path.join(relative_path.as_ref()))
.await
.expect("test disk path should be stat-able")
{
return true;
}
}
false
}
async fn any_disk_has_object_metadata(disk_paths: &[PathBuf], bucket: &str) -> bool {
for disk_path in disk_paths {
if super::has_xlmeta_files(&disk_path.join(bucket))
.await
.expect("object metadata scan should succeed")
{
return true;
}
}
false
}
async fn write_metadata_less_part_on_all_disks(disk_paths: &[PathBuf], bucket: &str, object: &str) {
for disk_path in disk_paths {
let data_dir = Uuid::new_v4();
let part_path = disk_path.join(bucket).join(object).join(data_dir.to_string()).join("part.1");
tokio::fs::create_dir_all(part_path.parent().expect("part path should have a parent"))
.await
.expect("metadata-less data dir should be created");
tokio::fs::write(part_path, b"orphan shard")
.await
.expect("metadata-less part should be written");
}
}
async fn write_bucket_metadata_marker(disk_paths: &[PathBuf], metadata_prefix: &str) {
for disk_path in disk_paths {
let marker_path = disk_path.join(metadata_prefix).join("config.json");
tokio::fs::create_dir_all(marker_path.parent().expect("metadata marker path should have a parent"))
.await
.expect("metadata marker parent should be created");
tokio::fs::write(marker_path, b"bucket metadata")
.await
.expect("metadata marker should be written");
}
}
#[test]
fn should_not_override_when_metadata_created_is_unix_epoch() {
assert!(!should_override_created_from_metadata(OffsetDateTime::UNIX_EPOCH));
}
#[test]
fn should_override_when_metadata_created_is_valid_time() {
let created = OffsetDateTime::from_unix_timestamp(1704067200).expect("valid timestamp");
assert!(should_override_created_from_metadata(created));
}
#[test]
fn table_bucket_delete_guard_rejects_remaining_catalog_metadata() {
let err = validate_table_bucket_delete_allowed("table-bucket", true, true).unwrap_err();
assert!(matches!(err, StorageError::BucketNotEmpty(bucket) if bucket == "table-bucket"));
assert!(validate_table_bucket_delete_allowed("table-bucket", true, false).is_ok());
assert!(validate_table_bucket_delete_allowed("regular-bucket", false, true).is_ok());
}
#[test]
fn bucket_delete_metadata_cleanup_removes_internal_table_catalog_prefix() {
let prefixes = bucket_delete_metadata_cleanup_prefixes("analytics");
assert!(prefixes.contains(&table_bucket_catalog_metadata_prefix("analytics")));
assert!(prefixes.contains(&"buckets/analytics".to_string()));
}
#[test]
fn bucket_delete_marker_path_uses_internal_deleted_bucket_metadata_prefix() {
assert_eq!(bucket_deleted_marker_prefix("analytics"), "buckets/.deleted/analytics");
assert_eq!(
bucket_deleted_marker_volume("analytics"),
format!("{RUSTFS_META_BUCKET}/{BUCKET_META_PREFIX}/.deleted/analytics")
);
}
#[test]
fn bucket_listing_selects_request_and_scanner_fanout() {
assert_eq!(bucket_list_set_concurrency(0, SCANNER_BUCKET_LIST_SET_CONCURRENCY), 1);
assert_eq!(bucket_list_set_concurrency(2, SCANNER_BUCKET_LIST_SET_CONCURRENCY), 2);
assert_eq!(
bucket_list_set_concurrency(100, SCANNER_BUCKET_LIST_SET_CONCURRENCY),
SCANNER_BUCKET_LIST_SET_CONCURRENCY
);
assert_eq!(bucket_list_set_concurrency(100, usize::MAX), 100);
}
#[tokio::test]
async fn metadata_less_residue_scan_ignores_empty_dirs_and_reports_uuid_data() {
let root = tempfile::tempdir().expect("temporary bucket root should be created");
let bucket_path = root.path().join("bucket");
tokio::fs::create_dir_all(bucket_path.join("empty/child"))
.await
.expect("empty directory residue should be created");
let empty = scan_metadata_less_residue(&bucket_path)
.await
.expect("empty residue scan should succeed");
assert!(!empty.has_residue_without_xlmeta());
let data_dir = Uuid::new_v4();
let part_path = bucket_path.join("object").join(data_dir.to_string()).join("part.1");
tokio::fs::create_dir_all(part_path.parent().expect("part path should have a parent"))
.await
.expect("data dir should be created");
tokio::fs::write(&part_path, b"orphan shard")
.await
.expect("part file should be written");
let residue = scan_metadata_less_residue(&bucket_path)
.await
.expect("metadata-less residue scan should succeed");
assert!(residue.has_residue_without_xlmeta());
assert_eq!(residue.files, 1);
assert_eq!(residue.uuid_data_dirs, 1);
let sample = residue.sample.as_deref().expect("part sample should be recorded");
assert!(sample.starts_with("object/"));
assert!(sample.ends_with("/part.1"));
}
#[tokio::test]
async fn metadata_less_residue_scan_shares_one_entry_budget_across_roots() {
let root = tempfile::tempdir().expect("temporary diagnostic roots should be created");
let first_root = root.path().join("disk-a");
let second_root = root.path().join("disk-b");
tokio::fs::create_dir_all(&first_root)
.await
.expect("first diagnostic root should be created");
tokio::fs::create_dir_all(&second_root)
.await
.expect("second diagnostic root should be created");
for index in 0..3 {
std::fs::write(first_root.join(format!("first-{index}")), b"").expect("first-root fixture should be written");
std::fs::write(second_root.join(format!("second-{index}")), b"").expect("second-root fixture should be written");
}
let mut budget = BucketDeleteDiagnosticBudget::with_limits(4, Duration::from_secs(5));
let first = scan_metadata_less_residue_with_budget(&first_root, &mut budget)
.await
.expect("first root should fit the shared budget");
assert!(!first.diagnostic_truncated);
let second = scan_metadata_less_residue_with_budget(&second_root, &mut budget)
.await
.expect("second root should stop at the remaining shared budget");
assert!(second.diagnostic_truncated);
assert!(first.entries_scanned + second.entries_scanned <= 4);
}
#[tokio::test]
async fn metadata_less_residue_scan_honors_an_expired_request_deadline() {
let root = tempfile::tempdir().expect("temporary diagnostic root should be created");
std::fs::write(root.path().join("orphan"), b"").expect("deadline fixture should be written");
let mut budget = BucketDeleteDiagnosticBudget::with_limits(8, Duration::ZERO);
let scan = scan_metadata_less_residue_with_budget(root.path(), &mut budget)
.await
.expect("an expired diagnostic budget should fail closed without an IO error");
assert!(scan.diagnostic_truncated);
assert_eq!(scan.entries_scanned, 0);
assert_eq!(scan.diagnostic_bytes_read, 0);
}
#[tokio::test]
async fn metadata_less_residue_scan_stops_at_diagnostic_budget() {
let root = tempfile::tempdir().expect("temporary bucket root should be created");
let bucket_path = root.path().join("bucket");
tokio::fs::create_dir_all(&bucket_path)
.await
.expect("budget fixture directory should be created");
for index in 0..(BUCKET_DELETE_DIAGNOSTIC_MAX_ENTRIES + 32) {
std::fs::write(bucket_path.join(format!("orphan-{index:05}")), b"").expect("budget fixture file should be created");
}
let residue = scan_metadata_less_residue(&bucket_path)
.await
.expect("budgeted residue scan should fail closed without an IO error");
assert!(residue.diagnostic_truncated);
assert!(residue.has_residue_without_xlmeta());
assert!(!residue.xlmeta_found);
assert!(residue.entries_scanned <= BUCKET_DELETE_DIAGNOSTIC_MAX_ENTRIES);
assert!(residue.files <= BUCKET_DELETE_DIAGNOSTIC_MAX_ENTRIES);
assert_eq!(residue.diagnostic_bytes_read, 0);
}
#[tokio::test(start_paused = true)]
async fn bucket_residue_scan_distinguishes_visible_and_tier_free_xlmeta() {
let (clock_guard_tx, clock_guard_rx) = std::sync::mpsc::channel::<()>();
let clock_guard = tokio::task::spawn_blocking(move || clock_guard_rx.recv());
let root = tempfile::tempdir().expect("temporary bucket root should be created");
let bucket_path = root.path().join("bucket");
let visible_path = bucket_path.join("visible").join(STORAGE_FORMAT_FILE);
tokio::fs::create_dir_all(visible_path.parent().expect("visible xl.meta should have a parent"))
.await
.expect("visible object directory should be created");
let mut visible = FileMeta::new();
visible
.add_version(FileInfo {
version_id: Some(Uuid::new_v4()),
mod_time: Some(OffsetDateTime::now_utc()),
..Default::default()
})
.expect("visible version should encode");
tokio::fs::write(&visible_path, visible.marshal_msg().expect("visible xl.meta should marshal"))
.await
.expect("visible xl.meta should be written");
let visible_scan = scan_metadata_less_residue(&bucket_path)
.await
.expect("visible xl.meta scan should succeed");
assert!(!visible_scan.diagnostic_truncated, "{visible_scan:?}");
assert_eq!(visible_scan.xlmeta_blocker, Some(BucketDeleteBlockerKind::VisibleVersion));
tokio::fs::remove_dir_all(bucket_path.join("visible"))
.await
.expect("visible fixture should be removed");
let free_path = bucket_path.join("free").join(STORAGE_FORMAT_FILE);
tokio::fs::create_dir_all(free_path.parent().expect("free xl.meta should have a parent"))
.await
.expect("free-version object directory should be created");
let source_version_id = Uuid::new_v4();
let mut free = FileMeta::new();
free.add_version(FileInfo {
version_id: Some(source_version_id),
transition_status: TRANSITION_COMPLETE.to_string(),
transitioned_objname: "remote/object".to_string(),
transition_version_id: Some(Uuid::new_v4()),
transition_tier: "WARM".to_string(),
mod_time: Some(OffsetDateTime::now_utc()),
..Default::default()
})
.expect("transitioned source should encode");
let mut delete = FileInfo {
version_id: Some(source_version_id),
mod_time: Some(OffsetDateTime::now_utc()),
..Default::default()
};
delete.set_tier_free_version_id(&Uuid::new_v4().to_string());
free.delete_version(&delete)
.expect("transitioned source delete should create a free-version");
tokio::fs::write(&free_path, free.marshal_msg().expect("free-version xl.meta should marshal"))
.await
.expect("free-version xl.meta should be written");
let free_scan = scan_metadata_less_residue(&bucket_path)
.await
.expect("free-version xl.meta scan should succeed");
assert!(!free_scan.diagnostic_truncated, "{free_scan:?}");
assert_eq!(free_scan.xlmeta_blocker, Some(BucketDeleteBlockerKind::TierFreeVersion));
tokio::fs::remove_dir_all(bucket_path.join("free"))
.await
.expect("free-version fixture should be removed");
let exact_limit_path = bucket_path.join("exact-limit").join(STORAGE_FORMAT_FILE);
tokio::fs::create_dir_all(exact_limit_path.parent().expect("exact-limit xl.meta should have a parent"))
.await
.expect("exact-limit object directory should be created");
let exact_limit = tokio::fs::File::create(&exact_limit_path)
.await
.expect("exact-limit xl.meta should be created");
exact_limit
.set_len(BUCKET_DELETE_XLMETA_DIAGNOSTIC_MAX_BYTES)
.await
.expect("exact-limit xl.meta should be extended without allocating its contents");
let exact_limit_scan = scan_metadata_less_residue(&bucket_path)
.await
.expect("exact-limit xl.meta scan should remain fail closed");
assert!(!exact_limit_scan.diagnostic_truncated, "{exact_limit_scan:?}");
assert_eq!(exact_limit_scan.xlmeta_blocker, Some(BucketDeleteBlockerKind::UnknownXlMeta));
assert_eq!(exact_limit_scan.diagnostic_bytes_read, BUCKET_DELETE_XLMETA_DIAGNOSTIC_MAX_BYTES);
tokio::fs::remove_dir_all(bucket_path.join("exact-limit"))
.await
.expect("exact-limit fixture should be removed");
let oversized_path = bucket_path.join("oversized").join(STORAGE_FORMAT_FILE);
tokio::fs::create_dir_all(oversized_path.parent().expect("oversized xl.meta should have a parent"))
.await
.expect("oversized object directory should be created");
let oversized = tokio::fs::File::create(&oversized_path)
.await
.expect("oversized xl.meta should be created");
oversized
.set_len(BUCKET_DELETE_XLMETA_DIAGNOSTIC_MAX_BYTES + 1)
.await
.expect("oversized xl.meta should be extended without allocating its contents");
let oversized_scan = scan_metadata_less_residue(&bucket_path)
.await
.expect("oversized xl.meta scan should remain fail closed");
assert!(!oversized_scan.diagnostic_truncated, "{oversized_scan:?}");
assert_eq!(oversized_scan.xlmeta_blocker, Some(BucketDeleteBlockerKind::UnknownXlMeta));
assert_eq!(oversized_scan.diagnostic_bytes_read, 0);
assert!(oversized_scan.diagnostic_bytes_read <= BUCKET_DELETE_XLMETA_DIAGNOSTIC_MAX_BYTES);
drop(clock_guard_tx);
let _ = clock_guard
.await
.expect("classification clock guard should exit after its sender is dropped");
let first_io_started = Arc::new(AtomicBool::new(false));
let mut budget = BucketDeleteDiagnosticBudget::new().with_first_io_delay(
BUCKET_DELETE_DIAGNOSTIC_MAX_ELAPSED + Duration::from_millis(100),
first_io_started.clone(),
);
let truncated = scan_metadata_less_residue_with_budget(&bucket_path, &mut budget)
.await
.expect("a diagnostic timeout should return a partial scan");
assert!(first_io_started.load(Ordering::SeqCst));
assert!(truncated.diagnostic_truncated, "{truncated:?}");
assert_eq!(truncated.xlmeta_blocker, None);
assert_eq!(truncated.entries_scanned, 0);
assert_eq!(truncated.diagnostic_bytes_read, 0);
}
#[tokio::test]
#[serial]
async fn bucket_namespace_reads_keep_pre_expansion_bucket_visible() {
let (_temp_dir, ecstore) = setup_multi_pool_bucket_test_env().await;
let bucket = format!("pre-expansion-{}", Uuid::new_v4().simple());
let object = "existing-object";
ecstore.pools[0].disk_set[0]
.make_bucket(&bucket, &MakeBucketOptions::default())
.await
.expect("bucket should be created in the original pool only");
let mut reader = PutObjReader::from_vec(b"object written before pool expansion".to_vec());
ecstore.pools[0]
.put_object(&bucket, object, &mut reader, &ObjectOptions::default())
.await
.expect("object should be written in the original pool only");
let buckets = ecstore
.list_bucket(&BucketOptions::default())
.await
.expect("S3 bucket listing should remain available after adding an empty pool");
assert!(buckets.iter().any(|entry| entry.name == bucket));
let info = ecstore
.get_bucket_info(&bucket, &BucketOptions::default())
.await
.expect("bucket validation should accept a bucket present in the original pool");
assert_eq!(info.name, bucket);
metadata_sys::init_bucket_metadata_sys(ecstore.clone(), vec![bucket.clone()]).await;
let listed = ecstore
.clone()
.list_objects_v2(&bucket, "", None, None, 1000, false, None, false)
.await
.expect("ListObjectsV2 should remain available after adding an empty pool");
assert!(listed.objects.iter().any(|entry| entry.name == object));
ecstore.pools[1].disk_set[0]
.get_bucket_info(&bucket, &BucketOptions::default())
.await
.expect("metadata initialization should heal the bucket volume into the expansion pool");
let object_info = ecstore
.get_object_info(&bucket, object, &ObjectOptions::default())
.await
.expect("an object in the original pool should remain readable after expansion");
assert_eq!(object_info.name, object);
}
#[tokio::test]
#[serial]
async fn bucket_creation_accepts_exact_quorum_in_every_set() {
let (_temp_dir, ecstore) = setup_multi_pool_bucket_test_env().await;
let bucket = format!("exact-set-quorum-{}", Uuid::new_v4().simple());
let original_set = Arc::clone(&ecstore.pools[0].disk_set[0]);
let offline = take_set_disks_offline(&ecstore, &original_set, &[0]).await;
ecstore
.make_bucket_on_sets(&bucket, &MakeBucketOptions::default())
.await
.expect("three of four disks should satisfy the original set quorum");
restore_set_disks(&ecstore, &original_set, offline).await;
for pool in &ecstore.pools {
pool.disk_set[0]
.get_bucket_info(&bucket, &BucketOptions::default())
.await
.expect("every set should expose a bucket created at its exact quorum");
}
}
#[tokio::test]
#[serial]
async fn bucket_creation_rejects_cross_pool_quorum_subsidy() {
let (_temp_dir, ecstore) = setup_multi_pool_bucket_test_env().await;
metadata_sys::init_bucket_metadata_sys(ecstore.clone(), Vec::new()).await;
let bucket = format!("create-set-quorum-minus-one-{}", Uuid::new_v4().simple());
let original_set = Arc::clone(&ecstore.pools[0].disk_set[0]);
let offline = take_set_disks_offline(&ecstore, &original_set, &[0, 1]).await;
let err = ecstore
.make_bucket(&bucket, &MakeBucketOptions::default())
.await
.expect_err("a healthy expansion pool must not subsidize a failed original set");
restore_set_disks(&ecstore, &original_set, offline).await;
assert!(
matches!(&err, StorageError::InsufficientWriteQuorum(name, object) if name == &bucket && object.is_empty()),
"unexpected error: {err}"
);
for pool in &ecstore.pools {
assert_eq!(
pool.disk_set[0]
.get_bucket_info(&bucket, &BucketOptions::default())
.await
.expect_err("failed creation must not leave an authoritative bucket"),
StorageError::VolumeNotFound
);
}
}
#[tokio::test]
#[serial]
async fn bucket_creation_prioritizes_hard_set_error_over_existing_bucket() {
let (_temp_dir, ecstore) = setup_multi_pool_bucket_test_env().await;
metadata_sys::init_bucket_metadata_sys(ecstore.clone(), Vec::new()).await;
let bucket = format!("create-hard-error-{}", Uuid::new_v4().simple());
ecstore.pools[0].disk_set[0]
.make_bucket(&bucket, &MakeBucketOptions::default())
.await
.expect("the original pool should contain the bucket");
let expansion_set = Arc::clone(&ecstore.pools[1].disk_set[0]);
let offline = take_set_disks_offline(&ecstore, &expansion_set, &[0, 1]).await;
let err = ecstore
.make_bucket(&bucket, &MakeBucketOptions::default())
.await
.expect_err("a hard set failure must outrank BucketExists from another set");
restore_set_disks(&ecstore, &expansion_set, offline).await;
assert!(
matches!(&err, StorageError::InsufficientWriteQuorum(name, object) if name == &bucket && object.is_empty()),
"unexpected error: {err}"
);
ecstore.pools[0].disk_set[0]
.get_bucket_info(&bucket, &BucketOptions::default())
.await
.expect("failure in another set must not roll back a pre-existing bucket");
}
#[tokio::test]
#[serial]
async fn bucket_delete_rejects_cross_pool_quorum_subsidy_before_mutation() {
let (_temp_dir, ecstore) = setup_multi_pool_bucket_test_env().await;
metadata_sys::init_bucket_metadata_sys(ecstore.clone(), Vec::new()).await;
let bucket = format!("delete-set-quorum-minus-one-{}", Uuid::new_v4().simple());
ecstore
.make_bucket(&bucket, &MakeBucketOptions::default())
.await
.expect("bucket should be created in every set before deletion");
let original_set = Arc::clone(&ecstore.pools[0].disk_set[0]);
let offline = take_set_disks_offline(&ecstore, &original_set, &[0, 1, 2]).await;
let err = ecstore
.delete_bucket(
&bucket,
&DeleteBucketOptions {
force: true,
..Default::default()
},
)
.await
.expect_err("a healthy expansion pool must not subsidize a failed original set deletion");
restore_set_disks(&ecstore, &original_set, offline).await;
assert!(
matches!(&err, StorageError::InsufficientWriteQuorum(name, object) if name == &bucket && object.is_empty()),
"unexpected error: {err}"
);
for pool in &ecstore.pools {
pool.disk_set[0]
.get_bucket_info(&bucket, &BucketOptions::default())
.await
.expect("failed deletion preflight must preserve every bucket volume");
}
let (_, persisted) = metadata_sys::get_config_from_disk_with_presence_in(&ecstore.ctx, &bucket)
.await
.expect("failed deletion must preserve bucket metadata");
assert!(persisted);
}
#[tokio::test]
#[serial]
async fn bucket_delete_rolls_back_sets_after_partial_failure() {
let (_temp_dir, ecstore) = setup_multi_pool_bucket_test_env().await;
let bucket = format!("delete-set-rollback-{}", Uuid::new_v4().simple());
ecstore
.make_bucket_on_sets(&bucket, &MakeBucketOptions::default())
.await
.expect("bucket should be created in every set before deletion");
let original_set = Arc::clone(&ecstore.pools[0].disk_set[0]);
let offline = take_set_disks_offline(&ecstore, &original_set, &[0, 1]).await;
let err = ecstore
.delete_bucket_on_sets(&bucket, &DeleteBucketOptions::default())
.await
.expect_err("a failed set must fail the complete bucket deletion");
restore_set_disks(&ecstore, &original_set, offline).await;
assert_eq!(err, StorageError::ErasureWriteQuorum);
for pool in &ecstore.pools {
pool.disk_set[0]
.get_bucket_info(&bucket, &BucketOptions::default())
.await
.expect("delete rollback should restore the bucket namespace in every set");
}
}
#[tokio::test]
#[serial]
async fn bucket_config_update_accepts_pre_expansion_bucket() {
let (_temp_dir, ecstore) = setup_multi_pool_bucket_test_env().await;
metadata_sys::init_bucket_metadata_sys(ecstore.clone(), Vec::new()).await;
let bucket = format!("config-pre-expansion-{}", Uuid::new_v4().simple());
ecstore
.make_bucket(&bucket, &MakeBucketOptions::default())
.await
.expect("bucket should be created with authoritative metadata");
ecstore.pools[1].disk_set[0]
.delete_bucket(&bucket, &DeleteBucketOptions::default())
.await
.expect("the expansion pool bucket volume should be removed for the regression state");
let policy = br#"{"Version":"2012-10-17","Statement":[]}"#.to_vec();
ecstore
.update_bucket_metadata_config(&bucket, crate::bucket::metadata::BUCKET_POLICY_CONFIG, policy.clone())
.await
.expect("config mutation should validate existence per erasure set");
let (stored, _) = ecstore
.get_bucket_policy_raw(&bucket)
.await
.expect("updated bucket policy should remain readable");
assert_eq!(stored.as_bytes(), policy);
}
#[tokio::test]
#[serial]
async fn bucket_metadata_init_recreates_pre_expansion_bucket_in_new_pool() {
let (_temp_dir, ecstore) = setup_multi_pool_bucket_test_env().await;
let bucket = format!("metadata-expansion-{}", Uuid::new_v4().simple());
ecstore.pools[0].disk_set[0]
.make_bucket(&bucket, &MakeBucketOptions::default())
.await
.expect("bucket should be created in the original pool only");
assert_eq!(
ecstore.pools[1].disk_set[0]
.get_bucket_info(&bucket, &BucketOptions::default())
.await
.expect_err("new pool should initially have no bucket volume"),
StorageError::VolumeNotFound
);
metadata_sys::init_bucket_metadata_sys(ecstore.clone(), vec![bucket.clone()]).await;
ecstore.pools[1].disk_set[0]
.get_bucket_info(&bucket, &BucketOptions::default())
.await
.expect("metadata initialization should recreate the bucket volume in the new pool");
}
#[tokio::test]
#[serial]
async fn bucket_metadata_init_repairs_half_created_expansion_pool() {
for complete_pool in 0..2 {
let (temp_dir, ecstore) = setup_multi_pool_bucket_test_env().await;
let bucket = format!("partial-expansion-{}", Uuid::new_v4().simple());
for pool_index in 0..2 {
let present_disks = if pool_index == complete_pool { 4 } else { 2 };
for disk_index in 0..present_disks {
tokio::fs::create_dir(
temp_dir
.path()
.join(format!("pool{pool_index}-disk{disk_index}"))
.join(&bucket),
)
.await
.expect("fixture bucket volume should be created");
}
}
assert_eq!(
ecstore
.get_bucket_info_from_sets(&bucket, &BucketOptions::default())
.await
.expect_err("request validation must reject a half-created expansion set"),
StorageError::ErasureWriteQuorum
);
metadata_sys::init_bucket_metadata_sys(ecstore.clone(), vec![bucket.clone()]).await;
for pool_index in 0..2 {
for disk_index in 0..4 {
assert!(
temp_dir
.path()
.join(format!("pool{pool_index}-disk{disk_index}"))
.join(&bucket)
.is_dir(),
"metadata initialization must heal every missing expansion volume"
);
}
}
ecstore
.get_bucket_info_from_sets(&bucket, &BucketOptions::default())
.await
.expect("strict request validation should succeed after volume repair");
}
}
#[tokio::test]
#[serial]
async fn bucket_metadata_init_does_not_combine_partial_set_evidence() {
let (temp_dir, ecstore) = setup_multi_pool_bucket_test_env().await;
let bucket = format!("no-quorum-expansion-{}", Uuid::new_v4().simple());
for pool_index in 0..2 {
for disk_index in 0..2 {
tokio::fs::create_dir(
temp_dir
.path()
.join(format!("pool{pool_index}-disk{disk_index}"))
.join(&bucket),
)
.await
.expect("fixture bucket volume should be created");
}
}
assert_eq!(
ecstore
.bucket_exists_for_heal(&bucket)
.await
.expect_err("repair must require a complete quorum within one set"),
StorageError::ErasureWriteQuorum
);
metadata_sys::init_bucket_metadata_sys(ecstore.clone(), vec![bucket.clone()]).await;
for pool_index in 0..2 {
for disk_index in 0..4 {
assert_eq!(
temp_dir
.path()
.join(format!("pool{pool_index}-disk{disk_index}"))
.join(&bucket)
.is_dir(),
disk_index < 2,
"unproven bucket generations must not recreate missing volumes"
);
}
}
}
#[tokio::test]
#[serial]
async fn bucket_metadata_init_does_not_recreate_stale_bucket_name() {
let (_temp_dir, ecstore) = setup_multi_pool_bucket_test_env().await;
let bucket = format!("stale-expansion-{}", Uuid::new_v4().simple());
metadata_sys::init_bucket_metadata_sys(ecstore.clone(), vec![bucket.clone()]).await;
for pool in &ecstore.pools {
let err = pool.disk_set[0]
.get_bucket_info(&bucket, &BucketOptions::default())
.await
.expect_err("a stale startup listing must not recreate a deleted bucket");
assert_eq!(err, StorageError::VolumeNotFound);
}
}
#[tokio::test]
#[serial]
async fn bucket_info_read_quorum_tracks_erasure_layout() {
for (drive_count, parity) in [(2, 1), (3, 1), (4, 2), (5, 2), (6, 3), (8, 4), (6, 2), (12, 6)] {
let (_temp_dir, store) = setup_bucket_quorum_test_env(&[drive_count], Some(parity)).await;
metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
let bucket = format!("read-quorum-{drive_count}-{parity}");
let object = "uncached-object";
let body = b"erasure read quorum must follow the persisted layout".repeat(32_768);
store
.make_bucket(&bucket, &MakeBucketOptions::default())
.await
.expect("healthy namespace should accept bucket creation");
store
.put_object(&bucket, object, &mut PutObjReader::from_vec(body.clone()), &ObjectOptions::default())
.await
.expect("healthy erasure set should accept the seed object");
let set = &store.pools[0].disk_set[0];
let lock = set
.new_ns_lock(&bucket, object)
.await
.expect("seed namespace lock should resolve");
drop(
lock.get_write_lock(Duration::from_secs(30))
.await
.expect("seed physical fanout must finish before taking disks offline"),
);
if (drive_count, parity) == (6, 3) {
let mut kvs = rustfs_config::server_config::KVS::new();
kvs.insert(crate::config::storageclass::CLASS_STANDARD.to_string(), "EC:2".to_string());
set.set_test_storage_class_config(
crate::config::storageclass::lookup_config_for_pools_without_env(&kvs, &[drive_count])
.expect("a later storage-class change must not raise old objects' read quorum"),
);
}
let offline_indexes = (0..parity).collect::<Vec<_>>();
let offline = take_set_disks_offline(&store, set, &offline_indexes).await;
let info = store
.get_bucket_info(&bucket, &BucketOptions::default())
.await
.expect("bucket validation must admit the object's exact read quorum");
assert_eq!(info.name, bucket);
let mut reader = store
.get_object_reader(&bucket, object, None, Default::default(), &ObjectOptions::default())
.await
.expect("the persisted layout should remain readable at its exact data-shard quorum");
let mut restored = Vec::new();
reader
.stream
.read_to_end(&mut restored)
.await
.expect("quorum read should reconstruct the body");
assert_eq!(restored, body, "layout {drive_count}/{parity} must retain exact object contents");
drop(reader);
if drive_count - parity == drive_count / 2 {
let error = store
.get_bucket_info_from_sets(&bucket, &BucketOptions::default())
.await
.expect_err("bucket mutations must retain their majority namespace check");
assert_eq!(error, StorageError::ErasureWriteQuorum);
}
let below_quorum = take_set_disks_offline(&store, set, &[parity]).await;
let read = store
.get_object_reader(&bucket, object, None, Default::default(), &ObjectOptions::default())
.await;
match read {
Ok(mut reader) => assert!(
reader.stream.read_to_end(&mut Vec::new()).await.is_err(),
"layout {drive_count}/{parity} must reject fewer than its data-shard quorum"
),
Err(error) => assert!(
matches!(error, StorageError::ErasureReadQuorum | StorageError::InsufficientReadQuorum(_, _)),
"a missing shard must report read quorum loss, got {error}"
),
}
restore_set_disks(&store, set, below_quorum).await;
restore_set_disks(&store, set, offline).await;
}
}
#[tokio::test]
#[serial]
async fn lazy_metadata_load_uses_read_quorum_for_non_lock_bucket() {
let (_temp_dir, store) = setup_bucket_quorum_test_env(&[4], Some(2)).await;
metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
let bucket = format!("lazy-read-quorum-{}", Uuid::new_v4().simple());
store
.make_bucket(&bucket, &MakeBucketOptions::default())
.await
.expect("healthy namespace should accept bucket creation");
metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
let set = &store.pools[0].disk_set[0];
let offline = take_set_disks_offline(&store, set, &[0, 1]).await;
metadata_sys::get_on_demand_migration_config_in(&store, &bucket)
.await
.expect("cold metadata load should admit the exact namespace read quorum");
assert!(
metadata_sys::get_in(&store.ctx, &bucket).await.is_ok(),
"the lazy load should publish authoritative metadata"
);
assert_eq!(
store
.get_bucket_info_from_sets(&bucket, &BucketOptions::default())
.await
.expect_err("bucket mutations must retain their majority namespace check"),
StorageError::ErasureWriteQuorum
);
restore_set_disks(&store, set, offline).await;
}
#[tokio::test]
#[serial]
async fn cold_bucket_reads_succeed_below_write_quorum() {
let (_temp_dir, store) = setup_bucket_quorum_test_env(&[4], Some(2)).await;
metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
let bucket = format!("cold-read-quorum-{}", Uuid::new_v4().simple());
let object = "seed-object";
let body = b"cold bucket reads must survive lost write quorum".to_vec();
store
.make_bucket(&bucket, &MakeBucketOptions::default())
.await
.expect("healthy namespace should accept bucket creation");
store
.put_object(&bucket, object, &mut PutObjReader::from_vec(body.clone()), &ObjectOptions::default())
.await
.expect("healthy erasure set should accept the seed object");
metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
assert!(
metadata_sys::get_in(&store.ctx, &bucket).await.is_err(),
"the replacement metadata system must start without this bucket"
);
let set = &store.pools[0].disk_set[0];
let lock = set
.new_ns_lock(&bucket, object)
.await
.expect("seed namespace lock should resolve");
drop(
lock.get_write_lock(Duration::from_secs(30))
.await
.expect("seed physical fanout must finish before taking disks offline"),
);
let offline = take_set_disks_offline(&store, set, &[0, 1]).await;
let info = store
.get_bucket_info(&bucket, &BucketOptions::default())
.await
.expect("HEAD bucket must succeed at read quorum before the bucket is cached");
assert_eq!(info.name, bucket);
let mut reader = store
.get_object_reader(&bucket, object, None, Default::default(), &ObjectOptions::default())
.await
.expect("GET must succeed for a bucket that was not loaded at startup");
let mut restored = Vec::new();
reader
.stream
.read_to_end(&mut restored)
.await
.expect("quorum read should reconstruct the body");
assert_eq!(restored, body);
drop(reader);
let object_info = store
.get_object_info(&bucket, object, &ObjectOptions::default())
.await
.expect("HEAD object must succeed at read quorum");
assert_eq!(object_info.name, object);
assert_eq!(object_info.size, i64::try_from(body.len()).expect("body length fits i64"));
let listed = store
.clone()
.list_objects_v2(&bucket, "", None, None, 100, false, None, false)
.await
.expect("List must succeed at read quorum");
assert!(
listed.objects.iter().any(|item| item.name == object),
"list should include the seed object, got {:?}",
listed.objects.iter().map(|item| item.name.clone()).collect::<Vec<_>>()
);
assert!(
metadata_sys::get_in(&store.ctx, &bucket).await.is_ok(),
"the cold load should publish authoritative metadata"
);
let write_error = store
.put_object(&bucket, "rejected-object", &mut PutObjReader::from_vec(body), &ObjectOptions::default())
.await
.expect_err("writes must still fail without write quorum");
assert!(
matches!(
write_error,
StorageError::InsufficientWriteQuorum(_, _) | StorageError::ErasureWriteQuorum
),
"writes must fail with a write-quorum error, got {write_error}"
);
restore_set_disks(&store, set, offline).await;
}
#[tokio::test]
#[serial]
async fn lazy_metadata_load_rejects_below_read_quorum() {
let (_temp_dir, store) = setup_bucket_quorum_test_env(&[4], Some(2)).await;
metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
let bucket = format!("lazy-below-quorum-{}", Uuid::new_v4().simple());
store
.make_bucket(&bucket, &MakeBucketOptions::default())
.await
.expect("healthy namespace should accept bucket creation");
metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
let set = &store.pools[0].disk_set[0];
let offline = take_set_disks_offline(&store, set, &[0, 1, 2]).await;
let error = metadata_sys::get_on_demand_migration_config_in(&store, &bucket)
.await
.expect_err("cold metadata load must reject read quorum minus one");
assert!(
matches!(error, StorageError::ErasureReadQuorum | StorageError::InsufficientReadQuorum(_, _)),
"below read quorum must fail with a read-quorum error, got {error}"
);
assert!(
metadata_sys::get_in(&store.ctx, &bucket).await.is_err(),
"a below-quorum load must not publish metadata"
);
restore_set_disks(&store, set, offline).await;
}
#[tokio::test]
#[serial]
async fn lazy_metadata_load_keeps_lock_creation_intent_fail_closed() {
let (temp_dir, store) = setup_bucket_quorum_test_env(&[4], Some(2)).await;
metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
let bucket = format!("lazy-lock-intent-{}", Uuid::new_v4().simple());
let mut intent = BucketMetadata::new(&bucket);
intent.lock_enabled = true;
metadata_sys::set_new_bucket_metadata_intent_in(&store.ctx, intent)
.await
.expect("Object Lock creation intent should persist metadata and its incarnation sidecar");
let set = &store.pools[0].disk_set[0];
let offline = take_set_disks_offline(&store, set, &[0, 1]).await;
assert_eq!(
store
.make_bucket_on_sets(&bucket, &MakeBucketOptions::default())
.await
.expect_err("physical creation must not commit below write quorum"),
StorageError::ErasureWriteQuorum
);
assert!(!temp_dir.path().join("pool0-disk0").join(&bucket).exists());
assert!(!temp_dir.path().join("pool0-disk1").join(&bucket).exists());
assert!(temp_dir.path().join("pool0-disk2").join(&bucket).is_dir());
assert!(temp_dir.path().join("pool0-disk3").join(&bucket).is_dir());
metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
let error = metadata_sys::get_on_demand_migration_config_in(&store, &bucket)
.await
.expect_err("Object Lock intent must not become visible at read quorum");
assert_eq!(error, StorageError::ErasureWriteQuorum);
assert!(
metadata_sys::get_in(&store.ctx, &bucket).await.is_err(),
"the rejected Object Lock intent must not enter the metadata cache"
);
assert_eq!(
store
.get_bucket_info(&bucket, &BucketOptions::default())
.await
.expect_err("bucket-info read fallback must also reject a pending Object Lock intent"),
StorageError::ErasureWriteQuorum
);
restore_set_disks(&store, set, offline).await;
}
#[tokio::test]
#[serial]
async fn lazy_metadata_load_admits_committed_lock_bucket_at_read_quorum() {
let (_temp_dir, store) = setup_bucket_quorum_test_env(&[4], Some(2)).await;
metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
let bucket = format!("lazy-committed-lock-{}", Uuid::new_v4().simple());
store
.make_bucket(
&bucket,
&MakeBucketOptions {
lock_enabled: true,
..Default::default()
},
)
.await
.expect("healthy namespace should accept Object Lock bucket creation");
let (metadata, persisted) = metadata_sys::get_config_from_disk_with_presence_in(&store.ctx, &bucket)
.await
.expect("read committed Object Lock metadata");
assert!(persisted && metadata.bucket_creation_committed);
metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
let set = &store.pools[0].disk_set[0];
let offline = take_set_disks_offline(&store, set, &[0, 1]).await;
metadata_sys::get_on_demand_migration_config_in(&store, &bucket)
.await
.expect("committed Object Lock metadata should load at read quorum");
assert!(
metadata_sys::get_in(&store.ctx, &bucket).await.is_ok(),
"committed Object Lock metadata should be published"
);
restore_set_disks(&store, set, offline).await;
}
#[tokio::test]
#[serial]
async fn creation_commit_backfill_runs_under_the_config_write_fence() {
let (_temp_dir, store) = setup_bucket_quorum_test_env(&[4], Some(2)).await;
metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
let bucket = format!("lazy-commit-backfill-{}", Uuid::new_v4().simple());
let mut intent = BucketMetadata::new(&bucket);
intent.lock_enabled = true;
metadata_sys::set_new_bucket_metadata_intent_in(&store.ctx, intent)
.await
.expect("persist pending Object Lock intent");
store
.make_bucket_on_sets(&bucket, &MakeBucketOptions::default())
.await
.expect("physical creation should commit at full write quorum");
metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
metadata_sys::get_object_lock_config_state_in(&store.ctx, &bucket)
.await
.expect("the fenced authority path should backfill the creation commit");
let (metadata, persisted) = metadata_sys::get_config_from_disk_with_presence_in(&store.ctx, &bucket)
.await
.expect("read backfilled creation commit");
assert!(persisted && metadata.bucket_creation_committed);
metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
let set = &store.pools[0].disk_set[0];
let offline = take_set_disks_offline(&store, set, &[0, 1]).await;
metadata_sys::get_on_demand_migration_config_in(&store, &bucket)
.await
.expect("backfilled Object Lock bucket should load at read quorum");
restore_set_disks(&store, set, offline).await;
}
#[tokio::test]
#[serial]
async fn enabling_object_lock_on_existing_bucket_establishes_commitment() {
let (_temp_dir, store) = setup_bucket_quorum_test_env(&[4], Some(2)).await;
metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
let bucket = format!("enable-lock-legacy-{}", Uuid::new_v4().simple());
store
.make_bucket(&bucket, &MakeBucketOptions::default())
.await
.expect("create a versioned non-lock bucket");
let mut legacy = metadata_sys::get_in(&store.ctx, &bucket)
.await
.expect("cached metadata")
.as_ref()
.clone();
legacy.lock_enabled = false;
legacy.object_lock_config_xml = Vec::new();
legacy.object_lock_config = None;
legacy.bucket_creation_committed = false;
metadata_sys::set_new_bucket_metadata_intent_in(&store.ctx, legacy)
.await
.expect("persist legacy-shaped metadata");
let lock_xml = crate::bucket::utils::serialize(&*crate::store::ENABLED_OBJECT_LOCK_CONFIG)
.expect("serialize Object Lock configuration");
metadata_sys::update_in(&store.ctx, &bucket, "object-lock.xml", lock_xml)
.await
.expect("enabling Object Lock on an existing bucket must succeed");
assert!(
matches!(
metadata_sys::get_object_lock_config_state_in(&store.ctx, &bucket)
.await
.expect("read the freshly enabled Object Lock state"),
metadata_sys::ObjectLockConfigState::Configured { .. }
),
"a healthy cluster must not report ErasureWriteQuorum after enabling Object Lock"
);
let (metadata, persisted) = metadata_sys::get_config_from_disk_with_presence_in(&store.ctx, &bucket)
.await
.expect("read persisted metadata");
assert!(persisted && metadata.bucket_creation_committed);
}
#[tokio::test]
#[serial]
async fn creation_commit_backfill_preserves_concurrent_config_update() {
let (_temp_dir, store) = setup_bucket_quorum_test_env(&[4], Some(2)).await;
metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
let bucket = format!("commit-backfill-pack-{}", Uuid::new_v4().simple());
store
.make_bucket(&bucket, &MakeBucketOptions::default())
.await
.expect("create bucket");
let mut stale = metadata_sys::get_in(&store.ctx, &bucket)
.await
.expect("cached metadata")
.as_ref()
.clone();
stale.bucket_creation_committed = false;
metadata_sys::set_new_bucket_metadata_intent_in(&store.ctx, stale.clone())
.await
.expect("persist the stale pre-upgrade snapshot");
let mut newer = stale.clone();
newer.lock_enabled = true;
newer.object_lock_config_xml = crate::bucket::utils::serialize(&*crate::store::ENABLED_OBJECT_LOCK_CONFIG)
.expect("serialize Object Lock configuration");
newer
.save_with_store(store.clone())
.await
.expect("persist the newer configuration");
metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
metadata_sys::get_object_lock_config_state_in(&store.ctx, &bucket)
.await
.expect("fenced commit migration");
let (metadata, persisted) = metadata_sys::get_config_from_disk_with_presence_in(&store.ctx, &bucket)
.await
.expect("read persisted metadata");
assert!(persisted && metadata.bucket_creation_committed);
assert!(
metadata.lock_enabled && !metadata.object_lock_config_xml.is_empty(),
"the acknowledged Object Lock update must survive the backfill"
);
}
#[tokio::test]
#[serial]
async fn creation_commit_backfill_keeps_recreated_generation() {
let (_temp_dir, store) = setup_bucket_quorum_test_env(&[4], Some(2)).await;
metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
let bucket = format!("commit-backfill-recreate-{}", Uuid::new_v4().simple());
store
.make_bucket(&bucket, &MakeBucketOptions::default())
.await
.expect("create bucket");
let old_incarnation = store.bucket_incarnation_id_from_disk(&bucket).await.expect("old incarnation");
let mut stale = metadata_sys::get_in(&store.ctx, &bucket)
.await
.expect("cached metadata")
.as_ref()
.clone();
stale.lock_enabled = true;
stale.object_lock_config_xml = crate::bucket::utils::serialize(&*crate::store::ENABLED_OBJECT_LOCK_CONFIG)
.expect("serialize Object Lock configuration");
stale.bucket_creation_committed = false;
stale.save_with_store(store.clone()).await.expect("persist stale metadata");
store
.delete_bucket(&bucket, &crate::storage_api_contracts::bucket::DeleteBucketOptions::default())
.await
.expect("delete bucket");
store
.make_bucket(
&bucket,
&MakeBucketOptions {
lock_enabled: true,
..Default::default()
},
)
.await
.expect("recreate bucket with Object Lock");
let new_incarnation = store.bucket_incarnation_id_from_disk(&bucket).await.expect("new incarnation");
assert_ne!(old_incarnation, new_incarnation);
metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
metadata_sys::get_object_lock_config_state_in(&store.ctx, &bucket)
.await
.expect("fenced backfill on the replacement generation");
let (metadata, persisted) = metadata_sys::get_config_from_disk_with_presence_in(&store.ctx, &bucket)
.await
.expect("read persisted metadata");
assert!(persisted);
assert_eq!(
metadata.bucket_incarnation_id, new_incarnation,
"the replacement generation must stay authoritative"
);
assert!(metadata.bucket_creation_committed);
}
#[tokio::test]
#[serial]
async fn bucket_info_read_quorum_is_scoped_to_each_erasure_set() {
let (_temp_dir, store) = setup_bucket_quorum_test_env(&[4, 6], None).await;
metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
let bucket = "read-quorum-mixed-pools";
store
.make_bucket(bucket, &MakeBucketOptions::default())
.await
.expect("healthy pools should accept bucket creation");
let first_set = &store.pools[0].disk_set[0];
let second_set = &store.pools[1].disk_set[0];
let first_offline = take_set_disks_offline(&store, first_set, &[0, 1]).await;
let second_offline = take_set_disks_offline(&store, second_set, &[0, 1, 2]).await;
store
.get_bucket_info(bucket, &BucketOptions::default())
.await
.expect("each set independently satisfies its namespace read quorum");
for (set, extra_disk) in [(first_set, 2), (second_set, 3)] {
let extra_offline = take_set_disks_offline(&store, set, &[extra_disk]).await;
assert_eq!(
store
.get_bucket_info(bucket, &BucketOptions::default())
.await
.expect_err("another pool must not subsidize a set below its read quorum"),
StorageError::ErasureReadQuorum
);
restore_set_disks(&store, set, extra_offline).await;
}
restore_set_disks(&store, first_set, first_offline).await;
restore_set_disks(&store, second_set, second_offline).await;
}
#[tokio::test]
#[serial]
async fn bucket_info_read_quorum_requires_authoritative_metadata() {
for state in ["missing", "corrupt", "foreign", "incarnation"] {
let (_temp_dir, store) = setup_bucket_quorum_test_env(&[4], None).await;
metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
let bucket = format!("read-quorum-{state}-metadata");
let mut metadata = if state == "missing" {
store
.make_bucket_on_sets(&bucket, &MakeBucketOptions::default())
.await
.expect("simulate directories left before bucket metadata is published");
BucketMetadata::new(&bucket)
} else {
store
.make_bucket(&bucket, &MakeBucketOptions::default())
.await
.expect("healthy bucket should publish metadata");
metadata_sys::get_in(&store.ctx, &bucket)
.await
.expect("seed metadata should be cached")
.as_ref()
.clone()
};
let path = metadata.save_file_path();
match state {
"corrupt" => crate::config::com::save_config(store.clone(), &path, b"corrupt".to_vec())
.await
.expect("persist corrupt metadata while the cached copy remains valid"),
"foreign" => {
metadata.name = "different-bucket".to_string();
let mut encoded = vec![1, 0, 1, 0];
encoded.extend(metadata.marshal_msg().expect("foreign metadata should encode"));
crate::config::com::save_config(store.clone(), &path, encoded)
.await
.expect("persist metadata for a different bucket at the requested path");
}
"incarnation" => crate::bucket::metadata::save_bucket_incarnation(store.clone(), &bucket, Uuid::new_v4())
.await
.expect("persist a different bucket generation"),
_ => {}
}
let set = &store.pools[0].disk_set[0];
let offline = take_set_disks_offline(&store, set, &[0, 1]).await;
let error = store
.get_bucket_info(&bucket, &BucketOptions::default())
.await
.expect_err("read admission must not trust residual directories or cached metadata");
match state {
"missing" => assert_eq!(error, StorageError::ErasureReadQuorum),
"foreign" => assert_eq!(error, StorageError::FileCorrupt),
"incarnation" => assert!(error.to_string().contains("sidecar does not match bucket metadata")),
"corrupt" => assert!(error.to_string().contains("format invalid"), "unexpected corruption error: {error}"),
_ => unreachable!(),
}
restore_set_disks(&store, set, offline).await;
}
}
#[tokio::test]
#[serial]
async fn bucket_info_read_quorum_accepts_persisted_legacy_metadata() {
let (_temp_dir, store) = setup_bucket_quorum_test_env(&[4], None).await;
metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
let bucket = "interop";
store
.make_bucket_on_sets(bucket, &MakeBucketOptions::default())
.await
.expect("legacy bucket directories should exist");
let hex = include_str!("../../tests/fixtures/minio/bucket_metadata.blob.hex")
.split_whitespace()
.collect::<String>();
let body = (0..hex.len())
.step_by(2)
.map(|index| u8::from_str_radix(&hex[index..index + 2], 16).expect("pinned MinIO metadata fixture"))
.collect();
crate::config::com::save_config(store.clone(), &BucketMetadata::new(bucket).save_file_path(), body)
.await
.expect("legacy metadata should be persisted without an incarnation sidecar");
let set = &store.pools[0].disk_set[0];
let offline = take_set_disks_offline(&store, set, &[0, 1]).await;
let info = store
.get_bucket_info(bucket, &BucketOptions::default())
.await
.expect("persisted MinIO metadata should authorize reads at the namespace read quorum");
assert_eq!(info.name, bucket);
assert!(info.versioning);
assert!(info.object_locking);
restore_set_disks(&store, set, offline).await;
}
#[tokio::test]
#[serial]
async fn bucket_namespace_reads_report_missing_when_every_set_is_absent() {
let (_temp_dir, ecstore) = setup_multi_pool_bucket_test_env().await;
let bucket = format!("absent-{}", Uuid::new_v4().simple());
let buckets = ecstore
.list_bucket(&BucketOptions::default())
.await
.expect("an empty healthy namespace should remain listable");
assert!(buckets.iter().all(|entry| entry.name != bucket));
let err = ecstore
.get_bucket_info(&bucket, &BucketOptions::default())
.await
.expect_err("a bucket absent from every set must remain missing");
assert_eq!(err, StorageError::VolumeNotFound);
}
#[tokio::test]
#[serial]
async fn bucket_namespace_reads_fail_closed_when_any_set_loses_quorum() {
let (_temp_dir, ecstore) = setup_multi_pool_bucket_test_env().await;
metadata_sys::init_bucket_metadata_sys(ecstore.clone(), Vec::new()).await;
let bucket = format!("degraded-expansion-{}", Uuid::new_v4().simple());
ecstore.pools[0].disk_set[0]
.make_bucket(&bucket, &MakeBucketOptions::default())
.await
.expect("bucket should be created in the original pool only");
ecstore.pools[1].disk_set[0].disks.write().await[0] = None;
ecstore.pools[1].disk_set[0].disks.write().await[1] = None;
ecstore.pools[1].disk_set[0].disks.write().await[2] = None;
let list_err = ecstore
.list_bucket(&BucketOptions::default())
.await
.expect_err("listing must not hide an unavailable expansion pool");
assert_eq!(list_err, StorageError::ErasureWriteQuorum);
let info_err = ecstore
.get_bucket_info(&bucket, &BucketOptions::default())
.await
.expect_err("bucket validation must fail when an expansion pool is unavailable");
assert_eq!(info_err, StorageError::ErasureReadQuorum);
}
#[tokio::test]
#[serial]
async fn scanner_bucket_listing_unions_every_erasure_set() {
let (_temp_dir, ecstore) = setup_multi_pool_bucket_test_env().await;
let bucket = format!("second-pool-only-{}", Uuid::new_v4().simple());
ecstore.pools[1].disk_set[0]
.make_bucket(&bucket, &MakeBucketOptions::default())
.await
.expect("bucket should be created in the second pool only");
let listing = ecstore
.list_bucket_for_scanner(&crate::storage_api_contracts::bucket::BucketOptions {
no_metadata: true,
..Default::default()
})
.await
.expect("scanner should enumerate every pool and set");
assert!(listing.topology_complete);
assert!(listing.buckets.iter().any(|entry| entry.name == bucket));
assert_eq!(listing.set_buckets.len(), 2);
assert!(
listing
.set_buckets
.iter()
.find(|scope| scope.pool_index == 0 && scope.set_index == 0)
.is_some_and(|scope| scope.buckets.is_empty())
);
assert!(
listing
.set_buckets
.iter()
.find(|scope| scope.pool_index == 1 && scope.set_index == 0)
.is_some_and(|scope| scope.buckets.iter().any(|entry| entry.name == bucket))
);
}
#[tokio::test]
async fn scanner_bucket_listing_marks_degraded_set_incomplete() {
let (_temp_dir, ecstore) = setup_multi_pool_bucket_test_env().await;
let bucket = format!("degraded-set-{}", Uuid::new_v4().simple());
let set = &ecstore.pools[0].disk_set[0];
set.make_bucket(&bucket, &MakeBucketOptions::default())
.await
.expect("bucket should be created before a disk is removed");
set.disks.write().await[0] = None;
let listing = ecstore
.list_bucket_for_scanner(&crate::storage_api_contracts::bucket::BucketOptions {
no_metadata: true,
..Default::default()
})
.await
.expect("a degraded set with quorum should still return candidate buckets");
assert!(listing.buckets.iter().any(|entry| entry.name == bucket));
assert!(!listing.topology_complete);
}
#[tokio::test]
async fn scanner_bucket_listing_marks_divergent_disk_views_incomplete() {
let (temp_dir, ecstore) = setup_multi_pool_bucket_test_env().await;
let bucket = format!("divergent-set-{}", Uuid::new_v4().simple());
ecstore.pools[0].disk_set[0]
.make_bucket(&bucket, &MakeBucketOptions::default())
.await
.expect("bucket should be created before disk views diverge");
for disk_index in 0..2 {
tokio::fs::remove_dir_all(temp_dir.path().join(format!("pool0-disk{disk_index}")).join(&bucket))
.await
.expect("test bucket directory should be removed from a minority disk view");
}
let listing = ecstore
.list_bucket_for_scanner(&crate::storage_api_contracts::bucket::BucketOptions {
no_metadata: true,
..Default::default()
})
.await
.expect("responsive disks should still produce a scanner candidate listing");
assert!(listing.buckets.iter().all(|entry| entry.name != bucket));
assert!(!listing.topology_complete);
}
#[tokio::test]
#[serial]
async fn bucket_delete_mark_delete_removes_empty_bucket_and_keeps_deleted_marker() {
let (disk_paths, ecstore) = setup_bucket_delete_test_env().await;
let bucket = format!("bucket-mark-delete-{}", Uuid::new_v4().simple());
ecstore
.make_bucket(&bucket, &MakeBucketOptions::default())
.await
.expect("bucket should be created");
for disk_path in &disk_paths {
tokio::fs::create_dir_all(disk_path.join(&bucket).join("empty-directory/nested/leaf"))
.await
.expect("empty directory remnant should be created");
}
assert!(metadata_sys::get_in(&ecstore.ctx, &bucket).await.is_ok());
let generation_before_delete = ecstore.scanner_namespace_mutation_generation();
ecstore
.delete_bucket(
&bucket,
&DeleteBucketOptions {
srdelete_op: SRBucketDeleteOp::MarkDelete,
..Default::default()
},
)
.await
.expect("MarkDelete should remove an empty bucket");
assert_eq!(
ecstore.scanner_namespace_mutation_generation(),
generation_before_delete.saturating_add(2),
"successful bucket deletion should fence scanner namespace activity before and after deletion"
);
assert!(
!any_disk_path_exists(&disk_paths, &bucket).await,
"MarkDelete should remove the bucket volume"
);
assert!(
any_disk_path_exists(&disk_paths, bucket_deleted_marker_volume(&bucket)).await,
"MarkDelete should persist the deleted-bucket marker"
);
assert!(
metadata_sys::get_in(&ecstore.ctx, &bucket).await.is_err(),
"deleted bucket metadata must be removed from the local cache"
);
let buckets = ecstore
.list_bucket(&BucketOptions::default())
.await
.expect("bucket listing should succeed after MarkDelete");
assert!(!buckets.iter().any(|info| info.name == bucket));
ecstore
.delete_all(RUSTFS_META_BUCKET, &bucket_deleted_marker_prefix(&bucket))
.await
.expect("deleted-bucket marker should be removed to simulate a partial failure");
assert!(!any_disk_path_exists(&disk_paths, bucket_deleted_marker_volume(&bucket)).await);
ecstore
.delete_bucket(
&bucket,
&DeleteBucketOptions {
srdelete_op: SRBucketDeleteOp::MarkDelete,
..Default::default()
},
)
.await
.expect("retried MarkDelete should recreate a missing tombstone");
assert!(any_disk_path_exists(&disk_paths, bucket_deleted_marker_volume(&bucket)).await);
}
#[tokio::test]
#[serial]
async fn bucket_delete_mark_delete_rejects_non_empty_bucket_without_force() {
let (disk_paths, ecstore) = setup_bucket_delete_test_env().await;
let bucket = format!("bucket-mark-delete-non-empty-{}", Uuid::new_v4().simple());
create_bucket_with_object(&ecstore, &bucket, "object.txt").await;
let err = ecstore
.delete_bucket(
&bucket,
&DeleteBucketOptions {
srdelete_op: SRBucketDeleteOp::MarkDelete,
..Default::default()
},
)
.await
.expect_err("MarkDelete should reject a non-empty bucket without force");
assert!(matches!(err, StorageError::BucketNotEmpty(name) if name == bucket));
assert!(any_disk_has_object_metadata(&disk_paths, &bucket).await);
assert!(!any_disk_path_exists(&disk_paths, bucket_deleted_marker_volume(&bucket)).await);
}
#[tokio::test]
#[serial]
async fn bucket_delete_mark_delete_rejects_hidden_object_paths_without_force() {
let (disk_paths, ecstore) = setup_bucket_delete_test_env().await;
let bucket = format!("bucket-mark-delete-hidden-{}", Uuid::new_v4().simple());
create_bucket_with_object(&ecstore, &bucket, ".well-known/acme-challenge").await;
let mut reader = PutObjReader::from_vec(b"delete bucket semantics".to_vec());
ecstore
.put_object(&bucket, ".rustfs.sys/object", &mut reader, &ObjectOptions::default())
.await
.expect("second hidden object should be written");
let err = ecstore
.delete_bucket(
&bucket,
&DeleteBucketOptions {
srdelete_op: SRBucketDeleteOp::MarkDelete,
..Default::default()
},
)
.await
.expect_err("MarkDelete should reject a hidden object path without force");
assert!(matches!(err, StorageError::BucketNotEmpty(name) if name == bucket));
assert!(any_disk_has_object_metadata(&disk_paths, &bucket).await);
}
#[tokio::test]
#[serial]
async fn bucket_delete_purge_removes_bucket_data_and_internal_metadata() {
let (disk_paths, ecstore) = setup_bucket_delete_test_env().await;
let bucket = format!("bucket-purge-{}", Uuid::new_v4().simple());
let object = "object.txt";
let metadata_prefix = format!("{RUSTFS_META_BUCKET}/{BUCKET_META_PREFIX}/{bucket}");
create_bucket_with_object(&ecstore, &bucket, object).await;
write_bucket_metadata_marker(&disk_paths, &metadata_prefix).await;
ecstore
.mark_bucket_deleted(&bucket)
.await
.expect("deleted-bucket marker should be created");
assert!(any_disk_path_exists(&disk_paths, &metadata_prefix).await);
assert!(any_disk_path_exists(&disk_paths, bucket_deleted_marker_volume(&bucket)).await);
let generation_before_delete = ecstore.scanner_namespace_mutation_generation();
ecstore
.delete_bucket(
&bucket,
&DeleteBucketOptions {
force: true,
srdelete_op: SRBucketDeleteOp::Purge,
..Default::default()
},
)
.await
.expect("Purge should force-delete bucket data");
assert_eq!(
ecstore.scanner_namespace_mutation_generation(),
generation_before_delete.saturating_add(2),
"successful bucket purge should fence scanner namespace activity before and after deletion"
);
assert!(!any_disk_path_exists(&disk_paths, &bucket).await, "Purge should remove the bucket volume");
assert!(
!any_disk_path_exists(&disk_paths, &metadata_prefix).await,
"Purge should remove bucket metadata prefix"
);
assert!(
!any_disk_path_exists(&disk_paths, bucket_deleted_marker_volume(&bucket)).await,
"Purge should remove the deleted-bucket marker"
);
assert!(
metadata_sys::get_in(&ecstore.ctx, &bucket).await.is_err(),
"purged bucket metadata must be removed from the local cache"
);
write_bucket_metadata_marker(&disk_paths, &metadata_prefix).await;
ecstore
.mark_bucket_deleted(&bucket)
.await
.expect("stale deleted-bucket marker should be recreated");
ecstore
.delete_bucket(
&bucket,
&DeleteBucketOptions {
force: true,
srdelete_op: SRBucketDeleteOp::Purge,
..Default::default()
},
)
.await
.expect("retried Purge should remove stale metadata without a bucket volume");
assert!(!any_disk_path_exists(&disk_paths, &metadata_prefix).await);
assert!(!any_disk_path_exists(&disk_paths, bucket_deleted_marker_volume(&bucket)).await);
}
#[tokio::test(flavor = "multi_thread")]
#[serial]
async fn force_create_enables_object_lock_on_an_existing_bucket() {
let (_disk_paths, ecstore) = setup_bucket_delete_test_env().await;
let bucket = format!("bucket-force-lock-{}", Uuid::new_v4().simple());
ecstore
.make_bucket(&bucket, &MakeBucketOptions::default())
.await
.expect("plain bucket should be created");
assert!(
!metadata_sys::get_in(&ecstore.ctx, &bucket)
.await
.expect("metadata should load")
.lock_enabled,
"test setup: the bucket must start unlocked"
);
ecstore
.make_bucket(
&bucket,
&MakeBucketOptions {
force_create: true,
lock_enabled: true,
..Default::default()
},
)
.await
.expect("force create with lock_enabled should succeed on an existing bucket");
let meta = metadata_sys::get_in(&ecstore.ctx, &bucket)
.await
.expect("metadata should load after the lock enable");
assert!(meta.lock_enabled, "Object Lock must be enabled on the existing bucket");
assert!(
!meta.object_lock_config_xml.is_empty(),
"the Object Lock configuration must be persisted, not just the flag"
);
assert!(
!meta.versioning_config_xml.is_empty(),
"Object Lock requires versioning, so that must be persisted too"
);
}
#[tokio::test(flavor = "multi_thread")]
#[serial]
async fn issue_8003_force_create_adopts_existing_incarnation_sidecar() {
let (disk_paths, ecstore) = setup_bucket_delete_test_env().await;
let bucket = format!("bucket-8003-sidecar-{}", Uuid::new_v4().simple());
ecstore
.make_bucket(&bucket, &MakeBucketOptions::default())
.await
.expect("bucket should be created");
let stored = metadata_sys::get_in(&ecstore.ctx, &bucket)
.await
.expect("metadata should load")
.bucket_incarnation_id;
assert!(!stored.is_nil());
let metadata_path = format!("{BUCKET_META_PREFIX}/{bucket}/{}", crate::bucket::metadata::BUCKET_METADATA_FILE);
crate::config::com::delete_config(ecstore.clone(), &metadata_path)
.await
.expect("simulate the interrupted migration by removing only .metadata.bin");
assert!(!any_disk_path_exists(&disk_paths, format!("{RUSTFS_META_BUCKET}/{metadata_path}")).await);
metadata_sys::remove_bucket_metadata_in(&ecstore.ctx, &bucket)
.await
.expect("drop the cached copy so the next read hits disk");
ecstore
.make_bucket(
&bucket,
&MakeBucketOptions {
force_create: true,
..Default::default()
},
)
.await
.expect("force create must repair the sidecar-only bucket");
let (repaired, persisted) = metadata_sys::get_config_from_disk_with_presence_in(&ecstore.ctx, &bucket)
.await
.expect("repaired metadata should load");
assert!(persisted, "force create must persist .metadata.bin again");
assert!(repaired.bucket_incarnation_sidecar);
assert_eq!(
repaired.bucket_incarnation_id, stored,
"the repair must keep the incarnation the sidecar already published"
);
}
#[tokio::test(flavor = "multi_thread")]
#[serial]
async fn make_bucket_inherits_default_durability() {
temp_env::async_with_vars([(crate::bucket::durability::ENV_NEW_BUCKET_DURABILITY_MODE, None::<&str>)], async {
let (_disk_paths, ecstore) = setup_bucket_delete_test_env().await;
let bucket = format!("bucket-default-durability-{}", Uuid::new_v4().simple());
ecstore
.make_bucket(&bucket, &MakeBucketOptions::default())
.await
.expect("new bucket should be created");
let metadata = metadata_sys::get_in(&ecstore.ctx, &bucket)
.await
.expect("metadata should load for the new bucket");
assert!(metadata.durability_config().is_none());
})
.await;
}
#[tokio::test(flavor = "multi_thread")]
#[serial]
async fn force_create_existing_bucket_keeps_durability_override() {
let (_disk_paths, ecstore) = setup_bucket_delete_test_env().await;
let bucket = format!("bucket-force-durability-{}", Uuid::new_v4().simple());
temp_env::async_with_vars([(crate::bucket::durability::ENV_NEW_BUCKET_DURABILITY_MODE, Some("inherit"))], async {
ecstore
.make_bucket(&bucket, &MakeBucketOptions::default())
.await
.expect("plain bucket should be created without a durability override");
})
.await;
assert!(
metadata_sys::get_in(&ecstore.ctx, &bucket)
.await
.expect("metadata should load after initial create")
.durability_config()
.is_none(),
"test setup: the existing bucket must start without an override"
);
temp_env::async_with_vars([(crate::bucket::durability::ENV_NEW_BUCKET_DURABILITY_MODE, None::<&str>)], async {
ecstore
.make_bucket(
&bucket,
&MakeBucketOptions {
force_create: true,
lock_enabled: true,
..Default::default()
},
)
.await
.expect("force create should update existing bucket metadata");
})
.await;
let metadata = metadata_sys::get_in(&ecstore.ctx, &bucket)
.await
.expect("metadata should load after force create");
assert!(metadata.lock_enabled, "force create sanity check: Object Lock should be enabled");
assert!(
metadata.durability_config().is_none(),
"force create must not apply the new-bucket default to existing bucket metadata"
);
}
#[tokio::test(flavor = "multi_thread")]
#[serial]
async fn bucket_delete_succeeds_after_the_last_object_version_is_deleted() {
let (disk_paths, ecstore) = setup_bucket_delete_test_env().await;
let bucket = format!("bucket-delete-after-drain-{}", Uuid::new_v4().simple());
let object = "object.txt";
create_bucket_with_object(&ecstore, &bucket, object).await;
ecstore
.delete_object(&bucket, object, ObjectOptions::default())
.await
.expect("client delete of the only object should succeed");
assert!(
!any_disk_has_object_metadata(&disk_paths, &bucket).await,
"deleting the last version must not leave xl.meta on disk: DeleteBucket scans the raw \
bucket directory, so residue here is reported to clients as BucketNotEmpty"
);
ecstore
.delete_bucket(&bucket, &DeleteBucketOptions::default())
.await
.expect("DeleteBucket must succeed once the client has drained the bucket");
}
#[tokio::test(flavor = "multi_thread")]
#[serial]
async fn bucket_delete_defers_zero_evidence_diagnostic_timeout_to_physical_empty_check() {
let (disk_paths, ecstore) = setup_bucket_delete_test_env().await;
let bucket = format!("bucket-del-diag-{}", Uuid::new_v4().simple());
ecstore
.make_bucket(&bucket, &MakeBucketOptions::default())
.await
.expect("empty bucket should be created");
let first_io_started = Arc::new(AtomicBool::new(false));
let diagnostic_budget = BucketDeleteDiagnosticBudget::new().with_first_io_delay(
BUCKET_DELETE_DIAGNOSTIC_MAX_ELAPSED + Duration::from_millis(100),
first_io_started.clone(),
);
ecstore
.handle_delete_bucket_with_diagnostic_budget(&bucket, &DeleteBucketOptions::default(), diagnostic_budget)
.await
.expect("a zero-evidence diagnostic timeout must defer to the physical empty check");
assert!(
first_io_started.load(Ordering::SeqCst),
"the regression must delay the first diagnostic read_dir"
);
assert!(
!any_disk_path_exists(&disk_paths, &bucket).await,
"the physical empty check should allow deletion of the actually empty bucket"
);
}
#[tokio::test(flavor = "multi_thread")]
#[serial]
async fn bucket_delete_preserves_unobserved_residue_after_diagnostic_timeout() {
let (disk_paths, ecstore) = setup_bucket_delete_test_env().await;
let bucket = format!("bucket-del-residue-{}", Uuid::new_v4().simple());
let object = "object.txt";
ecstore
.make_bucket(&bucket, &MakeBucketOptions::default())
.await
.expect("bucket should be created");
write_metadata_less_part_on_all_disks(&disk_paths, &bucket, object).await;
let first_io_started = Arc::new(AtomicBool::new(false));
let diagnostic_budget = BucketDeleteDiagnosticBudget::new().with_first_io_delay(
BUCKET_DELETE_DIAGNOSTIC_MAX_ELAPSED + Duration::from_millis(100),
first_io_started.clone(),
);
let err = ecstore
.handle_delete_bucket_with_diagnostic_budget(&bucket, &DeleteBucketOptions::default(), diagnostic_budget)
.await
.expect_err("physical empty-bucket enforcement must reject unobserved residue");
assert!(
first_io_started.load(Ordering::SeqCst),
"the regression must time out before observing the residue"
);
match &err {
StorageError::BucketNotEmpty(err_bucket) | StorageError::BucketNotEmptyWithDetails { bucket: err_bucket, .. } => {
assert_eq!(err_bucket, &bucket)
}
other => panic!("expected an S3-compatible BucketNotEmpty error, got {other:?}"),
}
assert!(
any_disk_path_exists(&disk_paths, Path::new(&bucket).join(object)).await,
"physical empty-bucket enforcement must preserve unobserved residue"
);
ecstore
.delete_bucket(
&bucket,
&DeleteBucketOptions {
force: true,
..Default::default()
},
)
.await
.expect("explicit force should clean up the test-owned residue");
}
#[tokio::test(flavor = "multi_thread")]
#[serial]
async fn bucket_delete_succeeds_after_listing_and_deleting_an_unversioned_overwrite() {
let (disk_paths, ecstore) = setup_bucket_delete_test_env().await;
let bucket = format!("bucket-delete-after-overwrite-{}", Uuid::new_v4().simple());
let object = "object.txt";
ecstore
.make_bucket(&bucket, &MakeBucketOptions::default())
.await
.expect("unversioned bucket should be created");
let mut first_reader = PutObjReader::from_vec(b"version A".to_vec());
let first = ecstore
.put_object(&bucket, object, &mut first_reader, &ObjectOptions::default())
.await
.expect("version A should be written");
let mut second_reader = PutObjReader::from_vec(b"version B".to_vec());
let second = ecstore
.put_object(&bucket, object, &mut second_reader, &ObjectOptions::default())
.await
.expect("version B should overwrite version A");
assert_ne!(first.data_dir, second.data_dir, "the overwrite must publish a new body generation");
let listing = ecstore
.clone()
.list_object_versions(&bucket, "", None, None, None, 1000)
.await
.expect("the overwritten object should remain listable for teardown");
assert_eq!(listing.objects.len(), 1, "an unversioned overwrite should expose one current version");
let current = &listing.objects[0];
assert_eq!(current.name, object);
assert!(current.is_latest, "the listed null version must be current");
assert_eq!(current.version_id, None, "an unversioned object must be exposed as the null version");
assert_eq!(
current.data_dir, second.data_dir,
"listing must expose version B, not the overwritten body"
);
for version in listing.objects {
let version_id = version.version_id.map(|version_id| version_id.to_string());
ecstore
.delete_object(
&bucket,
&version.name,
ObjectOptions {
version_id,
..Default::default()
},
)
.await
.expect("each version returned by teardown listing should be deletable");
}
assert!(
!any_disk_has_object_metadata(&disk_paths, &bucket).await,
"deleting the listed null version must remove every xl.meta"
);
ecstore
.delete_bucket(&bucket, &DeleteBucketOptions::default())
.await
.expect("DeleteBucket should succeed after the listed overwrite is deleted");
}
#[tokio::test]
#[serial]
async fn bucket_delete_default_s3_delete_still_rejects_non_empty_bucket() {
let (disk_paths, ecstore) = setup_bucket_delete_test_env().await;
let bucket = format!("bucket-s3-delete-{}", Uuid::new_v4().simple());
let object = "object.txt";
create_bucket_with_object(&ecstore, &bucket, object).await;
let generation_before_delete = ecstore.scanner_namespace_mutation_generation();
let err = ecstore
.delete_bucket(&bucket, &DeleteBucketOptions::default())
.await
.expect_err("default S3 DeleteBucket should reject non-empty buckets");
assert!(matches!(err, StorageError::BucketNotEmpty(name) if name == bucket));
assert_eq!(
ecstore.scanner_namespace_mutation_generation(),
generation_before_delete,
"failed bucket deletion must not advance scanner namespace activity"
);
assert!(
any_disk_has_object_metadata(&disk_paths, &bucket).await,
"failed default S3 DeleteBucket must keep object data"
);
assert!(
metadata_sys::get_in(&ecstore.ctx, &bucket).await.is_ok(),
"failed default S3 DeleteBucket must keep metadata cache"
);
}
#[tokio::test(flavor = "multi_thread")]
#[serial]
async fn bucket_delete_reports_metadata_less_residue_without_removing_it() {
let (disk_paths, ecstore) = setup_bucket_delete_test_env().await;
let bucket = format!("bucket-delete-orphan-datadir-{}", Uuid::new_v4().simple());
let object = "object.txt";
ecstore
.make_bucket(&bucket, &MakeBucketOptions::default())
.await
.expect("bucket should be created");
write_metadata_less_part_on_all_disks(&disk_paths, &bucket, object).await;
assert!(
!any_disk_has_object_metadata(&disk_paths, &bucket).await,
"test setup must reproduce a bucket with data dirs but no xl.meta"
);
let err = ecstore
.delete_bucket(&bucket, &DeleteBucketOptions::default())
.await
.expect_err("metadata-less data dirs must block ordinary DeleteBucket");
match err {
StorageError::BucketNotEmptyWithDetails {
bucket: err_bucket,
details,
} => {
assert_eq!(err_bucket, bucket);
assert!(
details.contains("metadata-less on-disk residue"),
"diagnostic should identify metadata-less residue, got: {details}"
);
assert!(details.contains("files="), "diagnostic should include a bounded count");
assert!(details.contains("uuid_data_dirs="), "diagnostic should include data-dir count");
}
other => panic!("expected detailed BucketNotEmpty, got {other:?}"),
}
assert!(
any_disk_path_exists(&disk_paths, Path::new(&bucket).join(object)).await,
"ordinary DeleteBucket must preserve metadata-less data until an explicit operator action"
);
}
#[tokio::test]
#[serial]
async fn bucket_delete_fences_put_started_after_empty_scan() {
let (_, ecstore) = setup_bucket_delete_test_env().await;
let bucket = format!("bucket-delete-empty-scan-race-{}", Uuid::new_v4().simple());
let object = "committed-after-empty-scan";
let payload = b"object started after DeleteBucket empty scan".to_vec();
ecstore
.make_bucket(&bucket, &MakeBucketOptions::default())
.await
.expect("bucket should be created");
let barrier = install_delete_bucket_empty_scan_barrier();
let delete_store = ecstore.clone();
let delete_bucket = bucket.clone();
let delete = tokio::spawn(async move {
delete_store
.delete_bucket(&delete_bucket, &DeleteBucketOptions::default())
.await
});
barrier.wait_until_paused().await;
let put_store = ecstore.clone();
let put_bucket = bucket.clone();
let mut put = tokio::spawn(async move {
let mut put_reader = PutObjReader::from_vec(payload);
put_store
.put_object(&put_bucket, object, &mut put_reader, &ObjectOptions::default())
.await
});
assert!(
tokio::time::timeout(Duration::from_millis(100), &mut put).await.is_err(),
"PUT must wait behind the DeleteBucket lifecycle fence"
);
barrier.release();
delete
.await
.expect("DeleteBucket task should join")
.expect("DeleteBucket should commit while holding the lifecycle fence");
let err = put
.await
.expect("PUT task should join")
.expect_err("PUT must not recreate an object in the deleted bucket");
assert!(matches!(err, StorageError::BucketNotFound(name) if name == bucket));
}
#[tokio::test]
#[serial]
async fn bucket_recreation_does_not_publish_unverified_usage() {
let (_, ecstore) = setup_bucket_delete_test_env().await;
let bucket = format!("bucket-usage-generation-{}", Uuid::new_v4().simple());
ecstore
.make_bucket(&bucket, &MakeBucketOptions::default())
.await
.expect("bucket should be created");
let mut snapshot = DataUsageInfo {
last_update: Some(SystemTime::now()),
buckets_count: 1,
..Default::default()
};
snapshot.buckets_usage.insert(
bucket.clone(),
BucketUsageInfo {
size: 42,
objects_count: 1,
versions_count: 1,
..Default::default()
},
);
snapshot.usage_snapshot_complete = true;
snapshot.bucket_sizes.insert(bucket.clone(), 42);
snapshot.calculate_totals();
crate::data_usage::store_data_usage_in_backend(snapshot, ecstore.clone())
.await
.expect("usage snapshot should be stored");
ecstore
.delete_bucket(&bucket, &DeleteBucketOptions::default())
.await
.expect("empty bucket should be deleted");
let deleted = crate::data_usage::load_data_usage_from_backend(ecstore.clone())
.await
.expect("usage snapshot should remain readable");
assert!(!deleted.buckets_usage.contains_key(&bucket));
ecstore
.make_bucket(&bucket, &MakeBucketOptions::default())
.await
.expect("same bucket name should be recreated after delete returns");
crate::data_usage::record_bucket_object_write_memory(&bucket, None, 84).await;
let mut recreated = crate::data_usage::load_data_usage_from_backend(ecstore.clone())
.await
.expect("recreated bucket usage base should load");
crate::data_usage::apply_bucket_usage_memory_overlay(&mut recreated).await;
assert!(
!recreated.buckets_usage.contains_key(&bucket),
"a request-path delta without an authoritative baseline must remain unavailable"
);
assert_eq!(crate::data_usage::get_bucket_usage_memory(&bucket).await, None);
}
#[tokio::test]
#[serial]
async fn bucket_create_removes_stale_usage_before_physical_creation() {
let (_, ecstore) = setup_bucket_delete_test_env().await;
let bucket = format!("bucket-create-stale-usage-{}", Uuid::new_v4().simple());
let mut snapshot = DataUsageInfo {
last_update: Some(SystemTime::now()),
buckets_count: 1,
..Default::default()
};
snapshot.buckets_usage.insert(
bucket.clone(),
BucketUsageInfo {
size: 42,
objects_count: 1,
versions_count: 1,
..Default::default()
},
);
snapshot.usage_snapshot_complete = true;
snapshot.bucket_sizes.insert(bucket.clone(), 42);
snapshot.calculate_totals();
crate::data_usage::store_data_usage_in_backend(snapshot, ecstore.clone())
.await
.expect("stale usage fixture should be stored");
ecstore
.make_bucket(&bucket, &MakeBucketOptions::default())
.await
.expect("CreateBucket should succeed");
let persisted = crate::data_usage::load_data_usage_from_backend(ecstore.clone())
.await
.expect("usage snapshot should remain readable");
assert!(
!persisted.buckets_usage.contains_key(&bucket),
"a newly created bucket must not inherit the predecessor generation's usage"
);
assert!(
ecstore.get_bucket_info(&bucket, &BucketOptions::default()).await.is_ok(),
"physical creation should happen after the usage fence succeeds"
);
}
#[tokio::test]
#[serial]
async fn failed_create_rollback_does_not_run_unfenced_usage_cleanup() {
let (_, ecstore) = setup_bucket_delete_test_env().await;
let bucket = format!("bucket-create-rollback-{}", Uuid::new_v4().simple());
ecstore
.peer_sys
.make_bucket(&bucket, &MakeBucketOptions::default())
.await
.expect("the partial-create fixture should expose a physical bucket");
let mut snapshot = DataUsageInfo {
last_update: Some(SystemTime::now()),
buckets_count: 1,
..Default::default()
};
snapshot.buckets_usage.insert(
bucket.clone(),
BucketUsageInfo {
size: 42,
objects_count: 1,
versions_count: 1,
..Default::default()
},
);
snapshot.usage_snapshot_complete = true;
snapshot.bucket_sizes.insert(bucket.clone(), 42);
snapshot.calculate_totals();
crate::data_usage::store_data_usage_in_backend(snapshot, ecstore.clone())
.await
.expect("the usage fixture should be stored");
crate::bucket::metadata::save_bucket_incarnation(ecstore.clone(), &bucket, Uuid::new_v4())
.await
.expect("partial create should have an incarnation sidecar");
ecstore.rollback_failed_bucket_creation(&bucket, None, None).await;
assert!(
ecstore
.peer_sys
.get_bucket_info(&bucket, &BucketOptions::default())
.await
.is_err(),
"failed-create rollback should remove the partial physical bucket"
);
assert!(
crate::bucket::metadata::load_bucket_incarnation(ecstore.clone(), &bucket)
.await
.expect("rollback sidecar lookup should succeed")
.is_none(),
"failed-create rollback must remove the orphan incarnation sidecar"
);
let persisted = crate::data_usage::load_data_usage_from_backend(ecstore.clone())
.await
.expect("the usage snapshot should remain readable");
assert!(
persisted.buckets_usage.contains_key(&bucket),
"failed-create rollback must not start an unfenced usage cleanup"
);
crate::data_usage::store_data_usage_in_backend(DataUsageInfo::default(), ecstore.clone())
.await
.expect("the rollback usage fixture should be cleared");
}
#[tokio::test]
#[serial]
async fn bucket_create_fails_closed_when_usage_snapshot_cannot_be_fenced() {
let (_, ecstore) = setup_bucket_delete_test_env().await;
let deleted_bucket = format!("bucket-delete-corrupt-usage-{}", Uuid::new_v4().simple());
ecstore
.make_bucket(&deleted_bucket, &MakeBucketOptions::default())
.await
.expect("bucket should be created before corrupting usage");
let usage_path = format!("{BUCKET_META_PREFIX}/{DATA_USAGE_OBJECT_NAME}");
crate::config::com::save_config(ecstore.clone(), &usage_path, b"{".to_vec())
.await
.expect("corrupt usage fixture should be stored");
ecstore
.delete_bucket(&deleted_bucket, &DeleteBucketOptions::default())
.await
.expect("usage snapshot corruption must not block DeleteBucket");
assert!(
ecstore
.get_bucket_info(&deleted_bucket, &crate::storage_api_contracts::bucket::BucketOptions::default())
.await
.is_err(),
"successful DeleteBucket must remove the physical bucket"
);
let new_bucket = format!("bucket-create-corrupt-usage-{}", Uuid::new_v4().simple());
let create_err = ecstore
.make_bucket(&new_bucket, &MakeBucketOptions::default())
.await
.expect_err("MakeBucket must fail when stale usage cannot be fenced");
assert!(!create_err.to_string().is_empty(), "the usage snapshot failure should be surfaced");
assert!(
ecstore
.get_bucket_info(&new_bucket, &crate::storage_api_contracts::bucket::BucketOptions::default())
.await
.is_err(),
"failed MakeBucket must not expose a new physical bucket"
);
let restored = serde_json::to_vec(&DataUsageInfo {
last_update: Some(SystemTime::now()),
..Default::default()
})
.expect("restored usage fixture should encode");
crate::config::com::save_config(ecstore, &usage_path, restored)
.await
.expect("usage fixture should be restored after the failure-path test");
}
fn bucket_logs_at_default_level(emit: impl FnOnce()) -> String {
use std::sync::{Arc, Mutex};
use tracing_subscriber::EnvFilter;
use tracing_subscriber::fmt::MakeWriter;
use tracing_subscriber::layer::SubscriberExt;
#[derive(Clone, Default)]
struct CapturedLogs {
buffer: Arc<Mutex<Vec<u8>>>,
}
struct CapturedLogWriter {
buffer: Arc<Mutex<Vec<u8>>>,
}
impl std::io::Write for CapturedLogWriter {
fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
self.buffer
.lock()
.expect("captured logs mutex should not be poisoned")
.extend_from_slice(buf);
Ok(buf.len())
}
fn flush(&mut self) -> std::io::Result<()> {
Ok(())
}
}
impl<'a> MakeWriter<'a> for CapturedLogs {
type Writer = CapturedLogWriter;
fn make_writer(&'a self) -> Self::Writer {
CapturedLogWriter {
buffer: Arc::clone(&self.buffer),
}
}
}
let logs = CapturedLogs::default();
let subscriber = tracing_subscriber::registry()
.with(EnvFilter::new(rustfs_config::DEFAULT_LOG_LEVEL))
.with(
tracing_subscriber::fmt::layer()
.with_writer(logs.clone())
.with_ansi(false)
.without_time(),
);
let _guard = tracing::subscriber::set_default(subscriber);
let _callsite_pin = crate::test_tracing::pin_callsite_interest_for_test();
emit();
let buffer = logs
.buffer
.lock()
.expect("captured logs mutex should not be poisoned")
.clone();
String::from_utf8(buffer).expect("captured logs should be valid UTF-8")
}
fn residue_sample(sample: &str) -> BucketMetadataLessResidue {
BucketMetadataLessResidue {
files: 1,
entries_scanned: 3,
sample: Some(sample.to_string()),
..Default::default()
}
}
#[test]
fn unreachable_residue_is_reported_at_the_default_log_level() {
for kind in [
BucketDeleteBlockerKind::UnknownXlMeta,
BucketDeleteBlockerKind::OrphanDirectory,
BucketDeleteBlockerKind::DiagnosticBudgetExceeded,
] {
let logs = bucket_logs_at_default_level(|| {
record_bucket_delete_blocker("drained-bucket", kind, &residue_sample("obj/8f2c/xl.meta"));
});
assert!(logs.contains("drained-bucket"), "the bucket must be named for {kind:?}: {logs}");
assert!(logs.contains(kind.as_str()), "the blocker kind must be named for {kind:?}: {logs}");
assert!(
logs.contains("obj/8f2c/xl.meta"),
"the residue sample is the only pointer to the leftover state for {kind:?}: {logs}"
);
}
}
#[test]
fn client_visible_blockers_stay_below_the_default_log_level() {
for kind in [
BucketDeleteBlockerKind::VisibleVersion,
BucketDeleteBlockerKind::TierFreeVersion,
] {
let logs = bucket_logs_at_default_level(|| {
record_bucket_delete_blocker("still-full-bucket", kind, &residue_sample("obj/xl.meta"));
});
assert!(logs.is_empty(), "an ordinary non-empty bucket must not log an error for {kind:?}: {logs}");
}
}
#[test]
fn blocker_kinds_split_by_whether_the_client_can_reach_the_residue() {
assert!(BucketDeleteBlockerKind::VisibleVersion.is_client_visible());
assert!(BucketDeleteBlockerKind::TierFreeVersion.is_client_visible());
assert!(!BucketDeleteBlockerKind::UnknownXlMeta.is_client_visible());
assert!(!BucketDeleteBlockerKind::OrphanDirectory.is_client_visible());
assert!(!BucketDeleteBlockerKind::DiagnosticBudgetExceeded.is_client_visible());
}
}