#[cfg(test)]
mod capacity_dedup_tests {
use crate::core::pools::{
fallback_free_capacity_dedup, fallback_total_capacity_dedup, get_total_usable_capacity, get_total_usable_capacity_free,
};
#[test]
fn test_single_disk_no_duplication() {
let disks = vec![rustfs_madmin::Disk {
endpoint: "node1".to_string(),
drive_path: "/mnt/disk1".to_string(),
pool_index: 0,
set_index: 0,
disk_index: 0,
total_space: 2_000_000_000_000, available_space: 500_000_000_000,
used_space: 1_500_000_000_000,
state: "ok".to_string(),
..Default::default()
}];
let info = rustfs_madmin::StorageInfo {
backend: rustfs_madmin::BackendInfo {
standard_sc_data: vec![1],
..Default::default()
},
disks: disks.clone(),
..Default::default()
};
let total = get_total_usable_capacity(&disks, &info);
let free = get_total_usable_capacity_free(&disks, &info);
assert_eq!(total, 2_000_000_000_000, "Total capacity should be 2TB");
assert_eq!(free, 500_000_000_000, "Free capacity should be 500GB");
}
#[test]
fn test_duplicate_disk_entries_deduped() {
let mut disks = Vec::new();
for _ in 0..232 {
disks.push(rustfs_madmin::Disk {
endpoint: "node1".to_string(),
drive_path: "/mnt/disk1".to_string(),
pool_index: 0,
set_index: 0,
disk_index: 0,
total_space: 2_000_000_000_000,
available_space: 500_000_000_000,
used_space: 1_500_000_000_000,
state: "ok".to_string(),
..Default::default()
});
}
let info = rustfs_madmin::StorageInfo {
backend: rustfs_madmin::BackendInfo {
standard_sc_data: vec![1],
..Default::default()
},
disks: disks.clone(),
..Default::default()
};
let total = get_total_usable_capacity(&disks, &info);
let free = get_total_usable_capacity_free(&disks, &info);
assert_eq!(total, 2_000_000_000_000, "Duplicate disks should be counted only once");
assert_eq!(free, 500_000_000_000, "Free capacity should not be multiplied");
}
#[test]
fn test_four_node_ec_2_2_reports_stable_usable_capacity() {
const TIB: u64 = 1 << 40;
const DISK_TOTAL: u64 = 2 * TIB;
const DISK_FREE: u64 = TIB / 5;
let disks = vec![
rustfs_madmin::Disk {
endpoint: "node1".to_string(),
drive_path: "/media/rustfs-01".to_string(),
pool_index: 0,
set_index: 0,
disk_index: 0,
total_space: DISK_TOTAL,
available_space: DISK_FREE,
used_space: DISK_TOTAL - DISK_FREE,
state: "ok".to_string(),
..Default::default()
},
rustfs_madmin::Disk {
endpoint: "node2".to_string(),
drive_path: "/media/rustfs-01".to_string(),
pool_index: 0,
set_index: 0,
disk_index: 1,
total_space: DISK_TOTAL,
available_space: DISK_FREE,
used_space: DISK_TOTAL - DISK_FREE,
state: "ok".to_string(),
..Default::default()
},
rustfs_madmin::Disk {
endpoint: "node3".to_string(),
drive_path: "/media/rustfs-01".to_string(),
pool_index: 0,
set_index: 0,
disk_index: 2,
total_space: DISK_TOTAL,
available_space: DISK_FREE,
used_space: DISK_TOTAL - DISK_FREE,
state: "ok".to_string(),
..Default::default()
},
rustfs_madmin::Disk {
endpoint: "node4".to_string(),
drive_path: "/media/rustfs-01".to_string(),
pool_index: 0,
set_index: 0,
disk_index: 3,
total_space: DISK_TOTAL,
available_space: DISK_FREE,
used_space: DISK_TOTAL - DISK_FREE,
state: "ok".to_string(),
..Default::default()
},
];
let info = rustfs_madmin::StorageInfo {
backend: rustfs_madmin::BackendInfo {
standard_sc_data: vec![2], standard_sc_parity: Some(2), ..Default::default()
},
disks: disks.clone(),
..Default::default()
};
let total = get_total_usable_capacity(&disks, &info);
let free = get_total_usable_capacity_free(&disks, &info);
let used = total.saturating_sub(free);
let expected_total = usize::try_from(4 * TIB).expect("4 TiB must fit the supported platform's usize");
let expected_free = usize::try_from(2 * DISK_FREE).expect("usable free capacity must fit usize");
let single_node_used = usize::try_from(DISK_TOTAL - DISK_FREE).expect("single-node used capacity must fit usize");
assert_eq!(total, expected_total, "usable total must count the two data disks");
assert_eq!(free, expected_free, "usable free must count the two data disks");
assert_eq!(used, 2 * single_node_used, "usable used must be approximately 3.6 TiB");
assert_ne!(used, single_node_used, "used must not collapse to one node's approximately 1.8 TiB");
assert_ne!(used, 4 * single_node_used, "used must not report the approximately 7.2 TiB raw aggregate");
}
#[test]
fn test_fallback_dedup() {
let mut disks = Vec::new();
for _ in 0..100 {
disks.push(rustfs_madmin::Disk {
endpoint: "node1".to_string(),
drive_path: "/mnt/disk1".to_string(),
total_space: 2_000_000_000_000,
available_space: 500_000_000_000,
state: "ok".to_string(),
..Default::default()
});
}
let total = fallback_total_capacity_dedup(&disks);
let free = fallback_free_capacity_dedup(&disks);
assert_eq!(total, 2_000_000_000_000);
assert_eq!(free, 500_000_000_000);
}
}
#[cfg(all(test, feature = "test-util"))]
mod decommission_lock_order_tests {
use crate::bucket::lifecycle::lifecycle::TRANSITION_PENDING;
use crate::core::pools::{
DECOMMISSION_CAPACITY_TARGET_LOCK_PREFIX, DecommissionCapacityLockOrderBarrier, DecommissionCapacityOwner,
DecommissionErasureLayout, DecommissionPoolCapacityInfo, DecommissionTestFaultGuard, POOL_META_NAME,
decommission_capacity_mutation_id, set_decommission_capacity_info_overrides_for_test,
};
use crate::data_movement;
use crate::disk::RUSTFS_META_BUCKET;
use crate::object_api::{ObjectOptions, PutObjReader};
use crate::services::rebalance::{
test_three_pool_stores_with_isolated_node_contexts,
test_three_pool_stores_with_three_disk_sets_with_isolated_node_contexts,
};
use crate::services::tier::test_util::register_mock_tier;
use crate::set_disk::{
DeleteObjectCommitBarrier, MultipartCommitBarrier, MultipartCommitPause, NewMultipartUploadCommitObservation,
PutObjectCommitBarrier, PutObjectCommitPause,
};
use crate::storage_api_contracts::bucket::{BucketOperations, MakeBucketOptions};
use crate::storage_api_contracts::multipart::MultipartOperations as _;
use crate::storage_api_contracts::namespace::NamespaceLocking as _;
use crate::storage_api_contracts::object::{ObjectIO, ObjectOperations as _};
use http::HeaderMap;
use rustfs_filemeta::MetaCacheEntry;
use std::collections::HashMap;
use std::future::Future;
use std::sync::{
Arc, Condvar, Mutex as StdMutex,
atomic::{AtomicUsize, Ordering},
};
use std::time::Duration;
use tokio::io::AsyncReadExt;
use tokio_util::sync::CancellationToken;
fn run_large_stack_async_test<C, F>(name: &str, case: C)
where
C: FnOnce() -> F + Send + 'static,
F: Future<Output = ()> + 'static,
{
const STACK_SIZE: usize = if cfg!(debug_assertions) {
8 * rustfs_config::DEFAULT_THREAD_STACK_SIZE
} else if cfg!(target_os = "macos") {
2 * rustfs_config::DEFAULT_THREAD_STACK_SIZE
} else {
rustfs_config::DEFAULT_THREAD_STACK_SIZE
};
std::thread::Builder::new()
.name(name.to_string())
.stack_size(STACK_SIZE)
.spawn(move || {
let runtime = tokio::runtime::Builder::new_multi_thread()
.enable_all()
.worker_threads(2)
.thread_stack_size(STACK_SIZE)
.build()
.expect("large-stack ecstore test runtime should build");
runtime.block_on(case());
})
.expect("large-stack ecstore test thread should spawn")
.join()
.expect("large-stack ecstore test thread should complete");
}
fn run_large_stack_current_thread_async_test<C, F>(name: &str, case: C)
where
C: FnOnce() -> F + Send + 'static,
F: Future<Output = ()> + 'static,
{
const STACK_SIZE: usize = if cfg!(debug_assertions) {
8 * rustfs_config::DEFAULT_THREAD_STACK_SIZE
} else if cfg!(target_os = "macos") {
2 * rustfs_config::DEFAULT_THREAD_STACK_SIZE
} else {
rustfs_config::DEFAULT_THREAD_STACK_SIZE
};
std::thread::Builder::new()
.name(name.to_string())
.stack_size(STACK_SIZE)
.spawn(move || {
let runtime = tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.expect("large-stack current-thread ecstore test runtime should build");
runtime.block_on(case());
})
.expect("large-stack current-thread ecstore test should spawn")
.join()
.expect("large-stack current-thread ecstore test should complete");
}
#[derive(Clone, Copy)]
enum ExternalObjectMutation {
Put,
CompleteMultipart,
Copy,
PutTags,
DeleteTags,
PutMetadata,
Delete,
Restore,
}
impl ExternalObjectMutation {
fn label(self) -> &'static str {
match self {
Self::Put => "put",
Self::CompleteMultipart => "complete-multipart",
Self::Copy => "copy",
Self::PutTags => "put-tags",
Self::DeleteTags => "delete-tags",
Self::PutMetadata => "put-metadata",
Self::Delete => "delete",
Self::Restore => "restore",
}
}
}
fn decommission_capacity_owner(meta: &crate::core::pools::PoolMeta) -> DecommissionCapacityOwner {
let reservation = meta.pools[0]
.decommission
.as_ref()
.and_then(|info| info.capacity_reservation.as_ref())
.expect("test decommission capacity reservation should exist");
DecommissionCapacityOwner {
source_pool_index: 0,
operation_id: reservation.operation_id,
generation: reservation.generation,
owner_nonce: reservation.owner_nonce,
mutation_id: None,
}
}
fn test_bucket(label: &str) -> String {
format!("dc-{label}-{}", &uuid::Uuid::new_v4().simple().to_string()[..12])
}
async fn new_multipart_upload(
store: &Arc<crate::store::ECStore>,
pool_index: usize,
bucket: &str,
object: &str,
mut opts: ObjectOptions,
) -> crate::error::Result<crate::storage_api_contracts::multipart::MultipartUploadResult> {
let lifecycle_guard = store.acquire_bucket_lifecycle_read_lock(bucket).await?;
opts.add_bucket_lifecycle_lock_guard(&lifecycle_guard);
store.pools[pool_index].new_multipart_upload(bucket, object, &opts).await
}
async fn multipart_options(store: &Arc<crate::store::ECStore>, bucket: &str, mut opts: ObjectOptions) -> ObjectOptions {
let lifecycle_guard = store
.acquire_bucket_lifecycle_read_lock(bucket)
.await
.expect("multipart test should acquire the bucket lifecycle fence");
opts.add_bucket_lifecycle_lock_guard(&lifecycle_guard);
opts
}
#[derive(Debug)]
struct CapacityLeaseLossControl {
calls: AtomicUsize,
fail_refresh: std::sync::atomic::AtomicBool,
}
impl CapacityLeaseLossControl {
fn arm(&self) {
self.fail_refresh.store(true, Ordering::Release);
}
fn load(&self, ordering: Ordering) -> usize {
self.calls.load(ordering)
}
}
#[derive(Debug)]
struct CapacityLeaseLossClient {
control: Arc<CapacityLeaseLossControl>,
active: tokio::sync::Mutex<HashMap<rustfs_lock::LockId, rustfs_lock::ObjectKey>>,
}
fn is_capacity_lease_resource(resource: &rustfs_lock::ObjectKey) -> bool {
resource.bucket.as_ref() == RUSTFS_META_BUCKET
&& (resource.object.as_ref() == POOL_META_NAME
|| resource
.object
.as_ref()
.strip_prefix(DECOMMISSION_CAPACITY_TARGET_LOCK_PREFIX)
.is_some_and(|suffix| suffix.starts_with('/')))
}
#[async_trait::async_trait]
impl rustfs_lock::LockClient for CapacityLeaseLossClient {
async fn acquire_lock(&self, request: &rustfs_lock::LockRequest) -> rustfs_lock::Result<rustfs_lock::LockResponse> {
self.active
.lock()
.await
.insert(request.lock_id.clone(), request.resource.clone());
let now = std::time::SystemTime::now();
Ok(rustfs_lock::LockResponse::success(
rustfs_lock::LockInfo {
id: request.lock_id.clone(),
resource: request.resource.clone(),
lock_type: request.lock_type,
status: rustfs_lock::LockStatus::Acquired,
owner: request.owner.clone(),
acquired_at: now,
expires_at: now + request.ttl,
last_refreshed: now,
metadata: request.metadata.clone(),
priority: request.priority,
wait_start_time: None,
},
Duration::ZERO,
))
}
async fn release(&self, lock_id: &rustfs_lock::LockId) -> rustfs_lock::Result<bool> {
Ok(self.active.lock().await.remove(lock_id).is_some())
}
async fn refresh(&self, lock_id: &rustfs_lock::LockId) -> rustfs_lock::Result<bool> {
let resource = self.active.lock().await.get(lock_id).cloned();
if resource.as_ref().is_some_and(is_capacity_lease_resource) {
self.control.calls.fetch_add(1, Ordering::Release);
return Ok(!self.control.fail_refresh.load(Ordering::Acquire));
}
Ok(resource.is_some())
}
async fn force_release(&self, lock_id: &rustfs_lock::LockId) -> rustfs_lock::Result<bool> {
self.release(lock_id).await
}
async fn check_status(&self, _lock_id: &rustfs_lock::LockId) -> rustfs_lock::Result<Option<rustfs_lock::LockInfo>> {
Ok(None)
}
async fn get_stats(&self) -> rustfs_lock::Result<rustfs_lock::LockStats> {
Ok(rustfs_lock::LockStats::default())
}
async fn close(&self) -> rustfs_lock::Result<()> {
Ok(())
}
async fn is_online(&self) -> bool {
true
}
async fn is_local(&self) -> bool {
false
}
}
async fn store_with_capacity_lease_loss(
other_store: &Arc<crate::store::ECStore>,
) -> (Arc<crate::store::ECStore>, Arc<CapacityLeaseLossControl>) {
let mut pool = (*other_store.pools[0]).clone();
let mut set = (*pool.disk_set[0]).clone();
let refresh_calls = Arc::new(CapacityLeaseLossControl {
calls: AtomicUsize::new(0),
fail_refresh: std::sync::atomic::AtomicBool::new(false),
});
set.lockers = (0..set.lockers.len().max(1))
.map(|_| {
Arc::new(CapacityLeaseLossClient {
control: Arc::clone(&refresh_calls),
active: tokio::sync::Mutex::new(HashMap::new()),
}) as Arc<dyn rustfs_lock::LockClient>
})
.collect();
pool.disk_set[0] = Arc::new(set);
let mut pools = other_store.pools.clone();
pools[0] = Arc::new(pool);
let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
ctx.update_erasure_type(crate::layout::endpoints::SetupType::DistErasure)
.await;
*ctx.local_disk_map().write().await = other_store.ctx.local_disk_map().read().await.clone();
let endpoints = other_store
.ctx
.endpoints()
.expect("lease-loss test store should have endpoint topology");
ctx.set_endpoints(endpoints.clone());
if let Some(deployment_id) = other_store.ctx.deployment_id() {
ctx.set_deployment_id(deployment_id);
}
for pool in &mut pools {
Arc::make_mut(pool).set_instance_ctx_for_test(Arc::clone(&ctx));
}
let store = Arc::new(crate::store::ECStore {
id: uuid::Uuid::new_v4(),
disk_map: other_store.disk_map.clone(),
pools,
peer_sys: crate::cluster::rpc::S3PeerSys::new_with_instance_ctx(&endpoints, Arc::clone(&ctx)),
pool_meta: tokio::sync::RwLock::new(other_store.pool_meta.read().await.clone()),
rebalance_meta: tokio::sync::RwLock::new(other_store.rebalance_meta.read().await.clone()),
decommission_cancelers: tokio::sync::RwLock::new(vec![None; other_store.pools.len()]),
start_gate: tokio::sync::Mutex::new(()),
pool_meta_save_gate: tokio::sync::Mutex::new(
other_store.pool_meta_save_gate.lock().await.independent_clone_for_test(),
),
ctx,
bucket_fence_registry: Arc::default(),
});
crate::bucket::metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
(store, refresh_calls)
}
async fn assert_lock_order(mutation: ExternalObjectMutation) {
assert_lock_order_with_tail(mutation, false).await;
}
#[tokio::test]
#[serial_test::serial]
async fn no_active_external_commit_probe_blocks_decommission_activation_until_commit_owns_it() {
let (_temp_dirs, store, _other_store) = test_three_pool_stores_with_isolated_node_contexts(None).await;
let bucket = test_bucket("no-active-probe");
let object = "probe-target.bin";
store
.make_bucket(&bucket, &MakeBucketOptions::default())
.await
.expect("create no-active probe bucket");
let layout = DecommissionErasureLayout { data: 1, parity: 0 };
set_decommission_capacity_info_overrides_for_test(
store.id,
vec![vec![
DecommissionPoolCapacityInfo::for_test(0, layout, 0, 1024, 1024),
DecommissionPoolCapacityInfo::for_test(1, layout, 1024, 1024, 0),
DecommissionPoolCapacityInfo::for_test(2, layout, 1024, 1024, 0),
]],
);
let barrier = DecommissionCapacityLockOrderBarrier::install(store.id, store.id);
barrier.pause_external_object_capacity_probe();
let put_store = Arc::clone(&store);
let put = tokio::spawn(async move {
let mut data = PutObjReader::from_vec(b"staged no-active commit".to_vec());
put_store.pools[2]
.put_object(
&bucket,
object,
&mut data,
&ObjectOptions {
decommission_capacity_admission: Some(Arc::clone(&put_store)),
..Default::default()
},
)
.await
});
barrier.wait_until_external_object_capacity_probe_acquired().await;
let activation_store = Arc::clone(&store);
let mut activation = tokio::spawn(async move {
activation_store
.save_current_pool_meta_for_decommission_start(&[0], Vec::new())
.await
});
assert!(
tokio::time::timeout(std::time::Duration::from_millis(50), &mut activation)
.await
.is_err(),
"decommission activation must wait behind the no-active commit probe"
);
barrier.release_external_object_capacity_probe();
tokio::time::timeout(std::time::Duration::from_secs(30), put)
.await
.expect("the staged external PUT should finish after probe release")
.expect("the staged external PUT should not panic")
.expect("the staged external PUT should commit");
drop(barrier);
tokio::time::timeout(std::time::Duration::from_secs(30), activation)
.await
.expect("decommission activation should proceed after the commit releases its probe")
.expect("decommission activation should not panic")
.expect("decommission activation should commit after the probe release");
}
#[test]
#[serial_test::serial]
fn staged_external_put_rechecks_retiring_source_on_another_node() {
run_large_stack_current_thread_async_test("staged-retiring-source", || async {
let (_temp_dirs, store, other_store) = test_three_pool_stores_with_isolated_node_contexts(None).await;
let bucket = test_bucket("staged-source");
let object = "selected-before-retirement.bin";
let original = b"original source object";
store
.make_bucket(&bucket, &MakeBucketOptions::default())
.await
.expect("create staged source bucket");
store.pools[0]
.put_object(&bucket, object, &mut PutObjReader::from_vec(original.to_vec()), &ObjectOptions::default())
.await
.expect("seed the source selected before retirement");
let layout = DecommissionErasureLayout { data: 1, parity: 0 };
set_decommission_capacity_info_overrides_for_test(
other_store.id,
vec![vec![
DecommissionPoolCapacityInfo::for_test(0, layout, 0, 1024, 1024),
DecommissionPoolCapacityInfo::for_test(1, layout, 4096, 4096, 0),
DecommissionPoolCapacityInfo::for_test(2, layout, 0, 4096, 4096),
]],
);
let barrier = DecommissionCapacityLockOrderBarrier::install(store.id, store.id);
barrier.pause_external_object_commit_phase();
let put_store = Arc::clone(&store);
let put_bucket = bucket.clone();
let put = tokio::spawn(async move {
put_store
.put_object(
&put_bucket,
object,
&mut PutObjReader::from_vec(b"must not replace a retiring source".to_vec()),
&ObjectOptions::default(),
)
.await
});
tokio::time::timeout(Duration::from_secs(30), barrier.wait_until_external_object_commit_phase_started())
.await
.expect("public PUT must stage before its decommission commit probe");
assert!(!store.pool_meta.read().await.is_suspended(0));
other_store
.save_current_pool_meta_for_decommission_start(&[0], Vec::new())
.await
.expect("the other node should activate retirement before the staged PUT commits");
assert!(other_store.pool_meta.read().await.is_suspended(0));
assert!(
!store.pool_meta.read().await.is_suspended(0),
"the writer's local snapshot must remain stale to exercise the durable admission probe"
);
barrier.release_external_object_commit_phase();
let result = tokio::time::timeout(Duration::from_secs(30), put)
.await
.expect("staged PUT must finish after the commit probe is released")
.expect("staged PUT must not panic");
assert!(
matches!(result, Err(crate::error::Error::SlowDown)),
"a staged PUT must retry pool selection instead of committing to a newly retiring source: {result:?}"
);
let mut reader = store.pools[0]
.get_object_reader(&bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
.await
.expect("the original source must remain readable after admission rejects the replacement");
let mut body = Vec::new();
reader
.stream
.read_to_end(&mut body)
.await
.expect("read the full retained source body");
assert_eq!(body, original);
});
}
#[test]
#[serial_test::serial]
fn reserved_target_shares_business_io_and_retains_source_after_capacity_loss() {
run_large_stack_current_thread_async_test("shared-decommission-capacity", || async {
for lose_capacity in [false, true] {
let (_temp_dirs, store, other_store) =
test_three_pool_stores_with_three_disk_sets_with_isolated_node_contexts(None).await;
let bucket = test_bucket("shared-capacity");
let object = "migrating-source.bin";
let business_object = "business-write.bin";
let multipart_object = "business-multipart.bin";
let source_body = vec![0x35; 256 * 1024];
let business_body = vec![0x57; 64 * 1024];
store
.make_bucket(&bucket, &MakeBucketOptions::default())
.await
.expect("create shared-capacity bucket");
store.pools[0]
.put_object(
&bucket,
object,
&mut PutObjReader::from_vec(source_body.clone()),
&ObjectOptions::default(),
)
.await
.expect("seed the retiring source");
store.pools[2]
.put_object(
&bucket,
business_object,
&mut PutObjReader::from_vec(b"previous business value".to_vec()),
&ObjectOptions::default(),
)
.await
.expect("pin the public overwrite to the migration target");
let multipart_opts = ObjectOptions {
expected_bucket_incarnation_id: Some(
store
.bucket_incarnation_id(&bucket)
.await
.expect("load the multipart bucket identity"),
),
..Default::default()
};
let routing_upload = new_multipart_upload(&store, 2, &bucket, multipart_object, multipart_opts.clone())
.await
.expect("pin subsequent public multipart creation to the migration target");
let layout = DecommissionErasureLayout { data: 1, parity: 0 };
let target_total = source_body.len() * 8;
let capacities = vec![
DecommissionPoolCapacityInfo::for_test(0, layout, 0, source_body.len() * 2, source_body.len() * 2),
DecommissionPoolCapacityInfo::for_test(1, layout, 0, target_total, target_total),
DecommissionPoolCapacityInfo::for_test(2, layout, target_total, target_total, 0),
];
set_decommission_capacity_info_overrides_for_test(store.id, vec![capacities.clone()]);
store
.save_current_pool_meta_for_decommission_start(&[0], Vec::new())
.await
.expect("activate source retirement");
*other_store.pool_meta.write().await = store.pool_meta.read().await.clone();
let before = other_store.pool_meta.read().await.clone();
let reservation = before.pools[0]
.decommission
.as_ref()
.expect("active source")
.capacity_reservation
.as_ref()
.expect("durable reservation");
assert_eq!(
reservation.model_version, 2,
"exercise migration I/O outside the global metadata write lock"
);
assert_eq!(reservation.targets[0].pool_index, 2);
let barrier =
crate::set_disk::rename_fanout_barrier::arm(object, 0, crate::set_disk::rename_fanout_barrier::PHASE_RENAME);
let migration_store = Arc::clone(&store);
let migration_bucket = bucket.clone();
let migration = tokio::spawn(async move {
migration_store
.decommission_entry_for_test_with_bucket_incarnation(
0,
MetaCacheEntry {
name: object.to_string(),
..Default::default()
},
migration_bucket,
migration_store.pools[0].get_disks_by_key(object),
)
.await
});
tokio::time::timeout(Duration::from_secs(30), barrier.wait_until_paused())
.await
.expect("migration must reach target publication");
assert!(!migration.is_finished());
let mut pending = crate::core::pools::PoolMeta::default();
pending
.load_no_lock_from_replicas(other_store.pools.clone())
.await
.expect("read migration intent from the other node");
let pending_reservation = pending.pools[0]
.decommission
.as_ref()
.expect("active source")
.capacity_reservation
.as_ref()
.expect("pending reservation")
.clone();
assert_eq!(pending_reservation.pending_target_physical_bytes, source_body.len());
assert_eq!(pending_reservation.consumed_target_physical_bytes, 0);
tokio::time::timeout(
Duration::from_secs(30),
other_store.put_object(
&bucket,
business_object,
&mut PutObjReader::from_vec(business_body.clone()),
&ObjectOptions::default(),
),
)
.await
.expect("business PUT must finish without waiting for the migration target gate")
.expect("a reserved healthy pool must accept ordinary PUT");
tokio::time::timeout(Duration::from_secs(30), async {
let upload = other_store
.new_multipart_upload(&bucket, multipart_object, &ObjectOptions::default())
.await
.expect("the reserved target must accept public multipart creation");
assert_ne!(upload.upload_id, routing_upload.upload_id);
let lifecycle_guard = other_store
.acquire_bucket_lifecycle_read_lock(&bucket)
.await
.expect("fence the exact-pool multipart placement check");
let mut lookup_opts = multipart_opts.clone();
lookup_opts.add_bucket_lifecycle_lock_guard(&lifecycle_guard);
other_store.pools[2]
.get_multipart_info(&bucket, multipart_object, &upload.upload_id, &lookup_opts)
.await
.expect("public multipart creation must actually select the reserved target");
drop(lifecycle_guard);
let mut final_part = None;
for payload in [vec![0x18; business_body.len()], business_body.clone()] {
final_part = Some(
other_store
.put_object_part(
&bucket,
multipart_object,
&upload.upload_id,
1,
&mut PutObjReader::from_vec(payload),
&ObjectOptions::default(),
)
.await
.expect("the reserved target must accept UploadPart and replacement of the same part"),
);
}
let part = final_part.expect("the replacement part must be present");
Arc::clone(&other_store)
.complete_multipart_upload(
&bucket,
multipart_object,
&upload.upload_id,
vec![crate::storage_api_contracts::multipart::CompletePart {
part_num: part.part_num,
etag: part.etag,
..Default::default()
}],
&ObjectOptions::default(),
)
.await
.expect("the reserved target must accept multipart completion");
other_store
.abort_multipart_upload(&bucket, multipart_object, &routing_upload.upload_id, &ObjectOptions::default())
.await
.expect("ordinary multipart cleanup must not consume the migration's pending intent");
})
.await
.expect("business multipart operations must finish while migration I/O is paused");
assert!(!migration.is_finished(), "business publication must overlap paused migration I/O");
let mut after_business = crate::core::pools::PoolMeta::default();
after_business
.load_no_lock_from_replicas(other_store.pools.clone())
.await
.expect("reload the shared-capacity ledger");
assert_eq!(
after_business.pools[0]
.decommission
.as_ref()
.expect("active source")
.capacity_reservation
.as_ref(),
Some(&pending_reservation),
"ordinary PUT and multipart operations must not settle or consume the migration's pending identity"
);
let mut after_capacity = capacities;
let free = if lose_capacity {
0
} else {
target_total - source_body.len() - business_body.len() * 2
};
after_capacity[2] = DecommissionPoolCapacityInfo::for_test(2, layout, free, target_total, target_total - free);
set_decommission_capacity_info_overrides_for_test(store.id, vec![after_capacity]);
barrier.release();
drop(barrier);
let migrated = tokio::time::timeout(Duration::from_secs(30), migration)
.await
.expect("migration must finish after publication resumes")
.expect("migration task must not panic");
if lose_capacity {
let err =
migrated.expect_err("capacity loss must prevent source cleanup, even after the target write commits");
assert!(err.to_string().contains("capacity"), "unexpected migration error: {err}");
} else {
migrated.expect("shared-capacity migration should finish when space remains sufficient");
}
let mut persisted = crate::core::pools::PoolMeta::default();
persisted
.load_no_lock_from_replicas(other_store.pools.clone())
.await
.expect("reload finalized migration state");
let info = persisted.pools[0].decommission.as_ref().expect("source state");
let reservation = info.capacity_reservation.as_ref().expect("migration ledger");
assert_eq!(reservation.pending_target_physical_bytes, 0);
assert_eq!(
reservation.consumed_target_physical_bytes,
source_body.len(),
"foreign writes must not count as committed source bytes"
);
assert_eq!(reservation.committed_data_bytes, source_body.len());
assert_eq!(info.capacity_blocked_reason.is_some(), lose_capacity);
for (pool, key, expected) in [
(2, business_object, &business_body),
(2, multipart_object, &business_body),
(2, object, &source_body),
] {
let mut reader = other_store.pools[pool]
.get_object_reader(&bucket, key, None, HeaderMap::new(), &ObjectOptions::default())
.await
.expect("all acknowledged target objects must remain readable");
let mut actual = Vec::new();
reader.read_to_end(&mut actual).await.expect("read the complete target body");
assert_eq!(&actual, expected);
}
if lose_capacity {
let mut source = other_store.pools[0]
.get_object_reader(&bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
.await
.expect("capacity-blocked migration must retain its source");
let mut actual = Vec::new();
source
.read_to_end(&mut actual)
.await
.expect("read the complete retained source");
assert_eq!(actual, source_body);
other_store
.put_object(
&bucket,
business_object,
&mut PutObjReader::from_vec(business_body.clone()),
&ObjectOptions::default(),
)
.await
.expect("a capacity-blocked migration must not itself make the healthy target read-only");
} else {
let err = other_store.pools[0]
.get_object_info(&bucket, object, &ObjectOptions::default())
.await
.expect_err("successful migration must clean the exact source");
assert!(crate::error::is_err_object_not_found(&err));
}
}
});
}
#[test]
#[serial_test::serial]
fn mixed_batch_delete_admits_only_marker_destinations_during_retirement() {
run_large_stack_current_thread_async_test("batch-marker-admission", || async {
use crate::storage_api_contracts::object::ObjectToDelete;
for marker_target in [1, 2] {
let (_temp_dirs, store, _other_store) = test_three_pool_stores_with_isolated_node_contexts(None).await;
let bucket = test_bucket("batch-marker");
store
.make_bucket(
&bucket,
&MakeBucketOptions {
versioning_enabled: true,
..Default::default()
},
)
.await
.expect("create a versioned batch-delete bucket");
let source_version = uuid::Uuid::new_v4();
for (pool, object, version) in [
(0, "purge-source", source_version),
(marker_target, "mark-active", uuid::Uuid::new_v4()),
] {
store.pools[pool]
.put_object(
&bucket,
object,
&mut PutObjReader::from_vec(b"version to delete".to_vec()),
&ObjectOptions {
versioned: true,
version_id: Some(version.to_string()),
..Default::default()
},
)
.await
.expect("seed each exact batch-delete destination");
}
let layout = DecommissionErasureLayout { data: 1, parity: 0 };
set_decommission_capacity_info_overrides_for_test(
store.id,
vec![vec![
DecommissionPoolCapacityInfo::for_test(0, layout, 0, 1024, 1024),
DecommissionPoolCapacityInfo::for_test(1, layout, 4096, 4096, 0),
DecommissionPoolCapacityInfo::for_test(2, layout, 0, 4096, 4096),
]],
);
store
.save_current_pool_meta_for_decommission_start(&[0], Vec::new())
.await
.expect("reserve pool 1 while pool 0 retires and pool 2 remains unreserved");
let (deleted, errors) = store
.delete_objects(
&bucket,
vec![
ObjectToDelete {
object_name: "mark-active".to_string(),
..Default::default()
},
ObjectToDelete {
object_name: "purge-source".to_string(),
version_id: Some(source_version),
..Default::default()
},
],
ObjectOptions::default(),
)
.await;
assert_eq!(errors.len(), 2);
assert!(
errors.iter().all(Option::is_none),
"the unrelated retiring/reserved pools must not reject marker admission: {errors:?}"
);
assert_eq!(deleted.len(), 2);
assert_eq!(deleted[0].object_name, "mark-active");
assert!(deleted[0].delete_marker);
assert!(
deleted[0].version_id.is_none(),
"a latest-version delete does not request an explicit version"
);
assert!(
deleted[0].delete_marker_version_id.is_some(),
"the newly created marker must have its own version identity"
);
assert_eq!(deleted[1].object_name, "purge-source");
assert!(!deleted[1].delete_marker);
assert_eq!(deleted[1].version_id, Some(source_version));
assert!(
matches!(
store.pools[0]
.get_object_info(
&bucket,
"purge-source",
&ObjectOptions {
version_id: Some(source_version.to_string()),
..Default::default()
},
)
.await,
Err(crate::error::Error::ObjectNotFound(..) | crate::error::Error::VersionNotFound(..))
),
"an exact source deletion must retain its capacity-release path"
);
}
});
}
#[tokio::test]
#[serial_test::serial]
async fn public_upload_part_holds_decommission_capacity_until_rename() {
let (_temp_dirs, store, other_store) = test_three_pool_stores_with_isolated_node_contexts(None).await;
let bucket = test_bucket("public-part-probe");
let object = "public-part.bin";
store
.make_bucket(&bucket, &MakeBucketOptions::default())
.await
.expect("create public UploadPart probe bucket");
let incarnation = store
.bucket_incarnation_id(&bucket)
.await
.expect("load public UploadPart bucket incarnation");
let upload = new_multipart_upload(
&store,
2,
&bucket,
object,
ObjectOptions {
expected_bucket_incarnation_id: Some(incarnation),
..Default::default()
},
)
.await
.expect("create public UploadPart probe upload");
let layout = DecommissionErasureLayout { data: 1, parity: 0 };
set_decommission_capacity_info_overrides_for_test(
other_store.id,
vec![vec![
DecommissionPoolCapacityInfo::for_test(0, layout, 0, 1024, 1024),
DecommissionPoolCapacityInfo::for_test(1, layout, 1024, 1024, 0),
DecommissionPoolCapacityInfo::for_test(2, layout, 1024, 1024, 0),
]],
);
let barrier = MultipartCommitBarrier::install(&bucket, object, MultipartCommitPause::PutPartAfterCapacityAdmission);
let put_store = Arc::clone(&other_store);
let put_bucket = bucket.clone();
let upload_id = upload.upload_id.clone();
let put = tokio::spawn(async move {
let mut data = PutObjReader::from_vec(b"public staged part".to_vec());
put_store
.put_object_part(
&put_bucket,
object,
&upload_id,
1,
&mut data,
&ObjectOptions {
expected_bucket_incarnation_id: Some(incarnation),
..Default::default()
},
)
.await
});
barrier.wait_until_paused().await;
let activation_store = Arc::clone(&other_store);
let mut activation = tokio::spawn(async move {
activation_store
.save_current_pool_meta_for_decommission_start(&[0], Vec::new())
.await
});
assert!(
tokio::time::timeout(std::time::Duration::from_millis(50), &mut activation)
.await
.is_err(),
"decommission activation must wait behind the public UploadPart capacity guard"
);
barrier.release();
let part = tokio::time::timeout(std::time::Duration::from_secs(30), put)
.await
.expect("public UploadPart should finish after the commit barrier release")
.expect("public UploadPart task should not panic")
.expect("public UploadPart should commit");
drop(barrier);
tokio::time::timeout(std::time::Duration::from_secs(30), activation)
.await
.expect("decommission activation should proceed after public UploadPart commit")
.expect("decommission activation task should not panic")
.expect("decommission activation should commit after public UploadPart");
let listed = other_store
.list_object_parts(
&bucket,
object,
&upload.upload_id,
None,
100,
&ObjectOptions {
expected_bucket_incarnation_id: Some(incarnation),
..Default::default()
},
)
.await
.expect("public UploadPart result should remain readable");
assert_eq!(listed.parts.len(), 1, "public UploadPart should publish exactly one part");
assert_eq!(listed.parts[0].part_num, part.part_num);
assert_eq!(listed.parts[0].etag, part.etag);
}
#[tokio::test]
#[serial_test::serial]
async fn public_upload_part_stages_before_decommission_capacity_admission() {
let (_temp_dirs, store, other_store) = test_three_pool_stores_with_isolated_node_contexts(None).await;
let bucket = test_bucket("public-part-staged");
let object = "public-part-staged.bin";
store
.make_bucket(&bucket, &MakeBucketOptions::default())
.await
.expect("create staged public UploadPart bucket");
let incarnation = store
.bucket_incarnation_id(&bucket)
.await
.expect("load staged public UploadPart bucket incarnation");
let upload = new_multipart_upload(
&store,
2,
&bucket,
object,
ObjectOptions {
expected_bucket_incarnation_id: Some(incarnation),
..Default::default()
},
)
.await
.expect("create staged public UploadPart upload");
let barrier = MultipartCommitBarrier::install(&bucket, object, MultipartCommitPause::PutPartBeforeLockLost);
let put_store = Arc::clone(&other_store);
let put_bucket = bucket.clone();
let upload_id = upload.upload_id.clone();
let put = tokio::spawn(async move {
let mut data = PutObjReader::from_vec(b"public staged before admission".to_vec());
put_store
.put_object_part(
&put_bucket,
object,
&upload_id,
1,
&mut data,
&ObjectOptions {
expected_bucket_incarnation_id: Some(incarnation),
..Default::default()
},
)
.await
});
barrier.wait_until_paused().await;
let probe_store = Arc::clone(&other_store);
let pool_meta_lock = probe_store
.new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME)
.await
.expect("create the public UploadPart capacity guard probe");
let pool_meta_guard = tokio::time::timeout(
std::time::Duration::from_secs(1),
pool_meta_lock.get_write_lock(std::time::Duration::from_secs(30)),
)
.await
.expect("public UploadPart must not hold a capacity read guard during staging")
.expect("public UploadPart capacity guard probe should acquire");
drop(pool_meta_guard);
barrier.release();
let part = tokio::time::timeout(std::time::Duration::from_secs(30), put)
.await
.expect("staged public UploadPart should finish after the commit barrier release")
.expect("staged public UploadPart task should not panic")
.expect("staged public UploadPart should commit");
drop(barrier);
let listed = other_store
.list_object_parts(
&bucket,
object,
&upload.upload_id,
None,
100,
&ObjectOptions {
expected_bucket_incarnation_id: Some(incarnation),
..Default::default()
},
)
.await
.expect("staged public UploadPart result should remain readable");
assert_eq!(listed.parts.len(), 1, "staged public UploadPart should publish exactly one part");
assert_eq!(listed.parts[0].part_num, part.part_num);
assert_eq!(listed.parts[0].etag, part.etag);
}
#[tokio::test]
#[serial_test::serial]
async fn public_put_holds_decommission_capacity_through_early_ack_tail() {
temp_env::async_with_vars([(crate::set_disk::ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async {
let (_temp_dirs, store, other_store) =
test_three_pool_stores_with_three_disk_sets_with_isolated_node_contexts(None).await;
let bucket = test_bucket("public-put-tail");
let object = "public-put-tail.bin";
let body = vec![0x71; 256 * 1024];
store
.make_bucket(&bucket, &MakeBucketOptions::default())
.await
.expect("create public PUT tail bucket");
let mut initial = PutObjReader::from_vec(vec![0x69; body.len()]);
store.pools[2]
.put_object(&bucket, object, &mut initial, &ObjectOptions::default())
.await
.expect("seed the unreserved public PUT target");
let layout = DecommissionErasureLayout { data: 1, parity: 0 };
let target_total = body.len().saturating_mul(4);
let capacity_snapshot = vec![
DecommissionPoolCapacityInfo::for_test(0, layout, 0, body.len(), body.len()),
DecommissionPoolCapacityInfo::for_test(1, layout, target_total, target_total, 0),
DecommissionPoolCapacityInfo::for_test(2, layout, target_total, target_total, 0),
];
set_decommission_capacity_info_overrides_for_test(store.id, vec![capacity_snapshot]);
store
.save_current_pool_meta_for_decommission_start(&[0], Vec::new())
.await
.expect("activate the public PUT tail reservation");
*other_store.pool_meta.write().await = store.pool_meta.read().await.clone();
let reservation_target = store.pool_meta.read().await.pools[0]
.decommission
.as_ref()
.and_then(|info| info.capacity_reservation.as_ref())
.expect("public PUT tail reservation should exist")
.targets[0]
.pool_index;
assert_eq!(reservation_target, 1, "the public PUT target must remain unreserved");
let tail_barrier =
crate::set_disk::rename_fanout_barrier::arm(object, 0, crate::set_disk::rename_fanout_barrier::PHASE_RENAME);
let put_store = Arc::clone(&other_store);
let put_bucket = bucket.clone();
let put_body = body.clone();
let expected_body = body.clone();
let mut put = tokio::spawn(async move {
let mut data = PutObjReader::from_vec(put_body);
put_store
.put_object(&put_bucket, object, &mut data, &ObjectOptions::default())
.await
});
tail_barrier.wait_until_paused().await;
tokio::time::timeout(Duration::from_secs(30), &mut put)
.await
.expect("public PUT should return its early quorum acknowledgement")
.expect("public PUT task should not panic")
.expect("public PUT should commit its quorum");
let capacity_lock = other_store
.new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME)
.await
.expect("create the public PUT capacity probe");
let mut capacity_probe = tokio::spawn(async move { capacity_lock.get_write_lock(Duration::from_secs(30)).await });
assert!(
tokio::time::timeout(Duration::from_millis(100), &mut capacity_probe)
.await
.is_err(),
"public PUT early-ACK tail must retain its staged capacity read fence"
);
tail_barrier.release();
drop(tail_barrier);
let capacity_guard = tokio::time::timeout(Duration::from_secs(30), capacity_probe)
.await
.expect("public PUT capacity probe should finish after its tail")
.expect("public PUT capacity probe should not panic")
.expect("public PUT capacity probe should acquire after its tail");
drop(capacity_guard);
let mut reader = other_store
.get_object_reader(&bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
.await
.expect("public PUT result should remain readable");
let mut committed = Vec::new();
reader
.stream
.read_to_end(&mut committed)
.await
.expect("public PUT result body should be readable");
assert_eq!(committed, expected_body, "public PUT should publish the requested bytes");
})
.await;
}
#[tokio::test]
#[serial_test::serial]
async fn public_put_fences_capacity_lease_loss_before_publication() {
let (_temp_dirs, store, other_store) = test_three_pool_stores_with_isolated_node_contexts(None).await;
let bucket = test_bucket("public-put-lease-loss");
let object = "public-put-lease-loss.bin";
let body = vec![0x72; 256 * 1024];
let old_body = vec![0x69; body.len()];
store
.make_bucket(&bucket, &MakeBucketOptions::default())
.await
.expect("create public PUT lease-loss bucket");
let incarnation = store
.bucket_incarnation_id(&bucket)
.await
.expect("load public PUT lease-loss bucket incarnation");
let mut old_target = PutObjReader::from_vec(old_body.clone());
store.pools[2]
.put_object(&bucket, object, &mut old_target, &ObjectOptions::default())
.await
.expect("seed the public PUT lease-loss target");
let layout = DecommissionErasureLayout { data: 1, parity: 0 };
let target_total = body.len().saturating_mul(4);
let capacity_snapshot = || {
vec![
DecommissionPoolCapacityInfo::for_test(0, layout, 0, body.len(), body.len()),
DecommissionPoolCapacityInfo::for_test(1, layout, target_total, target_total, 0),
DecommissionPoolCapacityInfo::for_test(2, layout, 0, target_total, target_total),
]
};
set_decommission_capacity_info_overrides_for_test(store.id, vec![capacity_snapshot()]);
store
.save_current_pool_meta_for_decommission_start(&[0], Vec::new())
.await
.expect("activate the public PUT lease-loss reservation");
*other_store.pool_meta.write().await = store.pool_meta.read().await.clone();
let (lossy_store, refresh_calls) = store_with_capacity_lease_loss(&other_store).await;
set_decommission_capacity_info_overrides_for_test(
lossy_store.id,
vec![capacity_snapshot(), capacity_snapshot(), capacity_snapshot()],
);
let barrier = PutObjectCommitBarrier::install(&bucket, object, PutObjectCommitPause::BeforeQuotaRename);
tokio::time::pause();
let put_store = Arc::clone(&lossy_store);
let put_bucket = bucket.clone();
let put_body = body.clone();
let put = tokio::spawn(async move {
let mut data = PutObjReader::from_vec(put_body);
put_store
.put_object(
&put_bucket,
object,
&mut data,
&ObjectOptions {
expected_bucket_incarnation_id: Some(incarnation),
..Default::default()
},
)
.await
});
barrier.wait_until_paused().await;
tokio::task::yield_now().await;
refresh_calls.arm();
tokio::time::advance(Duration::from_secs(11)).await;
tokio::task::yield_now().await;
assert!(
refresh_calls.load(Ordering::Acquire) > 0,
"the public PUT capacity lease must lose refresh quorum"
);
barrier.release();
let err = tokio::time::timeout(Duration::from_secs(30), put)
.await
.expect("public PUT should finish after the commit barrier release")
.expect("public PUT task should not panic")
.expect_err("lost public PUT capacity lease must fence before publication");
assert!(!err.to_string().is_empty(), "public PUT lease loss should remain observable");
drop(barrier);
let mut target = lossy_store
.get_object_reader(
&bucket,
object,
None,
HeaderMap::new(),
&ObjectOptions {
expected_bucket_incarnation_id: Some(incarnation),
..Default::default()
},
)
.await
.expect("the old public PUT target should remain readable");
let mut committed = Vec::new();
target
.stream
.read_to_end(&mut committed)
.await
.expect("the old public PUT target should drain");
assert_eq!(committed, old_body, "lost public PUT capacity lease must preserve the prior target");
}
#[tokio::test]
#[serial_test::serial]
async fn public_cross_object_copy_fences_capacity_lease_loss_before_publication() {
let (_temp_dirs, store, other_store) = test_three_pool_stores_with_isolated_node_contexts(None).await;
let bucket = test_bucket("public-copy-lease-loss");
let source_object = "public-copy-source.bin";
let target_object = "public-copy-target.bin";
let source_body = vec![0x73; 256 * 1024];
let old_target_body = vec![0x69; source_body.len()];
store
.make_bucket(&bucket, &MakeBucketOptions::default())
.await
.expect("create public CopyObject lease-loss bucket");
let incarnation = store
.bucket_incarnation_id(&bucket)
.await
.expect("load public CopyObject lease-loss bucket incarnation");
let mut source = PutObjReader::from_vec(source_body.clone());
store.pools[2]
.put_object(&bucket, source_object, &mut source, &ObjectOptions::default())
.await
.expect("seed the public CopyObject source");
let mut old_target = PutObjReader::from_vec(old_target_body.clone());
store.pools[2]
.put_object(&bucket, target_object, &mut old_target, &ObjectOptions::default())
.await
.expect("seed the public CopyObject target");
let mut copy_reader = store.pools[2]
.get_object_reader(
&bucket,
source_object,
None,
HeaderMap::new(),
&ObjectOptions {
no_lock: true,
..Default::default()
},
)
.await
.expect("read the public CopyObject source");
let mut copy_info = copy_reader.object_info.clone();
let mut copy_body = Vec::new();
copy_reader
.stream
.read_to_end(&mut copy_body)
.await
.expect("stage the public CopyObject source bytes");
assert_eq!(copy_body, source_body);
copy_info.put_object_reader = Some(PutObjReader::from_vec(copy_body));
let layout = DecommissionErasureLayout { data: 1, parity: 0 };
let target_total = source_body.len().saturating_mul(4);
let capacity_snapshot = || {
vec![
DecommissionPoolCapacityInfo::for_test(0, layout, 0, source_body.len(), source_body.len()),
DecommissionPoolCapacityInfo::for_test(1, layout, target_total, target_total, 0),
DecommissionPoolCapacityInfo::for_test(2, layout, 0, target_total, target_total),
]
};
set_decommission_capacity_info_overrides_for_test(store.id, vec![capacity_snapshot()]);
store
.save_current_pool_meta_for_decommission_start(&[0], Vec::new())
.await
.expect("activate the public CopyObject capacity reservation");
*other_store.pool_meta.write().await = store.pool_meta.read().await.clone();
let (lossy_store, refresh_calls) = store_with_capacity_lease_loss(&other_store).await;
set_decommission_capacity_info_overrides_for_test(
lossy_store.id,
vec![capacity_snapshot(), capacity_snapshot(), capacity_snapshot()],
);
let barrier = PutObjectCommitBarrier::install(&bucket, target_object, PutObjectCommitPause::BeforeQuotaRename);
tokio::time::pause();
let copy_store = Arc::clone(&lossy_store);
let copy_bucket = bucket.clone();
let copy = tokio::spawn(async move {
copy_store
.copy_object(
©_bucket,
source_object,
©_bucket,
target_object,
&mut copy_info,
&ObjectOptions::default(),
&ObjectOptions {
expected_bucket_incarnation_id: Some(incarnation),
..Default::default()
},
)
.await
});
barrier.wait_until_paused().await;
tokio::task::yield_now().await;
refresh_calls.arm();
tokio::time::advance(Duration::from_secs(11)).await;
tokio::task::yield_now().await;
assert!(
refresh_calls.load(Ordering::Acquire) > 0,
"the public CopyObject capacity lease must lose refresh quorum"
);
barrier.release();
let err = tokio::time::timeout(Duration::from_secs(30), copy)
.await
.expect("public CopyObject should finish after the commit barrier release")
.expect("public CopyObject task should not panic")
.expect_err("lost public CopyObject capacity lease must fence before publication");
assert!(!err.to_string().is_empty(), "public CopyObject lease loss should remain observable");
drop(barrier);
tokio::time::resume();
let mut target_reader = lossy_store.pools[2]
.get_object_reader(&bucket, target_object, None, HeaderMap::new(), &ObjectOptions::default())
.await
.expect("the old public CopyObject target should remain readable");
let mut committed = Vec::new();
target_reader
.stream
.read_to_end(&mut committed)
.await
.expect("the old public CopyObject target body should remain readable");
assert_eq!(committed, old_target_body, "lost CopyObject capacity lease must not publish new bytes");
}
async fn assert_same_object_copy_fences_capacity_lease_loss(transitioned: bool) {
let (_temp_dirs, store, other_store) = test_three_pool_stores_with_isolated_node_contexts(None).await;
let bucket = test_bucket("public-self-copy-loss");
let object = "public-self-copy-lease-loss.bin";
let source_body = vec![0x74; 256 * 1024];
store
.make_bucket(&bucket, &MakeBucketOptions::default())
.await
.expect("create public self-copy lease-loss bucket");
let incarnation = store
.bucket_incarnation_id(&bucket)
.await
.expect("load public self-copy lease-loss bucket incarnation");
let (mut copy_info, source_opts, destination_opts, expected_body, expected_tier, expected_current_version) =
if transitioned {
let mut original = PutObjReader::from_vec(source_body.clone());
let original_info = store.pools[2]
.put_object(&bucket, object, &mut original, &ObjectOptions::default())
.await
.expect("seed the transitioned self-copy source");
let tier_name = format!("SELFCOPY{}", &uuid::Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
let _backend = register_mock_tier(&store.pools[2].instance_ctx().tier_config_mgr(), &tier_name).await;
store.pools[2]
.transition_object(
&bucket,
object,
&ObjectOptions {
no_lock: true,
transition: crate::bucket::lifecycle::lifecycle::TransitionOptions {
status: TRANSITION_PENDING.to_string(),
tier: tier_name.clone(),
etag: original_info.etag.clone().expect("transition source should have an ETag"),
..Default::default()
},
mod_time: original_info.mod_time,
..Default::default()
},
)
.await
.expect("seed the transitioned self-copy source");
let mut copy_info = store.pools[2]
.get_object_info(&bucket, object, &ObjectOptions::default())
.await
.expect("read the transitioned self-copy source metadata");
assert_eq!(copy_info.transitioned_object.tier, tier_name);
assert!(!copy_info.metadata_only, "transitioned self-copy must use the reader PUT path");
let mut reader = store.pools[2]
.get_object_reader(
&bucket,
object,
None,
HeaderMap::new(),
&ObjectOptions {
no_lock: true,
..Default::default()
},
)
.await
.expect("read the transitioned self-copy source body");
let mut copy_body = Vec::new();
reader
.stream
.read_to_end(&mut copy_body)
.await
.expect("stage the transitioned self-copy source body");
assert_eq!(copy_body, source_body);
copy_info.put_object_reader = Some(PutObjReader::from_vec(copy_body));
(
copy_info,
ObjectOptions::default(),
ObjectOptions {
expected_bucket_incarnation_id: Some(incarnation),
..Default::default()
},
source_body.clone(),
Some(tier_name),
None,
)
} else {
let historical_version = uuid::Uuid::new_v4().to_string();
let current_version = uuid::Uuid::new_v4().to_string();
let current_body = vec![0x68; source_body.len()];
let mut historical = PutObjReader::from_vec(source_body.clone());
store.pools[2]
.put_object(
&bucket,
object,
&mut historical,
&ObjectOptions {
versioned: true,
version_id: Some(historical_version.clone()),
..Default::default()
},
)
.await
.expect("seed the historical self-copy source");
let mut current = PutObjReader::from_vec(current_body.clone());
store.pools[2]
.put_object(
&bucket,
object,
&mut current,
&ObjectOptions {
versioned: true,
version_id: Some(current_version.clone()),
..Default::default()
},
)
.await
.expect("seed the current self-copy target");
let mut copy_info = store.pools[2]
.get_object_info(
&bucket,
object,
&ObjectOptions {
versioned: true,
version_id: Some(historical_version.clone()),
..Default::default()
},
)
.await
.expect("read the historical self-copy source metadata");
let mut reader = store.pools[2]
.get_object_reader(
&bucket,
object,
None,
HeaderMap::new(),
&ObjectOptions {
no_lock: true,
versioned: true,
version_id: Some(historical_version.clone()),
..Default::default()
},
)
.await
.expect("read the historical self-copy source body");
let mut copy_body = Vec::new();
reader
.stream
.read_to_end(&mut copy_body)
.await
.expect("stage the historical self-copy source body");
assert_eq!(copy_body, source_body);
copy_info.put_object_reader = Some(PutObjReader::from_vec(copy_body));
(
copy_info,
ObjectOptions {
versioned: true,
version_id: Some(historical_version),
..Default::default()
},
ObjectOptions {
versioned: true,
expected_current_version_id: Some(current_version.clone()),
expected_bucket_incarnation_id: Some(incarnation),
..Default::default()
},
current_body,
None,
Some(current_version),
)
};
let layout = DecommissionErasureLayout { data: 1, parity: 0 };
let target_total = source_body.len().saturating_mul(4);
let capacity_snapshot = || {
vec![
DecommissionPoolCapacityInfo::for_test(0, layout, 0, source_body.len(), source_body.len()),
DecommissionPoolCapacityInfo::for_test(1, layout, target_total, target_total, 0),
DecommissionPoolCapacityInfo::for_test(2, layout, target_total, target_total, 0),
]
};
set_decommission_capacity_info_overrides_for_test(store.id, vec![capacity_snapshot()]);
store
.save_current_pool_meta_for_decommission_start(&[0], Vec::new())
.await
.expect("activate the same-object CopyObject capacity reservation");
*other_store.pool_meta.write().await = store.pool_meta.read().await.clone();
let (lossy_store, refresh_calls) = store_with_capacity_lease_loss(&other_store).await;
set_decommission_capacity_info_overrides_for_test(
lossy_store.id,
vec![capacity_snapshot(), capacity_snapshot(), capacity_snapshot()],
);
let barrier = PutObjectCommitBarrier::install(&bucket, object, PutObjectCommitPause::BeforeQuotaRename);
tokio::time::pause();
let copy_store = Arc::clone(&lossy_store);
let copy_bucket = bucket.clone();
let copy = tokio::spawn(async move {
copy_store
.copy_object(
©_bucket,
object,
©_bucket,
object,
&mut copy_info,
&source_opts,
&destination_opts,
)
.await
});
barrier.wait_until_paused().await;
tokio::task::yield_now().await;
refresh_calls.arm();
tokio::time::advance(Duration::from_secs(11)).await;
tokio::task::yield_now().await;
assert!(
refresh_calls.load(Ordering::Acquire) > 0,
"the same-object CopyObject capacity lease must lose refresh quorum"
);
barrier.release();
let err = tokio::time::timeout(Duration::from_secs(30), copy)
.await
.expect("same-object CopyObject should finish after the commit barrier release")
.expect("same-object CopyObject task should not panic")
.expect_err("lost same-object CopyObject capacity lease must fence before publication");
assert!(!err.to_string().is_empty(), "same-object CopyObject lease loss should remain observable");
drop(barrier);
tokio::time::resume();
let target_info = lossy_store.pools[2]
.get_object_info(&bucket, object, &ObjectOptions::default())
.await
.expect("the failed same-object CopyObject target should remain readable");
if let Some(expected_tier) = expected_tier {
assert_eq!(
target_info.transitioned_object.tier, expected_tier,
"lost transitioned self-copy capacity lease must not publish a local target"
);
}
if let Some(expected_current_version) = expected_current_version {
assert_eq!(
target_info.version_id.map(|version| version.to_string()),
Some(expected_current_version),
"lost historical self-copy capacity lease must not publish a new current version"
);
}
let mut target_reader = store.pools[2]
.get_object_reader(&bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
.await
.expect("the failed same-object CopyObject target body should remain readable");
let mut committed = Vec::new();
target_reader
.stream
.read_to_end(&mut committed)
.await
.expect("the failed same-object CopyObject target body should drain");
assert_eq!(committed, expected_body, "lost same-object CopyObject lease must preserve target bytes");
}
#[tokio::test]
#[serial_test::serial]
async fn public_historical_self_copy_fences_capacity_lease_loss_before_publication() {
assert_same_object_copy_fences_capacity_lease_loss(false).await;
}
#[tokio::test]
#[serial_test::serial]
async fn public_transitioned_self_copy_fences_capacity_lease_loss_before_publication() {
assert_same_object_copy_fences_capacity_lease_loss(true).await;
}
#[test]
#[serial_test::serial]
fn data_movement_multipart_restart_cleans_exact_zero_delta_upload_with_recovery_marker() {
run_large_stack_async_test(
"multipart-zero-delta-restart",
data_movement_multipart_restart_cleans_exact_zero_delta_upload_with_recovery_marker_case,
);
}
async fn data_movement_multipart_restart_cleans_exact_zero_delta_upload_with_recovery_marker_case() {
let (_temp_dirs, store, _other_store) = test_three_pool_stores_with_isolated_node_contexts(None).await;
let bucket = test_bucket("multipart-zero-delta-restart");
let object = "staged-without-statfs-delta.bin";
store
.make_bucket(&bucket, &MakeBucketOptions::default())
.await
.expect("create the zero-delta multipart restart bucket");
let incarnation = store
.bucket_incarnation_id(&bucket)
.await
.expect("load the zero-delta multipart bucket incarnation");
let layout = DecommissionErasureLayout { data: 1, parity: 0 };
let capacity_snapshot = || {
vec![
DecommissionPoolCapacityInfo::for_test(0, layout, 0, 8, 8),
DecommissionPoolCapacityInfo::for_test(1, layout, 0, 16, 16),
DecommissionPoolCapacityInfo::for_test(2, layout, 32, 32, 0),
]
};
set_decommission_capacity_info_overrides_for_test(store.id, (0..8).map(|_| capacity_snapshot()).collect());
store
.save_current_pool_meta_for_decommission_start(&[0], Vec::new())
.await
.expect("activate the zero-delta multipart reservation");
let target_pool_index = store.pool_meta.read().await.pools[0]
.decommission
.as_ref()
.and_then(|info| info.capacity_reservation.as_ref())
.and_then(|reservation| reservation.targets.first())
.expect("zero-delta fixture should reserve one target")
.pool_index;
assert_eq!(target_pool_index, 2);
let owner = decommission_capacity_owner(&*store.pool_meta.read().await).with_mutation_id(uuid::Uuid::new_v4());
let exact_version = uuid::Uuid::new_v4();
let foreign_version = uuid::Uuid::new_v4();
let exact_mod_time = time::OffsetDateTime::UNIX_EPOCH + time::Duration::seconds(11);
let foreign_mod_time = time::OffsetDateTime::UNIX_EPOCH + time::Duration::seconds(12);
let exact_identity = format!("v1:{exact_version}:{}", exact_mod_time.unix_timestamp_nanos());
let foreign_identity = format!("v1:{foreign_version}:{}", foreign_mod_time.unix_timestamp_nanos());
let upload_opts = |identity: &str, version_id: uuid::Uuid, mod_time: time::OffsetDateTime| {
let mut opts = ObjectOptions {
data_movement: true,
src_pool_idx: 0,
versioned: true,
version_id: Some(version_id.to_string()),
mod_time: Some(mod_time),
expected_bucket_incarnation_id: Some(incarnation),
..Default::default()
};
rustfs_utils::http::insert_str(
&mut opts.user_defined,
rustfs_utils::http::SUFFIX_DATA_MOVEMENT_UPLOAD,
identity.to_string(),
);
owner.apply_to(&mut opts);
opts
};
let mut cleanup_opts = ObjectOptions {
data_movement: true,
src_pool_idx: 0,
versioned: true,
version_id: Some(exact_version.to_string()),
mod_time: Some(exact_mod_time),
expected_bucket_incarnation_id: Some(incarnation),
..Default::default()
};
owner.apply_to(&mut cleanup_opts);
store.reset_data_movement_multipart_discovery_count_for_test();
store
.reconcile_multipart_uploads_for_data_movement(target_pool_index, &bucket, object, &exact_identity, &cleanup_opts)
.await
.expect("a fresh mutation without recovery state should take the metadata fast path");
assert_eq!(store.data_movement_multipart_discovery_count_for_test(), 0);
let exact_upload_opts = upload_opts(&exact_identity, exact_version, exact_mod_time);
let staging_store = Arc::clone(&store);
let staging_bucket = bucket.clone();
let exact_err = store
.run_decommission_capacity_temporary_mutation(target_pool_index, Some(owner), Some(1), || async move {
new_multipart_upload(&staging_store, target_pool_index, &staging_bucket, object, exact_upload_opts).await?;
Err::<(), crate::error::Error>(crate::error::Error::OperationCanceled)
})
.await
.expect_err("the fixture should fail after staging the exact zero-delta upload");
assert!(matches!(exact_err, crate::error::Error::OperationCanceled));
let foreign_upload = new_multipart_upload(
&store,
target_pool_index,
&bucket,
object,
upload_opts(&foreign_identity, foreign_version, foreign_mod_time),
)
.await
.expect("stage the foreign zero-delta upload");
{
let pool_meta = store.pool_meta.read().await;
let reservation = pool_meta.pools[0]
.decommission
.as_ref()
.and_then(|info| info.capacity_reservation.as_ref())
.expect("zero-delta reservation should remain active");
assert_eq!(reservation.pending_target_physical_bytes, 1);
assert_eq!(reservation.inflight_target_physical_bytes, 0);
assert_eq!(reservation.targets[0].pending_mutation_id, owner.mutation_id);
assert_eq!(reservation.targets[0].temporary_mutations.len(), 1);
assert_eq!(reservation.targets[0].temporary_mutations[0].mutation_id, owner.mutation_id.unwrap());
assert_eq!(reservation.targets[0].temporary_mutations[0].physical_bytes, 0);
}
store
.reconcile_multipart_uploads_for_data_movement(target_pool_index, &bucket, object, &exact_identity, &cleanup_opts)
.await
.expect("restart should remove the exact upload through its zero-byte recovery marker");
assert_eq!(store.data_movement_multipart_discovery_count_for_test(), 1);
let set = store.pools[target_pool_index].get_disks_by_key(object);
let exact_remaining = set
.data_movement_multipart_upload_ids(&bucket, object, Some(incarnation), &exact_identity)
.await
.expect("scan for the exact zero-delta upload after reconciliation");
assert!(exact_remaining.is_empty(), "the exact stale upload must be removed");
let foreign_remaining = set
.data_movement_multipart_upload_ids(&bucket, object, Some(incarnation), &foreign_identity)
.await
.expect("scan for the foreign upload after reconciliation");
assert_eq!(
foreign_remaining,
vec![foreign_upload.upload_id],
"identity-scoped cleanup must preserve a foreign upload"
);
let pool_meta = store.pool_meta.read().await;
let reservation = pool_meta.pools[0]
.decommission
.as_ref()
.and_then(|info| info.capacity_reservation.as_ref())
.expect("zero-delta reconciliation must preserve the reservation");
assert_eq!(reservation.pending_target_physical_bytes, 0);
assert_eq!(reservation.inflight_target_physical_bytes, 0);
assert!(reservation.targets[0].temporary_mutations.is_empty());
drop(pool_meta);
store
.reconcile_multipart_uploads_for_data_movement(target_pool_index, &bucket, object, &exact_identity, &cleanup_opts)
.await
.expect("repeated cleanup should take the fast path after removing the marker");
assert_eq!(store.data_movement_multipart_discovery_count_for_test(), 1);
}
#[tokio::test]
#[serial_test::serial]
async fn multipart_cleanup_rediscovers_uploads_and_proves_target_under_capacity_gate() {
let (_temp_dirs, store, _other_store) = test_three_pool_stores_with_isolated_node_contexts(None).await;
let bucket = test_bucket("multipart-cleanup-target-race");
let object = "published-between-cleanup-proof-and-finalize.bin";
let body = vec![0x5a; 64 * 1024];
let version_id = uuid::Uuid::new_v4();
let mod_time = time::OffsetDateTime::UNIX_EPOCH + time::Duration::seconds(19);
let upload_identity = format!("v1:{version_id}:{}", mod_time.unix_timestamp_nanos());
store
.make_bucket(&bucket, &MakeBucketOptions::default())
.await
.expect("create the cleanup target-race bucket");
let incarnation = store
.bucket_incarnation_id(&bucket)
.await
.expect("load the cleanup target-race bucket incarnation");
let layout = DecommissionErasureLayout { data: 1, parity: 0 };
set_decommission_capacity_info_overrides_for_test(
store.id,
vec![vec![
DecommissionPoolCapacityInfo::for_test(0, layout, 0, body.len(), body.len()),
DecommissionPoolCapacityInfo::for_test(1, layout, 0, body.len().saturating_mul(2), body.len().saturating_mul(2)),
DecommissionPoolCapacityInfo::for_test(2, layout, body.len().saturating_mul(2), body.len().saturating_mul(2), 0),
]],
);
store
.save_current_pool_meta_for_decommission_start(&[0], Vec::new())
.await
.expect("activate the cleanup target-race reservation");
let target_pool_index = store.pool_meta.read().await.pools[0]
.decommission
.as_ref()
.and_then(|info| info.capacity_reservation.as_ref())
.and_then(|reservation| reservation.targets.first())
.expect("cleanup target-race fixture should reserve one target")
.pool_index;
assert_eq!(target_pool_index, 2);
let owner = decommission_capacity_owner(&*store.pool_meta.read().await).with_mutation_id(uuid::Uuid::new_v4());
let mut upload_opts = ObjectOptions {
data_movement: true,
src_pool_idx: 0,
versioned: true,
version_id: Some(version_id.to_string()),
mod_time: Some(mod_time),
expected_bucket_incarnation_id: Some(incarnation),
..Default::default()
};
rustfs_utils::http::insert_str(
&mut upload_opts.user_defined,
rustfs_utils::http::SUFFIX_DATA_MOVEMENT_UPLOAD,
upload_identity.clone(),
);
owner.apply_to(&mut upload_opts);
let concurrent_upload_opts = upload_opts.clone();
let mut cleanup_opts = ObjectOptions {
data_movement: true,
src_pool_idx: 0,
versioned: true,
version_id: Some(version_id.to_string()),
mod_time: Some(mod_time),
expected_bucket_incarnation_id: Some(incarnation),
..Default::default()
};
owner.apply_to(&mut cleanup_opts);
let staging_store = Arc::clone(&store);
let staging_bucket = bucket.clone();
store
.run_decommission_capacity_temporary_mutation(target_pool_index, Some(owner), None, || async move {
new_multipart_upload(&staging_store, target_pool_index, &staging_bucket, object, upload_opts).await?;
Err::<(), crate::error::Error>(crate::error::Error::OperationCanceled)
})
.await
.expect_err("the fixture should fail after staging its recoverable upload");
store
.run_decommission_capacity_temporary_mutation(target_pool_index, Some(owner), Some(body.len()), || async {
Err::<(), crate::error::Error>(crate::error::Error::OperationCanceled)
})
.await
.expect_err("the fixture should retain an exact pending capacity intent");
{
let pool_meta = store.pool_meta.read().await;
let reservation = pool_meta.pools[0]
.decommission
.as_ref()
.and_then(|info| info.capacity_reservation.as_ref())
.expect("cleanup target-race reservation should remain active");
assert!(reservation.pending_target_physical_bytes > 0);
assert_eq!(reservation.targets[0].pending_mutation_id, owner.mutation_id);
assert!(
reservation.targets[0]
.temporary_mutations
.iter()
.any(|mutation| mutation.mutation_id == owner.mutation_id.unwrap())
);
}
let target_lock = store
.new_ns_lock(
RUSTFS_META_BUCKET,
&format!("{DECOMMISSION_CAPACITY_TARGET_LOCK_PREFIX}/{target_pool_index}"),
)
.await
.expect("create the cleanup target-race gate");
let target_guard = target_lock
.get_write_lock(Duration::from_secs(30))
.await
.expect("hold the cleanup target-race gate");
let barrier = DecommissionCapacityLockOrderBarrier::install(store.id, store.id);
barrier.disable_owner_pause();
barrier.pause_target_gate_acquire(target_pool_index);
let mut cleanup = tokio::spawn({
let cleanup_store = Arc::clone(&store);
let cleanup_bucket = bucket.clone();
let cleanup_identity = upload_identity.clone();
async move {
cleanup_store
.reconcile_multipart_uploads_for_data_movement(
target_pool_index,
&cleanup_bucket,
object,
&cleanup_identity,
&cleanup_opts,
)
.await
}
});
tokio::select! {
_ = barrier.wait_until_target_gate_acquire_paused() => {}
result = &mut cleanup => panic!("cleanup finished before its target-gate proof point: {result:?}"),
}
let concurrent_upload = new_multipart_upload(&store, target_pool_index, &bucket, object, concurrent_upload_opts)
.await
.expect("stage a same-identity upload after cleanup reaches its target-gate boundary");
let mut target_data = PutObjReader::from_vec(body);
let published = store.pools[target_pool_index]
.put_object(
&bucket,
object,
&mut target_data,
&ObjectOptions {
data_movement: true,
src_pool_idx: 0,
versioned: true,
version_id: Some(version_id.to_string()),
mod_time: Some(mod_time),
expected_bucket_incarnation_id: Some(incarnation),
..Default::default()
},
)
.await
.expect("publish the exact owned target while cleanup waits for its capacity gate");
assert!(crate::data_movement::is_owned_data_movement_target(&published));
drop(target_guard);
barrier.release_target_gate_acquire();
tokio::time::timeout(Duration::from_secs(30), cleanup)
.await
.expect("cleanup should finish after acquiring its target gate")
.expect("cleanup task should not panic")
.expect("cleanup should preserve the now-published target intent");
drop(barrier);
let remaining_uploads = store.pools[target_pool_index]
.get_disks_by_key(object)
.data_movement_multipart_upload_ids(&bucket, object, Some(incarnation), &upload_identity)
.await
.expect("check same-identity uploads after gated cleanup");
assert!(
remaining_uploads.is_empty(),
"gated cleanup must rediscover and remove the concurrent same-identity upload {}",
concurrent_upload.upload_id
);
let pool_meta = store.pool_meta.read().await;
let reservation = pool_meta.pools[0]
.decommission
.as_ref()
.and_then(|info| info.capacity_reservation.as_ref())
.expect("cleanup target-race reservation should remain active");
assert!(
reservation.pending_target_physical_bytes > 0,
"cleanup must not clear capacity for a target published before its gated proof"
);
assert_eq!(reservation.targets[0].pending_mutation_id, owner.mutation_id);
assert!(reservation.targets[0].temporary_mutations.is_empty());
}
#[test]
#[serial_test::serial]
fn data_movement_multipart_restart_reconciles_published_capacity_before_new_upload() {
run_large_stack_current_thread_async_test(
"multipart-published-capacity-restart",
data_movement_multipart_restart_reconciles_published_capacity_before_new_upload_case,
);
}
async fn data_movement_multipart_restart_reconciles_published_capacity_before_new_upload_case() {
let (_temp_dirs, store, other_store) =
test_three_pool_stores_with_three_disk_sets_with_isolated_node_contexts(None).await;
let bucket = test_bucket("multipart-restart");
let object = "published-multipart-before-capacity-save.bin";
let next_object = "next-after-published-multipart-reconcile.bin";
let first_part = vec![0x61; 5 * 1024 * 1024];
let second_part = vec![0x62; 1];
let mut body = first_part.clone();
body.extend_from_slice(&second_part);
let source_version = uuid::Uuid::new_v4().to_string();
let next_version = uuid::Uuid::new_v4().to_string();
let source_mod_time = time::OffsetDateTime::UNIX_EPOCH + time::Duration::seconds(41);
store
.make_bucket(&bucket, &MakeBucketOptions::default())
.await
.expect("create the multipart restart reconciliation bucket");
let incarnation = store
.bucket_incarnation_id(&bucket)
.await
.expect("load the multipart restart reconciliation bucket incarnation");
let source_upload = new_multipart_upload(
&store,
0,
&bucket,
object,
ObjectOptions {
versioned: true,
version_id: Some(source_version.clone()),
mod_time: Some(source_mod_time),
expected_bucket_incarnation_id: Some(incarnation),
..Default::default()
},
)
.await
.expect("create the real source multipart upload");
let source_multipart_opts = multipart_options(
&store,
&bucket,
ObjectOptions {
versioned: true,
version_id: Some(source_version.clone()),
mod_time: Some(source_mod_time),
expected_bucket_incarnation_id: Some(incarnation),
..Default::default()
},
)
.await;
let mut source_parts = Vec::with_capacity(2);
for (part_num, part_body) in [(1, first_part.as_slice()), (2, second_part.as_slice())] {
let mut part_reader = PutObjReader::from_vec(part_body.to_vec());
let part = store.pools[0]
.put_object_part(
&bucket,
object,
&source_upload.upload_id,
part_num,
&mut part_reader,
&source_multipart_opts,
)
.await
.expect("stage the real source multipart part");
source_parts.push(crate::storage_api_contracts::multipart::CompletePart {
part_num: part.part_num,
etag: part.etag,
..Default::default()
});
}
let source_info = store.pools[0]
.clone()
.complete_multipart_upload(&bucket, object, &source_upload.upload_id, source_parts, &source_multipart_opts)
.await
.expect("complete the real source multipart object");
assert!(source_info.is_multipart(), "the source fixture must take the multipart migration path");
let layout = DecommissionErasureLayout { data: 1, parity: 0 };
let target_total = body.len().saturating_mul(4);
let capacity_snapshot = || {
vec![
DecommissionPoolCapacityInfo::for_test(0, layout, 0, body.len(), body.len()),
DecommissionPoolCapacityInfo::for_test(1, layout, 0, target_total, target_total),
DecommissionPoolCapacityInfo::for_test(2, layout, target_total, target_total, 0),
]
};
set_decommission_capacity_info_overrides_for_test(store.id, vec![capacity_snapshot()]);
store
.save_current_pool_meta_for_decommission_start(&[0], Vec::new())
.await
.expect("activate the source multipart capacity reservation");
*other_store.pool_meta.write().await = store.pool_meta.read().await.clone();
let owner = decommission_capacity_owner(&*store.pool_meta.read().await);
let source_reader = store.pools[0]
.get_object_reader(
&bucket,
object,
None,
HeaderMap::new(),
&ObjectOptions {
versioned: true,
version_id: Some(source_version.clone()),
no_lock: true,
data_movement: true,
raw_data_movement_read: true,
..Default::default()
},
)
.await
.expect("read the real source multipart object for migration");
let (lossy_store, refresh_calls) = store_with_capacity_lease_loss(&other_store).await;
set_decommission_capacity_info_overrides_for_test(lossy_store.id, (0..40).map(|_| capacity_snapshot()).collect());
let barrier = MultipartCommitBarrier::install(&bucket, object, MultipartCommitPause::AfterObjectPublication);
let mut migration = tokio::spawn({
let migration_store = Arc::clone(&lossy_store);
let migration_bucket = bucket.clone();
async move {
data_movement::migrate_decommission_object(
migration_store,
0,
migration_bucket,
source_reader,
Some(incarnation),
"multipart_restart_reconcile_initial",
Some(owner),
)
.await
}
});
tokio::select! {
() = barrier.wait_until_paused() => {}
result = &mut migration => {
panic!("initial multipart migration completed before reaching the publication barrier: {result:?}");
}
}
tokio::time::pause();
let published = lossy_store.pools[2]
.get_object_info(
&bucket,
object,
&ObjectOptions {
versioned: true,
version_id: Some(source_version.clone()),
no_lock: true,
..Default::default()
},
)
.await
.expect("the multipart target must publish before capacity progress save");
assert!(published.is_multipart(), "published target must retain multipart identity");
assert_eq!(published.version_id.map(|version| version.to_string()), Some(source_version.clone()));
for prefix in [
rustfs_utils::http::RUSTFS_INTERNAL_PREFIX,
rustfs_utils::http::MINIO_INTERNAL_PREFIX,
] {
assert_eq!(
published
.user_defined
.get(&format!("{prefix}{}", rustfs_utils::http::SUFFIX_MULTIPART_UPLOAD_ID)),
Some(&source_upload.upload_id),
"migration must preserve the source completion identity under both internal prefixes"
);
}
let mut published_reader = lossy_store.pools[2]
.get_object_reader(
&bucket,
object,
None,
HeaderMap::new(),
&ObjectOptions {
versioned: true,
version_id: Some(source_version.clone()),
no_lock: true,
..Default::default()
},
)
.await
.expect("the published multipart target should be readable before reconciliation");
let mut published_body = Vec::new();
published_reader
.stream
.read_to_end(&mut published_body)
.await
.expect("the published multipart target body should be readable");
assert_eq!(published_body, body, "the published multipart target must retain the full source body");
refresh_calls.arm();
tokio::time::advance(Duration::from_secs(11)).await;
tokio::task::yield_now().await;
assert!(
refresh_calls.load(Ordering::Acquire) > 0,
"the final multipart capacity save must observe its lost lease"
);
barrier.release();
tokio::time::resume();
let initial_err = tokio::time::timeout(Duration::from_secs(30), migration)
.await
.expect("the initial multipart migration should finish after publication release")
.expect("the initial multipart migration task should not panic")
.expect_err("the published multipart target must report its failed capacity progress save");
assert!(
!initial_err.to_string().is_empty(),
"the failed multipart capacity save must remain observable"
);
drop(barrier);
let mut failed_persisted = crate::core::pools::PoolMeta::default();
failed_persisted
.load_no_lock_from_replicas(lossy_store.pools.clone())
.await
.expect("the published multipart failure must leave readable durable metadata");
let failed_reservation = failed_persisted.pools[0]
.decommission
.as_ref()
.and_then(|info| info.capacity_reservation.as_ref())
.expect("the failed multipart capacity intent must remain durable");
assert_eq!(
failed_reservation.pending_target_physical_bytes,
body.len(),
"the published target must retain its durable capacity intent for restart reconciliation"
);
assert_eq!(failed_reservation.inflight_target_physical_bytes, 0);
assert_eq!(failed_reservation.consumed_target_physical_bytes, 0);
*other_store.pool_meta.write().await = failed_persisted.clone();
let retry_owner = decommission_capacity_owner(&failed_persisted);
let new_upload_observation = NewMultipartUploadCommitObservation::install(&bucket, object);
let retry_reader = other_store.pools[0]
.get_object_reader(
&bucket,
object,
None,
HeaderMap::new(),
&ObjectOptions {
versioned: true,
version_id: Some(source_version.clone()),
no_lock: true,
data_movement: true,
raw_data_movement_read: true,
..Default::default()
},
)
.await
.expect("restart should reread the exact multipart source version");
tokio::time::timeout(
Duration::from_secs(30),
data_movement::migrate_decommission_object(
Arc::clone(&other_store),
0,
bucket.clone(),
retry_reader,
Some(incarnation),
"multipart_restart_reconcile_retry",
Some(retry_owner),
),
)
.await
.expect("published multipart retry should not deadlock")
.expect("published multipart retry should reconcile its durable capacity intent");
assert!(
!new_upload_observation.committed(),
"published multipart retry must reconcile before starting a new MPU"
);
let mut reconciled = crate::core::pools::PoolMeta::default();
reconciled
.load_no_lock_from_replicas(other_store.pools.clone())
.await
.expect("reconciled multipart capacity metadata should remain durable");
let reconciled_reservation = reconciled.pools[0]
.decommission
.as_ref()
.and_then(|info| info.capacity_reservation.as_ref())
.expect("reconciled multipart capacity reservation should remain present");
assert_eq!(reconciled_reservation.pending_target_physical_bytes, 0);
assert_eq!(reconciled_reservation.inflight_target_physical_bytes, 0);
assert_eq!(reconciled_reservation.consumed_target_physical_bytes, body.len());
assert_eq!(reconciled_reservation.committed_data_bytes, body.len());
let mut next_data = PutObjReader::from_vec(body.clone());
other_store.pools[0]
.put_object(
&bucket,
next_object,
&mut next_data,
&ObjectOptions {
versioned: true,
version_id: Some(next_version.clone()),
expected_bucket_incarnation_id: Some(incarnation),
..Default::default()
},
)
.await
.expect("seed the next source object for the multipart capacity oracle");
let next_reader = other_store.pools[0]
.get_object_reader(
&bucket,
next_object,
None,
HeaderMap::new(),
&ObjectOptions {
versioned: true,
version_id: Some(next_version.clone()),
no_lock: true,
data_movement: true,
raw_data_movement_read: true,
..Default::default()
},
)
.await
.expect("read the next source object for the multipart capacity oracle");
let next_result = tokio::time::timeout(
Duration::from_secs(30),
data_movement::migrate_decommission_object(
Arc::clone(&other_store),
0,
bucket.clone(),
next_reader,
Some(incarnation),
"multipart_restart_reconcile_next",
Some(retry_owner),
),
)
.await
.expect("the next-object multipart capacity oracle should finish promptly");
assert!(next_result.is_err(), "consumed multipart capacity must reject the next exact-fit object");
let next_target_err = other_store.pools[2]
.get_object_info(
&bucket,
next_object,
&ObjectOptions {
versioned: true,
version_id: Some(next_version),
no_lock: true,
..Default::default()
},
)
.await
.expect_err("the next object must not publish after multipart reconciliation");
assert!(
crate::error::is_err_object_not_found(&next_target_err) || crate::error::is_err_version_not_found(&next_target_err),
"capacity rejection must leave the next multipart target absent: {next_target_err}"
);
}
#[test]
#[serial_test::serial]
fn data_movement_multipart_restart_cleans_partial_upload_before_exact_fit_retry() {
run_large_stack_current_thread_async_test(
"multipart-partial-upload-restart",
data_movement_multipart_restart_cleans_partial_upload_before_exact_fit_retry_case,
);
}
async fn data_movement_multipart_restart_cleans_partial_upload_before_exact_fit_retry_case() {
let (_temp_dirs, store, other_store) = test_three_pool_stores_with_isolated_node_contexts(None).await;
let bucket = test_bucket("multipart-part-restart");
let object = "published-part-before-capacity-save.bin";
let first_part = vec![0x71; 5 * 1024 * 1024];
let second_part = vec![0x72; 1];
let mut body = first_part.clone();
body.extend_from_slice(&second_part);
let source_version = uuid::Uuid::new_v4().to_string();
let source_mod_time = time::OffsetDateTime::UNIX_EPOCH + time::Duration::seconds(71);
store
.make_bucket(&bucket, &MakeBucketOptions::default())
.await
.expect("create the partial multipart restart bucket");
let incarnation = store
.bucket_incarnation_id(&bucket)
.await
.expect("load the partial multipart restart bucket incarnation");
let source_upload = new_multipart_upload(
&store,
0,
&bucket,
object,
ObjectOptions {
versioned: true,
version_id: Some(source_version.clone()),
mod_time: Some(source_mod_time),
expected_bucket_incarnation_id: Some(incarnation),
..Default::default()
},
)
.await
.expect("create the partial-restart source upload");
let source_multipart_opts = multipart_options(
&store,
&bucket,
ObjectOptions {
versioned: true,
version_id: Some(source_version.clone()),
mod_time: Some(source_mod_time),
expected_bucket_incarnation_id: Some(incarnation),
..Default::default()
},
)
.await;
let mut source_parts = Vec::with_capacity(2);
for (part_num, part_body) in [(1, first_part.as_slice()), (2, second_part.as_slice())] {
let mut reader = PutObjReader::from_vec(part_body.to_vec());
let part = store.pools[0]
.put_object_part(&bucket, object, &source_upload.upload_id, part_num, &mut reader, &source_multipart_opts)
.await
.expect("stage the partial-restart source part");
source_parts.push(crate::storage_api_contracts::multipart::CompletePart {
part_num: part.part_num,
etag: part.etag,
..Default::default()
});
}
let source_info = store.pools[0]
.clone()
.complete_multipart_upload(&bucket, object, &source_upload.upload_id, source_parts, &source_multipart_opts)
.await
.expect("complete the partial-restart source object");
assert!(source_info.is_multipart(), "the fixture must use multipart data movement");
let layout = DecommissionErasureLayout { data: 1, parity: 0 };
let source_estimate = body.len().saturating_add(1);
let target_total = source_estimate.saturating_mul(2);
let capacity_snapshot = || {
vec![
DecommissionPoolCapacityInfo::for_test(0, layout, 0, source_estimate, source_estimate),
DecommissionPoolCapacityInfo::for_test(1, layout, 0, target_total, target_total),
DecommissionPoolCapacityInfo::for_test(2, layout, target_total, target_total, 0),
]
};
set_decommission_capacity_info_overrides_for_test(store.id, vec![capacity_snapshot()]);
store
.save_current_pool_meta_for_decommission_start(&[0], Vec::new())
.await
.expect("activate the exact-fit partial multipart reservation");
*other_store.pool_meta.write().await = store.pool_meta.read().await.clone();
let owner = decommission_capacity_owner(&*store.pool_meta.read().await);
let source_reader = store.pools[0]
.get_object_reader(
&bucket,
object,
None,
HeaderMap::new(),
&ObjectOptions {
versioned: true,
version_id: Some(source_version.clone()),
no_lock: true,
data_movement: true,
raw_data_movement_read: true,
..Default::default()
},
)
.await
.expect("read the partial-restart source object");
let (lossy_store, refresh_calls) = store_with_capacity_lease_loss(&other_store).await;
set_decommission_capacity_info_overrides_for_test(lossy_store.id, (0..40).map(|_| capacity_snapshot()).collect());
let part_barrier = MultipartCommitBarrier::install(&bucket, object, MultipartCommitPause::PutPartAfterRename);
let abort_barrier = data_movement::DataMovementMultipartAbortBarrier::install(&bucket, object);
let migration = tokio::spawn({
let migration_store = Arc::clone(&lossy_store);
let migration_bucket = bucket.clone();
async move {
data_movement::migrate_decommission_object(
migration_store,
0,
migration_bucket,
source_reader,
Some(incarnation),
"multipart_partial_restart_initial",
Some(owner),
)
.await
}
});
part_barrier.wait_until_paused().await;
tokio::time::pause();
tokio::task::yield_now().await;
refresh_calls.arm();
tokio::time::advance(Duration::from_secs(11)).await;
tokio::task::yield_now().await;
assert!(
refresh_calls.load(Ordering::Acquire) > 0,
"the published UploadPart capacity save must lose its lease"
);
part_barrier.release();
abort_barrier.wait_until_paused().await;
migration.abort();
let _ = migration.await;
drop(abort_barrier);
drop(part_barrier);
tokio::time::resume();
let upload_identity = format!("v1:{source_version}:{}", source_mod_time.unix_timestamp_nanos());
let stale_uploads = lossy_store.pools[2]
.get_disks_by_key(object)
.data_movement_multipart_upload_ids(&bucket, object, Some(incarnation), &upload_identity)
.await
.expect("discover the published partial upload after the simulated crash");
assert_eq!(stale_uploads.len(), 1, "the failed UploadPart must leave exactly one recoverable upload");
let mut failed_persisted = crate::core::pools::PoolMeta::default();
failed_persisted
.load_no_lock_from_replicas(lossy_store.pools.clone())
.await
.expect("load the durable partial UploadPart capacity intent");
let failed_reservation = failed_persisted.pools[0]
.decommission
.as_ref()
.and_then(|info| info.capacity_reservation.as_ref())
.expect("the partial UploadPart reservation should remain durable");
assert!(failed_reservation.pending_target_physical_bytes > 0);
assert_eq!(failed_reservation.consumed_target_physical_bytes, 0);
*other_store.pool_meta.write().await = failed_persisted.clone();
set_decommission_capacity_info_overrides_for_test(other_store.id, (0..80).map(|_| capacity_snapshot()).collect());
let retry_owner = decommission_capacity_owner(&failed_persisted);
let retry_reader = other_store.pools[0]
.get_object_reader(
&bucket,
object,
None,
HeaderMap::new(),
&ObjectOptions {
versioned: true,
version_id: Some(source_version.clone()),
no_lock: true,
data_movement: true,
raw_data_movement_read: true,
..Default::default()
},
)
.await
.expect("restart should reread the exact partial multipart source");
let new_upload = NewMultipartUploadCommitObservation::install(&bucket, object);
let retry_barrier = MultipartCommitBarrier::install(&bucket, object, MultipartCommitPause::NewUploadBeforeLockLost);
let mut retry = tokio::spawn({
let retry_store = Arc::clone(&other_store);
let retry_bucket = bucket.clone();
async move {
data_movement::migrate_decommission_object(
retry_store,
0,
retry_bucket,
retry_reader,
Some(incarnation),
"multipart_partial_restart_retry",
Some(retry_owner),
)
.await
}
});
tokio::select! {
_ = retry_barrier.wait_until_paused() => {}
result = &mut retry => {
panic!("partial multipart retry completed before reaching the new-upload barrier: {result:?}");
}
}
let uploads_before_retry = other_store.pools[2]
.get_disks_by_key(object)
.data_movement_multipart_upload_ids(&bucket, object, Some(incarnation), &upload_identity)
.await
.expect("scan for the old partial upload before the replacement upload commits");
assert!(
uploads_before_retry.is_empty(),
"restart must delete the old partial upload before publishing a replacement upload"
);
retry_barrier.release();
tokio::time::timeout(Duration::from_secs(30), retry)
.await
.expect("the exact-fit partial multipart retry should not deadlock")
.expect("the exact-fit partial multipart retry task should not panic")
.expect("the exact-fit partial multipart retry should converge");
drop(retry_barrier);
assert!(new_upload.committed(), "restart must create a fresh upload after cleaning the old one");
let remaining_uploads = other_store.pools[2]
.get_disks_by_key(object)
.data_movement_multipart_upload_ids(&bucket, object, Some(incarnation), &upload_identity)
.await
.expect("scan for residual partial uploads after restart convergence");
assert!(
remaining_uploads.is_empty(),
"restart must remove the original partial upload before retrying"
);
let mut reconciled = crate::core::pools::PoolMeta::default();
reconciled
.load_no_lock_from_replicas(other_store.pools.clone())
.await
.expect("reload exact-fit multipart progress after restart");
let reservation = reconciled.pools[0]
.decommission
.as_ref()
.and_then(|info| info.capacity_reservation.as_ref())
.expect("the exact-fit multipart reservation should remain readable");
assert_eq!(reservation.pending_target_physical_bytes, 0);
assert_eq!(reservation.inflight_target_physical_bytes, 0);
assert_eq!(reservation.consumed_target_physical_bytes, body.len());
assert_eq!(reservation.committed_data_bytes, body.len());
let mut target = other_store.pools[2]
.get_object_reader(
&bucket,
object,
None,
HeaderMap::new(),
&ObjectOptions {
versioned: true,
version_id: Some(source_version),
no_lock: true,
..Default::default()
},
)
.await
.expect("the exact-fit retry target should be readable");
let mut target_body = Vec::new();
target
.stream
.read_to_end(&mut target_body)
.await
.expect("read the exact-fit retry target body");
assert_eq!(target_body, body);
}
#[tokio::test]
#[serial_test::serial]
async fn data_movement_multipart_abort_fences_capacity_lease_loss_before_delete() {
let (_temp_dirs, store, other_store) = test_three_pool_stores_with_isolated_node_contexts(None).await;
let bucket = test_bucket("multipart-abort-fence");
let object = "abort-capacity-lease-before-delete.bin";
let layout = DecommissionErasureLayout { data: 1, parity: 0 };
let capacity_snapshot = || {
vec![
DecommissionPoolCapacityInfo::for_test(0, layout, 0, 2, 2),
DecommissionPoolCapacityInfo::for_test(1, layout, 0, 4, 4),
DecommissionPoolCapacityInfo::for_test(2, layout, 4, 4, 0),
]
};
store
.make_bucket(&bucket, &MakeBucketOptions::default())
.await
.expect("create the abort fence bucket");
let incarnation = store
.bucket_incarnation_id(&bucket)
.await
.expect("load the abort fence incarnation");
set_decommission_capacity_info_overrides_for_test(store.id, vec![capacity_snapshot()]);
store
.save_current_pool_meta_for_decommission_start(&[0], Vec::new())
.await
.expect("activate the abort fence reservation");
let owner = decommission_capacity_owner(&*store.pool_meta.read().await).with_mutation_id(uuid::Uuid::new_v4());
let version_id = uuid::Uuid::new_v4().to_string();
let mod_time = time::OffsetDateTime::UNIX_EPOCH + time::Duration::seconds(83);
let mut upload_opts = ObjectOptions {
data_movement: true,
src_pool_idx: 0,
versioned: true,
version_id: Some(version_id.clone()),
mod_time: Some(mod_time),
expected_bucket_incarnation_id: Some(incarnation),
..Default::default()
};
let upload_identity = data_movement::data_movement_upload_identity_from_options(&upload_opts);
rustfs_utils::http::insert_str(
&mut upload_opts.user_defined,
rustfs_utils::http::SUFFIX_DATA_MOVEMENT_UPLOAD,
upload_identity,
);
let upload = new_multipart_upload(&store, 2, &bucket, object, upload_opts)
.await
.expect("create the abort fence staging upload");
*other_store.pool_meta.write().await = store.pool_meta.read().await.clone();
let (lossy_store, refresh_calls) = store_with_capacity_lease_loss(&other_store).await;
set_decommission_capacity_info_overrides_for_test(lossy_store.id, vec![capacity_snapshot(), capacity_snapshot()]);
let mut abort_opts = ObjectOptions {
data_movement: true,
src_pool_idx: 0,
versioned: true,
version_id: Some(version_id),
mod_time: Some(mod_time),
expected_bucket_incarnation_id: Some(incarnation),
..Default::default()
};
owner.apply_to(&mut abort_opts);
let barrier = MultipartCommitBarrier::install(&bucket, object, MultipartCommitPause::AbortBeforeDelete);
tokio::time::pause();
let abort = tokio::spawn({
let abort_store = Arc::clone(&lossy_store);
let abort_bucket = bucket.clone();
let abort_upload_id = upload.upload_id.clone();
let abort_opts = abort_opts.clone();
async move {
abort_store
.abort_multipart_upload_for_data_movement(2, &abort_bucket, object, &abort_upload_id, &abort_opts)
.await
}
});
barrier.wait_until_paused().await;
tokio::task::yield_now().await;
refresh_calls.arm();
tokio::time::advance(Duration::from_secs(11)).await;
tokio::task::yield_now().await;
assert!(
refresh_calls.load(Ordering::Acquire) > 0,
"the abort delete must observe capacity lease refresh loss"
);
barrier.release();
let err = tokio::time::timeout(Duration::from_secs(30), abort)
.await
.expect("the fenced abort should finish after release")
.expect("the fenced abort task should not panic")
.expect_err("lost capacity lease must reject the staging delete");
assert!(!err.to_string().is_empty());
drop(barrier);
tokio::time::resume();
let upload_info = lossy_store.pools[2]
.list_object_parts(
&bucket,
object,
&upload.upload_id,
None,
100,
&ObjectOptions {
data_movement: true,
expected_bucket_incarnation_id: Some(incarnation),
..Default::default()
},
)
.await
.expect("the fenced abort must leave the staging upload readable");
assert!(upload_info.parts.is_empty());
}
#[test]
#[serial_test::serial]
fn unversioned_decommission_multipart_uses_reserved_target() {
run_large_stack_current_thread_async_test(
"unversioned-decommission-multipart-target",
unversioned_decommission_multipart_uses_reserved_target_case,
);
}
async fn unversioned_decommission_multipart_uses_reserved_target_case() {
let (_temp_dirs, store, _other_store) = test_three_pool_stores_with_isolated_node_contexts(None).await;
let bucket = test_bucket("unversioned-decommission-multipart-target");
let object = "unversioned-existing-unreserved-target.bin";
let layout = DecommissionErasureLayout { data: 1, parity: 0 };
let capacity_snapshot = || {
vec![
DecommissionPoolCapacityInfo::for_test(0, layout, 0, 1, 1),
DecommissionPoolCapacityInfo::for_test(1, layout, 2, 2, 0),
DecommissionPoolCapacityInfo::for_test(2, layout, 100, 100, 0),
]
};
store
.make_bucket(&bucket, &MakeBucketOptions::default())
.await
.expect("create the unversioned multipart target bucket");
let incarnation = store
.bucket_incarnation_id(&bucket)
.await
.expect("load the unversioned multipart target incarnation");
let mut existing = PutObjReader::from_vec(b"existing unreserved target".to_vec());
store.pools[2]
.put_object(
&bucket,
object,
&mut existing,
&ObjectOptions {
expected_bucket_incarnation_id: Some(incarnation),
..Default::default()
},
)
.await
.expect("seed the unreserved target object");
set_decommission_capacity_info_overrides_for_test(store.id, vec![capacity_snapshot()]);
store
.save_current_pool_meta_for_decommission_start(&[0], Vec::new())
.await
.expect("activate the unversioned multipart reservation");
{
let pool_meta = store.pool_meta.read().await;
let targets = &pool_meta.pools[0]
.decommission
.as_ref()
.and_then(|info| info.capacity_reservation.as_ref())
.expect("the unversioned multipart reservation should exist")
.targets;
assert_eq!(targets.len(), 1, "only pool 1 should be reserved");
assert_eq!(targets[0].pool_index, 1);
}
let owner = decommission_capacity_owner(&*store.pool_meta.read().await).with_mutation_id(uuid::Uuid::new_v4());
let mut upload_opts = ObjectOptions {
data_movement: true,
src_pool_idx: 0,
versioned: false,
version_id: None,
mod_time: Some(time::OffsetDateTime::UNIX_EPOCH + time::Duration::seconds(97)),
expected_bucket_incarnation_id: Some(incarnation),
..Default::default()
};
owner.apply_to(&mut upload_opts);
let (_, target_pool_idx, _) = store
.handle_new_multipart_upload_with_pool_idx(&bucket, object, &upload_opts, None)
.await
.expect("unversioned decommission multipart must use the reserved target");
assert_eq!(target_pool_idx, 1, "the reservation target must override the existing unreserved object");
}
#[test]
#[serial_test::serial]
fn data_movement_multipart_abort_restart_reconciles_inflight_after_release_save_loss() {
run_large_stack_current_thread_async_test(
"multipart-abort-restart",
data_movement_multipart_abort_restart_reconciles_inflight_after_release_save_loss_case,
);
}
async fn data_movement_multipart_abort_restart_reconciles_inflight_after_release_save_loss_case() {
let (_temp_dirs, store, other_store) = test_three_pool_stores_with_isolated_node_contexts(None).await;
let bucket = test_bucket("multipart-abort-restart");
let object = "deleted-upload-before-release-save.bin";
let body = vec![0x79; 256 * 1024];
let layout = DecommissionErasureLayout { data: 1, parity: 0 };
let source_estimate = body.len().saturating_add(1);
let target_total = source_estimate.saturating_mul(2);
let capacity_snapshot = |target_free| {
vec![
DecommissionPoolCapacityInfo::for_test(0, layout, 0, source_estimate, source_estimate),
DecommissionPoolCapacityInfo::for_test(1, layout, 0, target_total, target_total),
DecommissionPoolCapacityInfo::for_test(2, layout, target_free, target_total, target_total - target_free),
]
};
store
.make_bucket(&bucket, &MakeBucketOptions::default())
.await
.expect("create the abort restart bucket");
let incarnation = store
.bucket_incarnation_id(&bucket)
.await
.expect("load the abort restart incarnation");
set_decommission_capacity_info_overrides_for_test(store.id, vec![capacity_snapshot(target_total)]);
store
.save_current_pool_meta_for_decommission_start(&[0], Vec::new())
.await
.expect("activate the abort restart reservation");
let owner = decommission_capacity_owner(&*store.pool_meta.read().await).with_mutation_id(uuid::Uuid::new_v4());
let version_id = uuid::Uuid::new_v4().to_string();
let mod_time = time::OffsetDateTime::UNIX_EPOCH + time::Duration::seconds(91);
let mut new_opts = ObjectOptions {
data_movement: true,
src_pool_idx: 0,
versioned: true,
version_id: Some(version_id.clone()),
mod_time: Some(mod_time),
expected_bucket_incarnation_id: Some(incarnation),
..Default::default()
};
let upload_identity = data_movement::data_movement_upload_identity_from_options(&new_opts);
rustfs_utils::http::insert_str(
&mut new_opts.user_defined,
rustfs_utils::http::SUFFIX_DATA_MOVEMENT_UPLOAD,
upload_identity,
);
owner.apply_to(&mut new_opts);
set_decommission_capacity_info_overrides_for_test(
store.id,
vec![
capacity_snapshot(target_total),
capacity_snapshot(target_total),
capacity_snapshot(target_total - 1),
],
);
let (upload, target_pool_idx, _) = store
.handle_new_multipart_upload_with_pool_idx(&bucket, object, &new_opts, None)
.await
.expect("create the capacity-accounted data movement upload");
assert_eq!(target_pool_idx, 2);
let mut part_opts = ObjectOptions {
data_movement: true,
src_pool_idx: 0,
part_number: Some(1),
expected_bucket_incarnation_id: Some(incarnation),
..Default::default()
};
owner.apply_to(&mut part_opts);
set_decommission_capacity_info_overrides_for_test(
store.id,
vec![
capacity_snapshot(target_total - 1),
capacity_snapshot(target_total - 1),
capacity_snapshot(target_total - 1 - body.len()),
],
);
let mut part_reader = PutObjReader::from_vec(body.clone());
store
.put_object_part_for_data_movement(target_pool_idx, &bucket, object, &upload.upload_id, &mut part_reader, &part_opts)
.await
.expect("persist the scoped temporary multipart inflight bytes");
let persisted_before_abort = store.pool_meta.read().await.clone();
let before_reservation = persisted_before_abort.pools[0]
.decommission
.as_ref()
.and_then(|info| info.capacity_reservation.as_ref())
.expect("the abort fixture reservation should exist");
assert_eq!(before_reservation.pending_target_physical_bytes, 0);
assert_eq!(before_reservation.inflight_target_physical_bytes, body.len() + 1);
assert_eq!(before_reservation.targets[0].temporary_mutations.len(), 1);
assert_eq!(
before_reservation.targets[0].temporary_mutations[0].mutation_id,
owner
.mutation_id
.expect("the abort fixture owner should have a mutation identity")
);
*other_store.pool_meta.write().await = persisted_before_abort;
let (lossy_store, refresh_calls) = store_with_capacity_lease_loss(&other_store).await;
set_decommission_capacity_info_overrides_for_test(
lossy_store.id,
vec![
capacity_snapshot(target_total - 1 - body.len()),
capacity_snapshot(target_total - 1 - body.len()),
],
);
let mut abort_opts = ObjectOptions {
data_movement: true,
src_pool_idx: 0,
versioned: true,
version_id: Some(version_id),
mod_time: Some(mod_time),
expected_bucket_incarnation_id: Some(incarnation),
..Default::default()
};
owner.apply_to(&mut abort_opts);
let barrier = MultipartCommitBarrier::install(&bucket, object, MultipartCommitPause::AbortAfterDelete);
tokio::time::pause();
let abort = tokio::spawn({
let abort_store = Arc::clone(&lossy_store);
let abort_bucket = bucket.clone();
let abort_object = object.to_string();
let abort_upload_id = upload.upload_id.clone();
let abort_opts = abort_opts.clone();
async move {
abort_store
.abort_multipart_upload_for_data_movement(
target_pool_idx,
&abort_bucket,
&abort_object,
&abort_upload_id,
&abort_opts,
)
.await
}
});
barrier.wait_until_paused().await;
tokio::task::yield_now().await;
refresh_calls.arm();
tokio::time::advance(Duration::from_secs(11)).await;
tokio::task::yield_now().await;
assert!(
refresh_calls.load(Ordering::Acquire) > 0,
"the abort release save must observe capacity lease loss after deletion"
);
barrier.release();
let abort_err = tokio::time::timeout(Duration::from_secs(30), abort)
.await
.expect("the fenced abort should finish after release")
.expect("the fenced abort task should not panic")
.expect_err("the deleted upload must report its lost final capacity save");
assert!(!abort_err.to_string().is_empty());
drop(barrier);
tokio::time::resume();
let missing = lossy_store.pools[target_pool_idx]
.list_object_parts(
&bucket,
object,
&upload.upload_id,
None,
100,
&ObjectOptions {
data_movement: true,
..Default::default()
},
)
.await
.expect_err("the first abort must durably delete the staging upload");
assert!(crate::error::is_err_invalid_upload_id(&missing));
let mut failed_persisted = crate::core::pools::PoolMeta::default();
failed_persisted
.load_no_lock_from_replicas(lossy_store.pools.clone())
.await
.expect("reload inflight capacity after the abort release save loss");
let failed_reservation = failed_persisted.pools[0]
.decommission
.as_ref()
.and_then(|info| info.capacity_reservation.as_ref())
.expect("the failed abort reservation should remain durable");
assert_eq!(failed_reservation.inflight_target_physical_bytes, body.len() + 1);
assert_eq!(failed_reservation.targets[0].temporary_mutations.len(), 1);
assert_eq!(
failed_reservation.targets[0].temporary_mutations[0].mutation_id,
owner
.mutation_id
.expect("the failed abort owner should retain its mutation identity")
);
*other_store.pool_meta.write().await = failed_persisted;
set_decommission_capacity_info_overrides_for_test(
other_store.id,
vec![
capacity_snapshot(target_total - 1 - body.len()),
capacity_snapshot(target_total - 1 - body.len()),
],
);
other_store
.abort_multipart_upload_for_data_movement(target_pool_idx, &bucket, object, &upload.upload_id, &abort_opts)
.await
.expect("restart must treat the missing upload as a confirmed scoped release");
let mut reconciled = crate::core::pools::PoolMeta::default();
reconciled
.load_no_lock_from_replicas(other_store.pools.clone())
.await
.expect("reload capacity after idempotent abort reconciliation");
let reservation = reconciled.pools[0]
.decommission
.as_ref()
.and_then(|info| info.capacity_reservation.as_ref())
.expect("the reconciled abort reservation should remain readable");
assert_eq!(reservation.pending_target_physical_bytes, 0);
assert_eq!(reservation.inflight_target_physical_bytes, 0);
assert!(reservation.targets[0].temporary_mutations.is_empty());
assert_eq!(reservation.consumed_target_physical_bytes, 0);
}
#[tokio::test]
#[serial_test::serial]
async fn public_complete_multipart_fences_inner_capacity_lease_loss() {
let (_temp_dirs, store, other_store) = test_three_pool_stores_with_isolated_node_contexts(None).await;
let bucket = test_bucket("public-complete-loss");
let object = "public-complete-lease-loss.bin";
store
.make_bucket(&bucket, &MakeBucketOptions::default())
.await
.expect("create public CompleteMultipartUpload lease-loss bucket");
let incarnation = store
.bucket_incarnation_id(&bucket)
.await
.expect("load public CompleteMultipartUpload bucket incarnation");
let upload = new_multipart_upload(
&store,
2,
&bucket,
object,
ObjectOptions {
expected_bucket_incarnation_id: Some(incarnation),
..Default::default()
},
)
.await
.expect("create public CompleteMultipartUpload lease-loss upload");
let multipart_opts = multipart_options(
&store,
&bucket,
ObjectOptions {
expected_bucket_incarnation_id: Some(incarnation),
..Default::default()
},
)
.await;
let body = b"public complete staged part".to_vec();
let mut data = PutObjReader::from_vec(body.clone());
let part = store.pools[2]
.put_object_part(&bucket, object, &upload.upload_id, 1, &mut data, &multipart_opts)
.await
.expect("stage public CompleteMultipartUpload lease-loss part");
let completed_parts = vec![crate::storage_api_contracts::multipart::CompletePart {
part_num: part.part_num,
etag: part.etag,
..Default::default()
}];
let layout = DecommissionErasureLayout { data: 1, parity: 0 };
let target_total = body.len().saturating_mul(4);
let capacity_snapshot = || {
vec![
DecommissionPoolCapacityInfo::for_test(0, layout, 0, body.len(), body.len()),
DecommissionPoolCapacityInfo::for_test(1, layout, target_total, target_total, 0),
DecommissionPoolCapacityInfo::for_test(2, layout, 0, target_total, target_total),
]
};
set_decommission_capacity_info_overrides_for_test(store.id, vec![capacity_snapshot()]);
store
.save_current_pool_meta_for_decommission_start(&[0], Vec::new())
.await
.expect("activate the public CompleteMultipartUpload capacity reservation");
*other_store.pool_meta.write().await = store.pool_meta.read().await.clone();
let (lossy_store, refresh_calls) = store_with_capacity_lease_loss(&other_store).await;
set_decommission_capacity_info_overrides_for_test(
lossy_store.id,
vec![capacity_snapshot(), capacity_snapshot(), capacity_snapshot()],
);
let barrier = MultipartCommitBarrier::install(&bucket, object, MultipartCommitPause::BeforeQuotaRename);
tokio::time::pause();
let complete_store = Arc::clone(&lossy_store);
let complete_bucket = bucket.clone();
let complete_upload_id = upload.upload_id.clone();
let complete = tokio::spawn(async move {
complete_store
.complete_multipart_upload(
&complete_bucket,
object,
&complete_upload_id,
completed_parts,
&ObjectOptions {
expected_bucket_incarnation_id: Some(incarnation),
..Default::default()
},
)
.await
});
barrier.wait_until_paused().await;
tokio::task::yield_now().await;
refresh_calls.arm();
tokio::time::advance(Duration::from_secs(11)).await;
tokio::task::yield_now().await;
assert!(
refresh_calls.load(Ordering::Acquire) > 0,
"the inner CompleteMultipartUpload capacity guard must lose refresh quorum"
);
barrier.release();
let err = tokio::time::timeout(Duration::from_secs(30), complete)
.await
.expect("public CompleteMultipartUpload should finish after the commit barrier release")
.expect("public CompleteMultipartUpload task should not panic")
.expect_err("lost inner capacity lease must fence CompleteMultipartUpload before rename");
assert!(
err.to_string().contains("quota_reservation"),
"inner capacity lease loss should report a quota reservation fence error: {err}"
);
drop(barrier);
let object_err = lossy_store
.get_object_info(
&bucket,
object,
&ObjectOptions {
expected_bucket_incarnation_id: Some(incarnation),
..Default::default()
},
)
.await
.expect_err("failed CompleteMultipartUpload must not publish the object");
assert!(
crate::error::is_err_object_not_found(&object_err),
"failed CompleteMultipartUpload should leave the target object absent: {object_err}"
);
lossy_store
.get_multipart_info(
&bucket,
object,
&upload.upload_id,
&ObjectOptions {
expected_bucket_incarnation_id: Some(incarnation),
..Default::default()
},
)
.await
.expect("failed CompleteMultipartUpload must leave the MPU retryable");
let listed = lossy_store
.list_object_parts(
&bucket,
object,
&upload.upload_id,
None,
100,
&ObjectOptions {
expected_bucket_incarnation_id: Some(incarnation),
..Default::default()
},
)
.await
.expect("failed CompleteMultipartUpload should preserve the staged part");
assert_eq!(listed.parts.len(), 1, "failed CompleteMultipartUpload must preserve the MPU part");
}
#[tokio::test]
#[serial_test::serial]
async fn data_movement_complete_fences_capacity_lease_loss_before_publication() {
let (_temp_dirs, store, other_store) = test_three_pool_stores_with_isolated_node_contexts(None).await;
let object = "data-movement-complete-lease-loss.bin";
let body = vec![0x4d; 64];
let layout = DecommissionErasureLayout { data: 1, parity: 0 };
let capacity_snapshot = || {
vec![
DecommissionPoolCapacityInfo::for_test(0, layout, 0, 1024, 1024),
DecommissionPoolCapacityInfo::for_test(1, layout, 0, 4096, 4096),
DecommissionPoolCapacityInfo::for_test(2, layout, 4096, 4096, 0),
]
};
set_decommission_capacity_info_overrides_for_test(store.id, vec![capacity_snapshot()]);
store
.save_current_pool_meta_for_decommission_start(&[0], Vec::new())
.await
.expect("activate the data-movement capacity reservation");
*other_store.pool_meta.write().await = store.pool_meta.read().await.clone();
let owner = decommission_capacity_owner(&*store.pool_meta.read().await);
let reserved_target = store.pool_meta.read().await.pools[0]
.decommission
.as_ref()
.and_then(|info| info.capacity_reservation.as_ref())
.expect("data-movement reservation should exist")
.targets[0]
.pool_index;
assert_eq!(reserved_target, 2, "the exact-fit data-movement target must be pool 2");
let upload = new_multipart_upload(
&store,
2,
RUSTFS_META_BUCKET,
object,
ObjectOptions {
data_movement: true,
src_pool_idx: 0,
..Default::default()
},
)
.await
.expect("create the staged data-movement upload");
let multipart_opts = multipart_options(
&store,
RUSTFS_META_BUCKET,
ObjectOptions {
data_movement: true,
src_pool_idx: 0,
..Default::default()
},
)
.await;
let mut data = PutObjReader::from_vec(body.clone());
let part = store.pools[2]
.put_object_part(RUSTFS_META_BUCKET, object, &upload.upload_id, 1, &mut data, &multipart_opts)
.await
.expect("stage the data-movement completion part");
let completed_parts = vec![crate::storage_api_contracts::multipart::CompletePart {
part_num: part.part_num,
etag: part.etag,
..Default::default()
}];
let (lossy_store, refresh_calls) = store_with_capacity_lease_loss(&other_store).await;
set_decommission_capacity_info_overrides_for_test(
lossy_store.id,
vec![capacity_snapshot(), capacity_snapshot(), capacity_snapshot()],
);
let mut complete_opts = ObjectOptions {
data_movement: true,
src_pool_idx: 0,
..ObjectOptions::with_capacity_expected_data_bytes(Some(body.len()))
};
owner.apply_to(&mut complete_opts);
let barrier = MultipartCommitBarrier::install(RUSTFS_META_BUCKET, object, MultipartCommitPause::BeforeQuotaRename);
let complete_store = Arc::clone(&lossy_store);
let complete_upload_id = upload.upload_id.clone();
let complete = tokio::spawn(async move {
complete_store
.complete_multipart_upload_for_data_movement(
(2, None),
RUSTFS_META_BUCKET,
object,
&complete_upload_id,
completed_parts,
&complete_opts,
)
.await
});
barrier.wait_until_paused().await;
tokio::time::pause();
tokio::task::yield_now().await;
refresh_calls.arm();
tokio::time::advance(Duration::from_secs(11)).await;
tokio::task::yield_now().await;
assert!(
refresh_calls.load(Ordering::Acquire) > 0,
"the data-movement capacity lease must lose refresh quorum"
);
barrier.release();
let err = tokio::time::timeout(Duration::from_secs(30), complete)
.await
.expect("data-movement CompleteMultipartUpload should finish after the commit barrier release")
.expect("data-movement CompleteMultipartUpload task should not panic")
.expect_err("a lost data-movement capacity lease must fence before publication");
assert!(!err.to_string().is_empty(), "data-movement lease loss should remain observable");
drop(barrier);
let object_err = lossy_store.pools[2]
.get_object_info(
RUSTFS_META_BUCKET,
object,
&ObjectOptions {
data_movement: true,
..Default::default()
},
)
.await
.expect_err("lost data-movement capacity lease must not publish the object");
assert!(
crate::error::is_err_object_not_found(&object_err),
"data-movement lease loss should leave the target object absent: {object_err}"
);
lossy_store.pools[2]
.get_multipart_info(
RUSTFS_META_BUCKET,
object,
&upload.upload_id,
&ObjectOptions {
data_movement: true,
..Default::default()
},
)
.await
.expect("data-movement lease loss must leave the staged MPU retryable");
let listed = lossy_store.pools[2]
.list_object_parts(
RUSTFS_META_BUCKET,
object,
&upload.upload_id,
None,
100,
&ObjectOptions {
data_movement: true,
..Default::default()
},
)
.await
.expect("data-movement lease loss should preserve the staged part");
assert_eq!(listed.parts.len(), 1, "data-movement lease loss must preserve the MPU part");
let meta = lossy_store.pool_meta.read().await;
let reservation = meta.pools[0]
.decommission
.as_ref()
.and_then(|info| info.capacity_reservation.as_ref())
.expect("data-movement reservation should remain durable after the fenced failure");
assert_eq!(
reservation.consumed_target_physical_bytes, 0,
"a pre-rename data-movement failure must not consume durable target capacity"
);
drop(meta);
let mut persisted = crate::core::pools::PoolMeta::default();
persisted
.load_no_lock_from_replicas(lossy_store.pools.clone())
.await
.expect("the fenced data-movement failure should leave readable pool metadata");
let persisted_reservation = persisted.pools[0]
.decommission
.as_ref()
.and_then(|info| info.capacity_reservation.as_ref())
.expect("the fenced data-movement reservation should remain persisted");
assert_eq!(
persisted_reservation.consumed_target_physical_bytes, 0,
"the pre-rename data-movement failure must not persist consumed target capacity"
);
}
async fn assert_data_movement_complete_waits_for_capacity_tail() {
let (_temp_dirs, store, other_store) =
test_three_pool_stores_with_three_disk_sets_with_isolated_node_contexts(None).await;
let object = format!("data-movement-complete-tail-{}.bin", uuid::Uuid::new_v4());
let body = vec![0x5a; 64];
let layout = DecommissionErasureLayout { data: 1, parity: 0 };
let capacity_snapshot = || {
vec![
DecommissionPoolCapacityInfo::for_test(0, layout, 0, 1024, 1024),
DecommissionPoolCapacityInfo::for_test(1, layout, 0, 4096, 4096),
DecommissionPoolCapacityInfo::for_test(2, layout, 4096, 4096, 0),
]
};
set_decommission_capacity_info_overrides_for_test(store.id, vec![capacity_snapshot()]);
store
.save_current_pool_meta_for_decommission_start(&[0], Vec::new())
.await
.expect("activate the data-movement tail capacity reservation");
*other_store.pool_meta.write().await = store.pool_meta.read().await.clone();
let owner = decommission_capacity_owner(&*store.pool_meta.read().await);
let reserved_target = store.pool_meta.read().await.pools[0]
.decommission
.as_ref()
.and_then(|info| info.capacity_reservation.as_ref())
.expect("data-movement tail reservation should exist")
.targets[0]
.pool_index;
assert_eq!(reserved_target, 2, "the data-movement tail target must be pool 2");
let upload = new_multipart_upload(
&store,
2,
RUSTFS_META_BUCKET,
&object,
ObjectOptions {
data_movement: true,
src_pool_idx: 0,
..Default::default()
},
)
.await
.expect("create the staged data-movement tail upload");
let multipart_opts = multipart_options(
&store,
RUSTFS_META_BUCKET,
ObjectOptions {
data_movement: true,
src_pool_idx: 0,
..Default::default()
},
)
.await;
let mut data = PutObjReader::from_vec(body.clone());
let part = store.pools[2]
.put_object_part(RUSTFS_META_BUCKET, &object, &upload.upload_id, 1, &mut data, &multipart_opts)
.await
.expect("stage the data-movement tail part");
let completed_parts = vec![crate::storage_api_contracts::multipart::CompletePart {
part_num: part.part_num,
etag: part.etag,
..Default::default()
}];
set_decommission_capacity_info_overrides_for_test(
other_store.id,
vec![capacity_snapshot(), capacity_snapshot(), capacity_snapshot()],
);
let mut complete_opts = ObjectOptions {
data_movement: true,
src_pool_idx: 0,
http_preconditions: Some(data_movement::data_movement_target_precondition()),
..ObjectOptions::with_capacity_expected_data_bytes(Some(body.len()))
};
owner.apply_to(&mut complete_opts);
let tail_barrier =
crate::set_disk::rename_fanout_barrier::arm(&object, 0, crate::set_disk::rename_fanout_barrier::PHASE_RENAME);
let complete_store = Arc::clone(&other_store);
let complete_object = object.clone();
let complete_upload_id = upload.upload_id.clone();
let complete = tokio::spawn(async move {
complete_store
.complete_multipart_upload_for_data_movement(
(2, None),
RUSTFS_META_BUCKET,
&complete_object,
&complete_upload_id,
completed_parts,
&complete_opts,
)
.await
});
tail_barrier.wait_until_paused().await;
{
let meta = other_store.pool_meta.read().await;
let reservation = meta.pools[0]
.decommission
.as_ref()
.and_then(|info| info.capacity_reservation.as_ref())
.expect("paused data-movement completion reservation should remain present");
assert!(
reservation.pending_target_physical_bytes > 0 || reservation.inflight_target_physical_bytes > 0,
"paused data-movement completion must retain pending or inflight capacity"
);
assert_eq!(
reservation.consumed_target_physical_bytes, 0,
"paused data-movement completion must not persist consumed capacity"
);
}
let target_lock = other_store
.new_ns_lock(
RUSTFS_META_BUCKET,
&format!("{DECOMMISSION_CAPACITY_TARGET_LOCK_PREFIX}/{reserved_target}"),
)
.await
.expect("create the data-movement tail target-fence probe");
let mut target_probe = tokio::spawn(async move { target_lock.get_write_lock(std::time::Duration::from_secs(30)).await });
assert!(
tokio::time::timeout(std::time::Duration::from_millis(100), &mut target_probe)
.await
.is_err(),
"data-movement completion must retain its target fence while the tail is paused"
);
target_probe.abort();
let _ = target_probe.await;
let pool_meta_lock = other_store
.new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME)
.await
.expect("create the data-movement tail pool metadata probe");
let pool_meta_guard = tokio::time::timeout(
std::time::Duration::from_secs(1),
pool_meta_lock.get_write_lock(std::time::Duration::from_secs(30)),
)
.await
.expect("per-target data movement must leave pool metadata available during target I/O")
.expect("pool metadata probe should acquire during the target tail");
drop(pool_meta_guard);
tail_barrier.release();
drop(tail_barrier);
tokio::time::timeout(std::time::Duration::from_secs(30), complete)
.await
.expect("data-movement CompleteMultipartUpload should finish after its tail resumes")
.expect("data-movement CompleteMultipartUpload task should not panic")
.expect("data-movement CompleteMultipartUpload should commit after its tail resumes");
let target_lock = other_store
.new_ns_lock(
RUSTFS_META_BUCKET,
&format!("{DECOMMISSION_CAPACITY_TARGET_LOCK_PREFIX}/{reserved_target}"),
)
.await
.expect("create the post-tail target-fence probe");
let target_guard = tokio::time::timeout(
std::time::Duration::from_secs(30),
target_lock.get_write_lock(std::time::Duration::from_secs(30)),
)
.await
.expect("target-fence probe should finish after data-movement tail completion")
.expect("target-fence probe should acquire after data-movement tail completion");
drop(target_guard);
let meta = other_store.pool_meta.read().await;
let reservation = meta.pools[0]
.decommission
.as_ref()
.and_then(|info| info.capacity_reservation.as_ref())
.expect("completed data-movement tail reservation should remain present");
assert_eq!(reservation.pending_target_physical_bytes, 0);
assert_eq!(reservation.inflight_target_physical_bytes, 0);
assert!(
reservation.consumed_target_physical_bytes > 0,
"data-movement completion should consume capacity only after its tail"
);
drop(meta);
let mut persisted = crate::core::pools::PoolMeta::default();
persisted
.load_no_lock_from_replicas(other_store.pools.clone())
.await
.expect("data-movement tail completion should persist readable pool metadata");
let persisted_reservation = persisted.pools[0]
.decommission
.as_ref()
.and_then(|info| info.capacity_reservation.as_ref())
.expect("persisted data-movement tail reservation should remain present");
assert_eq!(persisted_reservation.pending_target_physical_bytes, 0);
assert_eq!(persisted_reservation.inflight_target_physical_bytes, 0);
assert!(persisted_reservation.consumed_target_physical_bytes > 0);
}
#[tokio::test]
#[serial_test::serial]
async fn data_movement_complete_does_not_release_capacity_before_early_ack_tail() {
temp_env::async_with_vars([(crate::set_disk::ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async {
assert_data_movement_complete_waits_for_capacity_tail().await
})
.await;
}
#[tokio::test]
#[serial_test::serial]
async fn data_movement_put_fences_capacity_lease_loss_before_publication() {
temp_env::async_with_vars([(crate::set_disk::ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async {
assert_data_movement_put_fences_capacity_lease_loss_before_publication().await
})
.await;
}
async fn assert_data_movement_put_fences_capacity_lease_loss_before_publication() {
let (_temp_dirs, store, other_store) =
test_three_pool_stores_with_three_disk_sets_with_isolated_node_contexts(None).await;
let object = "data-movement-put-lease-loss.bin";
let body = vec![0x52; 256 * 1024];
let layout = DecommissionErasureLayout { data: 1, parity: 0 };
let target_total = body.len().saturating_mul(4);
let capacity_snapshot = || {
vec![
DecommissionPoolCapacityInfo::for_test(0, layout, 0, body.len(), body.len()),
DecommissionPoolCapacityInfo::for_test(1, layout, 0, target_total, target_total),
DecommissionPoolCapacityInfo::for_test(2, layout, target_total, target_total, 0),
]
};
set_decommission_capacity_info_overrides_for_test(store.id, vec![capacity_snapshot()]);
store
.save_current_pool_meta_for_decommission_start(&[0], Vec::new())
.await
.expect("activate the data-movement PUT reservation");
*other_store.pool_meta.write().await = store.pool_meta.read().await.clone();
let owner = decommission_capacity_owner(&*store.pool_meta.read().await);
let reservation_target = store.pool_meta.read().await.pools[0]
.decommission
.as_ref()
.and_then(|info| info.capacity_reservation.as_ref())
.expect("data-movement PUT reservation should exist")
.targets[0]
.pool_index;
assert_eq!(reservation_target, 2, "data-movement PUT should use the reserved target");
let (lossy_store, refresh_calls) = store_with_capacity_lease_loss(&other_store).await;
set_decommission_capacity_info_overrides_for_test(
lossy_store.id,
vec![capacity_snapshot(), capacity_snapshot(), capacity_snapshot()],
);
let version_id = uuid::Uuid::new_v4().to_string();
let mut put_opts = ObjectOptions {
data_movement: true,
versioned: true,
version_id: Some(version_id.clone()),
src_pool_idx: 0,
..ObjectOptions::with_capacity_expected_data_bytes(Some(body.len()))
};
owner.apply_to(&mut put_opts);
let barrier = PutObjectCommitBarrier::install(RUSTFS_META_BUCKET, object, PutObjectCommitPause::BeforeQuotaRename);
let put_store = Arc::clone(&lossy_store);
let put_opts = put_opts.clone();
let put_body = body.clone();
let put = tokio::spawn(async move {
let mut data = PutObjReader::from_vec(put_body);
put_store
.put_object_for_data_movement(RUSTFS_META_BUCKET, object, &mut data, &put_opts, None)
.await
});
barrier.wait_until_paused().await;
tokio::time::pause();
tokio::task::yield_now().await;
refresh_calls.arm();
tokio::time::advance(Duration::from_secs(11)).await;
tokio::task::yield_now().await;
assert!(
refresh_calls.load(Ordering::Acquire) > 0,
"the data-movement PUT capacity lease must lose refresh quorum"
);
barrier.release();
tokio::time::resume();
let (_target_idx, result) = tokio::time::timeout(Duration::from_secs(2), put)
.await
.expect("data-movement PUT should finish after the commit barrier release")
.expect("data-movement PUT task should not panic")
.expect("data-movement PUT should select its reserved target");
let err = result.expect_err("a lost data-movement PUT capacity lease must fence before publication");
assert!(!err.to_string().is_empty(), "data-movement PUT lease loss should remain observable");
drop(barrier);
let object_err = lossy_store.pools[2]
.get_object_info(
RUSTFS_META_BUCKET,
object,
&ObjectOptions {
versioned: true,
version_id: Some(version_id.clone()),
..Default::default()
},
)
.await
.expect_err("lost data-movement PUT capacity lease must not publish the object");
assert!(
matches!(&object_err, crate::error::Error::VersionNotFound(b, o, v)
if b == RUSTFS_META_BUCKET && o == object && v == &version_id),
"data-movement PUT lease loss should leave the target object absent: {object_err}"
);
let meta = lossy_store.pool_meta.read().await;
let reservation = meta.pools[0]
.decommission
.as_ref()
.and_then(|info| info.capacity_reservation.as_ref())
.expect("data-movement PUT reservation should remain after the fenced failure");
assert!(
reservation.pending_target_physical_bytes > 0,
"failed data-movement PUT must retain its durable pending reservation"
);
assert_eq!(reservation.inflight_target_physical_bytes, 0);
assert_eq!(reservation.consumed_target_physical_bytes, 0);
let live_progress = (
reservation.pending_target_physical_bytes,
reservation.inflight_target_physical_bytes,
reservation.consumed_target_physical_bytes,
reservation.observed_target_physical_bytes,
reservation.committed_data_bytes,
);
drop(meta);
let mut persisted = crate::core::pools::PoolMeta::default();
persisted
.load_no_lock_from_replicas(lossy_store.pools.clone())
.await
.expect("the fenced data-movement PUT failure should leave readable pool metadata");
let persisted_reservation = persisted.pools[0]
.decommission
.as_ref()
.and_then(|info| info.capacity_reservation.as_ref())
.expect("the fenced data-movement PUT reservation should remain persisted");
assert_eq!(
(
persisted_reservation.pending_target_physical_bytes,
persisted_reservation.inflight_target_physical_bytes,
persisted_reservation.consumed_target_physical_bytes,
persisted_reservation.observed_target_physical_bytes,
persisted_reservation.committed_data_bytes,
),
live_progress,
"fenced data-movement PUT progress must match its durable replica"
);
*other_store.pool_meta.write().await = persisted.clone();
set_decommission_capacity_info_overrides_for_test(
other_store.id,
vec![
capacity_snapshot(),
capacity_snapshot(),
capacity_snapshot(),
capacity_snapshot(),
capacity_snapshot(),
capacity_snapshot(),
capacity_snapshot(),
capacity_snapshot(),
],
);
let retry_owner = decommission_capacity_owner(&persisted);
let mut retry_opts = ObjectOptions {
data_movement: true,
versioned: true,
version_id: Some(version_id.clone()),
src_pool_idx: 0,
..ObjectOptions::with_capacity_expected_data_bytes(Some(body.len()))
};
retry_owner.apply_to(&mut retry_opts);
let mut retry_data = PutObjReader::from_vec(body.clone());
let (retry_target, retry_result) = other_store
.put_object_for_data_movement(RUSTFS_META_BUCKET, object, &mut retry_data, &retry_opts, None)
.await
.expect("same-mutation retry should reach the staged target write");
assert_eq!(retry_target, 2);
retry_result.expect("same-mutation retry should recover the pending intent and publish");
let retry_info = other_store.pools[2]
.get_object_info(
RUSTFS_META_BUCKET,
object,
&ObjectOptions {
versioned: true,
version_id: Some(version_id),
..Default::default()
},
)
.await
.expect("same-mutation retry should publish a readable target");
assert_eq!(retry_info.size, i64::try_from(body.len()).expect("test body size fits i64"));
let next_object = "data-movement-put-after-retry.bin";
let next_version_id = uuid::Uuid::new_v4().to_string();
let mut next_opts = ObjectOptions {
data_movement: true,
versioned: true,
version_id: Some(next_version_id.clone()),
src_pool_idx: 0,
..ObjectOptions::with_capacity_expected_data_bytes(Some(body.len()))
};
retry_owner.apply_to(&mut next_opts);
let mut next_data = PutObjReader::from_vec(body);
let next_result = other_store
.put_object_for_data_movement(RUSTFS_META_BUCKET, next_object, &mut next_data, &next_opts, None)
.await;
assert!(next_result.is_err(), "a different mutation must not reuse A's recovered capacity");
let next_target_err = other_store.pools[2]
.get_object_info(
RUSTFS_META_BUCKET,
next_object,
&ObjectOptions {
versioned: true,
version_id: Some(next_version_id.clone()),
..Default::default()
},
)
.await
.expect_err("the next mutation must remain unpublished after A recovery");
assert!(
matches!(&next_target_err, crate::error::Error::VersionNotFound(b, o, v)
if b == RUSTFS_META_BUCKET && o == next_object && v == &next_version_id),
"the next mutation must leave its exact target version absent: {next_target_err}"
);
}
#[tokio::test]
#[serial_test::serial]
async fn data_movement_put_holds_capacity_through_successful_early_ack_tail() {
temp_env::async_with_vars([(crate::set_disk::ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async {
assert_data_movement_put_holds_capacity_through_successful_tail().await
})
.await;
}
async fn assert_data_movement_put_holds_capacity_through_successful_tail() {
let (_temp_dirs, store, other_store) =
test_three_pool_stores_with_three_disk_sets_with_isolated_node_contexts(None).await;
let object = "data-movement-put-success-tail.bin";
let body = vec![0x53; 256 * 1024];
let layout = DecommissionErasureLayout { data: 1, parity: 0 };
let target_total = body.len().saturating_mul(4);
let capacity_snapshot = || {
vec![
DecommissionPoolCapacityInfo::for_test(0, layout, 0, body.len(), body.len()),
DecommissionPoolCapacityInfo::for_test(1, layout, 0, target_total, target_total),
DecommissionPoolCapacityInfo::for_test(2, layout, target_total, target_total, 0),
]
};
set_decommission_capacity_info_overrides_for_test(store.id, vec![capacity_snapshot()]);
store
.save_current_pool_meta_for_decommission_start(&[0], Vec::new())
.await
.expect("activate the successful data-movement PUT reservation");
*other_store.pool_meta.write().await = store.pool_meta.read().await.clone();
let owner = decommission_capacity_owner(&*store.pool_meta.read().await);
let reservation_target = store.pool_meta.read().await.pools[0]
.decommission
.as_ref()
.and_then(|info| info.capacity_reservation.as_ref())
.expect("successful data-movement PUT reservation should exist")
.targets[0]
.pool_index;
assert_eq!(reservation_target, 2, "successful data-movement PUT should use the reserved target");
set_decommission_capacity_info_overrides_for_test(
other_store.id,
vec![capacity_snapshot(), capacity_snapshot(), capacity_snapshot()],
);
let version_id = uuid::Uuid::new_v4().to_string();
let mut put_opts = ObjectOptions {
data_movement: true,
versioned: true,
version_id: Some(version_id.clone()),
src_pool_idx: 0,
..ObjectOptions::with_capacity_expected_data_bytes(Some(body.len()))
};
owner.apply_to(&mut put_opts);
let tail_barrier =
crate::set_disk::rename_fanout_barrier::arm(object, 0, crate::set_disk::rename_fanout_barrier::PHASE_RENAME);
let put_store = Arc::clone(&other_store);
let put_opts = put_opts.clone();
let put_body = body.clone();
let mut put = tokio::spawn(async move {
let mut data = PutObjReader::from_vec(put_body);
put_store
.put_object_for_data_movement(RUSTFS_META_BUCKET, object, &mut data, &put_opts, None)
.await
});
tail_barrier.wait_until_paused().await;
assert!(
tokio::time::timeout(Duration::from_millis(100), &mut put).await.is_err(),
"owner-bearing data-movement PUT must wait for the rename tail instead of early-ACKing"
);
let target_lock = other_store
.new_ns_lock(
RUSTFS_META_BUCKET,
&format!("{DECOMMISSION_CAPACITY_TARGET_LOCK_PREFIX}/{reservation_target}"),
)
.await
.expect("create the data-movement PUT tail target-fence probe");
let mut target_probe = tokio::spawn(async move { target_lock.get_write_lock(Duration::from_secs(30)).await });
assert!(
tokio::time::timeout(Duration::from_millis(100), &mut target_probe)
.await
.is_err(),
"data-movement PUT tail must retain its target fence"
);
target_probe.abort();
let _ = target_probe.await;
let pool_meta_lock = other_store
.new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME)
.await
.expect("create the data-movement PUT tail pool metadata probe");
let pool_meta_guard =
tokio::time::timeout(Duration::from_secs(1), pool_meta_lock.get_write_lock(Duration::from_secs(30)))
.await
.expect("per-target data movement must leave pool metadata available during target I/O")
.expect("pool metadata probe should acquire during the target tail");
drop(pool_meta_guard);
{
let meta = other_store.pool_meta.read().await;
let reservation = meta.pools[0]
.decommission
.as_ref()
.and_then(|info| info.capacity_reservation.as_ref())
.expect("paused successful data-movement PUT reservation should remain present");
assert!(
reservation.pending_target_physical_bytes > 0,
"paused successful data-movement PUT must retain pending capacity"
);
assert_eq!(reservation.inflight_target_physical_bytes, 0);
assert_eq!(reservation.consumed_target_physical_bytes, 0);
}
tail_barrier.release();
drop(tail_barrier);
let (target_pool_idx, put_result) = tokio::time::timeout(Duration::from_secs(30), put)
.await
.expect("data-movement PUT should finish after its rename tail resumes")
.expect("data-movement PUT task should not panic")
.expect("data-movement PUT should select its reserved target");
let committed = put_result.expect("data-movement PUT should commit after its rename tail resumes");
assert_eq!(target_pool_idx, 2);
assert_eq!(committed.version_id.map(|id| id.to_string()), Some(version_id.clone()));
let target = other_store.pools[2]
.get_object_info(
RUSTFS_META_BUCKET,
object,
&ObjectOptions {
versioned: true,
version_id: Some(version_id),
..Default::default()
},
)
.await
.expect("successful data-movement PUT should remain readable after its tail");
assert!(!target.delete_marker);
let target_lock = other_store
.new_ns_lock(
RUSTFS_META_BUCKET,
&format!("{DECOMMISSION_CAPACITY_TARGET_LOCK_PREFIX}/{reservation_target}"),
)
.await
.expect("create the post-tail data-movement target-fence probe");
let target_guard = tokio::time::timeout(Duration::from_secs(30), target_lock.get_write_lock(Duration::from_secs(30)))
.await
.expect("target-fence probe should finish after the data-movement PUT tail")
.expect("target-fence probe should acquire after the data-movement PUT tail");
drop(target_guard);
let meta = other_store.pool_meta.read().await;
let reservation = meta.pools[0]
.decommission
.as_ref()
.and_then(|info| info.capacity_reservation.as_ref())
.expect("completed data-movement PUT reservation should remain present");
assert_eq!(reservation.pending_target_physical_bytes, 0);
assert_eq!(reservation.inflight_target_physical_bytes, 0);
assert!(
reservation.consumed_target_physical_bytes > 0,
"data-movement PUT should consume capacity only after its tail"
);
}
#[tokio::test]
#[serial_test::serial]
async fn target_gate_retry_reloads_exact_source_identity_before_copying() {
let (_temp_dirs, store, other_store) =
test_three_pool_stores_with_three_disk_sets_with_isolated_node_contexts(None).await;
let bucket = test_bucket("target-gate-source-change");
let object = "source-changes-while-target-busy.bin";
let initial_body = vec![0x41; 64 * 1024];
let replacement_body = vec![0x42; initial_body.len()];
store
.make_bucket(&bucket, &MakeBucketOptions::default())
.await
.expect("create the target-gate source-change bucket");
let incarnation = store
.bucket_incarnation_id(&bucket)
.await
.expect("load the target-gate source-change bucket incarnation");
let initial_mod_time = time::OffsetDateTime::now_utc();
let mut initial_data = PutObjReader::from_vec(initial_body);
store.pools[0]
.put_object(
&bucket,
object,
&mut initial_data,
&ObjectOptions {
mod_time: Some(initial_mod_time),
expected_bucket_incarnation_id: Some(incarnation),
..Default::default()
},
)
.await
.expect("seed the initial source identity");
let layout = DecommissionErasureLayout { data: 1, parity: 0 };
let target_total = replacement_body.len().saturating_mul(4);
set_decommission_capacity_info_overrides_for_test(
store.id,
vec![vec![
DecommissionPoolCapacityInfo::for_test(0, layout, 0, replacement_body.len(), replacement_body.len()),
DecommissionPoolCapacityInfo::for_test(1, layout, 0, target_total, target_total),
DecommissionPoolCapacityInfo::for_test(2, layout, target_total, target_total, 0),
]],
);
store
.save_current_pool_meta_for_decommission_start(&[0], Vec::new())
.await
.expect("activate the target-gate source-change reservation");
*other_store.pool_meta.write().await = store.pool_meta.read().await.clone();
set_decommission_capacity_info_overrides_for_test(
other_store.id,
vec![vec![
DecommissionPoolCapacityInfo::for_test(0, layout, 0, replacement_body.len(), replacement_body.len()),
DecommissionPoolCapacityInfo::for_test(1, layout, 0, target_total, target_total),
DecommissionPoolCapacityInfo::for_test(2, layout, target_total, target_total, 0),
]],
);
let target_pool_index = store.pool_meta.read().await.pools[0]
.decommission
.as_ref()
.and_then(|info| info.capacity_reservation.as_ref())
.and_then(|reservation| reservation.targets.first())
.expect("the source-change reservation should allocate a target")
.pool_index;
let in_memory_reservation = other_store.pool_meta.read().await.pools[0]
.decommission
.as_ref()
.and_then(|info| info.capacity_reservation.as_ref())
.expect("the source-change reservation should be published in memory")
.clone();
let mut persisted = crate::core::pools::PoolMeta::default();
persisted
.load_no_lock_from_replicas(other_store.pools.clone())
.await
.expect("the source-change reservation should reload from durable replicas");
let persisted_reservation = persisted.pools[0]
.decommission
.as_ref()
.and_then(|info| info.capacity_reservation.as_ref())
.expect("the durable source-change reservation should remain present");
assert_eq!(
(
persisted_reservation.operation_id,
persisted_reservation.generation,
persisted_reservation.owner_nonce,
persisted_reservation.model_version,
persisted_reservation.targets.clone(),
),
(
in_memory_reservation.operation_id,
in_memory_reservation.generation,
in_memory_reservation.owner_nonce,
in_memory_reservation.model_version,
in_memory_reservation.targets.clone(),
),
"the worker owner identity must match the durable reservation"
);
assert!(
persisted_reservation.expires_at > time::OffsetDateTime::now_utc(),
"the durable reservation lease must remain active before the worker starts"
);
let target_lock = other_store
.new_ns_lock(
RUSTFS_META_BUCKET,
&format!("{DECOMMISSION_CAPACITY_TARGET_LOCK_PREFIX}/{target_pool_index}"),
)
.await
.expect("create the target-gate source-change lock");
let target_guard = target_lock
.get_write_lock(Duration::from_secs(30))
.await
.expect("hold the target gate before starting migration");
let retry_observer = DecommissionCapacityLockOrderBarrier::install(other_store.id, other_store.id);
let source_set = other_store.pools[0].get_disks_by_key(object);
let cancel_token = CancellationToken::new();
let canceled_store = Arc::clone(&other_store);
let canceled_bucket = bucket.clone();
let canceled_set = Arc::clone(&source_set);
let canceled_token = cancel_token.clone();
let mut canceled_worker = tokio::spawn(async move {
canceled_store
.decommission_entry_with_retry_state_for_test(
canceled_token,
0,
MetaCacheEntry {
name: object.to_string(),
..Default::default()
},
canceled_bucket,
canceled_set,
None,
Arc::new(AtomicUsize::new(0)),
)
.await
});
tokio::select! {
_ = retry_observer.wait_until_target_gate_retry() => {}
result = &mut canceled_worker => {
panic!("cancellation probe finished before observing target contention: {result:?}");
}
}
assert_eq!(
retry_observer.target_gate_inline_retries(),
crate::core::pools::DECOMMISSION_MUTATION_GATE_MAX_INLINE_ATTEMPTS - 1,
"inline target-gate retries must stop at their bounded attempt budget"
);
tokio::time::sleep(Duration::from_millis(800)).await;
assert_eq!(
retry_observer.target_gate_exact_reloads(),
0,
"a long-held target gate must not trigger periodic exact quorum reloads"
);
cancel_token.cancel();
let canceled_err = tokio::time::timeout(Duration::from_secs(2), canceled_worker)
.await
.expect("target-gate wait should observe cancellation without waiting for release")
.expect("cancellation probe task should not panic")
.expect_err("the canceled target-gate wait must stop the entry");
assert!(
crate::error::is_err_operation_canceled(&canceled_err),
"unexpected target-gate cancellation error: {canceled_err:?}"
);
let worker_store = Arc::clone(&other_store);
let worker_bucket = bucket.clone();
let mut worker = tokio::spawn(async move {
worker_store
.decommission_entry_for_test(
0,
MetaCacheEntry {
name: object.to_string(),
..Default::default()
},
worker_bucket,
source_set,
)
.await
});
tokio::select! {
_ = retry_observer.wait_until_target_gate_retry() => {}
result = &mut worker => {
panic!("decommission entry finished before observing target contention: {result:?}");
}
}
let mut replacement_data = PutObjReader::from_vec(replacement_body.clone());
other_store.pools[0]
.put_object(
&bucket,
object,
&mut replacement_data,
&ObjectOptions {
mod_time: Some(initial_mod_time + time::Duration::seconds(1)),
expected_bucket_incarnation_id: Some(incarnation),
..Default::default()
},
)
.await
.expect("replace the source after the target-busy attempt releases its source snapshot");
retry_observer.release_owner();
drop(target_guard);
tokio::time::timeout(Duration::from_secs(30), worker)
.await
.expect("source-change retry should finish after releasing the target gate")
.expect("source-change retry task should not panic")
.expect("source-change retry should converge on the replacement identity");
assert_eq!(
retry_observer.target_gate_exact_reloads(),
1,
"source identity should be reloaded exactly once after the target gate becomes available"
);
drop(retry_observer);
let mut target_reader = other_store.pools[target_pool_index]
.get_object_reader(&bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
.await
.expect("replacement source should be migrated to the reserved target");
let mut target_body = Vec::new();
target_reader
.stream
.read_to_end(&mut target_body)
.await
.expect("read the migrated replacement body");
assert_eq!(target_body, replacement_body, "the stale pre-contention snapshot must never be copied");
let source_err = other_store.pools[0]
.get_object_info(&bucket, object, &ObjectOptions::default())
.await
.expect_err("the converged replacement source should be removed after migration");
assert!(
matches!(source_err, crate::error::Error::ObjectNotFound(_, _)),
"unexpected source state after replacement migration: {source_err:?}"
);
}
#[tokio::test]
#[serial_test::serial]
async fn target_gate_retry_pins_or_releases_permits_when_target_rank_changes() {
let (_temp_dirs, store, other_store) =
test_three_pool_stores_with_three_disk_sets_with_isolated_node_contexts(None).await;
let bucket = test_bucket("target-gate-two-waiters");
let objects = ["first-waiter.bin", "second-waiter.bin"];
let body = vec![0x54; 64 * 1024];
store
.make_bucket(&bucket, &MakeBucketOptions::default())
.await
.expect("create the two-waiter target-gate bucket");
let incarnation = store
.bucket_incarnation_id(&bucket)
.await
.expect("load the two-waiter bucket incarnation");
for object in objects {
let mut source_data = PutObjReader::from_vec(body.clone());
store.pools[0]
.put_object(
&bucket,
object,
&mut source_data,
&ObjectOptions {
expected_bucket_incarnation_id: Some(incarnation),
..Default::default()
},
)
.await
.expect("seed a same-target waiter source");
}
let layout = DecommissionErasureLayout { data: 1, parity: 0 };
let source_bytes = body.len().saturating_mul(4);
let first_target_capacity = body.len().saturating_mul(5);
let second_target_capacity = body.len().saturating_mul(3);
let capacity_snapshot = vec![
DecommissionPoolCapacityInfo::for_test(0, layout, 0, source_bytes, source_bytes),
DecommissionPoolCapacityInfo::for_test(1, layout, first_target_capacity, first_target_capacity, 0),
DecommissionPoolCapacityInfo::for_test(2, layout, second_target_capacity, second_target_capacity, 0),
];
set_decommission_capacity_info_overrides_for_test(store.id, vec![capacity_snapshot.clone()]);
store
.save_current_pool_meta_for_decommission_start(&[0], Vec::new())
.await
.expect("activate the two-waiter capacity reservation");
*other_store.pool_meta.write().await = store.pool_meta.read().await.clone();
set_decommission_capacity_info_overrides_for_test(other_store.id, vec![capacity_snapshot]);
let capacity_owner = {
let pool_meta = other_store.pool_meta.read().await;
decommission_capacity_owner(&pool_meta)
};
let target_pool_index = other_store
.select_decommission_capacity_target_pool(capacity_owner, body.len())
.await
.expect("select the initially largest target allocation");
let target_pool_indices = other_store.pool_meta.read().await.pools[0]
.decommission
.as_ref()
.and_then(|info| info.capacity_reservation.as_ref())
.expect("the two-waiter reservation should remain active")
.targets
.iter()
.map(|target| target.pool_index)
.collect::<Vec<_>>();
assert_eq!(target_pool_indices.len(), 2, "the rank-change fixture needs two eligible targets");
let alternate_target_pool_index = target_pool_indices
.iter()
.copied()
.find(|pool_index| *pool_index != target_pool_index)
.expect("the rank-change fixture should have an alternate target");
let target_lock = other_store
.new_ns_lock(
RUSTFS_META_BUCKET,
&format!("{DECOMMISSION_CAPACITY_TARGET_LOCK_PREFIX}/{target_pool_index}"),
)
.await
.expect("create the two-waiter target gate");
let target_guard = target_lock
.get_write_lock(Duration::from_secs(30))
.await
.expect("hold the target gate before starting both waiters");
let retry_observer = DecommissionCapacityLockOrderBarrier::install(other_store.id, other_store.id);
let mut workers = Vec::new();
for object in objects {
let worker_store = Arc::clone(&other_store);
let worker_bucket = bucket.clone();
let source_set = other_store.pools[0].get_disks_by_key(object);
workers.push(tokio::spawn(async move {
worker_store
.decommission_entry_for_test(
0,
MetaCacheEntry {
name: object.to_string(),
..Default::default()
},
worker_bucket,
source_set,
)
.await
}));
}
retry_observer.wait_until_target_gate_retries(2).await;
assert_eq!(
retry_observer.target_gate_exact_reloads(),
0,
"waiters must not poll exact source metadata while the target gate is held"
);
let holder_consumed = body.len().saturating_add(1);
{
let mut pool_meta = other_store.pool_meta.write().await;
let source_pool = &mut pool_meta.pools[0];
let reservation = source_pool
.decommission
.as_mut()
.and_then(|info| info.capacity_reservation.as_mut())
.expect("the holder commit should update the active reservation");
let target = reservation
.targets
.iter_mut()
.find(|target| target.pool_index == target_pool_index)
.expect("the holder commit target should remain allocated");
target.consumed_physical_bytes = target.consumed_physical_bytes.saturating_add(holder_consumed);
target.observed_physical_bytes = target.observed_physical_bytes.saturating_add(holder_consumed);
reservation.committed_data_bytes = reservation.committed_data_bytes.saturating_add(holder_consumed);
reservation.consumed_target_physical_bytes =
reservation.consumed_target_physical_bytes.saturating_add(holder_consumed);
reservation.observed_target_physical_bytes =
reservation.observed_target_physical_bytes.saturating_add(holder_consumed);
reservation.prediction_error_bytes = 0;
source_pool.last_update = time::OffsetDateTime::now_utc();
}
other_store
.save_current_pool_meta_for_test(&[0])
.await
.expect("persist the holder commit while it owns the original target gate");
assert_eq!(
other_store
.select_decommission_capacity_target_pool(capacity_owner, body.len())
.await
.expect("reselect after the holder commit"),
alternate_target_pool_index,
"the holder commit should flip the dynamic target ranking"
);
drop(target_guard);
retry_observer.wait_until_owner_paused().await;
assert_eq!(
retry_observer.target_gate_exact_reloads(),
1,
"the first waiter must retain its original target despite the changed ranking"
);
let alternate_probe = other_store
.new_ns_lock(
RUSTFS_META_BUCKET,
&format!("{DECOMMISSION_CAPACITY_TARGET_LOCK_PREFIX}/{alternate_target_pool_index}"),
)
.await
.expect("create the alternate target-gate probe")
.get_write_lock_quiet(Duration::from_millis(300))
.await
.expect("a pinned waiter must not hold the alternate target gate");
drop(alternate_probe);
retry_observer.release_owner();
retry_observer.wait_until_owner_paused().await;
assert_eq!(
retry_observer.target_gate_exact_reloads(),
2,
"the second waiter should reload exactly once before releasing its exhausted original permit"
);
let original_probe = other_store
.new_ns_lock(
RUSTFS_META_BUCKET,
&format!("{DECOMMISSION_CAPACITY_TARGET_LOCK_PREFIX}/{target_pool_index}"),
)
.await
.expect("create the original target-gate probe")
.get_write_lock_quiet(Duration::from_millis(300))
.await
.expect("a reselected waiter must release its original target gate before taking the alternate");
drop(original_probe);
retry_observer.release_owner();
for worker in workers {
tokio::time::timeout(Duration::from_secs(30), worker)
.await
.expect("same-target waiter should finish after permit transfer")
.expect("same-target waiter task should not panic")
.expect("same-target waiter migration should succeed");
}
assert_eq!(
retry_observer.target_gate_exact_reloads(),
2,
"formal mutations must consume their permits without re-entering target-busy recovery"
);
drop(retry_observer);
for object in objects {
let mut target_copies = 0;
for pool_index in &target_pool_indices {
if other_store.pools[*pool_index]
.get_object_info(&bucket, object, &ObjectOptions::default())
.await
.is_ok()
{
target_copies += 1;
}
}
assert_eq!(target_copies, 1, "each waiter should publish on exactly one reserved target");
let source_err = other_store.pools[0]
.get_object_info(&bucket, object, &ObjectOptions::default())
.await
.expect_err("each migrated waiter source should be removed");
assert!(matches!(source_err, crate::error::Error::ObjectNotFound(_, _)));
}
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial_test::serial]
async fn later_version_target_busy_does_not_replay_an_earlier_version() {
let (_temp_dirs, store, other_store) =
test_three_pool_stores_with_three_disk_sets_with_isolated_node_contexts(None).await;
let bucket = test_bucket("target-gate-later-version");
let object = "two-versions-one-busy.bin";
let bodies = [vec![0x31; 64 * 1024], vec![0x32; 64 * 1024]];
store
.make_bucket(&bucket, &MakeBucketOptions::default())
.await
.expect("create the later-version target-gate bucket");
let incarnation = store
.bucket_incarnation_id(&bucket)
.await
.expect("load the later-version bucket incarnation");
for body in &bodies {
let mut source_data = PutObjReader::from_vec(body.clone());
store.pools[0]
.put_object(
&bucket,
object,
&mut source_data,
&ObjectOptions {
versioned: true,
version_id: Some(uuid::Uuid::new_v4().to_string()),
expected_bucket_incarnation_id: Some(incarnation),
..Default::default()
},
)
.await
.expect("seed a source version");
}
let layout = DecommissionErasureLayout { data: 1, parity: 0 };
let source_bytes = bodies.iter().map(Vec::len).sum::<usize>();
let target_total = source_bytes.saturating_mul(4);
let capacity_snapshot = vec![
DecommissionPoolCapacityInfo::for_test(0, layout, 0, source_bytes, source_bytes),
DecommissionPoolCapacityInfo::for_test(1, layout, 0, target_total, target_total),
DecommissionPoolCapacityInfo::for_test(2, layout, target_total, target_total, 0),
];
set_decommission_capacity_info_overrides_for_test(store.id, vec![capacity_snapshot.clone()]);
store
.save_current_pool_meta_for_decommission_start(&[0], Vec::new())
.await
.expect("activate the later-version capacity reservation");
*other_store.pool_meta.write().await = store.pool_meta.read().await.clone();
set_decommission_capacity_info_overrides_for_test(other_store.id, vec![capacity_snapshot]);
let target_pool_index = store.pool_meta.read().await.pools[0]
.decommission
.as_ref()
.and_then(|info| info.capacity_reservation.as_ref())
.and_then(|reservation| reservation.targets.first())
.expect("the later-version reservation should allocate a target")
.pool_index;
let successful_copies = Arc::new(AtomicUsize::new(0));
let first_copy_ready = Arc::new(tokio::sync::Notify::new());
let first_copy_release = Arc::new((StdMutex::new(false), Condvar::new()));
let hook_bucket = bucket.clone();
let hook_object = object.to_string();
let hook_copies = Arc::clone(&successful_copies);
let hook_ready = Arc::clone(&first_copy_ready);
let hook_release = Arc::clone(&first_copy_release);
let fault_guard = DecommissionTestFaultGuard::install(Arc::new(move |stage, bucket, object, _attempt, succeeded| {
if succeeded && stage == "migrate_object" && bucket == hook_bucket && object == hook_object {
let copy_number = hook_copies.fetch_add(1, Ordering::AcqRel) + 1;
if copy_number == 1 {
hook_ready.notify_one();
let (released, wake) = &*hook_release;
let mut released = released.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
while !*released {
released = wake.wait(released).unwrap_or_else(std::sync::PoisonError::into_inner);
}
}
}
false
}));
let worker_store = Arc::clone(&other_store);
let worker_bucket = bucket.clone();
let source_set = other_store.pools[0].get_disks_by_key(object);
let mut worker = tokio::spawn(async move {
worker_store
.decommission_entry_for_test(
0,
MetaCacheEntry {
name: object.to_string(),
..Default::default()
},
worker_bucket,
source_set,
)
.await
});
tokio::time::timeout(Duration::from_secs(30), first_copy_ready.notified())
.await
.expect("the first version should complete its target copy");
let target_lock = other_store
.new_ns_lock(
RUSTFS_META_BUCKET,
&format!("{DECOMMISSION_CAPACITY_TARGET_LOCK_PREFIX}/{target_pool_index}"),
)
.await
.expect("create the later-version target gate");
let target_guard = target_lock
.get_write_lock(Duration::from_secs(30))
.await
.expect("hold the target gate after the first version copy");
let retry_observer = DecommissionCapacityLockOrderBarrier::install(other_store.id, other_store.id);
{
let (released, wake) = &*first_copy_release;
*released.lock().unwrap_or_else(std::sync::PoisonError::into_inner) = true;
wake.notify_one();
}
tokio::select! {
_ = retry_observer.wait_until_target_gate_retry() => {}
result = &mut worker => panic!("later-version worker finished before target contention: {result:?}"),
}
assert_eq!(
successful_copies.load(Ordering::Acquire),
1,
"the already copied version must not be replayed while the later version waits"
);
retry_observer.release_owner();
drop(target_guard);
tokio::time::timeout(Duration::from_secs(30), worker)
.await
.expect("later-version worker should finish after target release")
.expect("later-version worker task should not panic")
.expect("later-version migration should succeed");
assert_eq!(
successful_copies.load(Ordering::Acquire),
2,
"each source version should complete exactly one target copy"
);
drop(retry_observer);
drop(fault_guard);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial_test::serial]
async fn target_busy_does_not_reset_the_ordinary_copy_failure_budget() {
let (_temp_dirs, store, other_store) =
test_three_pool_stores_with_three_disk_sets_with_isolated_node_contexts(None).await;
let bucket = test_bucket("target-gate-copy-budget");
let object = "copy-fails-around-target-busy.bin";
let body = vec![0x43; 64 * 1024];
store
.make_bucket(&bucket, &MakeBucketOptions::default())
.await
.expect("create the copy-budget target-gate bucket");
let incarnation = store
.bucket_incarnation_id(&bucket)
.await
.expect("load the copy-budget bucket incarnation");
let mut source_data = PutObjReader::from_vec(body.clone());
store.pools[0]
.put_object(
&bucket,
object,
&mut source_data,
&ObjectOptions {
expected_bucket_incarnation_id: Some(incarnation),
..Default::default()
},
)
.await
.expect("seed the copy-budget source");
let layout = DecommissionErasureLayout { data: 1, parity: 0 };
let source_bytes = body.len().saturating_mul(3);
let target_total = body.len().saturating_mul(8);
let capacity_snapshot = vec![
DecommissionPoolCapacityInfo::for_test(0, layout, 0, source_bytes, source_bytes),
DecommissionPoolCapacityInfo::for_test(1, layout, 0, target_total, target_total),
DecommissionPoolCapacityInfo::for_test(2, layout, target_total, target_total, 0),
];
set_decommission_capacity_info_overrides_for_test(store.id, vec![capacity_snapshot.clone()]);
store
.save_current_pool_meta_for_decommission_start(&[0], Vec::new())
.await
.expect("activate the copy-budget capacity reservation");
*other_store.pool_meta.write().await = store.pool_meta.read().await.clone();
set_decommission_capacity_info_overrides_for_test(other_store.id, vec![capacity_snapshot]);
let target_pool_index = store.pool_meta.read().await.pools[0]
.decommission
.as_ref()
.and_then(|info| info.capacity_reservation.as_ref())
.and_then(|reservation| reservation.targets.first())
.expect("the copy-budget reservation should allocate a target")
.pool_index;
let observed_attempts = Arc::new(StdMutex::new(Vec::new()));
let fault_ready = Arc::new(tokio::sync::Notify::new());
let fault_release = Arc::new((StdMutex::new(0usize), Condvar::new()));
let hook_bucket = bucket.clone();
let hook_object = object.to_string();
let hook_attempts = Arc::clone(&observed_attempts);
let hook_ready = Arc::clone(&fault_ready);
let hook_release = Arc::clone(&fault_release);
let fault_guard = DecommissionTestFaultGuard::install(Arc::new(move |stage, bucket, object, attempt, succeeded| {
if stage != "migrate_object" || bucket != hook_bucket || object != hook_object {
return false;
}
let copy_number = {
let mut attempts = hook_attempts.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
attempts.push(attempt);
attempts.len()
};
if copy_number <= 2 {
hook_ready.notify_one();
let (released, wake) = &*hook_release;
let mut released = released.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
while *released < copy_number {
released = wake.wait(released).unwrap_or_else(std::sync::PoisonError::into_inner);
}
return succeeded;
}
false
}));
let worker_store = Arc::clone(&other_store);
let worker_bucket = bucket.clone();
let source_set = other_store.pools[0].get_disks_by_key(object);
let mut worker = tokio::spawn(async move {
worker_store
.decommission_entry_for_test(
0,
MetaCacheEntry {
name: object.to_string(),
..Default::default()
},
worker_bucket,
source_set,
)
.await
});
tokio::time::timeout(Duration::from_secs(30), fault_ready.notified())
.await
.expect("the first successful copy should reach fault injection");
let target_lock = other_store
.new_ns_lock(
RUSTFS_META_BUCKET,
&format!("{DECOMMISSION_CAPACITY_TARGET_LOCK_PREFIX}/{target_pool_index}"),
)
.await
.expect("create the copy-budget target gate");
let target_guard = target_lock
.get_write_lock(Duration::from_secs(30))
.await
.expect("hold the target gate after the first injected copy failure");
let retry_observer = DecommissionCapacityLockOrderBarrier::install(other_store.id, other_store.id);
retry_observer.disable_owner_pause();
{
let (released, wake) = &*fault_release;
*released.lock().unwrap_or_else(std::sync::PoisonError::into_inner) = 1;
wake.notify_all();
}
tokio::select! {
_ = retry_observer.wait_until_target_gate_retry() => {}
result = &mut worker => panic!("copy-budget worker finished before target contention: {result:?}"),
}
drop(target_guard);
tokio::time::timeout(Duration::from_secs(30), fault_ready.notified())
.await
.expect("the post-contention copy should reach the second injected failure");
assert_eq!(
*observed_attempts.lock().unwrap_or_else(std::sync::PoisonError::into_inner),
vec![1, 2],
"target contention must not reset the attempt consumed before it"
);
drop(retry_observer);
{
let (released, wake) = &*fault_release;
*released.lock().unwrap_or_else(std::sync::PoisonError::into_inner) = 2;
wake.notify_all();
}
tokio::time::timeout(Duration::from_secs(30), worker)
.await
.expect("copy-budget worker should finish on its final attempt")
.expect("copy-budget worker task should not panic")
.expect("copy-budget migration should succeed within its original retry budget");
assert_eq!(
*observed_attempts.lock().unwrap_or_else(std::sync::PoisonError::into_inner),
vec![1, 2, 3],
"ordinary copy failures before and after contention must consume one shared budget"
);
drop(fault_guard);
}
#[tokio::test]
#[serial_test::serial]
async fn exact_delete_reconciles_pending_capacity_before_removing_replicas() {
let (_temp_dirs, store, _other_store) = test_three_pool_stores_with_isolated_node_contexts(None).await;
let bucket = test_bucket("exact-delete-capacity");
let object = "published-before-exact-delete.bin";
let body = vec![0x55; 64 * 1024];
let version_id = uuid::Uuid::new_v4().to_string();
store
.make_bucket(&bucket, &MakeBucketOptions::default())
.await
.expect("create the exact-delete reconciliation bucket");
let incarnation = store
.bucket_incarnation_id(&bucket)
.await
.expect("load the exact-delete bucket incarnation");
let mut source_data = PutObjReader::from_vec(body.clone());
let source = store.pools[0]
.put_object(
&bucket,
object,
&mut source_data,
&ObjectOptions {
versioned: true,
version_id: Some(version_id.clone()),
expected_bucket_incarnation_id: Some(incarnation),
..Default::default()
},
)
.await
.expect("seed the exact source version");
let layout = DecommissionErasureLayout { data: 1, parity: 0 };
let target_total = body.len().saturating_mul(4);
set_decommission_capacity_info_overrides_for_test(
store.id,
vec![vec![
DecommissionPoolCapacityInfo::for_test(0, layout, 0, body.len(), body.len()),
DecommissionPoolCapacityInfo::for_test(1, layout, 0, target_total, target_total),
DecommissionPoolCapacityInfo::for_test(2, layout, target_total, target_total, 0),
]],
);
store
.save_current_pool_meta_for_decommission_start(&[0], Vec::new())
.await
.expect("activate the exact-delete capacity reservation");
let owner = decommission_capacity_owner(&*store.pool_meta.read().await);
let target_pool_index = store.pool_meta.read().await.pools[0]
.decommission
.as_ref()
.and_then(|info| info.capacity_reservation.as_ref())
.expect("the exact-delete capacity reservation should exist")
.targets[0]
.pool_index;
assert_eq!(target_pool_index, 2);
let target_options = ObjectOptions {
versioned: true,
version_id: Some(version_id.clone()),
mod_time: source.mod_time,
preserve_etag: source.etag.clone(),
user_defined: (*source.user_defined).clone(),
data_movement: true,
src_pool_idx: 0,
expected_bucket_incarnation_id: Some(incarnation),
..Default::default()
};
let mut target_data = PutObjReader::from_vec(body.clone());
let target = store.pools[target_pool_index]
.put_object(&bucket, object, &mut target_data, &target_options)
.await
.expect("publish the target version before capacity progress");
assert_eq!(target.version_id, source.version_id);
assert_eq!(target.mod_time, source.mod_time);
assert_eq!(target.size, source.size);
let source_version_id = source.version_id.map(|version_id| version_id.to_string());
let mutation_id = decommission_capacity_mutation_id(
owner,
&source.bucket,
&source.name,
source_version_id.as_deref(),
source.delete_marker,
source.mod_time,
);
{
let mut pool_meta = store.pool_meta.write().await;
let source_pool = &mut pool_meta.pools[0];
let reservation = source_pool
.decommission
.as_mut()
.and_then(|info| info.capacity_reservation.as_mut())
.expect("the exact-delete capacity reservation should remain active");
let target = reservation
.targets
.iter_mut()
.find(|target| target.pool_index == target_pool_index)
.expect("the exact-delete target allocation should exist");
target.pending_physical_bytes = body.len();
target.pending_mutation_id = Some(mutation_id);
reservation.pending_target_physical_bytes = body.len();
source_pool.last_update = time::OffsetDateTime::now_utc();
}
store
.save_current_pool_meta_for_test(&[0])
.await
.expect("persist the simulated post-commit capacity intent");
let exact_delete_options = ObjectOptions {
versioned: true,
version_id: Some(version_id.clone()),
expected_bucket_incarnation_id: Some(incarnation),
..Default::default()
};
store.pools[target_pool_index]
.delete_object(&bucket, object, exact_delete_options.clone())
.await
.expect("remove the target evidence before the fail-closed exact delete");
let delete_err = store
.delete_object(&bucket, object, exact_delete_options.clone())
.await
.expect_err("exact delete must fail while its pending target evidence is absent");
assert!(matches!(delete_err, crate::error::Error::DecommissionCapacityBlocked { .. }));
store.pools[0]
.get_object_info(
&bucket,
object,
&ObjectOptions {
versioned: true,
version_id: Some(version_id.clone()),
no_lock: true,
..Default::default()
},
)
.await
.expect("a failed reconciliation must preserve the source evidence");
let mut failed = crate::core::pools::PoolMeta::default();
failed
.load_no_lock_from_replicas(store.pools.clone())
.await
.expect("the failed exact delete must preserve readable capacity metadata");
let failed_reservation = failed.pools[0]
.decommission
.as_ref()
.and_then(|info| info.capacity_reservation.as_ref())
.expect("the failed exact delete reservation should remain present");
assert_eq!(failed_reservation.pending_target_physical_bytes, body.len());
assert_eq!(failed_reservation.consumed_target_physical_bytes, 0);
let mut replacement_target_data = PutObjReader::from_vec(body.clone());
store.pools[target_pool_index]
.put_object(&bucket, object, &mut replacement_target_data, &target_options)
.await
.expect("restore the equivalent target evidence for the exact-delete retry");
store
.delete_object(&bucket, object, exact_delete_options)
.await
.expect("the exact delete should reconcile capacity before removing replicas");
let mut persisted = crate::core::pools::PoolMeta::default();
persisted
.load_no_lock_from_replicas(store.pools.clone())
.await
.expect("the exact-delete reconciliation should remain durable");
let reservation = persisted.pools[0]
.decommission
.as_ref()
.and_then(|info| info.capacity_reservation.as_ref())
.expect("the reconciled reservation should remain present");
assert_eq!(reservation.pending_target_physical_bytes, 0);
assert_eq!(reservation.consumed_target_physical_bytes, body.len());
assert_eq!(reservation.committed_data_bytes, body.len());
for pool_index in [0, target_pool_index] {
let err = store.pools[pool_index]
.get_object_info(
&bucket,
object,
&ObjectOptions {
versioned: true,
version_id: Some(version_id.clone()),
no_lock: true,
..Default::default()
},
)
.await
.expect_err("the exact version should be absent after reconciliation and delete");
assert!(crate::error::is_err_object_not_found(&err) || crate::error::is_err_version_not_found(&err));
}
}
#[test]
#[serial_test::serial]
fn scanner_backlog_cas_keeps_fences_after_waiter_cancellation_until_rename_drains() {
run_large_stack_current_thread_async_test("scanner-backlog-canceled-waiter", async || {
temp_env::async_with_vars([(crate::set_disk::ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async {
let (_temp_dirs, writer, other) =
test_three_pool_stores_with_three_disk_sets_with_isolated_node_contexts(None).await;
let object = "buckets/.scanner-pause-backlog.json";
let set_index = 1;
let body = vec![0x37; 1024];
assert!(
!writer.pools[0].disk_set[0]
.shares_namespace_lock_domain(&writer.pools[0].disk_set[set_index])
.await
);
let rename_tasks = crate::set_disk::rename_fanout_barrier::observe_tasks(object);
let tail =
crate::set_disk::rename_fanout_barrier::arm(object, 0, crate::set_disk::rename_fanout_barrier::PHASE_RENAME);
let put_store = Arc::clone(&writer);
let put_body = body.clone();
let mut put = tokio::spawn(async move {
put_store
.save_scanner_pause_backlog_replica(0, set_index, put_body, Default::default())
.await
});
tokio::time::timeout(Duration::from_secs(30), tail.wait_until_paused())
.await
.expect("the native write must reach its held rename");
tokio::time::timeout(Duration::from_secs(30), async {
while rename_tasks.running() != 1 {
tokio::task::yield_now().await;
}
})
.await
.expect("the other disks must reach quorum before canceling the waiter");
assert!(
tokio::time::timeout(Duration::from_millis(100), &mut put).await.is_err(),
"native replica publication must await the entire rename tail"
);
put.abort();
assert!(put.await.expect_err("the scanner waiter must be canceled").is_cancelled());
let capacity_lock = other
.new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME)
.await
.expect("capacity lock probe");
let object_lock = other
.new_ns_lock(RUSTFS_META_BUCKET, object)
.await
.expect("fixed object lock probe");
let mut capacity_probe = tokio::spawn(async move { capacity_lock.get_write_lock(Duration::from_secs(30)).await });
let mut object_probe = tokio::spawn(async move { object_lock.get_write_lock(Duration::from_secs(30)).await });
for (label, probe) in [("capacity", &mut capacity_probe), ("fixed object", &mut object_probe)] {
assert!(
tokio::time::timeout(Duration::from_millis(100), probe).await.is_err(),
"canceling the scanner waiter must retain its {label} fence while rename is pending"
);
}
tail.release();
drop(tail);
for probe in [capacity_probe, object_probe] {
drop(
tokio::time::timeout(Duration::from_secs(30), probe)
.await
.expect("publication fence must drain after rename")
.expect("lock probe must not panic")
.expect("publication fence must eventually be released"),
);
}
let mut reader = writer.pools[0].disk_set[set_index]
.get_object_reader(RUSTFS_META_BUCKET, object, None, HeaderMap::new(), &ObjectOptions::default())
.await
.expect("canceled waiter must leave the committed replica readable");
let mut actual = Vec::new();
reader
.read_to_end(&mut actual)
.await
.expect("read the full native replica after tail drain");
assert_eq!(actual, body);
})
.await;
});
}
#[test]
#[serial_test::serial]
fn scanner_backlog_cas_rejects_lost_capacity_lease_before_publication() {
run_large_stack_current_thread_async_test("scanner-backlog-lease-loss", async || {
let (_temp_dirs, writer, other) = test_three_pool_stores_with_isolated_node_contexts(None).await;
let object = "buckets/.scanner-pause-backlog.json";
let body = b"native source before lease loss".to_vec();
let original = writer
.save_scanner_pause_backlog_replica(2, 1, body.clone(), Default::default())
.await
.expect("seed the exact native replica set");
let (lossy, refresh_calls) = store_with_capacity_lease_loss(&other).await;
let barrier = PutObjectCommitBarrier::install(RUSTFS_META_BUCKET, object, PutObjectCommitPause::BeforeQuotaRename);
let put = tokio::spawn(async move {
lossy
.save_scanner_pause_backlog_replica(
2,
1,
b"must not commit after lease loss".to_vec(),
crate::storage_api_contracts::object::HTTPPreconditions {
if_match: original.etag,
..Default::default()
},
)
.await
});
tokio::time::timeout(Duration::from_secs(30), barrier.wait_until_paused())
.await
.expect("native CAS must reach its commit barrier");
tokio::time::pause();
tokio::task::yield_now().await;
refresh_calls.arm();
tokio::time::advance(Duration::from_secs(11)).await;
tokio::task::yield_now().await;
assert!(
refresh_calls.load(Ordering::Acquire) > 0,
"the durable metadata lease must lose refresh quorum"
);
barrier.release();
tokio::time::resume();
let err = tokio::time::timeout(Duration::from_secs(30), put)
.await
.expect("native CAS must finish after the barrier release")
.expect("native CAS task must not panic")
.expect_err("a lost outer capacity lease must reject native publication");
assert!(
matches!(err, crate::error::Error::NamespaceLockQuorumUnavailable { .. }),
"unexpected lease error: {err}"
);
drop(barrier);
let mut reader = writer.pools[2].disk_set[1]
.get_object_reader(RUSTFS_META_BUCKET, object, None, HeaderMap::new(), &ObjectOptions::default())
.await
.expect("the preexisting replica must survive lease loss");
let mut actual = Vec::new();
reader.read_to_end(&mut actual).await.expect("read the full retained replica");
assert_eq!(actual, body);
});
}
#[test]
#[serial_test::serial]
fn scanner_backlog_cas_rejects_a_retiring_source_on_a_stale_node() {
run_large_stack_current_thread_async_test("scanner-backlog-source-fence", async || {
let (_temp_dirs, store, writer) = test_three_pool_stores_with_three_disk_sets_with_isolated_node_contexts(None).await;
let object = "buckets/.scanner-pause-backlog.json";
let body = b"frozen native scanner replica".to_vec();
let source_set_index = (writer.pools[0].get_disks_by_key(object).set_index + 1) % writer.pools[0].disk_set.len();
assert_ne!(
source_set_index,
writer.pools[0].get_disks_by_key(object).set_index,
"exercise a non-routed native set"
);
let original = writer
.save_scanner_pause_backlog_replica(
0,
source_set_index,
body.clone(),
crate::storage_api_contracts::object::HTTPPreconditions {
if_none_match: Some("*".to_string()),
..Default::default()
},
)
.await
.expect("seed a native scanner replica before retirement");
assert!(
writer
.scanner_pause_backlog_writable_set_disks()
.await
.iter()
.any(|set| set.pool_index == 0)
);
let layout = DecommissionErasureLayout { data: 1, parity: 0 };
let target_total = body.len() * 8;
set_decommission_capacity_info_overrides_for_test(
store.id,
vec![vec![
DecommissionPoolCapacityInfo::for_test(0, layout, 0, body.len() * 2, body.len() * 2),
DecommissionPoolCapacityInfo::for_test(1, layout, 0, target_total, target_total),
DecommissionPoolCapacityInfo::for_test(2, layout, target_total, target_total, 0),
]],
);
store
.save_current_pool_meta_for_decommission_start(&[0], Vec::new())
.await
.expect("another node durably retires the selected source");
assert!(
writer.pool_meta.read().await.pools[0].decommission.is_none(),
"the writer must retain a stale snapshot"
);
assert!(
writer
.scanner_pause_backlog_writable_set_disks()
.await
.iter()
.any(|set| set.pool_index == 0)
);
let result = writer
.save_scanner_pause_backlog_replica(
0,
source_set_index,
b"late native scanner update".to_vec(),
crate::storage_api_contracts::object::HTTPPreconditions {
if_match: original.etag.clone(),
..Default::default()
},
)
.await;
assert!(
matches!(result, Err(crate::error::Error::SlowDown)),
"late native source publication must fail: {result:?}"
);
let mut source = writer.pools[0].disk_set[source_set_index]
.get_object_reader(RUSTFS_META_BUCKET, object, None, HeaderMap::new(), &ObjectOptions::default())
.await
.expect("the original source must remain readable");
assert_eq!(source.object_info.etag, original.etag);
let mut actual = Vec::new();
source
.read_to_end(&mut actual)
.await
.expect("read the entire retained source");
assert_eq!(actual, body);
for set in &writer.pools[2].disk_set {
let target_body = format!("surviving native scanner set {}", set.set_index).into_bytes();
let committed = writer
.save_scanner_pause_backlog_replica(
2,
set.set_index,
target_body.clone(),
crate::storage_api_contracts::object::HTTPPreconditions {
if_none_match: Some("*".to_string()),
..Default::default()
},
)
.await
.expect("every reserved healthy target set must still accept scanner replicas");
let conflict = writer
.save_scanner_pause_backlog_replica(
2,
set.set_index,
b"must not bypass CAS".to_vec(),
crate::storage_api_contracts::object::HTTPPreconditions {
if_match: Some("stale-native-revision".to_string()),
..Default::default()
},
)
.await
.expect_err("capacity admission must retain the native writer's CAS");
assert!(matches!(conflict, crate::error::Error::PreconditionFailed));
let mut target = set
.get_object_reader(RUSTFS_META_BUCKET, object, None, HeaderMap::new(), &ObjectOptions::default())
.await
.expect("read the actual replica set, not the hash-routed set");
assert_eq!(target.object_info.etag, committed.etag);
let mut actual = Vec::new();
target
.read_to_end(&mut actual)
.await
.expect("read the complete native target");
assert_eq!(actual, target_body);
}
for (pool_index, set_index) in [(writer.pools.len(), 0), (0, writer.pools[0].disk_set.len())] {
assert!(matches!(
writer
.save_scanner_pause_backlog_replica(pool_index, set_index, Vec::new(), Default::default())
.await,
Err(crate::error::Error::InvalidArgument(_, _, _))
));
}
store
.decommission_cancel(0)
.await
.expect("cancel retirement before restoring native membership");
writer
.save_scanner_pause_backlog_replica(
0,
source_set_index,
b"canceled source membership repair".to_vec(),
crate::storage_api_contracts::object::HTTPPreconditions {
if_match: original.etag,
..Default::default()
},
)
.await
.expect("cancel must retain scanner's existing native membership repair contract");
});
}
#[test]
#[serial_test::serial]
fn data_movement_existing_replica_reconciles_capacity_and_cleans_source() {
data_movement_existing_replica_reconciles_capacity_case(false);
}
#[test]
#[serial_test::serial]
fn data_movement_existing_replica_outside_reservation_uses_reserved_target() {
data_movement_existing_replica_reconciles_capacity_case(true);
}
fn data_movement_existing_replica_reconciles_capacity_case(existing_outside_reservation: bool) {
run_large_stack_current_thread_async_test("reserved-replica-reconcile", async move || {
let (_temp_dirs, store, other_store) =
test_three_pool_stores_with_three_disk_sets_with_isolated_node_contexts(None).await;
let object = "buckets/reserved-replica-routing.json";
let body = br#"{"schemaVersion":1,"generation":2}"#.to_vec();
let old_body = br#"{"schemaVersion":1,"generation":1}"#.to_vec();
let source_time = time::OffsetDateTime::UNIX_EPOCH + time::Duration::seconds(20);
let target_time = source_time;
let target_pool_index = if existing_outside_reservation { 1 } else { 2 };
let mut replicas = vec![(0, body.clone(), source_time), (target_pool_index, old_body, target_time)];
if existing_outside_reservation {
replicas.push((
2,
br#"{"schemaVersion":1,"generation":3}"#.to_vec(),
source_time + time::Duration::seconds(10),
));
}
for (pool_index, payload, mod_time) in replicas.iter().cloned() {
store.pools[pool_index]
.put_object(
RUSTFS_META_BUCKET,
object,
&mut PutObjReader::from_vec(payload),
&ObjectOptions {
max_parity: true,
mod_time: Some(mod_time),
..Default::default()
},
)
.await
.expect("seed existing replicas with independent write times");
}
let layout = DecommissionErasureLayout { data: 1, parity: 0 };
let target_total = body.len() * 8;
let capacities = vec![
DecommissionPoolCapacityInfo::for_test(0, layout, 0, body.len() * 2, body.len() * 2),
DecommissionPoolCapacityInfo::for_test(
1,
layout,
if existing_outside_reservation { target_total } else { 0 },
target_total,
if existing_outside_reservation { 0 } else { target_total },
),
DecommissionPoolCapacityInfo::for_test(2, layout, target_total, target_total, 0),
];
set_decommission_capacity_info_overrides_for_test(store.id, vec![capacities.clone()]);
store
.save_current_pool_meta_for_decommission_start(&[0], Vec::new())
.await
.expect("activate the source reservation");
let owner = decommission_capacity_owner(&*store.pool_meta.read().await);
let reserved_snapshot = store.pool_meta.read().await.clone();
let reservation = reserved_snapshot.pools[0]
.decommission
.as_ref()
.and_then(|info| info.capacity_reservation.as_ref())
.expect("active source reservation");
assert_eq!(
reservation.targets.iter().map(|target| target.pool_index).collect::<Vec<_>>(),
vec![target_pool_index],
"the fixture must reserve exactly one target"
);
let source_reader = store.pools[0]
.get_object_reader(
RUSTFS_META_BUCKET,
object,
None,
HeaderMap::new(),
&ObjectOptions {
no_lock: true,
data_movement: true,
raw_data_movement_read: true,
..Default::default()
},
)
.await
.expect("read the frozen source replica");
let conflict = data_movement::migrate_decommission_object(
Arc::clone(&store),
0,
RUSTFS_META_BUCKET.to_string(),
source_reader,
None,
"reserved_replica_conflict",
Some(owner),
)
.await
.expect_err("a different older existing record must retain its source and capacity intent");
assert!(conflict.to_string().contains("Precondition failed"), "unexpected conflict: {conflict}");
let reserved_snapshot = store.pool_meta.read().await.clone();
let mut selection_opts = ObjectOptions {
data_movement: true,
src_pool_idx: 0,
..Default::default()
};
assert_eq!(
store
.select_data_movement_pool_idx(RUSTFS_META_BUCKET, object, body.len() as i64, &selection_opts, true)
.await
.expect("selection without a capacity owner retains existing-replica routing"),
2
);
owner.apply_to(&mut selection_opts);
for stale_owner in [
DecommissionCapacityOwner {
owner_nonce: uuid::Uuid::new_v4(),
..owner
},
DecommissionCapacityOwner {
generation: owner.generation + 1,
..owner
},
] {
let mut stale_opts = selection_opts.clone();
stale_owner.apply_to(&mut stale_opts);
assert!(
matches!(
store
.select_data_movement_pool_idx(RUSTFS_META_BUCKET, object, body.len() as i64, &stale_opts, true)
.await,
Err(crate::error::Error::DecommissionCapacityBlocked { .. })
),
"a stale owner must not fall back to another target"
);
}
{
let mut meta = store.pool_meta.write().await;
meta.pools[0]
.decommission
.as_mut()
.unwrap()
.capacity_reservation
.as_mut()
.unwrap()
.expires_at = time::OffsetDateTime::now_utc() - time::Duration::seconds(1);
}
assert!(
matches!(
store
.select_data_movement_pool_idx(RUSTFS_META_BUCKET, object, body.len() as i64, &selection_opts, true)
.await,
Err(crate::error::Error::DecommissionCapacityBlocked { .. })
),
"an expired owner must not fall back to another target"
);
*store.pool_meta.write().await = reserved_snapshot.clone();
if !existing_outside_reservation {
{
let mut meta = store.pool_meta.write().await;
let target = &mut meta.pools[0]
.decommission
.as_mut()
.unwrap()
.capacity_reservation
.as_mut()
.unwrap()
.targets[0];
target.consumed_physical_bytes = target.reserved_physical_bytes;
}
assert_eq!(
store
.select_data_movement_pool_idx(RUSTFS_META_BUCKET, object, body.len() as i64, &selection_opts, true)
.await
.expect("an existing reserved replica can still be selected after capacity was consumed"),
target_pool_index
);
*store.pool_meta.write().await = reserved_snapshot;
}
let mut persisted = crate::core::pools::PoolMeta::default();
persisted
.load_no_lock_from_replicas(store.pools.clone())
.await
.expect("reload the unresolved intent");
assert_eq!(
persisted.pools[0]
.decommission
.as_ref()
.expect("source state")
.capacity_reservation
.as_ref()
.expect("durable capacity")
.pending_target_physical_bytes,
body.len()
);
for (pool_index, payload, mod_time) in &replicas {
let mut reader = store.pools[*pool_index]
.get_object_reader(RUSTFS_META_BUCKET, object, None, HeaderMap::new(), &ObjectOptions::default())
.await
.expect("a refused existing record replacement must preserve every replica");
assert_eq!(reader.object_info.mod_time, Some(*mod_time));
let mut actual = Vec::new();
reader
.read_to_end(&mut actual)
.await
.expect("read the unchanged existing record");
assert_eq!(&actual, payload);
}
let previous = store.pools[target_pool_index]
.get_object_info(RUSTFS_META_BUCKET, object, &ObjectOptions::default())
.await
.expect("read the existing writer's CAS revision");
let replacement = store.pools[target_pool_index]
.put_object(
RUSTFS_META_BUCKET,
object,
&mut PutObjReader::from_vec(body.clone()),
&ObjectOptions {
max_parity: true,
mod_time: Some(target_time),
http_preconditions: Some(crate::storage_api_contracts::object::HTTPPreconditions {
if_match: previous.etag,
..Default::default()
}),
..Default::default()
},
)
.await
.expect("existing CAS converges the payload without a migration marker");
assert!(!data_movement::is_owned_data_movement_target(&replacement));
*other_store.pool_meta.write().await = persisted;
set_decommission_capacity_info_overrides_for_test(other_store.id, vec![capacities]);
tokio::time::timeout(
Duration::from_secs(30),
other_store.decommission_entry_for_test(
0,
MetaCacheEntry {
name: object.to_string(),
..Default::default()
},
RUSTFS_META_BUCKET.to_string(),
other_store.pools[0].get_disks_by_key(object),
),
)
.await
.expect("replica conflict recovery must be bounded")
.expect("identical existing replica should finish migration on the reloaded node");
let mut reconciled = crate::core::pools::PoolMeta::default();
reconciled
.load_no_lock_from_replicas(other_store.pools.clone())
.await
.expect("reload reconciled capacity");
let reservation = reconciled.pools[0]
.decommission
.as_ref()
.expect("source state")
.capacity_reservation
.as_ref()
.expect("reconciled capacity");
assert_eq!(reservation.pending_target_physical_bytes, 0);
assert_eq!(reservation.committed_data_bytes, body.len());
assert_eq!(reservation.consumed_target_physical_bytes, body.len());
assert!(reservation.targets.iter().all(|target| target.pending_mutation_id.is_none()));
assert_eq!(
other_store.pool_meta.read().await.pools[0]
.decommission
.as_ref()
.expect("worker progress")
.items_decommission_failed,
0
);
let missing = other_store.pools[0]
.get_object_info(RUSTFS_META_BUCKET, object, &ObjectOptions::default())
.await
.expect_err("the source should be cleaned only after equivalent-target capacity reconciliation");
assert!(crate::error::is_err_object_not_found(&missing));
let mut target_reader = other_store.pools[target_pool_index]
.get_object_reader(RUSTFS_META_BUCKET, object, None, HeaderMap::new(), &ObjectOptions::default())
.await
.expect("the surviving replica should remain readable");
assert_eq!(
target_reader.object_info.mod_time,
Some(target_time),
"recovery must not overwrite the existing target"
);
let mut actual = Vec::new();
target_reader
.read_to_end(&mut actual)
.await
.expect("read surviving ledger bytes");
assert_eq!(actual, body);
if existing_outside_reservation {
let (_, outside_body, outside_time) = replicas.last().expect("unreserved existing replica");
let mut outside = other_store.pools[2]
.get_object_reader(RUSTFS_META_BUCKET, object, None, HeaderMap::new(), &ObjectOptions::default())
.await
.expect("migration must leave the unreserved existing replica intact");
assert_eq!(outside.object_info.mod_time, Some(*outside_time));
let mut actual = Vec::new();
outside
.read_to_end(&mut actual)
.await
.expect("read the untouched unreserved replica");
assert_eq!(&actual, outside_body);
}
});
}
#[test]
#[serial_test::serial]
fn data_movement_equivalent_target_reconciles_published_capacity_after_restart() {
run_large_stack_current_thread_async_test(
"equivalent-target-capacity-restart",
data_movement_equivalent_target_reconciles_published_capacity_after_restart_case,
);
}
async fn data_movement_equivalent_target_reconciles_published_capacity_after_restart_case() {
let (_temp_dirs, store, other_store) =
test_three_pool_stores_with_three_disk_sets_with_isolated_node_contexts(None).await;
let bucket = test_bucket("equivalent-target");
let object = "published-before-capacity-save.bin";
let interleaved_object = "interleaved-before-capacity-reconcile.bin";
let next_object = "next-after-capacity-reconcile.bin";
let body = vec![0x54; 256 * 1024];
let source_version = uuid::Uuid::new_v4().to_string();
let interleaved_source_version = uuid::Uuid::new_v4().to_string();
let next_source_version = uuid::Uuid::new_v4().to_string();
store
.make_bucket(&bucket, &MakeBucketOptions::default())
.await
.expect("create the equivalent-target reconciliation bucket");
let incarnation = store
.bucket_incarnation_id(&bucket)
.await
.expect("load the equivalent-target reconciliation bucket incarnation");
let mut source_data = PutObjReader::from_vec(body.clone());
store.pools[0]
.put_object(
&bucket,
object,
&mut source_data,
&ObjectOptions {
versioned: true,
version_id: Some(source_version.clone()),
expected_bucket_incarnation_id: Some(incarnation),
..Default::default()
},
)
.await
.expect("seed the source object whose target will publish before the save");
let mut interleaved_source_data = PutObjReader::from_vec(body.clone());
store.pools[0]
.put_object(
&bucket,
interleaved_object,
&mut interleaved_source_data,
&ObjectOptions {
versioned: true,
version_id: Some(interleaved_source_version.clone()),
expected_bucket_incarnation_id: Some(incarnation),
..Default::default()
},
)
.await
.expect("seed the interleaved source object");
let layout = DecommissionErasureLayout { data: 1, parity: 0 };
let target_total = body.len().saturating_mul(4);
let capacity_snapshot = || {
vec![
DecommissionPoolCapacityInfo::for_test(0, layout, 0, body.len(), body.len()),
DecommissionPoolCapacityInfo::for_test(1, layout, 0, target_total, target_total),
DecommissionPoolCapacityInfo::for_test(2, layout, target_total, target_total, 0),
]
};
set_decommission_capacity_info_overrides_for_test(store.id, vec![capacity_snapshot()]);
store
.save_current_pool_meta_for_decommission_start(&[0], Vec::new())
.await
.expect("activate the source capacity reservation");
*other_store.pool_meta.write().await = store.pool_meta.read().await.clone();
let owner = decommission_capacity_owner(&*store.pool_meta.read().await);
let (lossy_store, refresh_calls) = store_with_capacity_lease_loss(&other_store).await;
set_decommission_capacity_info_overrides_for_test(
lossy_store.id,
vec![capacity_snapshot(), capacity_snapshot(), capacity_snapshot()],
);
let source_reader = store.pools[0]
.get_object_reader(
&bucket,
object,
None,
HeaderMap::new(),
&ObjectOptions {
versioned: true,
version_id: Some(source_version.clone()),
no_lock: true,
data_movement: true,
raw_data_movement_read: true,
..Default::default()
},
)
.await
.expect("read the source object for the first migration attempt");
let barrier = PutObjectCommitBarrier::install(&bucket, object, PutObjectCommitPause::AfterRenameHandoff);
let migration = tokio::spawn({
let migration_store = Arc::clone(&lossy_store);
let migration_bucket = bucket.clone();
async move {
data_movement::migrate_decommission_object(
migration_store,
0,
migration_bucket,
source_reader,
Some(incarnation),
"equivalent_target_reconcile_initial",
Some(owner),
)
.await
}
});
barrier.wait_until_paused().await;
tokio::time::pause();
let published = lossy_store.pools[2]
.get_object_info(
&bucket,
object,
&ObjectOptions {
versioned: true,
version_id: Some(source_version.clone()),
no_lock: true,
..Default::default()
},
)
.await
.expect("the target must be published before capacity progress save");
assert_eq!(published.version_id.map(|version| version.to_string()), Some(source_version.clone()));
refresh_calls.arm();
tokio::time::advance(Duration::from_secs(11)).await;
tokio::task::yield_now().await;
assert!(
refresh_calls.load(Ordering::Acquire) > 0,
"the initial capacity progress save must observe its lost lease"
);
barrier.release();
tokio::time::resume();
let initial_err = tokio::time::timeout(Duration::from_secs(30), migration)
.await
.expect("the initial migration should finish after the commit barrier release")
.expect("the initial migration task should not panic")
.expect_err("the published target must report its failed capacity progress save");
assert!(!initial_err.to_string().is_empty(), "the failed capacity save must remain observable");
drop(barrier);
let mut failed_persisted = crate::core::pools::PoolMeta::default();
failed_persisted
.load_no_lock_from_replicas(lossy_store.pools.clone())
.await
.expect("the published target failure must leave readable durable metadata");
let failed_reservation = failed_persisted.pools[0]
.decommission
.as_ref()
.and_then(|info| info.capacity_reservation.as_ref())
.expect("the failed target capacity intent must remain durable");
assert!(failed_reservation.pending_target_physical_bytes > 0);
assert!(
failed_reservation
.targets
.iter()
.any(|target| target.pending_mutation_id.is_some())
);
assert_eq!(failed_reservation.consumed_target_physical_bytes, 0);
*other_store.pool_meta.write().await = failed_persisted.clone();
set_decommission_capacity_info_overrides_for_test(
other_store.id,
vec![
capacity_snapshot(),
capacity_snapshot(),
capacity_snapshot(),
capacity_snapshot(),
capacity_snapshot(),
capacity_snapshot(),
capacity_snapshot(),
capacity_snapshot(),
capacity_snapshot(),
],
);
let retry_owner = decommission_capacity_owner(&failed_persisted);
let interleaved_reader = other_store.pools[0]
.get_object_reader(
&bucket,
interleaved_object,
None,
HeaderMap::new(),
&ObjectOptions {
versioned: true,
version_id: Some(interleaved_source_version.clone()),
no_lock: true,
data_movement: true,
raw_data_movement_read: true,
..Default::default()
},
)
.await
.expect("read the interleaved source object while the first intent is pending");
let interleaved_result = tokio::time::timeout(
Duration::from_secs(30),
data_movement::migrate_decommission_object(
Arc::clone(&other_store),
0,
bucket.clone(),
interleaved_reader,
Some(incarnation),
"equivalent_target_reconcile_interleaved",
Some(retry_owner),
),
)
.await
.expect("a different mutation must be rejected without waiting on the pending intent");
assert!(interleaved_result.is_err(), "B must not reuse A's pending capacity intent");
assert!(
interleaved_result
.as_ref()
.expect_err("B must not succeed while A's pending intent is unresolved")
.to_string()
.contains("unresolved target capacity intent")
);
let interleaved_target_err = other_store.pools[2]
.get_object_info(
&bucket,
interleaved_object,
&ObjectOptions {
versioned: true,
version_id: Some(interleaved_source_version),
no_lock: true,
..Default::default()
},
)
.await
.expect_err("B must not publish while A's pending intent is unresolved");
assert!(
crate::error::is_err_object_not_found(&interleaved_target_err)
|| crate::error::is_err_version_not_found(&interleaved_target_err)
);
let target_lock = other_store
.new_ns_lock(RUSTFS_META_BUCKET, &format!("{DECOMMISSION_CAPACITY_TARGET_LOCK_PREFIX}/2"))
.await
.expect("create the equivalent-target retry gate");
let target_guard = target_lock
.get_write_lock(Duration::from_secs(30))
.await
.expect("hold the equivalent-target gate before restart retry");
let retry_observer = DecommissionCapacityLockOrderBarrier::install(other_store.id, other_store.id);
retry_observer.disable_owner_pause();
let retry_set = other_store.pools[0].get_disks_by_key(object);
let mut retry = tokio::spawn({
let retry_store = Arc::clone(&other_store);
let retry_bucket = bucket.clone();
async move {
retry_store
.decommission_entry_for_test(
0,
MetaCacheEntry {
name: object.to_string(),
..Default::default()
},
retry_bucket,
retry_set,
)
.await
}
});
tokio::select! {
_ = retry_observer.wait_until_target_gate_retry() => {}
result = &mut retry => panic!("equivalent-target retry finished before target contention: {result:?}"),
}
drop(target_guard);
tokio::time::timeout(Duration::from_secs(30), retry)
.await
.expect("equivalent-target retry should not deadlock after permit transfer")
.expect("equivalent-target retry task should not panic")
.expect("equivalent target retry should reconcile the pending capacity intent");
assert_eq!(
retry_observer.target_gate_exact_reloads(),
1,
"equivalent reconciliation must consume its exact target permit without re-entering busy recovery"
);
drop(retry_observer);
let mut reconciled = crate::core::pools::PoolMeta::default();
reconciled
.load_no_lock_from_replicas(other_store.pools.clone())
.await
.expect("reconciled capacity metadata should remain durable");
let reconciled_reservation = reconciled.pools[0]
.decommission
.as_ref()
.and_then(|info| info.capacity_reservation.as_ref())
.expect("reconciled capacity reservation should remain present");
assert_eq!(reconciled_reservation.pending_target_physical_bytes, 0);
assert_eq!(reconciled_reservation.inflight_target_physical_bytes, 0);
assert_eq!(reconciled_reservation.consumed_target_physical_bytes, body.len());
assert_eq!(reconciled_reservation.committed_data_bytes, body.len());
let mut next_source_data = PutObjReader::from_vec(body.clone());
store.pools[0]
.put_object(
&bucket,
next_object,
&mut next_source_data,
&ObjectOptions {
versioned: true,
version_id: Some(next_source_version.clone()),
expected_bucket_incarnation_id: Some(incarnation),
..Default::default()
},
)
.await
.expect("seed the next source object for the capacity oracle");
let next_reader = other_store.pools[0]
.get_object_reader(
&bucket,
next_object,
None,
HeaderMap::new(),
&ObjectOptions {
versioned: true,
version_id: Some(next_source_version.clone()),
no_lock: true,
data_movement: true,
raw_data_movement_read: true,
..Default::default()
},
)
.await
.expect("read the next source object for the capacity oracle");
let next_result = tokio::time::timeout(
Duration::from_secs(30),
data_movement::migrate_decommission_object(
Arc::clone(&other_store),
0,
bucket.clone(),
next_reader,
Some(incarnation),
"equivalent_target_reconcile_next_object",
Some(retry_owner),
),
)
.await
.expect("the next object capacity oracle should finish promptly");
assert!(next_result.is_err(), "consumed capacity must reject the next exact-fit object");
let next_target_err = other_store.pools[2]
.get_object_info(
&bucket,
next_object,
&ObjectOptions {
versioned: true,
version_id: Some(next_source_version),
no_lock: true,
..Default::default()
},
)
.await
.expect_err("the next object must not publish after the reservation is consumed");
assert!(
crate::error::is_err_object_not_found(&next_target_err) || crate::error::is_err_version_not_found(&next_target_err),
"capacity rejection must leave the next target absent: {next_target_err}"
);
let mut final_persisted = crate::core::pools::PoolMeta::default();
final_persisted
.load_no_lock_from_replicas(other_store.pools.clone())
.await
.expect("the next-object capacity rejection must preserve durable reconciliation");
let final_reservation = final_persisted.pools[0]
.decommission
.as_ref()
.and_then(|info| info.capacity_reservation.as_ref())
.expect("the final capacity reservation should remain present");
assert_eq!(final_reservation.pending_target_physical_bytes, 0);
assert_eq!(final_reservation.consumed_target_physical_bytes, body.len());
}
#[tokio::test]
#[serial_test::serial]
async fn data_movement_equivalent_delete_marker_reconciles_pending_capacity_after_restart() {
let (_temp_dirs, store, other_store) = test_three_pool_stores_with_isolated_node_contexts(None).await;
let bucket = test_bucket("equivalent-delete-marker");
let object = "published-delete-marker.bin";
let next_object = "next-delete-marker.bin";
store
.make_bucket(&bucket, &MakeBucketOptions::default())
.await
.expect("create the equivalent delete-marker bucket");
let incarnation = store
.bucket_incarnation_id(&bucket)
.await
.expect("load the equivalent delete-marker bucket incarnation");
let mut source_body = PutObjReader::from_vec(b"delete marker source".to_vec());
store.pools[0]
.put_object(
&bucket,
object,
&mut source_body,
&ObjectOptions {
versioned: true,
expected_bucket_incarnation_id: Some(incarnation),
..Default::default()
},
)
.await
.expect("seed the source object");
let source_marker = store.pools[0]
.delete_object(
&bucket,
object,
ObjectOptions {
versioned: true,
expected_bucket_incarnation_id: Some(incarnation),
..Default::default()
},
)
.await
.expect("create the source delete marker");
let source_version_id = source_marker.version_id.expect("source marker must have a version id");
let source_mod_time = source_marker.mod_time.expect("source marker must have a modification time");
let layout = DecommissionErasureLayout { data: 1, parity: 0 };
let capacity_snapshot = || {
vec![
DecommissionPoolCapacityInfo::for_test(0, layout, 0, 1, 1),
DecommissionPoolCapacityInfo::for_test(1, layout, 0, 1, 1),
DecommissionPoolCapacityInfo::for_test(2, layout, 2, 2, 0),
]
};
set_decommission_capacity_info_overrides_for_test(store.id, vec![capacity_snapshot()]);
store
.save_current_pool_meta_for_decommission_start(&[0], Vec::new())
.await
.expect("activate the delete-marker capacity reservation");
*other_store.pool_meta.write().await = store.pool_meta.read().await.clone();
let owner = decommission_capacity_owner(&*store.pool_meta.read().await);
let mut move_opts = ObjectOptions {
data_movement: true,
delete_marker: true,
versioned: true,
version_id: Some(source_version_id.to_string()),
mod_time: Some(source_mod_time),
src_pool_idx: 0,
expected_bucket_incarnation_id: Some(incarnation),
..Default::default()
};
owner.apply_to(&mut move_opts);
let (lossy_store, refresh_calls) = store_with_capacity_lease_loss(&other_store).await;
set_decommission_capacity_info_overrides_for_test(
lossy_store.id,
vec![
capacity_snapshot(),
capacity_snapshot(),
capacity_snapshot(),
capacity_snapshot(),
capacity_snapshot(),
capacity_snapshot(),
],
);
let barrier = DeleteObjectCommitBarrier::install_after_publish(&bucket, object);
let publish_store = Arc::clone(&lossy_store);
let publish_bucket = bucket.clone();
let publish_opts = move_opts.clone();
let publish = tokio::spawn(async move { publish_store.delete_object(&publish_bucket, object, publish_opts).await });
barrier.wait_until_paused().await;
tokio::time::pause();
tokio::task::yield_now().await;
refresh_calls.arm();
tokio::time::advance(Duration::from_secs(11)).await;
tokio::task::yield_now().await;
barrier.release();
let publish_err = tokio::time::timeout(Duration::from_secs(30), publish)
.await
.expect("the published marker should finish after the deterministic barrier release")
.expect("the published marker task should not panic")
.expect_err("the final capacity progress save must observe the lost lease");
assert!(!publish_err.to_string().is_empty());
tokio::time::resume();
drop(barrier);
let published = lossy_store.pools[2]
.get_object_info(
&bucket,
object,
&ObjectOptions {
versioned: true,
version_id: Some(source_version_id.to_string()),
expected_bucket_incarnation_id: Some(incarnation),
..Default::default()
},
)
.await
.expect("the target marker must be visible before the simulated restart");
assert!(published.delete_marker);
let mut failed_persisted = crate::core::pools::PoolMeta::default();
failed_persisted
.load_no_lock_from_replicas(lossy_store.pools.clone())
.await
.expect("published marker metadata should remain readable");
let failed_reservation = failed_persisted.pools[0]
.decommission
.as_ref()
.and_then(|info| info.capacity_reservation.as_ref())
.expect("the source reservation should remain durable");
assert_eq!(failed_reservation.pending_target_physical_bytes, 1);
assert_eq!(failed_reservation.consumed_target_physical_bytes, 0);
assert!(
failed_reservation
.targets
.iter()
.any(|target| target.pending_mutation_id.is_some())
);
*other_store.pool_meta.write().await = failed_persisted.clone();
let mut restart_persisted = crate::core::pools::PoolMeta::default();
restart_persisted
.load_no_lock_from_replicas(other_store.pools.clone())
.await
.expect("the pending marker intent should survive restart");
assert_eq!(
restart_persisted.pools[0]
.decommission
.as_ref()
.and_then(|info| info.capacity_reservation.as_ref())
.expect("the restarted reservation should exist")
.pending_target_physical_bytes,
1
);
*other_store.pool_meta.write().await = restart_persisted.clone();
let mut next_source_body = PutObjReader::from_vec(b"next delete marker".to_vec());
store.pools[0]
.put_object(
&bucket,
next_object,
&mut next_source_body,
&ObjectOptions {
versioned: true,
expected_bucket_incarnation_id: Some(incarnation),
..Default::default()
},
)
.await
.expect("seed the next source object");
let next_source_marker = store.pools[0]
.delete_object(
&bucket,
next_object,
ObjectOptions {
versioned: true,
expected_bucket_incarnation_id: Some(incarnation),
..Default::default()
},
)
.await
.expect("create the next source delete marker");
let next_source_version_id = next_source_marker.version_id.expect("next marker version id");
let next_owner = decommission_capacity_owner(&restart_persisted);
let mut next_opts = ObjectOptions {
data_movement: true,
delete_marker: true,
versioned: true,
version_id: Some(next_source_version_id.to_string()),
mod_time: next_source_marker.mod_time,
src_pool_idx: 0,
expected_bucket_incarnation_id: Some(incarnation),
..Default::default()
};
next_owner.apply_to(&mut next_opts);
let next_before_reconcile = other_store
.delete_object(&bucket, next_object, next_opts.clone())
.await
.expect_err("a next mutation must not reuse the pending marker intent");
assert!(next_before_reconcile.to_string().contains("decommission_capacity_blocked"));
other_store
.delete_object(&bucket, object, move_opts)
.await
.expect("the equivalent target retry must reconcile the pending marker intent");
let mut reconciled = crate::core::pools::PoolMeta::default();
reconciled
.load_no_lock_from_replicas(other_store.pools.clone())
.await
.expect("reconciled marker metadata should remain durable");
let reconciled_reservation = reconciled.pools[0]
.decommission
.as_ref()
.and_then(|info| info.capacity_reservation.as_ref())
.expect("the reconciled reservation should exist");
assert_eq!(reconciled_reservation.pending_target_physical_bytes, 0);
assert_eq!(reconciled_reservation.consumed_target_physical_bytes, 1);
let next_after_reconcile = other_store
.delete_object(&bucket, next_object, next_opts)
.await
.expect_err("the consumed exact-fit capacity must reject the next marker");
assert!(next_after_reconcile.to_string().contains("decommission_capacity_blocked"));
let next_target_err = other_store.pools[2]
.get_object_info(
&bucket,
next_object,
&ObjectOptions {
versioned: true,
version_id: Some(next_source_version_id.to_string()),
expected_bucket_incarnation_id: Some(incarnation),
..Default::default()
},
)
.await
.expect_err("the rejected next marker must remain unpublished");
assert!(
crate::error::is_err_object_not_found(&next_target_err) || crate::error::is_err_version_not_found(&next_target_err)
);
}
#[tokio::test]
#[serial_test::serial]
async fn data_movement_delete_marker_fences_capacity_lease_loss_before_publication() {
let (_temp_dirs, store, other_store) = test_three_pool_stores_with_isolated_node_contexts(None).await;
let bucket = test_bucket("delete-marker-loss");
let object = "delete-marker-lease-loss.bin";
store
.make_bucket(&bucket, &MakeBucketOptions::default())
.await
.expect("create delete-marker lease-loss bucket");
let incarnation = store
.bucket_incarnation_id(&bucket)
.await
.expect("load delete-marker lease-loss bucket incarnation");
let mut body = PutObjReader::from_vec(b"delete-marker source".to_vec());
store.pools[0]
.put_object(
&bucket,
object,
&mut body,
&ObjectOptions {
versioned: true,
expected_bucket_incarnation_id: Some(incarnation),
..Default::default()
},
)
.await
.expect("seed the source object for delete-marker movement");
let source_marker = store.pools[0]
.delete_object(
&bucket,
object,
ObjectOptions {
versioned: true,
expected_bucket_incarnation_id: Some(incarnation),
..Default::default()
},
)
.await
.expect("create the source delete marker");
assert!(source_marker.delete_marker, "the source version must be a delete marker");
let source_version_id = source_marker
.version_id
.expect("the source delete marker must have a version id");
let source_mod_time = source_marker
.mod_time
.expect("the source delete marker must have a modification time");
let layout = DecommissionErasureLayout { data: 1, parity: 0 };
let capacity_snapshot = || {
vec![
DecommissionPoolCapacityInfo::for_test(0, layout, 0, 1024, 1024),
DecommissionPoolCapacityInfo::for_test(1, layout, 0, 4096, 4096),
DecommissionPoolCapacityInfo::for_test(2, layout, 4096, 4096, 0),
]
};
set_decommission_capacity_info_overrides_for_test(store.id, vec![capacity_snapshot()]);
store
.save_current_pool_meta_for_decommission_start(&[0], Vec::new())
.await
.expect("activate the delete-marker capacity reservation");
*other_store.pool_meta.write().await = store.pool_meta.read().await.clone();
let owner = decommission_capacity_owner(&*store.pool_meta.read().await);
assert_eq!(
store.pool_meta.read().await.pools[0]
.decommission
.as_ref()
.and_then(|info| info.capacity_reservation.as_ref())
.expect("delete-marker reservation should exist")
.targets[0]
.pool_index,
2
);
let (lossy_store, refresh_calls) = store_with_capacity_lease_loss(&other_store).await;
set_decommission_capacity_info_overrides_for_test(
lossy_store.id,
vec![capacity_snapshot(), capacity_snapshot(), capacity_snapshot()],
);
let mut move_opts = ObjectOptions {
data_movement: true,
delete_marker: true,
versioned: true,
version_id: Some(source_version_id.to_string()),
mod_time: Some(source_mod_time),
src_pool_idx: 0,
expected_bucket_incarnation_id: Some(incarnation),
..Default::default()
};
owner.apply_to(&mut move_opts);
let barrier = DeleteObjectCommitBarrier::install(&bucket, object);
let delete_store = Arc::clone(&lossy_store);
let delete_bucket = bucket.clone();
let delete = tokio::spawn(async move { delete_store.delete_object(&delete_bucket, object, move_opts).await });
barrier.wait_until_paused().await;
tokio::time::pause();
tokio::task::yield_now().await;
refresh_calls.arm();
tokio::time::advance(Duration::from_secs(11)).await;
tokio::task::yield_now().await;
assert!(
refresh_calls.load(Ordering::Acquire) > 0,
"the delete-marker capacity lease must lose refresh quorum"
);
barrier.release();
let err = tokio::time::timeout(Duration::from_secs(30), delete)
.await
.expect("delete-marker movement should finish after the commit barrier release")
.expect("delete-marker movement task should not panic")
.expect_err("lost delete-marker capacity lease must fence before publication");
assert!(!err.to_string().is_empty(), "delete-marker lease loss should remain observable");
drop(barrier);
let target_err = lossy_store.pools[2]
.get_object_info(
&bucket,
object,
&ObjectOptions {
versioned: true,
version_id: Some(source_version_id.to_string()),
expected_bucket_incarnation_id: Some(incarnation),
..Default::default()
},
)
.await
.expect_err("lost delete-marker capacity lease must not publish the target marker");
assert!(
crate::error::is_err_object_not_found(&target_err) || crate::error::is_err_version_not_found(&target_err),
"target delete marker should remain absent after the fenced failure: {target_err}"
);
let source = lossy_store.pools[0]
.get_object_info(
&bucket,
object,
&ObjectOptions {
versioned: true,
version_id: Some(source_version_id.to_string()),
expected_bucket_incarnation_id: Some(incarnation),
..Default::default()
},
)
.await
.expect("the source delete marker must remain intact");
assert!(source.delete_marker);
assert_eq!(source.mod_time, Some(source_mod_time));
let meta = lossy_store.pool_meta.read().await;
let reservation = meta.pools[0]
.decommission
.as_ref()
.and_then(|info| info.capacity_reservation.as_ref())
.expect("delete-marker reservation should remain after the fenced failure");
assert!(reservation.pending_target_physical_bytes > 0);
assert_eq!(reservation.inflight_target_physical_bytes, 0);
assert_eq!(reservation.consumed_target_physical_bytes, 0);
let live_progress = (
reservation.pending_target_physical_bytes,
reservation.inflight_target_physical_bytes,
reservation.consumed_target_physical_bytes,
reservation.observed_target_physical_bytes,
reservation.committed_data_bytes,
);
drop(meta);
let mut persisted = crate::core::pools::PoolMeta::default();
persisted
.load_no_lock_from_replicas(lossy_store.pools.clone())
.await
.expect("the fenced delete-marker failure should leave readable pool metadata");
let persisted_reservation = persisted.pools[0]
.decommission
.as_ref()
.and_then(|info| info.capacity_reservation.as_ref())
.expect("the fenced delete-marker reservation should remain persisted");
assert_eq!(
(
persisted_reservation.pending_target_physical_bytes,
persisted_reservation.inflight_target_physical_bytes,
persisted_reservation.consumed_target_physical_bytes,
persisted_reservation.observed_target_physical_bytes,
persisted_reservation.committed_data_bytes,
),
live_progress,
"fenced delete-marker progress must match its durable replica"
);
}
#[tokio::test]
#[serial_test::serial]
async fn data_movement_upload_part_fences_capacity_lease_loss_before_publication() {
let (_temp_dirs, store, other_store) = test_three_pool_stores_with_isolated_node_contexts(None).await;
let object = "data-movement-part-lease-loss.bin";
let body = vec![0x63; 256 * 1024];
let layout = DecommissionErasureLayout { data: 1, parity: 0 };
let target_total = body.len().saturating_mul(4);
let capacity_snapshot = || {
vec![
DecommissionPoolCapacityInfo::for_test(0, layout, 0, body.len(), body.len()),
DecommissionPoolCapacityInfo::for_test(1, layout, 0, target_total, target_total),
DecommissionPoolCapacityInfo::for_test(2, layout, target_total, target_total, 0),
]
};
set_decommission_capacity_info_overrides_for_test(store.id, vec![capacity_snapshot()]);
store
.save_current_pool_meta_for_decommission_start(&[0], Vec::new())
.await
.expect("activate the data-movement UploadPart reservation");
*other_store.pool_meta.write().await = store.pool_meta.read().await.clone();
let owner = decommission_capacity_owner(&*store.pool_meta.read().await);
let reservation_target = store.pool_meta.read().await.pools[0]
.decommission
.as_ref()
.and_then(|info| info.capacity_reservation.as_ref())
.expect("data-movement UploadPart reservation should exist")
.targets[0]
.pool_index;
assert_eq!(reservation_target, 2, "data-movement UploadPart should use the reserved target");
let upload = new_multipart_upload(
&store,
2,
RUSTFS_META_BUCKET,
object,
ObjectOptions {
data_movement: true,
src_pool_idx: 0,
versioned: true,
version_id: Some(uuid::Uuid::new_v4().to_string()),
..Default::default()
},
)
.await
.expect("create the data-movement UploadPart target upload");
let (lossy_store, refresh_calls) = store_with_capacity_lease_loss(&other_store).await;
set_decommission_capacity_info_overrides_for_test(
lossy_store.id,
vec![capacity_snapshot(), capacity_snapshot(), capacity_snapshot()],
);
let mut part_opts = ObjectOptions {
data_movement: true,
src_pool_idx: 0,
part_number: Some(1),
..ObjectOptions::with_capacity_expected_data_bytes(Some(body.len()))
};
owner.apply_to(&mut part_opts);
let barrier = MultipartCommitBarrier::install(RUSTFS_META_BUCKET, object, MultipartCommitPause::PutPartBeforeLockLost);
let part_store = Arc::clone(&lossy_store);
let part_upload_id = upload.upload_id.clone();
let part_opts = part_opts.clone();
let part_body = body.clone();
let part = tokio::spawn(async move {
let mut data = PutObjReader::from_vec(part_body);
part_store
.put_object_part_for_data_movement(2, RUSTFS_META_BUCKET, object, &part_upload_id, &mut data, &part_opts)
.await
});
barrier.wait_until_paused().await;
tokio::time::pause();
tokio::task::yield_now().await;
refresh_calls.arm();
tokio::time::advance(Duration::from_secs(11)).await;
tokio::task::yield_now().await;
assert!(
refresh_calls.load(Ordering::Acquire) > 0,
"the data-movement UploadPart capacity lease must lose refresh quorum"
);
barrier.release();
let err = tokio::time::timeout(Duration::from_secs(30), part)
.await
.expect("data-movement UploadPart should finish after the commit barrier release")
.expect("data-movement UploadPart task should not panic")
.expect_err("a lost data-movement UploadPart capacity lease must fence before publication");
assert!(
!err.to_string().is_empty(),
"data-movement UploadPart lease loss should remain observable"
);
drop(barrier);
let listed = lossy_store.pools[2]
.list_object_parts(
RUSTFS_META_BUCKET,
object,
&upload.upload_id,
None,
100,
&ObjectOptions {
data_movement: true,
..Default::default()
},
)
.await
.expect("the fenced data-movement UploadPart should leave the MPU readable");
assert!(
listed.parts.is_empty(),
"lost data-movement UploadPart capacity lease must not publish a part"
);
let meta = lossy_store.pool_meta.read().await;
let reservation = meta.pools[0]
.decommission
.as_ref()
.and_then(|info| info.capacity_reservation.as_ref())
.expect("data-movement UploadPart reservation should remain after the fenced failure");
assert!(reservation.pending_target_physical_bytes > 0);
assert_eq!(reservation.inflight_target_physical_bytes, 0);
assert_eq!(reservation.consumed_target_physical_bytes, 0);
let live_progress = (
reservation.pending_target_physical_bytes,
reservation.inflight_target_physical_bytes,
reservation.consumed_target_physical_bytes,
reservation.observed_target_physical_bytes,
reservation.committed_data_bytes,
);
drop(meta);
let mut persisted = crate::core::pools::PoolMeta::default();
persisted
.load_no_lock_from_replicas(lossy_store.pools.clone())
.await
.expect("the fenced data-movement UploadPart failure should leave readable pool metadata");
let persisted_reservation = persisted.pools[0]
.decommission
.as_ref()
.and_then(|info| info.capacity_reservation.as_ref())
.expect("the fenced data-movement UploadPart reservation should remain persisted");
assert_eq!(
(
persisted_reservation.pending_target_physical_bytes,
persisted_reservation.inflight_target_physical_bytes,
persisted_reservation.consumed_target_physical_bytes,
persisted_reservation.observed_target_physical_bytes,
persisted_reservation.committed_data_bytes,
),
live_progress,
"fenced data-movement UploadPart progress must match its durable replica"
);
}
async fn assert_lock_order_with_tail(mutation: ExternalObjectMutation, pause_restore_tail: bool) {
let (_temp_dirs, store, other_store) = if pause_restore_tail {
test_three_pool_stores_with_three_disk_sets_with_isolated_node_contexts(None).await
} else {
test_three_pool_stores_with_isolated_node_contexts(None).await
};
let bucket = test_bucket(&format!("{}-order", mutation.label()));
let object = "shared-object.bin";
let copy_source_object = "copy-source.bin";
let source_version = uuid::Uuid::new_v4().to_string();
let source_body = b"decommission source body".to_vec();
store
.make_bucket(&bucket, &MakeBucketOptions::default())
.await
.expect("create lock-order bucket");
let incarnation = store.bucket_incarnation_id(&bucket).await.expect("load bucket incarnation");
let mut existing = PutObjReader::from_vec(b"existing unreserved target".to_vec());
let existing_info = store.pools[2]
.put_object(&bucket, object, &mut existing, &ObjectOptions::default())
.await
.expect("seed the ordinary PUT target");
let mut source = PutObjReader::from_vec(source_body.clone());
store.pools[0]
.put_object(
&bucket,
object,
&mut source,
&ObjectOptions {
versioned: true,
version_id: Some(source_version.clone()),
..Default::default()
},
)
.await
.expect("seed the decommission source version");
let restore_get_barrier = if matches!(mutation, ExternalObjectMutation::Restore) {
let tier_name = format!("ORDERRESTORE{}", &uuid::Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
let source_tiers = store.tier_config_mgr();
let restore_tiers = other_store.tier_config_mgr();
let backend = register_mock_tier(&source_tiers, &tier_name).await;
let tier_config = source_tiers.read().await.tiers[&tier_name].clone();
restore_tiers.write().await.tiers.insert(tier_name.clone(), tier_config);
crate::services::tier::tier::TierConfigMgr::install_test_driver_in(
&restore_tiers,
&tier_name,
Box::new(backend.clone()),
)
.await
.expect("install the same mock tier on the restoring peer");
store.pools[2]
.transition_object(
&bucket,
object,
&ObjectOptions {
no_lock: true,
transition: crate::bucket::lifecycle::lifecycle::TransitionOptions {
status: TRANSITION_PENDING.to_string(),
tier: tier_name,
etag: existing_info.etag.clone().expect("the restore target should have an ETag"),
..Default::default()
},
mod_time: existing_info.mod_time,
..Default::default()
},
)
.await
.expect("seed the transitioned restore target");
Some(backend.arm_get_barrier().await)
} else {
None
};
let delete_marker_version = if matches!(mutation, ExternalObjectMutation::Delete) {
let marker = store.pools[2]
.delete_object(
&bucket,
object,
ObjectOptions {
versioned: true,
..Default::default()
},
)
.await
.expect("seed the ordinary target delete marker");
Some(
marker
.version_id
.expect("the versioned target delete should return its marker version")
.to_string(),
)
} else {
None
};
let prepared_multipart = if matches!(mutation, ExternalObjectMutation::CompleteMultipart) {
let upload = new_multipart_upload(
&store,
2,
&bucket,
object,
ObjectOptions {
expected_bucket_incarnation_id: Some(incarnation),
..Default::default()
},
)
.await
.expect("create the ordinary multipart upload");
let multipart_opts = multipart_options(
&store,
&bucket,
ObjectOptions {
expected_bucket_incarnation_id: Some(incarnation),
..Default::default()
},
)
.await;
let mut data = PutObjReader::from_vec(b"ordinary multipart replacement".to_vec());
let part = store.pools[2]
.put_object_part(&bucket, object, &upload.upload_id, 1, &mut data, &multipart_opts)
.await
.expect("stage the ordinary multipart part");
Some((
upload.upload_id,
vec![crate::storage_api_contracts::multipart::CompletePart {
part_num: part.part_num,
etag: part.etag,
..Default::default()
}],
))
} else {
None
};
let copy_body_size = if matches!(mutation, ExternalObjectMutation::Copy) {
if pause_restore_tail {
256 * 1024
} else {
b"ordinary copy replacement".len()
}
} else {
0
};
let prepared_copy = if matches!(mutation, ExternalObjectMutation::Copy) {
let copy_body = vec![0x6a; copy_body_size];
let mut copy_source = PutObjReader::from_vec(copy_body.clone());
store.pools[2]
.put_object(&bucket, copy_source_object, &mut copy_source, &ObjectOptions::default())
.await
.expect("seed the ordinary copy source");
let mut copy_reader = store.pools[2]
.get_object_reader(
&bucket,
copy_source_object,
None,
HeaderMap::new(),
&ObjectOptions {
no_lock: true,
..Default::default()
},
)
.await
.expect("read the ordinary copy source");
let mut copy_info = copy_reader.object_info.clone();
let mut copy_data = Vec::new();
copy_reader
.stream
.read_to_end(&mut copy_data)
.await
.expect("drain the ordinary copy source");
assert_eq!(copy_data, copy_body);
copy_info.put_object_reader = Some(PutObjReader::from_vec(copy_data));
Some(copy_info)
} else {
None
};
let layout = DecommissionErasureLayout { data: 1, parity: 0 };
let target_total = source_body.len().max(copy_body_size) * 4;
let capacity_snapshot = |target_free| {
vec![
DecommissionPoolCapacityInfo::for_test(0, layout, 0, source_body.len(), source_body.len()),
DecommissionPoolCapacityInfo::for_test(
1,
layout,
target_free,
target_total,
target_total.saturating_sub(target_free),
),
DecommissionPoolCapacityInfo::for_test(2, layout, target_total, target_total, 0),
]
};
set_decommission_capacity_info_overrides_for_test(store.id, vec![capacity_snapshot(target_total)]);
store
.save_current_pool_meta_for_decommission_start(&[0], Vec::new())
.await
.expect("activate the source reservation");
*other_store.pool_meta.write().await = store.pool_meta.read().await.clone();
let owner = decommission_capacity_owner(&*store.pool_meta.read().await);
set_decommission_capacity_info_overrides_for_test(
store.id,
vec![
capacity_snapshot(target_total),
capacity_snapshot(target_total),
capacity_snapshot(target_total - source_body.len()),
],
);
let source_reader = store.pools[0]
.get_object_reader(
&bucket,
object,
None,
HeaderMap::new(),
&ObjectOptions {
versioned: true,
version_id: Some(source_version.clone()),
no_lock: true,
data_movement: true,
raw_data_movement_read: true,
..Default::default()
},
)
.await
.expect("read the exact source version");
let barrier = DecommissionCapacityLockOrderBarrier::install(store.id, other_store.id);
let mut migration = tokio::spawn({
let migration_store = Arc::clone(&store);
let migration_bucket = bucket.clone();
async move {
let result = data_movement::migrate_decommission_object(
migration_store,
0,
migration_bucket,
source_reader,
Some(incarnation),
"decommission_object_lock_order",
Some(owner),
)
.await;
(result, std::time::Instant::now())
}
});
barrier.wait_until_owner_paused().await;
let ordinary_launched_at = std::time::Instant::now();
let mut ordinary_mutation = tokio::spawn({
let ordinary_store = Arc::clone(&other_store);
let ordinary_bucket = bucket.clone();
async move {
let ordinary_started_at = std::time::Instant::now();
let result = match mutation {
ExternalObjectMutation::Put => {
let mut data = PutObjReader::from_vec(b"ordinary replacement".to_vec());
ordinary_store
.put_object(&ordinary_bucket, object, &mut data, &ObjectOptions::default())
.await
.map(|_| ())
}
ExternalObjectMutation::CompleteMultipart => {
let (upload_id, completed_parts) =
prepared_multipart.expect("multipart mutation should have a staged upload");
ordinary_store
.complete_multipart_upload(
&ordinary_bucket,
object,
&upload_id,
completed_parts,
&ObjectOptions::default(),
)
.await
.map(|_| ())
}
ExternalObjectMutation::Copy => {
let mut copy_info = prepared_copy.expect("copy mutation should have a source reader");
ordinary_store
.copy_object(
&ordinary_bucket,
copy_source_object,
&ordinary_bucket,
object,
&mut copy_info,
&ObjectOptions::default(),
&ObjectOptions::default(),
)
.await
.map(|_| ())
}
ExternalObjectMutation::PutTags => ordinary_store
.put_object_tags(&ordinary_bucket, object, "ordinary=tag", &ObjectOptions::default())
.await
.map(|_| ()),
ExternalObjectMutation::DeleteTags => ordinary_store
.delete_object_tags(&ordinary_bucket, object, &ObjectOptions::default())
.await
.map(|_| ()),
ExternalObjectMutation::PutMetadata => ordinary_store
.put_object_metadata(&ordinary_bucket, object, &ObjectOptions::default())
.await
.map(|_| ()),
ExternalObjectMutation::Delete => {
let version_id = delete_marker_version.expect("delete mutation should have a marker version");
ordinary_store
.delete_object(
&ordinary_bucket,
object,
ObjectOptions {
versioned: true,
version_id: Some(version_id.clone()),
expected_current_version_id: Some(version_id),
..Default::default()
},
)
.await
.map(|_| ())
}
ExternalObjectMutation::Restore => {
let mut opts = ObjectOptions::default();
opts.transition.restore_request.days = Some(1);
ordinary_store
.restore_transitioned_object(&ordinary_bucket, object, &opts)
.await
}
};
(result, ordinary_started_at, std::time::Instant::now())
}
});
if let Some(get_barrier) = restore_get_barrier.as_ref() {
tokio::select! {
() = get_barrier.wait_until_paused() => {}
result = &mut ordinary_mutation => panic!("restore finished before the tier GET barrier: {result:?}"),
}
let read_opts = ObjectOptions {
skip_decommissioned: true,
..Default::default()
};
tokio::time::timeout(
std::time::Duration::from_secs(1),
other_store.get_object_info(&bucket, object, &read_opts),
)
.await
.expect("HEAD must not wait for the paused tier restore download")
.expect("HEAD should remain readable during the paused tier restore download");
let mut reader = tokio::time::timeout(
std::time::Duration::from_secs(1),
other_store.get_object_reader(&bucket, object, None, HeaderMap::new(), &read_opts),
)
.await
.expect("GET must not wait for the paused tier restore download")
.expect("GET should remain readable during the paused tier restore download");
let mut body = Vec::new();
tokio::time::timeout(std::time::Duration::from_secs(1), reader.stream.read_to_end(&mut body))
.await
.expect("GET body must not wait for the paused tier restore download")
.expect("GET body should remain readable during the paused tier restore download");
assert_eq!(body, b"existing unreserved target");
get_barrier.release();
}
if matches!(
mutation,
ExternalObjectMutation::Restore
| ExternalObjectMutation::Put
| ExternalObjectMutation::CompleteMultipart
| ExternalObjectMutation::Copy
) {
if pause_restore_tail {
barrier.pause_external_object_commit_phase();
}
barrier.wait_until_external_object_commit_phase_started().await;
} else {
barrier.wait_until_external_capacity_released().await;
}
let ordinary_lock_phase_observed_at = std::time::Instant::now();
assert!(!ordinary_mutation.is_finished());
let owner_released_at = std::time::Instant::now();
let (migration_result, migration_finished_at) = if pause_restore_tail {
barrier.release_owner();
tokio::time::timeout(std::time::Duration::from_secs(30), &mut migration)
.await
.expect("migration must finish before the early-ACK tail is released")
.expect("migration task should join")
} else {
barrier.release_owner();
tokio::time::timeout(std::time::Duration::from_secs(30), migration)
.await
.expect("migration must not deadlock with the ordinary object mutation")
.expect("migration task should join")
};
migration_result.expect("migration should commit to its reserved target");
let (ordinary_result, ordinary_started_at, ordinary_finished_at) = if pause_restore_tail {
let handoff_barrier = PutObjectCommitBarrier::install(&bucket, object, PutObjectCommitPause::AfterRenameHandoff);
let tail_barrier =
crate::set_disk::rename_fanout_barrier::arm(object, 0, crate::set_disk::rename_fanout_barrier::PHASE_RENAME);
barrier.release_external_object_commit_phase();
tokio::time::timeout(std::time::Duration::from_secs(30), handoff_barrier.wait_until_paused())
.await
.expect("restore should reach the quorum handoff before the tail probe");
handoff_barrier.release();
drop(handoff_barrier);
tokio::time::timeout(std::time::Duration::from_secs(30), tail_barrier.wait_until_paused())
.await
.expect("restore should pause a tail rename after quorum publication");
let object_lock = other_store
.new_ns_lock(&bucket, object)
.await
.expect("create the shared-domain object guard probe");
let mut object_probe =
tokio::spawn(async move { object_lock.get_write_lock(std::time::Duration::from_secs(30)).await });
assert!(
tokio::time::timeout(std::time::Duration::from_millis(50), &mut object_probe)
.await
.is_err(),
"early-ACK tail must retain the decommission namespace guard"
);
let capacity_lock = other_store
.new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME)
.await
.expect("create the decommission capacity guard probe");
let mut capacity_probe =
tokio::spawn(async move { capacity_lock.get_write_lock(std::time::Duration::from_secs(30)).await });
assert!(
tokio::time::timeout(std::time::Duration::from_millis(50), &mut capacity_probe)
.await
.is_err(),
"early-ACK tail must retain the decommission capacity guard"
);
capacity_probe.abort();
let _ = capacity_probe.await;
tail_barrier.release();
drop(tail_barrier);
let object_guard = tokio::time::timeout(std::time::Duration::from_secs(30), object_probe)
.await
.expect("object guard probe should finish after the tail")
.expect("object guard probe should join")
.expect("object guard probe should acquire after the tail");
drop(object_guard);
let capacity_lock = other_store
.new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME)
.await
.expect("create the post-tail decommission capacity guard probe");
let capacity_guard = tokio::time::timeout(
std::time::Duration::from_secs(30),
capacity_lock.get_write_lock(std::time::Duration::from_secs(30)),
)
.await
.expect("capacity guard probe should finish after the tail")
.expect("capacity guard probe should acquire after the tail");
drop(capacity_guard);
tokio::time::timeout(std::time::Duration::from_secs(30), &mut ordinary_mutation)
.await
.expect("restore should return its early quorum acknowledgement")
.expect("restore task should join")
} else {
tokio::time::timeout(std::time::Duration::from_secs(30), ordinary_mutation)
.await
.expect("ordinary object mutation must finish after the migration releases its object fence")
.expect("ordinary object mutation task should join")
};
drop(barrier);
ordinary_result.unwrap_or_else(|error| {
panic!(
"ordinary object mutation should commit to the unreserved target: {error:?}; \
ordinary_start_after_launch={:?}, lock_phase_observed_after_launch={:?}, \
owner_release_after_launch={:?}, migration_after_owner_release={:?}, \
ordinary_finish_after_launch={:?}, ordinary_finished_before_migration={}",
ordinary_started_at.duration_since(ordinary_launched_at),
ordinary_lock_phase_observed_at.duration_since(ordinary_launched_at),
owner_released_at.duration_since(ordinary_launched_at),
migration_finished_at.duration_since(owner_released_at),
ordinary_finished_at.duration_since(ordinary_launched_at),
ordinary_finished_at < migration_finished_at,
);
});
store.pools[1]
.get_object_info(
&bucket,
object,
&ObjectOptions {
versioned: true,
version_id: Some(source_version),
..Default::default()
},
)
.await
.expect("migration should preserve the source version on the reserved target");
store.pools[2]
.get_object_info(&bucket, object, &ObjectOptions::default())
.await
.expect("ordinary object mutation should stay on the unreserved target");
if matches!(mutation, ExternalObjectMutation::Restore) {
let mut restored = store.pools[2]
.get_object_reader(&bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
.await
.expect("restored target should be readable");
let mut restored_body = Vec::new();
restored
.stream
.read_to_end(&mut restored_body)
.await
.expect("restored target body should be readable");
assert_eq!(restored_body, b"existing unreserved target");
}
}
async fn assert_same_object_historical_copy_lock_order() {
let (_temp_dirs, store, other_store) = test_three_pool_stores_with_isolated_node_contexts(None).await;
let bucket = test_bucket("same-object-copy-order");
let object = "same-object.bin";
let migration_version = uuid::Uuid::new_v4().to_string();
let historical_version = uuid::Uuid::new_v4().to_string();
let current_version = uuid::Uuid::new_v4().to_string();
let migration_body = b"decommission migration body".to_vec();
store
.make_bucket(&bucket, &MakeBucketOptions::default())
.await
.expect("create same-object copy lock-order bucket");
let incarnation = store
.bucket_incarnation_id(&bucket)
.await
.expect("load same-object copy bucket incarnation");
let mut migration_source = PutObjReader::from_vec(migration_body.clone());
store.pools[0]
.put_object(
&bucket,
object,
&mut migration_source,
&ObjectOptions {
versioned: true,
version_id: Some(migration_version.clone()),
..Default::default()
},
)
.await
.expect("seed the migration source version");
let mut historical_source = PutObjReader::from_vec(b"historical copy body".to_vec());
store.pools[2]
.put_object(
&bucket,
object,
&mut historical_source,
&ObjectOptions {
versioned: true,
version_id: Some(historical_version.clone()),
..Default::default()
},
)
.await
.expect("seed the historical copy version");
let mut current_source = PutObjReader::from_vec(b"current copy body".to_vec());
store.pools[2]
.put_object(
&bucket,
object,
&mut current_source,
&ObjectOptions {
versioned: true,
version_id: Some(current_version.clone()),
..Default::default()
},
)
.await
.expect("seed the current copy version");
let mut copy_info = store.pools[2]
.get_object_info(
&bucket,
object,
&ObjectOptions {
version_id: Some(historical_version.clone()),
versioned: true,
..Default::default()
},
)
.await
.expect("historical copy version should be readable");
assert_eq!(
store.pools[2]
.get_object_info(
&bucket,
object,
&ObjectOptions {
versioned: true,
..Default::default()
},
)
.await
.expect("current copy version should be readable")
.version_id
.map(|version| version.to_string()),
Some(current_version.clone())
);
let layout = DecommissionErasureLayout { data: 1, parity: 0 };
let target_total = migration_body.len() * 4;
let capacity_snapshot = |target_free| {
vec![
DecommissionPoolCapacityInfo::for_test(0, layout, 0, migration_body.len(), migration_body.len()),
DecommissionPoolCapacityInfo::for_test(
1,
layout,
target_free,
target_total,
target_total.saturating_sub(target_free),
),
DecommissionPoolCapacityInfo::for_test(2, layout, 0, target_total, target_total),
]
};
set_decommission_capacity_info_overrides_for_test(store.id, vec![capacity_snapshot(target_total)]);
store
.save_current_pool_meta_for_decommission_start(&[0], Vec::new())
.await
.expect("activate the same-object copy source reservation");
*other_store.pool_meta.write().await = store.pool_meta.read().await.clone();
let owner = decommission_capacity_owner(&*store.pool_meta.read().await);
set_decommission_capacity_info_overrides_for_test(
store.id,
vec![
capacity_snapshot(target_total),
capacity_snapshot(target_total),
capacity_snapshot(target_total - migration_body.len()),
],
);
let source_reader = store.pools[0]
.get_object_reader(
&bucket,
object,
None,
HeaderMap::new(),
&ObjectOptions {
versioned: true,
version_id: Some(migration_version),
no_lock: true,
data_movement: true,
raw_data_movement_read: true,
..Default::default()
},
)
.await
.expect("read the migration source version");
let barrier = DecommissionCapacityLockOrderBarrier::install(store.id, other_store.id);
let migration = tokio::spawn({
let migration_store = Arc::clone(&store);
let migration_bucket = bucket.clone();
async move {
let result = data_movement::migrate_decommission_object(
migration_store,
0,
migration_bucket,
source_reader,
Some(incarnation),
"same_object_copy_lock_order",
Some(owner),
)
.await;
(result, std::time::Instant::now())
}
});
barrier.wait_until_owner_paused().await;
let ordinary_launched_at = std::time::Instant::now();
let copy_task = tokio::spawn({
let copy_store = Arc::clone(&other_store);
let copy_bucket = bucket.clone();
let source_opts = ObjectOptions {
versioned: true,
version_id: Some(historical_version),
..Default::default()
};
let destination_opts = ObjectOptions {
expected_current_version_id: Some(current_version),
versioned: true,
..Default::default()
};
async move {
let ordinary_started_at = std::time::Instant::now();
let result = copy_store
.copy_object(
©_bucket,
object,
©_bucket,
object,
&mut copy_info,
&source_opts,
&destination_opts,
)
.await
.map(|_| ());
(result, ordinary_started_at, std::time::Instant::now())
}
});
barrier.wait_until_external_capacity_released().await;
let ordinary_lock_phase_observed_at = std::time::Instant::now();
assert!(
!copy_task.is_finished(),
"historical same-object CopyObject must wait behind the migration object fence"
);
let owner_released_at = std::time::Instant::now();
barrier.release_owner();
drop(barrier);
let (migration_result, migration_finished_at) = tokio::time::timeout(std::time::Duration::from_secs(30), migration)
.await
.expect("migration must not deadlock with historical same-object CopyObject")
.expect("migration task should join");
migration_result.expect("migration should commit to its reserved target");
let (ordinary_result, ordinary_started_at, ordinary_finished_at) =
tokio::time::timeout(std::time::Duration::from_secs(30), copy_task)
.await
.expect("historical same-object CopyObject must finish after the migration releases its object fence")
.expect("CopyObject task should join");
ordinary_result.unwrap_or_else(|error| {
panic!(
"CopyObject should preserve the expected current-version precondition: {error:?}; \
ordinary_start_after_launch={:?}, lock_phase_observed_after_launch={:?}, \
owner_release_after_launch={:?}, migration_after_owner_release={:?}, \
ordinary_finish_after_launch={:?}, ordinary_finished_before_migration={}",
ordinary_started_at.duration_since(ordinary_launched_at),
ordinary_lock_phase_observed_at.duration_since(ordinary_launched_at),
owner_released_at.duration_since(ordinary_launched_at),
migration_finished_at.duration_since(owner_released_at),
ordinary_finished_at.duration_since(ordinary_launched_at),
ordinary_finished_at < migration_finished_at,
);
});
}
#[tokio::test]
#[serial_test::serial]
async fn active_decommission_migration_does_not_invert_capacity_and_unreserved_put_locks() {
assert_lock_order(ExternalObjectMutation::Put).await;
}
#[tokio::test]
#[serial_test::serial]
async fn active_decommission_migration_does_not_invert_capacity_and_multipart_complete_locks() {
assert_lock_order(ExternalObjectMutation::CompleteMultipart).await;
}
#[tokio::test]
#[serial_test::serial]
async fn active_decommission_migration_does_not_invert_capacity_and_cross_object_copy_locks() {
assert_lock_order(ExternalObjectMutation::Copy).await;
}
#[tokio::test]
#[serial_test::serial]
async fn public_cross_object_copy_holds_decommission_capacity_through_early_ack_tail() {
temp_env::async_with_vars([(crate::set_disk::ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async {
assert_lock_order_with_tail(ExternalObjectMutation::Copy, true).await
})
.await;
}
#[tokio::test]
#[serial_test::serial]
async fn active_decommission_migration_does_not_invert_capacity_and_same_object_historical_copy_locks() {
assert_same_object_historical_copy_lock_order().await;
}
#[tokio::test]
#[serial_test::serial]
async fn active_decommission_migration_does_not_invert_capacity_and_put_object_tag_locks() {
assert_lock_order(ExternalObjectMutation::PutTags).await;
}
#[tokio::test]
#[serial_test::serial]
async fn active_decommission_migration_does_not_invert_capacity_and_delete_object_tag_locks() {
assert_lock_order(ExternalObjectMutation::DeleteTags).await;
}
#[tokio::test]
#[serial_test::serial]
async fn active_decommission_migration_does_not_invert_capacity_and_put_object_metadata_locks() {
assert_lock_order(ExternalObjectMutation::PutMetadata).await;
}
#[tokio::test]
#[serial_test::serial]
async fn active_decommission_migration_does_not_invert_capacity_and_delete_object_locks() {
assert_lock_order(ExternalObjectMutation::Delete).await;
}
#[test]
#[serial_test::serial]
fn active_decommission_migration_does_not_invert_capacity_and_restore_locks() {
run_large_stack_async_test("decommission-restore-lock-order", || async {
temp_env::async_with_vars([(crate::set_disk::ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async {
assert_lock_order_with_tail(ExternalObjectMutation::Restore, true).await
})
.await;
});
}
}