1use super::*;
16use crate::core::pools::{
17 PoolMetaReplicaState, PoolMetaWriteState, local_decommission_queue_prefix, persist_pool_meta_identity_for_attested_pools,
18 persist_pool_meta_identity_for_startup, pool_meta_has_active_decommission,
19};
20use crate::runtime::instance::InstanceContext;
21use crate::runtime::sources as runtime_sources;
22use crate::storage_api_contracts::object::EcstoreObjectIO;
23use rustfs_config::server_config::KVS;
24use rustfs_credentials::{RPC_SECRET_REQUIRED_OPERATOR_MESSAGE, try_get_rpc_token};
25use std::future::Future;
26use tracing::{debug, error, info, warn};
27
28const LOG_COMPONENT_ECSTORE: &str = "ecstore";
29const LOG_SUBSYSTEM_STORE_INIT: &str = "store_init";
30const EVENT_DECOMMISSION_RESUME_RETRY: &str = "decommission_resume_retry";
31const EVENT_STORE_FORMAT_RETRY: &str = "store_format_retry";
32const EVENT_ECSTORE_INIT_STATUS: &str = "ecstore_init_status";
33const EVENT_STORE_RPC_SECRET_PREFLIGHT_FAILED: &str = "store_rpc_secret_preflight_failed";
34
35fn pool_first_endpoint_is_local(pool: &crate::layout::endpoints::PoolEndpoints) -> bool {
36 pool.endpoints.as_ref().first().is_some_and(|endpoint| endpoint.is_local)
37}
38
39fn startup_pool_drive_counts(endpoint_pools: &EndpointServerPools) -> Vec<usize> {
40 endpoint_pools.as_ref().iter().map(|pool| pool.drives_per_set).collect()
41}
42
43fn resolve_startup_pool_defaults(endpoint_pools: &EndpointServerPools) -> Result<Vec<usize>> {
44 resolve_startup_pool_defaults_with(endpoint_pools, ECStore::validate_startup_storage_class)
45}
46
47fn resolve_startup_pool_defaults_with(
48 endpoint_pools: &EndpointServerPools,
49 validate: impl FnOnce(&EndpointServerPools) -> Result<()>,
50) -> Result<Vec<usize>> {
51 validate(endpoint_pools)?;
52 let drive_counts = startup_pool_drive_counts(endpoint_pools);
53 drive_counts.into_iter().map(ec_drives_no_config).collect()
54}
55
56fn preflight_startup_rpc_secret(endpoint_pools: &EndpointServerPools) -> Result<()> {
61 preflight_startup_rpc_secret_with(endpoint_pools, try_get_rpc_token)
62}
63
64fn preflight_startup_rpc_secret_with(
65 endpoint_pools: &EndpointServerPools,
66 resolve_rpc_token: impl FnOnce() -> std::io::Result<String>,
67) -> Result<()> {
68 let has_remote_endpoint = endpoint_pools
69 .as_ref()
70 .iter()
71 .flat_map(|pool| pool.endpoints.as_ref().iter())
72 .any(|endpoint| !endpoint.is_local);
73
74 if !has_remote_endpoint {
75 return Ok(());
76 }
77
78 match resolve_rpc_token() {
79 Ok(_) => Ok(()),
80 Err(err) => {
81 error!(
86 event = EVENT_STORE_RPC_SECRET_PREFLIGHT_FAILED,
87 component = LOG_COMPONENT_ECSTORE,
88 subsystem = LOG_SUBSYSTEM_STORE_INIT,
89 "RPC auth secret resolution failed: {err}; {RPC_SECRET_REQUIRED_OPERATOR_MESSAGE}"
90 );
91 Err(Error::other(format!(
92 "store init aborted: endpoints include remote nodes but {err}; {RPC_SECRET_REQUIRED_OPERATOR_MESSAGE}"
93 )))
94 }
95 }
96}
97
98const LOCAL_DECOMMISSION_RESUME_RETRY_DELAY: Duration = Duration::from_secs(30);
99const LOCAL_DECOMMISSION_WATCHDOG_INTERVAL: Duration = Duration::from_secs(30);
100const LOCAL_DECOMMISSION_WATCHDOG_MAX_RETRY_DELAY: Duration = Duration::from_secs(60 * 5);
101const REBALANCE_INITIAL_RESUME_DELAY: Duration = Duration::from_secs(10);
102const REBALANCE_RESUME_RETRY_DELAY: Duration = Duration::from_secs(10);
103
104fn should_retry_format_load(err: &Error) -> bool {
105 !matches!(
106 err,
107 Error::CorruptedFormat | Error::UnsupportedSnsdExpansion { .. } | Error::PoolTopologyMismatch { .. }
108 )
109}
110
111fn should_auto_start_rebalance_after_init(decommission_running: bool, rebalance_resume_required: bool) -> bool {
112 rebalance_resume_required && !decommission_running
113}
114
115fn should_defer_rebalance_auto_start(distributed: bool, fleet_proof_available: bool) -> bool {
116 distributed && !fleet_proof_available
117}
118
119async fn wait_for_local_decommission_resume_delay(rx: &CancellationToken, delay: Duration) -> bool {
120 tokio::select! {
121 _ = rx.cancelled() => false,
122 _ = tokio::time::sleep(delay) => true,
123 }
124}
125
126fn local_decommission_watchdog_retry_delay(consecutive_failures: u32) -> Duration {
127 let exponent = consecutive_failures.saturating_sub(1).min(4);
128 LOCAL_DECOMMISSION_RESUME_RETRY_DELAY
129 .saturating_mul(1_u32 << exponent)
130 .min(LOCAL_DECOMMISSION_WATCHDOG_MAX_RETRY_DELAY)
131}
132
133async fn wait_for_rebalance_resume_delay(rx: &CancellationToken, delay: Duration) -> bool {
134 tokio::select! {
135 _ = rx.cancelled() => false,
136 _ = tokio::time::sleep(delay) => true,
137 }
138}
139
140async fn wait_for_rebalance_resume_retry(rx: &CancellationToken) -> bool {
141 wait_for_rebalance_resume_delay(rx, REBALANCE_RESUME_RETRY_DELAY).await
142}
143
144fn resolve_store_init_stage_result(result: Result<()>, stage: &str) -> Result<()> {
145 result.map_err(|err| Error::other(format!("store init failed during {stage}: {err}")))
146}
147
148async fn load_pool_meta_for_startup<S>(
149 pools: Vec<Arc<S>>,
150 write_state: &mut PoolMetaWriteState,
151) -> Result<(PoolMeta, PoolMetaReplicaState)>
152where
153 S: EcstoreObjectIO,
154{
155 let mut meta = PoolMeta::default();
156 let replica_state = meta
157 .load_for_startup_observing(pools, write_state)
158 .await
159 .map_err(|err| Error::other_with_context("store init failed during load_pool_meta", err))?;
160 write_state.observe_replicas(replica_state);
161 write_state
162 .ensure_missing_metadata_can_initialize()
163 .map_err(|err| Error::other(format!("store init failed during classify_pool_meta_absence: {err}")))?;
164 Ok((meta, replica_state))
165}
166
167async fn establish_pool_meta_bootstrap_identity_if_proven<S>(
168 pools: Vec<Arc<S>>,
169 write_state: &mut PoolMetaWriteState,
170 elected_writer: bool,
171) -> Result<()>
172where
173 S: EcstoreObjectIO,
174{
175 if elected_writer && write_state.bootstrap_identity_proven() {
176 return persist_pool_meta_identity_for_startup(pools, write_state, false).await;
177 }
178 if write_state.bootstrap_identity_proven() {
179 return Ok(());
180 }
181 let attested = write_state.attested_pool_indices();
186 if attested.is_empty() {
187 return Ok(());
188 }
189 persist_pool_meta_identity_for_attested_pools(pools, write_state, &attested).await
190}
191
192async fn save_validated_pool_meta_for_startup<S>(
193 meta: &PoolMeta,
194 pools: Vec<Arc<S>>,
195 write_state: &mut PoolMetaWriteState,
196) -> Result<PoolMeta>
197where
198 S: EcstoreObjectIO,
199{
200 meta.save_for_startup_observing(pools, write_state)
201 .await
202 .map_err(|err| Error::other(format!("store init failed during save_validated_pool_meta: {err}")))
203}
204
205async fn persist_pool_meta_for_startup_if_safe<S>(
206 meta: &PoolMeta,
207 pools: Vec<Arc<S>>,
208 replica_state: PoolMetaReplicaState,
209 write_state: &mut PoolMetaWriteState,
210 topology_update: bool,
211 elected_writer: bool,
212) -> Result<PoolMeta>
213where
214 S: EcstoreObjectIO,
215{
216 if !elected_writer {
217 return Ok(meta.clone());
218 }
219 let should_write = topology_update || (replica_state.needs_repair && replica_state.repair_write_safe);
220 if topology_update {
221 replica_state.ensure_write_safe("store init failed during save_validated_pool_meta")?;
222 }
223 if should_write || write_state.identity_requires_repair() {
224 write_state.ensure_write_safe("store init failed during save_validated_pool_meta")?;
225 }
226 let mut committed = meta.clone();
227 if should_write {
228 if write_state.bootstrap_identity_proven() || write_state.identity_is_pending() {
229 persist_pool_meta_identity_for_startup(pools.clone(), write_state, false)
230 .await
231 .map_err(|err| Error::other(format!("store init failed during prepare_pool_meta_identity: {err}")))?;
232 }
233 committed = save_validated_pool_meta_for_startup(meta, pools.clone(), write_state).await?;
234 }
235 if should_write || write_state.identity_requires_repair() {
236 persist_pool_meta_identity_for_startup(pools, write_state, true)
237 .await
238 .map_err(|err| Error::other(format!("store init failed during commit_pool_meta_identity: {err}")))?;
239 }
240 Ok(committed)
241}
242
243async fn run_local_decommission_watchdog<F, Fut>(rx: CancellationToken, mut reconcile: F)
244where
245 F: FnMut() -> Fut,
246 Fut: Future<Output = Result<()>>,
247{
248 let mut consecutive_failures = 0_u32;
249
250 loop {
251 if rx.is_cancelled() {
252 return;
253 }
254
255 let delay = match reconcile().await {
256 Ok(()) => {
257 consecutive_failures = 0;
258 LOCAL_DECOMMISSION_WATCHDOG_INTERVAL
259 }
260 Err(err) => {
261 consecutive_failures = consecutive_failures.saturating_add(1);
262 let retry_delay = local_decommission_watchdog_retry_delay(consecutive_failures);
263 warn!(
264 event = EVENT_DECOMMISSION_RESUME_RETRY,
265 component = LOG_COMPONENT_ECSTORE,
266 subsystem = LOG_SUBSYSTEM_STORE_INIT,
267 consecutive_failures,
268 retry_delay_secs = retry_delay.as_secs(),
269 error = %err,
270 "Retrying decommission worker recovery"
271 );
272 retry_delay
273 }
274 };
275
276 if !wait_for_local_decommission_resume_delay(&rx, delay).await {
277 return;
278 }
279 }
280}
281
282async fn reconcile_local_decommission_after_init(store: &Arc<ECStore>, rx: CancellationToken) -> Result<()> {
283 store
284 .ensure_pool_meta_side_effects_safe("decommission worker recovery blocked while pool metadata requires recovery")
285 .await?;
286 if store.has_active_local_decommission_worker().await {
287 return Ok(());
288 }
289 store.refresh_pool_status_meta().await?;
290 let resume_required = pool_meta_has_active_decommission(&*store.pool_meta.read().await);
291 if resume_required {
292 crate::core::pools::acquire_pool_activation_fleet_proof(&store.ctx).await?;
293 }
294 store.spawn_missing_local_decommission_routines_with_token(rx).await
295}
296
297async fn supervise_local_decommission_after_init(store: Arc<ECStore>, rx: CancellationToken) {
298 run_local_decommission_watchdog(rx.clone(), || {
299 let store = store.clone();
300 let worker_rx = rx.clone();
301 async move { reconcile_local_decommission_after_init(&store, worker_rx).await }
302 })
303 .await;
304}
305
306async fn resume_rebalance_after_init(store: Arc<ECStore>, rx: CancellationToken) {
307 if !wait_for_rebalance_resume_delay(&rx, REBALANCE_INITIAL_RESUME_DELAY).await {
308 return;
309 }
310
311 loop {
312 if rx.is_cancelled() {
313 return;
314 }
315
316 let resume_required = store
317 .rebalance_meta
318 .read()
319 .await
320 .as_ref()
321 .is_some_and(crate::services::rebalance::rebalance_requires_worker_activation);
322 if !resume_required {
323 return;
324 }
325
326 if should_defer_rebalance_auto_start(
327 store.ctx.is_dist_erasure().await,
328 crate::services::notification_sys::acquire_cross_pool_fence_fleet_proof().is_some(),
329 ) {
330 if !wait_for_rebalance_resume_retry(&rx).await {
331 return;
332 }
333 continue;
334 }
335
336 match store.start_rebalance().await {
337 Ok(()) => return,
338 Err(err) if crate::core::pools::is_pool_activation_fleet_proof_error(&err) => {
339 warn!(
340 event = EVENT_ECSTORE_INIT_STATUS,
341 component = LOG_COMPONENT_ECSTORE,
342 subsystem = LOG_SUBSYSTEM_STORE_INIT,
343 stage = "start_rebalance",
344 state = "retrying",
345 reason = "fleet_capability_proof_unavailable",
346 error = %err,
347 retry_delay_secs = REBALANCE_RESUME_RETRY_DELAY.as_secs(),
348 "Retrying deferred rebalance auto-start"
349 );
350 if !wait_for_rebalance_resume_retry(&rx).await {
351 return;
352 }
353 }
354 Err(err) => {
355 error!(
356 event = EVENT_ECSTORE_INIT_STATUS,
357 component = LOG_COMPONENT_ECSTORE,
358 subsystem = LOG_SUBSYSTEM_STORE_INIT,
359 stage = "start_rebalance",
360 state = "failed",
361 reason = "deferred_resume_failed",
362 error = %err,
363 "Failed to resume rebalance after store initialization"
364 );
365 return;
366 }
367 }
368 }
369}
370
371impl ECStore {
372 pub fn background_cancel_token(&self) -> Option<CancellationToken> {
374 self.ctx.background_cancel_token()
375 }
376
377 pub fn validate_startup_storage_class(endpoint_pools: &EndpointServerPools) -> Result<()> {
379 let drive_counts = startup_pool_drive_counts(endpoint_pools);
380 storageclass::lookup_config_for_pools(&KVS::new(), &drive_counts).map(|_| ())
381 }
382
383 #[allow(clippy::new_ret_no_self)]
384 #[instrument(level = "debug", skip(endpoint_pools))]
385 pub async fn new(address: SocketAddr, endpoint_pools: EndpointServerPools, ctx: CancellationToken) -> Result<Arc<Self>> {
386 Self::new_with_instance_ctx(address, endpoint_pools, ctx, crate::runtime::instance::bootstrap_ctx()).await
387 }
388
389 #[allow(clippy::new_ret_no_self)]
397 #[instrument(level = "debug", skip(endpoint_pools, instance_ctx))]
398 pub async fn new_with_instance_ctx(
399 address: SocketAddr,
400 endpoint_pools: EndpointServerPools,
401 ctx: CancellationToken,
402 instance_ctx: Arc<InstanceContext>,
403 ) -> Result<Arc<Self>> {
404 Box::pin(Self::new_with_instance_ctx_inner(address, endpoint_pools, ctx, instance_ctx)).await
405 }
406
407 async fn new_with_instance_ctx_inner(
408 address: SocketAddr,
409 endpoint_pools: EndpointServerPools,
410 ctx: CancellationToken,
411 instance_ctx: Arc<InstanceContext>,
412 ) -> Result<Arc<Self>> {
413 instance_ctx.bind_background_cancel_token(ctx.clone());
414
415 let default_pool_parities = resolve_startup_pool_defaults(&endpoint_pools)?;
422
423 preflight_startup_rpc_secret(&endpoint_pools)?;
424
425 let mut deployment_id = None;
426 let mut pool_meta_bootstrap_authorities = Vec::new();
427
428 let mut pools = Vec::with_capacity(endpoint_pools.as_ref().len());
431 let mut disk_map = HashMap::with_capacity(endpoint_pools.as_ref().len());
432
433 let mut local_disks = Vec::new();
434
435 debug!(
436 event = EVENT_ECSTORE_INIT_STATUS,
437 component = LOG_COMPONENT_ECSTORE,
438 subsystem = LOG_SUBSYSTEM_STORE_INIT,
439 address = %address,
440 "Initializing ECStore address"
441 );
442 let mut host = address.ip().to_string();
443 if host.is_empty() {
444 host = runtime_sources::rustfs_host().await
445 }
446 let mut port = address.port().to_string();
447 if port.is_empty() {
448 port = runtime_sources::rustfs_port().to_string()
449 }
450 debug!(
451 event = EVENT_ECSTORE_INIT_STATUS,
452 component = LOG_COMPONENT_ECSTORE,
453 subsystem = LOG_SUBSYSTEM_STORE_INIT,
454 host = %host,
455 port = %port,
456 "Initializing ECStore host"
457 );
458 init_local_peer(&endpoint_pools, &host, &port).await;
459
460 for (i, pool_eps) in endpoint_pools.as_ref().iter().enumerate() {
463 let pool_first_is_local = pool_first_endpoint_is_local(pool_eps);
464 let parity_drives = default_pool_parities
465 .get(i)
466 .copied()
467 .ok_or_else(|| Error::other(format!("store init failed to resolve default parity for pool {i}")))?;
468
469 let (mut disks, errs) = init_format::init_disks(
476 &pool_eps.endpoints,
477 &DiskOption {
478 cleanup: true,
479 health_check: true,
480 },
481 )
482 .await;
483
484 check_disk_fatal_errs(&errs)?;
485
486 let loaded_format = {
487 let mut times = 0;
488 let mut interval = 1;
489 loop {
490 match init_format::connect_load_init_formats_with_instance_ctx(
491 &instance_ctx,
492 pool_first_is_local,
493 &mut disks,
494 pool_eps.set_count,
495 pool_eps.drives_per_set,
496 deployment_id,
497 )
498 .await
499 {
500 Ok(fm) => break Ok(fm),
501 Err(e) if !should_retry_format_load(&e) => break Err(e),
502 Err(e) if times >= 10 => {
504 break Err(Error::other(format!("store init failed to load formats after {times} retries: {e}")));
505 }
506 Err(_) => {}
508 }
509 times += 1;
510 if interval < 16 {
511 interval *= 2;
512 }
513 debug!(
514 event = EVENT_STORE_FORMAT_RETRY,
515 component = LOG_COMPONENT_ECSTORE,
516 subsystem = LOG_SUBSYSTEM_STORE_INIT,
517 retry_count = times,
518 retry_delay_secs = interval,
519 "Retrying storage format load"
520 );
521 select! {
522 _ = tokio::signal::ctrl_c() => {
523 info!(
524 event = EVENT_STORE_FORMAT_RETRY,
525 component = LOG_COMPONENT_ECSTORE,
526 subsystem = LOG_SUBSYSTEM_STORE_INIT,
527 reason = "ctrl_c",
528 "Interrupted storage format retry loop"
529 );
530 exit(0);
531 }
532 _ = sleep(Duration::from_secs(interval)) => {
533 }
534 }
535 for disk in disks.iter().flatten() {
538 disk.reset_health_for_store_init_retry();
539 }
540 }
541 }?;
542 pool_meta_bootstrap_authorities.push(loaded_format.pool_meta_bootstrap_authority);
546 let fm = loaded_format.format;
547
548 for disk in disks.iter().flatten() {
550 disk.enable_health_check();
551 }
552
553 if deployment_id.is_none() {
554 deployment_id = Some(fm.id);
555 }
556
557 if deployment_id != Some(fm.id) {
558 return Err(Error::other("store init failed: deployment IDs do not match across pools"));
559 }
560
561 if deployment_id.is_some_and(|id| id.is_nil()) {
562 deployment_id = Some(Uuid::new_v4());
563 }
564
565 for disk in disks.iter() {
566 if disk.is_some() && disk.as_ref().expect("operation should succeed").is_local() {
567 local_disks.push(disk.as_ref().expect("operation should succeed").clone());
568 }
569 }
570
571 let sets = Sets::new_with_instance_ctx(disks.clone(), pool_eps, &fm, i, parity_drives, instance_ctx.clone()).await?;
572 pools.push(sets);
573
574 disk_map.insert(i, disks);
575 }
576
577 if !instance_ctx.is_dist_erasure().await {
579 runtime_sources::record_local_disks(&instance_ctx, local_disks).await;
580 }
581
582 let deployment_id = deployment_id.ok_or_else(|| Error::other("store init failed: deployment id is not initialized"))?;
583 let peer_sys = S3PeerSys::new_with_instance_ctx(&endpoint_pools, instance_ctx.clone());
584 let mut pool_meta = PoolMeta::new(&pools, &PoolMeta::default());
585 pool_meta.dont_save = true;
586 let elected_bootstrap_writer = pools
587 .first()
588 .is_some_and(|pool| pool_first_endpoint_is_local(&pool.endpoints));
589 let pool_meta_write_state = PoolMetaWriteState::for_startup_with_pool_bootstrap_authorities(
590 deployment_id,
591 pool_meta_bootstrap_authorities,
592 elected_bootstrap_writer,
593 );
594
595 let decommission_cancelers = RwLock::new(vec![None; pools.len()]);
596 let ec = Arc::new(ECStore {
597 id: deployment_id,
598 disk_map,
599 pools,
600 peer_sys,
601 pool_meta: RwLock::new(pool_meta),
602 rebalance_meta: RwLock::new(None),
603 decommission_cancelers,
604 start_gate: Mutex::new(()),
605 pool_meta_save_gate: Mutex::new(pool_meta_write_state),
606 ctx: instance_ctx.clone(),
610 bucket_fence_registry: std::sync::Arc::default(),
611 });
612
613 if instance_ctx.deployment_id().is_none() {
615 instance_ctx.set_deployment_id(deployment_id);
616 }
617
618 let wait_sec = 5;
619 let mut exit_count = 0;
620 loop {
621 if let Err(err) = ec.init(ctx.clone()).await {
622 error!("init err: {}", err);
623 error!("retry after {} second", wait_sec);
624 sleep(Duration::from_secs(wait_sec)).await;
625
626 if exit_count > 10 {
627 return Err(Error::other("store init failed: init retry budget exhausted"));
628 }
629
630 exit_count += 1;
631
632 continue;
633 }
634
635 break;
636 }
637
638 runtime_sources::publish_object_store(ec.clone()).await;
639
640 Ok(ec)
641 }
642
643 #[instrument(level = "debug", skip(self, rx))]
644 pub async fn init(self: &Arc<Self>, rx: CancellationToken) -> Result<()> {
645 runtime_sources::ensure_boot_time().await;
646
647 let should_persist_pool_meta = self
648 .pools
649 .first()
650 .is_some_and(|pool| pool_first_endpoint_is_local(&pool.endpoints));
651 #[cfg(feature = "e2e-test-hooks")]
652 let startup_attempt = uuid::Uuid::new_v4();
653 let (meta, pool_meta_replica_state) = {
654 let mut write_state = self.pool_meta_save_gate.lock().await;
655 establish_pool_meta_bootstrap_identity_if_proven(self.pools.clone(), &mut write_state, should_persist_pool_meta)
656 .await
657 .map_err(|err| Error::other(format!("store init failed during establish_pool_meta_bootstrap_identity: {err}")))?;
658 let load = load_pool_meta_for_startup(self.pools.clone(), &mut write_state);
659 #[cfg(feature = "e2e-test-hooks")]
660 let load = crate::core::pools::startup_cas_test_scope(startup_attempt, "load", &self.pools, load);
661 load.await?
662 };
663 let update = meta.validate(self.pools.clone())?;
664 #[cfg(feature = "e2e-test-hooks")]
665 crate::core::pools::startup_cas_test_observe(serde_json::json!({
666 "kind": "startup-classifier", "attempt": startup_attempt,
667 "elected_writer": should_persist_pool_meta,
668 "needs_repair": pool_meta_replica_state.needs_repair,
669 "repair_write_safe": pool_meta_replica_state.repair_write_safe,
670 "topology_update": update,
671 }));
672 let endpoints = runtime_sources::endpoint_pools_or_default();
673
674 let mut installed_pool_meta = if update {
675 PoolMeta::new(&self.pools, &meta)
676 } else {
677 meta.clone()
678 };
679 {
682 let mut write_state = self.pool_meta_save_gate.lock().await;
683 let persist = persist_pool_meta_for_startup_if_safe(
684 &installed_pool_meta,
685 self.pools.clone(),
686 pool_meta_replica_state,
687 &mut write_state,
688 update,
689 should_persist_pool_meta,
690 );
691 #[cfg(feature = "e2e-test-hooks")]
692 let persist = crate::core::pools::startup_cas_test_scope(startup_attempt, "persist", &self.pools, persist);
693 installed_pool_meta = persist.await?;
694 }
695
696 {
697 let mut pool_meta = self.pool_meta.write().await;
698 *pool_meta = installed_pool_meta.clone();
699 }
700
701 resolve_store_init_stage_result(self.load_rebalance_meta().await, "load_rebalance_meta")?;
702 let rebalance_resume_required = {
703 let rebalance_meta = self.rebalance_meta.read().await;
704 rebalance_meta
705 .as_ref()
706 .is_some_and(crate::services::rebalance::rebalance_requires_worker_activation)
707 };
708 let decommission_running =
709 pool_meta_has_active_decommission(&installed_pool_meta) || self.is_decommission_running().await;
710 let distributed = self.ctx.is_dist_erasure().await;
711 let fleet_proof_available = crate::services::notification_sys::acquire_cross_pool_fence_fleet_proof().is_some();
712 let mut rebalance_auto_start_deferred = false;
713 if should_auto_start_rebalance_after_init(decommission_running, rebalance_resume_required) {
714 if should_defer_rebalance_auto_start(distributed, fleet_proof_available) {
715 rebalance_auto_start_deferred = true;
716 warn!(
717 event = EVENT_ECSTORE_INIT_STATUS,
718 component = LOG_COMPONENT_ECSTORE,
719 subsystem = LOG_SUBSYSTEM_STORE_INIT,
720 stage = "start_rebalance",
721 state = "deferred",
722 reason = "fleet_capability_proof_unavailable",
723 retry_delay_secs = REBALANCE_RESUME_RETRY_DELAY.as_secs(),
724 "Deferred rebalance auto-start until a live fleet capability proof is available"
725 );
726 } else if let Err(err) = self.start_rebalance().await {
727 if crate::core::pools::is_pool_activation_fleet_proof_error(&err) {
728 rebalance_auto_start_deferred = true;
729 warn!(
730 event = EVENT_ECSTORE_INIT_STATUS,
731 component = LOG_COMPONENT_ECSTORE,
732 subsystem = LOG_SUBSYSTEM_STORE_INIT,
733 stage = "start_rebalance",
734 state = "deferred",
735 reason = "fleet_capability_proof_changed",
736 error = %err,
737 retry_delay_secs = REBALANCE_RESUME_RETRY_DELAY.as_secs(),
738 "Deferred rebalance auto-start after the fleet capability proof changed"
739 );
740 } else {
741 return resolve_store_init_stage_result(Err(err), "start_rebalance");
742 }
743 }
744 } else if decommission_running && rebalance_resume_required {
745 warn!(
746 event = EVENT_ECSTORE_INIT_STATUS,
747 component = LOG_COMPONENT_ECSTORE,
748 subsystem = LOG_SUBSYSTEM_STORE_INIT,
749 stage = "start_rebalance",
750 reason = "active_decommission",
751 "Deferred rebalance auto-start during store init because decommission is active"
752 );
753 }
754
755 let _ = self.scanner_data_usage_publication_blocked().await;
759
760 let pools = installed_pool_meta.return_resumable_pools();
761 let mut pool_indices = Vec::with_capacity(pools.len());
762
763 for p in pools.iter() {
764 if let Some(idx) = endpoints.get_pool_idx(&p.cmd_line) {
765 pool_indices.push(idx);
766 } else {
767 return Err(Error::other(format!(
768 "store init failed to resolve resumable decommission pool `{}` from current endpoints",
769 p.cmd_line
770 )));
771 }
772 }
773
774 let local_pool_indices = local_decommission_queue_prefix(&endpoints, &pool_indices)?;
775 let has_local_decommission_leadership = endpoints.as_ref().iter().any(pool_first_endpoint_is_local);
776 let pool_meta_write_safe = self
777 .ensure_pool_meta_side_effects_safe("decommission resume blocked while pool metadata requires recovery")
778 .await
779 .is_ok();
780 if !pool_meta_replica_state.repair_write_safe || !pool_meta_write_safe {
781 warn!(
782 event = EVENT_DECOMMISSION_RESUME_RETRY,
783 component = LOG_COMPONENT_ECSTORE,
784 subsystem = LOG_SUBSYSTEM_STORE_INIT,
785 state = "blocked",
786 pool_indices = ?local_pool_indices,
787 reason = "pool_meta_write_blocked",
788 "Decommission watchdog waiting for pool metadata recovery"
789 );
790 }
791 if has_local_decommission_leadership {
792 tokio::spawn(supervise_local_decommission_after_init(self.clone(), rx.clone()));
795 }
796
797 let recovery_store = self.clone();
798 let recovery_rx = rx.clone();
799 tokio::spawn(async move {
800 let mut delay = std::time::Duration::from_secs(5);
801 loop {
802 tokio::select! {
803 _ = recovery_rx.cancelled() => return,
804 _ = tokio::time::sleep(delay) => {}
805 }
806 let result = tokio::select! {
807 _ = recovery_rx.cancelled() => return,
808 result = tokio::time::timeout(std::time::Duration::from_secs(30), recovery_store.recover_pool_meta_transaction()) => result,
809 };
810 delay = match result {
811 Ok(Ok(_)) => std::time::Duration::from_secs(5),
812 failure => {
813 let error = match failure {
814 Ok(Err(error)) => error,
815 _ => Error::Timeout,
816 };
817 recovery_store.record_pool_meta_recovery_failure(error);
818 (delay * 2).min(std::time::Duration::from_secs(60))
819 }
820 };
821 }
822 });
823
824 runtime_sources::init_bucket_monitor_for_current_endpoints();
825 crate::bucket::bucket_target_sys::BucketTargetSys::get().start_heartbeat();
826
827 init_background_expiry(self.clone()).await;
828 crate::bucket::lifecycle::bucket_lifecycle_ops::init_background_stale_multipart_upload_cleanup(self.clone());
829
830 TransitionState::init(self.clone()).await;
831 crate::services::tier::tier::try_migrate_tiering_config(self.clone()).await;
832
833 if let Err(err) = runtime_sources::init_tier_config_mgr(self.clone()).await {
834 info!("TierConfigMgr init error: {}", err);
835 }
836
837 if rebalance_auto_start_deferred {
838 let store = self.clone();
839 tokio::spawn(resume_rebalance_after_init(store, rx));
840 }
841
842 Ok(())
843 }
844
845 pub fn init_local_disks() {}
846
847 pub fn single_pool(&self) -> bool {
848 self.pools.len() == 1
849 }
850
851 pub fn supports_atomic_create_only_write_back(&self) -> bool {
854 !self.ctx.lock_manager().is_disabled() && self.pools.len() == 1 && self.pools[0].disk_set.len() == 1
855 }
856}
857
858#[cfg(test)]
859mod tests {
860 use super::{
861 LOCAL_DECOMMISSION_RESUME_RETRY_DELAY, LOCAL_DECOMMISSION_WATCHDOG_MAX_RETRY_DELAY, PoolMetaWriteState,
862 establish_pool_meta_bootstrap_identity_if_proven, load_pool_meta_for_startup, local_decommission_watchdog_retry_delay,
863 persist_pool_meta_for_startup_if_safe, pool_first_endpoint_is_local, pool_meta_has_active_decommission,
864 preflight_startup_rpc_secret_with, resolve_startup_pool_defaults_with, resolve_store_init_stage_result,
865 run_local_decommission_watchdog, save_validated_pool_meta_for_startup, should_auto_start_rebalance_after_init,
866 should_defer_rebalance_auto_start, should_retry_format_load, wait_for_local_decommission_resume_delay,
867 };
868 use crate::core::pools::PoolMetaBootstrapAuthority;
869 #[cfg(feature = "test-util")]
870 use crate::disk::DiskAPI;
871 #[cfg(feature = "test-util")]
872 use crate::{
873 bucket::lifecycle::{
874 DurableIlmRecordCheckpoint, ILM_META_PREFIX, ValidatedDurableIlmRecord,
875 bucket_lifecycle_ops::{ExpiryState, ManualTransitionRunOptions, recover_manual_transition_jobs_once},
876 lifecycle::{TRANSITION_PENDING, TransitionOptions},
877 manual_transition_job::{
878 ManualTransitionJobRecord, ManualTransitionScopeAdmission, ManualTransitionTaskRecord,
879 ManualTransitionWorkerResult, ManualTransitionWorkerResultRecord, manual_transition_job_record_object_name,
880 manual_transition_scope_record_object_name, manual_transition_task_object_name,
881 manual_transition_worker_result_object_name, manual_transition_worker_result_task_key,
882 },
883 recovery_control::{
884 IlmRecoveryClassification, IlmRecoveryControl, IlmRecoveryControlIdentity, IlmRecoveryErrorCode,
885 IlmRecoveryProtocol, MAX_RECOVERY_ATTEMPTS, list_recovery_controls, load_recovery_control,
886 observe_recovery_source, recovery_control_record_object_name, save_recovery_control_if_absent,
887 },
888 recovery_disposition::{
889 IlmRecoveryDispositionExecutionOutcome, IlmRecoveryDispositionState, RecoveryDispositionCrashStage,
890 dry_run_recovery_disposition, execute_recovery_disposition, inject_recovery_disposition_crash_once,
891 load_recovery_disposition,
892 },
893 recovery_disposition_runtime::garbage_collect_completed_recovery_disposition,
894 recovery_export::{
895 create_recovery_export, inspect_recovery_export_observation, load_recovery_export,
896 recovery_export_record_object_name,
897 },
898 tier_delete_journal::{
899 DecommissionCheckpointTargetFailureHook, TIER_DELETE_DISPATCH_MANIFEST_PREFIX, TIER_DELETE_JOURNAL_PREFIX,
900 TierDeleteChunkTestBarrier, TierDeleteChunkTestStage, TierDeleteDispatchBatchLimitGuard,
901 TierDeleteDispatchManifestState, TierDeleteDispatchMemberReadTestHook, TierDeleteDispatchMemberReadTestStage,
902 TierDeleteDispatchRollbackTestHook, complete_tier_delete_dispatch, encode_tier_delete_journal_entry,
903 install_test_tier_delete_dispatch_fixture, persist_tier_delete_journal_entry, prepare_tier_delete_dispatch,
904 recover_test_tier_delete_dispatch_manifest, recover_test_tier_delete_dispatch_manifest_with_page_budget,
905 recover_test_tier_delete_journal_pass_with_budget, recover_tier_delete_dispatch_manifests,
906 recover_tier_delete_journal_entries, test_tier_delete_dispatch_manifest_checkpoint,
907 test_tier_delete_dispatch_manifest_state, tier_delete_dispatch_manifest_operation_lock_held_for_test,
908 tier_delete_dispatch_manifest_recovery_count_for_test, tier_delete_dispatch_manifest_recovery_inflight_for_test,
909 tier_delete_journal_object_name,
910 },
911 tier_free_version_recovery::recover_tier_free_versions,
912 tier_sweeper::{
913 Jentry, TierDeleteJournalState, TierDeleteSourceIdentity, transitioned_delete_journal_entry_for_source,
914 },
915 transition_transaction::{
916 TRANSITION_TRANSACTION_RECORD_PREFIX, TransitionCleanupDecision, TransitionCleanupProof, TransitionOperatorError,
917 TransitionOperatorProbe, TransitionRecoveryClaimBarrier, TransitionRecoveryTerminalBarrier,
918 TransitionRemoteVersion, TransitionSourceIdentity, TransitionSourceVersionMode, TransitionTransaction,
919 TransitionTransactionInit, TransitionTransactionState, delete_transition_candidate_for_operator,
920 finalize_missing_transition_transaction_for_operator, inspect_transition_recovery_retry_for_operator,
921 inspect_transition_transaction_for_operator, load_transition_transaction_record,
922 recover_transition_transaction_records, recover_transition_transaction_records_at,
923 retry_transition_recovery_for_operator, save_transition_transaction_record,
924 save_transition_transaction_record_if_current, transition_recovery_control_id,
925 transition_transaction_record_object_name,
926 },
927 validate_durable_ilm_record,
928 },
929 bucket::metadata::{BUCKET_LIFECYCLE_CONFIG, BUCKET_VERSIONING_CONFIG},
930 config::com,
931 core::pools::DecomBucketInfo,
932 data_movement::SourceCleanupDeleteBarrier,
933 disk::{BUCKET_META_PREFIX, RUSTFS_META_BUCKET, STORAGE_FORMAT_FILE},
934 runtime::{global::set_object_store_resolver, sources as runtime_sources},
935 services::notification_sys::{
936 acquire_tier_delete_journal_fleet_proof, acquire_transition_transaction_compaction_fleet_proof,
937 install_transition_transaction_compaction_fleet_proof_for_test,
938 transition_transaction_compaction_fleet_proof_matches,
939 },
940 services::tier::{
941 test_util::{MockWarmBackend, MockWarmOp, TransitionCleanupStoreBarrier, register_mock_tier},
942 tier::{
943 ERR_TIER_BACKEND_IN_USE, ERR_TIER_INVALID_CONFIG, TIER_CONFIG_FILE, TierConfigMgr,
944 ensure_no_authoritative_tier_references_for_test, tier_config_candidate_digest,
945 },
946 tier_config::{TierConfig, TierType, TierWasabi},
947 tier_mutation_intent::{
948 TIER_MUTATION_INTENT_RECORD_PREFIX, TierMutationIntent, TierMutationIntentKind, TierMutationIntentState,
949 TierMutationIntentTarget, advance_tier_mutation_intent_record_idempotent, delete_tier_mutation_intent_record,
950 list_tier_mutation_intent_records, load_tier_mutation_intent_record, load_tier_mutation_intent_record_with_etag,
951 save_tier_mutation_intent_record, save_tier_mutation_intent_record_if_current,
952 },
953 tier_mutation_peer::{TierMutationPeerError, TierMutationPeerState, handle_tier_mutation_peer_request},
954 tier_probe_intent::{
955 TierProbeIntent, TierProbeIntentState, TierProbeOperationIdentity, TierProbeOwnerFence, TierProbeRemoteVersion,
956 delete_tier_probe_intent_record_if_current, load_tier_probe_intent_record,
957 save_tier_probe_intent_record_if_absent, save_tier_probe_intent_record_if_current,
958 },
959 warm_backend::{TransitionCandidateProbe, WarmBackend},
960 },
961 set_disk::{
962 SetDiskTransitionTransactionKillPoint as TransitionTransactionKillPoint,
963 SetDiskTransitionTransactionKillPointBarrier as TransitionTransactionKillPointBarrier,
964 SetDiskTransitionTransactionMutationKind as TransitionTransactionMutationKind,
965 SetDiskTransitionTransactionMutationObservation as TransitionTransactionMutationObservation,
966 SetDiskTransitionTransactionMutationProbe as TransitionTransactionMutationProbe,
967 SetDiskTransitionUploadedCommitBarrier as TransitionUploadedCommitBarrier,
968 },
969 storage_api_contracts::list::ListOperations as _,
970 };
971 #[cfg(feature = "test-util")]
972 use crate::{bucket::replication::ReplicationObjectBridge, storage_api_contracts::bucket::DeleteBucketOptions};
973 use crate::{
974 bucket::replication::{ReplicationState, ReplicationStatusType, replication_statuses_map},
975 core::pools::{
976 DecommissionErasureLayout, DecommissionPoolCapacityInfo, POOL_META_IDENTITY_NAME, POOL_META_NAME, POOL_META_VERSION,
977 PoolDecommissionInfo, PoolMeta, PoolStatus, pending_pool_meta_identity_for_test,
978 pool_meta_identity_initialized_for_test, pool_meta_v3_commit_state_for_test,
979 set_decommission_capacity_info_overrides_for_test,
980 },
981 disk::endpoint::Endpoint,
982 error::{Error, Result, StorageError},
983 io_support::rio::{WritePlan, compression_metadata_value},
984 layout::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints},
985 object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader},
986 services::rebalance::{RebalStatus, RebalanceInfo, RebalanceMeta, RebalanceStats},
987 storage_api_contracts::{
988 bucket::{BucketOperations as _, MakeBucketOptions},
989 multipart::MultipartOperations as _,
990 object::{ObjectIO, ObjectOperations as _, ObjectToDelete},
991 range::HTTPRangeSpec,
992 },
993 };
994 #[cfg(feature = "test-util")]
995 use futures::{StreamExt as _, TryStreamExt as _};
996 use http::HeaderMap;
997 use rustfs_config::server_config::KVS;
998 #[cfg(feature = "test-util")]
999 use rustfs_filemeta::{FileInfo, FileMeta};
1000 use rustfs_filemeta::{FileInfoVersions, MetaCacheEntry};
1001 #[cfg(feature = "test-util")]
1002 use rustfs_protos::{TIER_MUTATION_RPC_PROTOCOL_VERSION, TierMutationRpcPhase};
1003 use rustfs_rio::{Checksum, ChecksumType};
1004 #[cfg(feature = "test-util")]
1005 use rustfs_s3_client::transition_api::ReaderImpl;
1006 use rustfs_utils::{
1007 CompressionAlgorithm,
1008 http::{SUFFIX_COMPRESSION, insert_str},
1009 };
1010 use std::{
1011 collections::HashMap,
1012 future::Future,
1013 io::Cursor,
1014 sync::{
1015 Arc, Mutex,
1016 atomic::{AtomicBool, AtomicUsize, Ordering},
1017 },
1018 time::Duration,
1019 };
1020 use time::OffsetDateTime;
1021 use tokio::io::AsyncReadExt;
1022 use tokio_util::sync::CancellationToken;
1023
1024 fn run_large_stack_async_test<C, F>(name: &str, case: C)
1025 where
1026 C: FnOnce() -> F + Send + 'static,
1027 F: Future<Output = ()> + 'static,
1028 {
1029 const STACK_SIZE: usize = if cfg!(debug_assertions) {
1030 8 * rustfs_config::DEFAULT_THREAD_STACK_SIZE
1031 } else if cfg!(target_os = "macos") {
1032 2 * rustfs_config::DEFAULT_THREAD_STACK_SIZE
1033 } else {
1034 rustfs_config::DEFAULT_THREAD_STACK_SIZE
1035 };
1036 std::thread::Builder::new()
1037 .name(name.to_string())
1038 .stack_size(STACK_SIZE)
1039 .spawn(move || {
1040 let runtime = tokio::runtime::Builder::new_multi_thread()
1041 .enable_all()
1042 .worker_threads(2)
1043 .thread_stack_size(STACK_SIZE)
1044 .build()
1045 .expect("large-stack store test runtime should build");
1046 runtime.block_on(case());
1047 })
1048 .expect("large-stack store test thread should spawn")
1049 .join()
1050 .expect("large-stack store test thread should complete");
1051 }
1052 use uuid::Uuid;
1053
1054 #[test]
1055 fn new_with_instance_ctx_future_remains_stack_bounded() {
1056 let future = crate::store::ECStore::new_with_instance_ctx(
1057 std::net::SocketAddr::from(([127, 0, 0, 1], 0)),
1058 EndpointServerPools(Vec::new()),
1059 CancellationToken::new(),
1060 Arc::new(crate::runtime::instance::InstanceContext::new()),
1061 );
1062 let future_size = std::mem::size_of_val(&future);
1063
1064 assert!(
1065 future_size <= 4 * 1024,
1066 "ECStore constructor future must remain stack-bounded; measured {future_size} bytes"
1067 );
1068 }
1069
1070 fn startup_pool_meta_payload(meta: &PoolMeta) -> Vec<u8> {
1071 meta.encode_config_data_for_test().expect("pool metadata should encode")
1072 }
1073
1074 #[derive(Debug)]
1075 struct StartupPoolMetaStorage {
1076 read_payload: Vec<u8>,
1077 read_error: bool,
1078 read_without_lock: AtomicBool,
1079 wrote_without_lock: AtomicBool,
1080 wrote_with_max_parity: AtomicBool,
1081 written_payload: Mutex<Option<Vec<u8>>>,
1082 revision: AtomicUsize,
1083 pool_meta_write_attempts: AtomicUsize,
1084 fail_pool_meta_write_at: AtomicUsize,
1085 pool_meta_written_versions: Mutex<Vec<u16>>,
1086 objects: Mutex<HashMap<String, (Vec<u8>, String)>>,
1087 }
1088
1089 impl StartupPoolMetaStorage {
1090 fn new(read_payload: Vec<u8>) -> Self {
1091 let mut objects = HashMap::new();
1092 if !read_payload.is_empty() {
1093 objects.insert(POOL_META_NAME.to_string(), (read_payload.clone(), "startup-pool-meta-0".to_string()));
1094 }
1095 Self {
1096 read_payload,
1097 read_error: false,
1098 read_without_lock: AtomicBool::new(false),
1099 wrote_without_lock: AtomicBool::new(false),
1100 wrote_with_max_parity: AtomicBool::new(false),
1101 written_payload: Mutex::new(None),
1102 revision: AtomicUsize::new(0),
1103 pool_meta_write_attempts: AtomicUsize::new(0),
1104 fail_pool_meta_write_at: AtomicUsize::new(0),
1105 pool_meta_written_versions: Mutex::new(Vec::new()),
1106 objects: Mutex::new(objects),
1107 }
1108 }
1109
1110 fn unreadable() -> Self {
1111 Self {
1112 read_payload: Vec::new(),
1113 read_error: true,
1114 read_without_lock: AtomicBool::new(false),
1115 wrote_without_lock: AtomicBool::new(false),
1116 wrote_with_max_parity: AtomicBool::new(false),
1117 written_payload: Mutex::new(None),
1118 revision: AtomicUsize::new(0),
1119 pool_meta_write_attempts: AtomicUsize::new(0),
1120 fail_pool_meta_write_at: AtomicUsize::new(0),
1121 pool_meta_written_versions: Mutex::new(Vec::new()),
1122 objects: Mutex::new(HashMap::new()),
1123 }
1124 }
1125
1126 fn object_info(&self, bucket: &str, object: &str, size: usize, etag: String) -> ObjectInfo {
1127 ObjectInfo {
1128 bucket: bucket.to_string(),
1129 name: object.to_string(),
1130 size: size as i64,
1131 actual_size: size as i64,
1132 etag: Some(etag),
1133 ..Default::default()
1134 }
1135 }
1136 }
1137
1138 #[async_trait::async_trait]
1139 impl ObjectIO for StartupPoolMetaStorage {
1140 type Error = Error;
1141 type RangeSpec = HTTPRangeSpec;
1142 type HeaderMap = HeaderMap;
1143 type ObjectOptions = ObjectOptions;
1144 type ObjectInfo = ObjectInfo;
1145 type GetObjectReader = GetObjectReader;
1146 type PutObjectReader = PutObjReader;
1147
1148 async fn get_object_reader(
1149 &self,
1150 bucket: &str,
1151 object: &str,
1152 _range: Option<HTTPRangeSpec>,
1153 _h: HeaderMap,
1154 opts: &ObjectOptions,
1155 ) -> Result<GetObjectReader> {
1156 assert!(opts.no_lock, "store init pool metadata load must not require namespace locks");
1157 self.read_without_lock.store(true, Ordering::SeqCst);
1158 if self.read_error {
1159 return Err(Error::other("pool metadata read quorum unavailable"));
1160 }
1161 let Some((payload, etag)) = self
1162 .objects
1163 .lock()
1164 .unwrap_or_else(std::sync::PoisonError::into_inner)
1165 .get(object)
1166 .cloned()
1167 else {
1168 return Err(Error::FileNotFound);
1169 };
1170
1171 Ok(GetObjectReader {
1172 stream: Box::new(Cursor::new(payload.clone())),
1173 object_info: self.object_info(bucket, object, payload.len(), etag),
1174 buffered_body: None,
1175 body_source: Default::default(),
1176 })
1177 }
1178
1179 async fn put_object(
1180 &self,
1181 bucket: &str,
1182 object: &str,
1183 data: &mut PutObjReader,
1184 opts: &ObjectOptions,
1185 ) -> Result<ObjectInfo> {
1186 assert!(opts.no_lock, "store init pool metadata save must not require namespace locks");
1187 self.wrote_without_lock.store(true, Ordering::SeqCst);
1188 self.wrote_with_max_parity.store(opts.max_parity, Ordering::SeqCst);
1189 if object == POOL_META_NAME {
1190 let attempt = self.pool_meta_write_attempts.fetch_add(1, Ordering::SeqCst) + 1;
1191 if self.fail_pool_meta_write_at.load(Ordering::SeqCst) == attempt {
1192 return Err(Error::Timeout);
1193 }
1194 }
1195 let current_etag = self
1196 .objects
1197 .lock()
1198 .unwrap_or_else(std::sync::PoisonError::into_inner)
1199 .get(object)
1200 .map(|(_, etag)| etag.clone());
1201 if opts
1202 .http_preconditions
1203 .as_ref()
1204 .and_then(|preconditions| preconditions.if_none_match_value())
1205 == Some("*")
1206 && current_etag.is_some()
1207 {
1208 return Err(Error::PreconditionFailed);
1209 }
1210 if let Some(expected) = opts
1211 .http_preconditions
1212 .as_ref()
1213 .and_then(|preconditions| preconditions.if_match_value())
1214 && current_etag.as_deref() != Some(expected)
1215 {
1216 return Err(Error::PreconditionFailed);
1217 }
1218 let mut payload = Vec::new();
1219 data.stream.read_to_end(&mut payload).await?;
1220 let size = payload.len();
1221 if object == POOL_META_NAME {
1222 *self.written_payload.lock().unwrap_or_else(std::sync::PoisonError::into_inner) = Some(payload.clone());
1223 if payload.len() >= 4 {
1224 self.pool_meta_written_versions
1225 .lock()
1226 .unwrap_or_else(std::sync::PoisonError::into_inner)
1227 .push(u16::from_le_bytes([payload[2], payload[3]]));
1228 }
1229 }
1230 let etag = format!("startup-pool-meta-{}", self.revision.fetch_add(1, Ordering::SeqCst) + 1);
1231 self.objects
1232 .lock()
1233 .unwrap_or_else(std::sync::PoisonError::into_inner)
1234 .insert(object.to_string(), (payload, etag.clone()));
1235 Ok(self.object_info(bucket, object, size, etag))
1236 }
1237 }
1238
1239 fn init_test_pool_meta(decommission: Option<PoolDecommissionInfo>) -> PoolMeta {
1240 PoolMeta {
1241 version: POOL_META_VERSION,
1242 pools: vec![PoolStatus {
1243 id: 0,
1244 cmd_line: "pool-0".to_string(),
1245 last_update: OffsetDateTime::UNIX_EPOCH,
1246 decommission,
1247 }],
1248 dont_save: false,
1249 }
1250 }
1251
1252 #[tokio::test]
1253 async fn test_fresh_bootstrap_pool_meta_save_reaches_v3_cas_without_blocking_itself() {
1254 let storage = Arc::new(StartupPoolMetaStorage::new(Vec::new()));
1255 let mut write_state = PoolMetaWriteState::for_startup(Uuid::new_v4(), true);
1256 establish_pool_meta_bootstrap_identity_if_proven(vec![storage.clone()], &mut write_state, true)
1257 .await
1258 .expect("the elected fresh bootstrap should persist its identity before loading pool metadata");
1259
1260 let (loaded, replica_state) = load_pool_meta_for_startup(vec![storage.clone()], &mut write_state)
1261 .await
1262 .expect("startup pool metadata load should tolerate missing metadata without locks");
1263 assert!(loaded.pools.is_empty());
1264 assert!(!replica_state.needs_repair);
1265 assert!(replica_state.repair_write_safe);
1266 assert!(storage.read_without_lock.load(Ordering::SeqCst));
1267
1268 let meta = PoolMeta {
1269 version: POOL_META_VERSION,
1270 pools: Vec::new(),
1271 dont_save: false,
1272 };
1273 save_validated_pool_meta_for_startup(&meta, vec![storage.clone()], &mut write_state)
1274 .await
1275 .expect("startup pool metadata save should bypass locks");
1276 assert!(storage.wrote_without_lock.load(Ordering::SeqCst));
1277 assert!(storage.wrote_with_max_parity.load(Ordering::SeqCst));
1278 assert_eq!(
1279 storage.pool_meta_write_attempts.load(Ordering::SeqCst),
1280 2,
1281 "fresh bootstrap should reach both V3 prepare and commit CAS writes"
1282 );
1283 assert_eq!(
1284 *storage
1285 .pool_meta_written_versions
1286 .lock()
1287 .unwrap_or_else(std::sync::PoisonError::into_inner),
1288 vec![3, 3]
1289 );
1290 write_state
1291 .ensure_write_safe("fresh bootstrap publication")
1292 .expect("successful startup publication must disarm the transaction guard");
1293 }
1294
1295 #[tokio::test]
1296 async fn test_legacy_adoption_pool_meta_bootstrap_reaches_v3_cas() {
1297 let deployment_id = Uuid::new_v4();
1298 let storage = Arc::new(StartupPoolMetaStorage::new(Vec::new()));
1299 let mut write_state =
1300 PoolMetaWriteState::for_startup_with_bootstrap_authority(deployment_id, PoolMetaBootstrapAuthority::LegacyAdoption);
1301
1302 establish_pool_meta_bootstrap_identity_if_proven(vec![storage.clone()], &mut write_state, true)
1303 .await
1304 .expect("verified legacy adoption should persist a nonce-bound identity before loading pool metadata");
1305 let (loaded, replica_state) = load_pool_meta_for_startup(vec![storage.clone()], &mut write_state)
1306 .await
1307 .expect("verified legacy adoption should authorize initially missing pool metadata");
1308 assert!(loaded.pools.is_empty());
1309
1310 let committed = persist_pool_meta_for_startup_if_safe(
1311 &init_test_pool_meta(None),
1312 vec![storage.clone()],
1313 replica_state,
1314 &mut write_state,
1315 true,
1316 true,
1317 )
1318 .await
1319 .expect("verified legacy adoption should publish initial pool metadata");
1320 assert_eq!(committed.pools[0].cmd_line, "pool-0");
1321
1322 let objects = storage.objects.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
1323 assert!(objects.contains_key(POOL_META_NAME));
1324 let identity = objects
1325 .get(POOL_META_IDENTITY_NAME)
1326 .map(|(payload, _)| payload.clone())
1327 .expect("legacy adoption should commit the bootstrap identity");
1328 assert!(pool_meta_identity_initialized_for_test(&identity).expect("decode committed identity"));
1329 }
1330
1331 #[tokio::test]
1332 async fn test_non_elected_legacy_adoption_cannot_initialize_pool_meta() {
1333 let storage = Arc::new(StartupPoolMetaStorage::new(Vec::new()));
1334 let mut write_state =
1335 PoolMetaWriteState::for_startup_with_bootstrap_authority(Uuid::new_v4(), PoolMetaBootstrapAuthority::LegacyAdoption);
1336
1337 establish_pool_meta_bootstrap_identity_if_proven(vec![storage.clone()], &mut write_state, false)
1338 .await
1339 .expect("a non-elected distributed node must not create legacy adoption authority");
1340 let err = load_pool_meta_for_startup(vec![storage.clone()], &mut write_state)
1341 .await
1342 .expect_err("legacy adoption still requires the elected writer to create a durable identity");
1343 assert!(err.to_string().contains("no durable bootstrap identity"));
1344 assert!(
1345 !storage
1346 .objects
1347 .lock()
1348 .unwrap_or_else(std::sync::PoisonError::into_inner)
1349 .contains_key(POOL_META_NAME),
1350 "classification must not create pool metadata on non-elected nodes"
1351 );
1352 }
1353
1354 #[tokio::test]
1355 async fn test_unproven_pending_identity_cannot_authorize_all_missing_pool_meta() {
1356 let deployment_id = Uuid::new_v4();
1357 let storage = Arc::new(StartupPoolMetaStorage::new(Vec::new()));
1358 let mut proven_bootstrap = PoolMetaWriteState::for_startup(deployment_id, true);
1359 establish_pool_meta_bootstrap_identity_if_proven(vec![storage.clone()], &mut proven_bootstrap, true)
1360 .await
1361 .expect("fresh topology proof should persist a nonce-bound pending identity");
1362
1363 let identity = storage
1364 .objects
1365 .lock()
1366 .unwrap_or_else(std::sync::PoisonError::into_inner)
1367 .get(POOL_META_IDENTITY_NAME)
1368 .map(|(payload, _)| payload.clone())
1369 .expect("pending identity should be durable");
1370 assert!(!pool_meta_identity_initialized_for_test(&identity).expect("decode pending identity"));
1371
1372 let mut unproven_restart = PoolMetaWriteState::for_startup(deployment_id, false);
1373 let err = load_pool_meta_for_startup(vec![storage.clone()], &mut unproven_restart)
1374 .await
1375 .expect_err("a pending identity alone must not authorize an all-missing restart");
1376 assert!(err.to_string().contains("no verified fresh-bootstrap proof"));
1377 unproven_restart
1378 .ensure_write_safe("unproven pending identity restart")
1379 .expect_err("the rejected restart must latch the write gate");
1380 assert!(
1381 !storage
1382 .objects
1383 .lock()
1384 .unwrap_or_else(std::sync::PoisonError::into_inner)
1385 .contains_key(POOL_META_NAME),
1386 "classification must not create pool metadata"
1387 );
1388 }
1389
1390 #[tokio::test]
1391 async fn test_nonfresh_identity_repair_crash_never_persists_pending_bootstrap_authority() {
1392 let deployment_id = Uuid::new_v4();
1393 let initial = init_test_pool_meta(None);
1394 let storage = Arc::new(StartupPoolMetaStorage::new(startup_pool_meta_payload(&initial)));
1395 let mut write_state = PoolMetaWriteState::for_startup(deployment_id, false);
1396 let (loaded, replica_state) = load_pool_meta_for_startup(vec![storage.clone()], &mut write_state)
1397 .await
1398 .expect("an existing pool metadata snapshot may repair its missing identity");
1399
1400 storage.fail_pool_meta_write_at.store(1, Ordering::SeqCst);
1401 persist_pool_meta_for_startup_if_safe(&loaded, vec![storage.clone()], replica_state, &mut write_state, true, true)
1402 .await
1403 .expect_err("the injected crash boundary should stop before the pool metadata replacement");
1404
1405 let identity = storage
1406 .objects
1407 .lock()
1408 .unwrap_or_else(std::sync::PoisonError::into_inner)
1409 .get(POOL_META_IDENTITY_NAME)
1410 .map(|(payload, _)| payload.clone())
1411 .expect("identity repair should be durable before the failed topology write");
1412 assert!(
1413 pool_meta_identity_initialized_for_test(&identity).expect("decode repaired identity"),
1414 "an existing cluster must never leave pending bootstrap authority at this crash boundary"
1415 );
1416
1417 storage
1418 .objects
1419 .lock()
1420 .unwrap_or_else(std::sync::PoisonError::into_inner)
1421 .remove(POOL_META_NAME);
1422 let mut restarted = PoolMetaWriteState::for_startup(deployment_id, false);
1423 let err = load_pool_meta_for_startup(vec![storage], &mut restarted)
1424 .await
1425 .expect_err("wiping pool metadata after the crash must require recovery");
1426 assert!(err.to_string().contains("initialized cluster identity exists"));
1427 }
1428
1429 #[tokio::test]
1430 #[serial_test::serial(pool_meta_version_env)]
1431 async fn test_existing_legacy_identity_and_replica_repair_respects_disabled_v3_gates() {
1432 temp_env::async_with_vars(
1433 [
1434 (rustfs_config::ENV_POOL_META_V3_WRITE, None::<&str>),
1435 (rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, None::<&str>),
1436 ],
1437 async {
1438 let deployment_id = Uuid::new_v4();
1439 let corrupt = Arc::new(StartupPoolMetaStorage::new(vec![0, 1, 2]));
1440 let legacy = init_test_pool_meta(None);
1441 let valid = Arc::new(StartupPoolMetaStorage::new(startup_pool_meta_payload(&legacy)));
1442 let mut write_state = PoolMetaWriteState::for_startup(deployment_id, false);
1443 let (loaded, replica_state) = load_pool_meta_for_startup(vec![corrupt.clone(), valid.clone()], &mut write_state)
1444 .await
1445 .expect("existing V2 metadata should remain readable while its identity is missing");
1446 assert!(replica_state.needs_repair);
1447
1448 let committed = persist_pool_meta_for_startup_if_safe(
1449 &loaded,
1450 vec![corrupt.clone(), valid.clone()],
1451 replica_state,
1452 &mut write_state,
1453 true,
1454 true,
1455 )
1456 .await
1457 .expect("legacy replica and identity repair should succeed without crossing the V3 gate");
1458 assert_eq!(committed.version, POOL_META_VERSION);
1459 for storage in [corrupt, valid] {
1460 assert_eq!(
1461 *storage
1462 .pool_meta_written_versions
1463 .lock()
1464 .unwrap_or_else(std::sync::PoisonError::into_inner),
1465 vec![POOL_META_VERSION],
1466 "legacy repair must write exactly one V2 snapshot"
1467 );
1468 let identity = storage
1469 .objects
1470 .lock()
1471 .unwrap_or_else(std::sync::PoisonError::into_inner)
1472 .get(POOL_META_IDENTITY_NAME)
1473 .map(|(payload, _)| payload.clone())
1474 .expect("identity repair should persist on every pool");
1475 assert!(pool_meta_identity_initialized_for_test(&identity).expect("decode repaired identity"));
1476 }
1477 },
1478 )
1479 .await;
1480 }
1481
1482 fn startup_object(storage: &StartupPoolMetaStorage, object: &str) -> Option<Vec<u8>> {
1483 storage
1484 .objects
1485 .lock()
1486 .unwrap_or_else(std::sync::PoisonError::into_inner)
1487 .get(object)
1488 .map(|(payload, _)| payload.clone())
1489 }
1490
1491 fn error_chain_text(err: &Error) -> String {
1495 let mut parts = vec![err.to_string()];
1496 let mut current: Option<&(dyn std::error::Error + 'static)> = Some(err);
1497 while let Some(error) = current {
1498 current = if let Some(io) = error.downcast_ref::<std::io::Error>() {
1499 io.get_ref().map(|inner| inner as &(dyn std::error::Error + 'static))
1500 } else {
1501 error.source()
1502 };
1503 if let Some(next) = current {
1504 parts.push(next.to_string());
1505 }
1506 }
1507 parts.join(" <- ")
1508 }
1509
1510 fn inject_startup_object(storage: &StartupPoolMetaStorage, object: &str, payload: Vec<u8>) {
1511 storage
1512 .objects
1513 .lock()
1514 .unwrap_or_else(std::sync::PoisonError::into_inner)
1515 .insert(object.to_string(), (payload, format!("injected-{object}")));
1516 }
1517
1518 fn init_test_pool_meta_with_pools(pool_count: usize) -> PoolMeta {
1519 PoolMeta {
1520 version: POOL_META_VERSION,
1521 pools: (0..pool_count)
1522 .map(|id| PoolStatus {
1523 id,
1524 cmd_line: format!("pool-{id}"),
1525 last_update: OffsetDateTime::UNIX_EPOCH,
1526 decommission: None,
1527 })
1528 .collect(),
1529 dont_save: false,
1530 }
1531 }
1532
1533 fn two_pool_creator_states(deployment_id: Uuid) -> (PoolMetaWriteState, PoolMetaWriteState) {
1536 let node0 = PoolMetaWriteState::for_startup_with_pool_bootstrap_authorities(
1537 deployment_id,
1538 vec![PoolMetaBootstrapAuthority::Fresh, PoolMetaBootstrapAuthority::None],
1539 true,
1540 );
1541 let node1 = PoolMetaWriteState::for_startup_with_pool_bootstrap_authorities(
1542 deployment_id,
1543 vec![PoolMetaBootstrapAuthority::None, PoolMetaBootstrapAuthority::Fresh],
1544 false,
1545 );
1546 (node0, node1)
1547 }
1548
1549 #[tokio::test]
1550 async fn test_two_pool_bootstrap_with_distinct_format_creators_converges_through_creator_attestation() {
1551 let deployment_id = Uuid::new_v4();
1552 let pool0 = Arc::new(StartupPoolMetaStorage::new(Vec::new()));
1553 let pool1 = Arc::new(StartupPoolMetaStorage::new(Vec::new()));
1554 let pools = vec![pool0.clone(), pool1.clone()];
1555 let (mut node0, mut node1) = two_pool_creator_states(deployment_id);
1556 assert!(!node0.bootstrap_identity_proven(), "reading pool1's format is not deployment-wide proof");
1557 assert!(!node1.bootstrap_identity_proven());
1558
1559 establish_pool_meta_bootstrap_identity_if_proven(pools.clone(), &mut node1, false)
1562 .await
1563 .expect("a non-first creator without a durable nonce writes nothing");
1564 assert!(startup_object(&pool0, POOL_META_IDENTITY_NAME).is_none());
1565 assert!(startup_object(&pool1, POOL_META_IDENTITY_NAME).is_none());
1566 let err = load_pool_meta_for_startup(pools.clone(), &mut node1)
1567 .await
1568 .expect_err("nothing durable authorizes a non-elected node");
1569 assert!(err.to_string().contains("bootstrap pending"), "{err}");
1570 node1
1571 .ensure_write_safe("waiting non-elected creator")
1572 .expect("waiting for the elected writer must not latch the write gate");
1573
1574 establish_pool_meta_bootstrap_identity_if_proven(pools.clone(), &mut node0, true)
1577 .await
1578 .expect("the first pool's creator mints the pending identity");
1579 let minted = startup_object(&pool0, POOL_META_IDENTITY_NAME).expect("pool0 pending identity");
1580 assert!(!pool_meta_identity_initialized_for_test(&minted).expect("decode pending identity"));
1581 assert!(
1582 startup_object(&pool1, POOL_META_IDENTITY_NAME).is_none(),
1583 "node0 holds no first-hand proof for pool1 and must not attest it"
1584 );
1585 let err = load_pool_meta_for_startup(pools.clone(), &mut node0)
1586 .await
1587 .expect_err("an unattested pool keeps the elected writer from publishing");
1588 assert!(err.to_string().contains("waiting for every pool creator"), "{err}");
1589 node0
1590 .ensure_write_safe("waiting elected writer")
1591 .expect("waiting for creators must not latch the write gate");
1592 assert!(startup_object(&pool0, POOL_META_NAME).is_none());
1593
1594 establish_pool_meta_bootstrap_identity_if_proven(pools.clone(), &mut node1, false)
1597 .await
1598 .expect("the pool1 creator attests with the durable nonce");
1599 assert_eq!(startup_object(&pool1, POOL_META_IDENTITY_NAME).as_deref(), Some(minted.as_slice()));
1600 let err = load_pool_meta_for_startup(pools.clone(), &mut node1)
1601 .await
1602 .expect_err("a complete pending set never unlocks a non-elected node");
1603 assert!(err.to_string().contains("waiting for the elected writer to publish"), "{err}");
1604 node1
1605 .ensure_write_safe("attested non-elected creator")
1606 .expect("waiting for pool.bin must not latch the write gate");
1607 assert!(startup_object(&pool0, POOL_META_NAME).is_none());
1608
1609 establish_pool_meta_bootstrap_identity_if_proven(pools.clone(), &mut node0, true)
1612 .await
1613 .expect("re-establishing an already minted identity is idempotent");
1614 let (_, replica_state) = load_pool_meta_for_startup(pools.clone(), &mut node0)
1615 .await
1616 .expect("complete creator attestation authorizes the initial pool metadata write");
1617 persist_pool_meta_for_startup_if_safe(
1618 &init_test_pool_meta_with_pools(2),
1619 pools.clone(),
1620 replica_state,
1621 &mut node0,
1622 true,
1623 true,
1624 )
1625 .await
1626 .expect("the elected writer publishes pool.bin and commits the identity");
1627 for pool in [&pool0, &pool1] {
1628 assert!(startup_object(pool, POOL_META_NAME).is_some());
1629 let identity = startup_object(pool, POOL_META_IDENTITY_NAME).expect("committed identity");
1630 assert!(pool_meta_identity_initialized_for_test(&identity).expect("decode committed identity"));
1631 }
1632
1633 let before = startup_object(&pool1, POOL_META_IDENTITY_NAME);
1635 establish_pool_meta_bootstrap_identity_if_proven(pools.clone(), &mut node1, false)
1636 .await
1637 .expect("an initialized deployment never reopens bootstrap");
1638 assert_eq!(startup_object(&pool1, POOL_META_IDENTITY_NAME), before);
1639 load_pool_meta_for_startup(pools, &mut node1)
1640 .await
1641 .expect("published pool metadata admits the non-elected node");
1642 node1
1643 .ensure_write_safe("converged non-elected creator")
1644 .expect("no latch remains after convergence");
1645 }
1646
1647 #[tokio::test]
1648 async fn test_two_pool_bootstrap_rejects_pending_replicas_from_different_bootstraps() {
1649 let deployment_id = Uuid::new_v4();
1650 let pool0 = Arc::new(StartupPoolMetaStorage::new(Vec::new()));
1651 let pool1 = Arc::new(StartupPoolMetaStorage::new(Vec::new()));
1652 let pools = vec![pool0.clone(), pool1.clone()];
1653 let (mut node0, _) = two_pool_creator_states(deployment_id);
1654 establish_pool_meta_bootstrap_identity_if_proven(pools.clone(), &mut node0, true)
1655 .await
1656 .expect("the first pool's creator mints the pending identity");
1657 inject_startup_object(
1658 &pool1,
1659 POOL_META_IDENTITY_NAME,
1660 pending_pool_meta_identity_for_test(deployment_id, 1, Uuid::new_v4()).expect("encode foreign pending identity"),
1661 );
1662
1663 let err = load_pool_meta_for_startup(pools.clone(), &mut node0)
1664 .await
1665 .expect_err("a pending replica bound to another bootstrap nonce must fail closed");
1666 let chain = error_chain_text(&err);
1667 assert!(chain.contains("disagree on fresh-bootstrap proof"), "{chain}");
1668 node0
1669 .ensure_write_safe("split bootstrap")
1670 .expect_err("a split bootstrap latches the write gate");
1671 assert!(startup_object(&pool0, POOL_META_NAME).is_none());
1672 }
1673
1674 #[tokio::test]
1675 async fn test_two_pool_bootstrap_treats_corrupt_creator_replica_as_recovery_not_waiting() {
1676 let deployment_id = Uuid::new_v4();
1677 let pool0 = Arc::new(StartupPoolMetaStorage::new(Vec::new()));
1678 let pool1 = Arc::new(StartupPoolMetaStorage::new(Vec::new()));
1679 let pools = vec![pool0.clone(), pool1.clone()];
1680 let (mut node0, _) = two_pool_creator_states(deployment_id);
1681 establish_pool_meta_bootstrap_identity_if_proven(pools.clone(), &mut node0, true)
1682 .await
1683 .expect("the first pool's creator mints the pending identity");
1684 let mut corrupt = pending_pool_meta_identity_for_test(deployment_id, 1, Uuid::new_v4()).expect("encode identity");
1687 corrupt.truncate(4);
1688 corrupt.extend_from_slice(b"not a cluster identity");
1689 inject_startup_object(&pool1, POOL_META_IDENTITY_NAME, corrupt);
1690
1691 let err = load_pool_meta_for_startup(pools.clone(), &mut node0)
1692 .await
1693 .expect_err("a corrupt replica is not a creator that is still catching up");
1694 let chain = error_chain_text(&err);
1695 assert!(chain.contains("no verified fresh-bootstrap proof"), "{chain}");
1696 node0
1697 .ensure_write_safe("corrupt attestation")
1698 .expect_err("a corrupt attestation latches the write gate");
1699 assert!(startup_object(&pool0, POOL_META_NAME).is_none());
1700 }
1701
1702 #[tokio::test]
1703 async fn test_elected_restart_without_first_hand_proof_cannot_reuse_a_complete_pending_set() {
1704 let deployment_id = Uuid::new_v4();
1705 let pool0 = Arc::new(StartupPoolMetaStorage::new(Vec::new()));
1706 let pool1 = Arc::new(StartupPoolMetaStorage::new(Vec::new()));
1707 let pools = vec![pool0.clone(), pool1.clone()];
1708 let (mut node0, mut node1) = two_pool_creator_states(deployment_id);
1709 establish_pool_meta_bootstrap_identity_if_proven(pools.clone(), &mut node0, true)
1710 .await
1711 .expect("mint");
1712 establish_pool_meta_bootstrap_identity_if_proven(pools.clone(), &mut node1, false)
1713 .await
1714 .expect("attest");
1715 assert_eq!(
1716 startup_object(&pool0, POOL_META_IDENTITY_NAME),
1717 startup_object(&pool1, POOL_META_IDENTITY_NAME),
1718 "both creators attested the same pending identity"
1719 );
1720
1721 let mut restarted = PoolMetaWriteState::for_startup_with_pool_bootstrap_authorities(
1724 deployment_id,
1725 vec![PoolMetaBootstrapAuthority::None, PoolMetaBootstrapAuthority::None],
1726 true,
1727 );
1728 establish_pool_meta_bootstrap_identity_if_proven(pools.clone(), &mut restarted, true)
1729 .await
1730 .expect("a restart without first-hand proof writes nothing");
1731 let err = load_pool_meta_for_startup(pools.clone(), &mut restarted)
1732 .await
1733 .expect_err("a pending set alone never authorizes a writer without first-hand proof");
1734 assert!(err.to_string().contains("no verified fresh-bootstrap proof"), "{err}");
1735 restarted
1736 .ensure_write_safe("unproven restart")
1737 .expect_err("the rejected restart latches the write gate");
1738 assert!(startup_object(&pool0, POOL_META_NAME).is_none());
1739 }
1740
1741 #[tokio::test]
1742 async fn test_fresh_pool_joining_an_initialized_deployment_never_reopens_bootstrap() {
1743 let deployment_id = Uuid::new_v4();
1744 let pool0 = Arc::new(StartupPoolMetaStorage::new(Vec::new()));
1745 let mut founder = PoolMetaWriteState::for_startup(deployment_id, true);
1746 establish_pool_meta_bootstrap_identity_if_proven(vec![pool0.clone()], &mut founder, true)
1747 .await
1748 .expect("the founder mints");
1749 let (_, replica_state) = load_pool_meta_for_startup(vec![pool0.clone()], &mut founder)
1750 .await
1751 .expect("the founder may initialize");
1752 persist_pool_meta_for_startup_if_safe(
1753 &init_test_pool_meta(None),
1754 vec![pool0.clone()],
1755 replica_state,
1756 &mut founder,
1757 true,
1758 true,
1759 )
1760 .await
1761 .expect("the founder commits");
1762 let founded = startup_object(&pool0, POOL_META_IDENTITY_NAME).expect("committed identity");
1763 assert!(pool_meta_identity_initialized_for_test(&founded).expect("decode committed identity"));
1764
1765 let pool1 = Arc::new(StartupPoolMetaStorage::new(Vec::new()));
1768 let pools = vec![pool0.clone(), pool1.clone()];
1769 for elected in [false, true] {
1770 let mut joiner = PoolMetaWriteState::for_startup_with_pool_bootstrap_authorities(
1771 deployment_id,
1772 vec![PoolMetaBootstrapAuthority::None, PoolMetaBootstrapAuthority::Fresh],
1773 elected,
1774 );
1775 establish_pool_meta_bootstrap_identity_if_proven(pools.clone(), &mut joiner, elected)
1776 .await
1777 .expect("an initialized deployment ignores first-hand proof for a new pool");
1778 assert!(
1779 startup_object(&pool1, POOL_META_IDENTITY_NAME).is_none(),
1780 "no pending identity may be written to an expansion pool"
1781 );
1782 assert_eq!(startup_object(&pool0, POOL_META_IDENTITY_NAME).as_deref(), Some(founded.as_slice()));
1783 let (_, replica_state) = load_pool_meta_for_startup(pools.clone(), &mut joiner)
1784 .await
1785 .expect("published pool metadata admits the joiner");
1786 joiner
1787 .ensure_write_safe("expansion joiner")
1788 .expect("joining never latches the write gate");
1789 if elected {
1790 persist_pool_meta_for_startup_if_safe(
1791 &init_test_pool_meta_with_pools(2),
1792 pools.clone(),
1793 replica_state,
1794 &mut joiner,
1795 true,
1796 true,
1797 )
1798 .await
1799 .expect("the topology update repairs the new pool's replicas");
1800 let identity = startup_object(&pool1, POOL_META_IDENTITY_NAME).expect("expansion pool identity");
1801 assert!(pool_meta_identity_initialized_for_test(&identity).expect("decode repaired identity"));
1802 assert!(startup_object(&pool1, POOL_META_NAME).is_some());
1803 }
1804 }
1805 }
1806
1807 #[tokio::test]
1808 async fn test_store_init_distinguishes_fresh_deployment_from_wiped_lagging_node() {
1809 let deployment_id = Uuid::new_v4();
1810 let storage = Arc::new(StartupPoolMetaStorage::new(Vec::new()));
1811 let mut fresh_state = PoolMetaWriteState::for_startup(deployment_id, true);
1812 establish_pool_meta_bootstrap_identity_if_proven(vec![storage.clone()], &mut fresh_state, true)
1813 .await
1814 .expect("fresh topology proof should become a durable pending identity");
1815 let (_, replica_state) = load_pool_meta_for_startup(vec![storage.clone()], &mut fresh_state)
1816 .await
1817 .expect("new formats with no identity may initialize pool metadata exactly once");
1818 let committed = persist_pool_meta_for_startup_if_safe(
1819 &init_test_pool_meta(None),
1820 vec![storage.clone()],
1821 replica_state,
1822 &mut fresh_state,
1823 true,
1824 true,
1825 )
1826 .await
1827 .expect("fresh deployment should durably commit identity and pool metadata");
1828 assert_eq!(committed.pools[0].cmd_line, "pool-0");
1829 {
1830 let objects = storage.objects.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
1831 assert!(objects.contains_key(POOL_META_NAME));
1832 assert!(objects.contains_key(POOL_META_IDENTITY_NAME));
1833 }
1834
1835 storage
1836 .objects
1837 .lock()
1838 .unwrap_or_else(std::sync::PoisonError::into_inner)
1839 .remove(POOL_META_NAME);
1840 let mut wiped_pool_state = PoolMetaWriteState::for_startup(deployment_id, false);
1841 let err = load_pool_meta_for_startup(vec![storage.clone()], &mut wiped_pool_state)
1842 .await
1843 .expect_err("an initialized identity must prevent an empty pool metadata rebuild");
1844 assert!(err.to_string().contains("initialized cluster identity exists"));
1845
1846 storage
1847 .objects
1848 .lock()
1849 .unwrap_or_else(std::sync::PoisonError::into_inner)
1850 .remove(POOL_META_IDENTITY_NAME);
1851 let mut wiped_all_state = PoolMetaWriteState::for_startup(deployment_id, false);
1852 let err = load_pool_meta_for_startup(vec![storage], &mut wiped_all_state)
1853 .await
1854 .expect_err("existing formats without identity or pool metadata must require recovery");
1855 assert!(err.to_string().contains("no durable bootstrap identity"));
1856
1857 let distributed_node = Arc::new(StartupPoolMetaStorage::new(Vec::new()));
1858 let mut non_elected_state = PoolMetaWriteState::for_startup(Uuid::new_v4(), true);
1859 establish_pool_meta_bootstrap_identity_if_proven(vec![distributed_node.clone()], &mut non_elected_state, false)
1860 .await
1861 .expect("a non-elected distributed node must not create bootstrap authority");
1862 let err = load_pool_meta_for_startup(vec![distributed_node], &mut non_elected_state)
1863 .await
1864 .expect_err("a distributed node without durable identity or pool metadata must require recovery");
1865 assert!(err.to_string().contains("no durable bootstrap identity"));
1866 }
1867
1868 #[tokio::test]
1869 async fn test_store_init_resumes_pending_v3_bootstrap_without_legacy_overwrite() {
1870 let deployment_id = Uuid::new_v4();
1871 let storage = Arc::new(StartupPoolMetaStorage::new(Vec::new()));
1872 let mut first_bootstrap = PoolMetaWriteState::for_startup(deployment_id, true);
1873 establish_pool_meta_bootstrap_identity_if_proven(vec![storage.clone()], &mut first_bootstrap, true)
1874 .await
1875 .expect("fresh bootstrap should persist a pending identity");
1876 let (_, replica_state) = load_pool_meta_for_startup(vec![storage.clone()], &mut first_bootstrap)
1877 .await
1878 .expect("the durable pending identity should authorize the initial pool metadata write");
1879
1880 storage.fail_pool_meta_write_at.store(2, Ordering::SeqCst);
1881 persist_pool_meta_for_startup_if_safe(
1882 &init_test_pool_meta(None),
1883 vec![storage.clone()],
1884 replica_state,
1885 &mut first_bootstrap,
1886 true,
1887 true,
1888 )
1889 .await
1890 .expect_err("the injected crash boundary should leave only the V3 prepare record");
1891
1892 let (pending_pool, pending_identity) = {
1893 let objects = storage.objects.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
1894 (
1895 objects
1896 .get(POOL_META_NAME)
1897 .map(|(payload, _)| payload.clone())
1898 .expect("the V3 prepare record should be durable"),
1899 objects
1900 .get(POOL_META_IDENTITY_NAME)
1901 .map(|(payload, _)| payload.clone())
1902 .expect("the bootstrap identity should be durable"),
1903 )
1904 };
1905 assert_eq!(pool_meta_v3_commit_state_for_test(pending_pool).expect("decode pending V3"), (1, false));
1906 assert!(!pool_meta_identity_initialized_for_test(&pending_identity).expect("decode pending identity"));
1907
1908 storage.fail_pool_meta_write_at.store(0, Ordering::SeqCst);
1909 let mut restarted = PoolMetaWriteState::for_startup(deployment_id, false);
1910 let (loaded, replica_state) = load_pool_meta_for_startup(vec![storage.clone()], &mut restarted)
1911 .await
1912 .expect("restart should recover the predecessor embedded in the pending V3 record");
1913 assert!(loaded.pools.is_empty());
1914 assert!(replica_state.needs_repair);
1915 let committed = persist_pool_meta_for_startup_if_safe(
1916 &init_test_pool_meta(None),
1917 vec![storage.clone()],
1918 replica_state,
1919 &mut restarted,
1920 true,
1921 true,
1922 )
1923 .await
1924 .expect("restart should finish generation 1 before promoting the identity");
1925 assert_eq!(committed.version, 3);
1926
1927 let (committed_pool, committed_identity) = {
1928 let objects = storage.objects.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
1929 (
1930 objects
1931 .get(POOL_META_NAME)
1932 .map(|(payload, _)| payload.clone())
1933 .expect("the committed V3 record should be durable"),
1934 objects
1935 .get(POOL_META_IDENTITY_NAME)
1936 .map(|(payload, _)| payload.clone())
1937 .expect("the committed identity should be durable"),
1938 )
1939 };
1940 assert_eq!(
1941 pool_meta_v3_commit_state_for_test(committed_pool).expect("decode committed V3"),
1942 (1, true)
1943 );
1944 assert!(pool_meta_identity_initialized_for_test(&committed_identity).expect("decode committed identity"));
1945 assert!(
1946 storage
1947 .pool_meta_written_versions
1948 .lock()
1949 .unwrap_or_else(std::sync::PoisonError::into_inner)
1950 .iter()
1951 .all(|version| *version == 3),
1952 "bootstrap recovery must never overwrite a pending V3 record with legacy metadata"
1953 );
1954 }
1955
1956 #[tokio::test]
1957 async fn test_store_init_pool_meta_falls_back_from_corrupt_first_replica() {
1958 let corrupt = Arc::new(StartupPoolMetaStorage::new(vec![0, 1, 2]));
1959 let expected = init_test_pool_meta(None);
1960 let backup = Arc::new(StartupPoolMetaStorage::new(startup_pool_meta_payload(&expected)));
1961 let mut write_state = PoolMetaWriteState::default();
1962
1963 let (loaded, replica_state) = load_pool_meta_for_startup(vec![corrupt.clone(), backup.clone()], &mut write_state)
1964 .await
1965 .expect("startup should select the validated backup replica");
1966
1967 assert!(replica_state.needs_repair);
1968 assert!(replica_state.repair_write_safe);
1969 assert_eq!(loaded.pools.len(), 1);
1970 assert_eq!(loaded.pools[0].cmd_line, expected.pools[0].cmd_line);
1971 assert!(corrupt.read_without_lock.load(Ordering::SeqCst));
1972 assert!(backup.read_without_lock.load(Ordering::SeqCst));
1973
1974 persist_pool_meta_for_startup_if_safe(
1975 &loaded,
1976 vec![corrupt.clone(), backup.clone()],
1977 replica_state,
1978 &mut write_state,
1979 false,
1980 true,
1981 )
1982 .await
1983 .expect("the elected startup writer should repair validated corrupt replicas");
1984
1985 let corrupt_write = corrupt
1986 .written_payload
1987 .lock()
1988 .unwrap_or_else(std::sync::PoisonError::into_inner)
1989 .clone()
1990 .expect("corrupt replica should be repaired");
1991 let backup_write = backup
1992 .written_payload
1993 .lock()
1994 .unwrap_or_else(std::sync::PoisonError::into_inner)
1995 .clone()
1996 .expect("backup replica should receive the same canonical snapshot");
1997 assert_eq!(corrupt_write, backup_write);
1998 assert_ne!(corrupt_write, corrupt.read_payload);
1999 }
2000
2001 #[tokio::test]
2002 async fn test_store_init_pool_meta_does_not_repair_unreadable_replica() {
2003 let valid = Arc::new(StartupPoolMetaStorage::new(startup_pool_meta_payload(&init_test_pool_meta(None))));
2004 let unreadable = Arc::new(StartupPoolMetaStorage::unreadable());
2005 let mut write_state = PoolMetaWriteState::default();
2006
2007 let (loaded, replica_state) = load_pool_meta_for_startup(vec![valid.clone(), unreadable.clone()], &mut write_state)
2008 .await
2009 .expect("startup should use a validated replica without overwriting an unreadable copy");
2010 assert!(replica_state.needs_repair);
2011 assert!(!replica_state.repair_write_safe);
2012
2013 persist_pool_meta_for_startup_if_safe(
2014 &loaded,
2015 vec![valid.clone(), unreadable.clone()],
2016 replica_state,
2017 &mut write_state,
2018 false,
2019 true,
2020 )
2021 .await
2022 .expect("an unreadable copy should defer repair when no topology write is needed");
2023 assert!(!valid.wrote_without_lock.load(Ordering::SeqCst));
2024 assert!(!unreadable.wrote_without_lock.load(Ordering::SeqCst));
2025
2026 let err = persist_pool_meta_for_startup_if_safe(
2027 &loaded,
2028 vec![valid.clone(), unreadable.clone()],
2029 replica_state,
2030 &mut write_state,
2031 true,
2032 true,
2033 )
2034 .await
2035 .expect_err("a topology update must not overwrite an unreadable replica");
2036 assert!(err.to_string().contains("cannot overwrite an unreadable replica"));
2037 assert!(!valid.wrote_without_lock.load(Ordering::SeqCst));
2038 assert!(!unreadable.wrote_without_lock.load(Ordering::SeqCst));
2039 }
2040
2041 #[tokio::test]
2042 async fn test_store_init_pool_meta_stays_blocked_after_all_replicas_were_unreadable() {
2043 let unreadable_a = Arc::new(StartupPoolMetaStorage::unreadable());
2044 let unreadable_b = Arc::new(StartupPoolMetaStorage::unreadable());
2045 let mut write_state = PoolMetaWriteState::default();
2046
2047 load_pool_meta_for_startup(vec![unreadable_a, unreadable_b], &mut write_state)
2048 .await
2049 .expect_err("startup must fail when no readable pool metadata replica exists");
2050
2051 let repaired = Arc::new(StartupPoolMetaStorage::new(startup_pool_meta_payload(&init_test_pool_meta(None))));
2052 let (loaded, replica_state) = load_pool_meta_for_startup(vec![repaired.clone()], &mut write_state)
2053 .await
2054 .expect("a later startup retry may read the repaired replica");
2055 assert!(replica_state.repair_write_safe);
2056
2057 let err =
2058 persist_pool_meta_for_startup_if_safe(&loaded, vec![repaired.clone()], replica_state, &mut write_state, true, true)
2059 .await
2060 .expect_err("the same store instance must not write after observing unreadable replicas");
2061 assert!(
2062 err.to_string()
2063 .contains("restart after all replicas are readable and consistent")
2064 );
2065 assert!(!repaired.wrote_without_lock.load(Ordering::SeqCst));
2066 }
2067
2068 #[test]
2069 fn test_pool_first_endpoint_is_local_respects_local_flag() {
2070 let mut local_endpoint = Endpoint::try_from("http://127.0.0.1:9000/data").expect("endpoint should parse");
2071 local_endpoint.is_local = true;
2072 let pool = PoolEndpoints {
2073 legacy: false,
2074 set_count: 1,
2075 drives_per_set: 1,
2076 endpoints: Endpoints::from(vec![local_endpoint]),
2077 cmd_line: "pool-0".to_string(),
2078 platform: String::new(),
2079 };
2080
2081 assert!(pool_first_endpoint_is_local(&pool));
2082 }
2083
2084 #[test]
2085 fn test_pool_first_endpoint_is_local_rejects_missing_endpoint() {
2086 let pool = PoolEndpoints {
2087 legacy: false,
2088 set_count: 1,
2089 drives_per_set: 1,
2090 endpoints: Endpoints::from(Vec::<Endpoint>::new()),
2091 cmd_line: "pool-0".to_string(),
2092 platform: String::new(),
2093 };
2094
2095 assert!(!pool_first_endpoint_is_local(&pool));
2096 }
2097
2098 #[test]
2099 fn test_local_decommission_watchdog_retry_delay_is_bounded() {
2100 assert_eq!(local_decommission_watchdog_retry_delay(1), LOCAL_DECOMMISSION_RESUME_RETRY_DELAY);
2101 assert_eq!(
2102 local_decommission_watchdog_retry_delay(u32::MAX),
2103 LOCAL_DECOMMISSION_WATCHDOG_MAX_RETRY_DELAY
2104 );
2105 }
2106
2107 #[tokio::test(start_paused = true)]
2108 async fn test_local_decommission_watchdog_cancelled_start_does_not_reconcile() {
2109 let rx = CancellationToken::new();
2110 rx.cancel();
2111 run_local_decommission_watchdog(rx, || async {
2112 panic!("cancelled startup must not schedule persisted work");
2113 })
2114 .await;
2115 }
2116
2117 #[tokio::test]
2118 #[serial_test::serial]
2119 async fn test_local_decommission_recovery_waits_for_live_fleet_proof_before_reserving_worker() {
2120 let (_temp_dirs, store, _other_store) = crate::services::rebalance::test_two_pool_stores(None).await;
2121 mark_test_pool_decommissioning(&store, 0).await;
2122 assert!(store.ctx.is_dist_erasure().await);
2123 let worker_rx = CancellationToken::new();
2124
2125 {
2126 let _proof_guard = crate::services::notification_sys::without_cross_pool_fence_fleet_proof_for_test();
2127 let err = super::reconcile_local_decommission_after_init(&store, worker_rx.clone())
2128 .await
2129 .expect_err("cold distributed recovery must wait for live fleet proof");
2130 assert!(
2131 crate::core::pools::is_pool_activation_fleet_proof_error(&err),
2132 "recovery must reach the live fleet proof gate: {err:?}"
2133 );
2134 assert!(store.decommission_cancelers.read().await.iter().all(Option::is_none));
2135 assert!(pool_meta_has_active_decommission(&*store.pool_meta.read().await));
2136 }
2137
2138 super::reconcile_local_decommission_after_init(&store, worker_rx.clone())
2139 .await
2140 .expect("restored fleet proof should admit the persisted worker");
2141 assert!(store.has_active_local_decommission_worker().await);
2142 worker_rx.cancel();
2143 }
2144
2145 #[tokio::test(start_paused = true)]
2146 async fn test_local_decommission_watchdog_retries_general_failures_until_cancelled() {
2147 let rx = CancellationToken::new();
2148 let attempts = Arc::new(AtomicUsize::new(0));
2149 let task = tokio::spawn(run_local_decommission_watchdog(rx.clone(), {
2150 let attempts = attempts.clone();
2151 let rx = rx.clone();
2152 move || {
2153 let attempts = attempts.clone();
2154 let rx = rx.clone();
2155 async move {
2156 match attempts.fetch_add(1, Ordering::SeqCst) {
2157 0 => Err(StorageError::other("pool activation requires a live fleet capability proof")),
2158 1 => Err(StorageError::SlowDown),
2159 _ => {
2160 rx.cancel();
2161 Ok(())
2162 }
2163 }
2164 }
2165 }
2166 }));
2167
2168 tokio::task::yield_now().await;
2169 assert_eq!(attempts.load(Ordering::SeqCst), 1);
2170 tokio::time::advance(LOCAL_DECOMMISSION_RESUME_RETRY_DELAY).await;
2171 tokio::task::yield_now().await;
2172 assert_eq!(attempts.load(Ordering::SeqCst), 2);
2173 tokio::time::advance(local_decommission_watchdog_retry_delay(2)).await;
2174 task.await.expect("watchdog task should exit after cancellation");
2175 assert_eq!(attempts.load(Ordering::SeqCst), 3);
2176 }
2177
2178 #[tokio::test(start_paused = true)]
2179 async fn test_local_decommission_watchdog_rescans_after_success() {
2180 let rx = CancellationToken::new();
2181 let attempts = Arc::new(AtomicUsize::new(0));
2182 let task = tokio::spawn(run_local_decommission_watchdog(rx.clone(), {
2183 let attempts = attempts.clone();
2184 let rx = rx.clone();
2185 move || {
2186 let attempts = attempts.clone();
2187 let rx = rx.clone();
2188 async move {
2189 if attempts.fetch_add(1, Ordering::SeqCst) == 1 {
2190 rx.cancel();
2191 }
2192 Ok(())
2193 }
2194 }
2195 }));
2196
2197 tokio::task::yield_now().await;
2198 assert_eq!(attempts.load(Ordering::SeqCst), 1);
2199 tokio::time::advance(super::LOCAL_DECOMMISSION_WATCHDOG_INTERVAL).await;
2200 task.await.expect("watchdog task should exit after cancellation");
2201 assert_eq!(attempts.load(Ordering::SeqCst), 2);
2202 }
2203
2204 #[test]
2205 fn test_should_retry_format_load_rejects_permanent_corruption() {
2206 assert!(!should_retry_format_load(&StorageError::CorruptedFormat));
2207 assert!(should_retry_format_load(&StorageError::ErasureReadQuorum));
2208 assert!(should_retry_format_load(&StorageError::FirstDiskWait));
2209 }
2210
2211 #[test]
2212 fn test_should_retry_format_load_rejects_permanent_topology_errors() {
2213 for error in [
2214 StorageError::UnsupportedSnsdExpansion { configured_drives: 4 },
2215 StorageError::PoolTopologyMismatch {
2216 stored_drives: 4,
2217 stored_set_drive_count: 4,
2218 configured_drives: 8,
2219 configured_set_drive_count: 8,
2220 },
2221 ] {
2222 assert!(!should_retry_format_load(&error), "topology errors require operator action: {error}");
2223 }
2224 for error in [
2225 StorageError::DiskNotFound,
2226 StorageError::Timeout,
2227 StorageError::RemoteNotInitialized,
2228 StorageError::NotFirstDisk,
2229 StorageError::other(std::io::Error::from(std::io::ErrorKind::ConnectionRefused)),
2230 ] {
2231 assert!(
2232 should_retry_format_load(&error),
2233 "transient failures retain their existing retry path: {error}"
2234 );
2235 }
2236 }
2237
2238 #[test]
2239 fn test_should_auto_start_rebalance_after_init_allows_active_rebalance_without_decommission() {
2240 assert!(should_auto_start_rebalance_after_init(false, true));
2241 }
2242
2243 #[test]
2244 fn test_should_auto_start_rebalance_after_init_rejects_active_decommission() {
2245 assert!(!should_auto_start_rebalance_after_init(true, true));
2246 }
2247
2248 #[test]
2249 fn test_should_auto_start_rebalance_after_init_rejects_terminal_or_missing_rebalance() {
2250 assert!(!should_auto_start_rebalance_after_init(false, false));
2251 }
2252
2253 #[test]
2254 fn test_should_defer_rebalance_auto_start_only_without_distributed_fleet_proof() {
2255 assert!(should_defer_rebalance_auto_start(true, false));
2256 assert!(!should_defer_rebalance_auto_start(true, true));
2257 assert!(!should_defer_rebalance_auto_start(false, false));
2258 }
2259
2260 #[test]
2261 fn test_store_init_recovery_skips_rebalance_when_decommission_metadata_is_active() {
2262 let pool_meta = init_test_pool_meta(Some(PoolDecommissionInfo {
2263 start_time: Some(OffsetDateTime::UNIX_EPOCH),
2264 complete: false,
2265 failed: false,
2266 canceled: false,
2267 ..Default::default()
2268 }));
2269 let rebalance_meta = Some(RebalanceMeta {
2270 pool_stats: vec![RebalanceStats {
2271 participating: true,
2272 info: RebalanceInfo {
2273 status: RebalStatus::Started,
2274 ..Default::default()
2275 },
2276 ..Default::default()
2277 }],
2278 ..Default::default()
2279 });
2280
2281 assert!(!should_auto_start_rebalance_after_init(
2282 pool_meta_has_active_decommission(&pool_meta),
2283 rebalance_meta
2284 .as_ref()
2285 .is_some_and(crate::services::rebalance::rebalance_requires_worker_activation)
2286 ));
2287 }
2288
2289 #[test]
2290 fn test_store_init_recovery_allows_active_rebalance_without_decommission() {
2291 let pool_meta = init_test_pool_meta(None);
2292 let rebalance_meta = Some(RebalanceMeta {
2293 pool_stats: vec![RebalanceStats {
2294 participating: true,
2295 info: RebalanceInfo {
2296 status: RebalStatus::Started,
2297 ..Default::default()
2298 },
2299 ..Default::default()
2300 }],
2301 ..Default::default()
2302 });
2303
2304 assert!(should_auto_start_rebalance_after_init(
2305 pool_meta_has_active_decommission(&pool_meta),
2306 rebalance_meta
2307 .as_ref()
2308 .is_some_and(crate::services::rebalance::rebalance_requires_worker_activation)
2309 ));
2310 }
2311
2312 #[test]
2313 fn test_store_init_recovery_skips_completed_rebalance_metadata() {
2314 let pool_meta = init_test_pool_meta(None);
2315 let rebalance_meta = Some(RebalanceMeta {
2316 pool_stats: vec![RebalanceStats {
2317 participating: true,
2318 info: RebalanceInfo {
2319 status: RebalStatus::Completed,
2320 ..Default::default()
2321 },
2322 ..Default::default()
2323 }],
2324 ..Default::default()
2325 });
2326
2327 assert!(!should_auto_start_rebalance_after_init(
2328 pool_meta_has_active_decommission(&pool_meta),
2329 rebalance_meta
2330 .as_ref()
2331 .is_some_and(crate::services::rebalance::rebalance_requires_worker_activation)
2332 ));
2333 }
2334
2335 #[test]
2336 fn test_resolve_store_init_stage_result_passthrough_ok() {
2337 resolve_store_init_stage_result(Ok(()), "load_rebalance_meta").expect("successful stage should pass through");
2338 }
2339
2340 #[test]
2341 fn test_resolve_store_init_stage_result_wraps_error_context() {
2342 let err = resolve_store_init_stage_result(Err(StorageError::SlowDown), "start_rebalance")
2343 .expect_err("failed stage should be wrapped");
2344 let err_message = err.to_string();
2345 assert!(err_message.contains("store init failed during start_rebalance"));
2346 assert!(err_message.contains(&StorageError::SlowDown.to_string()));
2347 }
2348
2349 #[tokio::test]
2350 async fn test_wait_for_local_decommission_resume_delay_returns_true_after_delay() {
2351 let rx = CancellationToken::new();
2352 assert!(wait_for_local_decommission_resume_delay(&rx, Duration::from_millis(1)).await);
2353 }
2354
2355 #[tokio::test]
2356 async fn test_wait_for_local_decommission_resume_delay_returns_false_when_cancelled() {
2357 let rx = CancellationToken::new();
2358 rx.cancel();
2359 assert!(!wait_for_local_decommission_resume_delay(&rx, Duration::from_secs(1)).await);
2360 }
2361
2362 #[test]
2363 fn test_pool_first_endpoint_is_local_uses_pool_scope_for_expansion() {
2364 let mut remote_endpoint = Endpoint::try_from("http://127.0.0.2:9000/data1").expect("remote endpoint should parse");
2365 remote_endpoint.is_local = false;
2366
2367 let mut local_endpoint = Endpoint::try_from("http://127.0.0.1:9000/data1").expect("local endpoint should parse");
2368 local_endpoint.is_local = true;
2369
2370 let endpoints = EndpointServerPools::from(vec![
2371 PoolEndpoints {
2372 legacy: false,
2373 set_count: 1,
2374 drives_per_set: 1,
2375 endpoints: Endpoints::from(vec![remote_endpoint]),
2376 cmd_line: "pool-0".to_string(),
2377 platform: String::new(),
2378 },
2379 PoolEndpoints {
2380 legacy: false,
2381 set_count: 1,
2382 drives_per_set: 1,
2383 endpoints: Endpoints::from(vec![local_endpoint]),
2384 cmd_line: "pool-1".to_string(),
2385 platform: String::new(),
2386 },
2387 ]);
2388
2389 assert!(!endpoints.first_local(), "cluster first endpoint is intentionally remote");
2390 assert!(
2391 pool_first_endpoint_is_local(endpoints.as_ref().get(1).expect("second pool should exist")),
2392 "the expanded pool should be initialized by its own first local endpoint"
2393 );
2394 }
2395
2396 fn rpc_preflight_pools(pools: Vec<Vec<Endpoint>>) -> EndpointServerPools {
2397 EndpointServerPools::from(
2398 pools
2399 .into_iter()
2400 .enumerate()
2401 .map(|(pool_index, endpoints)| PoolEndpoints {
2402 legacy: false,
2403 set_count: 1,
2404 drives_per_set: endpoints.len(),
2405 endpoints: Endpoints::from(endpoints),
2406 cmd_line: format!("pool-{pool_index}"),
2407 platform: String::new(),
2408 })
2409 .collect::<Vec<_>>(),
2410 )
2411 }
2412
2413 fn parsed_local_endpoint() -> Endpoint {
2414 let mut endpoint = Endpoint::try_from("http://127.0.0.1:9000/data").expect("local endpoint should parse");
2415 endpoint.is_local = true;
2416 endpoint
2417 }
2418
2419 fn parsed_remote_endpoint() -> Endpoint {
2420 let mut endpoint = Endpoint::try_from("http://10.0.0.2:9000/data").expect("remote endpoint should parse");
2421 endpoint.is_local = false;
2422 endpoint
2423 }
2424
2425 #[test]
2426 fn test_rpc_secret_preflight_aborts_for_remote_endpoints_without_secret() {
2427 let pools = rpc_preflight_pools(vec![vec![parsed_local_endpoint(), parsed_remote_endpoint()]]);
2430
2431 let err = preflight_startup_rpc_secret_with(&pools, || {
2432 Err(std::io::Error::other(rustfs_credentials::RPC_SECRET_REQUIRED_MESSAGE))
2433 })
2434 .expect_err("remote endpoints without an RPC secret must abort startup before the format retry loop");
2435
2436 let message = err.to_string();
2437 assert!(message.contains("store init aborted"), "unexpected error: {message}");
2438 assert!(
2439 message.contains(rustfs_credentials::RPC_SECRET_REQUIRED_MESSAGE),
2440 "error must state the resolution failure: {message}"
2441 );
2442 assert!(
2443 message.contains(rustfs_credentials::RPC_SECRET_REQUIRED_OPERATOR_MESSAGE),
2444 "error must carry the operator remediation guidance: {message}"
2445 );
2446 }
2447
2448 #[test]
2449 fn test_rpc_secret_preflight_aborts_for_remote_endpoint_in_later_pool() {
2450 let pools = rpc_preflight_pools(vec![vec![parsed_local_endpoint()], vec![parsed_remote_endpoint()]]);
2453
2454 let err = preflight_startup_rpc_secret_with(&pools, || {
2455 Err(std::io::Error::other(rustfs_credentials::RPC_SECRET_REQUIRED_MESSAGE))
2456 })
2457 .expect_err("a remote endpoint in a later pool must abort startup without an RPC secret");
2458
2459 let message = err.to_string();
2460 assert!(message.contains("store init aborted"), "unexpected error: {message}");
2461 }
2462
2463 #[test]
2464 fn test_rpc_secret_preflight_skips_resolution_for_local_only_endpoints() {
2465 preflight_startup_rpc_secret_with(&rpc_preflight_pools(vec![vec![parsed_local_endpoint()]]), || {
2466 panic!("single-node topology must not resolve an RPC secret")
2467 })
2468 .expect("local-only topology must start without an RPC secret");
2469 }
2470
2471 #[test]
2472 fn test_rpc_secret_preflight_accepts_remote_endpoints_with_resolved_secret() {
2473 preflight_startup_rpc_secret_with(&rpc_preflight_pools(vec![vec![parsed_remote_endpoint()]]), || {
2474 Ok("resolved-secret".to_string())
2475 })
2476 .expect("a resolvable RPC secret must not block startup");
2477 }
2478
2479 #[tokio::test]
2480 #[serial_test::serial(storage_class_env)]
2481 async fn test_new_with_instance_ctx_aborts_before_format_retry_loop_without_rpc_secret() {
2482 if rustfs_credentials::try_get_rpc_token().is_ok() {
2488 return;
2489 }
2490
2491 let mut remote_endpoint = parsed_remote_endpoint();
2492 remote_endpoint.set_pool_index(0);
2493 remote_endpoint.set_set_index(0);
2494 remote_endpoint.set_disk_index(0);
2495
2496 let err = crate::store::ECStore::new_with_instance_ctx(
2497 "127.0.0.1:0".parse().expect("test address"),
2498 rpc_preflight_pools(vec![vec![remote_endpoint]]),
2499 CancellationToken::new(),
2500 Arc::new(crate::runtime::instance::InstanceContext::new()),
2501 )
2502 .await
2503 .expect_err("distributed init without an RPC secret must abort before the format retry loop");
2504
2505 let message = err.to_string();
2506 assert!(message.contains("store init aborted"), "unexpected error: {message}");
2507 assert!(
2508 message.contains(rustfs_credentials::RPC_SECRET_REQUIRED_OPERATOR_MESSAGE),
2509 "abort must carry the operator remediation guidance: {message}"
2510 );
2511 }
2512
2513 fn endpoint_pools_with_drive_counts(counts: &[usize]) -> EndpointServerPools {
2514 EndpointServerPools::from(
2515 counts
2516 .iter()
2517 .enumerate()
2518 .map(|(pool_index, &drives_per_set)| PoolEndpoints {
2519 legacy: false,
2520 set_count: 1,
2521 drives_per_set,
2522 endpoints: Endpoints::from(Vec::new()),
2523 cmd_line: format!("pool-{pool_index}"),
2524 platform: String::new(),
2525 })
2526 .collect::<Vec<_>>(),
2527 )
2528 }
2529
2530 #[test]
2531 fn startup_pool_defaults_are_resolved_per_pool() {
2532 let validate = |pools: &EndpointServerPools| {
2533 let drive_counts: Vec<_> = pools.as_ref().iter().map(|pool| pool.drives_per_set).collect();
2534 crate::config::storageclass::lookup_config_for_pools_without_env(&KVS::new(), &drive_counts).map(|_| ())
2535 };
2536 let defaults = resolve_startup_pool_defaults_with(&endpoint_pools_with_drive_counts(&[4, 2]), validate)
2537 .expect("heterogeneous topology should resolve");
2538 assert_eq!(defaults, vec![2, 1]);
2539
2540 let defaults = resolve_startup_pool_defaults_with(&endpoint_pools_with_drive_counts(&[4, 6]), validate)
2541 .expect("heterogeneous topology should resolve");
2542 assert_eq!(defaults, vec![2, 3]);
2543 }
2544
2545 #[test]
2546 fn startup_pool_defaults_validate_explicit_environment_for_every_pool() {
2547 let validate = |pools: &EndpointServerPools| {
2548 let drive_counts: Vec<_> = pools.as_ref().iter().map(|pool| pool.drives_per_set).collect();
2549 let mut kvs = KVS::new();
2550 kvs.insert(crate::config::storageclass::CLASS_STANDARD.to_string(), "EC:2".to_string());
2551 crate::config::storageclass::lookup_config_for_pools_without_env(&kvs, &drive_counts).map(|_| ())
2552 };
2553 let err = resolve_startup_pool_defaults_with(&endpoint_pools_with_drive_counts(&[4, 2]), validate)
2554 .expect_err("explicit EC:2 must fail before any two-drive pool I/O");
2555 assert!(err.to_string().contains("pool 1") && err.to_string().contains("2 drives"));
2556 }
2557
2558 #[test]
2559 #[serial_test::serial(storage_class_env)]
2560 fn startup_pool_defaults_validate_environment_without_changing_metadata_fallback() {
2561 temp_env::with_vars(
2562 [
2563 (crate::config::storageclass::STANDARD_ENV, Some("EC:1")),
2564 (crate::config::storageclass::RRS_ENV, None),
2565 (crate::config::storageclass::OPTIMIZE_ENV, None),
2566 (crate::config::storageclass::INLINE_BLOCK_ENV, None),
2567 ],
2568 || {
2569 let runtime = crate::config::storageclass::lookup_config_for_pools(&KVS::new(), &[6, 4])
2570 .expect("explicit standard parity must resolve the runtime candidate");
2571 assert_eq!(runtime.parities_for_sc(crate::config::storageclass::STANDARD), Some(vec![1, 1]));
2572
2573 let defaults = super::resolve_startup_pool_defaults(&endpoint_pools_with_drive_counts(&[6, 4]))
2574 .expect("explicit standard parity must validate for every pool");
2575 assert_eq!(defaults, vec![3, 2]);
2576 },
2577 );
2578 }
2579
2580 async fn without_storage_class_env<F: Future>(future: F) -> F::Output {
2581 temp_env::async_with_vars(
2582 [
2583 (crate::config::storageclass::STANDARD_ENV, None::<&str>),
2584 (crate::config::storageclass::RRS_ENV, None::<&str>),
2585 (crate::config::storageclass::OPTIMIZE_ENV, None::<&str>),
2586 (crate::config::storageclass::INLINE_BLOCK_ENV, None::<&str>),
2587 ],
2588 future,
2589 )
2590 .await
2591 }
2592
2593 async fn build_isolated_test_store(
2595 temp_dir: &std::path::Path,
2596 cmd_line: &str,
2597 pool_drive_counts: &[usize],
2598 ) -> (
2599 Arc<crate::runtime::instance::InstanceContext>,
2600 Arc<crate::store::ECStore>,
2601 CancellationToken,
2602 ) {
2603 build_isolated_test_store_with_shutdown(temp_dir, cmd_line, pool_drive_counts, CancellationToken::new()).await
2604 }
2605
2606 async fn build_isolated_test_store_with_shutdown(
2607 temp_dir: &std::path::Path,
2608 cmd_line: &str,
2609 pool_drive_counts: &[usize],
2610 shutdown: CancellationToken,
2611 ) -> (
2612 Arc<crate::runtime::instance::InstanceContext>,
2613 Arc<crate::store::ECStore>,
2614 CancellationToken,
2615 ) {
2616 let pool_layouts = pool_drive_counts
2617 .iter()
2618 .map(|&drives_per_set| (1, drives_per_set))
2619 .collect::<Vec<_>>();
2620 build_isolated_test_store_with_layout(temp_dir, cmd_line, &pool_layouts, shutdown, None).await
2621 }
2622
2623 async fn build_isolated_test_store_with_layout(
2624 temp_dir: &std::path::Path,
2625 cmd_line: &str,
2626 pool_layouts: &[(usize, usize)],
2627 shutdown: CancellationToken,
2628 instance_ctx: Option<Arc<crate::runtime::instance::InstanceContext>>,
2629 ) -> (
2630 Arc<crate::runtime::instance::InstanceContext>,
2631 Arc<crate::store::ECStore>,
2632 CancellationToken,
2633 ) {
2634 let mut pools = Vec::with_capacity(pool_layouts.len());
2635 for (pool_index, &(set_count, drives_per_set)) in pool_layouts.iter().enumerate() {
2636 let mut endpoints = Vec::with_capacity(set_count * drives_per_set);
2637 for set_index in 0..set_count {
2638 for disk_index in 0..drives_per_set {
2639 let path = temp_dir.join(format!("pool{pool_index}/set{set_index}/disk{disk_index}"));
2640 tokio::fs::create_dir_all(&path).await.expect("create disk dir");
2641 let mut endpoint =
2642 Endpoint::try_from(path.to_str().expect("disk path should be utf-8")).expect("local endpoint");
2643 endpoint.set_pool_index(pool_index);
2644 endpoint.set_set_index(set_index);
2645 endpoint.set_disk_index(disk_index);
2646 endpoints.push(endpoint);
2647 }
2648 }
2649 pools.push(PoolEndpoints {
2650 legacy: false,
2651 set_count,
2652 drives_per_set,
2653 endpoints: Endpoints::from(endpoints),
2654 cmd_line: format!("{cmd_line}-pool-{pool_index}"),
2655 platform: "test".to_string(),
2656 });
2657 }
2658 let endpoint_pools = EndpointServerPools(pools);
2659 crate::services::notification_sys::install_cross_pool_fence_fleet_proof_for_test();
2660
2661 let instance_ctx = instance_ctx.unwrap_or_else(|| Arc::new(crate::runtime::instance::InstanceContext::new()));
2662 crate::store::init_local_disks_with_instance_ctx(&instance_ctx, endpoint_pools.clone())
2663 .await
2664 .expect("register local disks into the fresh context");
2665
2666 let store = crate::store::ECStore::new_with_instance_ctx(
2667 "127.0.0.1:0".parse().expect("test address"),
2668 endpoint_pools,
2669 shutdown.clone(),
2670 instance_ctx.clone(),
2671 )
2672 .await
2673 .expect("store should build around the fresh context");
2674
2675 let layout = DecommissionErasureLayout { data: 2, parity: 2 };
2679 let snapshot: Vec<DecommissionPoolCapacityInfo> = store
2680 .pools
2681 .iter()
2682 .enumerate()
2683 .map(|(pool_index, _)| DecommissionPoolCapacityInfo::for_test(pool_index, layout, 1 << 40, 1 << 40, 1 << 30))
2684 .collect();
2685 set_decommission_capacity_info_overrides_for_test(store.id, (0..128).map(|_| snapshot.clone()).collect());
2686
2687 (instance_ctx, store, shutdown)
2688 }
2689
2690 #[cfg(feature = "test-util")]
2691 #[tokio::test]
2692 #[serial_test::serial(storage_class_env)]
2693 async fn cross_key_materialized_copy_does_not_publish_remote_tuple_ownership() {
2694 let temp_dir = tempfile::tempdir().expect("create materialized-copy store dir");
2695 let (ctx, store, shutdown) =
2696 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "copy-materialized", &[4])).await;
2697 crate::bucket::metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
2698
2699 let bucket = format!("copy-materialized-{}", Uuid::new_v4());
2700 let source_object = "source-restored.bin";
2701 let target_object = "target-local.bin";
2702 let payload = b"materialized copy must own only its new local bytes".to_vec();
2703 store
2704 .make_bucket(&bucket, &MakeBucketOptions::default())
2705 .await
2706 .expect("copy test bucket should be created");
2707
2708 let tier_name = "COPY-MATERIALIZED-TIER";
2709 register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
2710 let lease = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
2711 .await
2712 .expect("mock tier generation should be available");
2713 let backend_identity = lease.backend_identity();
2714 drop(lease);
2715
2716 let mut source_metadata = HashMap::from([
2717 ("content-type".to_string(), "application/octet-stream".to_string()),
2718 (
2719 "x-amz-restore".to_string(),
2720 "ongoing-request=\"false\", expiry-date=\"2099-01-01T00:00:00Z\"".to_string(),
2721 ),
2722 ]);
2723 for (suffix, value) in [
2724 (
2725 rustfs_utils::http::SUFFIX_TRANSITION_STATUS,
2726 crate::bucket::lifecycle::core::TRANSITION_COMPLETE.to_string(),
2727 ),
2728 (
2729 rustfs_utils::http::SUFFIX_TRANSITIONED_OBJECTNAME,
2730 "remote/source-restored.bin".to_string(),
2731 ),
2732 (rustfs_utils::http::SUFFIX_TRANSITION_TIER, tier_name.to_string()),
2733 (rustfs_utils::http::SUFFIX_TRANSITIONED_VERSION_ID, "remote-source-version".to_string()),
2734 (
2735 rustfs_utils::http::SUFFIX_TRANSITIONED_VERSION_STATE,
2736 rustfs_filemeta::TransitionVersionState::Exact.as_str().to_string(),
2737 ),
2738 (
2739 rustfs_utils::http::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
2740 rustfs_utils::crypto::hex(backend_identity),
2741 ),
2742 (rustfs_utils::http::SUFFIX_TRANSITION_TRANSACTION_ID, Uuid::new_v4().to_string()),
2743 ] {
2744 rustfs_utils::http::metadata_compat::insert_str(&mut source_metadata, suffix, value);
2745 }
2746
2747 let mut source_body = PutObjReader::from_vec(payload.clone());
2748 store
2749 .put_object(
2750 &bucket,
2751 source_object,
2752 &mut source_body,
2753 &ObjectOptions {
2754 user_defined: source_metadata,
2755 ..Default::default()
2756 },
2757 )
2758 .await
2759 .expect("restored transitioned source should be written");
2760
2761 let source_opts = ObjectOptions::default();
2762 let mut source_info = store
2763 .get_object_info(&bucket, source_object, &source_opts)
2764 .await
2765 .expect("source metadata should be readable");
2766 assert_eq!(
2767 source_info.transitioned_object.status,
2768 crate::bucket::lifecycle::core::TRANSITION_COMPLETE
2769 );
2770 assert_eq!(source_info.transitioned_object.tier, tier_name);
2771 assert!(source_info.data_dir.is_some(), "restored source must retain local data");
2772 source_info.put_object_reader = Some(PutObjReader::from_vec(payload.clone()));
2773
2774 let copied = store
2775 .copy_object(
2776 &bucket,
2777 source_object,
2778 &bucket,
2779 target_object,
2780 &mut source_info,
2781 &source_opts,
2782 &ObjectOptions::default(),
2783 )
2784 .await
2785 .expect("cross-key materialized copy should succeed");
2786 assert!(copied.transitioned_object.status.is_empty());
2787 assert!(copied.transitioned_object.name.is_empty());
2788 assert!(copied.transitioned_object.version_id.is_empty());
2789 assert!(copied.transitioned_object.tier.is_empty());
2790 assert!(!copied.transitioned_object.free_version);
2791 for suffix in [
2792 rustfs_utils::http::SUFFIX_TRANSITION_STATUS,
2793 rustfs_utils::http::SUFFIX_TRANSITIONED_OBJECTNAME,
2794 rustfs_utils::http::SUFFIX_TRANSITIONED_VERSION_ID,
2795 rustfs_utils::http::SUFFIX_TRANSITIONED_VERSION_STATE,
2796 rustfs_utils::http::SUFFIX_TRANSITION_TIER,
2797 rustfs_utils::http::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
2798 rustfs_utils::http::SUFFIX_TRANSITION_TRANSACTION_ID,
2799 ] {
2800 assert!(
2801 !rustfs_utils::http::metadata_compat::contains_key_str(copied.user_defined.as_ref(), suffix),
2802 "target retained protected source suffix {suffix}"
2803 );
2804 }
2805 assert_eq!(
2806 copied.user_defined.get("content-type").map(String::as_str),
2807 Some("application/octet-stream")
2808 );
2809
2810 let mut target_reader = store
2811 .get_object_reader(&bucket, target_object, None, HeaderMap::new(), &ObjectOptions::default())
2812 .await
2813 .expect("target object should be readable");
2814 let mut target_body = Vec::new();
2815 target_reader
2816 .stream
2817 .read_to_end(&mut target_body)
2818 .await
2819 .expect("target body should stream");
2820 assert_eq!(target_body, payload);
2821
2822 let source_after = store
2823 .get_object_info(&bucket, source_object, &source_opts)
2824 .await
2825 .expect("source metadata should remain readable after copy");
2826 assert_eq!(
2827 source_after.transitioned_object.status,
2828 crate::bucket::lifecycle::core::TRANSITION_COMPLETE
2829 );
2830 assert_eq!(source_after.transitioned_object.tier, tier_name);
2831 assert_eq!(source_after.transitioned_object.name, "remote/source-restored.bin");
2832 shutdown.cancel();
2833 }
2834
2835 #[cfg(feature = "test-util")]
2836 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2837 #[serial_test::serial(storage_class_env)]
2838 async fn tier_overwrite_put_and_self_copy_recover_persisted_cleanup_owners() {
2839 use crate::bucket::lifecycle::bucket_lifecycle_ops::ExpiryState;
2840 use crate::bucket::lifecycle::tier_free_version_recovery::recover_tier_free_versions;
2841 use rustfs_filemeta::TransitionVersionState::{Exact, KnownDisabled, SuspendedNull};
2842 use rustfs_s3_client::transition_api::ReaderImpl;
2843 use rustfs_utils::http::{
2844 SUFFIX_TRANSITION_STATUS, SUFFIX_TRANSITION_TIER, SUFFIX_TRANSITION_TIER_DESTINATION_ID,
2845 SUFFIX_TRANSITIONED_OBJECTNAME, SUFFIX_TRANSITIONED_VERSION_ID, SUFFIX_TRANSITIONED_VERSION_STATE, insert_str,
2846 };
2847
2848 let temp_dir = tempfile::tempdir().expect("create tier overwrite store");
2849 let (mut ctx, mut store, mut shutdown) =
2850 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "tier-overwrite", &[4])).await;
2851 crate::bucket::metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
2852 let tier = "OVERWRITE-TIER";
2853 let backend = register_mock_tier(&ctx.tier_config_mgr(), tier).await;
2854 let lease = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier)
2855 .await
2856 .expect("tier identity");
2857 let identity = rustfs_utils::crypto::hex(lease.backend_identity());
2858 drop(lease);
2859
2860 for state in [Exact, KnownDisabled, SuspendedNull] {
2861 for suspended in [false, true] {
2862 for self_copy in [false, true] {
2863 let bucket = format!("tier-overwrite-{}", Uuid::new_v4());
2864 let object = "object";
2865 let remote = format!("remote/{bucket}");
2866 let version = match state {
2867 Exact => "opaque-overwrite-version",
2868 SuspendedNull => "null",
2869 _ => "",
2870 };
2871 let payload = vec![0x5b; if suspended { 512 * 1024 } else { 257 }];
2872 store
2873 .make_bucket(&bucket, &MakeBucketOptions::default())
2874 .await
2875 .expect("create bucket");
2876 backend.set_put_remote_version(Some(version.to_string())).await;
2877 let lease = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier)
2878 .await
2879 .expect("seed tier lease");
2880 lease
2881 .put(
2882 &remote,
2883 ReaderImpl::Body(bytes::Bytes::from(payload.clone())),
2884 payload.len().try_into().expect("payload size"),
2885 )
2886 .await
2887 .expect("seed remote bytes");
2888 drop(lease);
2889 let mut metadata = HashMap::from([
2890 ("content-type".to_string(), "application/octet-stream".to_string()),
2891 (
2892 "x-amz-restore".to_string(),
2893 "ongoing-request=\"false\", expiry-date=\"2099-01-01T00:00:00Z\"".to_string(),
2894 ),
2895 ]);
2896 for (suffix, value) in [
2897 (SUFFIX_TRANSITION_STATUS, "complete"),
2898 (SUFFIX_TRANSITION_TIER, tier),
2899 (SUFFIX_TRANSITION_TIER_DESTINATION_ID, identity.as_str()),
2900 (SUFFIX_TRANSITIONED_OBJECTNAME, remote.as_str()),
2901 (SUFFIX_TRANSITIONED_VERSION_STATE, state.as_str()),
2902 ] {
2903 insert_str(&mut metadata, suffix, value.to_string());
2904 }
2905 if !version.is_empty() {
2906 insert_str(&mut metadata, SUFFIX_TRANSITIONED_VERSION_ID, version.to_string());
2907 }
2908 let options = ObjectOptions {
2909 version_suspended: suspended,
2910 ..Default::default()
2911 };
2912 store
2913 .put_object(
2914 &bucket,
2915 object,
2916 &mut PutObjReader::from_vec(payload.clone()),
2917 &ObjectOptions {
2918 user_defined: metadata,
2919 ..options.clone()
2920 },
2921 )
2922 .await
2923 .expect("seed transitioned source with locally restored bytes");
2924 backend.set_remove_failure(true);
2930 let expected = if self_copy {
2931 payload.clone()
2932 } else {
2933 vec![0x73; payload.len()]
2934 };
2935 let new_metadata = HashMap::from([
2936 ("content-type".to_string(), "text/plain".to_string()),
2937 ("x-amz-meta-replacement".to_string(), "kept".to_string()),
2938 ]);
2939 if self_copy {
2940 let mut source = store
2941 .get_object_info(&bucket, object, &options)
2942 .await
2943 .expect("self-copy source");
2944 source.metadata_only = false;
2945 source.user_defined = Arc::new(new_metadata);
2946 source.put_object_reader = Some(PutObjReader::from_vec(expected.clone()));
2947 store
2948 .copy_object(&bucket, object, &bucket, object, &mut source, &options, &options)
2949 .await
2950 .expect("materialized self-copy");
2951 } else {
2952 store
2953 .put_object(
2954 &bucket,
2955 object,
2956 &mut PutObjReader::from_vec(expected.clone()),
2957 &ObjectOptions {
2958 user_defined: new_metadata,
2959 ..options.clone()
2960 },
2961 )
2962 .await
2963 .expect("overwrite transitioned null version");
2964 }
2965
2966 let set = store.pools[0].get_disks_by_key(object);
2967 let versions = set
2968 .load_file_info_versions_exact(&bucket, object)
2969 .await
2970 .expect("read committed disk metadata")
2971 .expect("replacement metadata exists");
2972 let free: Vec<_> = versions
2973 .versions
2974 .iter()
2975 .chain(versions.free_versions.iter())
2976 .filter(|fi| fi.tier_free_version())
2977 .collect();
2978 assert_eq!(free.len(), 1, "{state:?}, suspended={suspended}, copy={self_copy}");
2979 assert_eq!(free[0].transitioned_objname, remote);
2980 assert_eq!(free[0].transition_version_state, state);
2981 assert!(backend.contains(&remote).await, "commit must not delete remote bytes before cleanup");
2982 let removed_before = backend.remove_count().await;
2983
2984 wait_for_expiry_workers_idle(&store).await;
2989
2990 let tier_config = ctx
2994 .tier_config_mgr()
2995 .read()
2996 .await
2997 .tiers
2998 .get(tier)
2999 .expect("tier configuration survives restart")
3000 .clone_with_credentials();
3001 drop(set);
3002 shutdown.cancel();
3003 drop(store);
3004 drop(ctx);
3005 (ctx, store, shutdown) =
3006 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "tier-overwrite-restart", &[4]))
3007 .await;
3008 crate::bucket::metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
3009 {
3010 let manager = ctx.tier_config_mgr();
3011 let mut manager = manager.write().await;
3012 manager.tiers.insert(tier.to_string(), tier_config);
3013 manager
3014 .install_test_driver(tier, Box::new(backend.clone()))
3015 .expect("rebind the same remote destination after restart");
3016 }
3017 let set = store.pools[0].get_disks_by_key(object);
3018 backend.set_remove_failure(true);
3021 ExpiryState::resize_workers(1, Arc::clone(&store)).await;
3022 let recovered = recover_tier_free_versions(Arc::clone(&store), 100, None, None)
3023 .await
3024 .expect("recover persisted cleanup owner");
3025 assert!(recovered.enqueued >= 1);
3026 wait_for_expiry_workers_idle(&store).await;
3027 assert!(backend.contains(&remote).await, "failed cleanup must retain remote bytes");
3028 assert_eq!(backend.remove_count().await, removed_before);
3029 backend.set_remove_failure(false);
3030 eprintln!("Recovering overwrite cleanup: state={state:?}, suspended={suspended}, copy={self_copy}");
3033 wait_for_tier_free_version_recovery(Arc::clone(&store), &backend, removed_before + 1).await;
3034 let versions = set
3035 .load_file_info_versions_exact(&bucket, object)
3036 .await
3037 .expect("read cleanup progress")
3038 .expect("new object must survive cleanup");
3039 assert!(
3040 versions
3041 .versions
3042 .iter()
3043 .chain(versions.free_versions.iter())
3044 .all(|fi| !fi.tier_free_version()),
3045 "cleanup must remove its owner: {state:?}, suspended={suspended}, copy={self_copy}"
3046 );
3047 assert!(!backend.contains(&remote).await);
3048 assert_eq!(backend.remove_count().await, removed_before + 1, "one successful remote DELETE per owner");
3049 assert_eq!(backend.remove_versions().await.last(), Some(&(remote.clone(), version.to_string())));
3050 let mut reader = store
3051 .get_object_reader(&bucket, object, None, HeaderMap::new(), &options)
3052 .await
3053 .expect("replacement remains readable");
3054 let mut actual = Vec::new();
3055 reader.stream.read_to_end(&mut actual).await.expect("read replacement bytes");
3056 assert_eq!(actual, expected);
3057 let current = store
3058 .get_object_info(&bucket, object, &options)
3059 .await
3060 .expect("replacement metadata");
3061 assert_eq!(current.user_defined.get("content-type").map(String::as_str), Some("text/plain"));
3062 assert_eq!(current.user_defined.get("x-amz-meta-replacement").map(String::as_str), Some("kept"));
3063 assert!(current.transitioned_object.status.is_empty());
3064 }
3065 }
3066 }
3067 shutdown.cancel();
3068 }
3069
3070 #[cfg(feature = "test-util")]
3071 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
3072 #[serial_test::serial(storage_class_env)]
3073 async fn copy_object_immediately_reads_small_completed_multipart_source() {
3074 let temp_dir = tempfile::tempdir().expect("create small multipart copy store dir");
3075 let (_ctx, store, shutdown) =
3076 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "small-multipart-copy", &[1])).await;
3077 crate::bucket::metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
3078
3079 let bucket = format!("small-multipart-copy-{}", Uuid::new_v4());
3080 let source_object = "docker/registry/v2/repositories/example/_uploads/upload-id/data";
3081 let target_object = "docker/registry/v2/blobs/sha256/c0/digest/data";
3082 let payload = vec![0xAB; 273];
3083
3084 store
3085 .make_bucket(&bucket, &MakeBucketOptions::default())
3086 .await
3087 .expect("create bucket for small multipart copy");
3088 let upload = store
3089 .new_multipart_upload(&bucket, source_object, &ObjectOptions::default())
3090 .await
3091 .expect("create source multipart upload");
3092 let mut part_reader = PutObjReader::from_vec(payload.clone());
3093 let part = store
3094 .put_object_part(&bucket, source_object, &upload.upload_id, 1, &mut part_reader, &ObjectOptions::default())
3095 .await
3096 .expect("stage small multipart source part");
3097 let completed = store
3098 .clone()
3099 .complete_multipart_upload(
3100 &bucket,
3101 source_object,
3102 &upload.upload_id,
3103 vec![crate::storage_api_contracts::multipart::CompletePart {
3104 part_num: part.part_num,
3105 etag: part.etag,
3106 ..Default::default()
3107 }],
3108 &ObjectOptions::default(),
3109 )
3110 .await
3111 .expect("complete the small multipart source");
3112 assert_eq!(completed.get_actual_size().expect("completed object logical size"), payload.len() as i64);
3113
3114 let source_reader = store
3115 .get_object_reader(&bucket, source_object, None, HeaderMap::new(), &ObjectOptions::default())
3116 .await
3117 .expect("completed multipart source should be immediately readable");
3118 let mut copy_info = source_reader.object_info.clone();
3119 let actual_size = copy_info.get_actual_size().expect("copy source logical size should resolve");
3120 assert_eq!(actual_size, payload.len() as i64);
3121 let copy_reader = rustfs_rio::HashReader::from_stream(source_reader.stream, actual_size, actual_size, None, None, false)
3122 .expect("copy source hash reader should build");
3123 copy_info.put_object_reader = Some(PutObjReader::new(copy_reader));
3124
3125 store
3126 .copy_object(
3127 &bucket,
3128 source_object,
3129 &bucket,
3130 target_object,
3131 &mut copy_info,
3132 &ObjectOptions::default(),
3133 &ObjectOptions::default(),
3134 )
3135 .await
3136 .expect("CopyObject should accept a freshly completed multipart source");
3137
3138 let mut target_reader = store
3139 .get_object_reader(&bucket, target_object, None, HeaderMap::new(), &ObjectOptions::default())
3140 .await
3141 .expect("copied target should be readable");
3142 let mut target_body = Vec::new();
3143 target_reader
3144 .stream
3145 .read_to_end(&mut target_body)
3146 .await
3147 .expect("target body should stream");
3148 assert_eq!(target_body, payload);
3149 shutdown.cancel();
3150 }
3151
3152 #[cfg(feature = "test-util")]
3153 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
3154 #[serial_test::serial(storage_class_env)]
3155 async fn unfinished_multipart_upload_is_not_copy_source_readable() {
3156 let temp_dir = tempfile::tempdir().expect("create unfinished multipart copy store dir");
3157 let (_ctx, store, shutdown) =
3158 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "unfinished-multipart-copy", &[1])).await;
3159 crate::bucket::metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
3160
3161 let bucket = format!("unfinished-multipart-copy-{}", Uuid::new_v4());
3162 let source_object = "docker/registry/v2/repositories/example/_uploads/upload-id/data";
3163 let payload = vec![0xCD; 273];
3164
3165 store
3166 .make_bucket(&bucket, &MakeBucketOptions::default())
3167 .await
3168 .expect("create bucket for unfinished multipart copy source");
3169 let upload = store
3170 .new_multipart_upload(&bucket, source_object, &ObjectOptions::default())
3171 .await
3172 .expect("create source multipart upload");
3173 let mut part_reader = PutObjReader::from_vec(payload);
3174 store
3175 .put_object_part(&bucket, source_object, &upload.upload_id, 1, &mut part_reader, &ObjectOptions::default())
3176 .await
3177 .expect("stage unfinished multipart source part");
3178
3179 let source_err = match store
3180 .get_object_reader(&bucket, source_object, None, HeaderMap::new(), &ObjectOptions::default())
3181 .await
3182 {
3183 Ok(_) => panic!("an uncompleted multipart upload must not be readable as a CopyObject source"),
3184 Err(err) => err,
3185 };
3186 assert!(
3187 matches!(
3188 source_err,
3189 StorageError::ObjectNotFound(_, _) | StorageError::FileNotFound | StorageError::VersionNotFound(_, _, _)
3190 ),
3191 "unexpected unfinished multipart source error: {source_err:?}"
3192 );
3193
3194 shutdown.cancel();
3195 }
3196
3197 #[tokio::test]
3198 #[serial_test::serial(storage_class_env)]
3199 async fn pool_metadata_preflight_recovery_preserves_single_and_multi_pool_public_mutations() {
3200 for layout in [vec![4], vec![4, 4]] {
3201 let temp_dir = tempfile::tempdir().unwrap();
3202 let (_ctx, store, shutdown) =
3203 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "pool-meta-retry", &layout)).await;
3204 crate::bucket::metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
3205 let bucket = format!("pool-meta-retry-{}", Uuid::new_v4());
3206 store.make_bucket(&bucket, &MakeBucketOptions::default()).await.unwrap();
3207 let mut saved_disks = Vec::new();
3208 for set in &store.pools[0].disk_set {
3209 let mut disks = set.disks.write().await;
3210 let count = disks.len();
3211 saved_disks.push((set.clone(), std::mem::replace(&mut *disks, vec![None; count])));
3212 }
3213 let indices = (0..layout.len()).collect::<Vec<_>>();
3214 let err = store.save_current_pool_meta_for_test(&indices).await.unwrap_err();
3215 assert_eq!(
3216 err.pool_metadata_failure().unwrap().kind,
3217 crate::error::PoolMetadataFailure::ReadUnavailable
3218 );
3219 for (set, disks) in saved_disks {
3220 *set.disks.write().await = disks;
3221 }
3222 store.save_current_pool_meta_for_test(&indices).await.unwrap();
3223 assert!(store.pool_meta_writes_ready().await);
3224
3225 let payload = b"pool metadata recovery payload".to_vec();
3226 store
3227 .put_object(&bucket, "put", &mut PutObjReader::from_vec(payload.clone()), &ObjectOptions::default())
3228 .await
3229 .unwrap();
3230 let mut reader = store
3231 .get_object_reader(&bucket, "put", None, HeaderMap::new(), &ObjectOptions::default())
3232 .await
3233 .unwrap();
3234 let mut actual = Vec::new();
3235 reader.stream.read_to_end(&mut actual).await.unwrap();
3236 assert_eq!(actual, payload);
3237 drop(reader);
3238 store.delete_object(&bucket, "put", ObjectOptions::default()).await.unwrap();
3239 assert!(crate::error::is_err_object_not_found(
3240 &store
3241 .get_object_info(&bucket, "put", &ObjectOptions::default())
3242 .await
3243 .unwrap_err()
3244 ));
3245
3246 let upload = store
3247 .new_multipart_upload(&bucket, "multipart", &ObjectOptions::default())
3248 .await
3249 .unwrap();
3250 let part = store
3251 .put_object_part(
3252 &bucket,
3253 "multipart",
3254 &upload.upload_id,
3255 1,
3256 &mut PutObjReader::from_vec(payload.clone()),
3257 &ObjectOptions::default(),
3258 )
3259 .await
3260 .unwrap();
3261 store
3262 .clone()
3263 .complete_multipart_upload(
3264 &bucket,
3265 "multipart",
3266 &upload.upload_id,
3267 vec![crate::storage_api_contracts::multipart::CompletePart {
3268 part_num: part.part_num,
3269 etag: part.etag,
3270 ..Default::default()
3271 }],
3272 &ObjectOptions::default(),
3273 )
3274 .await
3275 .unwrap();
3276 let mut reader = store
3277 .get_object_reader(&bucket, "multipart", None, HeaderMap::new(), &ObjectOptions::default())
3278 .await
3279 .unwrap();
3280 actual.clear();
3281 reader.stream.read_to_end(&mut actual).await.unwrap();
3282 assert_eq!(actual, payload);
3283 drop(reader);
3284 let upload = store
3285 .new_multipart_upload(&bucket, "abort", &ObjectOptions::default())
3286 .await
3287 .unwrap();
3288 store
3289 .abort_multipart_upload(&bucket, "abort", &upload.upload_id, &ObjectOptions::default())
3290 .await
3291 .unwrap();
3292 assert!(store.pool_meta_writes_ready().await);
3293 shutdown.cancel();
3294 }
3295 }
3296
3297 #[cfg(feature = "test-util")]
3298 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
3299 #[serial_test::serial(storage_class_env)]
3300 async fn complete_multipart_waits_for_tail_rename_before_copy_source_visibility() {
3301 let temp_dir = tempfile::tempdir().expect("create early-ack multipart copy store dir");
3302 let (_ctx, store, shutdown) =
3303 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "early-ack-multipart-copy", &[4])).await;
3304 crate::bucket::metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
3305
3306 let bucket = format!("early-ack-multipart-copy-{}", Uuid::new_v4());
3307 let source_object = "docker/registry/v2/repositories/example/_uploads/upload-id/data";
3308 let target_object = "docker/registry/v2/blobs/sha256/c0/digest/data";
3309 let payload = vec![0xCD; 273];
3310
3311 store
3312 .make_bucket(&bucket, &MakeBucketOptions::default())
3313 .await
3314 .expect("create bucket for early-ack multipart copy");
3315 let upload = store
3316 .new_multipart_upload(&bucket, source_object, &ObjectOptions::default())
3317 .await
3318 .expect("create source multipart upload");
3319 let mut part_reader = PutObjReader::from_vec(payload.clone());
3320 let part = store
3321 .put_object_part(&bucket, source_object, &upload.upload_id, 1, &mut part_reader, &ObjectOptions::default())
3322 .await
3323 .expect("stage small multipart source part");
3324 let completed_parts = vec![crate::storage_api_contracts::multipart::CompletePart {
3325 part_num: part.part_num,
3326 etag: part.etag,
3327 ..Default::default()
3328 }];
3329
3330 temp_env::async_with_vars([(crate::set_disk::ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async {
3331 let rename_tasks = crate::set_disk::rename_fanout_barrier::observe_tasks(source_object);
3332 let rename_barrier = crate::set_disk::rename_fanout_barrier::arm(
3333 source_object,
3334 0,
3335 crate::set_disk::rename_fanout_barrier::PHASE_RENAME,
3336 );
3337 let complete_store = Arc::clone(&store);
3338 let complete_bucket = bucket.clone();
3339 let complete_upload_id = upload.upload_id.clone();
3340 let mut complete = tokio::spawn(async move {
3341 complete_store
3342 .complete_multipart_upload(
3343 &complete_bucket,
3344 source_object,
3345 &complete_upload_id,
3346 completed_parts,
3347 &ObjectOptions::default(),
3348 )
3349 .await
3350 });
3351 tokio::time::timeout(Duration::from_secs(30), rename_barrier.wait_until_paused())
3352 .await
3353 .expect("multipart completion should pause one tail disk during rename");
3354 assert!(
3355 rename_tasks.running() >= 1,
3356 "the paused multipart tail disk must remain in flight before completion returns"
3357 );
3358 assert!(
3359 tokio::time::timeout(Duration::from_millis(100), &mut complete).await.is_err(),
3360 "CompleteMultipartUpload must not return while a copy source rename tail is still pending"
3361 );
3362 rename_barrier.release();
3363 complete
3364 .await
3365 .expect("multipart completion task should join")
3366 .expect("multipart completion should return after every rename tail finishes");
3367
3368 let source_reader = store
3369 .get_object_reader(&bucket, source_object, None, HeaderMap::new(), &ObjectOptions::default())
3370 .await
3371 .expect("completed multipart source should be immediately readable after success");
3372 let mut copy_info = source_reader.object_info.clone();
3373 let actual_size = copy_info.get_actual_size().expect("copy source logical size should resolve");
3374 assert_eq!(actual_size, payload.len() as i64);
3375 let copy_reader =
3376 rustfs_rio::HashReader::from_stream(source_reader.stream, actual_size, actual_size, None, None, false)
3377 .expect("copy source hash reader should build");
3378 copy_info.put_object_reader = Some(PutObjReader::new(copy_reader));
3379
3380 store
3381 .copy_object(
3382 &bucket,
3383 source_object,
3384 &bucket,
3385 target_object,
3386 &mut copy_info,
3387 &ObjectOptions::default(),
3388 &ObjectOptions::default(),
3389 )
3390 .await
3391 .expect("CopyObject should accept a freshly completed multipart source");
3392 })
3393 .await;
3394
3395 let mut target_reader = store
3396 .get_object_reader(&bucket, target_object, None, HeaderMap::new(), &ObjectOptions::default())
3397 .await
3398 .expect("copied target should be readable after tail release");
3399 let mut target_body = Vec::new();
3400 target_reader
3401 .stream
3402 .read_to_end(&mut target_body)
3403 .await
3404 .expect("target body should stream");
3405 assert_eq!(target_body, payload);
3406 shutdown.cancel();
3407 }
3408
3409 #[cfg(feature = "test-util")]
3410 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
3411 #[serial_test::serial(storage_class_env)]
3412 async fn early_ack_put_tails_block_scanner_publication_until_all_renames_finish() {
3413 use crate::storage_api_contracts::namespace::NamespaceLocking as _;
3414
3415 let temp_dir = tempfile::tempdir().expect("create scanner PUT tail store dir");
3416 let (ctx, store, shutdown) =
3417 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "scanner-put-tails", &[4])).await;
3418 crate::bucket::metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
3419 let bucket = format!("scanner-put-tails-{}", Uuid::new_v4());
3420 store
3421 .make_bucket(&bucket, &MakeBucketOptions::default())
3422 .await
3423 .expect("create scanner PUT tail bucket");
3424 let set = &store.pools[0].disk_set[0];
3425 let objects = [("scanner-tail-a", vec![0xA1; 273]), ("scanner-tail-b", vec![0xB2; 379])];
3426
3427 temp_env::async_with_vars([(crate::set_disk::ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async {
3428 let (active, blocked, movement_generation) = store.scanner_data_movement_activity().await;
3429 assert!(!active && !blocked);
3430 assert!(ctx.scanner_publication_state_allowed(), "the set admission cache should start allowed");
3431 let (old_lease, _) = store
3432 .acquire_scanner_publication_lease(movement_generation, crate::runtime::instance::SCANNER_PUBLICATION_LEASE_TTL)
3433 .await
3434 .expect("publication lease should be admitted before either PUT starts");
3435
3436 let barriers: Vec<_> = objects
3437 .iter()
3438 .map(|(object, _)| {
3439 crate::set_disk::rename_fanout_barrier::arm(object, 0, crate::set_disk::rename_fanout_barrier::PHASE_RENAME)
3440 })
3441 .collect();
3442 let trackers: Vec<_> = objects
3443 .iter()
3444 .map(|(object, _)| crate::set_disk::rename_fanout_barrier::observe_tasks(object))
3445 .collect();
3446 let puts: Vec<_> = objects
3447 .iter()
3448 .map(|(object, body)| {
3449 let put_store = Arc::clone(&store);
3450 let put_bucket = bucket.clone();
3451 let object = *object;
3452 let body = body.clone();
3453 tokio::spawn(async move {
3454 let mut reader = PutObjReader::from_vec(body);
3455 put_store
3456 .put_object(&put_bucket, object, &mut reader, &ObjectOptions::default())
3457 .await
3458 })
3459 })
3460 .collect();
3461 let committed = tokio::time::timeout(Duration::from_secs(30), async {
3462 for barrier in &barriers {
3463 barrier.wait_until_paused().await;
3464 }
3465 let mut committed = Vec::with_capacity(puts.len());
3466 for put in puts {
3467 committed.push(
3468 put.await
3469 .expect("early-ACK PUT task should join while its tail is paused")
3470 .expect("root PUT should return after quorum without waiting for its tail"),
3471 );
3472 }
3473 committed
3474 })
3475 .await
3476 .expect("both root PUTs must quorum-ACK while their tail disks remain paused");
3477
3478 assert!(trackers.iter().all(|tracker| tracker.running() >= 1));
3479 assert!(ctx.namespace_commits_pending());
3480 assert!(
3481 ctx.scanner_publication_state_allowed(),
3482 "pending PUT tails must not disable scanner namespace walks"
3483 );
3484 let (active, blocked, observed_movement_generation) = store.scanner_data_movement_activity().await;
3485 assert!(!active, "ordinary PUT tails are not decommission or rebalance work");
3486 assert!(!blocked, "ordinary PUT tails must not block the movement-only scan baseline");
3487 assert_eq!(observed_movement_generation, movement_generation);
3488 assert!(store.scanner_data_usage_publication_blocked().await);
3489 assert!(store.scanner_data_usage_publication_admission_guard().await.is_some());
3490 assert!(set.scanner_data_usage_publication_admission_guard().await.is_some());
3491 for error in [
3492 store
3493 .acquire_scanner_publication_lease(
3494 movement_generation,
3495 crate::runtime::instance::SCANNER_PUBLICATION_LEASE_TTL,
3496 )
3497 .await
3498 .expect_err("a new remote publication lease must reject pending PUT tails"),
3499 store
3500 .validate_scanner_publication_lease(old_lease, movement_generation)
3501 .await
3502 .expect_err("an existing remote lease must not bypass pending PUT tails"),
3503 store
3504 .acquire_scanner_publication_lease_guard(old_lease)
3505 .await
3506 .expect_err("target-side publication admission must reject pending PUT tails"),
3507 ] {
3508 assert!(
3509 error.to_string().contains("blocked"),
3510 "publication must fail because of active tails: {error}"
3511 );
3512 }
3513 store.release_scanner_publication_lease(old_lease).await;
3514
3515 for (index, barrier) in barriers.iter().enumerate() {
3516 let commit_generation = ctx.namespace_commit_generation();
3517 let namespace_generation = store.scanner_namespace_mutation_generation();
3518 barrier.release();
3519 tokio::time::timeout(Duration::from_secs(30), async {
3520 while trackers[index].running() != 0 || ctx.namespace_commit_generation() <= commit_generation {
3521 tokio::task::yield_now().await;
3522 }
3523 if index + 1 == barriers.len() {
3524 while ctx.namespace_commits_pending() {
3525 tokio::task::yield_now().await;
3526 }
3527 }
3528 })
3529 .await
3530 .expect("released tail must drain and publish its terminal namespace generation");
3531 assert!(store.scanner_namespace_mutation_generation() > namespace_generation);
3532 let pending = index + 1 < barriers.len();
3533 assert_eq!(ctx.namespace_commits_pending(), pending);
3534 assert_eq!(store.scanner_data_usage_publication_blocked().await, pending);
3535 assert!(!store.scanner_data_movement_activity().await.1);
3536 assert!(store.scanner_data_usage_publication_admission_guard().await.is_some());
3537 assert!(set.scanner_data_usage_publication_admission_guard().await.is_some());
3538 }
3539
3540 let (lease, generation) = store
3541 .acquire_scanner_publication_lease(movement_generation, crate::runtime::instance::SCANNER_PUBLICATION_LEASE_TTL)
3542 .await
3543 .expect("remote publication lease should resume after both tails drain");
3544 store
3545 .validate_scanner_publication_lease(lease, generation)
3546 .await
3547 .expect("a resumed remote publication lease should validate");
3548 drop(
3549 store
3550 .acquire_scanner_publication_lease_guard(lease)
3551 .await
3552 .expect("target-side publication admission should resume after both tails drain"),
3553 );
3554 assert!(store.release_scanner_publication_lease(lease).await);
3555
3556 let disks = set.disk_inventory().await;
3557 assert_eq!(disks.len(), 4);
3558 for ((object, body), committed) in objects.iter().zip(&committed) {
3559 let logical_size = i64::try_from(body.len()).expect("fixture payload size should fit i64");
3560 let etag = committed.etag.as_ref().expect("root PUT should return a committed ETag");
3561 for (disk_index, disk) in disks.iter().enumerate() {
3562 let file_info = disk
3563 .as_ref()
3564 .expect("every fixture disk should remain online")
3565 .read_version(
3566 "",
3567 &bucket,
3568 object,
3569 "",
3570 &crate::disk::ReadOptions {
3571 read_data: true,
3572 ..Default::default()
3573 },
3574 )
3575 .await
3576 .unwrap_or_else(|err| panic!("disk {disk_index} should publish {object} after its tail finishes: {err}"));
3577 assert_eq!(file_info.size, logical_size);
3578 assert_eq!(file_info.metadata.get(http::header::ETAG.as_str()), Some(etag));
3579 assert!(
3580 file_info.inline_data(),
3581 "small fixture payloads should have an inline shard on every disk"
3582 );
3583 let inline_data = file_info.data.as_ref().expect("every disk should retain its inline shard");
3584 let erasure = crate::erasure::coding::Erasure::try_new_with_options(
3585 file_info.erasure.data_blocks,
3586 file_info.erasure.parity_blocks,
3587 file_info.erasure.block_size,
3588 file_info.uses_legacy_checksum,
3589 )
3590 .expect("persisted erasure geometry should be valid");
3591 let shard_size =
3592 usize::try_from(erasure.shard_file_size(logical_size)).expect("fixture shard size should fit usize");
3593 crate::erasure::coding::bitrot_verify(
3594 Cursor::new(inline_data.clone()),
3595 inline_data.len(),
3596 shard_size,
3597 rustfs_utils::HashAlgorithm::HighwayHash256S,
3598 erasure.shard_size(),
3599 )
3600 .await
3601 .unwrap_or_else(|err| panic!("disk {disk_index} should retain a complete valid shard for {object}: {err}"));
3602 }
3603 let mut reader = store
3604 .get_object_reader(&bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
3605 .await
3606 .expect("fully drained PUT should be readable");
3607 let mut actual = Vec::new();
3608 reader.stream.read_to_end(&mut actual).await.expect("PUT body should drain");
3609 assert_eq!(&actual, body);
3610 }
3611
3612 let generation_before_internal_put = ctx.namespace_commit_generation();
3613 let internal_object = "scanner-tail-regression/internal-metadata";
3614 let internal_body = b"scanner metadata must not invalidate its own publication";
3615 let mut internal_reader = PutObjReader::from_vec(internal_body.to_vec());
3616 store
3617 .put_object(RUSTFS_META_BUCKET, internal_object, &mut internal_reader, &ObjectOptions::default())
3618 .await
3619 .expect("internal metadata PUT should commit without scanner self-invalidation");
3620 let internal_lock = set
3621 .new_ns_lock(RUSTFS_META_BUCKET, internal_object)
3622 .await
3623 .expect("internal metadata tail lock should be available");
3624 drop(
3625 internal_lock
3626 .get_write_lock(Duration::from_secs(30))
3627 .await
3628 .expect("internal metadata tail should drain"),
3629 );
3630 assert_eq!(ctx.namespace_commit_generation(), generation_before_internal_put);
3631 assert!(!ctx.namespace_commits_pending());
3632 assert!(store.scanner_data_usage_publication_admission_guard().await.is_some());
3633 assert!(set.scanner_data_usage_publication_admission_guard().await.is_some());
3634 let mut internal_reader = store
3635 .get_object_reader(RUSTFS_META_BUCKET, internal_object, None, HeaderMap::new(), &ObjectOptions::default())
3636 .await
3637 .expect("internal metadata should remain readable");
3638 let mut actual = Vec::new();
3639 internal_reader
3640 .stream
3641 .read_to_end(&mut actual)
3642 .await
3643 .expect("internal metadata body should drain");
3644 assert_eq!(actual, internal_body);
3645 })
3646 .await;
3647 shutdown.cancel();
3648 }
3649
3650 #[cfg(feature = "test-util")]
3651 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
3652 #[serial_test::serial(storage_class_env)]
3653 async fn cancelled_early_ack_put_keeps_scanner_publication_blocked_until_tail_finishes() {
3654 let temp_dir = tempfile::tempdir().expect("create cancelled scanner PUT tail store dir");
3655 let (ctx, store, shutdown) =
3656 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "scanner-cancelled-put-tail", &[4])).await;
3657 crate::bucket::metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
3658 let bucket = format!("scanner-cancelled-put-tail-{}", Uuid::new_v4());
3659 let object = "scanner-cancelled-tail";
3660 let body = vec![0xC3; 273];
3661 store
3662 .make_bucket(&bucket, &MakeBucketOptions::default())
3663 .await
3664 .expect("create cancelled scanner PUT tail bucket");
3665
3666 temp_env::async_with_vars([(crate::set_disk::ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async {
3667 let tracker = crate::set_disk::rename_fanout_barrier::observe_tasks(object);
3668 let tail =
3669 crate::set_disk::rename_fanout_barrier::arm(object, 0, crate::set_disk::rename_fanout_barrier::PHASE_RENAME);
3670 let quorum = crate::set_disk::PutObjectCommitBarrier::install(
3671 &bucket,
3672 object,
3673 crate::set_disk::PutObjectCommitPause::AfterRenameQuorum,
3674 );
3675 let handoff = crate::set_disk::PutObjectCommitBarrier::install(
3676 &bucket,
3677 object,
3678 crate::set_disk::PutObjectCommitPause::AfterRenameHandoff,
3679 );
3680 let put_store = Arc::clone(&store);
3681 let put_bucket = bucket.clone();
3682 let put_body = body.clone();
3683 let put = tokio::spawn(async move {
3684 let mut reader = PutObjReader::from_vec(put_body);
3685 put_store
3686 .put_object(&put_bucket, object, &mut reader, &ObjectOptions::default())
3687 .await
3688 });
3689 tokio::time::timeout(Duration::from_secs(30), tail.wait_until_paused())
3690 .await
3691 .expect("cancelled PUT should pause one disk before rename");
3692 quorum.wait_until_paused().await;
3693 put.abort();
3694 assert!(
3695 put.await
3696 .expect_err("caller should be cancelled after rename quorum")
3697 .is_cancelled()
3698 );
3699 quorum.release();
3700 handoff.wait_until_paused().await;
3701 assert!(tracker.running() >= 1);
3702 assert!(ctx.namespace_commits_pending());
3703 assert!(!store.scanner_data_movement_activity().await.1);
3704 assert!(store.scanner_data_usage_publication_blocked().await);
3705 assert!(store.scanner_data_usage_publication_admission_guard().await.is_some());
3706 assert!(
3707 store.pools[0].disk_set[0]
3708 .scanner_data_usage_publication_admission_guard()
3709 .await
3710 .is_some()
3711 );
3712 let generation = store.scanner_namespace_mutation_generation();
3713
3714 handoff.release();
3715 tail.release();
3716 tokio::time::timeout(Duration::from_secs(30), async {
3717 while tracker.running() != 0 || ctx.namespace_commits_pending() {
3718 tokio::task::yield_now().await;
3719 }
3720 })
3721 .await
3722 .expect("cancelled request's detached fanout must release scanner admission after finishing");
3723 assert!(store.scanner_namespace_mutation_generation() > generation);
3724 assert!(!store.scanner_data_usage_publication_blocked().await);
3725 assert!(store.scanner_data_usage_publication_admission_guard().await.is_some());
3726 for (disk_index, disk) in store.pools[0].disk_set[0].disk_inventory().await.iter().enumerate() {
3727 let file_info = disk
3728 .as_ref()
3729 .expect("cancelled PUT fixture disk should remain online")
3730 .read_version("", &bucket, object, "", &crate::disk::ReadOptions::default())
3731 .await
3732 .unwrap_or_else(|err| panic!("cancelled PUT must still publish on disk {disk_index}: {err}"));
3733 assert_eq!(file_info.size, i64::try_from(body.len()).expect("fixture body size should fit i64"));
3734 }
3735 let mut reader = store
3736 .get_object_reader(&bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
3737 .await
3738 .expect("a cancelled caller must not discard its quorum-committed object");
3739 let mut actual = Vec::new();
3740 reader
3741 .stream
3742 .read_to_end(&mut actual)
3743 .await
3744 .expect("cancelled PUT body should drain");
3745 assert_eq!(actual, body);
3746 })
3747 .await;
3748 shutdown.cancel();
3749 }
3750
3751 #[cfg(feature = "test-util")]
3752 #[test]
3753 #[serial_test::serial(storage_class_env)]
3754 fn data_movement_multipart_part_staging_holds_no_publication_lock_or_tier_lease() {
3755 run_large_stack_async_test(
3756 "multipart-part-staging-publication-fence",
3757 data_movement_multipart_part_staging_holds_no_publication_lock_or_tier_lease_case,
3758 );
3759 }
3760
3761 #[cfg(feature = "test-util")]
3762 async fn data_movement_multipart_part_staging_holds_no_publication_lock_or_tier_lease_case() {
3763 let temp_dir = tempfile::tempdir().expect("create multipart staging-fence store dir");
3764 let (ctx, store, shutdown) =
3765 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "mpu-staging-publication", &[4, 4])).await;
3766 crate::bucket::metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
3767
3768 let bucket = format!("mpu-staging-publication-{}", Uuid::new_v4());
3769 let object = "remote-owned.bin";
3770 let payload = b"multipart part staging must not publish an object".to_vec();
3771 store
3772 .make_bucket(&bucket, &MakeBucketOptions::default())
3773 .await
3774 .expect("multipart staging test bucket should be created");
3775
3776 let tier_name = "MPU-STAGING-TIER";
3777 register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
3778 let lease = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
3779 .await
3780 .expect("mock tier generation should be available");
3781 let backend_identity = lease.backend_identity();
3782 drop(lease);
3783
3784 let mut source_metadata = HashMap::from([(
3785 "x-amz-restore".to_string(),
3786 "ongoing-request=\"false\", expiry-date=\"2099-01-01T00:00:00Z\"".to_string(),
3787 )]);
3788 for (suffix, value) in [
3789 (
3790 rustfs_utils::http::SUFFIX_TRANSITION_STATUS,
3791 crate::bucket::lifecycle::core::TRANSITION_COMPLETE.to_string(),
3792 ),
3793 (rustfs_utils::http::SUFFIX_TRANSITIONED_OBJECTNAME, "remote/mpu.bin".to_string()),
3794 (rustfs_utils::http::SUFFIX_TRANSITION_TIER, tier_name.to_string()),
3795 (rustfs_utils::http::SUFFIX_TRANSITIONED_VERSION_ID, "remote-mpu-version".to_string()),
3796 (
3797 rustfs_utils::http::SUFFIX_TRANSITIONED_VERSION_STATE,
3798 rustfs_filemeta::TransitionVersionState::Exact.as_str().to_string(),
3799 ),
3800 (
3801 rustfs_utils::http::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
3802 rustfs_utils::crypto::hex(backend_identity),
3803 ),
3804 ] {
3805 rustfs_utils::http::metadata_compat::insert_str(&mut source_metadata, suffix, value);
3806 }
3807 let mut source_body = PutObjReader::from_vec(payload.clone());
3808 store.pools[0]
3809 .put_object(
3810 &bucket,
3811 object,
3812 &mut source_body,
3813 &ObjectOptions {
3814 user_defined: source_metadata,
3815 ..Default::default()
3816 },
3817 )
3818 .await
3819 .expect("remote-owned source should be written to the source pool");
3820 let source = store.pools[0]
3821 .get_object_info(&bucket, object, &ObjectOptions::default())
3822 .await
3823 .expect("remote-owned source metadata should be readable");
3824 let publication = store
3825 .acquire_remote_tuple_publication_fence(&bucket, 0, &source, true)
3826 .await
3827 .expect("multipart migration should capture its publication capability");
3828 assert_eq!(
3829 TierConfigMgr::active_operation_lease_count(&ctx.tier_config_mgr(), tier_name).await,
3830 0,
3831 "capturing the commit-late capability must not pin the tier generation"
3832 );
3833
3834 let bucket_incarnation_id = store
3835 .bucket_incarnation_id(&bucket)
3836 .await
3837 .expect("bucket incarnation should resolve");
3838 mark_test_pool_decommissioning(&store, 0).await;
3839 let capacity_owner = test_decommission_capacity_owner(store.as_ref(), 0)
3840 .await
3841 .with_mutation_id(Uuid::new_v4());
3842 let upload_metadata = source.user_defined.as_ref().clone();
3843 let mut staging_opts = ObjectOptions {
3844 data_movement: true,
3845 src_pool_idx: 0,
3846 versioned: source.version_id.is_some(),
3847 version_id: source.version_id.map(|version_id| version_id.to_string()),
3848 mod_time: source.mod_time,
3849 user_defined: upload_metadata,
3850 expected_bucket_incarnation_id: Some(bucket_incarnation_id),
3851 ..Default::default()
3852 };
3853 let upload_identity = crate::data_movement::data_movement_upload_identity_from_options(&staging_opts);
3854 rustfs_utils::http::insert_str(
3855 &mut staging_opts.user_defined,
3856 rustfs_utils::http::SUFFIX_DATA_MOVEMENT_UPLOAD,
3857 upload_identity,
3858 );
3859 capacity_owner.apply_to(&mut staging_opts);
3860 let (upload, target_pool_idx, staged_incarnation_id) = store
3861 .handle_new_multipart_upload_with_pool_idx(&bucket, object, &staging_opts, None)
3862 .await
3863 .expect("target multipart upload should be staged");
3864 assert_eq!(target_pool_idx, 1, "the active pool must receive the staged upload");
3865 assert_eq!(staged_incarnation_id, Some(bucket_incarnation_id));
3866
3867 let barrier = crate::set_disk::MultipartCommitBarrier::install(
3868 &bucket,
3869 object,
3870 crate::set_disk::MultipartCommitPause::PutPartBeforeLockLost,
3871 );
3872 let part_store = Arc::clone(&store);
3873 let part_bucket = bucket.clone();
3874 let upload_id = upload.upload_id.clone();
3875 let part_payload = payload.clone();
3876 let part = tokio::spawn(async move {
3877 let mut reader = PutObjReader::from_vec(part_payload);
3878 let mut part_opts = ObjectOptions {
3879 data_movement: true,
3880 src_pool_idx: 0,
3881 part_number: Some(1),
3882 expected_bucket_incarnation_id: Some(bucket_incarnation_id),
3883 ..Default::default()
3884 };
3885 capacity_owner.apply_to(&mut part_opts);
3886 part_store
3887 .put_object_part_for_data_movement(1, &part_bucket, object, &upload_id, &mut reader, &part_opts)
3888 .await
3889 });
3890 barrier.wait_until_paused().await;
3891
3892 assert_eq!(
3893 TierConfigMgr::active_operation_lease_count(&ctx.tier_config_mgr(), tier_name).await,
3894 0,
3895 "UploadPart staging must not pin the tier generation"
3896 );
3897 let encoded = rustfs_utils::path::encode_dir_object(object);
3898 let mut proof_opts = ObjectOptions::default();
3899 let proof = tokio::time::timeout(
3900 Duration::from_millis(250),
3901 store.acquire_all_physical_object_read_locks("tier_delete_journal_recovery", &bucket, &encoded, &mut proof_opts),
3902 )
3903 .await
3904 .expect("UploadPart staging must not hold any object publication write lock")
3905 .expect("the all-physical recovery proof should acquire every object namespace");
3906 let source_head = store.pools[0]
3907 .get_object_info(&bucket, object, &ObjectOptions::default())
3908 .await
3909 .expect("source HEAD should remain available while the part commit is paused");
3910 assert_eq!(source_head.etag, source.etag);
3911 assert!(
3912 store.pools[1]
3913 .get_object_info(&bucket, object, &ObjectOptions::default())
3914 .await
3915 .is_err(),
3916 "UploadPart staging must not publish target object metadata"
3917 );
3918 drop(proof);
3919
3920 barrier.release();
3921 drop(barrier);
3922 part.await
3923 .expect("UploadPart staging task should join")
3924 .expect("UploadPart staging should commit after the observation releases");
3925 let mut abort_opts = ObjectOptions {
3926 data_movement: true,
3927 src_pool_idx: 0,
3928 versioned: source.version_id.is_some(),
3929 version_id: source.version_id.map(|version_id| version_id.to_string()),
3930 mod_time: source.mod_time,
3931 expected_bucket_incarnation_id: Some(bucket_incarnation_id),
3932 ..Default::default()
3933 };
3934 capacity_owner.apply_to(&mut abort_opts);
3935 store
3936 .abort_multipart_upload_for_data_movement(1, &bucket, object, &upload.upload_id, &abort_opts)
3937 .await
3938 .expect("staged target upload should be removable without publishing");
3939 drop(publication);
3940 assert_eq!(TierConfigMgr::active_operation_lease_count(&ctx.tier_config_mgr(), tier_name).await, 0);
3941 shutdown.cancel();
3942 }
3943
3944 #[cfg(feature = "test-util")]
3945 struct OfflineTestDisks {
3946 disks: Vec<crate::disk::DiskStore>,
3947 }
3948
3949 #[cfg(feature = "test-util")]
3950 impl Drop for OfflineTestDisks {
3951 fn drop(&mut self) {
3952 for disk in &self.disks {
3953 disk.reset_health_for_store_init_retry();
3954 }
3955 }
3956 }
3957
3958 #[cfg(feature = "test-util")]
3959 async fn force_set_disks_offline_for_test(set: &Arc<crate::set_disk::SetDisks>) -> OfflineTestDisks {
3960 let disks = set
3961 .disks
3962 .read()
3963 .await
3964 .iter()
3965 .map(|disk| disk.clone().expect("fault-injection set should start fully online"))
3966 .collect::<Vec<_>>();
3967 for disk in &disks {
3968 disk.close().await.expect("fault injection should stop per-disk monitoring");
3969 disk.force_runtime_state_for_test(crate::disk::health_state::RuntimeDriveHealthState::Offline);
3970 }
3971
3972 set.connect_disks().await;
3976 let current = set.disks.read().await.clone();
3977 assert_eq!(current.len(), disks.len());
3978 for (slot, expected) in current.iter().zip(&disks) {
3979 let disk = slot.as_ref().expect("offline test slot must remain populated");
3980 assert!(Arc::ptr_eq(disk, expected), "endpoint monitor must not replace an offline test handle");
3981 assert_eq!(disk.runtime_state(), crate::disk::health_state::RuntimeDriveHealthState::Offline);
3982 assert!(
3983 disk.is_online().await,
3984 "the valid disk identity must prevent endpoint renewal while the IO health gate remains offline"
3985 );
3986 }
3987 OfflineTestDisks { disks }
3988 }
3989
3990 #[cfg(feature = "test-util")]
3991 async fn force_set_disk_range_offline_for_test(
3992 set: &Arc<crate::set_disk::SetDisks>,
3993 range: std::ops::Range<usize>,
3994 ) -> OfflineTestDisks {
3995 let disks = set
3996 .disks
3997 .read()
3998 .await
3999 .get(range)
4000 .expect("offline test range must fit the set")
4001 .iter()
4002 .map(|disk| disk.clone().expect("fault-injection disk should start online"))
4003 .collect::<Vec<_>>();
4004 for disk in &disks {
4005 disk.close().await.expect("fault injection should stop per-disk monitoring");
4006 disk.force_runtime_state_for_test(crate::disk::health_state::RuntimeDriveHealthState::Offline);
4007 }
4008 set.connect_disks().await;
4009 for disk in &disks {
4010 assert_eq!(disk.runtime_state(), crate::disk::health_state::RuntimeDriveHealthState::Offline);
4011 }
4012 OfflineTestDisks { disks }
4013 }
4014
4015 fn active_rebalance_meta_for_pool(pool_count: usize, active_pool_idx: usize) -> RebalanceMeta {
4016 let now = OffsetDateTime::now_utc();
4017 let mut pool_stats = vec![RebalanceStats::default(); pool_count];
4018 pool_stats[active_pool_idx] = RebalanceStats {
4019 participating: true,
4020 info: RebalanceInfo {
4021 start_time: Some(now),
4022 status: RebalStatus::Started,
4023 ..Default::default()
4024 },
4025 ..Default::default()
4026 };
4027
4028 RebalanceMeta {
4029 id: uuid::Uuid::new_v4().to_string(),
4030 pool_stats,
4031 ..Default::default()
4032 }
4033 }
4034
4035 #[tokio::test]
4036 #[serial_test::serial(storage_class_env)]
4037 async fn quota_object_fence_ignores_an_unrelated_offline_pool() {
4038 let temp_dir = tempfile::tempdir().expect("create quota fence store dir");
4039 let (_ctx, store, shutdown) =
4040 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "quota-object-fence", &[4, 4])).await;
4041 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
4042 let bucket = format!("quota-object-fence-{}", uuid::Uuid::new_v4());
4043 let object = "object.bin";
4044 store
4045 .make_bucket(&bucket, &MakeBucketOptions::default())
4046 .await
4047 .expect("create quota fence bucket");
4048 store.pools[1].disk_set[0].disks.write().await.fill(None);
4049
4050 crate::bucket::quota::reservation::fence_namespace_mutations_for_test(&store, &bucket, object, Some((0, 0)))
4051 .await
4052 .expect("the selected pool fence should ignore an unrelated offline pool");
4053 let err = crate::bucket::quota::reservation::fence_namespace_mutations_for_test(&store, &bucket, object, None)
4054 .await
4055 .expect_err("legacy reservations must conservatively fence every pool");
4056 assert!(matches!(err, StorageError::ErasureWriteQuorum));
4057
4058 shutdown.cancel();
4059 }
4060
4061 async fn migrate_versioned_decommission_test_object(
4062 store: &Arc<crate::store::ECStore>,
4063 bucket: &str,
4064 object: &str,
4065 payload: &[u8],
4066 op_label: &'static str,
4067 ) -> (uuid::Uuid, FileInfoVersions) {
4068 let mut source = PutObjReader::from_vec(payload.to_vec());
4069 let source_info = store.pools[0]
4070 .put_object(
4071 bucket,
4072 object,
4073 &mut source,
4074 &ObjectOptions {
4075 versioned: true,
4076 ..Default::default()
4077 },
4078 )
4079 .await
4080 .expect("write versioned source to the pool being decommissioned");
4081 let source_version = source_info.version_id.expect("versioned source must have a version ID");
4082 let expected_source_versions = store.pools[0]
4083 .get_disks_by_key(object)
4084 .load_file_info_versions_exact(bucket, object)
4085 .await
4086 .expect("source versions should be readable before migration")
4087 .expect("source versions should exist before migration");
4088 {
4089 let mut pool_meta = store.pool_meta.write().await;
4090 pool_meta.pools[0].decommission = Some(PoolDecommissionInfo {
4091 start_time: Some(OffsetDateTime::now_utc()),
4092 ..Default::default()
4093 });
4094 }
4095
4096 let barrier = crate::set_disk::PutObjectCommitBarrier::install(
4097 bucket,
4098 object,
4099 crate::set_disk::PutObjectCommitPause::AfterNamespace,
4100 );
4101 let migration_store = Arc::clone(store);
4102 let migration_bucket = bucket.to_string();
4103 let migration_object = object.to_string();
4104 let migration = tokio::spawn(async move {
4105 let source_reader = migration_store.pools[0]
4106 .get_object_reader(
4107 &migration_bucket,
4108 &migration_object,
4109 None,
4110 HeaderMap::new(),
4111 &ObjectOptions {
4112 versioned: true,
4113 version_id: Some(source_version.to_string()),
4114 no_lock: true,
4115 data_movement: true,
4116 raw_data_movement_read: true,
4117 ..Default::default()
4118 },
4119 )
4120 .await?;
4121 crate::data_movement::migrate_decommission_object(
4122 migration_store,
4123 0,
4124 migration_bucket,
4125 source_reader,
4126 None,
4127 op_label,
4128 None,
4129 )
4130 .await
4131 });
4132 barrier.wait_until_paused().await;
4133 barrier.release();
4134 migration
4135 .await
4136 .expect("versioned decommission migration task should join")
4137 .expect("versioned decommission migration should commit");
4138
4139 (source_version, expected_source_versions)
4140 }
4141
4142 fn set_test_decommission_capacity_override(store: &Arc<crate::store::ECStore>, pool_idx: usize) {
4143 let layout = DecommissionErasureLayout { data: 2, parity: 2 };
4144 let source_physical_bytes = 1024 * 1024 * 1024;
4145 let target_physical_bytes = source_physical_bytes * 8;
4146 let capacity: Vec<DecommissionPoolCapacityInfo> = store
4147 .pools
4148 .iter()
4149 .enumerate()
4150 .map(|(index, _)| {
4151 if index == pool_idx {
4152 DecommissionPoolCapacityInfo::for_test(index, layout, 0, source_physical_bytes, source_physical_bytes)
4153 } else {
4154 DecommissionPoolCapacityInfo::for_test(index, layout, target_physical_bytes, target_physical_bytes, 0)
4155 }
4156 })
4157 .collect();
4158 set_decommission_capacity_info_overrides_for_test(store.id, (0..128).map(|_| capacity.clone()).collect());
4159 }
4160
4161 async fn mark_test_pool_decommissioning(store: &Arc<crate::store::ECStore>, pool_idx: usize) {
4162 set_test_decommission_capacity_override(store, pool_idx);
4163 store
4164 .save_current_pool_meta_for_decommission_start(&[pool_idx], Vec::new())
4165 .await
4166 .expect("test decommission capacity reservation should activate");
4167 }
4168
4169 #[cfg(feature = "test-util")]
4170 async fn mark_test_pool_decommissioning_with_split_targets(store: &Arc<crate::store::ECStore>, pool_idx: usize) {
4171 let layout = DecommissionErasureLayout { data: 2, parity: 2 };
4172 let source_physical_bytes = 1024 * 1024 * 1024;
4173 let capacity = store
4174 .pools
4175 .iter()
4176 .enumerate()
4177 .map(|(index, _)| {
4178 if index == pool_idx {
4179 DecommissionPoolCapacityInfo::for_test(index, layout, 0, source_physical_bytes, source_physical_bytes)
4180 } else {
4181 DecommissionPoolCapacityInfo::for_test(index, layout, source_physical_bytes, source_physical_bytes, 0)
4185 }
4186 })
4187 .collect::<Vec<_>>();
4188 set_decommission_capacity_info_overrides_for_test(store.id, (0..128).map(|_| capacity.clone()).collect());
4189 store
4190 .save_current_pool_meta_for_decommission_start(&[pool_idx], Vec::new())
4191 .await
4192 .expect("split-target decommission capacity reservation should activate");
4193 }
4194
4195 #[cfg(feature = "test-util")]
4196 async fn test_decommission_capacity_owner(
4197 store: &crate::store::ECStore,
4198 source_pool_index: usize,
4199 ) -> crate::core::pools::DecommissionCapacityOwner {
4200 let pool_meta = store.pool_meta.read().await;
4201 let reservation = pool_meta.pools[source_pool_index]
4202 .decommission
4203 .as_ref()
4204 .and_then(|info| info.capacity_reservation.as_ref())
4205 .expect("test decommission capacity reservation should exist");
4206 crate::core::pools::DecommissionCapacityOwner {
4207 source_pool_index,
4208 operation_id: reservation.operation_id,
4209 generation: reservation.generation,
4210 owner_nonce: reservation.owner_nonce,
4211 mutation_id: None,
4212 }
4213 }
4214
4215 const DECOMMISSION_TEST_FAULT_STAGE_DELETE_MARKER: &str = "delete_marker_copy";
4216 const DECOMMISSION_TEST_FAULT_STAGE_MIGRATE_OBJECT: &str = "migrate_object";
4217 #[cfg(feature = "test-util")]
4218 const DECOMMISSION_TEST_FAULT_STAGE_TIERED: &str = "decommission_tiered_object";
4219
4220 fn decommission_retry_fault_hook(
4221 bucket: &str,
4222 object: &str,
4223 faults: Arc<AtomicUsize>,
4224 ) -> crate::core::pools::DecommissionTestFaultDecision {
4225 let target_bucket = bucket.to_string();
4226 let target_object = object.to_string();
4227 Arc::new(move |stage, bucket, object, attempt, succeeded| {
4228 if !succeeded
4229 || attempt >= crate::core::pools::DECOMMISSION_VERSION_COPY_ATTEMPTS
4230 || stage != DECOMMISSION_TEST_FAULT_STAGE_MIGRATE_OBJECT
4231 || bucket != target_bucket
4232 || object != target_object
4233 {
4234 return false;
4235 }
4236
4237 faults
4241 .fetch_update(Ordering::SeqCst, Ordering::SeqCst, |faults| {
4242 (faults < crate::core::pools::DECOMMISSION_VERSION_COPY_ATTEMPTS.saturating_sub(1))
4243 .then_some(faults.saturating_add(1))
4244 })
4245 .is_ok()
4246 })
4247 }
4248
4249 async fn seed_decommission_source(
4250 store: &Arc<crate::store::ECStore>,
4251 bucket: &str,
4252 object: &str,
4253 body: Vec<u8>,
4254 opts: &ObjectOptions,
4255 ) {
4256 let mut reader = PutObjReader::from_vec(body);
4257 store.pools[0]
4258 .put_object(bucket, object, &mut reader, opts)
4259 .await
4260 .expect("seed decommission source object");
4261 }
4262
4263 async fn run_decommission_entry_retry_test(
4264 store: &Arc<crate::store::ECStore>,
4265 rx: CancellationToken,
4266 bucket: &str,
4267 object: &str,
4268 expected_bucket_incarnation_id: Option<uuid::Uuid>,
4269 source_changed_exhaustions: Arc<AtomicUsize>,
4270 ) -> crate::error::Result<()> {
4271 let source_set = store.pools[0].get_disks_by_key(object);
4272 store
4273 .decommission_entry_with_retry_state_for_test(
4274 rx,
4275 0,
4276 MetaCacheEntry {
4277 name: object.to_string(),
4278 ..Default::default()
4279 },
4280 bucket.to_string(),
4281 source_set,
4282 expected_bucket_incarnation_id,
4283 source_changed_exhaustions,
4284 )
4285 .await
4286 }
4287
4288 async fn read_decommission_target_body(
4289 store: &Arc<crate::store::ECStore>,
4290 bucket: &str,
4291 object: &str,
4292 opts: &ObjectOptions,
4293 ) -> Vec<u8> {
4294 let mut reader = store.pools[1]
4295 .get_object_reader(bucket, object, None, HeaderMap::new(), opts)
4296 .await
4297 .expect("read decommission target object");
4298 let mut body = Vec::new();
4299 reader
4300 .stream
4301 .read_to_end(&mut body)
4302 .await
4303 .expect("drain decommission target body");
4304 body
4305 }
4306
4307 async fn assert_decommission_source_absent(
4308 store: &Arc<crate::store::ECStore>,
4309 bucket: &str,
4310 object: &str,
4311 opts: &ObjectOptions,
4312 ) {
4313 let err = store.pools[0]
4314 .get_object_info(bucket, object, opts)
4315 .await
4316 .expect_err("decommission source must be retained until target commit, then removed");
4317 assert!(
4318 matches!(err, StorageError::ObjectNotFound(_, _) | StorageError::VersionNotFound(_, _, _)),
4319 "unexpected decommission source result: {err:?}"
4320 );
4321 }
4322
4323 #[cfg(feature = "test-util")]
4324 async fn seed_transitioned_free_version(
4325 ctx: &Arc<crate::runtime::instance::InstanceContext>,
4326 store: &Arc<crate::store::ECStore>,
4327 bucket: &str,
4328 object: &str,
4329 ) -> (uuid::Uuid, uuid::Uuid) {
4330 let tier_name = format!("DECOMFREE{}", uuid::Uuid::new_v4().simple());
4331 register_mock_tier(&ctx.tier_config_mgr(), &tier_name).await;
4332
4333 let mut reader = PutObjReader::from_vec(b"transitioned source bytes".to_vec());
4334 let source = store.pools[0]
4335 .put_object(
4336 bucket,
4337 object,
4338 &mut reader,
4339 &ObjectOptions {
4340 versioned: true,
4341 ..Default::default()
4342 },
4343 )
4344 .await
4345 .expect("write transitioned decommission source");
4346 let source_version = source.version_id.expect("transitioned source must be versioned");
4347 store.pools[0]
4348 .transition_object(
4349 bucket,
4350 object,
4351 &ObjectOptions {
4352 versioned: true,
4353 version_id: Some(source_version.to_string()),
4354 transition: TransitionOptions {
4355 status: TRANSITION_PENDING.to_string(),
4356 tier: tier_name,
4357 etag: source.etag.clone().expect("transitioned source must have an ETag"),
4358 ..Default::default()
4359 },
4360 mod_time: source.mod_time,
4361 ..Default::default()
4362 },
4363 )
4364 .await
4365 .expect("transition source before decommission");
4366 store.pools[0]
4367 .delete_object(
4368 bucket,
4369 object,
4370 ObjectOptions {
4371 versioned: true,
4372 version_id: Some(source_version.to_string()),
4373 ..Default::default()
4374 },
4375 )
4376 .await
4377 .expect("delete transitioned source version");
4378
4379 let versions = store.pools[0]
4380 .get_disks_by_key(object)
4381 .load_file_info_versions_exact(bucket, object)
4382 .await
4383 .expect("source versions should decode after transition delete")
4384 .expect("source free version should remain after transition delete");
4385 let free_version = versions
4386 .versions
4387 .iter()
4388 .find(|version| version.tier_free_version())
4389 .and_then(|version| version.version_id)
4390 .expect("transition delete should create a free version");
4391 (source_version, free_version)
4392 }
4393
4394 async fn write_decommission_test_multipart_source(
4395 store: &Arc<crate::store::ECStore>,
4396 pool_idx: usize,
4397 bucket: &str,
4398 object: &str,
4399 ) {
4400 let pool = &store.pools[pool_idx];
4401 let upload = pool
4402 .new_multipart_upload(bucket, object, &ObjectOptions::default())
4403 .await
4404 .expect("create decommission multipart source upload");
4405 let first_part = vec![b'm'; 5 * 1024 * 1024];
4406 let second_part = b"decommission multipart tail".to_vec();
4407 let mut completed_parts = Vec::with_capacity(2);
4408 for (part_number, body) in [(1, first_part), (2, second_part)] {
4409 let mut reader = PutObjReader::from_vec(body);
4410 let part = pool
4411 .put_object_part(bucket, object, &upload.upload_id, part_number, &mut reader, &ObjectOptions::default())
4412 .await
4413 .expect("write decommission multipart source part");
4414 completed_parts.push(crate::storage_api_contracts::multipart::CompletePart {
4415 part_num: part.part_num,
4416 etag: part.etag,
4417 ..Default::default()
4418 });
4419 }
4420 pool.clone()
4421 .complete_multipart_upload(bucket, object, &upload.upload_id, completed_parts, &ObjectOptions::default())
4422 .await
4423 .expect("complete decommission multipart source object");
4424 }
4425
4426 struct DataMovementSourceSeed {
4427 pool_idx: usize,
4428 version_id: Option<Uuid>,
4429 body: Vec<u8>,
4430 multipart_split: Option<usize>,
4431 mod_time: OffsetDateTime,
4432 user_defined: HashMap<String, String>,
4433 }
4434
4435 async fn seed_data_movement_source_reader(
4436 store: &Arc<crate::store::ECStore>,
4437 bucket: &str,
4438 object: &str,
4439 seed: DataMovementSourceSeed,
4440 ) -> GetObjectReader {
4441 let DataMovementSourceSeed {
4442 pool_idx,
4443 version_id,
4444 body,
4445 multipart_split,
4446 mod_time,
4447 user_defined,
4448 } = seed;
4449 let pool = &store.pools[pool_idx];
4450 let write_opts = ObjectOptions {
4451 versioned: version_id.is_some(),
4452 version_id: version_id.map(|version_id| version_id.to_string()),
4453 mod_time: Some(mod_time),
4454 user_defined,
4455 ..Default::default()
4456 };
4457 if let Some(split) = multipart_split {
4458 assert!(split > 0 && split < body.len(), "multipart source split must create two non-empty parts");
4459 let upload = pool
4460 .new_multipart_upload(bucket, object, &write_opts)
4461 .await
4462 .expect("create physical data-movement source upload");
4463 let mut completed_parts = Vec::with_capacity(2);
4464 for (part_number, bytes) in [(1, &body[..split]), (2, &body[split..])] {
4465 let mut reader = PutObjReader::from_vec(bytes.to_vec());
4466 let part = pool
4467 .put_object_part(bucket, object, &upload.upload_id, part_number, &mut reader, &ObjectOptions::default())
4468 .await
4469 .expect("write physical data-movement source part");
4470 completed_parts.push(crate::storage_api_contracts::multipart::CompletePart {
4471 part_num: part.part_num,
4472 etag: part.etag,
4473 ..Default::default()
4474 });
4475 }
4476 pool.clone()
4477 .complete_multipart_upload(bucket, object, &upload.upload_id, completed_parts, &write_opts)
4478 .await
4479 .expect("complete physical data-movement source object");
4480 } else {
4481 let mut reader = PutObjReader::from_vec(body);
4482 pool.put_object(bucket, object, &mut reader, &write_opts)
4483 .await
4484 .expect("write physical data-movement source object");
4485 }
4486
4487 pool.get_object_reader(
4488 bucket,
4489 object,
4490 None,
4491 HeaderMap::new(),
4492 &ObjectOptions {
4493 versioned: version_id.is_some(),
4494 version_id: version_id.map(|version_id| version_id.to_string()),
4495 raw_data_movement_read: true,
4496 include_part_checksums: true,
4497 ..Default::default()
4498 },
4499 )
4500 .await
4501 .expect("read the physical data-movement source snapshot")
4502 }
4503
4504 async fn assert_pool_object_present(pool: &Arc<crate::core::sets::Sets>, bucket: &str, object: &str) {
4505 pool.get_object_info(bucket, object, &ObjectOptions::default())
4506 .await
4507 .expect("expected object generation must remain present");
4508 }
4509
4510 async fn assert_pool_object_absent(pool: &Arc<crate::core::sets::Sets>, bucket: &str, object: &str) {
4511 let err = pool
4512 .get_object_info(bucket, object, &ObjectOptions::default())
4513 .await
4514 .expect_err("fenced decommission target must remain absent");
4515 assert!(
4516 matches!(err, StorageError::ObjectNotFound(_, _) | StorageError::VersionNotFound(_, _, _)),
4517 "unexpected fenced target result: {err:?}"
4518 );
4519 }
4520
4521 async fn write_suspended_decommission_source(store: &Arc<crate::store::ECStore>, bucket: &str, object: &str) {
4522 let mut reader = PutObjReader::from_vec(b"suspended source generation".to_vec());
4523 let source = store.pools[0]
4524 .put_object(
4525 bucket,
4526 object,
4527 &mut reader,
4528 &ObjectOptions {
4529 version_suspended: true,
4530 mod_time: Some(OffsetDateTime::UNIX_EPOCH + time::Duration::SECOND),
4531 ..Default::default()
4532 },
4533 )
4534 .await
4535 .expect("write suspended null source version");
4536 assert!(
4537 source.version_id.is_none_or(|version_id| version_id.is_nil()),
4538 "suspended source must use the null version identity"
4539 );
4540 }
4541
4542 async fn assert_suspended_null_source_present(store: &Arc<crate::store::ECStore>, bucket: &str, object: &str) {
4543 let versions = store.pools[0]
4544 .get_disks_by_key(object)
4545 .load_file_info_versions_exact(bucket, object)
4546 .await
4547 .expect("suspended source versions should be readable")
4548 .expect("suspended source must exist before worker convergence");
4549 assert_eq!(versions.versions.len(), 1, "DELETE must not add a marker to the retiring source");
4550 let source = &versions.versions[0];
4551 assert!(
4552 !source.deleted && source.version_id.is_none_or(|version_id| version_id.is_nil()),
4553 "the source pool must retain its null data version until worker convergence"
4554 );
4555 assert_eq!(source.mod_time, Some(OffsetDateTime::UNIX_EPOCH + time::Duration::SECOND));
4556
4557 let mut reader = store.pools[0]
4558 .get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
4559 .await
4560 .expect("the retiring source must remain directly readable before worker convergence");
4561 let mut body = Vec::new();
4562 reader
4563 .stream
4564 .read_to_end(&mut body)
4565 .await
4566 .expect("read retained source bytes");
4567 assert_eq!(body, b"suspended source generation");
4568 }
4569
4570 async fn assert_suspended_null_delete_marker_visible(
4571 store: &Arc<crate::store::ECStore>,
4572 bucket: &str,
4573 object: &str,
4574 marker_mod_time: OffsetDateTime,
4575 ) {
4576 let versions = store.pools[1]
4577 .get_disks_by_key(object)
4578 .load_file_info_versions_exact(bucket, object)
4579 .await
4580 .expect("healthy target versions should be readable")
4581 .expect("the healthy target must retain the DELETE marker");
4582 assert_eq!(versions.versions.len(), 1, "the target must contain only the null delete marker");
4583 let marker = &versions.versions[0];
4584 assert!(marker.deleted, "migration must not replace the DELETE marker with source data");
4585 assert!(marker.version_id.is_none_or(|version_id| version_id.is_nil()));
4586 assert_eq!(marker.size, 0);
4587 assert_eq!(marker.mod_time, Some(marker_mod_time), "migration must preserve the marker generation");
4588 assert!(marker_mod_time > OffsetDateTime::UNIX_EPOCH + time::Duration::SECOND);
4589
4590 let head_err = store
4591 .get_object_info(bucket, object, &ObjectOptions::default())
4592 .await
4593 .expect_err("HEAD must observe the DELETE marker instead of the old null source");
4594 assert!(matches!(head_err, Error::ObjectNotFound(_, _)), "unexpected HEAD result: {head_err:?}");
4595 let get_err = match store
4596 .get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
4597 .await
4598 {
4599 Ok(_) => panic!("GET must not resurrect the deleted null source"),
4600 Err(err) => err,
4601 };
4602 assert!(matches!(get_err, Error::ObjectNotFound(_, _)), "unexpected GET result: {get_err:?}");
4603 }
4604
4605 #[tokio::test]
4606 #[serial_test::serial(storage_class_env)]
4607 async fn tag_updates_skip_active_rebalance_source_pool() {
4608 let temp_dir = tempfile::tempdir().expect("create writer-fencing store dir");
4609 let (_ctx, store, shutdown) =
4610 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "writer-fencing-tags", &[4, 4])).await;
4611 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
4612
4613 let bucket = format!("writer-fencing-tags-{}", uuid::Uuid::new_v4());
4614 let object = "tagged-object.bin";
4615 store
4616 .make_bucket(&bucket, &MakeBucketOptions::default())
4617 .await
4618 .expect("create writer fencing bucket");
4619
4620 let old_time = OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("fixed timestamp should be valid");
4621 let newer_time = old_time + time::Duration::seconds(10);
4622 let mut source_reader = PutObjReader::from_vec(b"source-body".to_vec());
4623 store.pools[0]
4624 .put_object(
4625 &bucket,
4626 object,
4627 &mut source_reader,
4628 &ObjectOptions {
4629 mod_time: Some(newer_time),
4630 ..Default::default()
4631 },
4632 )
4633 .await
4634 .expect("write newer source object");
4635 let mut target_reader = PutObjReader::from_vec(b"target-body".to_vec());
4636 store.pools[1]
4637 .put_object(
4638 &bucket,
4639 object,
4640 &mut target_reader,
4641 &ObjectOptions {
4642 mod_time: Some(old_time),
4643 ..Default::default()
4644 },
4645 )
4646 .await
4647 .expect("write older target object");
4648
4649 *store.rebalance_meta.write().await = Some(active_rebalance_meta_for_pool(store.pools.len(), 0));
4650 assert!(store.is_pool_rebalancing(0).await, "pool 0 must be marked as an active rebalance source");
4651
4652 let tags = "rebalance=target";
4653 assert_ne!(
4654 store.pools[0]
4655 .get_object_tags(&bucket, object, &ObjectOptions::default())
4656 .await
4657 .expect("source object tags should be readable before update"),
4658 tags,
4659 "source object must start without the target tag"
4660 );
4661 assert_ne!(
4662 store.pools[1]
4663 .get_object_tags(&bucket, object, &ObjectOptions::default())
4664 .await
4665 .expect("target object tags should be readable before update"),
4666 tags,
4667 "target object must start without the target tag"
4668 );
4669 let selected_pool = store
4670 .get_pool_idx_existing_with_opts(
4671 &bucket,
4672 object,
4673 &ObjectOptions {
4674 no_lock: true,
4675 metadata_chg: true,
4676 skip_decommissioned: true,
4677 skip_rebalancing: true,
4678 ..Default::default()
4679 },
4680 )
4681 .await
4682 .expect("writer lookup should select an existing non-rebalancing pool");
4683 assert_eq!(selected_pool, 1, "writer lookup must skip active rebalance pool 0");
4684
4685 let updated = store
4686 .put_object_tags(&bucket, object, tags, &ObjectOptions::default())
4687 .await
4688 .expect("tag update should use the non-rebalancing target pool");
4689 assert_eq!(
4690 updated.mod_time,
4691 Some(old_time),
4692 "tag update must return the non-rebalancing pool object rather than the newer active source"
4693 );
4694
4695 let target_tags = store.pools[1]
4696 .get_object_tags(&bucket, object, &ObjectOptions::default())
4697 .await
4698 .expect("target object tags should be readable");
4699 assert_eq!(target_tags, tags, "non-rebalancing pool must receive writer tag updates");
4700
4701 shutdown.cancel();
4702 }
4703
4704 #[tokio::test]
4705 #[serial_test::serial(storage_class_env)]
4706 async fn multipart_listing_skips_active_rebalance_source_pool() {
4707 let temp_dir = tempfile::tempdir().expect("create multipart writer-fencing store dir");
4708 let (_ctx, store, shutdown) =
4709 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "writer-fencing-multipart", &[4, 4])).await;
4710 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
4711
4712 let bucket = format!("writer-fencing-multipart-{}", uuid::Uuid::new_v4());
4713 store
4714 .make_bucket(&bucket, &MakeBucketOptions::default())
4715 .await
4716 .expect("create multipart writer fencing bucket");
4717
4718 let incarnation = store.bucket_incarnation_id(&bucket).await.expect("read bucket incarnation");
4719 let lifecycle_guard = store
4720 .acquire_bucket_lifecycle_read_lock(&bucket)
4721 .await
4722 .expect("acquire multipart test lifecycle fence");
4723 let mut upload_opts = ObjectOptions {
4724 expected_bucket_incarnation_id: Some(incarnation),
4725 ..Default::default()
4726 };
4727 upload_opts.add_bucket_lifecycle_lock_guard(&lifecycle_guard);
4728 let source_upload = store.pools[0]
4729 .new_multipart_upload(&bucket, "source-only.bin", &upload_opts)
4730 .await
4731 .expect("create source upload");
4732 let target_upload = store.pools[1]
4733 .new_multipart_upload(&bucket, "target-visible.bin", &upload_opts)
4734 .await
4735 .expect("create target upload");
4736
4737 *store.rebalance_meta.write().await = Some(active_rebalance_meta_for_pool(store.pools.len(), 0));
4738 assert!(store.is_pool_rebalancing(0).await, "pool 0 must be marked as an active rebalance source");
4739
4740 let listed = store
4741 .list_multipart_uploads(&bucket, "", None, None, None, 100)
4742 .await
4743 .expect("list multipart uploads");
4744 let listed_uploads: Vec<(&str, &str)> = listed
4745 .uploads
4746 .iter()
4747 .map(|upload| (upload.object.as_str(), upload.upload_id.as_str()))
4748 .collect();
4749
4750 assert!(
4751 !listed_uploads.contains(&("source-only.bin", source_upload.upload_id.as_str())),
4752 "active source pool upload must be hidden from multipart listing"
4753 );
4754 assert!(
4755 listed_uploads.contains(&("target-visible.bin", target_upload.upload_id.as_str())),
4756 "non-rebalancing pool upload must remain visible"
4757 );
4758
4759 shutdown.cancel();
4760 }
4761
4762 #[tokio::test]
4763 #[serial_test::serial(storage_class_env)]
4764 async fn suspended_decommission_source_multipart_remains_operable_until_drained() {
4765 let temp_dir = tempfile::tempdir().expect("create decommission multipart drain store dir");
4766 let (_ctx, store, shutdown) =
4767 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "decommission-multipart-drain", &[4, 4])).await;
4768 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
4769
4770 let bucket = format!("decommission-mp-drain-{}", uuid::Uuid::new_v4());
4771 let complete_object = "complete.bin";
4772 let abort_object = "abort.bin";
4773 store
4774 .make_bucket(&bucket, &MakeBucketOptions::default())
4775 .await
4776 .expect("create decommission multipart drain bucket");
4777
4778 let incarnation = store.bucket_incarnation_id(&bucket).await.expect("read bucket incarnation");
4779 let lifecycle_guard = store
4780 .acquire_bucket_lifecycle_read_lock(&bucket)
4781 .await
4782 .expect("acquire multipart creation lifecycle fence");
4783 let mut upload_opts = ObjectOptions {
4784 expected_bucket_incarnation_id: Some(incarnation),
4785 ..Default::default()
4786 };
4787 upload_opts.add_bucket_lifecycle_lock_guard(&lifecycle_guard);
4788 let complete_upload = store.pools[0]
4789 .new_multipart_upload(&bucket, complete_object, &upload_opts)
4790 .await
4791 .expect("create source upload to complete");
4792 let abort_upload = store.pools[0]
4793 .new_multipart_upload(&bucket, abort_object, &upload_opts)
4794 .await
4795 .expect("create source upload to abort");
4796 drop(lifecycle_guard);
4797
4798 mark_test_pool_decommissioning(&store, 0).await;
4799 let err = store
4800 .ensure_decommission_multipart_uploads_drained_for_test(0)
4801 .await
4802 .expect_err("an unresolved source multipart upload must block final decommission");
4803 let drain_error = err.to_string();
4804 assert!(
4805 drain_error.contains("still contains multipart upload") && drain_error.contains(&bucket),
4806 "the drain error must identify both the upload path and user bucket: {drain_error}"
4807 );
4808
4809 let listed = store
4810 .list_multipart_uploads(&bucket, "", None, None, None, 100)
4811 .await
4812 .expect("list uploads from suspended decommission source");
4813 assert!(
4814 listed
4815 .uploads
4816 .iter()
4817 .any(|upload| upload.upload_id.as_str() == complete_upload.upload_id.as_str()),
4818 "the upload selected before suspension must remain visible"
4819 );
4820 store
4821 .get_multipart_info(&bucket, complete_object, &complete_upload.upload_id, &ObjectOptions::default())
4822 .await
4823 .expect("read upload metadata from suspended decommission source");
4824
4825 let mut part_reader = PutObjReader::from_vec(b"multipart body".to_vec());
4826 let part = store
4827 .put_object_part(
4828 &bucket,
4829 complete_object,
4830 &complete_upload.upload_id,
4831 1,
4832 &mut part_reader,
4833 &ObjectOptions::default(),
4834 )
4835 .await
4836 .expect("write part to suspended decommission source");
4837 let parts = store
4838 .list_object_parts(&bucket, complete_object, &complete_upload.upload_id, None, 100, &ObjectOptions::default())
4839 .await
4840 .expect("list parts from suspended decommission source");
4841 assert_eq!(parts.parts.len(), 1);
4842 assert_eq!(parts.parts[0].etag.as_deref(), part.etag.as_deref());
4843
4844 store
4845 .clone()
4846 .complete_multipart_upload(
4847 &bucket,
4848 complete_object,
4849 &complete_upload.upload_id,
4850 vec![crate::storage_api_contracts::multipart::CompletePart {
4851 part_num: part.part_num,
4852 etag: part.etag,
4853 ..Default::default()
4854 }],
4855 &ObjectOptions::default(),
4856 )
4857 .await
4858 .expect("complete upload on suspended decommission source");
4859 store
4860 .abort_multipart_upload(&bucket, abort_object, &abort_upload.upload_id, &ObjectOptions::default())
4861 .await
4862 .expect("abort upload on suspended decommission source");
4863
4864 tokio::time::timeout(Duration::from_secs(5), async {
4865 loop {
4866 match store.ensure_decommission_multipart_uploads_drained_for_test(0).await {
4867 Ok(()) => break,
4868 Err(err) if err.to_string().contains("still contains multipart upload") => {
4869 tokio::time::sleep(Duration::from_millis(10)).await;
4870 }
4871 Err(err) => panic!("unexpected final decommission drain error: {err:?}"),
4872 }
4873 }
4874 })
4875 .await
4876 .expect("final decommission gate should open after upload cleanup converges");
4877 assert_pool_object_present(&store.pools[0], &bucket, complete_object).await;
4878
4879 shutdown.cancel();
4880 }
4881
4882 #[tokio::test]
4883 #[serial_test::serial(storage_class_env)]
4884 async fn active_multipart_upload_routes_before_faulted_suspended_source() {
4885 use sha2::Digest;
4886
4887 let temp_dir = tempfile::tempdir().expect("create active-first multipart routing store dir");
4888 let (_ctx, store, shutdown) =
4889 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "active-first-multipart-routing", &[4, 4]))
4890 .await;
4891 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
4892
4893 let bucket = format!("active-first-mp-route-{}", uuid::Uuid::new_v4());
4894 let object = "target-upload.bin";
4895 store
4896 .make_bucket(&bucket, &MakeBucketOptions::default())
4897 .await
4898 .expect("create active-first multipart routing bucket");
4899
4900 let incarnation = store.bucket_incarnation_id(&bucket).await.expect("read bucket incarnation");
4901 let lifecycle_guard = store
4902 .acquire_bucket_lifecycle_read_lock(&bucket)
4903 .await
4904 .expect("acquire multipart creation lifecycle fence");
4905 let mut upload_opts = ObjectOptions {
4906 expected_bucket_incarnation_id: Some(incarnation),
4907 ..Default::default()
4908 };
4909 upload_opts.add_bucket_lifecycle_lock_guard(&lifecycle_guard);
4910 let upload = store.pools[1]
4911 .new_multipart_upload(&bucket, object, &upload_opts)
4912 .await
4913 .expect("create upload in active target pool");
4914 drop(lifecycle_guard);
4915
4916 mark_test_pool_decommissioning(&store, 0).await;
4917 let upload_sha =
4921 hex_simd::encode_to_string(sha2::Sha256::digest(format!("{bucket}/{object}").as_bytes()), hex_simd::AsciiCase::Lower);
4922 let upload_uuid = crate::runtime::sources::upload_uuid_suffix(&upload.upload_id);
4923 for disk_index in 0..4 {
4924 let metadata_path = temp_dir
4925 .path()
4926 .join(format!("pool0/set0/disk{disk_index}"))
4927 .join(crate::disk::RUSTFS_META_MULTIPART_BUCKET)
4928 .join(&upload_sha)
4929 .join(&upload_uuid)
4930 .join(crate::disk::STORAGE_FORMAT_FILE);
4931 tokio::fs::create_dir_all(metadata_path.parent().expect("multipart metadata path should have a parent"))
4932 .await
4933 .expect("create corrupt source upload directory");
4934 tokio::fs::write(metadata_path, b"not-xl-meta")
4935 .await
4936 .expect("inject source upload metadata read failure");
4937 }
4938
4939 let source_result = store.pools[0]
4940 .get_multipart_info(&bucket, object, &upload.upload_id, &ObjectOptions::default())
4941 .await;
4942 let routed_result = store
4943 .get_multipart_info(&bucket, object, &upload.upload_id, &ObjectOptions::default())
4944 .await;
4945
4946 assert!(
4947 source_result
4948 .as_ref()
4949 .is_err_and(|err| !crate::error::is_err_invalid_upload_id(err)),
4950 "the suspended source must expose the injected hard read failure: {source_result:?}"
4951 );
4952 let routed = routed_result.expect("the active target UploadID must be resolved before the faulted suspended source");
4953 assert_eq!(routed.upload_id, upload.upload_id);
4954
4955 shutdown.cancel();
4956 }
4957
4958 #[tokio::test]
4959 #[serial_test::serial(storage_class_env)]
4960 async fn delete_objects_skips_active_rebalance_source_pool() {
4961 let temp_dir = tempfile::tempdir().expect("create batch-delete writer-fencing store dir");
4962 let (_ctx, store, shutdown) =
4963 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "batch-delete-rebalance", &[4, 4])).await;
4964 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
4965
4966 let bucket = format!("batch-delete-rebalance-{}", uuid::Uuid::new_v4());
4967 let object = "delete-me.bin";
4968 store
4969 .make_bucket(&bucket, &MakeBucketOptions::default())
4970 .await
4971 .expect("create batch delete rebalance bucket");
4972
4973 let mut source_reader = PutObjReader::from_vec(b"source-body".to_vec());
4974 store.pools[0]
4975 .put_object(&bucket, object, &mut source_reader, &ObjectOptions::default())
4976 .await
4977 .expect("write object on active source pool");
4978 let mut target_reader = PutObjReader::from_vec(b"target-body".to_vec());
4979 store.pools[1]
4980 .put_object(&bucket, object, &mut target_reader, &ObjectOptions::default())
4981 .await
4982 .expect("write object on non-rebalancing target pool");
4983
4984 let mut pool_stats = vec![RebalanceStats::default(); store.pools.len()];
4985 pool_stats[0] = RebalanceStats {
4986 participating: true,
4987 info: RebalanceInfo {
4988 start_time: Some(OffsetDateTime::now_utc()),
4989 status: RebalStatus::Started,
4990 ..Default::default()
4991 },
4992 ..Default::default()
4993 };
4994 *store.rebalance_meta.write().await = Some(RebalanceMeta {
4995 id: uuid::Uuid::new_v4().to_string(),
4996 pool_stats,
4997 ..Default::default()
4998 });
4999 assert!(store.is_pool_rebalancing(0).await, "pool 0 must be marked as an active rebalance source");
5000
5001 let (deleted, errs) = store
5002 .delete_objects(
5003 &bucket,
5004 vec![crate::storage_api_contracts::object::ObjectToDelete {
5005 object_name: object.to_string(),
5006 ..Default::default()
5007 }],
5008 ObjectOptions::default(),
5009 )
5010 .await;
5011 assert!(matches!(errs.as_slice(), [None]), "batch delete must not fail: {errs:?}");
5012 assert!(
5013 matches!(deleted.as_slice(), [deleted] if deleted.found && deleted.object_name == object),
5014 "batch delete must report the non-rebalancing pool deletion"
5015 );
5016
5017 store.pools[0]
5018 .get_object_info(&bucket, object, &ObjectOptions::default())
5019 .await
5020 .expect("active source pool object must not be deleted by DeleteObjects");
5021 let target_err = store.pools[1]
5022 .get_object_info(&bucket, object, &ObjectOptions::default())
5023 .await
5024 .expect_err("non-rebalancing target pool object must be deleted");
5025 assert!(
5026 matches!(target_err, StorageError::ObjectNotFound(_, _)),
5027 "target pool should report object not found after DeleteObjects, got {target_err:?}"
5028 );
5029
5030 shutdown.cancel();
5031 }
5032
5033 #[test]
5034 #[serial_test::serial(storage_class_env)]
5035 fn data_movement_conflicts_and_multipart_retries_converge_safely() {
5036 std::thread::Builder::new()
5039 .stack_size(16 * 1024 * 1024)
5040 .spawn(|| {
5041 let runtime = tokio::runtime::Builder::new_multi_thread()
5042 .enable_all()
5043 .worker_threads(2)
5044 .build()
5045 .expect("build data movement test runtime");
5046 runtime.block_on(async move {
5047 let temp_dir = tempfile::tempdir().expect("create data movement store dir");
5048 let (_ctx, store, _shutdown) = without_storage_class_env(build_isolated_test_store(
5049 temp_dir.path(),
5050 "data-movement-conflict-convergence",
5051 &[4, 4],
5052 ))
5053 .await;
5054 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
5055
5056 let bucket = format!("data-movement-conflict-{}", uuid::Uuid::new_v4());
5057 store
5058 .make_bucket(&bucket, &MakeBucketOptions::default())
5059 .await
5060 .expect("create data movement bucket");
5061 let source_mod_time = OffsetDateTime::UNIX_EPOCH + time::Duration::SECOND;
5062 let target_mod_time = source_mod_time + time::Duration::SECOND;
5063
5064 let object = "single-object";
5065 let target_body = b"newer client body".to_vec();
5066 let mut target_reader = PutObjReader::from_vec(target_body.clone());
5067 store.pools[1]
5068 .put_object(
5069 &bucket,
5070 object,
5071 &mut target_reader,
5072 &ObjectOptions {
5073 mod_time: Some(target_mod_time),
5074 ..Default::default()
5075 },
5076 )
5077 .await
5078 .expect("write newer single-part target");
5079
5080 let source_body = b"stale migration body".to_vec();
5081 let source_reader = seed_data_movement_source_reader(
5082 &store,
5083 &bucket,
5084 object,
5085 DataMovementSourceSeed {
5086 pool_idx: 0,
5087 version_id: None,
5088 body: source_body,
5089 multipart_split: None,
5090 mod_time: source_mod_time,
5091 user_defined: HashMap::new(),
5092 },
5093 )
5094 .await;
5095 crate::data_movement::migrate_object(
5096 store.clone(),
5097 0,
5098 bucket.clone(),
5099 source_reader,
5100 None,
5101 "test_data_movement",
5102 )
5103 .await
5104 .expect("newer single-part target should converge migration");
5105
5106 let mut reader = store.pools[1]
5107 .get_object_reader(&bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
5108 .await
5109 .expect("read converged single-part target");
5110 let mut body = Vec::new();
5111 reader.stream.read_to_end(&mut body).await.expect("drain single-part target");
5112 assert_eq!(body, target_body);
5113
5114 let multipart_object = "multipart-object";
5115 let multipart_target_body = b"newer multipart client body".to_vec();
5116 let mut multipart_target_reader = PutObjReader::from_vec(multipart_target_body.clone());
5117 store.pools[1]
5118 .put_object(
5119 &bucket,
5120 multipart_object,
5121 &mut multipart_target_reader,
5122 &ObjectOptions {
5123 mod_time: Some(target_mod_time),
5124 ..Default::default()
5125 },
5126 )
5127 .await
5128 .expect("write newer multipart target");
5129
5130 let first_part_size = 5 * 1024 * 1024;
5131 let mut multipart_source_body = vec![b'a'; first_part_size];
5132 multipart_source_body.push(b'b');
5133 let multipart_source_reader = seed_data_movement_source_reader(
5134 &store,
5135 &bucket,
5136 multipart_object,
5137 DataMovementSourceSeed {
5138 pool_idx: 0,
5139 version_id: None,
5140 body: multipart_source_body,
5141 multipart_split: Some(first_part_size),
5142 mod_time: source_mod_time,
5143 user_defined: HashMap::new(),
5144 },
5145 )
5146 .await;
5147 crate::data_movement::migrate_object(
5148 store.clone(),
5149 0,
5150 bucket.clone(),
5151 multipart_source_reader,
5152 None,
5153 "test_data_movement",
5154 )
5155 .await
5156 .expect("newer multipart target should converge migration");
5157
5158 let uploads = store.pools[1]
5159 .list_multipart_uploads(&bucket, multipart_object, None, None, None, 100)
5160 .await
5161 .expect("list target pool multipart uploads");
5162 assert!(uploads.uploads.is_empty(), "superseded migration staging must be aborted");
5163
5164 let mut reader = store.pools[1]
5165 .get_object_reader(&bucket, multipart_object, None, HeaderMap::new(), &ObjectOptions::default())
5166 .await
5167 .expect("read converged multipart target");
5168 let mut body = Vec::new();
5169 reader.stream.read_to_end(&mut body).await.expect("drain multipart target");
5170 assert_eq!(body, multipart_target_body);
5171
5172 let retry_object = "multipart-retry-object";
5173 let retry_object_etag = "0123456789abcdef0123456789abcdef".to_string();
5174 let retry_first_part_size = 5 * 1024 * 1024;
5175 let retry_object_mod_time =
5176 OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("fixed retry timestamp should be valid");
5177 let mut retry_source_body = vec![b'c'; retry_first_part_size];
5178 retry_source_body.push(b'd');
5179 let retry_first_checksum =
5180 Checksum::new_from_data(ChecksumType::CRC32C, &retry_source_body[..retry_first_part_size])
5181 .expect("first source part checksum should be created");
5182 let retry_second_checksum =
5183 Checksum::new_from_data(ChecksumType::CRC32C, &retry_source_body[retry_first_part_size..])
5184 .expect("second source part checksum should be created");
5185 let mut retry_checksum_parts = retry_first_checksum.raw.clone();
5186 retry_checksum_parts.extend_from_slice(&retry_second_checksum.raw);
5187 let mut retry_checksum_type = ChecksumType::CRC32C;
5188 retry_checksum_type
5189 .merge(ChecksumType::MULTIPART)
5190 .merge(ChecksumType::INCLUDES_MULTIPART);
5191 let retry_object_checksum = Checksum::new_from_data(retry_checksum_type, &retry_checksum_parts)
5192 .expect("source multipart checksum should be created");
5193 let retry_object_checksum_bytes = retry_object_checksum.to_bytes(&retry_checksum_parts);
5194 let mut retry_metadata = HashMap::from([("x-amz-meta-retry".to_string(), "stable".to_string())]);
5195 insert_str(
5196 &mut retry_metadata,
5197 SUFFIX_COMPRESSION,
5198 compression_metadata_value(CompressionAlgorithm::default()),
5199 );
5200 let retry_upload = store.pools[0]
5201 .new_multipart_upload(
5202 &bucket,
5203 retry_object,
5204 &ObjectOptions {
5205 user_defined: retry_metadata,
5206 want_checksum: Some(retry_object_checksum.clone()),
5207 ..Default::default()
5208 },
5209 )
5210 .await
5211 .expect("create source multipart upload");
5212 let mut retry_parts = Vec::with_capacity(2);
5213 for (part_number, plaintext, checksum) in [
5214 (1, &retry_source_body[..retry_first_part_size], &retry_first_checksum),
5215 (2, &retry_source_body[retry_first_part_size..], &retry_second_checksum),
5216 ] {
5217 let part_size = i64::try_from(plaintext.len()).expect("source part size should fit i64");
5218 let mut plaintext_reader = rustfs_rio::HashReader::from_stream(
5219 Cursor::new(plaintext.to_vec()),
5220 part_size,
5221 part_size,
5222 None,
5223 None,
5224 false,
5225 )
5226 .expect("create source part plaintext reader");
5227 plaintext_reader
5228 .add_non_trailing_checksum(Some(checksum.clone()), false)
5229 .expect("set source part checksum");
5230 let compressed = WritePlan::new()
5231 .with_compression(CompressionAlgorithm::default())
5232 .apply(plaintext_reader, part_size)
5233 .expect("compress source part");
5234 let mut reader = PutObjReader::new(compressed);
5235 let part = store.pools[0]
5236 .put_object_part(
5237 &bucket,
5238 retry_object,
5239 &retry_upload.upload_id,
5240 part_number,
5241 &mut reader,
5242 &ObjectOptions::default(),
5243 )
5244 .await
5245 .expect("write source multipart part");
5246 retry_parts.push(crate::storage_api_contracts::multipart::CompletePart {
5247 part_num: part.part_num,
5248 etag: part.etag,
5249 checksum_crc32c: Some(checksum.encoded.clone()),
5250 ..Default::default()
5251 });
5252 }
5253 store.pools[0]
5254 .clone()
5255 .complete_multipart_upload(
5256 &bucket,
5257 retry_object,
5258 &retry_upload.upload_id,
5259 retry_parts,
5260 &ObjectOptions {
5261 mod_time: Some(retry_object_mod_time),
5262 want_checksum: Some(retry_object_checksum),
5263 preserve_etag: Some(retry_object_etag.clone()),
5264 ..Default::default()
5265 },
5266 )
5267 .await
5268 .expect("complete source multipart object");
5269
5270 *store.rebalance_meta.write().await = Some(active_rebalance_meta_for_pool(store.pools.len(), 0));
5271 assert!(store.is_pool_rebalancing(0).await, "pool 0 must be marked as an active rebalance source");
5272
5273 let mut retry_reader = store.pools[0]
5274 .get_object_reader(
5275 &bucket,
5276 retry_object,
5277 None,
5278 HeaderMap::new(),
5279 &ObjectOptions {
5280 raw_data_movement_read: true,
5281 ..Default::default()
5282 },
5283 )
5284 .await
5285 .expect("read source multipart object for first migration");
5286 let mut retry_source_info = retry_reader.object_info.clone();
5287 let mut retry_source_parts = retry_source_info.parts.as_ref().clone();
5288 for (part, checksum) in retry_source_parts
5289 .iter_mut()
5290 .zip([&retry_first_checksum, &retry_second_checksum])
5291 {
5292 part.mod_time = Some(OffsetDateTime::UNIX_EPOCH);
5293 part.checksums = Some(HashMap::from([(ChecksumType::CRC32C.to_string(), checksum.encoded.clone())]));
5294 }
5295 retry_source_info.parts = Arc::new(retry_source_parts);
5296 assert_eq!(retry_source_info.etag.as_deref(), Some(retry_object_etag.as_str()));
5297 assert!(retry_source_info.is_multipart());
5298 assert!(retry_source_info.parts.iter().all(|part| part.checksums.is_some()));
5299 assert_eq!(retry_source_info.checksum.as_deref(), Some(retry_object_checksum_bytes.as_ref()));
5300 assert!(
5301 !retry_source_info
5302 .user_defined
5303 .contains_key(rustfs_rio::RUSTFS_MULTIPART_CHECKSUM)
5304 );
5305 assert!(
5306 !retry_source_info
5307 .user_defined
5308 .contains_key(rustfs_rio::RUSTFS_MULTIPART_CHECKSUM_TYPE)
5309 );
5310 retry_reader.object_info = retry_source_info.clone();
5311 temp_env::async_with_vars(
5312 [
5313 (rustfs_config::ENV_DATA_MOVEMENT_PART_CHECKSUMS_WRITE, None::<&str>),
5314 (rustfs_config::ENV_DATA_MOVEMENT_PART_CHECKSUMS_FLEET_CONFIRMED, None::<&str>),
5315 ],
5316 crate::data_movement::migrate_object(
5317 store.clone(),
5318 0,
5319 bucket.clone(),
5320 retry_reader,
5321 None,
5322 "test_data_movement_compatible_retry",
5323 ),
5324 )
5325 .await
5326 .expect("default multipart migration should not require the checksum sidecar capability");
5327
5328 let compatible_target = store.pools[1]
5329 .get_object_info(
5330 &bucket,
5331 retry_object,
5332 &ObjectOptions {
5333 include_part_checksums: true,
5334 ..Default::default()
5335 },
5336 )
5337 .await
5338 .expect("read compatible multipart target metadata");
5339 assert_eq!(compatible_target.checksum, retry_source_info.checksum);
5340 assert!(compatible_target.parts.iter().all(|part| part.checksums.is_none()));
5341 assert_eq!(
5342 rustfs_utils::http::get_consistent_str(
5343 &compatible_target.user_defined,
5344 rustfs_utils::http::SUFFIX_DATA_MOVED
5345 ),
5346 Some("true")
5347 );
5348 assert!(!rustfs_utils::http::contains_key_str(
5349 &compatible_target.user_defined,
5350 rustfs_utils::http::SUFFIX_PART_CHECKSUMS
5351 ));
5352 let mut compatible_reader = store.pools[1]
5353 .get_object_reader(&bucket, retry_object, None, HeaderMap::new(), &ObjectOptions::default())
5354 .await
5355 .expect("read compatible multipart target body");
5356 let mut compatible_body = Vec::new();
5357 compatible_reader
5358 .stream
5359 .read_to_end(&mut compatible_body)
5360 .await
5361 .expect("drain compatible multipart target body");
5362 assert_eq!(compatible_body, retry_source_body);
5363
5364 let mut compatible_retry_reader = store.pools[0]
5365 .get_object_reader(
5366 &bucket,
5367 retry_object,
5368 None,
5369 HeaderMap::new(),
5370 &ObjectOptions {
5371 raw_data_movement_read: true,
5372 ..Default::default()
5373 },
5374 )
5375 .await
5376 .expect("read source multipart object for compatible retry");
5377 compatible_retry_reader.object_info = retry_source_info.clone();
5378 temp_env::async_with_vars(
5379 [
5380 (rustfs_config::ENV_DATA_MOVEMENT_PART_CHECKSUMS_WRITE, Some("true")),
5381 (rustfs_config::ENV_DATA_MOVEMENT_PART_CHECKSUMS_FLEET_CONFIRMED, Some("true")),
5382 ],
5383 crate::data_movement::migrate_object(
5384 store.clone(),
5385 0,
5386 bucket.clone(),
5387 compatible_retry_reader,
5388 None,
5389 "test_data_movement_compatible_retry",
5390 ),
5391 )
5392 .await
5393 .expect("pre-gate multipart target should converge after enabling checksum persistence");
5394 let compatible_target_after_retry = store.pools[1]
5395 .get_object_info(
5396 &bucket,
5397 retry_object,
5398 &ObjectOptions {
5399 include_part_checksums: true,
5400 ..Default::default()
5401 },
5402 )
5403 .await
5404 .expect("read pre-gate target after fleet-confirmed retry");
5405 assert_eq!(compatible_target_after_retry.data_dir, compatible_target.data_dir);
5406 assert_eq!(compatible_target_after_retry.parts, compatible_target.parts);
5407 let compatible_uploads = store.pools[1]
5408 .list_multipart_uploads(&bucket, retry_object, None, None, None, 100)
5409 .await
5410 .expect("list compatible target multipart uploads");
5411 assert!(compatible_uploads.uploads.is_empty(), "compatible retry staging must be aborted");
5412
5413 store.pools[1]
5414 .delete_object(&bucket, retry_object, ObjectOptions::default())
5415 .await
5416 .expect("remove compatible target before fleet-confirmed migration");
5417 let mut retry_reader = store.pools[0]
5418 .get_object_reader(
5419 &bucket,
5420 retry_object,
5421 None,
5422 HeaderMap::new(),
5423 &ObjectOptions {
5424 raw_data_movement_read: true,
5425 ..Default::default()
5426 },
5427 )
5428 .await
5429 .expect("read source multipart object for fleet-confirmed migration");
5430 retry_reader.object_info = retry_source_info.clone();
5431 temp_env::async_with_vars(
5432 [
5433 (rustfs_config::ENV_DATA_MOVEMENT_PART_CHECKSUMS_WRITE, Some("true")),
5434 (rustfs_config::ENV_DATA_MOVEMENT_PART_CHECKSUMS_FLEET_CONFIRMED, Some("true")),
5435 ],
5436 crate::data_movement::migrate_object(
5437 store.clone(),
5438 0,
5439 bucket.clone(),
5440 retry_reader,
5441 None,
5442 "test_data_movement_retry",
5443 ),
5444 )
5445 .await
5446 .expect("first multipart migration should succeed");
5447
5448 let target_before_retry = store.pools[1]
5449 .get_object_info(
5450 &bucket,
5451 retry_object,
5452 &ObjectOptions {
5453 include_part_checksums: true,
5454 ..Default::default()
5455 },
5456 )
5457 .await
5458 .expect("read target metadata before retry");
5459 assert_eq!(target_before_retry.checksum, retry_source_info.checksum);
5460 assert_eq!(
5461 target_before_retry
5462 .parts
5463 .iter()
5464 .map(|part| (&part.number, &part.checksums))
5465 .collect::<Vec<_>>(),
5466 retry_source_info
5467 .parts
5468 .iter()
5469 .map(|part| (&part.number, &part.checksums))
5470 .collect::<Vec<_>>()
5471 );
5472 assert_eq!(retry_source_info.mod_time, target_before_retry.mod_time);
5473 assert!(
5474 retry_source_info.parts.iter().any(|source_part| {
5475 target_before_retry
5476 .parts
5477 .iter()
5478 .find(|target_part| target_part.number == source_part.number)
5479 .is_some_and(|target_part| target_part.mod_time != source_part.mod_time)
5480 }),
5481 "multipart migration must rewrite at least one target part timestamp"
5482 );
5483 let mut retry_reader = store.pools[0]
5484 .get_object_reader(
5485 &bucket,
5486 retry_object,
5487 None,
5488 HeaderMap::new(),
5489 &ObjectOptions {
5490 raw_data_movement_read: true,
5491 ..Default::default()
5492 },
5493 )
5494 .await
5495 .expect("read source multipart object for retry");
5496 retry_reader.object_info = retry_source_info;
5497 temp_env::async_with_vars(
5498 [
5499 (rustfs_config::ENV_DATA_MOVEMENT_PART_CHECKSUMS_WRITE, Some("true")),
5500 (rustfs_config::ENV_DATA_MOVEMENT_PART_CHECKSUMS_FLEET_CONFIRMED, Some("true")),
5501 ],
5502 crate::data_movement::migrate_object(
5503 store.clone(),
5504 0,
5505 bucket.clone(),
5506 retry_reader,
5507 None,
5508 "test_data_movement_retry",
5509 ),
5510 )
5511 .await
5512 .expect("equivalent multipart migration retry should converge");
5513
5514 let target_after_retry = store.pools[1]
5515 .get_object_info(&bucket, retry_object, &ObjectOptions::default())
5516 .await
5517 .expect("read target metadata after retry");
5518 assert_eq!(target_after_retry.data_dir, target_before_retry.data_dir);
5519 assert_eq!(target_after_retry.etag, target_before_retry.etag);
5520 assert_eq!(target_after_retry.checksum, target_before_retry.checksum);
5521 assert_eq!(target_after_retry.mod_time, target_before_retry.mod_time);
5522 assert!(!rustfs_utils::http::contains_key_str(
5523 &target_after_retry.user_defined,
5524 rustfs_utils::http::SUFFIX_PART_CHECKSUMS
5525 ));
5526 assert!(target_after_retry.parts.iter().all(|part| part.checksums.is_none()));
5527 let cached_target_after_retry = store.pools[1]
5528 .get_object_info(&bucket, retry_object, &ObjectOptions::default())
5529 .await
5530 .expect("read target metadata from the ordinary cache path");
5531 assert!(!rustfs_utils::http::contains_key_str(
5532 &cached_target_after_retry.user_defined,
5533 rustfs_utils::http::SUFFIX_PART_CHECKSUMS
5534 ));
5535 assert!(cached_target_after_retry.parts.iter().all(|part| part.checksums.is_none()));
5536 let hydrated_target_after_retry = store.pools[1]
5537 .get_object_info(
5538 &bucket,
5539 retry_object,
5540 &ObjectOptions {
5541 include_part_checksums: true,
5542 ..Default::default()
5543 },
5544 )
5545 .await
5546 .expect("hydrate target metadata after ordinary cache reads");
5547 assert_eq!(hydrated_target_after_retry.user_defined, target_before_retry.user_defined);
5548 assert_eq!(hydrated_target_after_retry.parts, target_before_retry.parts);
5549
5550 let mut target_reader = store.pools[1]
5551 .get_object_reader(&bucket, retry_object, None, HeaderMap::new(), &ObjectOptions::default())
5552 .await
5553 .expect("read target multipart body after retry");
5554 let mut target_body = Vec::new();
5555 target_reader
5556 .stream
5557 .read_to_end(&mut target_body)
5558 .await
5559 .expect("drain target multipart body after retry");
5560 assert_eq!(target_body, retry_source_body);
5561
5562 let uploads = store.pools[1]
5563 .list_multipart_uploads(&bucket, retry_object, None, None, None, 100)
5564 .await
5565 .expect("list retry target multipart uploads");
5566 assert!(uploads.uploads.is_empty(), "retry migration staging must be aborted");
5567
5568 Box::pin(async {
5569 store.pools[0]
5570 .delete_object(&bucket, retry_object, ObjectOptions::default())
5571 .await
5572 .expect("remove first-hop source before second migration");
5573 *store.rebalance_meta.write().await = Some(active_rebalance_meta_for_pool(store.pools.len(), 1));
5574 let second_hop_reader = store.pools[1]
5575 .get_object_reader(
5576 &bucket,
5577 retry_object,
5578 None,
5579 HeaderMap::new(),
5580 &ObjectOptions {
5581 data_movement: true,
5582 raw_data_movement_read: true,
5583 ..Default::default()
5584 },
5585 )
5586 .await
5587 .expect("read first target as the second migration source");
5588 assert!(
5589 second_hop_reader
5590 .object_info
5591 .parts
5592 .iter()
5593 .all(|part| part.checksums.is_some()),
5594 "data-movement source reads must hydrate persisted part checksums"
5595 );
5596 temp_env::async_with_vars(
5597 [
5598 (rustfs_config::ENV_DATA_MOVEMENT_PART_CHECKSUMS_WRITE, Some("true")),
5599 (rustfs_config::ENV_DATA_MOVEMENT_PART_CHECKSUMS_FLEET_CONFIRMED, Some("true")),
5600 ],
5601 crate::data_movement::migrate_object(
5602 store.clone(),
5603 1,
5604 bucket.clone(),
5605 second_hop_reader,
5606 None,
5607 "test_data_movement_second_hop",
5608 ),
5609 )
5610 .await
5611 .expect("second multipart migration should preserve part checksums");
5612
5613 let second_hop_target = store.pools[0]
5614 .get_object_info(
5615 &bucket,
5616 retry_object,
5617 &ObjectOptions {
5618 include_part_checksums: true,
5619 ..Default::default()
5620 },
5621 )
5622 .await
5623 .expect("read second-hop target metadata");
5624 assert_eq!(second_hop_target.checksum, target_before_retry.checksum);
5625 assert_eq!(
5626 second_hop_target
5627 .parts
5628 .iter()
5629 .map(|part| (&part.number, &part.checksums))
5630 .collect::<Vec<_>>(),
5631 target_before_retry
5632 .parts
5633 .iter()
5634 .map(|part| (&part.number, &part.checksums))
5635 .collect::<Vec<_>>()
5636 );
5637 })
5638 .await;
5639 });
5640 })
5641 .expect("spawn data movement test thread")
5642 .join()
5643 .expect("join data movement test thread");
5644 }
5645
5646 #[test]
5647 #[serial_test::serial(storage_class_env)]
5648 fn data_movement_multipart_replaces_only_unlocked_owned_generation() {
5649 std::thread::Builder::new()
5650 .stack_size(16 * 1024 * 1024)
5651 .spawn(|| {
5652 let runtime = tokio::runtime::Builder::new_multi_thread()
5653 .enable_all()
5654 .worker_threads(2)
5655 .build()
5656 .expect("build data movement replacement test runtime");
5657 runtime.block_on(async move {
5658 let temp_dir = tempfile::tempdir().expect("create data movement replacement store dir");
5659 let (_ctx, store, _shutdown) = without_storage_class_env(build_isolated_test_store(
5660 temp_dir.path(),
5661 "data-movement-stale-replacement",
5662 &[4, 4],
5663 ))
5664 .await;
5665 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
5666
5667 let bucket = format!("dm-stale-replacement-{}", uuid::Uuid::new_v4());
5668 store
5669 .make_bucket(
5670 &bucket,
5671 &MakeBucketOptions {
5672 lock_enabled: true,
5673 ..Default::default()
5674 },
5675 )
5676 .await
5677 .expect("create data movement replacement bucket");
5678 *store.rebalance_meta.write().await = Some(active_rebalance_meta_for_pool(store.pools.len(), 0));
5679
5680 let version_id = uuid::Uuid::new_v4();
5681 let old_time = OffsetDateTime::UNIX_EPOCH + time::Duration::SECOND;
5682 let new_time = old_time + time::Duration::SECOND;
5683 let first_part_size = 5 * 1024 * 1024;
5684 let mut old_body = vec![b'a'; first_part_size];
5685 old_body.push(b'b');
5686 let mut new_body = vec![b'c'; first_part_size];
5687 new_body.push(b'd');
5688
5689 let source_reader = |name: &'static str, body: Vec<u8>, mod_time, metadata: HashMap<String, String>| {
5690 seed_data_movement_source_reader(
5691 &store,
5692 &bucket,
5693 name,
5694 DataMovementSourceSeed {
5695 pool_idx: 0,
5696 version_id: Some(version_id),
5697 body,
5698 multipart_split: Some(first_part_size),
5699 mod_time,
5700 user_defined: metadata,
5701 },
5702 )
5703 };
5704
5705 let replaceable = "replaceable.bin";
5706 crate::data_movement::migrate_object(
5707 store.clone(),
5708 0,
5709 bucket.clone(),
5710 source_reader(
5711 replaceable,
5712 old_body.clone(),
5713 old_time,
5714 HashMap::from([("x-amz-meta-generation".to_string(), "old".to_string())]),
5715 )
5716 .await,
5717 None,
5718 "test_stale_target_seed",
5719 )
5720 .await
5721 .expect("seed old migrated target");
5722 let seeded_target = store.pools[1]
5723 .get_object_info(
5724 &bucket,
5725 replaceable,
5726 &ObjectOptions {
5727 versioned: true,
5728 version_id: Some(version_id.to_string()),
5729 ..Default::default()
5730 },
5731 )
5732 .await
5733 .expect("read old migrated target");
5734 let replacement_opts = ObjectOptions {
5735 data_movement: true,
5736 versioned: true,
5737 version_id: Some(version_id.to_string()),
5738 mod_time: Some(new_time),
5739 http_preconditions: Some(crate::data_movement::data_movement_target_precondition()),
5740 ..Default::default()
5741 };
5742 assert!(
5743 crate::data_movement::can_replace_stale_data_movement_target(&seeded_target, &replacement_opts),
5744 "seeded target should be replaceable: {seeded_target:?}"
5745 );
5746 crate::data_movement::migrate_object(
5747 store.clone(),
5748 0,
5749 bucket.clone(),
5750 source_reader(
5751 replaceable,
5752 new_body.clone(),
5753 new_time,
5754 HashMap::from([("x-amz-meta-generation".to_string(), "new".to_string())]),
5755 )
5756 .await,
5757 None,
5758 "test_stale_target_replace",
5759 )
5760 .await
5761 .expect("newer source generation should replace the old migrated target");
5762
5763 let replacement = store.pools[1]
5764 .get_object_info(
5765 &bucket,
5766 replaceable,
5767 &ObjectOptions {
5768 versioned: true,
5769 version_id: Some(version_id.to_string()),
5770 ..Default::default()
5771 },
5772 )
5773 .await
5774 .expect("read replaced target metadata");
5775 assert_eq!(replacement.mod_time, Some(new_time));
5776 assert_eq!(replacement.user_defined.get("x-amz-meta-generation").map(String::as_str), Some("new"));
5777 let mut replacement_reader = store.pools[1]
5778 .get_object_reader(
5779 &bucket,
5780 replaceable,
5781 None,
5782 HeaderMap::new(),
5783 &ObjectOptions {
5784 versioned: true,
5785 version_id: Some(version_id.to_string()),
5786 ..Default::default()
5787 },
5788 )
5789 .await
5790 .expect("read replaced target body");
5791 let mut replacement_body = Vec::new();
5792 replacement_reader
5793 .stream
5794 .read_to_end(&mut replacement_body)
5795 .await
5796 .expect("drain replaced target body");
5797 assert_eq!(replacement_body, new_body);
5798
5799 let client_target = "client-target.bin";
5800 let client_body = b"client-owned exact version".to_vec();
5801 let mut client_reader = PutObjReader::from_vec(client_body.clone());
5802 store.pools[1]
5803 .put_object(
5804 &bucket,
5805 client_target,
5806 &mut client_reader,
5807 &ObjectOptions {
5808 versioned: true,
5809 version_id: Some(version_id.to_string()),
5810 mod_time: Some(old_time),
5811 user_defined: HashMap::from([("x-amz-meta-owner".to_string(), "client".to_string())]),
5812 ..Default::default()
5813 },
5814 )
5815 .await
5816 .expect("seed client-owned exact version");
5817 let err = crate::data_movement::migrate_object(
5818 store.clone(),
5819 0,
5820 bucket.clone(),
5821 source_reader(client_target, new_body.clone(), new_time, HashMap::new()).await,
5822 None,
5823 "test_client_target_reject",
5824 )
5825 .await
5826 .expect_err("data movement must not replace a client-owned exact version");
5827 assert!(err.to_string().contains("complete_multipart_upload"), "unexpected migration error: {err}");
5828 let mut preserved_reader = store.pools[1]
5829 .get_object_reader(
5830 &bucket,
5831 client_target,
5832 None,
5833 HeaderMap::new(),
5834 &ObjectOptions {
5835 versioned: true,
5836 version_id: Some(version_id.to_string()),
5837 ..Default::default()
5838 },
5839 )
5840 .await
5841 .expect("read preserved client target");
5842 let mut preserved_body = Vec::new();
5843 preserved_reader
5844 .stream
5845 .read_to_end(&mut preserved_body)
5846 .await
5847 .expect("drain preserved client target");
5848 assert_eq!(preserved_body, client_body);
5849
5850 let acknowledged_target = "acknowledged-target.bin";
5851 crate::data_movement::migrate_object(
5852 store.clone(),
5853 0,
5854 bucket.clone(),
5855 source_reader(
5856 acknowledged_target,
5857 old_body.clone(),
5858 old_time,
5859 HashMap::from([("x-amz-meta-generation".to_string(), "old".to_string())]),
5860 )
5861 .await,
5862 None,
5863 "test_acknowledged_target_seed",
5864 )
5865 .await
5866 .expect("seed migrated target before tag acknowledgement");
5867 let target_version_opts = ObjectOptions {
5868 versioned: true,
5869 version_id: Some(version_id.to_string()),
5870 ..Default::default()
5871 };
5872 store
5873 .put_object_tags(&bucket, acknowledged_target, "acknowledged=true", &target_version_opts)
5874 .await
5875 .expect("tag update should acknowledge the migrated target");
5876 let acknowledged = store.pools[1]
5877 .get_object_info(&bucket, acknowledged_target, &target_version_opts)
5878 .await
5879 .expect("read acknowledged migrated target");
5880 let rustfs_data_moved_key = rustfs_utils::http::internal_key_rustfs(rustfs_utils::http::SUFFIX_DATA_MOVED);
5881 let minio_data_moved_key =
5882 format!("{}{}", rustfs_utils::http::MINIO_INTERNAL_PREFIX, rustfs_utils::http::SUFFIX_DATA_MOVED);
5883 assert_eq!(acknowledged.user_defined.get(&rustfs_data_moved_key).map(String::as_str), Some(""));
5884 assert_eq!(acknowledged.user_defined.get(&minio_data_moved_key).map(String::as_str), Some(""));
5885 assert_eq!(
5886 rustfs_utils::http::get_consistent_str(&acknowledged.user_defined, rustfs_utils::http::SUFFIX_DATA_MOVED),
5887 None
5888 );
5889
5890 let err = crate::data_movement::migrate_object(
5891 store.clone(),
5892 0,
5893 bucket.clone(),
5894 source_reader(
5895 acknowledged_target,
5896 new_body.clone(),
5897 new_time,
5898 HashMap::from([("x-amz-meta-generation".to_string(), "new".to_string())]),
5899 )
5900 .await,
5901 None,
5902 "test_acknowledged_target_reject",
5903 )
5904 .await
5905 .expect_err("data movement must not replace a target acknowledged by a tag update");
5906 assert!(err.to_string().contains("complete_multipart_upload"), "unexpected migration error: {err}");
5907
5908 let acknowledged_tags = store.pools[1]
5909 .get_object_tags(&bucket, acknowledged_target, &target_version_opts)
5910 .await
5911 .expect("read preserved acknowledged target tags");
5912 assert_eq!(acknowledged_tags, "acknowledged=true");
5913 let mut acknowledged_reader = store.pools[1]
5914 .get_object_reader(&bucket, acknowledged_target, None, HeaderMap::new(), &target_version_opts)
5915 .await
5916 .expect("read preserved acknowledged target body");
5917 let mut acknowledged_body = Vec::new();
5918 acknowledged_reader
5919 .stream
5920 .read_to_end(&mut acknowledged_body)
5921 .await
5922 .expect("drain preserved acknowledged target body");
5923 assert_eq!(acknowledged_body, old_body);
5924
5925 let metadata_acknowledged_target = "metadata-acknowledged-target.bin";
5926 crate::data_movement::migrate_object(
5927 store.clone(),
5928 0,
5929 bucket.clone(),
5930 source_reader(
5931 metadata_acknowledged_target,
5932 old_body.clone(),
5933 old_time,
5934 HashMap::from([("x-amz-meta-generation".to_string(), "old".to_string())]),
5935 )
5936 .await,
5937 None,
5938 "test_metadata_acknowledged_target_seed",
5939 )
5940 .await
5941 .expect("seed migrated target before metadata acknowledgement");
5942 store
5943 .put_object_metadata(
5944 &bucket,
5945 metadata_acknowledged_target,
5946 &ObjectOptions {
5947 eval_metadata: Some(HashMap::from([(
5948 rustfs_filemeta::metadata_keys::OBJECT_LOCK_LEGAL_HOLD.to_string(),
5949 s3s::dto::ObjectLockLegalHoldStatus::OFF.to_string(),
5950 )])),
5951 ..target_version_opts.clone()
5952 },
5953 )
5954 .await
5955 .expect("legal hold metadata update should acknowledge the migrated target");
5956 let metadata_acknowledged = store.pools[1]
5957 .get_object_info(&bucket, metadata_acknowledged_target, &target_version_opts)
5958 .await
5959 .expect("read metadata-acknowledged migrated target");
5960 assert_eq!(
5961 metadata_acknowledged
5962 .user_defined
5963 .get(&rustfs_data_moved_key)
5964 .map(String::as_str),
5965 Some("")
5966 );
5967 assert_eq!(
5968 metadata_acknowledged
5969 .user_defined
5970 .get(&minio_data_moved_key)
5971 .map(String::as_str),
5972 Some("")
5973 );
5974 assert_eq!(
5975 metadata_acknowledged
5976 .user_defined
5977 .get(rustfs_filemeta::metadata_keys::OBJECT_LOCK_LEGAL_HOLD)
5978 .map(String::as_str),
5979 Some("OFF")
5980 );
5981 let err = crate::data_movement::migrate_object(
5982 store.clone(),
5983 0,
5984 bucket.clone(),
5985 source_reader(
5986 metadata_acknowledged_target,
5987 new_body.clone(),
5988 new_time,
5989 HashMap::from([("x-amz-meta-generation".to_string(), "new".to_string())]),
5990 )
5991 .await,
5992 None,
5993 "test_metadata_acknowledged_target_reject",
5994 )
5995 .await
5996 .expect_err("data movement must not replace a target acknowledged by a metadata update");
5997 assert!(err.to_string().contains("complete_multipart_upload"), "unexpected migration error: {err}");
5998
5999 let retain_until = (OffsetDateTime::now_utc() + time::Duration::days(1))
6000 .format(&time::format_description::well_known::Rfc3339)
6001 .expect("retain-until date should format");
6002 for (object, mode) in [
6003 ("compliance-target.bin", s3s::dto::ObjectLockRetentionMode::COMPLIANCE),
6004 ("governance-target.bin", s3s::dto::ObjectLockRetentionMode::GOVERNANCE),
6005 ] {
6006 let retained_metadata = HashMap::from([
6007 (rustfs_filemeta::metadata_keys::OBJECT_LOCK_MODE.to_string(), mode.to_string()),
6008 (
6009 rustfs_filemeta::metadata_keys::OBJECT_LOCK_RETAIN_UNTIL_DATE.to_string(),
6010 retain_until.clone(),
6011 ),
6012 ]);
6013 crate::data_movement::migrate_object(
6014 store.clone(),
6015 0,
6016 bucket.clone(),
6017 source_reader(object, old_body.clone(), old_time, retained_metadata.clone()).await,
6018 None,
6019 "test_retained_target_seed",
6020 )
6021 .await
6022 .expect("seed retained migrated target");
6023 let err = crate::data_movement::migrate_object(
6024 store.clone(),
6025 0,
6026 bucket.clone(),
6027 source_reader(object, new_body.clone(), new_time, retained_metadata).await,
6028 None,
6029 "test_retained_target_replace",
6030 )
6031 .await
6032 .expect_err("active retention must block stale target replacement");
6033 assert!(err.to_string().contains("complete_multipart_upload"), "unexpected retention error: {err}");
6034
6035 let mut retained_reader = store.pools[1]
6036 .get_object_reader(
6037 &bucket,
6038 object,
6039 None,
6040 HeaderMap::new(),
6041 &ObjectOptions {
6042 versioned: true,
6043 version_id: Some(version_id.to_string()),
6044 ..Default::default()
6045 },
6046 )
6047 .await
6048 .expect("read retained target");
6049 let mut retained_body = Vec::new();
6050 retained_reader
6051 .stream
6052 .read_to_end(&mut retained_body)
6053 .await
6054 .expect("drain retained target");
6055 assert_eq!(retained_body, old_body);
6056 }
6057
6058 for object in [
6059 replaceable,
6060 client_target,
6061 acknowledged_target,
6062 metadata_acknowledged_target,
6063 "compliance-target.bin",
6064 "governance-target.bin",
6065 ] {
6066 let uploads = store.pools[1]
6067 .list_multipart_uploads(&bucket, object, None, None, None, 100)
6068 .await
6069 .expect("list target multipart staging");
6070 assert!(uploads.uploads.is_empty(), "data movement staging must be cleaned for {object}");
6071 }
6072 });
6073 })
6074 .expect("spawn data movement replacement test thread")
6075 .join()
6076 .expect("join data movement replacement test thread");
6077 }
6078
6079 #[tokio::test]
6080 #[serial_test::serial(storage_class_env)]
6081 async fn data_movement_delete_marker_retries_converge_safely() {
6082 let temp_dir = tempfile::tempdir().expect("create delete-marker data movement store dir");
6083 let (_ctx, store, _shutdown) =
6084 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "data-movement-delete-marker", &[4, 4])).await;
6085 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
6086
6087 let bucket = format!("dm-delete-{}", uuid::Uuid::new_v4());
6088 store
6089 .make_bucket(&bucket, &MakeBucketOptions::default())
6090 .await
6091 .expect("create delete-marker data movement bucket");
6092 let object = "delete-marker-retry";
6093 let version = uuid::Uuid::new_v4();
6094 let mod_time = OffsetDateTime::UNIX_EPOCH + time::Duration::SECOND;
6095 let replica_timestamp = mod_time + time::Duration::SECOND;
6096 let replication_timestamp = mod_time;
6097 let replica_timestamp_string = replica_timestamp
6098 .format(&time::format_description::well_known::Rfc3339)
6099 .expect("replica timestamp should format as RFC3339");
6100 let replication_timestamp_string = replication_timestamp
6101 .format(&time::format_description::well_known::Rfc3339)
6102 .expect("replication timestamp should format as RFC3339");
6103 let replication_status = "arn:minio:replication::TenantA:bucket=COMPLETED;";
6104 store.pools[0]
6105 .delete_object(
6106 &bucket,
6107 object,
6108 ObjectOptions {
6109 versioned: true,
6110 version_id: Some(version.to_string()),
6111 delete_marker: true,
6112 mod_time: Some(mod_time),
6113 delete_replication: Some(ReplicationState {
6114 replica_status: ReplicationStatusType::Replica,
6115 replica_timestamp: Some(replica_timestamp),
6116 replication_status_internal: Some(replication_status.to_string()),
6117 replication_timestamp: Some(replication_timestamp),
6118 targets: replication_statuses_map(replication_status),
6119 delete_marker: true,
6120 ..Default::default()
6121 }),
6122 ..Default::default()
6123 },
6124 )
6125 .await
6126 .expect("write source delete marker");
6127 let source = store.pools[0]
6128 .get_object_info(
6129 &bucket,
6130 object,
6131 &ObjectOptions {
6132 versioned: true,
6133 version_id: Some(version.to_string()),
6134 ..Default::default()
6135 },
6136 )
6137 .await
6138 .expect("read source delete marker replication metadata");
6139 assert_eq!(
6140 rustfs_utils::http::get_str(&source.user_defined, rustfs_utils::http::SUFFIX_REPLICA_STATUS).as_deref(),
6141 Some("REPLICA"),
6142 "source metadata: {:?}",
6143 source.user_defined
6144 );
6145
6146 *store.rebalance_meta.write().await = Some(active_rebalance_meta_for_pool(store.pools.len(), 0));
6147 let missing_object = "missing-delete-marker";
6148 let missing_version = uuid::Uuid::new_v4();
6149 let missing_opts = ObjectOptions {
6150 versioned: true,
6151 version_id: Some(missing_version.to_string()),
6152 delete_marker: true,
6153 mod_time: Some(mod_time),
6154 src_pool_idx: 0,
6155 data_movement: true,
6156 skip_decommissioned: true,
6157 ..Default::default()
6158 };
6159 let missing_source_err = store
6160 .delete_object(&bucket, missing_object, missing_opts.clone())
6161 .await
6162 .expect_err("data movement must not recreate a delete marker missing from the source");
6163 assert!(matches!(missing_source_err, StorageError::DataMovementOverwriteErr(_, _, _)));
6164 let missing_target_err = store.pools[1]
6165 .get_object_info(&bucket, missing_object, &missing_opts)
6166 .await
6167 .expect_err("source-missing preflight must not write a target delete marker");
6168 assert!(matches!(
6169 missing_target_err,
6170 StorageError::ObjectNotFound(_, _) | StorageError::VersionNotFound(_, _, _)
6171 ));
6172
6173 let movement_opts = ObjectOptions {
6174 versioned: true,
6175 version_id: Some(version.to_string()),
6176 delete_marker: true,
6177 mod_time: Some(mod_time),
6178 src_pool_idx: 0,
6179 data_movement: true,
6180 skip_decommissioned: true,
6181 ..Default::default()
6182 };
6183 for attempt in 1..=2 {
6184 let moved = store
6185 .delete_object(&bucket, object, movement_opts.clone())
6186 .await
6187 .unwrap_or_else(|err| panic!("delete marker migration attempt {attempt} should converge: {err}"));
6188 assert!(moved.delete_marker);
6189 assert_eq!(moved.version_id, Some(version));
6190 }
6191 let target = store.pools[1]
6192 .get_object_info(&bucket, object, &movement_opts)
6193 .await
6194 .expect("retry must retain the target delete marker");
6195 assert!(target.delete_marker);
6196 assert_eq!(target.mod_time, Some(mod_time));
6197 assert_eq!(
6198 rustfs_utils::http::get_str(&target.user_defined, rustfs_utils::http::SUFFIX_REPLICA_STATUS).as_deref(),
6199 Some("REPLICA")
6200 );
6201 assert_eq!(
6202 rustfs_utils::http::get_str(&target.user_defined, rustfs_utils::http::SUFFIX_REPLICA_TIMESTAMP).as_deref(),
6203 Some(replica_timestamp_string.as_str())
6204 );
6205 assert_eq!(
6206 rustfs_utils::http::get_str(&target.user_defined, rustfs_utils::http::SUFFIX_REPLICATION_STATUS).as_deref(),
6207 Some(replication_status)
6208 );
6209 assert_eq!(
6210 rustfs_utils::http::get_str(&target.user_defined, rustfs_utils::http::SUFFIX_REPLICATION_TIMESTAMP).as_deref(),
6211 Some(replication_timestamp_string.as_str())
6212 );
6213
6214 let null_object = "null-delete-marker-retry";
6215 let null_mod_time = mod_time + time::Duration::seconds(3);
6216 let null_marker = store.pools[0]
6217 .delete_object(
6218 &bucket,
6219 null_object,
6220 ObjectOptions {
6221 version_suspended: true,
6222 version_id: Some(uuid::Uuid::nil().to_string()),
6223 data_movement: true,
6224 delete_marker: true,
6225 mod_time: Some(null_mod_time),
6226 ..Default::default()
6227 },
6228 )
6229 .await
6230 .expect("write source null delete marker");
6231 assert_eq!(null_marker.version_id, Some(uuid::Uuid::nil()));
6232 let null_opts = ObjectOptions {
6233 version_suspended: true,
6234 version_id: Some(uuid::Uuid::nil().to_string()),
6235 delete_marker: true,
6236 mod_time: Some(null_mod_time),
6237 src_pool_idx: 0,
6238 data_movement: true,
6239 skip_decommissioned: true,
6240 ..Default::default()
6241 };
6242 for attempt in 1..=2 {
6243 let moved = store
6244 .delete_object(&bucket, null_object, null_opts.clone())
6245 .await
6246 .unwrap_or_else(|err| panic!("null delete marker migration attempt {attempt} should converge: {err}"));
6247 assert!(moved.delete_marker);
6248 assert_eq!(moved.version_id, Some(uuid::Uuid::nil()));
6249 }
6250 let target_null_marker = store.pools[1]
6251 .get_object_info(&bucket, null_object, &null_opts)
6252 .await
6253 .expect("target must retain the null delete marker identity");
6254 assert!(target_null_marker.delete_marker);
6255 assert_eq!(target_null_marker.version_id, Some(uuid::Uuid::nil()));
6256
6257 let explicit_object = "suspended-explicit-version-delete";
6258 let explicit_version = uuid::Uuid::new_v4();
6259 let mut explicit_reader = PutObjReader::from_vec(b"explicit version".to_vec());
6260 store.pools[0]
6261 .put_object(
6262 &bucket,
6263 explicit_object,
6264 &mut explicit_reader,
6265 &ObjectOptions {
6266 versioned: true,
6267 version_id: Some(explicit_version.to_string()),
6268 ..Default::default()
6269 },
6270 )
6271 .await
6272 .expect("write explicit version before suspended delete");
6273 store.pools[0]
6274 .delete_object(
6275 &bucket,
6276 explicit_object,
6277 ObjectOptions {
6278 version_suspended: true,
6279 version_id: Some(explicit_version.to_string()),
6280 ..Default::default()
6281 },
6282 )
6283 .await
6284 .expect("delete explicit version from suspended bucket");
6285 store.pools[0]
6286 .get_object_info(
6287 &bucket,
6288 explicit_object,
6289 &ObjectOptions {
6290 versioned: true,
6291 version_id: Some(explicit_version.to_string()),
6292 ..Default::default()
6293 },
6294 )
6295 .await
6296 .expect_err("suspended delete must remove the requested UUID version");
6297 }
6298
6299 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6300 #[serial_test::serial(storage_class_env)]
6301 async fn decommission_entry_carries_migration_and_cleanup_mutation_fences() {
6302 let temp_dir = tempfile::tempdir().expect("create decommission delete-fence store dir");
6303 let (_ctx, store, shutdown) = without_storage_class_env(build_isolated_test_store_with_layout(
6304 temp_dir.path(),
6305 "decommission-delete-fence",
6306 &[(2, 4), (1, 4)],
6307 CancellationToken::new(),
6308 None,
6309 ))
6310 .await;
6311 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
6312
6313 let bucket = format!("decom-delete-fence-{}", uuid::Uuid::new_v4());
6314 let object = (0..128)
6315 .map(|index| format!("object-{index}.bin"))
6316 .find(|candidate| store.pools[0].get_disks_by_key(candidate).set_index == 1)
6317 .expect("the deterministic object search should select source set 1");
6318 store
6319 .make_bucket(&bucket, &MakeBucketOptions::default())
6320 .await
6321 .expect("create decommission delete-fence bucket");
6322 let mut source = PutObjReader::from_vec(b"source generation".to_vec());
6323 store.pools[0]
6324 .put_object(&bucket, &object, &mut source, &ObjectOptions::default())
6325 .await
6326 .expect("write source object to the pool being decommissioned");
6327 mark_test_pool_decommissioning(&store, 0).await;
6328 assert!(store.is_suspended(0).await, "pool 0 must be a suspended decommission source");
6329
6330 let barrier = crate::set_disk::PutObjectCommitBarrier::install(
6331 &bucket,
6332 &object,
6333 crate::set_disk::PutObjectCommitPause::AfterNamespace,
6334 );
6335 let cleanup_barrier = crate::data_movement::SourceCleanupDeleteBarrier::install(&bucket, &object);
6336 let source_set = store.pools[0].get_disks_by_key(&object);
6337 assert_eq!(source_set.set_index, 1, "the source entry must exercise the non-fixed set cleanup lock");
6338 let worker_store = Arc::clone(&store);
6339 let worker_bucket = bucket.clone();
6340 let worker_object = object.clone();
6341 let worker = tokio::spawn(async move {
6342 worker_store
6343 .decommission_entry_for_test(
6344 0,
6345 MetaCacheEntry {
6346 name: worker_object,
6347 ..Default::default()
6348 },
6349 worker_bucket,
6350 source_set,
6351 )
6352 .await
6353 });
6354 barrier.wait_until_paused().await;
6355
6356 let delete_barrier = crate::store::object::DeleteAfterObjectLockSnapshotBarrier::install(&bucket);
6357 let delete_store = Arc::clone(&store);
6358 let delete_bucket = bucket.clone();
6359 let delete_object = object.clone();
6360 let delete = tokio::spawn(async move {
6361 delete_store
6362 .delete_object(&delete_bucket, &delete_object, ObjectOptions::default())
6363 .await
6364 });
6365 delete_barrier.wait_until_paused().await;
6366 delete_barrier.release_and_wait_until_namespace_pending().await;
6367 assert!(
6368 !delete_barrier.namespace_acquired() && !delete.is_finished(),
6369 "DELETE must remain before namespace acquisition behind the decommission worker's target-commit mutation fence"
6370 );
6371
6372 barrier.release();
6373 cleanup_barrier.wait_until_paused().await;
6374 drop(barrier);
6375
6376 let fixed_set = Arc::clone(&store.pools[0].disk_set[0]);
6377 let fixed_mutation_barrier = crate::set_disk::PutObjectCommitBarrier::install(
6378 &bucket,
6379 &object,
6380 crate::set_disk::PutObjectCommitPause::BeforeNamespace,
6381 );
6382 let mutation_bucket = bucket.clone();
6383 let mutation_object = object.clone();
6384 let fixed_mutation = tokio::spawn(async move {
6385 let mut reader = PutObjReader::from_vec(b"fixed-domain replacement".to_vec());
6386 fixed_set
6387 .put_object(&mutation_bucket, &mutation_object, &mut reader, &ObjectOptions::default())
6388 .await
6389 });
6390 fixed_mutation_barrier.wait_until_paused().await;
6391 fixed_mutation_barrier.release_and_wait_until_namespace_pending().await;
6392 assert!(
6393 !fixed_mutation_barrier.namespace_acquired() && !fixed_mutation.is_finished(),
6394 "the source cleanup must retain the fixed mutation fence before the set-0 mutation acquires its namespace"
6395 );
6396 fixed_mutation.abort();
6397 assert!(
6398 fixed_mutation
6399 .await
6400 .expect_err("the fixed-domain mutation should be canceled")
6401 .is_cancelled(),
6402 "the competing fixed-domain mutation must remain cancelable while blocked"
6403 );
6404 drop(fixed_mutation_barrier);
6405
6406 cleanup_barrier.release();
6407 worker
6408 .await
6409 .expect("decommission entry worker should join")
6410 .expect("decommission entry should migrate and clean its source");
6411 delete
6412 .await
6413 .expect("DELETE task should join")
6414 .expect("DELETE should remove the source and migrated target generations");
6415
6416 for pool in &store.pools {
6417 let err = pool
6418 .get_object_info(&bucket, &object, &ObjectOptions::default())
6419 .await
6420 .expect_err("DELETE must remove the source and migrated target copies");
6421 assert!(
6422 matches!(err, StorageError::ObjectNotFound(_, _)),
6423 "unexpected post-delete pool result: {err:?}"
6424 );
6425 }
6426 let err = store
6427 .get_object_info(&bucket, &object, &ObjectOptions::default())
6428 .await
6429 .expect_err("the deleted generation must not become visible again");
6430 assert!(
6431 matches!(err, StorageError::ObjectNotFound(_, _)),
6432 "unexpected post-delete store result: {err:?}"
6433 );
6434
6435 shutdown.cancel();
6436 }
6437
6438 #[test]
6439 fn decommission_retry_fault_budget_counts_successes_across_attempt_changes() {
6440 let cases: &[&[(usize, bool, bool)]] = &[
6441 &[(1, true, true), (2, true, true), (3, true, false)],
6442 &[(1, true, true), (1, true, true), (2, true, false)],
6443 &[(1, true, true), (3, true, false), (3, true, false)],
6444 &[(1, true, true), (2, false, false), (1, true, true), (2, true, false)],
6445 &[(1, true, true), (2, false, false), (3, true, false)],
6446 &[(3, true, false), (4, true, false)],
6447 ];
6448 for case in cases {
6449 let faults = Arc::new(AtomicUsize::new(0));
6450 let hook = decommission_retry_fault_hook("bucket", "object", Arc::clone(&faults));
6451
6452 for (stage, bucket, object, succeeded) in [
6453 ("other-stage", "bucket", "object", true),
6454 (DECOMMISSION_TEST_FAULT_STAGE_MIGRATE_OBJECT, "other-bucket", "object", true),
6455 (DECOMMISSION_TEST_FAULT_STAGE_MIGRATE_OBJECT, "bucket", "other-object", true),
6456 (DECOMMISSION_TEST_FAULT_STAGE_MIGRATE_OBJECT, "bucket", "object", false),
6457 ] {
6458 assert!(!hook(stage, bucket, object, 1, succeeded));
6459 }
6460 assert_eq!(faults.load(Ordering::SeqCst), 0, "unrelated or failed copies must not consume faults");
6461
6462 let mut expected_faults = 0;
6463 for &(attempt, succeeded, expected) in *case {
6464 assert_eq!(
6465 hook(DECOMMISSION_TEST_FAULT_STAGE_MIGRATE_OBJECT, "bucket", "object", attempt, succeeded),
6466 expected,
6467 "fault plan {case:?} at attempt {attempt}"
6468 );
6469 expected_faults += usize::from(expected);
6470 assert_eq!(faults.load(Ordering::SeqCst), expected_faults);
6471 }
6472 }
6473 }
6474
6475 #[test]
6476 #[serial_test::serial(storage_class_env)]
6477 fn decommission_entry_retries_source_changed_without_canceling_other_bucket() {
6478 let handle = std::thread::Builder::new()
6479 .name("decommission_entry_retries_source_changed_without_canceling_other_bucket".to_string())
6480 .stack_size(32 * 1024 * 1024)
6481 .spawn(|| {
6482 let runtime = tokio::runtime::Builder::new_multi_thread()
6483 .enable_all()
6484 .worker_threads(2)
6485 .build()
6486 .expect("test runtime should build");
6487 runtime.block_on(async {
6488 let temp_dir = tempfile::tempdir().expect("create decommission retry store dir");
6489 let (_ctx, store, shutdown) = without_storage_class_env(build_isolated_test_store(
6490 temp_dir.path(),
6491 "decommission-entry-retry",
6492 &[4, 4],
6493 ))
6494 .await;
6495 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
6496
6497 let changed_bucket = format!("decom-retry-a-{}", uuid::Uuid::new_v4());
6498 let other_bucket = format!("decom-retry-b-{}", uuid::Uuid::new_v4());
6499 for bucket in [&changed_bucket, &other_bucket] {
6500 store
6501 .make_bucket(bucket, &MakeBucketOptions::default())
6502 .await
6503 .expect("create decommission retry bucket");
6504 }
6505
6506 let changed_object = "changed.bin";
6507 let first_version = uuid::Uuid::new_v4();
6508 let second_version = uuid::Uuid::new_v4();
6509 let base_time = OffsetDateTime::now_utc();
6510 seed_decommission_source(
6511 &store,
6512 &changed_bucket,
6513 changed_object,
6514 b"first generation".to_vec(),
6515 &ObjectOptions {
6516 versioned: true,
6517 version_id: Some(first_version.to_string()),
6518 mod_time: Some(base_time),
6519 ..Default::default()
6520 },
6521 )
6522 .await;
6523 let other_object = "other.bin";
6524 seed_decommission_source(
6525 &store,
6526 &other_bucket,
6527 other_object,
6528 b"other bucket generation".to_vec(),
6529 &ObjectOptions::default(),
6530 )
6531 .await;
6532 mark_test_pool_decommissioning(&store, 0).await;
6533
6534 let mutation_calls = Arc::new(AtomicUsize::new(0));
6535 let mutation_calls_for_hook = Arc::clone(&mutation_calls);
6536 let mutation_store = Arc::clone(&store);
6537 let mutation_bucket = changed_bucket.clone();
6538 let _mutation_guard = crate::core::pools::DecommissionCleanupMutationGuard::install(Arc::new(
6539 move |bucket, object, attempt| {
6540 let is_target = bucket == mutation_bucket.as_str() && object == changed_object;
6541 let calls = Arc::clone(&mutation_calls_for_hook);
6542 let store = Arc::clone(&mutation_store);
6543 let bucket = mutation_bucket.clone();
6544 Box::pin(async move {
6545 if !is_target {
6546 return;
6547 }
6548 calls.fetch_add(1, Ordering::SeqCst);
6549 if attempt == 1 {
6550 seed_decommission_source(
6551 &store,
6552 &bucket,
6553 changed_object,
6554 b"second generation".to_vec(),
6555 &ObjectOptions {
6556 versioned: true,
6557 version_id: Some(second_version.to_string()),
6558 mod_time: Some(base_time + time::Duration::seconds(1)),
6559 ..Default::default()
6560 },
6561 )
6562 .await;
6563 }
6564 })
6565 },
6566 ));
6567
6568 let ordinary_faults = Arc::new(AtomicUsize::new(0));
6569 let fault_hook = decommission_retry_fault_hook(&other_bucket, other_object, Arc::clone(&ordinary_faults));
6570 let _fault_guard = crate::core::pools::DecommissionTestFaultGuard::install(fault_hook);
6571
6572 let rx = CancellationToken::new();
6573 let source_changed_exhaustions = Arc::new(AtomicUsize::new(0));
6574 let changed_incarnation = Some(
6575 store
6576 .bucket_incarnation_id(&changed_bucket)
6577 .await
6578 .expect("changed bucket incarnation"),
6579 );
6580 let other_incarnation = Some(
6581 store
6582 .bucket_incarnation_id(&other_bucket)
6583 .await
6584 .expect("other bucket incarnation"),
6585 );
6586 let (changed_result, other_result) = tokio::join!(
6587 run_decommission_entry_retry_test(
6588 &store,
6589 rx.clone(),
6590 &changed_bucket,
6591 changed_object,
6592 changed_incarnation,
6593 Arc::clone(&source_changed_exhaustions),
6594 ),
6595 run_decommission_entry_retry_test(
6596 &store,
6597 rx.clone(),
6598 &other_bucket,
6599 other_object,
6600 other_incarnation,
6601 Arc::clone(&source_changed_exhaustions),
6602 )
6603 );
6604 changed_result.expect("SourceChanged entry retry must converge");
6605 other_result.expect("other bucket entry must continue through ordinary copy retries");
6606
6607 assert_eq!(
6608 store.pool_meta.read().await.pools[0]
6609 .decommission
6610 .as_ref()
6611 .expect("decommission progress should remain available")
6612 .items_decommission_failed,
6613 0,
6614 "entry completion must not hide an exhausted copy failure"
6615 );
6616 assert!(!rx.is_cancelled(), "entry-level SourceChanged must not cancel the shared worker token");
6617 assert_eq!(mutation_calls.load(Ordering::SeqCst), 2, "entry must be re-listed after SourceChanged");
6618 assert_eq!(ordinary_faults.load(Ordering::SeqCst), 2, "ordinary copy must consume the retry budget");
6619 assert_eq!(source_changed_exhaustions.load(Ordering::SeqCst), 0);
6620
6621 for (version_id, expected_body) in [
6622 (first_version, b"first generation".as_slice()),
6623 (second_version, b"second generation".as_slice()),
6624 ] {
6625 let opts = ObjectOptions {
6626 versioned: true,
6627 version_id: Some(version_id.to_string()),
6628 ..Default::default()
6629 };
6630 assert_decommission_source_absent(&store, &changed_bucket, changed_object, &opts).await;
6631 assert_eq!(
6632 read_decommission_target_body(&store, &changed_bucket, changed_object, &opts).await,
6633 expected_body
6634 );
6635 }
6636 assert_decommission_source_absent(&store, &other_bucket, other_object, &ObjectOptions::default()).await;
6637 assert_eq!(
6638 read_decommission_target_body(&store, &other_bucket, other_object, &ObjectOptions::default()).await,
6639 b"other bucket generation"
6640 );
6641
6642 shutdown.cancel();
6643 });
6644 })
6645 .expect("spawn decommission retry test thread");
6646 if let Err(payload) = handle.join() {
6647 std::panic::resume_unwind(payload);
6648 }
6649 }
6650
6651 #[test]
6652 #[serial_test::serial(storage_class_env)]
6653 fn decommission_entry_exhausted_source_changed_retains_source_and_records_failure() {
6654 let handle = std::thread::Builder::new()
6655 .name("decommission_entry_exhausted_source_changed_retains_source_and_records_failure".to_string())
6656 .stack_size(32 * 1024 * 1024)
6657 .spawn(|| {
6658 let runtime = tokio::runtime::Builder::new_multi_thread()
6659 .enable_all()
6660 .worker_threads(2)
6661 .build()
6662 .expect("test runtime should build");
6663 runtime.block_on(async {
6664 let temp_dir = tempfile::tempdir().expect("create decommission exhaustion store dir");
6665 let (_ctx, store, shutdown) = without_storage_class_env(build_isolated_test_store(
6666 temp_dir.path(),
6667 "decommission-entry-exhaustion",
6668 &[4, 4],
6669 ))
6670 .await;
6671 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
6672
6673 let bucket = format!("decom-exhausted-{}", uuid::Uuid::new_v4());
6674 let object = "exhausted.bin";
6675 store
6676 .make_bucket(&bucket, &MakeBucketOptions::default())
6677 .await
6678 .expect("create decommission exhaustion bucket");
6679 let original_version = uuid::Uuid::new_v4();
6680 let base_time = OffsetDateTime::now_utc();
6681 seed_decommission_source(
6682 &store,
6683 &bucket,
6684 object,
6685 b"original generation".to_vec(),
6686 &ObjectOptions {
6687 versioned: true,
6688 version_id: Some(original_version.to_string()),
6689 mod_time: Some(base_time),
6690 ..Default::default()
6691 },
6692 )
6693 .await;
6694 mark_test_pool_decommissioning(&store, 0).await;
6695
6696 let mutation_calls = Arc::new(AtomicUsize::new(0));
6697 let mutation_calls_for_hook = Arc::clone(&mutation_calls);
6698 let mutation_store = Arc::clone(&store);
6699 let mutation_bucket = bucket.clone();
6700 let _mutation_guard = crate::core::pools::DecommissionCleanupMutationGuard::install(Arc::new(
6701 move |called_bucket, called_object, _attempt| {
6702 let is_target = called_bucket == mutation_bucket.as_str() && called_object == object;
6703 let calls = Arc::clone(&mutation_calls_for_hook);
6704 let store = Arc::clone(&mutation_store);
6705 let bucket = mutation_bucket.clone();
6706 Box::pin(async move {
6707 if !is_target {
6708 return;
6709 }
6710 let call = calls.fetch_add(1, Ordering::SeqCst) + 1;
6711 let offset = i64::try_from(call).expect("entry retry count should fit i64");
6712 seed_decommission_source(
6713 &store,
6714 &bucket,
6715 object,
6716 format!("concurrent generation {call}").into_bytes(),
6717 &ObjectOptions {
6718 versioned: true,
6719 version_id: Some(uuid::Uuid::new_v4().to_string()),
6720 mod_time: Some(base_time + time::Duration::seconds(offset)),
6721 ..Default::default()
6722 },
6723 )
6724 .await;
6725 })
6726 },
6727 ));
6728
6729 let rx = CancellationToken::new();
6730 let source_changed_exhaustions = Arc::new(AtomicUsize::new(0));
6731 let incarnation = Some(store.bucket_incarnation_id(&bucket).await.expect("bucket incarnation"));
6732 run_decommission_entry_retry_test(
6733 &store,
6734 rx.clone(),
6735 &bucket,
6736 object,
6737 incarnation,
6738 Arc::clone(&source_changed_exhaustions),
6739 )
6740 .await
6741 .expect("entry-level exhaustion must stay local below the pool threshold");
6742
6743 assert!(!rx.is_cancelled(), "one exhausted entry must not cancel other bucket workers");
6744 assert_eq!(mutation_calls.load(Ordering::SeqCst), crate::core::pools::DECOMMISSION_ENTRY_MAX_ATTEMPTS);
6745 assert_eq!(source_changed_exhaustions.load(Ordering::SeqCst), 1);
6746 store.pools[0]
6747 .get_object_info(
6748 &bucket,
6749 object,
6750 &ObjectOptions {
6751 versioned: true,
6752 version_id: Some(original_version.to_string()),
6753 ..Default::default()
6754 },
6755 )
6756 .await
6757 .expect("retry exhaustion must retain the original source version");
6758 let pool_meta = store.pool_meta.read().await;
6759 let info = pool_meta.pools[0]
6760 .decommission
6761 .as_ref()
6762 .expect("decommission progress must be initialized");
6763 assert_eq!(info.items_decommission_failed, 1, "exhausted entry must be visible as failed");
6764 drop(pool_meta);
6765
6766 shutdown.cancel();
6767 });
6768 })
6769 .expect("spawn decommission retry test thread");
6770 if let Err(payload) = handle.join() {
6771 std::panic::resume_unwind(payload);
6772 }
6773 }
6774
6775 #[test]
6776 #[serial_test::serial(storage_class_env)]
6777 fn decommission_entry_delete_marker_copy_retries_real_path() {
6778 let handle = std::thread::Builder::new()
6779 .name("decommission_entry_delete_marker_copy_retries_real_path".to_string())
6780 .stack_size(32 * 1024 * 1024)
6781 .spawn(|| {
6782 let runtime = tokio::runtime::Builder::new_multi_thread()
6783 .enable_all()
6784 .worker_threads(2)
6785 .build()
6786 .expect("test runtime should build");
6787 runtime.block_on(async {
6788 let temp_dir = tempfile::tempdir().expect("create delete marker retry store dir");
6789 let (_ctx, store, shutdown) = without_storage_class_env(build_isolated_test_store(
6790 temp_dir.path(),
6791 "decommission-delete-marker-retry",
6792 &[4, 4],
6793 ))
6794 .await;
6795 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
6796
6797 let bucket = format!("decom-marker-{}", uuid::Uuid::new_v4());
6798 let object = "marker.bin";
6799 store
6800 .make_bucket(&bucket, &MakeBucketOptions::default())
6801 .await
6802 .expect("create delete marker retry bucket");
6803 let data_version = uuid::Uuid::new_v4();
6804 let marker_version = uuid::Uuid::new_v4();
6805 let base_time = OffsetDateTime::now_utc();
6806 seed_decommission_source(
6807 &store,
6808 &bucket,
6809 object,
6810 b"delete marker data".to_vec(),
6811 &ObjectOptions {
6812 versioned: true,
6813 version_id: Some(data_version.to_string()),
6814 mod_time: Some(base_time),
6815 ..Default::default()
6816 },
6817 )
6818 .await;
6819 store.pools[0]
6820 .delete_object(
6821 &bucket,
6822 object,
6823 ObjectOptions {
6824 versioned: true,
6825 version_id: Some(marker_version.to_string()),
6826 delete_marker: true,
6827 mod_time: Some(base_time + time::Duration::seconds(1)),
6828 ..Default::default()
6829 },
6830 )
6831 .await
6832 .expect("seed source delete marker");
6833 mark_test_pool_decommissioning(&store, 0).await;
6834
6835 let fault_calls = Arc::new(AtomicUsize::new(0));
6836 let fault_calls_for_hook = Arc::clone(&fault_calls);
6837 let fault_bucket = bucket.clone();
6838 let _fault_guard = crate::core::pools::DecommissionTestFaultGuard::install(Arc::new(
6839 move |stage, called_bucket, called_object, attempt, succeeded| {
6840 let injected = succeeded
6841 && stage == DECOMMISSION_TEST_FAULT_STAGE_DELETE_MARKER
6842 && called_bucket == fault_bucket.as_str()
6843 && called_object == object
6844 && attempt < crate::core::pools::DECOMMISSION_VERSION_COPY_ATTEMPTS;
6845 if injected {
6846 fault_calls_for_hook.fetch_add(1, Ordering::SeqCst);
6847 }
6848 injected
6849 },
6850 ));
6851
6852 let incarnation = Some(store.bucket_incarnation_id(&bucket).await.expect("bucket incarnation"));
6853 run_decommission_entry_retry_test(
6854 &store,
6855 CancellationToken::new(),
6856 &bucket,
6857 object,
6858 incarnation,
6859 Arc::new(AtomicUsize::new(0)),
6860 )
6861 .await
6862 .expect("delete marker copy retries must converge");
6863
6864 assert_eq!(
6865 fault_calls.load(Ordering::SeqCst),
6866 crate::core::pools::DECOMMISSION_VERSION_COPY_ATTEMPTS - 1
6867 );
6868 let marker_opts = ObjectOptions {
6869 versioned: true,
6870 version_id: Some(marker_version.to_string()),
6871 ..Default::default()
6872 };
6873 let target_marker = store.pools[1]
6874 .get_object_info(&bucket, object, &marker_opts)
6875 .await
6876 .expect("target delete marker must exist");
6877 assert!(target_marker.delete_marker);
6878 assert_decommission_source_absent(&store, &bucket, object, &marker_opts).await;
6879
6880 let data_opts = ObjectOptions {
6881 versioned: true,
6882 version_id: Some(data_version.to_string()),
6883 ..Default::default()
6884 };
6885 assert_eq!(
6886 read_decommission_target_body(&store, &bucket, object, &data_opts).await,
6887 b"delete marker data"
6888 );
6889 assert_decommission_source_absent(&store, &bucket, object, &data_opts).await;
6890
6891 shutdown.cancel();
6892 });
6893 })
6894 .expect("spawn decommission retry test thread");
6895 if let Err(payload) = handle.join() {
6896 std::panic::resume_unwind(payload);
6897 }
6898 }
6899
6900 #[cfg(feature = "test-util")]
6901 #[test]
6902 #[serial_test::serial(storage_class_env)]
6903 fn decommission_entry_tiered_copy_retries_real_path() {
6904 let handle = std::thread::Builder::new()
6905 .name("decommission_entry_tiered_copy_retries_real_path".to_string())
6906 .stack_size(32 * 1024 * 1024)
6907 .spawn(|| {
6908 let runtime = tokio::runtime::Builder::new_multi_thread()
6909 .enable_all()
6910 .worker_threads(2)
6911 .build()
6912 .expect("test runtime should build");
6913 runtime.block_on(async {
6914 let temp_dir = tempfile::tempdir().expect("create tiered retry store dir");
6915 let (ctx, store, shutdown) = without_storage_class_env(build_isolated_test_store(
6916 temp_dir.path(),
6917 "decommission-tiered-retry",
6918 &[4, 4],
6919 ))
6920 .await;
6921 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
6922
6923 let bucket = format!("decom-tiered-{}", uuid::Uuid::new_v4());
6924 let object = "tiered.bin";
6925 store
6926 .make_bucket(&bucket, &MakeBucketOptions::default())
6927 .await
6928 .expect("create tiered retry bucket");
6929 let mut reader = PutObjReader::from_vec(b"tiered generation".to_vec());
6930 let original = store.pools[0]
6931 .put_object(&bucket, object, &mut reader, &ObjectOptions::default())
6932 .await
6933 .expect("seed tiered source object");
6934 let tier_name = format!("DECOM{}", &uuid::Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
6935 register_mock_tier(&ctx.tier_config_mgr(), &tier_name).await;
6936 store.pools[0]
6937 .transition_object(
6938 &bucket,
6939 object,
6940 &ObjectOptions {
6941 transition: TransitionOptions {
6942 status: TRANSITION_PENDING.to_string(),
6943 tier: tier_name,
6944 etag: original.etag.clone().expect("tiered source ETag"),
6945 ..Default::default()
6946 },
6947 version_id: original.version_id.map(|version_id| version_id.to_string()),
6948 mod_time: original.mod_time,
6949 ..Default::default()
6950 },
6951 )
6952 .await
6953 .expect("transition source object to mock tier");
6954 mark_test_pool_decommissioning(&store, 0).await;
6955
6956 let fault_calls = Arc::new(AtomicUsize::new(0));
6957 let fault_calls_for_hook = Arc::clone(&fault_calls);
6958 let fault_bucket = bucket.clone();
6959 let _fault_guard = crate::core::pools::DecommissionTestFaultGuard::install(Arc::new(
6960 move |stage, called_bucket, called_object, attempt, succeeded| {
6961 let injected = succeeded
6962 && stage == DECOMMISSION_TEST_FAULT_STAGE_TIERED
6963 && called_bucket == fault_bucket.as_str()
6964 && called_object == object
6965 && attempt < crate::core::pools::DECOMMISSION_VERSION_COPY_ATTEMPTS;
6966 if injected {
6967 fault_calls_for_hook.fetch_add(1, Ordering::SeqCst);
6968 }
6969 injected
6970 },
6971 ));
6972
6973 let incarnation = Some(store.bucket_incarnation_id(&bucket).await.expect("bucket incarnation"));
6974 run_decommission_entry_retry_test(
6975 &store,
6976 CancellationToken::new(),
6977 &bucket,
6978 object,
6979 incarnation,
6980 Arc::new(AtomicUsize::new(0)),
6981 )
6982 .await
6983 .expect("tiered copy retries must converge");
6984
6985 assert_eq!(
6986 fault_calls.load(Ordering::SeqCst),
6987 crate::core::pools::DECOMMISSION_VERSION_COPY_ATTEMPTS - 1
6988 );
6989 let target = store.pools[1]
6990 .get_object_info(&bucket, object, &ObjectOptions::default())
6991 .await
6992 .expect("tiered target metadata must exist");
6993 assert_eq!(target.transitioned_object.status, rustfs_filemeta::TRANSITION_COMPLETE);
6994 assert_decommission_source_absent(&store, &bucket, object, &ObjectOptions::default()).await;
6995
6996 shutdown.cancel();
6997 });
6998 })
6999 .expect("spawn decommission retry test thread");
7000 if let Err(payload) = handle.join() {
7001 std::panic::resume_unwind(payload);
7002 }
7003 }
7004
7005 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7006 #[serial_test::serial(storage_class_env)]
7007 async fn decommission_outer_fence_loss_blocks_target_put_commit() {
7008 let temp_dir = tempfile::tempdir().expect("create decommission PUT fence-loss store dir");
7009 let (_ctx, store, shutdown) =
7010 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "decommission-put-fence-loss", &[4, 4])).await;
7011 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
7012
7013 let bucket = format!("decom-put-fence-loss-{}", uuid::Uuid::new_v4());
7014 let object = "ordinary.bin";
7015 store
7016 .make_bucket(&bucket, &MakeBucketOptions::default())
7017 .await
7018 .expect("create decommission PUT fence-loss bucket");
7019 let mut source = PutObjReader::from_vec(b"source generation".to_vec());
7020 store.pools[0]
7021 .put_object(&bucket, object, &mut source, &ObjectOptions::default())
7022 .await
7023 .expect("write decommission PUT source");
7024 mark_test_pool_decommissioning(&store, 0).await;
7025
7026 let loss_hook = crate::store::object::DecommissionMutationFenceLossHook::install(
7027 &bucket,
7028 object,
7029 crate::store::object::DecommissionMutationFenceTestPhase::Migration,
7030 );
7031 let barrier = crate::set_disk::PutObjectCommitBarrier::install(
7032 &bucket,
7033 object,
7034 crate::set_disk::PutObjectCommitPause::BeforeQuotaRename,
7035 );
7036 let source_set = store.pools[0].get_disks_by_key(object);
7037 let worker_store = Arc::clone(&store);
7038 let worker_bucket = bucket.clone();
7039 let worker = tokio::spawn(async move {
7040 worker_store
7041 .decommission_entry_for_test(
7042 0,
7043 MetaCacheEntry {
7044 name: object.to_string(),
7045 ..Default::default()
7046 },
7047 worker_bucket,
7048 source_set,
7049 )
7050 .await
7051 });
7052
7053 barrier.wait_until_paused().await;
7054 loss_hook.mark_lost();
7055 barrier.release();
7056 drop(barrier);
7057 worker
7058 .await
7059 .expect("decommission PUT fence-loss worker should join")
7060 .expect("a fenced migration failure should remain retryable at entry scope");
7061
7062 assert_pool_object_absent(&store.pools[1], &bucket, object).await;
7063 assert_pool_object_present(&store.pools[0], &bucket, object).await;
7064 shutdown.cancel();
7065 }
7066
7067 #[test]
7068 #[serial_test::serial(storage_class_env)]
7069 fn decommission_outer_fence_loss_blocks_multipart_commits() {
7070 run_large_stack_async_test(
7071 "decommission-multipart-outer-fence-loss",
7072 decommission_outer_fence_loss_blocks_multipart_commits_case,
7073 );
7074 }
7075
7076 async fn decommission_outer_fence_loss_blocks_multipart_commits_case() {
7077 for (object, pause) in [
7078 ("complete.bin", crate::set_disk::MultipartCommitPause::BeforeLockLost),
7079 ("new-upload.bin", crate::set_disk::MultipartCommitPause::NewUploadBeforeLockLost),
7080 ] {
7081 let temp_dir = tempfile::tempdir().expect("create decommission multipart fence-loss store dir");
7085 let (_ctx, store, shutdown) = without_storage_class_env(build_isolated_test_store(
7086 temp_dir.path(),
7087 "decommission-multipart-fence-loss",
7088 &[4, 4],
7089 ))
7090 .await;
7091 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
7092
7093 let bucket = format!("decom-mpu-fence-loss-{}", uuid::Uuid::new_v4());
7094 store
7095 .make_bucket(&bucket, &MakeBucketOptions::default())
7096 .await
7097 .expect("create decommission multipart fence-loss bucket");
7098 write_decommission_test_multipart_source(&store, 0, &bucket, object).await;
7099 mark_test_pool_decommissioning(&store, 0).await;
7100
7101 let loss_hook = crate::store::object::DecommissionMutationFenceLossHook::install(
7102 &bucket,
7103 object,
7104 crate::store::object::DecommissionMutationFenceTestPhase::Migration,
7105 );
7106 let commit_observation = (pause == crate::set_disk::MultipartCommitPause::NewUploadBeforeLockLost)
7107 .then(|| crate::set_disk::NewMultipartUploadCommitObservation::install(&bucket, object));
7108 let barrier = crate::set_disk::MultipartCommitBarrier::install(&bucket, object, pause);
7109 let source_set = store.pools[0].get_disks_by_key(object);
7110 let recovery_source_set = Arc::clone(&source_set);
7111 let worker_store = Arc::clone(&store);
7112 let worker_bucket = bucket.clone();
7113 let mut worker = tokio::spawn(async move {
7114 worker_store
7115 .decommission_entry_for_test(
7116 0,
7117 MetaCacheEntry {
7118 name: object.to_string(),
7119 ..Default::default()
7120 },
7121 worker_bucket,
7122 source_set,
7123 )
7124 .await
7125 });
7126
7127 tokio::select! {
7128 _ = barrier.wait_until_paused() => {}
7129 result = &mut worker => {
7130 panic!("decommission multipart worker finished before the requested {pause:?} commit barrier: {result:?}");
7131 }
7132 }
7133 loss_hook.mark_lost();
7134 barrier.release();
7135 drop(barrier);
7136 worker
7137 .await
7138 .expect("decommission multipart fence-loss worker should join")
7139 .expect("a fenced multipart migration failure should remain retryable at entry scope");
7140
7141 if let Some(commit_observation) = commit_observation {
7142 assert!(
7143 !commit_observation.committed(),
7144 "NewMultipart must reject a lost outer decommission/capacity fence even though it does not acquire the final remote-tuple publication fence"
7145 );
7146 }
7147 assert_pool_object_absent(&store.pools[1], &bucket, object).await;
7148 assert_pool_object_present(&store.pools[0], &bucket, object).await;
7149 let uploads = store.pools[1]
7150 .list_multipart_uploads(&bucket, object, None, None, None, 100)
7151 .await
7152 .expect("list target multipart uploads after fenced migration");
7153 assert!(uploads.uploads.is_empty(), "fenced multipart migration must not retain target staging");
7154 drop(loss_hook);
7155 store
7156 .decommission_entry_for_test(
7157 0,
7158 MetaCacheEntry {
7159 name: object.to_string(),
7160 ..Default::default()
7161 },
7162 bucket.clone(),
7163 recovery_source_set,
7164 )
7165 .await
7166 .expect("same-mutation retry should recover the durable capacity intent");
7167
7168 shutdown.cancel();
7169 }
7170 }
7171
7172 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7173 #[serial_test::serial(storage_class_env)]
7174 async fn decommission_outer_fence_loss_blocks_source_cleanup_delete_commit() {
7175 let temp_dir = tempfile::tempdir().expect("create decommission cleanup fence-loss store dir");
7176 let (_ctx, store, shutdown) =
7177 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "decommission-cleanup-fence-loss", &[4, 4]))
7178 .await;
7179 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
7180
7181 let bucket = format!("decom-cleanup-fence-loss-{}", uuid::Uuid::new_v4());
7182 let object = "cleanup.bin";
7183 store
7184 .make_bucket(&bucket, &MakeBucketOptions::default())
7185 .await
7186 .expect("create decommission cleanup fence-loss bucket");
7187 let mut source = PutObjReader::from_vec(b"source generation".to_vec());
7188 store.pools[0]
7189 .put_object(&bucket, object, &mut source, &ObjectOptions::default())
7190 .await
7191 .expect("write decommission cleanup source");
7192 mark_test_pool_decommissioning(&store, 0).await;
7193
7194 let loss_hook = crate::store::object::DecommissionMutationFenceLossHook::install(
7195 &bucket,
7196 object,
7197 crate::store::object::DecommissionMutationFenceTestPhase::SourceCleanup,
7198 );
7199 let barrier = crate::data_movement::SourceCleanupDeleteBarrier::install(&bucket, object);
7200 let source_set = store.pools[0].get_disks_by_key(object);
7201 let worker_store = Arc::clone(&store);
7202 let worker_bucket = bucket.clone();
7203 let worker = tokio::spawn(async move {
7204 worker_store
7205 .decommission_entry_for_test(
7206 0,
7207 MetaCacheEntry {
7208 name: object.to_string(),
7209 ..Default::default()
7210 },
7211 worker_bucket,
7212 source_set,
7213 )
7214 .await
7215 });
7216
7217 barrier.wait_until_paused().await;
7218 loss_hook.mark_lost();
7219 barrier.release();
7220 drop(barrier);
7221 let err = worker
7222 .await
7223 .expect("decommission cleanup fence-loss worker should join")
7224 .expect_err("source cleanup must fail after its outer fence is lost");
7225 assert!(
7226 err.to_string().contains("delete_object_commit"),
7227 "cleanup failure must come from the delete commit fence: {err:?}"
7228 );
7229
7230 assert_pool_object_present(&store.pools[0], &bucket, object).await;
7231 assert_pool_object_present(&store.pools[1], &bucket, object).await;
7232 shutdown.cancel();
7233 }
7234
7235 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7236 #[serial_test::serial(storage_class_env)]
7237 async fn reverse_decommission_reuses_fixed_target_fence_for_put_and_multipart() {
7238 let temp_dir = tempfile::tempdir().expect("create reverse decommission store dir");
7239 let (_ctx, store, shutdown) = without_storage_class_env(build_isolated_test_store_with_layout(
7240 temp_dir.path(),
7241 "reverse-decommission-fixed-target",
7242 &[(1, 4), (1, 4)],
7243 CancellationToken::new(),
7244 None,
7245 ))
7246 .await;
7247 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
7248
7249 let bucket = format!("reverse-decom-fixed-target-{}", uuid::Uuid::new_v4());
7250 let object = "ordinary.bin";
7251 let object_body = b"reverse ordinary generation".to_vec();
7252 let self_copy_object = "source-only-self-copy.bin";
7253 let self_copy_body = b"source only copy generation".to_vec();
7254 let multipart_object = "multipart.bin";
7255 let first_part = vec![b'm'; 5 * 1024 * 1024];
7256 let second_part = b"reverse multipart tail".to_vec();
7257 let mut multipart_body = first_part.clone();
7258 multipart_body.extend_from_slice(&second_part);
7259
7260 store
7261 .make_bucket(&bucket, &MakeBucketOptions::default())
7262 .await
7263 .expect("create reverse decommission bucket");
7264 let mut source = PutObjReader::from_vec(object_body.clone());
7265 store.pools[1]
7266 .put_object(&bucket, object, &mut source, &ObjectOptions::default())
7267 .await
7268 .expect("write ordinary source object to pool 1");
7269 let mut self_copy_source = PutObjReader::from_vec(self_copy_body.clone());
7270 store.pools[1]
7271 .put_object(&bucket, self_copy_object, &mut self_copy_source, &ObjectOptions::default())
7272 .await
7273 .expect("write self-copy source object to pool 1");
7274
7275 let upload = store.pools[1]
7276 .new_multipart_upload(&bucket, multipart_object, &ObjectOptions::default())
7277 .await
7278 .expect("create source multipart upload in pool 1");
7279 let mut completed_parts = Vec::with_capacity(2);
7280 for (part_number, bytes) in [(1, first_part.as_slice()), (2, second_part.as_slice())] {
7281 let mut reader = PutObjReader::from_vec(bytes.to_vec());
7282 let part = store.pools[1]
7283 .put_object_part(
7284 &bucket,
7285 multipart_object,
7286 &upload.upload_id,
7287 part_number,
7288 &mut reader,
7289 &ObjectOptions::default(),
7290 )
7291 .await
7292 .expect("write source multipart part");
7293 completed_parts.push(crate::storage_api_contracts::multipart::CompletePart {
7294 part_num: part.part_num,
7295 etag: part.etag,
7296 ..Default::default()
7297 });
7298 }
7299 store.pools[1]
7300 .clone()
7301 .complete_multipart_upload(&bucket, multipart_object, &upload.upload_id, completed_parts, &ObjectOptions::default())
7302 .await
7303 .expect("complete source multipart object in pool 1");
7304
7305 {
7306 let mut pool_meta = store.pool_meta.write().await;
7307 pool_meta.pools[1].decommission = Some(PoolDecommissionInfo {
7308 start_time: Some(OffsetDateTime::now_utc()),
7309 ..Default::default()
7310 });
7311 }
7312 assert!(store.is_suspended(1).await, "pool 1 must be the reverse decommission source");
7313
7314 let self_copy_opts = ObjectOptions {
7315 no_lock: true,
7316 ..Default::default()
7317 };
7318 let mut self_copy_reader = store
7319 .get_object_reader(&bucket, self_copy_object, None, HeaderMap::new(), &self_copy_opts)
7320 .await
7321 .expect("read the suspended source before its target is committed");
7322 let mut self_copy_info = self_copy_reader.object_info.clone();
7323 let mut self_copy_source_body = Vec::new();
7324 self_copy_reader
7325 .stream
7326 .read_to_end(&mut self_copy_source_body)
7327 .await
7328 .expect("drain the suspended self-copy source");
7329 assert_eq!(self_copy_source_body, self_copy_body);
7330 self_copy_info.metadata_only = true;
7331 self_copy_info.put_object_reader = Some(PutObjReader::from_vec(self_copy_source_body));
7332 store
7333 .copy_object(
7334 &bucket,
7335 self_copy_object,
7336 &bucket,
7337 self_copy_object,
7338 &mut self_copy_info,
7339 &self_copy_opts,
7340 &self_copy_opts,
7341 )
7342 .await
7343 .expect("self-copy should read the suspended source and commit to an active pool");
7344 assert_pool_object_present(&store.pools[0], &bucket, self_copy_object).await;
7345 assert_pool_object_present(&store.pools[1], &bucket, self_copy_object).await;
7346
7347 let mut active_copy_reader = store
7348 .get_object_reader(&bucket, self_copy_object, None, HeaderMap::new(), &self_copy_opts)
7349 .await
7350 .expect("read the active self-copy target while the source remains");
7351 let active_copy_data_dir = active_copy_reader.object_info.data_dir;
7352 let mut active_copy_info = active_copy_reader.object_info.clone();
7353 let mut active_copy_body = Vec::new();
7354 active_copy_reader
7355 .stream
7356 .read_to_end(&mut active_copy_body)
7357 .await
7358 .expect("drain the active self-copy target");
7359 assert_eq!(active_copy_body, self_copy_body);
7360 active_copy_info.metadata_only = true;
7361 active_copy_info.put_object_reader = Some(PutObjReader::from_vec(active_copy_body));
7362 let active_copy_result = store
7363 .copy_object(
7364 &bucket,
7365 self_copy_object,
7366 &bucket,
7367 self_copy_object,
7368 &mut active_copy_info,
7369 &self_copy_opts,
7370 &self_copy_opts,
7371 )
7372 .await
7373 .expect("self-copy should keep using the committed active target");
7374 assert_eq!(active_copy_result.data_dir, active_copy_data_dir);
7375
7376 {
7377 let mut pool_meta = store.pool_meta.write().await;
7378 pool_meta.pools[1].decommission = None;
7379 }
7380 mark_test_pool_decommissioning(&store, 1).await;
7381
7382 let cleanup_barrier = crate::data_movement::SourceCleanupDeleteBarrier::install(&bucket, object);
7383 let commit_barrier = crate::set_disk::PutObjectCommitBarrier::install(
7384 &bucket,
7385 object,
7386 crate::set_disk::PutObjectCommitPause::AfterNamespace,
7387 );
7388 let source_set = store.pools[1].get_disks_by_key(object);
7389 let worker_store = Arc::clone(&store);
7390 let worker_bucket = bucket.clone();
7391 let worker = tokio::spawn(async move {
7392 worker_store
7393 .decommission_entry_for_test(
7394 1,
7395 MetaCacheEntry {
7396 name: object.to_string(),
7397 ..Default::default()
7398 },
7399 worker_bucket,
7400 source_set,
7401 )
7402 .await
7403 });
7404 commit_barrier.wait_until_paused().await;
7405
7406 let delete_barrier = crate::store::object::DeleteAfterObjectLockSnapshotBarrier::install(&bucket);
7407 let delete_store = Arc::clone(&store);
7408 let delete_bucket = bucket.clone();
7409 let delete = tokio::spawn(async move {
7410 delete_store
7411 .delete_object(&delete_bucket, object, ObjectOptions::default())
7412 .await
7413 });
7414 delete_barrier.wait_until_paused().await;
7415 delete_barrier.release_and_wait_until_namespace_pending().await;
7416 assert!(
7417 !delete_barrier.namespace_acquired() && !delete.is_finished(),
7418 "the reverse target commit must keep DELETE behind the fixed read fence"
7419 );
7420 delete.abort();
7421 assert!(
7422 delete
7423 .await
7424 .expect_err("the blocked DELETE should be canceled")
7425 .is_cancelled(),
7426 "canceling the blocked DELETE must not mutate either pool"
7427 );
7428 drop(delete_barrier);
7429
7430 commit_barrier.release();
7431 cleanup_barrier.wait_until_paused().await;
7432 drop(commit_barrier);
7433
7434 let read_opts = ObjectOptions {
7435 no_lock: true,
7436 ..Default::default()
7437 };
7438 let target_info = store.pools[0]
7439 .get_object_info(&bucket, object, &read_opts)
7440 .await
7441 .expect("read the committed active target before source cleanup");
7442 store.pools[1]
7443 .get_object_info(&bucket, object, &read_opts)
7444 .await
7445 .expect("the source must remain present until cleanup is released");
7446 let routed_info = store
7447 .get_object_info(&bucket, object, &read_opts)
7448 .await
7449 .expect("HEAD routing should prefer the active target before source cleanup");
7450 assert_eq!(routed_info.data_dir, target_info.data_dir);
7451
7452 let mut ranged_reader = store
7453 .get_object_reader(
7454 &bucket,
7455 object,
7456 Some(HTTPRangeSpec {
7457 is_suffix_length: false,
7458 start: 8,
7459 end: 15,
7460 }),
7461 HeaderMap::new(),
7462 &read_opts,
7463 )
7464 .await
7465 .expect("ranged GET routing should prefer the active target before source cleanup");
7466 assert_eq!(ranged_reader.object_info.data_dir, target_info.data_dir);
7467 let mut ranged_body = Vec::new();
7468 ranged_reader
7469 .stream
7470 .read_to_end(&mut ranged_body)
7471 .await
7472 .expect("drain the routed target range");
7473 assert_eq!(ranged_body, object_body[8..=15]);
7474
7475 cleanup_barrier.release();
7476 drop(cleanup_barrier);
7477 tokio::time::timeout(Duration::from_secs(60), worker)
7478 .await
7479 .expect("reverse ordinary decommission must not self-deadlock on the fixed target set")
7480 .expect("reverse ordinary decommission worker should join")
7481 .expect("reverse ordinary decommission should complete");
7482
7483 let mut ordinary_reader = store.pools[0]
7484 .get_object_reader(&bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
7485 .await
7486 .expect("read the ordinary object from the fixed target set");
7487 let mut ordinary_target_body = Vec::new();
7488 ordinary_reader
7489 .stream
7490 .read_to_end(&mut ordinary_target_body)
7491 .await
7492 .expect("drain the ordinary target body");
7493 assert_eq!(ordinary_target_body, object_body, "ordinary migration must preserve the full body");
7494 let ordinary_source_err = store.pools[1]
7495 .get_object_info(&bucket, object, &ObjectOptions::default())
7496 .await
7497 .expect_err("ordinary source generation must be cleaned after migration");
7498 assert!(matches!(ordinary_source_err, StorageError::ObjectNotFound(_, _)));
7499
7500 let multipart_source_set = store.pools[1].get_disks_by_key(multipart_object);
7501 let multipart_store = Arc::clone(&store);
7502 let multipart_bucket = bucket.clone();
7503 let multipart_worker = tokio::spawn(async move {
7504 multipart_store
7505 .decommission_entry_for_test(
7506 1,
7507 MetaCacheEntry {
7508 name: multipart_object.to_string(),
7509 ..Default::default()
7510 },
7511 multipart_bucket,
7512 multipart_source_set,
7513 )
7514 .await
7515 });
7516 tokio::time::timeout(Duration::from_secs(60), multipart_worker)
7517 .await
7518 .expect("reverse multipart decommission must not self-deadlock on new or complete")
7519 .expect("reverse multipart decommission worker should join")
7520 .expect("reverse multipart decommission should complete");
7521
7522 let target_info = store.pools[0]
7523 .get_object_info(&bucket, multipart_object, &ObjectOptions::default())
7524 .await
7525 .expect("read migrated multipart metadata from the fixed target set");
7526 assert!(target_info.is_multipart(), "migration must retain multipart identity");
7527 let mut multipart_reader = store.pools[0]
7528 .get_object_reader(&bucket, multipart_object, None, HeaderMap::new(), &ObjectOptions::default())
7529 .await
7530 .expect("read migrated multipart object from the fixed target set");
7531 let mut multipart_target_body = Vec::new();
7532 multipart_reader
7533 .stream
7534 .read_to_end(&mut multipart_target_body)
7535 .await
7536 .expect("drain the multipart target body");
7537 assert_eq!(multipart_target_body, multipart_body, "multipart migration must preserve the full body");
7538 let multipart_source_err = store.pools[1]
7539 .get_object_info(&bucket, multipart_object, &ObjectOptions::default())
7540 .await
7541 .expect_err("multipart source generation must be cleaned after migration");
7542 assert!(matches!(multipart_source_err, StorageError::ObjectNotFound(_, _)));
7543
7544 shutdown.cancel();
7545 }
7546
7547 #[tokio::test]
7548 #[serial_test::serial(storage_class_env)]
7549 async fn batch_delete_real_path_preserves_source_pool_errors_in_any_pool_order() {
7550 let temp_dir = tempfile::tempdir().expect("create batch delete pool-error store dir");
7551 let (_ctx, store, shutdown) =
7552 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "batch-delete-pool-errors", &[4, 4])).await;
7553 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
7554
7555 for source_pool_idx in [0, 1] {
7556 {
7557 let mut pool_meta = store.pool_meta.write().await;
7558 for pool in &mut pool_meta.pools {
7559 pool.decommission = None;
7560 }
7561 }
7562
7563 let bucket = format!("batch-del-pool-error-{source_pool_idx}-{}", uuid::Uuid::new_v4());
7564 let object_names = vec![
7565 format!("third-{source_pool_idx}.bin"),
7566 format!("first-{source_pool_idx}.bin"),
7567 format!("second-{source_pool_idx}.bin"),
7568 ];
7569 store
7570 .make_bucket(&bucket, &MakeBucketOptions::default())
7571 .await
7572 .expect("create batch delete pool-error bucket");
7573 for pool in &store.pools {
7574 for object_name in &object_names {
7575 let mut reader = PutObjReader::from_vec(format!("pool {} {object_name}", pool.pool_idx).into_bytes());
7576 pool.put_object(&bucket, object_name, &mut reader, &ObjectOptions::default())
7577 .await
7578 .expect("seed each object in both the source and active pools");
7579 }
7580 }
7581 {
7582 let mut pool_meta = store.pool_meta.write().await;
7583 pool_meta.pools[source_pool_idx].decommission = Some(PoolDecommissionInfo {
7584 start_time: Some(OffsetDateTime::now_utc()),
7585 ..Default::default()
7586 });
7587 }
7588 assert!(
7589 store.is_suspended(source_pool_idx).await,
7590 "the injected error pool must be the decommission source"
7591 );
7592
7593 let expected_errors = [
7594 StorageError::ErasureWriteQuorum,
7595 StorageError::NamespaceLockQuorumUnavailable {
7596 mode: "delete_objects_commit",
7597 bucket: bucket.clone(),
7598 object: object_names[1].clone(),
7599 required: 3,
7600 achieved: 2,
7601 },
7602 StorageError::ErasureWriteQuorum,
7603 ];
7604 let injection = crate::store::object::BatchDeletePoolErrorInjection::install(
7605 &bucket,
7606 source_pool_idx,
7607 object_names.iter().cloned().zip(expected_errors.iter().cloned()).collect(),
7608 );
7609 let requests = object_names
7610 .iter()
7611 .map(|object_name| ObjectToDelete {
7612 object_name: object_name.clone(),
7613 ..Default::default()
7614 })
7615 .collect();
7616
7617 let (deleted, errors) = store.delete_objects(&bucket, requests, ObjectOptions::default()).await;
7618
7619 assert_eq!(
7620 injection.observed(),
7621 object_names.len(),
7622 "the source pool must first complete every real delete"
7623 );
7624 assert_eq!(
7625 errors,
7626 expected_errors.iter().cloned().map(Some).collect::<Vec<_>>(),
7627 "a successful pool must not clear a source pool failure at any request index"
7628 );
7629 assert_eq!(
7630 deleted.iter().map(|object| object.object_name.as_str()).collect::<Vec<_>>(),
7631 object_names.iter().map(String::as_str).collect::<Vec<_>>(),
7632 "DeleteObjects must preserve request index mapping while aggregating pool failures"
7633 );
7634 assert!(
7635 deleted.iter().all(|object| object.found),
7636 "the injected source results must retain real delete success data"
7637 );
7638
7639 for pool in &store.pools {
7640 for object_name in &object_names {
7641 let error = pool
7642 .get_object_info(&bucket, object_name, &ObjectOptions::default())
7643 .await
7644 .expect_err("both the active and source pool delete calls must execute");
7645 assert!(
7646 matches!(error, StorageError::ObjectNotFound(_, _)),
7647 "unexpected residual object: {error:?}"
7648 );
7649 }
7650 }
7651 drop(injection);
7652 }
7653
7654 shutdown.cancel();
7655 }
7656
7657 #[tokio::test]
7658 #[serial_test::serial(storage_class_env)]
7659 async fn decommission_source_cleanup_holds_hashed_set_lock_across_preflight() {
7660 let temp_dir = tempfile::tempdir().expect("create multi-set decommission cleanup store dir");
7661 let (_ctx, store, shutdown) = without_storage_class_env(build_isolated_test_store_with_layout(
7662 temp_dir.path(),
7663 "multi-set-decommission-source-cleanup",
7664 &[(2, 4)],
7665 CancellationToken::new(),
7666 None,
7667 ))
7668 .await;
7669 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
7670
7671 let bucket = format!("decom-source-cleanup-lock-{}", uuid::Uuid::new_v4());
7672 let object = (0..128)
7673 .map(|index| format!("object-{index}.bin"))
7674 .find(|candidate| store.pools[0].get_disks_by_key(candidate).set_index == 1)
7675 .expect("the deterministic object search should select source set 1");
7676 store
7677 .make_bucket(&bucket, &MakeBucketOptions::default())
7678 .await
7679 .expect("create multi-set decommission cleanup bucket");
7680 let mut source = PutObjReader::from_vec(b"source generation".to_vec());
7681 store.pools[0]
7682 .put_object(&bucket, &object, &mut source, &ObjectOptions::default())
7683 .await
7684 .expect("write the source generation to set 1");
7685 let source_set = store.pools[0].get_disks_by_key(&object);
7686 assert_eq!(source_set.set_index, 1, "the source must not share the fixed set-0 namespace");
7687 let expected_source_versions = source_set
7688 .load_file_info_versions_exact(&bucket, &object)
7689 .await
7690 .expect("source versions should be readable")
7691 .expect("the source generation should exist");
7692
7693 let cleanup_barrier = crate::data_movement::SourceCleanupDeleteBarrier::install(&bucket, &object);
7694 let cleanup_store = Arc::clone(&store);
7695 let cleanup_bucket = bucket.clone();
7696 let cleanup_object = object.clone();
7697 let cleanup = tokio::spawn(async move {
7698 let mutation_fence = cleanup_store
7699 .acquire_decommission_source_cleanup_fence(&cleanup_bucket, &cleanup_object, source_set.as_ref())
7700 .await?;
7701 crate::data_movement::cleanup_source_entry_if_unchanged(
7702 source_set,
7703 &cleanup_bucket,
7704 &cleanup_object,
7705 &expected_source_versions,
7706 &[],
7707 crate::data_movement::SourceCleanupBucketFence {
7708 object_mutation_fence: Some(&mutation_fence),
7709 ..Default::default()
7710 },
7711 "test_multi_set_decommission_source_cleanup",
7712 )
7713 .await
7714 });
7715 cleanup_barrier.wait_until_paused().await;
7716
7717 let put_barrier = crate::set_disk::PutObjectCommitBarrier::install(
7718 &bucket,
7719 &object,
7720 crate::set_disk::PutObjectCommitPause::BeforeNamespace,
7721 );
7722 let mutation_pool = Arc::clone(&store.pools[0]);
7723 let mutation_bucket = bucket.clone();
7724 let mutation_object = object.clone();
7725 let replacement = b"replacement generation".to_vec();
7726 let expected_replacement = replacement.clone();
7727 let mutation = tokio::spawn(async move {
7728 let mut reader = PutObjReader::from_vec(replacement);
7729 mutation_pool
7730 .put_object(&mutation_bucket, &mutation_object, &mut reader, &ObjectOptions::default())
7731 .await
7732 });
7733 put_barrier.wait_until_paused().await;
7734 put_barrier.release_and_wait_until_namespace_pending().await;
7735 assert!(!mutation.is_finished(), "a source mutation must wait behind cleanup's set-1 write lock");
7736
7737 cleanup_barrier.release();
7738 cleanup
7739 .await
7740 .expect("source cleanup task should join")
7741 .expect("source cleanup should remove only the preflight generation");
7742 mutation
7743 .await
7744 .expect("source mutation task should join")
7745 .expect("source mutation should commit after cleanup releases the set lock");
7746
7747 let mut reader = store.pools[0]
7748 .get_object_reader(&bucket, &object, None, HeaderMap::new(), &ObjectOptions::default())
7749 .await
7750 .expect("the replacement generation must remain readable");
7751 let mut actual = Vec::new();
7752 reader
7753 .stream
7754 .read_to_end(&mut actual)
7755 .await
7756 .expect("read the replacement generation");
7757 assert_eq!(actual, expected_replacement, "cleanup must not delete the replacement generation");
7758
7759 shutdown.cancel();
7760 }
7761
7762 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7763 #[serial_test::serial(storage_class_env)]
7764 async fn versioned_delete_marker_survives_decommission_source_cleanup() {
7765 let temp_dir = tempfile::tempdir().expect("create versioned decommission delete-fence store dir");
7766 let (_ctx, store, shutdown) =
7767 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "versioned-decommission-delete-fence", &[4, 4]))
7768 .await;
7769 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
7770
7771 let bucket = format!("versioned-decom-delete-{}", uuid::Uuid::new_v4());
7772 let object = "object.bin";
7773 store
7774 .make_bucket(&bucket, &MakeBucketOptions::default())
7775 .await
7776 .expect("create versioned decommission delete-fence bucket");
7777 let (source_version, expected_source_versions) = migrate_versioned_decommission_test_object(
7778 &store,
7779 &bucket,
7780 object,
7781 b"source generation",
7782 "test_versioned_decommission_delete_fence",
7783 )
7784 .await;
7785
7786 let delete_barrier = crate::store::object::VersionedDeleteMarkerCommitBarrier::install(&bucket, object);
7787 let delete_store = Arc::clone(&store);
7788 let delete_bucket = bucket.clone();
7789 let delete = tokio::spawn(async move {
7790 delete_store
7791 .delete_object(
7792 &delete_bucket,
7793 object,
7794 ObjectOptions {
7795 versioned: true,
7796 ..Default::default()
7797 },
7798 )
7799 .await
7800 });
7801 delete_barrier.wait_until_paused().await;
7802 let cleanup_set = store.pools[0].get_disks_by_key(object);
7803 crate::data_movement::ensure_source_cleanup_versions_unchanged(
7804 Arc::clone(&cleanup_set),
7805 &bucket,
7806 object,
7807 &expected_source_versions,
7808 &[],
7809 "test_versioned_decommission_delete_fence",
7810 )
7811 .await
7812 .expect("the committed delete marker must not be published to the suspended source pool");
7813
7814 let cleanup_delete_barrier = crate::data_movement::SourceCleanupDeleteBarrier::install(&bucket, object);
7815 let cleanup_store = Arc::clone(&store);
7816 let cleanup_bucket = bucket.clone();
7817 let cleanup = tokio::spawn(async move {
7818 let mutation_fence = cleanup_store
7819 .acquire_decommission_source_cleanup_fence(&cleanup_bucket, object, cleanup_set.as_ref())
7820 .await?;
7821 crate::data_movement::cleanup_source_entry_if_unchanged(
7822 cleanup_set,
7823 &cleanup_bucket,
7824 object,
7825 &expected_source_versions,
7826 &[],
7827 crate::data_movement::SourceCleanupBucketFence {
7828 object_mutation_fence: Some(&mutation_fence),
7829 ..Default::default()
7830 },
7831 "test_versioned_decommission_delete_fence",
7832 )
7833 .await
7834 });
7835 cleanup_delete_barrier.wait_until_fence_pending().await;
7836 assert!(
7837 !cleanup_delete_barrier.is_paused(),
7838 "source cleanup must wait for the versioned DELETE mutation fence"
7839 );
7840
7841 delete_barrier.release();
7842 let marker = delete
7843 .await
7844 .expect("versioned DELETE task should join")
7845 .expect("versioned DELETE should publish a delete marker after migration");
7846 assert!(marker.delete_marker, "versioned DELETE must publish a delete marker");
7847 assert!(
7848 marker.version_id.is_some_and(|version_id| !version_id.is_nil()),
7849 "the delete marker must have a non-nil version ID"
7850 );
7851
7852 cleanup_delete_barrier.wait_until_paused().await;
7853 cleanup_delete_barrier.release();
7854 cleanup
7855 .await
7856 .expect("source cleanup task should join")
7857 .expect("source cleanup should preserve the active-pool delete marker");
7858
7859 let err = store
7860 .get_object_info(
7861 &bucket,
7862 object,
7863 &ObjectOptions {
7864 versioned: true,
7865 ..Default::default()
7866 },
7867 )
7868 .await
7869 .expect_err("the post-migration delete marker must hide the migrated version");
7870 assert!(
7871 matches!(err, StorageError::ObjectNotFound(_, _)),
7872 "unexpected latest-version result: {err:?}"
7873 );
7874 store
7875 .get_object_info(
7876 &bucket,
7877 object,
7878 &ObjectOptions {
7879 versioned: true,
7880 version_id: Some(source_version.to_string()),
7881 ..Default::default()
7882 },
7883 )
7884 .await
7885 .expect("the migrated source version must remain addressable below the delete marker");
7886 store.pools[0]
7887 .get_object_info(
7888 &bucket,
7889 object,
7890 &ObjectOptions {
7891 versioned: true,
7892 version_id: Some(source_version.to_string()),
7893 ..Default::default()
7894 },
7895 )
7896 .await
7897 .expect_err("source cleanup must remove the decommissioned source versions");
7898
7899 shutdown.cancel();
7900 }
7901
7902 #[cfg(feature = "test-util")]
7903 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7904 #[serial_test::serial(storage_class_env)]
7905 async fn decommission_entry_skips_cleanup_only_marker_when_free_version_is_present() {
7906 let temp_dir = tempfile::tempdir().expect("create free-version decommission store dir");
7907 let (ctx, store, shutdown) =
7908 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "decommission-free-marker", &[4, 4])).await;
7909 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
7910 let bucket = format!("decom-free-marker-{}", uuid::Uuid::new_v4());
7911 let object = "free-marker-object";
7912 store
7913 .make_bucket(&bucket, &MakeBucketOptions::default())
7914 .await
7915 .expect("create free-version decommission bucket");
7916 let (_, free_version) = seed_transitioned_free_version(&ctx, &store, &bucket, object).await;
7917 let source_free = store.pools[0]
7918 .get_disks_by_key(object)
7919 .load_file_info_versions_exact(&bucket, object)
7920 .await
7921 .expect("source free-version metadata should decode")
7922 .and_then(|versions| {
7923 versions
7924 .versions
7925 .into_iter()
7926 .find(|version| version.version_id == Some(free_version) && version.tier_free_version())
7927 })
7928 .expect("source free-version identity should be present before decommission");
7929 let mut target_reader = PutObjReader::from_vec(b"target ordinary bytes".to_vec());
7930 let target_w = store.pools[1]
7931 .put_object(
7932 &bucket,
7933 object,
7934 &mut target_reader,
7935 &ObjectOptions {
7936 versioned: true,
7937 ..Default::default()
7938 },
7939 )
7940 .await
7941 .expect("write unrelated target version");
7942 let target_w_version = target_w.version_id.expect("target version should have an id");
7943 let marker = store.pools[0]
7944 .delete_object(
7945 &bucket,
7946 object,
7947 ObjectOptions {
7948 versioned: true,
7949 ..Default::default()
7950 },
7951 )
7952 .await
7953 .expect("write cleanup-only delete marker");
7954 assert!(marker.delete_marker);
7955
7956 mark_test_pool_decommissioning(&store, 0).await;
7957 let source_set = store.pools[0].get_disks_by_key(object);
7958 store
7959 .decommission_entry_for_test_with_bucket_incarnation(
7960 0,
7961 MetaCacheEntry {
7962 name: object.to_string(),
7963 ..Default::default()
7964 },
7965 bucket.clone(),
7966 source_set.clone(),
7967 )
7968 .await
7969 .expect("real decommission entry should migrate the free version");
7970
7971 let target_versions = store.pools[1]
7972 .get_disks_by_key(object)
7973 .load_file_info_versions_exact(&bucket, object)
7974 .await
7975 .expect("target versions should decode")
7976 .expect("target free version should be present");
7977 assert!(
7978 target_versions
7979 .versions
7980 .iter()
7981 .any(|version| { version.version_id == Some(free_version) && version.tier_free_version() })
7982 );
7983 let migrated_free = target_versions
7984 .versions
7985 .iter()
7986 .find(|version| version.version_id == Some(free_version) && version.tier_free_version())
7987 .expect("migrated free-version identity should remain readable from target disks");
7988 assert!(
7989 crate::store::tiered_data_movement_source_matches(&source_free, migrated_free)
7990 .expect("migrated free-version identity should decode")
7991 );
7992 let retained_w = target_versions
7993 .versions
7994 .iter()
7995 .find(|version| version.version_id == Some(target_w_version))
7996 .expect("unrelated target version should remain");
7997 assert!(!retained_w.deleted && !retained_w.tier_free_version());
7998 assert_eq!(retained_w.size, target_w.size);
7999 assert_eq!(retained_w.get_etag(), target_w.etag);
8000 assert!(
8001 target_versions
8002 .versions
8003 .iter()
8004 .all(|version| { version.tier_free_version() || !version.deleted })
8005 );
8006 assert!(
8007 source_set
8008 .load_file_info_versions_exact(&bucket, object)
8009 .await
8010 .expect("source versions should be readable after cleanup")
8011 .is_none(),
8012 "successful free-version migration should permit source cleanup"
8013 );
8014 let (heal_versions, _, _) = store
8015 .heal_walk_versions_page(1, 0, &bucket, "", None, 2, 16, true)
8016 .await
8017 .expect("heal walk should decode the migrated free version");
8018 let free_version_string = free_version.to_string();
8019 let healed_free = heal_versions
8020 .iter()
8021 .find(|version| version.version_id.as_deref() == Some(free_version_string.as_str()))
8022 .expect("heal walk should surface the migrated free version");
8023 let healed_info = healed_free
8024 .lifecycle_object_info
8025 .as_ref()
8026 .expect("heal walk should retain lifecycle identity for the migrated free version");
8027 assert!(healed_info.transitioned_object.free_version);
8028 assert_eq!(healed_info.transitioned_object.tier, source_free.transition_tier);
8029 assert_eq!(healed_info.transitioned_object.name, source_free.transitioned_objname);
8030 shutdown.cancel();
8031 }
8032
8033 #[cfg(feature = "test-util")]
8034 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8035 #[serial_test::serial(storage_class_env)]
8036 async fn decommission_entry_allows_free_version_consumed_before_source_lock() {
8037 let temp_dir = tempfile::tempdir().expect("create consumed free-version store dir");
8038 let (ctx, store, shutdown) =
8039 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "decommission-free-consumed", &[4, 4])).await;
8040 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
8041 let bucket = format!("decom-free-consumed-{}", uuid::Uuid::new_v4());
8042 let object = "free-consumed-object";
8043 store
8044 .make_bucket(&bucket, &MakeBucketOptions::default())
8045 .await
8046 .expect("create consumed free-version bucket");
8047 let (_, free_version) = seed_transitioned_free_version(&ctx, &store, &bucket, object).await;
8048
8049 mark_test_pool_decommissioning(&store, 0).await;
8050 let source_set = store.pools[0].get_disks_by_key(object);
8051 let barrier = crate::store::object::DecommissionFreeVersionSourceRaceBarrier::install(&bucket, object);
8052 let decommission = tokio::spawn({
8053 let store = store.clone();
8054 let bucket = bucket.clone();
8055 let source_set = source_set.clone();
8056 async move {
8057 store
8058 .decommission_entry_for_test_with_bucket_incarnation(
8059 0,
8060 MetaCacheEntry {
8061 name: object.to_string(),
8062 ..Default::default()
8063 },
8064 bucket,
8065 source_set,
8066 )
8067 .await
8068 }
8069 });
8070
8071 barrier.wait_until_paused().await;
8072 store.pools[0]
8073 .delete_object(
8074 &bucket,
8075 object,
8076 ObjectOptions {
8077 versioned: true,
8078 version_id: Some(free_version.to_string()),
8079 incl_free_versions: true,
8080 ..Default::default()
8081 },
8082 )
8083 .await
8084 .expect("lifecycle should consume the source free version before decommission locks it");
8085 assert!(
8086 source_set
8087 .load_file_info_versions_exact(&bucket, object)
8088 .await
8089 .expect("consumed source metadata should remain readable")
8090 .is_none(),
8091 "the lifecycle delete should remove the source free version"
8092 );
8093
8094 barrier.release();
8095 decommission
8096 .await
8097 .expect("decommission task should join")
8098 .expect("a concurrently consumed free version should not fail source cleanup");
8099 assert!(
8100 store.pools[1]
8101 .get_disks_by_key(object)
8102 .load_file_info_versions_exact(&bucket, object)
8103 .await
8104 .expect("target metadata should remain readable")
8105 .is_none(),
8106 "an already consumed free version should not be recreated on the target"
8107 );
8108 shutdown.cancel();
8109 }
8110
8111 #[cfg(feature = "test-util")]
8112 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8113 #[serial_test::serial(storage_class_env)]
8114 async fn decommission_entry_rejects_subquorum_free_version_conflict_and_retains_source() {
8115 let temp_dir = tempfile::tempdir().expect("create sub-quorum free-version store dir");
8116 let (ctx, store, shutdown) =
8117 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "decommission-free-conflict", &[4, 4])).await;
8118 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
8119 let bucket = format!("decom-free-conflict-{}", uuid::Uuid::new_v4());
8120 let object = "free-conflict-object";
8121 store
8122 .make_bucket(&bucket, &MakeBucketOptions::default())
8123 .await
8124 .expect("create sub-quorum conflict bucket");
8125 let (_, free_version) = seed_transitioned_free_version(&ctx, &store, &bucket, object).await;
8126 let source_free = store.pools[0]
8127 .get_disks_by_key(object)
8128 .load_file_info_versions_exact(&bucket, object)
8129 .await
8130 .expect("source free version should decode before crash replay setup")
8131 .and_then(|versions| {
8132 versions
8133 .versions
8134 .into_iter()
8135 .find(|version| version.version_id == Some(free_version))
8136 })
8137 .expect("source free version should be available for crash replay setup");
8138
8139 let mut target_reader = PutObjReader::from_vec(b"target ordinary bytes".to_vec());
8140 let target = store.pools[1]
8141 .put_object(
8142 &bucket,
8143 object,
8144 &mut target_reader,
8145 &ObjectOptions {
8146 versioned: true,
8147 ..Default::default()
8148 },
8149 )
8150 .await
8151 .expect("seed ordinary target version");
8152 let target_version = target.version_id.expect("target version must have an ID");
8153 store.pools[1]
8157 .get_object_info(
8158 &bucket,
8159 object,
8160 &ObjectOptions {
8161 versioned: true,
8162 version_id: Some(target_version.to_string()),
8163 ..Default::default()
8164 },
8165 )
8166 .await
8167 .expect("target PUT tail should drain before per-disk fixture mutation");
8168 let target_disks = store.pools[1].get_disks_by_key(object).disks.read().await.clone();
8169 for disk in target_disks.iter().skip(1) {
8170 disk.as_ref()
8171 .expect("target crash replay quorum disk should be online")
8172 .write_metadata("", &bucket, object, source_free.clone())
8173 .await
8174 .expect("seed an equivalent free version on the target quorum");
8175 }
8176 let conflict_path = temp_dir
8177 .path()
8178 .join(format!("pool1/set0/disk0/{bucket}/{object}/{STORAGE_FORMAT_FILE}"));
8179 let encoded = tokio::fs::read(&conflict_path)
8180 .await
8181 .expect("target metadata should be readable");
8182 let mut metadata = FileMeta::load(&encoded).expect("target metadata should decode");
8183 let target_index = metadata
8184 .versions
8185 .iter()
8186 .position(|version| version.header.version_id == Some(target_version))
8187 .expect("target version should be present on the conflict disk");
8188 let mut target_meta = metadata.versions[target_index]
8189 .parse_version_meta()
8190 .expect("target version metadata should decode");
8191 target_meta
8192 .object
8193 .as_mut()
8194 .expect("target conflict must remain an ordinary object")
8195 .version_id = Some(free_version);
8196 metadata.versions[target_index] = target_meta.try_into().expect("conflict metadata should encode");
8197 let expected_conflict_meta = metadata.versions[target_index].meta.clone();
8198 let expected_conflict = metadata.versions[target_index]
8199 .into_fileinfo(&bucket, object, true)
8200 .expect("conflict metadata should decode as an ordinary object");
8201 let duplicate_free: rustfs_filemeta::FileMetaShallowVersion = rustfs_filemeta::FileMetaVersion::from(source_free.clone())
8202 .try_into()
8203 .expect("duplicate free metadata should encode");
8204 metadata.versions.insert(target_index, duplicate_free);
8205 tokio::fs::write(&conflict_path, metadata.marshal_msg().expect("conflict metadata should encode"))
8206 .await
8207 .expect("write sub-quorum conflict metadata");
8208
8209 let mut expected_target_metadata = Vec::with_capacity(4);
8210 for disk_index in 0..4 {
8211 let target_path = temp_dir
8212 .path()
8213 .join(format!("pool1/set0/disk{disk_index}/{bucket}/{object}/{STORAGE_FORMAT_FILE}"));
8214 let target_encoded = tokio::fs::read(&target_path)
8215 .await
8216 .expect("target fixture metadata should be readable before decommission");
8217 let target_meta = FileMeta::load(&target_encoded).expect("target fixture metadata should decode");
8218 let same_id = target_meta
8219 .versions
8220 .iter()
8221 .filter(|version| version.header.version_id == Some(free_version))
8222 .collect::<Vec<_>>();
8223 if disk_index == 0 {
8224 assert_eq!(same_id.len(), 2, "conflict fixture should contain both same-ID records");
8225 assert!(same_id[0].header.free_version());
8226 assert!(!same_id[1].header.free_version());
8227 } else {
8228 assert_eq!(same_id.len(), 1, "target fixture disk {disk_index} should contain one free version");
8229 assert!(same_id[0].header.free_version());
8230 }
8231 expected_target_metadata.push(target_encoded);
8232 }
8233
8234 mark_test_pool_decommissioning(&store, 0).await;
8235 let source_set = store.pools[0].get_disks_by_key(object);
8236 store
8237 .decommission_entry_for_test_with_bucket_incarnation(
8238 0,
8239 MetaCacheEntry {
8240 name: object.to_string(),
8241 ..Default::default()
8242 },
8243 bucket.clone(),
8244 source_set.clone(),
8245 )
8246 .await
8247 .expect("conflicted decommission entry should retain the source and retry later");
8248
8249 let source_versions = source_set
8250 .load_file_info_versions_exact(&bucket, object)
8251 .await
8252 .expect("retained source versions should decode")
8253 .expect("source free version should be retained after conflict");
8254 assert!(
8255 source_versions
8256 .versions
8257 .iter()
8258 .any(|version| { version.version_id == Some(free_version) && version.tier_free_version() })
8259 );
8260 let post_encoded = tokio::fs::read(&conflict_path)
8261 .await
8262 .expect("conflict metadata should remain readable");
8263 let post_metadata = FileMeta::load(&post_encoded).expect("post-conflict metadata should decode");
8264 let same_id = post_metadata
8265 .versions
8266 .iter()
8267 .filter(|version| version.header.version_id == Some(free_version))
8268 .collect::<Vec<_>>();
8269 assert_eq!(same_id.len(), 2, "conflict metadata should retain both same-ID records");
8270 assert_eq!(same_id.iter().filter(|version| version.header.free_version()).count(), 1);
8271 assert_eq!(same_id.iter().filter(|version| !version.header.free_version()).count(), 1);
8272 let post_conflict = same_id
8273 .into_iter()
8274 .find(|version| !version.header.free_version())
8275 .expect("ordinary conflict version must remain addressable by the source ID");
8276 let post_conflict_info = post_conflict
8277 .into_fileinfo(&bucket, object, true)
8278 .expect("post-conflict ordinary metadata should decode");
8279 assert!(!post_conflict_info.deleted && !post_conflict_info.tier_free_version());
8280 assert_eq!(post_conflict.meta, expected_conflict_meta);
8281 assert_eq!(post_conflict_info.size, expected_conflict.size);
8282 assert_eq!(post_conflict_info.data_dir, expected_conflict.data_dir);
8283 assert_eq!(post_conflict_info.metadata, expected_conflict.metadata);
8284 assert_eq!(post_conflict_info.get_etag(), expected_conflict.get_etag());
8285
8286 for (disk_index, expected_target_encoded) in expected_target_metadata.iter().enumerate() {
8287 let target_path = temp_dir
8288 .path()
8289 .join(format!("pool1/set0/disk{disk_index}/{bucket}/{object}/{STORAGE_FORMAT_FILE}"));
8290 let target_encoded = tokio::fs::read(&target_path)
8291 .await
8292 .expect("target metadata should remain readable");
8293 assert_eq!(
8294 &target_encoded, expected_target_encoded,
8295 "conflicted decommission must not mutate target disk {disk_index}"
8296 );
8297 let target_meta = FileMeta::load(&target_encoded).expect("target metadata should decode");
8298 let same_id = target_meta
8299 .versions
8300 .iter()
8301 .filter(|version| version.header.version_id == Some(free_version))
8302 .collect::<Vec<_>>();
8303 if disk_index == 0 {
8304 assert_eq!(same_id.len(), 2);
8305 assert!(same_id[0].header.free_version());
8306 assert!(!same_id[1].header.free_version());
8307 } else {
8308 assert_eq!(same_id.len(), 1);
8309 assert!(same_id[0].header.free_version());
8310 }
8311 }
8312 let sweep_err = store
8313 .check_after_decommission_for_test(0)
8314 .await
8315 .expect_err("final sweep must report the retained free version");
8316 assert!(
8317 sweep_err.to_string().contains("version(s) were found"),
8318 "unexpected final sweep error: {sweep_err}"
8319 );
8320 shutdown.cancel();
8321 }
8322
8323 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8324 #[serial_test::serial(storage_class_env)]
8325 async fn versioned_batch_delete_marker_skips_decommission_source() {
8326 let temp_dir = tempfile::tempdir().expect("create versioned batch decommission store dir");
8327 let (_ctx, store, shutdown) = without_storage_class_env(build_isolated_test_store(
8328 temp_dir.path(),
8329 "versioned-batch-decommission-delete-fence",
8330 &[4, 4, 4],
8331 ))
8332 .await;
8333 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
8334
8335 let bucket = format!("vbatch-decom-delete-{}", uuid::Uuid::new_v4());
8336 let object = "batch-object.bin";
8337 store
8338 .make_bucket(&bucket, &MakeBucketOptions::default())
8339 .await
8340 .expect("create versioned batch decommission bucket");
8341 let (_source_version, expected_source_versions) = migrate_versioned_decommission_test_object(
8342 &store,
8343 &bucket,
8344 object,
8345 b"batch source generation",
8346 "test_versioned_batch_decommission_delete_fence",
8347 )
8348 .await;
8349
8350 let delete_config_snapshot =
8351 Arc::new(crate::bucket::replication::DeleteReplicationConfigSnapshot::from_configs_for_test(
8352 s3s::dto::VersioningConfiguration {
8353 status: Some(s3s::dto::BucketVersioningStatus::from_static(s3s::dto::BucketVersioningStatus::ENABLED)),
8354 ..Default::default()
8355 },
8356 None,
8357 ));
8358 let delete_barrier = crate::store::object::VersionedDeleteMarkerCommitBarrier::install(&bucket, object);
8359 let delete_store = Arc::clone(&store);
8360 let delete_bucket = bucket.clone();
8361 let delete = tokio::spawn(async move {
8362 delete_store
8363 .delete_objects(
8364 &delete_bucket,
8365 vec![ObjectToDelete {
8366 object_name: object.to_string(),
8367 ..Default::default()
8368 }],
8369 ObjectOptions {
8370 delete_replication_config_snapshot: Some(delete_config_snapshot),
8371 ..Default::default()
8372 },
8373 )
8374 .await
8375 });
8376 delete_barrier.wait_until_paused().await;
8377
8378 let source_set = store.pools[0].get_disks_by_key(object);
8379 crate::data_movement::ensure_source_cleanup_versions_unchanged(
8380 source_set,
8381 &bucket,
8382 object,
8383 &expected_source_versions,
8384 &[],
8385 "test_versioned_batch_decommission_delete_fence",
8386 )
8387 .await
8388 .expect("batch DELETE must not publish a marker to the suspended source");
8389
8390 delete_barrier.release();
8391 let (deleted, errors) = delete.await.expect("versioned batch DELETE task should join");
8392 assert!(errors.iter().all(Option::is_none), "versioned batch DELETE should succeed: {errors:?}");
8393 assert_eq!(deleted.len(), 1);
8394 assert!(deleted[0].delete_marker, "versioned batch DELETE must return a marker");
8395 assert!(
8396 deleted[0]
8397 .delete_marker_version_id
8398 .is_some_and(|version_id| !version_id.is_nil()),
8399 "versioned batch DELETE marker must have a non-nil version ID"
8400 );
8401
8402 let mut active_marker_count = 0;
8403 for pool in store.pools.iter().skip(1) {
8404 let Some(versions) = pool
8405 .get_disks_by_key(object)
8406 .load_file_info_versions_exact(&bucket, object)
8407 .await
8408 .expect("active-pool versions should be readable")
8409 else {
8410 continue;
8411 };
8412 active_marker_count += versions.versions.iter().filter(|version| version.deleted).count();
8413 }
8414 assert_eq!(active_marker_count, 1, "batch DELETE must publish exactly one active-pool marker");
8415
8416 shutdown.cancel();
8417 }
8418
8419 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8420 #[serial_test::serial(storage_class_env)]
8421 async fn suspended_delete_marker_then_decommission_worker_converges_null_source() {
8422 let temp_dir = tempfile::tempdir().expect("create suspended decommission DELETE store dir");
8423 let (_ctx, store, shutdown) = without_storage_class_env(build_isolated_test_store(
8424 temp_dir.path(),
8425 "suspended-decommission-delete-convergence",
8426 &[4, 4],
8427 ))
8428 .await;
8429 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
8430
8431 let bucket = format!("suspended-decom-delete-{}", uuid::Uuid::new_v4());
8432 let object = "single.bin";
8433 store
8434 .make_bucket(&bucket, &MakeBucketOptions::default())
8435 .await
8436 .expect("create suspended decommission DELETE bucket");
8437 write_suspended_decommission_source(&store, &bucket, object).await;
8438 mark_test_pool_decommissioning(&store, 0).await;
8439
8440 let deleted = store
8441 .delete_object(
8442 &bucket,
8443 object,
8444 ObjectOptions {
8445 version_suspended: true,
8446 ..Default::default()
8447 },
8448 )
8449 .await
8450 .expect("a healthy reserved target must accept suspended DELETE");
8451 assert!(deleted.delete_marker);
8452 assert_eq!(deleted.version_id, Some(uuid::Uuid::nil()));
8453 let marker_mod_time = deleted.mod_time.expect("DELETE must return the marker generation");
8454 assert_suspended_null_source_present(&store, &bucket, object).await;
8455 assert_suspended_null_delete_marker_visible(&store, &bucket, object, marker_mod_time).await;
8456
8457 let source_set = store.pools[0].get_disks_by_key(object);
8458 let worker_store = Arc::clone(&store);
8459 let worker_bucket = bucket.clone();
8460 tokio::spawn(async move {
8461 worker_store
8462 .decommission_entry_for_test(
8463 0,
8464 MetaCacheEntry {
8465 name: object.to_string(),
8466 ..Default::default()
8467 },
8468 worker_bucket,
8469 source_set,
8470 )
8471 .await
8472 })
8473 .await
8474 .expect("suspended decommission worker should join")
8475 .expect("worker must converge the old null source behind the newer DELETE marker");
8476
8477 assert_decommission_source_absent(
8478 &store,
8479 &bucket,
8480 object,
8481 &ObjectOptions {
8482 version_suspended: true,
8483 ..Default::default()
8484 },
8485 )
8486 .await;
8487 assert_suspended_null_delete_marker_visible(&store, &bucket, object, marker_mod_time).await;
8488 shutdown.cancel();
8489 }
8490
8491 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8492 #[serial_test::serial(storage_class_env)]
8493 async fn suspended_batch_delete_marker_then_decommission_worker_converges_null_source() {
8494 let temp_dir = tempfile::tempdir().expect("create suspended batch decommission DELETE store dir");
8495 let (_ctx, store, shutdown) = without_storage_class_env(build_isolated_test_store(
8496 temp_dir.path(),
8497 "suspended-batch-decommission-delete-convergence",
8498 &[4, 4],
8499 ))
8500 .await;
8501 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
8502
8503 let bucket = format!("susp-batch-decom-delete-{}", uuid::Uuid::new_v4());
8504 let object = "batch.bin";
8505 store
8506 .make_bucket(&bucket, &MakeBucketOptions::default())
8507 .await
8508 .expect("create suspended batch decommission DELETE bucket");
8509 write_suspended_decommission_source(&store, &bucket, object).await;
8510 mark_test_pool_decommissioning(&store, 0).await;
8511
8512 let delete_config_snapshot =
8513 Arc::new(crate::bucket::replication::DeleteReplicationConfigSnapshot::from_configs_for_test(
8514 s3s::dto::VersioningConfiguration {
8515 status: Some(s3s::dto::BucketVersioningStatus::from_static(s3s::dto::BucketVersioningStatus::SUSPENDED)),
8516 ..Default::default()
8517 },
8518 None,
8519 ));
8520 let (deleted, errors) = store
8521 .delete_objects(
8522 &bucket,
8523 vec![ObjectToDelete {
8524 object_name: object.to_string(),
8525 ..Default::default()
8526 }],
8527 ObjectOptions {
8528 delete_replication_config_snapshot: Some(delete_config_snapshot),
8529 ..Default::default()
8530 },
8531 )
8532 .await;
8533 assert!(
8534 matches!(errors.as_slice(), [None]),
8535 "unexpected suspended batch DELETE result: {errors:?}"
8536 );
8537 assert_eq!(deleted.len(), 1);
8538 assert!(deleted[0].delete_marker);
8539 assert_eq!(deleted[0].object_name, object);
8540 assert!(
8541 deleted[0]
8542 .delete_marker_version_id
8543 .is_none_or(|version_id| version_id.is_nil()),
8544 "batch DELETE must retain the native null version identity"
8545 );
8546 let marker_mod_time = deleted[0]
8547 .delete_marker_mtime
8548 .expect("batch DELETE must return the marker generation");
8549 assert_suspended_null_source_present(&store, &bucket, object).await;
8550 assert_suspended_null_delete_marker_visible(&store, &bucket, object, marker_mod_time).await;
8551
8552 let source_set = store.pools[0].get_disks_by_key(object);
8553 let worker_store = Arc::clone(&store);
8554 let worker_bucket = bucket.clone();
8555 tokio::spawn(async move {
8556 worker_store
8557 .decommission_entry_for_test(
8558 0,
8559 MetaCacheEntry {
8560 name: object.to_string(),
8561 ..Default::default()
8562 },
8563 worker_bucket,
8564 source_set,
8565 )
8566 .await
8567 })
8568 .await
8569 .expect("suspended batch decommission worker should join")
8570 .expect("worker must converge the old null source behind the newer batch DELETE marker");
8571
8572 assert_decommission_source_absent(
8573 &store,
8574 &bucket,
8575 object,
8576 &ObjectOptions {
8577 version_suspended: true,
8578 ..Default::default()
8579 },
8580 )
8581 .await;
8582 assert_suspended_null_delete_marker_visible(&store, &bucket, object, marker_mod_time).await;
8583 shutdown.cancel();
8584 }
8585
8586 #[cfg(feature = "test-util")]
8587 #[tokio::test]
8588 #[serial_test::serial(storage_class_env)]
8589 async fn tiered_data_movement_rejects_a_stale_source_snapshot_before_target_write() {
8590 let temp_dir = tempfile::tempdir().expect("create stale-source data movement store dir");
8591 let (ctx, store, _shutdown) =
8592 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "stale-tier-source", &[4, 4])).await;
8593 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
8594 let bucket = "stale-tier-source-bucket";
8595 let object = "stale-tier-source-object";
8596 store
8597 .make_bucket(bucket, &MakeBucketOptions::default())
8598 .await
8599 .expect("bucket should be created");
8600 register_mock_tier(&ctx.tier_config_mgr(), "STALE-TIER").await;
8601 let incarnation = store
8602 .bucket_incarnation_id(bucket)
8603 .await
8604 .expect("bucket incarnation should exist");
8605 let version_id = uuid::Uuid::new_v4();
8606 let stale = FileInfo {
8607 volume: bucket.to_string(),
8608 name: object.to_string(),
8609 version_id: Some(version_id),
8610 transition_status: rustfs_filemeta::TRANSITION_COMPLETE.to_string(),
8611 transitioned_objname: "remote/stale-source".to_string(),
8612 transition_tier: "STALE-TIER".to_string(),
8613 transition_version_id: Some(uuid::Uuid::new_v4()),
8614 transition_version_state: rustfs_filemeta::TransitionVersionState::Exact,
8615 data_dir: Some(uuid::Uuid::new_v4()),
8616 mod_time: Some(OffsetDateTime::UNIX_EPOCH),
8617 size: 1,
8618 metadata: HashMap::new(),
8619 ..Default::default()
8620 };
8621 let err = store
8622 .decommission_tiered_object(
8623 bucket,
8624 object,
8625 &stale,
8626 &ObjectOptions {
8627 versioned: true,
8628 version_id: Some(version_id.to_string()),
8629 src_pool_idx: 0,
8630 data_movement: true,
8631 expected_bucket_incarnation_id: Some(incarnation),
8632 ..Default::default()
8633 },
8634 )
8635 .await
8636 .expect_err("a source removed after queue capture must fail closed");
8637 assert!(matches!(err, Error::ObjectNotFound(_, _) | Error::FileNotFound));
8638 let target_err = store.pools[1]
8639 .get_object_info(bucket, object, &ObjectOptions::default())
8640 .await
8641 .expect_err("stale source rejection must not write target metadata");
8642 assert!(matches!(
8643 target_err,
8644 StorageError::ObjectNotFound(_, _) | StorageError::VersionNotFound(_, _, _)
8645 ));
8646 }
8647
8648 #[tokio::test]
8649 #[serial_test::serial(storage_class_env)]
8650 async fn data_movement_put_conflict_validates_only_selected_target_pool() {
8651 let temp_dir = tempfile::tempdir().expect("create three-pool data movement store dir");
8652 let (_ctx, store, _shutdown) =
8653 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "data-movement-selected-target", &[4, 4, 4]))
8654 .await;
8655 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
8656
8657 let bucket = format!("dm-selected-target-{}", uuid::Uuid::new_v4());
8658 let object = "object.bin";
8659 let source_body = b"source-and-equivalent".to_vec();
8660 let conflicting_body = b"newer-conflicting-target".to_vec();
8661 let source_mod_time = OffsetDateTime::UNIX_EPOCH + time::Duration::SECOND;
8662 let version = uuid::Uuid::new_v4();
8663 store
8664 .make_bucket(&bucket, &MakeBucketOptions::default())
8665 .await
8666 .expect("create selected-target bucket");
8667 for (pool_idx, body, mod_time) in [
8668 (0, source_body.clone(), source_mod_time),
8669 (1, conflicting_body.clone(), source_mod_time + time::Duration::SECOND),
8670 (2, source_body.clone(), source_mod_time),
8671 ] {
8672 let mut reader = PutObjReader::from_vec(body);
8673 store.pools[pool_idx]
8674 .put_object(
8675 &bucket,
8676 object,
8677 &mut reader,
8678 &ObjectOptions {
8679 versioned: true,
8680 version_id: Some(version.to_string()),
8681 mod_time: Some(mod_time),
8682 ..Default::default()
8683 },
8684 )
8685 .await
8686 .expect("seed data movement pool");
8687 }
8688 *store.rebalance_meta.write().await = Some(active_rebalance_meta_for_pool(store.pools.len(), 0));
8689
8690 let source_reader = store.pools[0]
8691 .get_object_reader(
8692 &bucket,
8693 object,
8694 None,
8695 HeaderMap::new(),
8696 &ObjectOptions {
8697 versioned: true,
8698 version_id: Some(version.to_string()),
8699 raw_data_movement_read: true,
8700 ..Default::default()
8701 },
8702 )
8703 .await
8704 .expect("read source for selected-target migration");
8705 crate::data_movement::migrate_object(store.clone(), 0, bucket.clone(), source_reader, None, "test_selected_target")
8706 .await
8707 .expect_err("an equivalent object in another pool must not mask the selected target conflict");
8708
8709 let mut selected_target = store.pools[1]
8710 .get_object_reader(
8711 &bucket,
8712 object,
8713 None,
8714 HeaderMap::new(),
8715 &ObjectOptions {
8716 versioned: true,
8717 version_id: Some(version.to_string()),
8718 ..Default::default()
8719 },
8720 )
8721 .await
8722 .expect("read selected conflicting target");
8723 let mut selected_body = Vec::new();
8724 selected_target
8725 .stream
8726 .read_to_end(&mut selected_body)
8727 .await
8728 .expect("drain selected conflicting target");
8729 assert_eq!(selected_body, conflicting_body);
8730 }
8731
8732 #[tokio::test]
8733 #[serial_test::serial(storage_class_env)]
8734 async fn data_movement_target_remains_readable_before_source_cleanup() {
8735 let temp_dir = tempfile::tempdir().expect("create data movement read-window store dir");
8736 let (_ctx, store, shutdown) =
8737 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "data-movement-read-window", &[4, 4])).await;
8738 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
8739
8740 let bucket = format!("dm-read-window-{}", uuid::Uuid::new_v4());
8741 let object = "object.bin";
8742 let version = uuid::Uuid::new_v4();
8743 let mod_time = OffsetDateTime::UNIX_EPOCH + time::Duration::SECOND;
8744 store
8745 .make_bucket(&bucket, &MakeBucketOptions::default())
8746 .await
8747 .expect("create data movement read-window bucket");
8748
8749 let mut reader = PutObjReader::from_vec(b"source body".to_vec());
8750 store.pools[0]
8751 .put_object(
8752 &bucket,
8753 object,
8754 &mut reader,
8755 &ObjectOptions {
8756 versioned: true,
8757 version_id: Some(version.to_string()),
8758 mod_time: Some(mod_time),
8759 ..Default::default()
8760 },
8761 )
8762 .await
8763 .expect("seed data movement source");
8764 *store.rebalance_meta.write().await = Some(active_rebalance_meta_for_pool(store.pools.len(), 0));
8765
8766 let source_reader = store.pools[0]
8767 .get_object_reader(
8768 &bucket,
8769 object,
8770 None,
8771 HeaderMap::new(),
8772 &ObjectOptions {
8773 versioned: true,
8774 version_id: Some(version.to_string()),
8775 raw_data_movement_read: true,
8776 ..Default::default()
8777 },
8778 )
8779 .await
8780 .expect("read data movement source");
8781 let source_data_dir = source_reader.object_info.data_dir;
8782 crate::data_movement::migrate_object(store.clone(), 0, bucket.clone(), source_reader, None, "test_read_window")
8783 .await
8784 .expect("commit data movement target");
8785
8786 let target = store.pools[1]
8787 .get_object_info(
8788 &bucket,
8789 object,
8790 &ObjectOptions {
8791 versioned: true,
8792 version_id: Some(version.to_string()),
8793 ..Default::default()
8794 },
8795 )
8796 .await
8797 .expect("read committed data movement target");
8798 assert_ne!(source_data_dir, target.data_dir, "the target must use its own data directory");
8799 assert_eq!(
8800 rustfs_utils::http::get_consistent_str(&target.user_defined, rustfs_utils::http::SUFFIX_DATA_MOVED),
8801 Some("true")
8802 );
8803
8804 let resolved = store
8805 .get_object_info(
8806 &bucket,
8807 object,
8808 &ObjectOptions {
8809 versioned: true,
8810 version_id: Some(version.to_string()),
8811 ..Default::default()
8812 },
8813 )
8814 .await
8815 .expect("the equal-time migration window must remain readable");
8816 assert_eq!(resolved.data_dir, target.data_dir);
8817
8818 shutdown.cancel();
8819 }
8820
8821 #[tokio::test]
8822 #[serial_test::serial(storage_class_env)]
8823 async fn data_movement_multipart_conflict_validates_exact_version_target_pool() {
8824 let temp_dir = tempfile::tempdir().expect("create three-pool multipart data movement store dir");
8825 let (_ctx, store, _shutdown) =
8826 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "data-movement-multipart-target", &[4, 4, 4]))
8827 .await;
8828 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
8829
8830 let bucket = format!("dm-multipart-target-{}", uuid::Uuid::new_v4());
8831 let object = "object.bin";
8832 let source_version = uuid::Uuid::new_v4();
8833 let other_version = uuid::Uuid::new_v4();
8834 let source_mod_time = OffsetDateTime::UNIX_EPOCH + time::Duration::SECOND;
8835 store
8836 .make_bucket(&bucket, &MakeBucketOptions::default())
8837 .await
8838 .expect("create multipart selected-target bucket");
8839 let first_part_size = 5 * 1024 * 1024;
8840 let mut source_body = vec![b'a'; first_part_size];
8841 source_body.push(b'b');
8842 for (pool_idx, version_id, body, mod_time) in [
8843 (
8844 1,
8845 other_version,
8846 b"newer different version".to_vec(),
8847 source_mod_time + time::Duration::seconds(2),
8848 ),
8849 (2, source_version, b"conflicting exact version".to_vec(), source_mod_time),
8850 ] {
8851 let mut reader = PutObjReader::from_vec(body);
8852 store.pools[pool_idx]
8853 .put_object(
8854 &bucket,
8855 object,
8856 &mut reader,
8857 &ObjectOptions {
8858 versioned: true,
8859 version_id: Some(version_id.to_string()),
8860 mod_time: Some(mod_time),
8861 ..Default::default()
8862 },
8863 )
8864 .await
8865 .expect("seed multipart data movement pool");
8866 }
8867 let source_reader = seed_data_movement_source_reader(
8868 &store,
8869 &bucket,
8870 object,
8871 DataMovementSourceSeed {
8872 pool_idx: 0,
8873 version_id: Some(source_version),
8874 body: source_body,
8875 multipart_split: Some(first_part_size),
8876 mod_time: source_mod_time,
8877 user_defined: HashMap::new(),
8878 },
8879 )
8880 .await;
8881 *store.rebalance_meta.write().await = Some(active_rebalance_meta_for_pool(store.pools.len(), 0));
8882
8883 let completion_barrier = crate::store::multipart::DataMovementMultipartCompletionBarrier::install(&bucket);
8884 let migration_store = store.clone();
8885 let migration_bucket = bucket.clone();
8886 let migration = tokio::spawn(async move {
8887 crate::data_movement::migrate_object(
8888 migration_store,
8889 0,
8890 migration_bucket,
8891 source_reader,
8892 None,
8893 "test_multipart_selected_target",
8894 )
8895 .await
8896 });
8897 completion_barrier.wait_until_paused().await;
8898 let unrelated_set = store.pools[1].get_disks_by_key(object);
8899 let original_disks = {
8900 let mut disks = unrelated_set.disks.write().await;
8901 let original = disks.clone();
8902 for disk in disks.iter_mut().take(3) {
8903 *disk = None;
8904 }
8905 original
8906 };
8907 drop(completion_barrier);
8908 let err = migration
8909 .await
8910 .expect("multipart migration task should join")
8911 .expect_err("the exact-version target conflict must not be bypassed by a newer different version");
8912 *unrelated_set.disks.write().await = original_disks;
8913 let rendered = err.to_string();
8914 assert!(
8915 rendered.contains("complete_multipart_upload"),
8916 "the selected target should be reached despite an unrelated degraded pool: {rendered}"
8917 );
8918 assert!(
8919 !rendered.contains("put_object_part failed"),
8920 "part upload must not scan the unrelated degraded pool: {rendered}"
8921 );
8922
8923 let unexpected_version = store.pools[1]
8924 .get_object_info(
8925 &bucket,
8926 object,
8927 &ObjectOptions {
8928 versioned: true,
8929 version_id: Some(source_version.to_string()),
8930 ..Default::default()
8931 },
8932 )
8933 .await
8934 .expect_err("multipart migration must not duplicate the source version into the latest-version pool");
8935 assert!(matches!(
8936 unexpected_version,
8937 StorageError::ObjectNotFound(_, _) | StorageError::VersionNotFound(_, _, _)
8938 ));
8939 }
8940
8941 #[cfg(feature = "test-util")]
8942 #[tokio::test]
8943 #[serial_test::serial(storage_class_env)]
8944 async fn decommission_durable_ilm_target_read_error_is_not_masked_by_peer_success() {
8945 let temp_dir = tempfile::tempdir().expect("create temp store dir");
8946 let (_ctx, store, _shutdown) =
8947 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "durable-ilm-target-read-error", &[4, 4, 4]))
8948 .await;
8949 let job_id = uuid::Uuid::new_v4();
8950 let job =
8951 ManualTransitionJobRecord::new(job_id, "manual-target-read-error", &ManualTransitionRunOptions::default(), "owner");
8952 let path = manual_transition_job_record_object_name(job_id).expect("manual job path should build");
8953 let data = job.encode().expect("manual job should encode");
8954 for pool in &store.pools {
8955 com::save_config(pool.clone(), &path, data.clone())
8956 .await
8957 .expect("manual job fixture should persist in every pool");
8958 }
8959 store.pool_meta.write().await.pools[0].decommission = Some(PoolDecommissionInfo {
8960 start_time: Some(OffsetDateTime::now_utc()),
8961 ..Default::default()
8962 });
8963
8964 let failing_target = store.pools[2].get_disks_by_key(&path);
8965 let original_disks = {
8966 let mut disks = failing_target.disks.write().await;
8967 let original = disks.clone();
8968 for disk in disks.iter_mut().take(3) {
8969 *disk = None;
8970 }
8971 original
8972 };
8973 let error = store
8974 .verify_and_cleanup_decommissioned_durable_ilm_record_for_test(0, store.pools[0].get_disks_by_key(&path), &path)
8975 .await
8976 .expect_err("one target read-quorum error must fail closed despite another target success")
8977 .to_string();
8978 *failing_target.disks.write().await = original_disks;
8979
8980 assert!(error.contains(&path));
8981 assert!(error.contains("pool 2"));
8982 assert_eq!(
8983 com::read_config(store.pools[0].clone(), &path)
8984 .await
8985 .expect("target read error must retain the source"),
8986 data
8987 );
8988 assert_eq!(
8989 store
8990 .decommission_durable_ilm_receipt_count_for_test(0)
8991 .await
8992 .expect("failed target verification should not create a receipt"),
8993 0
8994 );
8995 }
8996
8997 #[cfg(feature = "test-util")]
8998 #[tokio::test]
8999 #[serial_test::serial(storage_class_env)]
9000 async fn decommission_durable_ilm_terminal_receipt_recovers_failed_source_cleanup() {
9001 let temp_dir = tempfile::tempdir().expect("create temp store dir");
9002 let (ctx, store, _shutdown) =
9003 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "durable-ilm-terminal-receipt", &[4, 4])).await;
9004 let tier_name = "DECOMMISSION-RECEIPT";
9005 let backend = register_transition_reconcile_test_tier(&ctx.tier_config_mgr(), tier_name).await;
9006 let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
9007 .await
9008 .expect("tier lease should resolve")
9009 .backend_identity();
9010 let entry = Jentry {
9011 persisted_version: 0,
9012 obj_name: "receipt-recovery-object".to_string(),
9013 version_id: "receipt-recovery-version".to_string(),
9014 tier_name: tier_name.to_string(),
9015 backend_identity: Some(backend_identity),
9016 version_id_exact: true,
9017 version_state: rustfs_filemeta::TransitionVersionState::Exact,
9018 state: TierDeleteJournalState::Committed,
9019 source: None,
9020 dispatch: None,
9021 };
9022 let path = tier_delete_journal_object_name(&entry);
9023 let data = encode_tier_delete_journal_entry(&entry).expect("tier journal should encode");
9024 com::save_config(store.pools[0].clone(), &path, data.clone())
9025 .await
9026 .expect("source tier journal should persist");
9027 com::save_config(store.pools[1].clone(), &path, data.clone())
9028 .await
9029 .expect("target tier journal should persist");
9030 let active_pool_meta = {
9031 let mut pool_meta = store.pool_meta.write().await;
9032 pool_meta.pools[0].decommission = Some(PoolDecommissionInfo {
9033 start_time: Some(OffsetDateTime::now_utc()),
9034 ..Default::default()
9035 });
9036 pool_meta.clone()
9037 };
9038 active_pool_meta
9039 .save(store.pools.clone())
9040 .await
9041 .expect("active decommission run identity should persist");
9042
9043 let source_set = store.pools[0].get_disks_by_key(&path);
9044 let barrier = SourceCleanupDeleteBarrier::install(RUSTFS_META_BUCKET, &path);
9045 let cleanup_store = store.clone();
9046 let cleanup_set = source_set.clone();
9047 let cleanup_path = path.clone();
9048 let cleanup = tokio::spawn(async move {
9049 cleanup_store
9050 .verify_and_cleanup_decommissioned_durable_ilm_record_for_test(0, cleanup_set, &cleanup_path)
9051 .await
9052 });
9053 barrier.wait_until_paused().await;
9054 let original_source_disks = {
9055 let mut disks = source_set.disks.write().await;
9056 let original = disks.clone();
9057 for disk in disks.iter_mut().take(3) {
9058 *disk = None;
9059 }
9060 original
9061 };
9062 barrier.release();
9063 let cleanup_error = cleanup
9064 .await
9065 .expect("source cleanup task should not panic")
9066 .expect_err("injected source delete quorum failure must fail cleanup")
9067 .to_string();
9068 *source_set.disks.write().await = original_source_disks;
9069 drop(barrier);
9070
9071 assert!(cleanup_error.contains("source durable ILM cleanup failed"));
9072 assert_eq!(
9073 store
9074 .decommission_durable_ilm_receipt_count_for_test(0)
9075 .await
9076 .expect("receipt should persist before source cleanup"),
9077 1
9078 );
9079 assert_eq!(
9080 com::read_config(store.pools[0].clone(), &path)
9081 .await
9082 .expect("failed cleanup must retain the source"),
9083 data
9084 );
9085
9086 let mut restarted_pool_meta = PoolMeta::default();
9087 restarted_pool_meta
9088 .load(store.pools[0].clone(), store.pools.clone())
9089 .await
9090 .expect("decommission run identity should reload after restart");
9091 *store.pool_meta.write().await = restarted_pool_meta;
9092 let stats = recover_tier_delete_journal_entries(store.clone(), 100, None)
9093 .await
9094 .expect("target recovery should commit terminal proof and delete the target");
9095 assert_eq!((stats.scanned, stats.deleted, stats.failed), (1, 1, 0));
9096 assert!(matches!(
9097 com::read_config(store.pools[1].clone(), &path).await,
9098 Err(Error::ConfigNotFound)
9099 ));
9100 assert_eq!(
9101 com::read_config(store.pools[0].clone(), &path)
9102 .await
9103 .expect("target recovery must not delete the decommission source"),
9104 data
9105 );
9106
9107 store
9108 .verify_and_cleanup_decommissioned_durable_ilm_record_for_test(0, source_set, &path)
9109 .await
9110 .expect("terminal receipt should authorize cleanup after target deletion");
9111 assert!(matches!(
9112 com::read_config(store.pools[0].clone(), &path).await,
9113 Err(Error::ConfigNotFound)
9114 ));
9115 assert!(backend.remove_versions().await.contains(&(entry.obj_name, entry.version_id)));
9116 }
9117
9118 #[cfg(feature = "test-util")]
9119 #[tokio::test]
9120 #[serial_test::serial(storage_class_env)]
9121 async fn decommission_final_sweep_blocks_cancel_until_source_cleanup_finishes() {
9122 let temp_dir = tempfile::tempdir().expect("create final sweep gate store dir");
9123 let (_ctx, store, _shutdown) =
9124 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "durable-ilm-final-sweep-gate", &[4, 4])).await;
9125 let job_id = uuid::Uuid::new_v4();
9126 let job = ManualTransitionJobRecord::new(job_id, "final-sweep-gate", &ManualTransitionRunOptions::default(), "owner");
9127 let path = manual_transition_job_record_object_name(job_id).expect("manual job path should build");
9128 let data = job.encode().expect("manual job should encode");
9129 for pool in &store.pools {
9130 com::save_config(pool.clone(), &path, data.clone())
9131 .await
9132 .expect("manual job fixture should persist in both pools");
9133 }
9134 let active_pool_meta = {
9135 let mut pool_meta = store.pool_meta.write().await;
9136 pool_meta.pools[0].decommission = Some(PoolDecommissionInfo {
9137 start_time: Some(OffsetDateTime::now_utc()),
9138 ..Default::default()
9139 });
9140 pool_meta.clone()
9141 };
9142 active_pool_meta
9143 .save(store.pools.clone())
9144 .await
9145 .expect("active decommission run identity should persist");
9146
9147 let barrier = SourceCleanupDeleteBarrier::install(RUSTFS_META_BUCKET, &path);
9148 let final_sweep = tokio::spawn({
9149 let store = store.clone();
9150 async move { store.check_after_decommission_for_test(0).await }
9151 });
9152 barrier.wait_until_paused().await;
9153
9154 let mut cancel = tokio::spawn({
9155 let store = store.clone();
9156 async move { store.decommission_cancel(0).await }
9157 });
9158 assert!(
9159 tokio::time::timeout(Duration::from_millis(100), &mut cancel).await.is_err(),
9160 "cancel must wait for the final sweep source cleanup"
9161 );
9162 {
9163 let pool_meta = store.pool_meta.read().await;
9164 let decommission = pool_meta.pools[0]
9165 .decommission
9166 .as_ref()
9167 .expect("decommission state should remain present");
9168 assert!(
9169 decommission.canceled,
9170 "cancel must publish its durable terminal state before waiting for the final sweep"
9171 );
9172 assert!(
9173 decommission.start_time.is_none(),
9174 "the durable cancel state must clear the active run identity before quiescence"
9175 );
9176 }
9177
9178 barrier.release();
9179 final_sweep
9180 .await
9181 .expect("final sweep task should not panic")
9182 .expect("final sweep should finish after the barrier releases");
9183 cancel
9184 .await
9185 .expect("cancel task should not panic")
9186 .expect("cancel should complete after the final sweep releases the operation gate");
9187 let pool_meta = store.pool_meta.read().await;
9188 let decommission = pool_meta.pools[0]
9189 .decommission
9190 .as_ref()
9191 .expect("decommission state should remain present");
9192 assert!(decommission.canceled);
9193 assert!(decommission.start_time.is_none());
9194 }
9195
9196 #[cfg(feature = "test-util")]
9197 #[tokio::test]
9198 #[serial_test::serial(storage_class_env)]
9199 async fn decommission_durable_ilm_recovery_keeps_multiple_active_sources() {
9200 let temp_dir = tempfile::tempdir().expect("create multi-source recovery store dir");
9201 let (ctx, store, _shutdown) = without_storage_class_env(build_isolated_test_store(
9202 temp_dir.path(),
9203 "durable-ilm-multi-source-recovery",
9204 &[4, 4, 4],
9205 ))
9206 .await;
9207 let tier_name = "DECOMMISSION-MULTI-SOURCE";
9208 let backend = register_transition_reconcile_test_tier(&ctx.tier_config_mgr(), tier_name).await;
9209 let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
9210 .await
9211 .expect("tier lease should resolve")
9212 .backend_identity();
9213 let entry = Jentry {
9214 persisted_version: 0,
9215 obj_name: "multi-source-recovery-object".to_string(),
9216 version_id: "multi-source-recovery-version".to_string(),
9217 tier_name: tier_name.to_string(),
9218 backend_identity: Some(backend_identity),
9219 version_id_exact: true,
9220 version_state: rustfs_filemeta::TransitionVersionState::Exact,
9221 state: TierDeleteJournalState::Committed,
9222 source: None,
9223 dispatch: None,
9224 };
9225 let path = tier_delete_journal_object_name(&entry);
9226 let data = encode_tier_delete_journal_entry(&entry).expect("tier journal should encode");
9227 for pool in &store.pools {
9228 com::save_config(pool.clone(), &path, data.clone())
9229 .await
9230 .expect("source and target tier journals should persist");
9231 }
9232
9233 let active_pool_meta = {
9234 let mut pool_meta = store.pool_meta.write().await;
9235 let start_time = OffsetDateTime::now_utc();
9236 for pool_idx in [0, 1] {
9237 pool_meta.pools[pool_idx].decommission = Some(PoolDecommissionInfo {
9238 start_time: Some(start_time),
9239 ..Default::default()
9240 });
9241 }
9242 pool_meta.clone()
9243 };
9244 active_pool_meta
9245 .save(store.pools.clone())
9246 .await
9247 .expect("multiple active decommission runs should persist");
9248 let mut restarted_pool_meta = PoolMeta::default();
9249 restarted_pool_meta
9250 .load(store.pools[0].clone(), store.pools.clone())
9251 .await
9252 .expect("multiple active decommission runs should reload");
9253 *store.pool_meta.write().await = restarted_pool_meta;
9254
9255 let record = validate_durable_ilm_record(&path, &data).expect("tier journal should validate");
9256 let source_zero_receipt = store
9257 .persist_decommission_durable_ilm_receipt_for_test(0, 1, &path, &record, false)
9258 .await
9259 .expect("source pool zero receipt should persist on the other active source");
9260 let source_one_receipt = store
9261 .persist_decommission_durable_ilm_receipt_for_test(1, 0, &path, &record, false)
9262 .await
9263 .expect("source pool one receipt should persist on the other active source");
9264 assert_ne!(
9265 source_zero_receipt, source_one_receipt,
9266 "active source runs must have distinct receipt paths"
9267 );
9268
9269 let stats = recover_tier_delete_journal_entries(store.clone(), 100, None)
9270 .await
9271 .expect("cross-source receipts should not remove active source journals");
9272 assert_eq!((stats.scanned, stats.deleted, stats.failed), (1, 1, 0));
9273 for pool in &store.pools {
9274 assert_eq!(
9275 com::read_config(pool.clone(), &path)
9276 .await
9277 .expect("cross-source receipts alone must retain every journal copy"),
9278 data
9279 );
9280 }
9281
9282 store
9283 .persist_decommission_durable_ilm_receipt_for_test(0, 2, &path, &record, false)
9284 .await
9285 .expect("source pool zero receipt should persist on the target");
9286 store
9287 .persist_decommission_durable_ilm_receipt_for_test(1, 2, &path, &record, false)
9288 .await
9289 .expect("source pool one receipt should persist on the target");
9290
9291 let stats = recover_tier_delete_journal_entries(store.clone(), 100, None)
9292 .await
9293 .expect("multi-source tier journal recovery should complete");
9294 assert_eq!((stats.scanned, stats.deleted, stats.failed), (1, 1, 0));
9295 assert_eq!(
9296 com::read_config(store.pools[0].clone(), &path)
9297 .await
9298 .expect("first active source must remain after target recovery"),
9299 data
9300 );
9301 assert_eq!(
9302 com::read_config(store.pools[1].clone(), &path)
9303 .await
9304 .expect("second active source must remain after target recovery"),
9305 data
9306 );
9307 assert!(matches!(
9308 com::read_config(store.pools[2].clone(), &path).await,
9309 Err(Error::ConfigNotFound)
9310 ));
9311 assert!(backend.remove_versions().await.contains(&(entry.obj_name, entry.version_id)));
9312 }
9313
9314 #[cfg(feature = "test-util")]
9315 #[tokio::test]
9316 #[serial_test::serial(storage_class_env)]
9317 async fn decommission_durable_ilm_receipt_pagination_fails_closed_on_second_page() {
9318 const RECEIPT_COUNT: usize = 1001;
9319
9320 let temp_dir = tempfile::tempdir().expect("create paginated receipt store dir");
9321 let (_ctx, store, _shutdown) =
9322 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "durable-ilm-receipt-pages", &[4, 4])).await;
9323 store.pool_meta.write().await.pools[0].decommission = Some(PoolDecommissionInfo {
9324 start_time: Some(OffsetDateTime::now_utc()),
9325 ..Default::default()
9326 });
9327
9328 futures::stream::iter(0..RECEIPT_COUNT)
9329 .map(|index| {
9330 let store = store.clone();
9331 async move {
9332 let id = format!("{index:064x}");
9333 let source_path = format!("ilm/tier-delete-journal/{id}.json");
9334 let record = ValidatedDurableIlmRecord {
9335 namespace: "tier-delete-journal",
9336 id_kind: "operation_id",
9337 id,
9338 checkpoint: DurableIlmRecordCheckpoint::TierDeleteJournal {
9339 content_sha256: format!("{:064x}", index + RECEIPT_COUNT),
9340 identity_sha256: "f".repeat(64),
9341 committed: false,
9342 dispatch_identity_sha256: None,
9343 state: None,
9344 },
9345 };
9346 store
9347 .persist_decommission_durable_ilm_receipt_for_test(0, 0, &source_path, &record, true)
9348 .await?;
9349 store
9350 .persist_decommission_durable_ilm_receipt_for_test(0, 1, &source_path, &record, true)
9351 .await?;
9352 Ok::<(), Error>(())
9353 }
9354 })
9355 .buffer_unordered(32)
9356 .try_collect::<Vec<_>>()
9357 .await
9358 .expect("more than one receipt page should persist");
9359 store
9360 .persist_decommission_durable_ilm_manifest_for_test(0)
9361 .await
9362 .expect("paginated source receipts should produce a manifest");
9363
9364 let target_receipts = store
9365 .decommission_durable_ilm_receipt_paths_for_test(0)
9366 .await
9367 .expect("paginated target receipts should list");
9368 assert_eq!(target_receipts.len(), RECEIPT_COUNT);
9369 let (target_pool_idx, second_page_path) = target_receipts
9370 .get(1000)
9371 .cloned()
9372 .expect("the real 1000-item page boundary should expose a second page receipt");
9373 let receipt_bytes = com::read_config(store.pools[target_pool_idx].clone(), &second_page_path)
9374 .await
9375 .expect("second page receipt should be readable");
9376
9377 com::delete_config(store.pools[target_pool_idx].clone(), &second_page_path)
9378 .await
9379 .expect("second page receipt should delete");
9380 let missing = store
9381 .complete_decommission(0)
9382 .await
9383 .expect_err("a missing second page receipt must block completion")
9384 .to_string();
9385 assert!(missing.contains(&second_page_path));
9386 assert!(
9387 !store.pool_meta.read().await.pools[0]
9388 .decommission
9389 .as_ref()
9390 .expect("source pool should remain in decommission")
9391 .complete
9392 );
9393 assert!(com::read_config(store.pools[0].clone(), &second_page_path).await.is_ok());
9394
9395 let full_tail = ObjectOptions {
9396 max_parity: true,
9397 write_completion: crate::object_api::WriteCompletion::TailDrained,
9398 ..Default::default()
9399 };
9400 com::save_config_with_opts(store.pools[target_pool_idx].clone(), &second_page_path, receipt_bytes.clone(), &full_tail)
9401 .await
9402 .expect("second page receipt should restore");
9403 com::save_config_with_opts(store.pools[target_pool_idx].clone(), &second_page_path, b"{corrupt".to_vec(), &full_tail)
9404 .await
9405 .expect("second page receipt should corrupt deterministically");
9406 let corrupt = store
9407 .complete_decommission(0)
9408 .await
9409 .expect_err("a corrupt second page receipt must block completion")
9410 .to_string();
9411 assert!(corrupt.contains(&second_page_path));
9412 assert!(corrupt.contains("invalid"));
9413 assert!(
9414 !store.pool_meta.read().await.pools[0]
9415 .decommission
9416 .as_ref()
9417 .expect("source pool should remain in decommission")
9418 .complete
9419 );
9420 assert!(com::read_config(store.pools[0].clone(), &second_page_path).await.is_ok());
9421 }
9422
9423 #[cfg(feature = "test-util")]
9424 #[test]
9425 #[serial_test::serial(storage_class_env)]
9426 fn decommission_migrates_and_verifies_registered_durable_ilm_records() {
9427 std::thread::Builder::new()
9428 .name("durable-ilm-decommission-test".to_string())
9429 .stack_size(16 * 1024 * 1024)
9430 .spawn(|| {
9431 let runtime = tokio::runtime::Builder::new_multi_thread()
9432 .enable_all()
9433 .worker_threads(2)
9434 .build()
9435 .expect("durable ILM decommission runtime should build");
9436 runtime.block_on(decommission_migrates_and_verifies_registered_durable_ilm_records_scenario());
9437 })
9438 .expect("durable ILM decommission scenario thread should spawn")
9439 .join()
9440 .expect("durable ILM decommission scenario should not panic");
9441 }
9442
9443 #[cfg(feature = "test-util")]
9444 async fn decommission_migrates_and_verifies_registered_durable_ilm_records_scenario() {
9445 let temp_dir = tempfile::tempdir().expect("create temp store dir");
9446 let (ctx, store, _shutdown) =
9447 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "durable-ilm-decommission", &[4, 4])).await;
9448
9449 let tier_name = "DECOMMISSION-ILM";
9450 let backend = register_transition_reconcile_test_tier(&ctx.tier_config_mgr(), tier_name).await;
9451 let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
9452 .await
9453 .expect("tier lease should resolve")
9454 .backend_identity();
9455 let tier_entry = Jentry {
9456 persisted_version: 0,
9457 obj_name: "decommissioned-remote-object".to_string(),
9458 version_id: "decommissioned-remote-version".to_string(),
9459 tier_name: tier_name.to_string(),
9460 backend_identity: Some(backend_identity),
9461 version_id_exact: true,
9462 version_state: rustfs_filemeta::TransitionVersionState::Exact,
9463 state: TierDeleteJournalState::Committed,
9464 source: None,
9465 dispatch: None,
9466 };
9467 let tier_path = tier_delete_journal_object_name(&tier_entry);
9468 let tier_bytes = encode_tier_delete_journal_entry(&tier_entry).expect("tier journal should encode");
9469
9470 let mut transaction = TransitionTransaction::new(TransitionTransactionInit {
9471 deployment_id: ctx.deployment_id().expect("test store should initialize deployment id"),
9472 transaction_id: uuid::Uuid::new_v4(),
9473 owner_epoch: uuid::Uuid::new_v4(),
9474 write_id: uuid::Uuid::new_v4(),
9475 source: TransitionSourceIdentity {
9476 bucket: "source-bucket".to_string(),
9477 object: "source-object".to_string(),
9478 version_id: Some(uuid::Uuid::new_v4()),
9479 data_dir: uuid::Uuid::new_v4(),
9480 mod_time_unix_nanos: 1_770_000_000_000_000_000,
9481 size: 42,
9482 etag: "source-etag".to_string(),
9483 version_mode: TransitionSourceVersionMode::Versioned,
9484 },
9485 tier_name: tier_name.to_string(),
9486 backend_fingerprint: backend_identity,
9487 not_after_unix_nanos: 1_780_000_000_000_000_000,
9488 })
9489 .expect("transition transaction should build");
9490 let regressed_transaction = transaction.clone();
9491 transaction
9492 .advance(transaction.fence(), TransitionTransactionState::UploadOutcomeUnknown, None)
9493 .expect("transition transaction should advance before migration");
9494 let transaction_path = transition_transaction_record_object_name(transaction.transaction_id)
9495 .expect("transition transaction path should build");
9496 let transaction_bytes = transaction.encode().expect("transition transaction should encode");
9497
9498 let manual_job_id = uuid::Uuid::new_v4();
9499 let manual_bucket = format!("manual-decommission-{}", manual_job_id.simple());
9500 let manual_options = ManualTransitionRunOptions {
9501 prefix: "logs/".to_string(),
9502 tier: Some(tier_name.to_string()),
9503 ..Default::default()
9504 };
9505 let mut manual_job = ManualTransitionJobRecord::new(manual_job_id, &manual_bucket, &manual_options, "old-owner");
9506 manual_job.scan_completed = true;
9507 manual_job.report.enqueued = 1;
9508 manual_job.lease_expires_at_unix_nanos = 0;
9509 let manual_scope = ManualTransitionScopeAdmission::from_job(&manual_job);
9510 let task_key = manual_transition_worker_result_task_key(&manual_bucket, "logs/a", None);
9511 let manual_task = ManualTransitionTaskRecord::new(manual_job_id, &task_key, &manual_bucket, "logs/a", None, tier_name);
9512 let manual_result =
9513 ManualTransitionWorkerResultRecord::new(manual_job_id, &task_key, ManualTransitionWorkerResult::Completed);
9514
9515 let manual_job_path = manual_transition_job_record_object_name(manual_job_id).expect("manual job path should build");
9516 let manual_scope_path =
9517 manual_transition_scope_record_object_name(&manual_scope.scope_key).expect("manual scope path should build");
9518 let manual_task_path =
9519 manual_transition_task_object_name(manual_job_id, &task_key).expect("manual task path should build");
9520 let manual_result_path = manual_transition_worker_result_object_name(manual_job_id, &task_key)
9521 .expect("manual worker result path should build");
9522 let manual_job_bytes = manual_job.encode().expect("manual job should encode");
9523 let manual_scope_bytes = serde_json::to_vec(&manual_scope).expect("manual scope should encode");
9524 let manual_task_bytes = manual_task.encode().expect("manual task should encode");
9525 let manual_result_bytes = manual_result.encode().expect("manual result should encode");
9526
9527 let records = vec![
9528 (tier_path.clone(), tier_bytes.clone()),
9529 (transaction_path.clone(), transaction_bytes.clone()),
9530 (manual_job_path.clone(), manual_job_bytes.clone()),
9531 (manual_scope_path.clone(), manual_scope_bytes.clone()),
9532 (manual_task_path.clone(), manual_task_bytes.clone()),
9533 (manual_result_path.clone(), manual_result_bytes.clone()),
9534 ];
9535 for (path, data) in &records {
9536 com::save_config(store.pools[0].clone(), path, data.clone())
9537 .await
9538 .expect("durable ILM source record should persist");
9539 }
9540
9541 let legacy_queue = [com::CONFIG_PREFIX, BUCKET_META_PREFIX]
9542 .into_iter()
9543 .map(|prefix| {
9544 DecomBucketInfo {
9545 name: RUSTFS_META_BUCKET.to_string(),
9546 prefix: prefix.to_string(),
9547 }
9548 .to_string()
9549 })
9550 .collect();
9551 let legacy_pool_meta = {
9552 let mut pool_meta = store.pool_meta.write().await;
9553 pool_meta.pools[0].decommission = Some(PoolDecommissionInfo {
9554 queued: true,
9555 queued_buckets: legacy_queue,
9556 ..Default::default()
9557 });
9558 pool_meta.clone()
9559 };
9560 legacy_pool_meta
9561 .save(store.pools.clone())
9562 .await
9563 .expect("legacy decommission queue should persist before restart");
9564 let mut restarted_pool_meta = PoolMeta::default();
9565 restarted_pool_meta
9566 .load(store.pools[0].clone(), store.pools.clone())
9567 .await
9568 .expect("legacy decommission queue should reload after restart");
9569 *store.pool_meta.write().await = restarted_pool_meta;
9570 set_test_decommission_capacity_override(&store, 0);
9571 store
9572 .promote_queued_decommission_for_test(0)
9573 .await
9574 .expect("legacy queued decommission should resume");
9575 let expected_ilm_queue = DecomBucketInfo {
9576 name: RUSTFS_META_BUCKET.to_string(),
9577 prefix: ILM_META_PREFIX.to_string(),
9578 }
9579 .to_string();
9580 {
9581 let pool_meta = store.pool_meta.read().await;
9582 let decommission = pool_meta.pools[0]
9583 .decommission
9584 .as_ref()
9585 .expect("decommission state should remain present");
9586 assert!(!decommission.queued);
9587 assert!(decommission.queued_buckets.contains(&expected_ilm_queue));
9588 }
9589
9590 let ilm_bucket = DecomBucketInfo {
9591 name: RUSTFS_META_BUCKET.to_string(),
9592 prefix: ILM_META_PREFIX.to_string(),
9593 };
9594 for _ in 0..2 {
9595 store
9596 .decommission_pool_for_test(CancellationToken::new(), 0, store.pools[0].clone(), ilm_bucket.clone())
9597 .await
9598 .expect("durable ILM decommission should be idempotent");
9599 }
9600 for (path, expected) in &records {
9601 assert_eq!(
9602 com::read_config(store.pools[0].clone(), path)
9603 .await
9604 .expect("source should remain until the final sweep"),
9605 *expected
9606 );
9607 assert_eq!(
9608 com::read_config(store.pools[1].clone(), path)
9609 .await
9610 .expect("target should contain the migrated record"),
9611 *expected
9612 );
9613 }
9614 assert_eq!(
9615 store
9616 .decommission_durable_ilm_receipt_count_for_test(0)
9617 .await
9618 .expect("no receipt should exist before the final sweep"),
9619 0
9620 );
9621 let isolated_tier_stats = recover_tier_delete_journal_entries(store.clone(), 100, None)
9622 .await
9623 .expect("tier recovery should retain a terminal record until its receipt is committed");
9624 assert!(isolated_tier_stats.scanned >= 1);
9625 assert_eq!(isolated_tier_stats.deleted, 0);
9626 assert!(isolated_tier_stats.failed >= 1);
9627 assert_eq!(
9628 com::read_config(store.pools[1].clone(), &tier_path)
9629 .await
9630 .expect("receipt isolation must retain the target tier journal"),
9631 tier_bytes
9632 );
9633
9634 com::delete_config(store.pools[1].clone(), &manual_job_path)
9635 .await
9636 .expect("target manual job should delete");
9637 let missing = store
9638 .verify_and_cleanup_decommissioned_durable_ilm_record_for_test(
9639 0,
9640 store.pools[0].get_disks_by_key(&manual_job_path),
9641 &manual_job_path,
9642 )
9643 .await
9644 .expect_err("missing target must block source cleanup");
9645 let missing = missing.to_string();
9646 assert!(missing.contains(&manual_job_path) && missing.contains(&manual_job_id.to_string()));
9647 assert_eq!(
9648 com::read_config(store.pools[0].clone(), &manual_job_path)
9649 .await
9650 .expect("missing target must retain source"),
9651 manual_job_bytes
9652 );
9653 com::save_config(store.pools[1].clone(), &manual_job_path, manual_job_bytes.clone())
9654 .await
9655 .expect("target manual job should restore");
9656
9657 com::save_config(store.pools[1].clone(), &transaction_path, b"{corrupt".to_vec())
9658 .await
9659 .expect("target transaction should corrupt deterministically");
9660 let corrupt = store
9661 .verify_and_cleanup_decommissioned_durable_ilm_record_for_test(
9662 0,
9663 store.pools[0].get_disks_by_key(&transaction_path),
9664 &transaction_path,
9665 )
9666 .await
9667 .expect_err("corrupt target must block source cleanup");
9668 let corrupt = corrupt.to_string();
9669 assert!(corrupt.contains(&transaction_path) && corrupt.contains(&transaction.transaction_id.to_string()));
9670 assert_eq!(
9671 com::read_config(store.pools[0].clone(), &transaction_path)
9672 .await
9673 .expect("corrupt target must retain source"),
9674 transaction_bytes
9675 );
9676 com::save_config(store.pools[1].clone(), &transaction_path, transaction_bytes.clone())
9677 .await
9678 .expect("target transaction should restore");
9679
9680 com::save_config(store.pools[1].clone(), &manual_scope_path, manual_scope_bytes.clone())
9681 .await
9682 .expect("target scope rewrite should invalidate cached metadata before the quorum check");
9683 let target_scope_set = store.pools[1].get_disks_by_key(&manual_scope_path);
9684 let original_target_scope_disks = {
9685 let mut disks = target_scope_set.disks.write().await;
9686 let original = disks.clone();
9687 for disk in disks.iter_mut().take(3) {
9688 *disk = None;
9689 }
9690 original
9691 };
9692 let quorum_error = store
9693 .verify_and_cleanup_decommissioned_durable_ilm_record_for_test(
9694 0,
9695 store.pools[0].get_disks_by_key(&manual_scope_path),
9696 &manual_scope_path,
9697 )
9698 .await
9699 .expect_err("target below read quorum must block source cleanup");
9700 *target_scope_set.disks.write().await = original_target_scope_disks;
9701 let quorum_error = quorum_error.to_string();
9702 assert!(quorum_error.contains(&manual_scope_path) && quorum_error.contains(&manual_job_id.to_string()));
9703 assert!(com::read_config(store.pools[0].clone(), &manual_scope_path).await.is_ok());
9704
9705 com::save_config(store.pools[1].clone(), &manual_task_path, manual_task_bytes.clone())
9706 .await
9707 .expect("target task rewrite should invalidate cached metadata before the quorum check");
9708 let manual_task_record =
9709 validate_durable_ilm_record(&manual_task_path, &manual_task_bytes).expect("manual task should validate");
9710 let manual_task_receipt_path = store
9711 .decommission_durable_ilm_receipt_path_for_test(0, &manual_task_path, &manual_task_record)
9712 .await
9713 .expect("manual task receipt path should resolve");
9714 let target_task_set = store.pools[1].get_disks_by_key(&manual_task_receipt_path);
9715 let original_target_task_disks = {
9716 let mut disks = target_task_set.disks.write().await;
9717 let original = disks.clone();
9718 for disk in disks.iter_mut().take(2) {
9719 *disk = None;
9720 }
9721 original
9722 };
9723 let receipt_quorum_error = store
9724 .verify_and_cleanup_decommissioned_durable_ilm_record_for_test(
9725 0,
9726 store.pools[0].get_disks_by_key(&manual_task_path),
9727 &manual_task_path,
9728 )
9729 .await
9730 .expect_err("target read quorum without receipt write quorum must retain the source");
9731 *target_task_set.disks.write().await = original_target_task_disks;
9732 let receipt_quorum_error = receipt_quorum_error.to_string();
9733 assert!(receipt_quorum_error.contains("receipt"));
9734 assert!(receipt_quorum_error.contains(&manual_task_path));
9735 assert!(receipt_quorum_error.contains(&manual_job_id.to_string()));
9736 assert!(com::read_config(store.pools[0].clone(), &manual_task_path).await.is_ok());
9737 store
9738 .verify_and_cleanup_decommissioned_durable_ilm_record_for_test(
9739 0,
9740 store.pools[0].get_disks_by_key(&manual_task_path),
9741 &manual_task_path,
9742 )
9743 .await
9744 .expect("healthy target should persist the receipt before source cleanup");
9745
9746 let unknown_path = "ilm/future-durable/jobs/one.json";
9747 com::save_config(store.pools[0].clone(), unknown_path, b"{}".to_vec())
9748 .await
9749 .expect("unknown durable ILM record should persist for the guard test");
9750 let unknown_migration = store
9751 .decommission_pool_for_test(CancellationToken::new(), 0, store.pools[0].clone(), ilm_bucket)
9752 .await
9753 .expect_err("unregistered durable ILM namespace must block migration");
9754 assert!(unknown_migration.to_string().contains(unknown_path));
9755 let unknown_final_sweep = store
9756 .check_after_decommission_for_test(0)
9757 .await
9758 .expect_err("unregistered durable ILM namespace must block completion");
9759 assert!(unknown_final_sweep.to_string().contains(unknown_path));
9760 com::delete_config(store.pools[0].clone(), unknown_path)
9761 .await
9762 .expect("unknown guard fixture should be removed before the successful final sweep");
9763
9764 store
9765 .check_after_decommission_for_test(0)
9766 .await
9767 .expect("production final sweep should validate every target before cleanup");
9768 assert_eq!(
9769 store
9770 .decommission_durable_ilm_receipt_count_for_test(0)
9771 .await
9772 .expect("durable ILM receipts should be listable"),
9773 records.len(),
9774 "every cleaned source record must have a durable validation receipt"
9775 );
9776 for (path, expected) in &records {
9777 assert!(
9778 matches!(com::read_config(store.pools[0].clone(), path).await, Err(Error::ConfigNotFound)),
9779 "final sweep should remove the validated source `{path}`"
9780 );
9781 assert_eq!(
9782 com::read_config(store.pools[1].clone(), path)
9783 .await
9784 .expect("final sweep must preserve the target"),
9785 *expected
9786 );
9787 }
9788
9789 let mut crash_restarted_pool_meta = PoolMeta::default();
9790 crash_restarted_pool_meta
9791 .load(store.pools[0].clone(), store.pools.clone())
9792 .await
9793 .expect("pool metadata should reload after the simulated pre-complete crash");
9794 *store.pool_meta.write().await = crash_restarted_pool_meta;
9795
9796 let (manual_job_receipt_pool, manual_job_receipt_path) = store
9797 .decommission_durable_ilm_receipt_paths_for_test(0)
9798 .await
9799 .expect("durable ILM receipt paths should be listable")
9800 .into_iter()
9801 .find(|(_, path)| path.contains(&manual_job_path))
9802 .expect("manual job should have one target receipt");
9803 let manual_job_receipt_bytes = com::read_config(store.pools[manual_job_receipt_pool].clone(), &manual_job_receipt_path)
9804 .await
9805 .expect("manual job receipt should be readable before deletion");
9806 com::delete_config(store.pools[manual_job_receipt_pool].clone(), &manual_job_receipt_path)
9807 .await
9808 .expect("manual job receipt should delete after source cleanup");
9809 com::delete_config(store.pools[1].clone(), &manual_job_path)
9810 .await
9811 .expect("post-crash target manual job should delete");
9812 let missing_after_crash = store
9813 .complete_decommission(0)
9814 .await
9815 .expect_err("completion must reject a missing target after source cleanup and restart")
9816 .to_string();
9817 assert!(missing_after_crash.contains("receipt"));
9818 assert!(missing_after_crash.contains(&manual_job_path));
9819 assert!(missing_after_crash.contains(&manual_job_id.to_string()));
9820 assert!(
9821 !store.pool_meta.read().await.pools[0]
9822 .decommission
9823 .as_ref()
9824 .expect("decommission state should survive restart")
9825 .complete
9826 );
9827 com::save_config(store.pools[1].clone(), &manual_job_path, manual_job_bytes.clone())
9828 .await
9829 .expect("post-crash target manual job should restore");
9830 com::save_config(
9831 store.pools[manual_job_receipt_pool].clone(),
9832 &manual_job_receipt_path,
9833 manual_job_receipt_bytes,
9834 )
9835 .await
9836 .expect("manual job receipt should restore after the missing-receipt check");
9837
9838 com::save_config(store.pools[1].clone(), &transaction_path, b"{corrupt".to_vec())
9839 .await
9840 .expect("post-crash target transaction should corrupt deterministically");
9841 let corrupt_after_crash = store
9842 .complete_decommission(0)
9843 .await
9844 .expect_err("completion must reject a corrupt target after source cleanup and restart")
9845 .to_string();
9846 assert!(corrupt_after_crash.contains(&transaction_path));
9847 assert!(corrupt_after_crash.contains(&transaction.transaction_id.to_string()));
9848 com::save_config(store.pools[1].clone(), &transaction_path, transaction_bytes.clone())
9849 .await
9850 .expect("post-crash target transaction should restore");
9851
9852 let mut wrong_manual_job = manual_job.clone();
9853 wrong_manual_job.job_id = uuid::Uuid::new_v4();
9854 com::save_config(
9855 store.pools[1].clone(),
9856 &manual_job_path,
9857 wrong_manual_job.encode().expect("wrong-id job should encode"),
9858 )
9859 .await
9860 .expect("post-crash target manual job should accept the wrong-id fixture");
9861 let wrong_id_after_crash = store
9862 .complete_decommission(0)
9863 .await
9864 .expect_err("completion must reject a target record with the wrong id")
9865 .to_string();
9866 assert!(wrong_id_after_crash.contains(&manual_job_path));
9867 assert!(wrong_id_after_crash.contains(&manual_job_id.to_string()));
9868 com::save_config(store.pools[1].clone(), &manual_job_path, manual_job_bytes.clone())
9869 .await
9870 .expect("post-crash target manual job should restore after the wrong-id check");
9871
9872 com::save_config(
9873 store.pools[1].clone(),
9874 &transaction_path,
9875 regressed_transaction
9876 .encode()
9877 .expect("regressed transition transaction should encode"),
9878 )
9879 .await
9880 .expect("post-crash target transaction should accept the regression fixture");
9881 let regression_after_crash = store
9882 .complete_decommission(0)
9883 .await
9884 .expect_err("completion must reject a lower transition transaction revision")
9885 .to_string();
9886 assert!(regression_after_crash.contains("generation mismatch"));
9887 assert!(regression_after_crash.contains(&transaction_path));
9888 assert!(regression_after_crash.contains(&transaction.transaction_id.to_string()));
9889 com::save_config(store.pools[1].clone(), &transaction_path, transaction_bytes.clone())
9890 .await
9891 .expect("post-crash target transaction should restore after the regression check");
9892
9893 let (manual_task_receipt_pool, manual_task_receipt_path) = store
9894 .decommission_durable_ilm_receipt_paths_for_test(0)
9895 .await
9896 .expect("durable ILM receipt paths should be listable")
9897 .into_iter()
9898 .find(|(_, path)| path.contains(&manual_task_path))
9899 .expect("manual task receipt should retain its reversible source path");
9900 let manual_task_receipt_bytes =
9901 com::read_config(store.pools[manual_task_receipt_pool].clone(), &manual_task_receipt_path)
9902 .await
9903 .expect("manual task receipt should be readable before corruption");
9904 com::save_config(
9905 store.pools[manual_task_receipt_pool].clone(),
9906 &manual_task_receipt_path,
9907 b"{corrupt".to_vec(),
9908 )
9909 .await
9910 .expect("manual task receipt should corrupt deterministically");
9911 let corrupt_receipt = store
9912 .complete_decommission(0)
9913 .await
9914 .expect_err("completion must fail closed on a corrupt receipt")
9915 .to_string();
9916 assert!(corrupt_receipt.contains(&manual_task_path));
9917 assert!(corrupt_receipt.contains(&manual_job_id.to_string()));
9918 com::save_config(
9919 store.pools[manual_task_receipt_pool].clone(),
9920 &manual_task_receipt_path,
9921 manual_task_receipt_bytes,
9922 )
9923 .await
9924 .expect("manual task receipt should restore after the corruption check");
9925
9926 let tier_stats = recover_tier_delete_journal_entries(store.clone(), 100, None)
9927 .await
9928 .expect("tier journal recovery should consume the migrated record before completion");
9929 assert_eq!((tier_stats.scanned, tier_stats.deleted, tier_stats.failed), (1, 1, 0));
9930 assert!(matches!(com::read_config(store.clone(), &tier_path).await, Err(Error::ConfigNotFound)));
9931
9932 let recovered_transition_version = "recovered-transition-version".to_string();
9933 backend
9934 .set_transition_candidate_probe_override(Some(TransitionCandidateProbe::VersionedPresent(
9935 recovered_transition_version.clone(),
9936 )))
9937 .await;
9938 let transaction_stats = recover_transition_transaction_records(store.clone(), 100, None)
9939 .await
9940 .expect("transition recovery should advance and consume the migrated transaction before completion");
9941 backend.set_transition_candidate_probe_override(None).await;
9942 assert_eq!(
9943 (
9944 transaction_stats.scanned,
9945 transaction_stats.recovered,
9946 transaction_stats.retained,
9947 transaction_stats.failed,
9948 ),
9949 (1, 1, 0, 0)
9950 );
9951 assert!(matches!(
9952 com::read_config(store.clone(), &transaction_path).await,
9953 Err(Error::ConfigNotFound)
9954 ));
9955 com::save_config(
9956 store.pools[1].clone(),
9957 &transaction_path,
9958 regressed_transaction
9959 .encode()
9960 .expect("post-terminal transition rollback should encode"),
9961 )
9962 .await
9963 .expect("target should accept the post-terminal rollback fixture");
9964 let post_terminal_regression = store
9965 .complete_decommission(0)
9966 .await
9967 .expect_err("terminal proof must not mask a lower transition revision")
9968 .to_string();
9969 assert!(post_terminal_regression.contains("generation mismatch"));
9970 assert!(post_terminal_regression.contains(&transaction_path));
9971 assert!(post_terminal_regression.contains(&transaction.transaction_id.to_string()));
9972 com::delete_config(store.pools[1].clone(), &transaction_path)
9973 .await
9974 .expect("post-terminal rollback fixture should be removed");
9975
9976 let manual_stats = recover_manual_transition_jobs_once(store.clone(), 100, None)
9977 .await
9978 .expect("manual recovery should advance the migrated job and consume its scope before completion");
9979 assert_eq!(
9980 (manual_stats.scanned, manual_stats.resumed, manual_stats.skipped, manual_stats.failed,),
9981 (1, 1, 0, 0)
9982 );
9983 assert!(matches!(
9984 com::read_config(store.clone(), &manual_scope_path).await,
9985 Err(Error::ConfigNotFound)
9986 ));
9987 let recovered_manual_job_bytes = com::read_config(store.pools[1].clone(), &manual_job_path)
9988 .await
9989 .expect("manual recovery should retain the advanced job record");
9990 assert_ne!(recovered_manual_job_bytes, manual_job_bytes);
9991
9992 com::save_config(store.pools[1].clone(), &manual_job_path, manual_job_bytes.clone())
9993 .await
9994 .expect("target manual job should accept the rollback fixture");
9995 let manual_regression = store
9996 .complete_decommission(0)
9997 .await
9998 .expect_err("completion must reject a manual job generation rollback")
9999 .to_string();
10000 assert!(manual_regression.contains("generation mismatch"));
10001 assert!(manual_regression.contains(&manual_job_path));
10002 assert!(manual_regression.contains(&manual_job_id.to_string()));
10003 com::save_config(store.pools[1].clone(), &manual_job_path, recovered_manual_job_bytes)
10004 .await
10005 .expect("target manual job should restore its recovered generation");
10006
10007 store
10008 .complete_decommission(0)
10009 .await
10010 .expect("completion should persist before receipt cleanup");
10011 assert!(
10012 store.pool_meta.read().await.pools[0]
10013 .decommission
10014 .as_ref()
10015 .expect("completed decommission state should remain present")
10016 .complete
10017 );
10018 assert_eq!(
10019 store
10020 .decommission_durable_ilm_receipt_count_for_test(0)
10021 .await
10022 .expect("receipt cleanup should be observable"),
10023 0
10024 );
10025 store
10026 .cleanup_decommission_durable_ilm_receipts_for_test(0)
10027 .await
10028 .expect("receipt cleanup should be idempotent");
10029 let removed_versions = backend.remove_versions().await;
10030 assert!(removed_versions.contains(&(tier_entry.obj_name.clone(), tier_entry.version_id.clone())));
10031 assert!(removed_versions.contains(&(transaction.remote_object.clone(), recovered_transition_version)));
10032 }
10033
10034 #[cfg(feature = "test-util")]
10035 async fn tier_delete_journal_count(store: Arc<crate::store::ECStore>) -> usize {
10036 store
10037 .list_objects_v2(RUSTFS_META_BUCKET, TIER_DELETE_JOURNAL_PREFIX, None, None, 100, false, None, false)
10038 .await
10039 .expect("tier delete journal should be listable")
10040 .objects
10041 .len()
10042 }
10043
10044 #[cfg(feature = "test-util")]
10045 async fn tier_delete_dispatch_manifest_count(store: Arc<crate::store::ECStore>) -> usize {
10046 store
10047 .list_objects_v2(
10048 RUSTFS_META_BUCKET,
10049 TIER_DELETE_DISPATCH_MANIFEST_PREFIX,
10050 None,
10051 None,
10052 100,
10053 false,
10054 None,
10055 false,
10056 )
10057 .await
10058 .expect("tier delete dispatch manifests should be listable")
10059 .objects
10060 .len()
10061 }
10062
10063 #[cfg(feature = "test-util")]
10064 async fn install_committed_tier_delete_journals(
10065 store: Arc<crate::store::ECStore>,
10066 tier_name: &str,
10067 backend_identity: [u8; 32],
10068 count: usize,
10069 ) -> Vec<Jentry> {
10070 let mut entries = Vec::with_capacity(count);
10071 for index in 0..count {
10072 let entry = Jentry {
10073 persisted_version: 0,
10074 obj_name: format!("remote/pass-drain-{index:06}.bin"),
10075 version_id: uuid::Uuid::new_v4().to_string(),
10076 tier_name: tier_name.to_string(),
10077 backend_identity: Some(backend_identity),
10078 version_id_exact: true,
10079 version_state: rustfs_filemeta::TransitionVersionState::Exact,
10080 state: TierDeleteJournalState::Committed,
10081 source: None,
10082 dispatch: None,
10083 };
10084 persist_tier_delete_journal_entry(store.clone(), &entry)
10085 .await
10086 .expect("committed tier-delete journal fixture should persist");
10087 entries.push(entry);
10088 }
10089 entries
10090 }
10091
10092 #[cfg(feature = "test-util")]
10093 fn synthetic_v6_dispatch_entry(
10094 bucket: &str,
10095 object: &str,
10096 tier_name: &str,
10097 backend_identity: [u8; 32],
10098 remote_version: &str,
10099 ) -> Jentry {
10100 Jentry {
10101 persisted_version: 0,
10102 obj_name: format!("remote/{object}"),
10103 version_id: remote_version.to_string(),
10104 tier_name: tier_name.to_string(),
10105 backend_identity: Some(backend_identity),
10106 version_id_exact: true,
10107 version_state: rustfs_filemeta::TransitionVersionState::Exact,
10108 state: TierDeleteJournalState::Prepared,
10109 source: Some(TierDeleteSourceIdentity {
10110 bucket: bucket.to_string(),
10111 object: object.to_string(),
10112 version_id: Some(uuid::Uuid::new_v4().to_string()),
10113 versioned: true,
10114 version_suspended: false,
10115 data_dir: Some(uuid::Uuid::new_v4().to_string()),
10116 etag: Some(format!("etag-{object}")),
10117 mod_time: Some(OffsetDateTime::UNIX_EPOCH.to_string()),
10118 }),
10119 dispatch: None,
10120 }
10121 }
10122
10123 #[cfg(feature = "test-util")]
10124 async fn install_aborting_dispatch_fixture(
10125 store: Arc<crate::store::ECStore>,
10126 bucket: &str,
10127 incarnation: uuid::Uuid,
10128 prefix: &str,
10129 tier_name: &str,
10130 backend_identity: [u8; 32],
10131 journal_count: usize,
10132 ) -> (String, Vec<Jentry>) {
10133 let entries = (0..journal_count)
10134 .map(|index| {
10135 (
10136 synthetic_v6_dispatch_entry(
10137 bucket,
10138 &format!("{prefix}{index:06}.bin"),
10139 tier_name,
10140 backend_identity,
10141 &uuid::Uuid::new_v4().to_string(),
10142 ),
10143 Some(TierDeleteJournalState::Prepared),
10144 )
10145 })
10146 .collect();
10147 install_test_tier_delete_dispatch_fixture(
10148 store,
10149 bucket,
10150 incarnation,
10151 prefix,
10152 entries,
10153 TierDeleteDispatchManifestState::Aborting,
10154 )
10155 .await
10156 .expect("Aborting rollback fixture should persist")
10157 }
10158
10159 #[cfg(feature = "test-util")]
10160 async fn drive_tier_delete_dispatch_restart_to_convergence(store: Arc<crate::store::ECStore>) {
10161 tokio::time::timeout(Duration::from_secs(30), async {
10162 let mut stable_empty_observations = 0;
10163 loop {
10164 recover_tier_delete_dispatch_manifests(store.clone(), 100, None)
10165 .await
10166 .expect("restarted manifest recovery pass should finish");
10167 recover_tier_delete_journal_entries(store.clone(), 100, None)
10168 .await
10169 .expect("restarted journal recovery pass should finish");
10170 if tier_delete_journal_count(store.clone()).await == 0
10171 && tier_delete_dispatch_manifest_count(store.clone()).await == 0
10172 && tier_delete_dispatch_manifest_recovery_count_for_test(&store) == 0
10173 {
10174 stable_empty_observations += 1;
10175 if stable_empty_observations == 10 {
10176 break;
10177 }
10178 } else {
10179 stable_empty_observations = 0;
10180 }
10181 tokio::time::sleep(Duration::from_millis(50)).await;
10182 }
10183 })
10184 .await
10185 .expect("same-disk restart recovery should converge without retained dispatch state");
10186 }
10187
10188 #[cfg(feature = "test-util")]
10189 #[tokio::test]
10190 #[serial_test::serial(storage_class_env)]
10191 async fn tier_delete_recovery_pass_drains_multiple_fast_manifest_pages() {
10192 const MANIFEST_COUNT: usize = 10;
10193
10194 let temp_dir = tempfile::tempdir().expect("create fast manifest pass recovery store dir");
10195 let mut instance_ctx = crate::runtime::instance::InstanceContext::new();
10196 instance_ctx.suppress_tier_delete_journal_recovery_for_test();
10197 let (ctx, store, shutdown) = without_storage_class_env(build_isolated_test_store_with_layout(
10198 temp_dir.path(),
10199 "tier-delete-fast-manifest-pass",
10200 &[(1, 4)],
10201 CancellationToken::new(),
10202 Some(Arc::new(instance_ctx)),
10203 ))
10204 .await;
10205 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
10206 let tier_name = "FAST-MANIFEST-PASS";
10207 let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
10208 let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
10209 .await
10210 .expect("fast manifest pass tier lease should resolve")
10211 .backend_identity();
10212 for index in 0..MANIFEST_COUNT {
10213 let bucket = format!("tier-delete-fast-manifest-pass-{index}");
10215 store
10216 .make_bucket(&bucket, &MakeBucketOptions::default())
10217 .await
10218 .expect("fast manifest pass bucket should be created");
10219 let incarnation = store
10220 .bucket_incarnation_id(&bucket)
10221 .await
10222 .expect("fast manifest pass bucket incarnation should resolve");
10223 install_aborting_dispatch_fixture(
10224 store.clone(),
10225 &bucket,
10226 incarnation,
10227 &format!("manifest-page-{index:06}/"),
10228 tier_name,
10229 backend_identity,
10230 1,
10231 )
10232 .await;
10233 }
10234 assert_eq!(tier_delete_dispatch_manifest_count(store.clone()).await, MANIFEST_COUNT);
10235 assert_eq!(tier_delete_journal_count(store.clone()).await, MANIFEST_COUNT);
10236
10237 let cancel = CancellationToken::new();
10238 let mut manifest_marker = None;
10239 let mut journal_marker = None;
10240 let stats = recover_test_tier_delete_journal_pass_with_budget(
10241 store.clone(),
10242 &cancel,
10243 &mut manifest_marker,
10244 &mut journal_marker,
10245 Duration::from_secs(30),
10246 )
10247 .await;
10248
10249 assert!(!stats.canceled);
10250 assert!(!stats.deadline_exhausted);
10251 assert!(
10252 stats.manifest_pages > 1,
10253 "one production pass must cross the default eight-manifest page limit"
10254 );
10255 assert_eq!(stats.manifests.scanned, MANIFEST_COUNT);
10256 assert_eq!(stats.manifests.deleted, MANIFEST_COUNT, "full recovery result: {stats:?}");
10257 assert_eq!(stats.manifests.failed, 0, "full recovery result: {stats:?}");
10258 assert_eq!(manifest_marker, None);
10259 assert_eq!(tier_delete_dispatch_manifest_count(store.clone()).await, 0);
10260 assert_eq!(tier_delete_journal_count(store).await, 0);
10261 assert_eq!(backend.remove_count().await, 0, "rollback recovery must not call the remote tier");
10262 shutdown.cancel();
10263 }
10264
10265 #[cfg(feature = "test-util")]
10266 #[tokio::test]
10267 #[serial_test::serial(storage_class_env)]
10268 async fn tier_delete_manual_pass_retains_manifest_owned_by_startup_recovery() {
10269 let temp_dir = tempfile::tempdir().expect("create automatic recovery ownership store dir");
10270 let (ctx, store, shutdown) =
10271 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "tier-delete-auto-owner", &[4])).await;
10272 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
10273 let bucket = "tier-delete-auto-owner-bucket";
10274 store
10275 .make_bucket(bucket, &MakeBucketOptions::default())
10276 .await
10277 .expect("automatic recovery bucket should be created");
10278 let incarnation = store.bucket_incarnation_id(bucket).await.expect("bucket incarnation");
10279 let tier_name = "AUTO-OWNER";
10280 let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
10281 let identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
10282 .await
10283 .expect("automatic recovery tier lease")
10284 .backend_identity();
10285
10286 let lifecycle_guard = store
10288 .acquire_bucket_lifecycle_write_lock(bucket)
10289 .await
10290 .expect("fixture lifecycle lock");
10291 let (manifest_name, entries) =
10292 install_aborting_dispatch_fixture(store.clone(), bucket, incarnation, "auto-owner/", tier_name, identity, 1).await;
10293 let journal_name = tier_delete_journal_object_name(&entries[0]);
10294 let hook = TierDeleteDispatchRollbackTestHook::install_slow_delete(&journal_name, &journal_name);
10295 drop(lifecycle_guard);
10296 ctx.wake_tier_delete_journal_recovery();
10297 tokio::time::timeout(Duration::from_secs(30), hook.wait_until_delete_paused())
10298 .await
10299 .expect("startup recovery should own the manifest before a manual pass");
10300 assert!(tier_delete_dispatch_manifest_recovery_inflight_for_test(&store, &manifest_name));
10301
10302 let stats = recover_tier_delete_dispatch_manifests(store.clone(), 8, None)
10303 .await
10304 .expect("manual recovery scan");
10305 assert_eq!(stats.scanned, 1, "{stats:?}");
10306 assert_eq!(stats.retained, 1, "{stats:?}");
10307 assert_eq!(stats.deleted, 0, "{stats:?}");
10308 assert_eq!(stats.failed, 0, "{stats:?}");
10309 assert_eq!(tier_delete_dispatch_manifest_count(store.clone()).await, 1);
10310 assert_eq!(tier_delete_journal_count(store.clone()).await, 1);
10311
10312 hook.release_delete();
10313 tokio::time::timeout(Duration::from_secs(30), async {
10314 loop {
10315 let manifest_gone = matches!(com::read_config(store.clone(), &manifest_name).await, Err(Error::ConfigNotFound));
10316 if manifest_gone && !tier_delete_dispatch_manifest_recovery_inflight_for_test(&store, &manifest_name) {
10317 break;
10318 }
10319 tokio::time::sleep(Duration::from_millis(10)).await;
10320 }
10321 })
10322 .await
10323 .expect("automatic recovery should converge without a manual retry");
10324 assert_eq!(tier_delete_dispatch_manifest_count(store.clone()).await, 0);
10325 assert_eq!(tier_delete_journal_count(store).await, 0);
10326 assert_eq!(backend.remove_count().await, 0, "rollback must not delete from the remote tier");
10327 shutdown.cancel();
10328 }
10329
10330 #[cfg(feature = "test-util")]
10331 #[tokio::test]
10332 #[serial_test::serial(storage_class_env)]
10333 async fn tier_delete_recovery_pass_drains_multiple_fast_journal_pages() {
10334 const JOURNAL_COUNT: usize = 25;
10335
10336 let temp_dir = tempfile::tempdir().expect("create fast pass recovery store dir");
10337 let (ctx, store, _shutdown) =
10338 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "tier-delete-fast-pass", &[4])).await;
10339 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
10340 let tier_name = "FAST-PASS-RECOVERY";
10341 let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
10342 let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
10343 .await
10344 .expect("fast pass tier lease should resolve")
10345 .backend_identity();
10346 let entries = install_committed_tier_delete_journals(store.clone(), tier_name, backend_identity, JOURNAL_COUNT).await;
10347 assert_eq!(tier_delete_journal_count(store.clone()).await, JOURNAL_COUNT);
10348
10349 let cancel = CancellationToken::new();
10350 let mut manifest_marker = None;
10351 let mut journal_marker = None;
10352 let stats = recover_test_tier_delete_journal_pass_with_budget(
10353 store.clone(),
10354 &cancel,
10355 &mut manifest_marker,
10356 &mut journal_marker,
10357 Duration::from_secs(30),
10358 )
10359 .await;
10360
10361 assert!(!stats.canceled);
10362 assert!(!stats.deadline_exhausted);
10363 assert!(
10364 stats.journal_pages > 1,
10365 "one production pass must cross the default eight-record page limit"
10366 );
10367 assert_eq!(stats.journals.scanned, JOURNAL_COUNT);
10368 assert_eq!(stats.journals.deleted, JOURNAL_COUNT);
10369 assert_eq!(stats.journals.failed, 0);
10370 assert_eq!(journal_marker, None);
10371 assert_eq!(tier_delete_journal_count(store).await, 0);
10372 assert_eq!(backend.exact_remove_count(), JOURNAL_COUNT);
10373 let mut removed = backend.remove_versions().await;
10374 removed.sort();
10375 let mut expected = entries
10376 .into_iter()
10377 .map(|entry| (entry.obj_name, entry.version_id))
10378 .collect::<Vec<_>>();
10379 expected.sort();
10380 assert_eq!(removed, expected);
10381 }
10382
10383 #[cfg(feature = "test-util")]
10384 #[tokio::test]
10385 #[serial_test::serial(storage_class_env)]
10386 async fn tier_delete_recovery_pass_rotates_failed_journal_pages_without_busy_loop() {
10387 const JOURNAL_COUNT: usize = 9;
10388
10389 let temp_dir = tempfile::tempdir().expect("create failed pass recovery store dir");
10390 let (ctx, store, _shutdown) =
10391 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "tier-delete-failed-pass", &[4])).await;
10392 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
10393 let tier_name = "FAILED-PASS-RECOVERY";
10394 let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
10395 let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
10396 .await
10397 .expect("failed pass tier lease should resolve")
10398 .backend_identity();
10399 install_committed_tier_delete_journals(store.clone(), tier_name, backend_identity, JOURNAL_COUNT).await;
10400 backend.set_remove_failure(true);
10401
10402 let cancel = CancellationToken::new();
10403 let mut manifest_marker = None;
10404 let mut journal_marker = None;
10405 let first = recover_test_tier_delete_journal_pass_with_budget(
10406 store.clone(),
10407 &cancel,
10408 &mut manifest_marker,
10409 &mut journal_marker,
10410 Duration::from_secs(30),
10411 )
10412 .await;
10413 assert_eq!(first.journal_pages, 1, "a no-progress failed page must stop the pass");
10414 assert_eq!(first.journals.scanned, 8);
10415 assert_eq!(first.journals.deleted, 0);
10416 assert_eq!(first.journals.failed, 8);
10417 assert!(journal_marker.is_some(), "a truncated failed page must retain its continuation marker");
10418 assert_eq!(tier_delete_journal_count(store.clone()).await, JOURNAL_COUNT);
10419
10420 let second = recover_test_tier_delete_journal_pass_with_budget(
10421 store.clone(),
10422 &cancel,
10423 &mut manifest_marker,
10424 &mut journal_marker,
10425 Duration::from_secs(30),
10426 )
10427 .await;
10428 assert_eq!(second.journal_pages, 1);
10429 assert_eq!(second.journals.scanned, 1);
10430 assert_eq!(second.journals.deleted, 0);
10431 assert_eq!(second.journals.failed, 1);
10432 assert_eq!(
10433 journal_marker, None,
10434 "end-of-list rotation must revisit the failed prefix on a later pass"
10435 );
10436 assert_eq!(tier_delete_journal_count(store.clone()).await, JOURNAL_COUNT);
10437
10438 backend.set_remove_failure(false);
10439 let third = recover_test_tier_delete_journal_pass_with_budget(
10440 store.clone(),
10441 &cancel,
10442 &mut manifest_marker,
10443 &mut journal_marker,
10444 Duration::from_secs(30),
10445 )
10446 .await;
10447 assert_eq!(third.journals.scanned, JOURNAL_COUNT);
10448 assert_eq!(third.journals.deleted, JOURNAL_COUNT);
10449 assert_eq!(third.journals.failed, 0);
10450 assert_eq!(tier_delete_journal_count(store).await, 0);
10451 }
10452
10453 #[cfg(feature = "test-util")]
10454 #[tokio::test]
10455 #[serial_test::serial(storage_class_env)]
10456 async fn tier_delete_journal_rejects_unbound_new_prepare_without_persisting() {
10457 let temp_dir = tempfile::tempdir().expect("create journal admission store dir");
10458 let (_ctx, store, _shutdown) =
10459 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "tier-journal-unbound-prepare", &[4])).await;
10460 let entry = Jentry {
10461 persisted_version: 0,
10462 obj_name: "remote/object".to_string(),
10463 version_id: uuid::Uuid::new_v4().to_string(),
10464 tier_name: "COLD-A".to_string(),
10465 backend_identity: Some([7; 32]),
10466 version_id_exact: true,
10467 version_state: rustfs_filemeta::TransitionVersionState::Exact,
10468 state: TierDeleteJournalState::Prepared,
10469 source: Some(TierDeleteSourceIdentity {
10470 bucket: "source-bucket".to_string(),
10471 object: "source-object".to_string(),
10472 version_id: Some(uuid::Uuid::new_v4().to_string()),
10473 versioned: true,
10474 version_suspended: false,
10475 data_dir: Some(uuid::Uuid::new_v4().to_string()),
10476 etag: Some("source-etag".to_string()),
10477 mod_time: Some(OffsetDateTime::UNIX_EPOCH.to_string()),
10478 }),
10479 dispatch: None,
10480 };
10481 let error = persist_tier_delete_journal_entry(store.clone(), &entry)
10482 .await
10483 .expect_err("a new source-owning journal without a v6 manifest binding must fail closed");
10484 assert!(error.to_string().contains("dispatch manifest binding"), "unexpected error: {error}");
10485 assert_eq!(tier_delete_journal_count(store).await, 0, "rejected prepare must not write a journal");
10486 }
10487
10488 #[cfg(feature = "test-util")]
10489 #[tokio::test]
10490 #[serial_test::serial(storage_class_env)]
10491 async fn dispatch_manifest_recovery_rolls_back_unapproved_crash_states_without_remote_io() {
10492 let temp_dir = tempfile::tempdir().expect("create dispatch rollback store dir");
10493 let (ctx, store, _shutdown) =
10494 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "dispatch-manifest-rollback", &[4])).await;
10495 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
10496 let bucket = "dispatch-manifest-rollback-bucket";
10497 store
10498 .make_bucket(bucket, &MakeBucketOptions::default())
10499 .await
10500 .expect("dispatch rollback bucket should be created");
10501 let incarnation = store
10502 .bucket_incarnation_id(bucket)
10503 .await
10504 .expect("bucket incarnation should resolve");
10505 let tier_name = "DISPATCH-ROLLBACK";
10506 let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
10507 let identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
10508 .await
10509 .expect("tier lease should resolve")
10510 .backend_identity();
10511
10512 let preparing = [
10513 ("preparing/a.bin", TierDeleteJournalState::Prepared),
10514 ("preparing/b.bin", TierDeleteJournalState::Dispatched),
10515 ]
10516 .into_iter()
10517 .map(|(object, state)| {
10518 (
10519 synthetic_v6_dispatch_entry(bucket, object, tier_name, identity, &uuid::Uuid::new_v4().to_string()),
10520 Some(state),
10521 )
10522 })
10523 .collect();
10524 install_test_tier_delete_dispatch_fixture(
10525 store.clone(),
10526 bucket,
10527 incarnation,
10528 "preparing/",
10529 preparing,
10530 TierDeleteDispatchManifestState::Preparing,
10531 )
10532 .await
10533 .expect("Preparing crash fixture should persist");
10534
10535 let aborting = [
10536 ("aborting/missing.bin", None),
10537 ("aborting/prepared.bin", Some(TierDeleteJournalState::Prepared)),
10538 ("aborting/dispatched.bin", Some(TierDeleteJournalState::Dispatched)),
10539 ]
10540 .into_iter()
10541 .map(|(object, state)| {
10542 (
10543 synthetic_v6_dispatch_entry(bucket, object, tier_name, identity, &uuid::Uuid::new_v4().to_string()),
10544 state,
10545 )
10546 })
10547 .collect();
10548 install_test_tier_delete_dispatch_fixture(
10549 store.clone(),
10550 bucket,
10551 incarnation,
10552 "aborting/",
10553 aborting,
10554 TierDeleteDispatchManifestState::Aborting,
10555 )
10556 .await
10557 .expect("Aborting crash fixture should persist");
10558 install_test_tier_delete_dispatch_fixture(
10559 store.clone(),
10560 bucket,
10561 incarnation,
10562 "empty/",
10563 Vec::new(),
10564 TierDeleteDispatchManifestState::Completed,
10565 )
10566 .await
10567 .expect("empty Completed crash fixture should persist");
10568
10569 let deleted = tokio::time::timeout(Duration::from_secs(30), async {
10570 let mut deleted = 0;
10571 loop {
10572 let stats = recover_tier_delete_dispatch_manifests(store.clone(), 100, None)
10573 .await
10574 .expect("manifest recovery should reconcile negative-seal crash states");
10575 deleted += stats.deleted;
10576 if tier_delete_dispatch_manifest_count(store.clone()).await == 0 {
10577 break deleted;
10578 }
10579 tokio::time::sleep(Duration::from_millis(10)).await;
10583 }
10584 })
10585 .await
10586 .expect("valid negative-seal crash states should eventually converge");
10587 assert_eq!(deleted, 3);
10588 assert_eq!(tier_delete_journal_count(store.clone()).await, 0);
10589 assert_eq!(tier_delete_dispatch_manifest_count(store).await, 0);
10590 assert_eq!(
10591 backend.remove_count().await,
10592 0,
10593 "rollback recovery must perform zero remote delete operations"
10594 );
10595 }
10596
10597 #[cfg(feature = "test-util")]
10598 #[tokio::test]
10599 #[serial_test::serial(storage_class_env)]
10600 async fn dispatch_manifest_recovery_quarantines_impossible_committed_rollback_set() {
10601 let temp_dir = tempfile::tempdir().expect("create dispatch quarantine store dir");
10602 let (ctx, store, _shutdown) =
10603 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "dispatch-manifest-quarantine", &[4])).await;
10604 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
10605 let bucket = "dispatch-manifest-quarantine-bucket";
10606 store
10607 .make_bucket(bucket, &MakeBucketOptions::default())
10608 .await
10609 .expect("dispatch quarantine bucket should be created");
10610 let incarnation = store
10611 .bucket_incarnation_id(bucket)
10612 .await
10613 .expect("bucket incarnation should resolve");
10614 let tier_name = "DISPATCH-QUARANTINE";
10615 let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
10616 let identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
10617 .await
10618 .expect("tier lease should resolve")
10619 .backend_identity();
10620 let entries = [TierDeleteJournalState::Prepared, TierDeleteJournalState::Committed]
10621 .into_iter()
10622 .enumerate()
10623 .map(|(index, state)| {
10624 (
10625 synthetic_v6_dispatch_entry(
10626 bucket,
10627 &format!("quarantine/{index}.bin"),
10628 tier_name,
10629 identity,
10630 &uuid::Uuid::new_v4().to_string(),
10631 ),
10632 Some(state),
10633 )
10634 })
10635 .collect();
10636 let (manifest_name, _) = install_test_tier_delete_dispatch_fixture(
10637 store.clone(),
10638 bucket,
10639 incarnation,
10640 "quarantine/",
10641 entries,
10642 TierDeleteDispatchManifestState::Aborting,
10643 )
10644 .await
10645 .expect("impossible rollback fixture should persist");
10646
10647 let stats = recover_tier_delete_dispatch_manifests(store.clone(), 100, None)
10648 .await
10649 .expect("quarantined manifest scan should finish");
10650 assert_eq!((stats.scanned, stats.deleted, stats.failed), (1, 0, 1));
10651 assert_eq!(
10652 tier_delete_journal_count(store.clone()).await,
10653 2,
10654 "validation must precede every deletion"
10655 );
10656 assert_eq!(tier_delete_dispatch_manifest_count(store.clone()).await, 1);
10657 assert_eq!(
10658 test_tier_delete_dispatch_manifest_state(store, &manifest_name)
10659 .await
10660 .expect("quarantined manifest should remain readable"),
10661 Some(TierDeleteDispatchManifestState::Aborting)
10662 );
10663 assert_eq!(backend.remove_count().await, 0);
10664 }
10665
10666 #[cfg(feature = "test-util")]
10667 #[tokio::test]
10668 #[serial_test::serial(storage_class_env)]
10669 async fn dispatch_manifest_rollback_bounded_concurrency_reaches_tail_behind_slow_member() {
10670 const JOURNAL_COUNT: usize = 65;
10671
10672 let temp_dir = tempfile::tempdir().expect("create bounded rollback store dir");
10673 let (ctx, store, _shutdown) =
10674 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "bounded-dispatch-rollback", &[4])).await;
10675 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
10676 let bucket = "bounded-dispatch-rollback-bucket";
10677 store
10678 .make_bucket(bucket, &MakeBucketOptions::default())
10679 .await
10680 .expect("bounded rollback bucket should be created");
10681 let incarnation = store
10682 .bucket_incarnation_id(bucket)
10683 .await
10684 .expect("bounded rollback bucket incarnation should resolve");
10685 let tier_name = "BOUNDED-DISPATCH-ROLLBACK";
10686 let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
10687 let identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
10688 .await
10689 .expect("bounded rollback tier lease should resolve")
10690 .backend_identity();
10691 let (manifest_name, entries) =
10692 install_aborting_dispatch_fixture(store.clone(), bucket, incarnation, "slow/", tier_name, identity, JOURNAL_COUNT)
10693 .await;
10694 let first_name = tier_delete_journal_object_name(entries.first().expect("slow rollback fixture should not be empty"));
10695 let last_name = tier_delete_journal_object_name(entries.last().expect("slow rollback fixture should not be empty"));
10696 let hook = TierDeleteDispatchRollbackTestHook::install_slow_delete(&first_name, &last_name);
10697 let worker_store = store.clone();
10698 let worker_manifest = manifest_name.clone();
10699 let worker =
10700 tokio::spawn(async move { recover_test_tier_delete_dispatch_manifest(worker_store, &worker_manifest).await });
10701 tokio::time::timeout(Duration::from_secs(30), hook.wait_until_delete_paused())
10702 .await
10703 .expect("the first rollback member should reach the slow hook");
10704 tokio::time::timeout(Duration::from_secs(30), hook.wait_until_delete_observed())
10705 .await
10706 .expect("a member beyond the first two concurrency windows should remain reachable");
10707 assert!(
10708 (2..=32).contains(&hook.max_in_flight()),
10709 "rollback delete concurrency must make progress without exceeding its bound; observed {}",
10710 hook.max_in_flight()
10711 );
10712 hook.release_delete();
10713 worker
10714 .await
10715 .expect("bounded rollback recovery task should join")
10716 .expect("bounded rollback recovery should converge after the slow member releases");
10717 drop(hook);
10718
10719 assert_eq!(tier_delete_journal_count(store.clone()).await, 0);
10720 assert_eq!(tier_delete_dispatch_manifest_count(store.clone()).await, 0);
10721 assert_eq!(backend.remove_count().await, 0, "rollback must not call the remote tier");
10722 }
10723
10724 #[cfg(feature = "test-util")]
10725 #[tokio::test]
10726 #[serial_test::serial(storage_class_env)]
10727 async fn aborted_dispatch_retry_defers_residual_scan_to_recovery() {
10728 const JOURNAL_COUNT: usize = 65;
10729
10730 let temp_dir = tempfile::tempdir().expect("create aborted retry store dir");
10731 let (ctx, store, shutdown) =
10732 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "aborted-dispatch-retry", &[4])).await;
10733 shutdown.cancel();
10734 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
10735 let bucket = "aborted-dispatch-retry-bucket";
10736 let prefix = "archive/";
10737 store
10738 .make_bucket(bucket, &MakeBucketOptions::default())
10739 .await
10740 .expect("aborted retry bucket should be created");
10741 let incarnation = store
10742 .bucket_incarnation_id(bucket)
10743 .await
10744 .expect("aborted retry bucket incarnation should resolve");
10745 let tier_name = "ABORTED-DISPATCH-RETRY";
10746 let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
10747 let identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
10748 .await
10749 .expect("aborted retry tier lease should resolve")
10750 .backend_identity();
10751 let aborted_entries = (0..JOURNAL_COUNT)
10752 .map(|index| {
10753 (
10754 synthetic_v6_dispatch_entry(
10755 bucket,
10756 &format!("{prefix}{index:06}.bin"),
10757 tier_name,
10758 identity,
10759 &uuid::Uuid::new_v4().to_string(),
10760 ),
10761 Some(TierDeleteJournalState::Prepared),
10762 )
10763 })
10764 .collect::<Vec<_>>();
10765 let entries = aborted_entries.iter().map(|(entry, _)| entry.clone()).collect::<Vec<_>>();
10766 let (manifest_name, _) = install_test_tier_delete_dispatch_fixture(
10767 store.clone(),
10768 bucket,
10769 incarnation,
10770 prefix,
10771 aborted_entries,
10772 TierDeleteDispatchManifestState::Aborted,
10773 )
10774 .await
10775 .expect("Aborted retry fixture should persist");
10776
10777 let lifecycle_guard = store
10778 .acquire_bucket_lifecycle_write_lock(bucket)
10779 .await
10780 .expect("aborted retry should acquire the bucket lifecycle fence");
10781 let mut fence_opts = ObjectOptions::default();
10782 fence_opts.add_bucket_lifecycle_lock_guard(&lifecycle_guard);
10783 let bucket_fence = fence_opts
10784 .bucket_lifecycle_lock_fence
10785 .clone()
10786 .expect("aborted retry should capture the bucket lifecycle fence");
10787 let fleet_proof = acquire_tier_delete_journal_fleet_proof().expect("aborted retry fixture should have a fleet proof");
10788 let hook = TierDeleteDispatchMemberReadTestHook::install_pause(TierDeleteDispatchMemberReadTestStage::Validation);
10789
10790 let error =
10791 match prepare_tier_delete_dispatch(store.clone(), bucket, incarnation, prefix, entries, fleet_proof, &bucket_fence)
10792 .await
10793 {
10794 Ok(_) => panic!("an Aborted manifest must be retained for bounded recovery cleanup"),
10795 Err(err) => err,
10796 };
10797
10798 assert!(
10799 error.to_string().contains("bounded recovery cleanup"),
10800 "unexpected aborted retry error: {error}"
10801 );
10802 assert_eq!(
10803 hook.entry_count(),
10804 0,
10805 "request retry must not scan the retained Aborted journal set under bucket write lock"
10806 );
10807 assert_eq!(
10808 test_tier_delete_dispatch_manifest_state(store.clone(), &manifest_name)
10809 .await
10810 .expect("retained Aborted manifest should remain readable"),
10811 Some(TierDeleteDispatchManifestState::Aborted)
10812 );
10813 assert_eq!(tier_delete_journal_count(store.clone()).await, JOURNAL_COUNT);
10814 assert_eq!(backend.remove_count().await, 0, "retry must not call the remote tier");
10815 drop(hook);
10816 drop(lifecycle_guard);
10817
10818 recover_test_tier_delete_dispatch_manifest(store.clone(), &manifest_name)
10819 .await
10820 .expect("bounded recovery should clean the retained Aborted manifest");
10821 assert_eq!(tier_delete_journal_count(store.clone()).await, 0);
10822 assert_eq!(tier_delete_dispatch_manifest_count(store).await, 0);
10823 }
10824
10825 #[cfg(feature = "test-util")]
10826 #[tokio::test]
10827 #[serial_test::serial(storage_class_env)]
10828 async fn dispatch_manifest_page_timeout_detaches_one_deduplicated_worker_until_convergence() {
10829 let temp_dir = tempfile::tempdir().expect("create detached rollback store dir");
10830 let (ctx, store, _shutdown) =
10831 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "detached-dispatch-rollback", &[4])).await;
10832 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
10833 let bucket = "detached-dispatch-rollback-bucket";
10834 store
10835 .make_bucket(bucket, &MakeBucketOptions::default())
10836 .await
10837 .expect("detached rollback bucket should be created");
10838 let incarnation = store
10839 .bucket_incarnation_id(bucket)
10840 .await
10841 .expect("detached rollback bucket incarnation should resolve");
10842 let tier_name = "DETACHED-DISPATCH-ROLLBACK";
10843 let _backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
10844 let identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
10845 .await
10846 .expect("detached rollback tier lease should resolve")
10847 .backend_identity();
10848 let (manifest_name, entries) =
10849 install_aborting_dispatch_fixture(store.clone(), bucket, incarnation, "detached/", tier_name, identity, 1).await;
10850 let journal_name = tier_delete_journal_object_name(&entries[0]);
10851 let hook = TierDeleteDispatchRollbackTestHook::install_slow_delete(&journal_name, &journal_name);
10852
10853 assert!(
10854 recover_test_tier_delete_dispatch_manifest_with_page_budget(
10855 store.clone(),
10856 &manifest_name,
10857 Duration::from_millis(10),
10858 )
10859 .await,
10860 "the synthetic page budget must elapse while the manifest worker continues"
10861 );
10862 tokio::time::timeout(Duration::from_secs(30), hook.wait_until_delete_paused())
10863 .await
10864 .expect("the detached manifest worker should continue into its delete phase");
10865 assert!(
10866 tier_delete_dispatch_manifest_recovery_inflight_for_test(&store, &manifest_name),
10867 "the detached worker must retain its per-store/object deduplication guard"
10868 );
10869 assert!(
10870 !recover_test_tier_delete_dispatch_manifest_with_page_budget(store.clone(), &manifest_name, Duration::from_secs(30),)
10871 .await,
10872 "a duplicate page observation should be retained without queuing another worker"
10873 );
10874
10875 hook.release_delete();
10876 tokio::time::timeout(Duration::from_secs(30), async {
10877 loop {
10878 let manifest_gone = matches!(com::read_config(store.clone(), &manifest_name).await, Err(Error::ConfigNotFound));
10879 if manifest_gone && !tier_delete_dispatch_manifest_recovery_inflight_for_test(&store, &manifest_name) {
10880 break;
10881 }
10882 tokio::time::sleep(Duration::from_millis(10)).await;
10883 }
10884 })
10885 .await
10886 .expect("the detached manifest worker should release its guard after convergence");
10887 drop(hook);
10888 assert_eq!(tier_delete_journal_count(store.clone()).await, 0);
10889 }
10890
10891 #[cfg(feature = "test-util")]
10892 #[tokio::test]
10893 #[serial_test::serial(storage_class_env)]
10894 async fn dispatch_manifest_detached_workers_are_store_bounded_and_shutdown_cancellable() {
10895 const MANIFEST_COUNT: usize = 6;
10896 const WORKER_LIMIT: usize = 4;
10897
10898 let temp_dir = tempfile::tempdir().expect("create bounded detached rollback store dir");
10899 let (ctx, store, shutdown) =
10900 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "bounded-detached-dispatch-rollback", &[4]))
10901 .await;
10902 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
10903 let tier_name = "BOUNDED-DETACHED-DISPATCH-ROLLBACK";
10904 let _backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
10905 let identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
10906 .await
10907 .expect("bounded detached rollback tier lease should resolve")
10908 .backend_identity();
10909 let mut manifest_names = Vec::with_capacity(MANIFEST_COUNT);
10910 for index in 0..MANIFEST_COUNT {
10911 let bucket = format!("bounded-detached-dispatch-rollback-{index}");
10912 store
10913 .make_bucket(&bucket, &MakeBucketOptions::default())
10914 .await
10915 .expect("bounded detached rollback bucket should be created");
10916 let incarnation = store
10917 .bucket_incarnation_id(&bucket)
10918 .await
10919 .expect("bounded detached rollback bucket incarnation should resolve");
10920 let (manifest_name, _) = install_aborting_dispatch_fixture(
10921 store.clone(),
10922 &bucket,
10923 incarnation,
10924 &format!("detached-{index}/"),
10925 tier_name,
10926 identity,
10927 1,
10928 )
10929 .await;
10930 manifest_names.push(manifest_name);
10931 }
10932 let hook = TierDeleteDispatchRollbackTestHook::install_pause_all_deletes();
10933
10934 let first_pass = futures::stream::iter(manifest_names.iter().cloned().map(|manifest_name| {
10935 let store = store.clone();
10936 async move {
10937 recover_test_tier_delete_dispatch_manifest_with_page_budget(store, &manifest_name, Duration::from_millis(10))
10938 .await
10939 }
10940 }))
10941 .buffer_unordered(MANIFEST_COUNT)
10942 .collect::<Vec<_>>()
10943 .await;
10944 tokio::time::timeout(Duration::from_secs(30), hook.wait_until_delete_pause_count(WORKER_LIMIT))
10945 .await
10946 .expect("every admitted detached worker should pause before its first mutation");
10947 assert_eq!(
10948 first_pass.into_iter().filter(|timed_out| *timed_out).count(),
10949 WORKER_LIMIT,
10950 "only the per-store worker budget may detach"
10951 );
10952 assert_eq!(
10953 tier_delete_dispatch_manifest_recovery_count_for_test(&store),
10954 WORKER_LIMIT,
10955 "detached workers must remain bounded per store"
10956 );
10957
10958 let repeated_pass = futures::stream::iter(manifest_names.iter().cloned().map(|manifest_name| {
10959 let store = store.clone();
10960 async move {
10961 recover_test_tier_delete_dispatch_manifest_with_page_budget(store, &manifest_name, Duration::from_secs(30)).await
10962 }
10963 }))
10964 .buffer_unordered(MANIFEST_COUNT)
10965 .collect::<Vec<_>>()
10966 .await;
10967 assert!(
10968 repeated_pass.into_iter().all(|timed_out| !timed_out),
10969 "duplicates and over-budget manifests must be retained without a waiter queue"
10970 );
10971 assert_eq!(tier_delete_dispatch_manifest_recovery_count_for_test(&store), WORKER_LIMIT);
10972
10973 shutdown.cancel();
10974 tokio::time::sleep(Duration::from_millis(50)).await;
10975 assert_eq!(
10976 tier_delete_dispatch_manifest_recovery_count_for_test(&store),
10977 WORKER_LIMIT,
10978 "shutdown must not drop a bounded batch before its admitted work drains"
10979 );
10980 assert_eq!(tier_delete_journal_count(store.clone()).await, MANIFEST_COUNT);
10981 assert_eq!(tier_delete_dispatch_manifest_count(store.clone()).await, MANIFEST_COUNT);
10982 hook.release_all_deletes();
10983 tokio::time::timeout(Duration::from_secs(30), async {
10984 while tier_delete_dispatch_manifest_recovery_count_for_test(&store) != 0 {
10985 tokio::time::sleep(Duration::from_millis(10)).await;
10986 }
10987 })
10988 .await
10989 .expect("shutdown should cancel every detached worker and release its registry permit");
10990 assert_eq!(tier_delete_journal_count(store.clone()).await, MANIFEST_COUNT);
10991 assert_eq!(tier_delete_dispatch_manifest_count(store.clone()).await, MANIFEST_COUNT);
10992 drop(hook);
10993 tokio::time::sleep(Duration::from_millis(50)).await;
10994 assert_eq!(
10995 tier_delete_journal_count(store.clone()).await,
10996 MANIFEST_COUNT,
10997 "releasing the old hook after shutdown must not resume a cancelled mutation"
10998 );
10999 assert_eq!(
11000 tier_delete_dispatch_manifest_recovery_count_for_test(&store),
11001 0,
11002 "the shutdown store must leave no registry permits behind"
11003 );
11004 drop(store);
11005 drop(ctx);
11006
11007 let (_restarted_ctx, restarted_store, restarted_shutdown) = without_storage_class_env(build_isolated_test_store(
11008 temp_dir.path(),
11009 "bounded-detached-dispatch-rollback-restart",
11010 &[4],
11011 ))
11012 .await;
11013 crate::bucket::metadata_sys::init_bucket_metadata_sys(restarted_store.clone(), Vec::new()).await;
11014 assert!(!restarted_shutdown.is_cancelled(), "the restarted store needs a fresh shutdown token");
11015 tokio::time::timeout(Duration::from_secs(30), async {
11016 loop {
11017 let active = tier_delete_dispatch_manifest_recovery_count_for_test(&restarted_store);
11018 let manifests = tier_delete_dispatch_manifest_count(restarted_store.clone()).await;
11019 if active > 0 || manifests < MANIFEST_COUNT {
11020 break;
11021 }
11022 tokio::task::yield_now().await;
11023 }
11024 })
11025 .await
11026 .expect("the restarted store should start a fresh manifest recovery");
11027 tokio::time::timeout(Duration::from_secs(30), async {
11028 while tier_delete_dispatch_manifest_recovery_count_for_test(&restarted_store) != 0 {
11029 tokio::time::sleep(Duration::from_millis(10)).await;
11030 }
11031 })
11032 .await
11033 .expect("the restarted store recovery should release every worker permit");
11034 if tier_delete_dispatch_manifest_count(restarted_store.clone()).await != 0 {
11035 let stats = recover_tier_delete_dispatch_manifests(restarted_store.clone(), 100, None)
11036 .await
11037 .expect("the restarted store should retry any conservatively retained manifest");
11038 assert_eq!(stats.failed, 0, "the restarted recovery must not quarantine a valid manifest");
11039 }
11040 assert_eq!(tier_delete_journal_count(restarted_store.clone()).await, 0);
11041 assert_eq!(tier_delete_dispatch_manifest_count(restarted_store).await, 0);
11042 restarted_shutdown.cancel();
11043 }
11044
11045 #[cfg(feature = "test-util")]
11046 #[tokio::test]
11047 #[serial_test::serial(storage_class_env)]
11048 async fn dispatch_manifest_shutdown_stops_aborting_validation_tail_reads() {
11049 const JOURNAL_COUNT: usize = 65;
11050 const ADMITTED_READS: usize = 32;
11051
11052 let temp_dir = tempfile::tempdir().expect("create validation-read shutdown store dir");
11053 let (ctx, store, shutdown) =
11054 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "dispatch-validation-read-shutdown", &[4]))
11055 .await;
11056 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
11057 let bucket = "dispatch-validation-read-shutdown-bucket";
11058 store
11059 .make_bucket(bucket, &MakeBucketOptions::default())
11060 .await
11061 .expect("validation-read shutdown bucket should be created");
11062 let incarnation = store
11063 .bucket_incarnation_id(bucket)
11064 .await
11065 .expect("validation-read shutdown bucket incarnation should resolve");
11066 let tier_name = "VALIDATION-READ-SHUTDOWN";
11067 let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
11068 let identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
11069 .await
11070 .expect("validation-read shutdown tier lease should resolve")
11071 .backend_identity();
11072 let (manifest_name, _) =
11073 install_aborting_dispatch_fixture(store.clone(), bucket, incarnation, "archive/", tier_name, identity, JOURNAL_COUNT)
11074 .await;
11075 let hook = TierDeleteDispatchMemberReadTestHook::install_pause(TierDeleteDispatchMemberReadTestStage::Validation);
11076 let worker_store = store.clone();
11077 let worker_manifest = manifest_name.clone();
11078 let worker = tokio::spawn(async move {
11079 recover_test_tier_delete_dispatch_manifest_with_page_budget(worker_store, &worker_manifest, Duration::from_secs(30))
11080 .await
11081 });
11082 tokio::time::timeout(Duration::from_secs(30), hook.wait_until_read_pause_count(ADMITTED_READS))
11083 .await
11084 .expect("the first validation read window should be admitted");
11085 assert_eq!(hook.entry_count(), ADMITTED_READS);
11086 assert_eq!(hook.in_flight(), ADMITTED_READS);
11087
11088 shutdown.cancel();
11089 tokio::time::sleep(Duration::from_millis(50)).await;
11090 assert_eq!(hook.entry_count(), ADMITTED_READS, "shutdown must not admit validation tail reads");
11091 assert_eq!(tier_delete_dispatch_manifest_recovery_count_for_test(&store), 1);
11092 assert!(tier_delete_dispatch_manifest_operation_lock_held_for_test(store.clone(), &manifest_name).await);
11093 assert!(crate::services::notification_sys::tier_delete_journal_fleet_proof_has_inflight_for_test());
11094 assert_eq!(tier_delete_journal_count(store.clone()).await, JOURNAL_COUNT);
11095 assert_eq!(backend.remove_count().await, 0);
11096
11097 hook.release_all_reads();
11098 assert!(
11099 !worker.await.expect("validation-read shutdown worker should join"),
11100 "cooperative shutdown should retain an unvalidated rollback"
11101 );
11102 assert_eq!(hook.entry_count(), ADMITTED_READS);
11103 assert_eq!(hook.in_flight(), 0);
11104 assert_eq!(tier_delete_dispatch_manifest_recovery_count_for_test(&store), 0);
11105 assert_eq!(
11106 tier_delete_journal_count(store.clone()).await,
11107 JOURNAL_COUNT,
11108 "no rollback DELETE may start"
11109 );
11110 assert_eq!(
11111 test_tier_delete_dispatch_manifest_state(store.clone(), &manifest_name)
11112 .await
11113 .expect("retained Aborting manifest should remain readable"),
11114 Some(TierDeleteDispatchManifestState::Aborting)
11115 );
11116 drop(hook);
11117 drop(store);
11118 drop(ctx);
11119
11120 let (_restarted_ctx, restarted_store, restarted_shutdown) = without_storage_class_env(build_isolated_test_store(
11121 temp_dir.path(),
11122 "dispatch-validation-read-shutdown-restart",
11123 &[4],
11124 ))
11125 .await;
11126 crate::bucket::metadata_sys::init_bucket_metadata_sys(restarted_store.clone(), Vec::new()).await;
11127 drive_tier_delete_dispatch_restart_to_convergence(restarted_store.clone()).await;
11128 restarted_shutdown.cancel();
11129 }
11130
11131 #[cfg(feature = "test-util")]
11132 #[tokio::test]
11133 #[serial_test::serial(storage_class_env)]
11134 async fn dispatch_manifest_shutdown_stops_authorized_tail_reads_before_receipt_or_cas() {
11135 const JOURNAL_COUNT: usize = 65;
11136 const ADMITTED_READS: usize = 32;
11137
11138 let temp_dir = tempfile::tempdir().expect("create authorized-read shutdown store dir");
11139 let (ctx, store, shutdown) =
11140 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "dispatch-authorized-read-shutdown", &[4]))
11141 .await;
11142 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
11143 let bucket = "dispatch-authorized-read-shutdown-bucket";
11144 store
11145 .make_bucket(bucket, &MakeBucketOptions::default())
11146 .await
11147 .expect("authorized-read shutdown bucket should be created");
11148 let incarnation = store
11149 .bucket_incarnation_id(bucket)
11150 .await
11151 .expect("authorized-read shutdown bucket incarnation should resolve");
11152 let tier_name = "AUTHORIZED-READ-SHUTDOWN";
11153 let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
11154 let identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
11155 .await
11156 .expect("authorized-read shutdown tier lease should resolve")
11157 .backend_identity();
11158 let entries = (0..JOURNAL_COUNT)
11159 .map(|index| {
11160 (
11161 synthetic_v6_dispatch_entry(
11162 bucket,
11163 &format!("archive/{index:06}.bin"),
11164 tier_name,
11165 identity,
11166 &uuid::Uuid::new_v4().to_string(),
11167 ),
11168 Some(TierDeleteJournalState::Committed),
11169 )
11170 })
11171 .collect();
11172 let (manifest_name, _) = install_test_tier_delete_dispatch_fixture(
11173 store.clone(),
11174 bucket,
11175 incarnation,
11176 "archive/",
11177 entries,
11178 TierDeleteDispatchManifestState::DispatchAuthorized,
11179 )
11180 .await
11181 .expect("authorized-read shutdown fixture should persist");
11182 let tier_config = ctx
11183 .tier_config_mgr()
11184 .read()
11185 .await
11186 .tiers
11187 .get(tier_name)
11188 .cloned()
11189 .expect("authorized-read tier config should remain available for restart");
11190 let hook = TierDeleteDispatchMemberReadTestHook::install_pause(TierDeleteDispatchMemberReadTestStage::Authorized);
11191 let worker_store = store.clone();
11192 let worker_manifest = manifest_name.clone();
11193 let worker = tokio::spawn(async move {
11194 recover_test_tier_delete_dispatch_manifest_with_page_budget(worker_store, &worker_manifest, Duration::from_secs(30))
11195 .await
11196 });
11197 tokio::time::timeout(Duration::from_secs(30), hook.wait_until_read_pause_count(ADMITTED_READS))
11198 .await
11199 .expect("the first authorized read window should be admitted");
11200
11201 shutdown.cancel();
11202 tokio::time::sleep(Duration::from_millis(50)).await;
11203 assert_eq!(hook.entry_count(), ADMITTED_READS, "shutdown must not admit authorized tail reads");
11204 assert_eq!(hook.in_flight(), ADMITTED_READS);
11205 assert_eq!(tier_delete_dispatch_manifest_recovery_count_for_test(&store), 1);
11206 assert!(tier_delete_dispatch_manifest_operation_lock_held_for_test(store.clone(), &manifest_name).await);
11207 assert!(crate::services::notification_sys::tier_delete_journal_fleet_proof_has_inflight_for_test());
11208
11209 hook.release_all_reads();
11210 assert!(
11211 !worker.await.expect("authorized-read shutdown worker should join"),
11212 "cooperative shutdown should retain an unscanned authorized manifest"
11213 );
11214 assert_eq!(hook.entry_count(), ADMITTED_READS);
11215 assert_eq!(hook.in_flight(), 0);
11216 assert_eq!(tier_delete_dispatch_manifest_recovery_count_for_test(&store), 0);
11217 assert_eq!(
11218 test_tier_delete_dispatch_manifest_state(store.clone(), &manifest_name)
11219 .await
11220 .expect("retained Authorized manifest should remain readable"),
11221 Some(TierDeleteDispatchManifestState::DispatchAuthorized),
11222 "cancellation before member reads must prevent the Completed CAS"
11223 );
11224 assert_eq!(
11225 hook.authorized_progress_count(),
11226 0,
11227 "cancellation before member reads must not attempt a decommission progress receipt"
11228 );
11229 assert_eq!(backend.remove_count().await, 0);
11230 drop(hook);
11231 drop(store);
11232 drop(ctx);
11233
11234 let (restarted_ctx, restarted_store, restarted_shutdown) = without_storage_class_env(build_isolated_test_store(
11235 temp_dir.path(),
11236 "dispatch-authorized-read-shutdown-restart",
11237 &[4],
11238 ))
11239 .await;
11240 {
11241 let tier_config_mgr = restarted_ctx.tier_config_mgr();
11242 let mut manager = tier_config_mgr.write().await;
11243 manager.tiers.insert(tier_name.to_string(), tier_config);
11244 manager
11245 .install_test_driver(tier_name, Box::new(backend.clone()))
11246 .expect("the exact authorized-read tier driver should reinstall after restart");
11247 }
11248 crate::bucket::metadata_sys::init_bucket_metadata_sys(restarted_store.clone(), Vec::new()).await;
11249 drive_tier_delete_dispatch_restart_to_convergence(restarted_store.clone()).await;
11250 assert_eq!(backend.remove_count().await, JOURNAL_COUNT);
11251 restarted_shutdown.cancel();
11252 }
11253
11254 #[cfg(feature = "test-util")]
11255 #[tokio::test]
11256 #[serial_test::serial(storage_class_env)]
11257 async fn dispatch_completion_cas_is_bounded_and_reaches_the_tail() {
11258 const JOURNAL_COUNT: usize = 65;
11259 const ADMITTED_COMMITS: usize = 32;
11260
11261 let temp_dir = tempfile::tempdir().expect("create dispatch completion concurrency store dir");
11262 let (ctx, store, shutdown) =
11263 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "dispatch-completion-concurrency", &[4])).await;
11264 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
11265 let bucket = "dispatch-completion-concurrency-bucket";
11266 let prefix = "archive/";
11267 store
11268 .make_bucket(bucket, &MakeBucketOptions::default())
11269 .await
11270 .expect("dispatch completion concurrency bucket should be created");
11271 let incarnation = store
11272 .bucket_incarnation_id(bucket)
11273 .await
11274 .expect("dispatch completion concurrency bucket incarnation should resolve");
11275 let tier_name = "DISPATCH-COMPLETION-CONCURRENCY";
11276 let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
11277 backend.set_remove_failure(true);
11278 let identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
11279 .await
11280 .expect("dispatch completion concurrency tier lease should resolve")
11281 .backend_identity();
11282 let entries = (0..JOURNAL_COUNT)
11283 .map(|index| {
11284 synthetic_v6_dispatch_entry(
11285 bucket,
11286 &format!("{prefix}{index:06}.bin"),
11287 tier_name,
11288 identity,
11289 &uuid::Uuid::new_v4().to_string(),
11290 )
11291 })
11292 .collect::<Vec<_>>();
11293 let (manifest_name, _) = install_test_tier_delete_dispatch_fixture(
11294 store.clone(),
11295 bucket,
11296 incarnation,
11297 prefix,
11298 entries
11299 .iter()
11300 .cloned()
11301 .map(|entry| (entry, Some(TierDeleteJournalState::Dispatched)))
11302 .collect(),
11303 TierDeleteDispatchManifestState::DispatchAuthorized,
11304 )
11305 .await
11306 .expect("dispatch completion concurrency fixture should persist");
11307
11308 let lifecycle_guard = store
11309 .acquire_bucket_lifecycle_write_lock(bucket)
11310 .await
11311 .expect("dispatch completion should acquire the bucket lifecycle fence");
11312 let mut fence_opts = ObjectOptions::default();
11313 fence_opts.add_bucket_lifecycle_lock_guard(&lifecycle_guard);
11314 let bucket_fence = fence_opts
11315 .bucket_lifecycle_lock_fence
11316 .clone()
11317 .expect("dispatch completion should capture the bucket lifecycle fence");
11318 let fleet_proof =
11319 acquire_tier_delete_journal_fleet_proof().expect("dispatch completion concurrency fixture should have a fleet proof");
11320 let prepared =
11321 prepare_tier_delete_dispatch(store.clone(), bucket, incarnation, prefix, entries, fleet_proof, &bucket_fence)
11322 .await
11323 .expect("the existing Authorized dispatch should reload");
11324 let active = prepared
11325 .consume(bucket, incarnation, prefix)
11326 .expect("the existing Authorized dispatch permit should be consumable");
11327 active
11328 .authorization()
11329 .mark_mutation_started(bucket, incarnation, prefix)
11330 .expect("the completion test should mark its synthetic local mutation");
11331
11332 let hook = TierDeleteDispatchMemberReadTestHook::install_pause(TierDeleteDispatchMemberReadTestStage::Completion);
11333 let worker_store = store.clone();
11334 let worker = tokio::spawn(async move { complete_tier_delete_dispatch(worker_store, &active, &bucket_fence).await });
11335 tokio::time::timeout(Duration::from_secs(30), hook.wait_until_read_pause_count(ADMITTED_COMMITS))
11336 .await
11337 .expect("the first completion CAS window should be admitted");
11338 tokio::time::sleep(Duration::from_millis(50)).await;
11339 assert_eq!(hook.entry_count(), ADMITTED_COMMITS, "completion must not admit an unbounded CAS tail");
11340 assert_eq!(hook.in_flight(), ADMITTED_COMMITS);
11341
11342 hook.release_all_reads();
11343 worker
11344 .await
11345 .expect("dispatch completion concurrency worker should join")
11346 .expect("every bounded completion CAS should succeed");
11347 assert_eq!(
11348 hook.entry_count(),
11349 JOURNAL_COUNT,
11350 "completion must eventually admit the entire journal set"
11351 );
11352 assert_eq!(hook.in_flight(), 0);
11353 assert_eq!(
11354 test_tier_delete_dispatch_manifest_state(store.clone(), &manifest_name)
11355 .await
11356 .expect("completed concurrency manifest should remain readable"),
11357 Some(TierDeleteDispatchManifestState::Completed)
11358 );
11359 drop(hook);
11360 shutdown.cancel();
11361 }
11362
11363 #[cfg(feature = "test-util")]
11364 #[tokio::test]
11365 #[serial_test::serial(storage_class_env)]
11366 async fn dispatch_manifest_shutdown_stops_completed_missing_journal_tail_reads() {
11367 const JOURNAL_COUNT: usize = 65;
11368 const ADMITTED_READS: usize = 32;
11369
11370 let temp_dir = tempfile::tempdir().expect("create completed-read shutdown store dir");
11371 let (ctx, store, shutdown) =
11372 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "dispatch-completed-read-shutdown", &[4])).await;
11373 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
11374 let bucket = "dispatch-completed-read-shutdown-bucket";
11375 store
11376 .make_bucket(bucket, &MakeBucketOptions::default())
11377 .await
11378 .expect("completed-read shutdown bucket should be created");
11379 let incarnation = store
11380 .bucket_incarnation_id(bucket)
11381 .await
11382 .expect("completed-read shutdown bucket incarnation should resolve");
11383 let tier_name = "COMPLETED-READ-SHUTDOWN";
11384 let _backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
11385 let identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
11386 .await
11387 .expect("completed-read shutdown tier lease should resolve")
11388 .backend_identity();
11389 let entries = (0..JOURNAL_COUNT)
11390 .map(|index| {
11391 (
11392 synthetic_v6_dispatch_entry(
11393 bucket,
11394 &format!("archive/{index:06}.bin"),
11395 tier_name,
11396 identity,
11397 &uuid::Uuid::new_v4().to_string(),
11398 ),
11399 None,
11400 )
11401 })
11402 .collect();
11403 let (manifest_name, _) = install_test_tier_delete_dispatch_fixture(
11404 store.clone(),
11405 bucket,
11406 incarnation,
11407 "archive/",
11408 entries,
11409 TierDeleteDispatchManifestState::Completed,
11410 )
11411 .await
11412 .expect("completed-read shutdown fixture should persist");
11413 let hook = TierDeleteDispatchMemberReadTestHook::install_pause(TierDeleteDispatchMemberReadTestStage::Completed);
11414 let worker_store = store.clone();
11415 let worker_manifest = manifest_name.clone();
11416 let worker = tokio::spawn(async move {
11417 recover_test_tier_delete_dispatch_manifest_with_page_budget(worker_store, &worker_manifest, Duration::from_secs(30))
11418 .await
11419 });
11420 tokio::time::timeout(Duration::from_secs(30), hook.wait_until_read_pause_count(ADMITTED_READS))
11421 .await
11422 .expect("the first Completed read window should be admitted");
11423
11424 shutdown.cancel();
11425 tokio::time::sleep(Duration::from_millis(50)).await;
11426 assert_eq!(hook.entry_count(), ADMITTED_READS, "shutdown must not admit later Completed chunks");
11427 assert_eq!(hook.in_flight(), ADMITTED_READS);
11428 assert_eq!(tier_delete_dispatch_manifest_recovery_count_for_test(&store), 1);
11429 assert!(tier_delete_dispatch_manifest_operation_lock_held_for_test(store.clone(), &manifest_name).await);
11430 assert!(crate::services::notification_sys::tier_delete_journal_fleet_proof_has_inflight_for_test());
11431
11432 hook.release_all_reads();
11433 assert!(
11434 !worker.await.expect("completed-read shutdown worker should join"),
11435 "cooperative shutdown should retain a partially scanned Completed manifest"
11436 );
11437 assert_eq!(hook.entry_count(), ADMITTED_READS);
11438 assert_eq!(hook.in_flight(), 0);
11439 assert_eq!(tier_delete_dispatch_manifest_recovery_count_for_test(&store), 0);
11440 assert_eq!(tier_delete_journal_count(store.clone()).await, 0);
11441 assert_eq!(
11442 tier_delete_dispatch_manifest_count(store.clone()).await,
11443 1,
11444 "manifest DELETE must not run"
11445 );
11446 assert_eq!(
11447 test_tier_delete_dispatch_manifest_state(store.clone(), &manifest_name)
11448 .await
11449 .expect("retained Completed manifest should remain readable"),
11450 Some(TierDeleteDispatchManifestState::Completed)
11451 );
11452 drop(hook);
11453 drop(store);
11454 drop(ctx);
11455
11456 let (_restarted_ctx, restarted_store, restarted_shutdown) = without_storage_class_env(build_isolated_test_store(
11457 temp_dir.path(),
11458 "dispatch-completed-read-shutdown-restart",
11459 &[4],
11460 ))
11461 .await;
11462 crate::bucket::metadata_sys::init_bucket_metadata_sys(restarted_store.clone(), Vec::new()).await;
11463 drive_tier_delete_dispatch_restart_to_convergence(restarted_store.clone()).await;
11464 restarted_shutdown.cancel();
11465 }
11466
11467 #[cfg(feature = "test-util")]
11468 #[tokio::test]
11469 #[serial_test::serial(storage_class_env)]
11470 async fn dispatch_manifest_shutdown_drains_preparing_cas_before_releasing_fences() {
11471 let temp_dir = tempfile::tempdir().expect("create preparing CAS shutdown store dir");
11472 let (ctx, store, shutdown) =
11473 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "dispatch-preparing-cas-shutdown", &[4])).await;
11474 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
11475 let bucket = "dispatch-preparing-cas-shutdown-bucket";
11476 store
11477 .make_bucket(bucket, &MakeBucketOptions::default())
11478 .await
11479 .expect("preparing CAS shutdown bucket should be created");
11480 let incarnation = store
11481 .bucket_incarnation_id(bucket)
11482 .await
11483 .expect("preparing CAS shutdown bucket incarnation should resolve");
11484 let entry = synthetic_v6_dispatch_entry(
11485 bucket,
11486 "archive/preparing-cas.bin",
11487 "PREPARING-CAS-SHUTDOWN",
11488 [31; 32],
11489 &uuid::Uuid::new_v4().to_string(),
11490 );
11491 let (manifest_name, _) = install_test_tier_delete_dispatch_fixture(
11492 store.clone(),
11493 bucket,
11494 incarnation,
11495 "archive/",
11496 vec![(entry, Some(TierDeleteJournalState::Prepared))],
11497 TierDeleteDispatchManifestState::Preparing,
11498 )
11499 .await
11500 .expect("preparing CAS shutdown fixture should persist");
11501 let barrier = crate::set_disk::PutObjectCommitBarrier::install(
11502 RUSTFS_META_BUCKET,
11503 &manifest_name,
11504 crate::set_disk::PutObjectCommitPause::BeforeQuotaRename,
11505 );
11506 let worker_store = store.clone();
11507 let worker_manifest = manifest_name.clone();
11508 let worker = tokio::spawn(async move {
11509 recover_test_tier_delete_dispatch_manifest_with_page_budget(worker_store, &worker_manifest, Duration::from_secs(30))
11510 .await
11511 });
11512 barrier.wait_until_paused().await;
11513
11514 shutdown.cancel();
11515 tokio::time::sleep(Duration::from_millis(50)).await;
11516 assert_eq!(
11517 tier_delete_dispatch_manifest_recovery_count_for_test(&store),
11518 1,
11519 "a paused CAS must retain its registry permit after shutdown"
11520 );
11521 assert!(
11522 tier_delete_dispatch_manifest_operation_lock_held_for_test(store.clone(), &manifest_name).await,
11523 "a paused CAS must retain its manifest operation lock after shutdown"
11524 );
11525 assert!(
11526 crate::services::notification_sys::tier_delete_journal_fleet_proof_has_inflight_for_test(),
11527 "a paused CAS must retain its fleet-proof generation permit"
11528 );
11529
11530 barrier.release();
11531 assert!(
11532 !worker.await.expect("preparing CAS shutdown worker should join"),
11533 "cooperative shutdown should retain the partially advanced manifest"
11534 );
11535 assert_eq!(tier_delete_dispatch_manifest_recovery_count_for_test(&store), 0);
11536 assert!(
11537 !tier_delete_dispatch_manifest_operation_lock_held_for_test(store.clone(), &manifest_name).await,
11538 "the operation lock may release only after the admitted CAS drains"
11539 );
11540 assert!(
11541 com::read_config(store.clone(), &manifest_name).await.is_ok(),
11542 "shutdown must retain the durable manifest for restart replay"
11543 );
11544 drop(barrier);
11545 drop(store);
11546 drop(ctx);
11547
11548 let (_restarted_ctx, restarted_store, restarted_shutdown) = without_storage_class_env(build_isolated_test_store(
11549 temp_dir.path(),
11550 "dispatch-preparing-cas-shutdown-restart",
11551 &[4],
11552 ))
11553 .await;
11554 crate::bucket::metadata_sys::init_bucket_metadata_sys(restarted_store.clone(), Vec::new()).await;
11555 tokio::time::timeout(Duration::from_secs(30), async {
11556 let mut stable_empty_observations = 0;
11557 loop {
11558 if tier_delete_journal_count(restarted_store.clone()).await == 0
11559 && tier_delete_dispatch_manifest_count(restarted_store.clone()).await == 0
11560 && tier_delete_dispatch_manifest_recovery_count_for_test(&restarted_store) == 0
11561 {
11562 stable_empty_observations += 1;
11563 if stable_empty_observations == 10 {
11564 break;
11565 }
11566 } else {
11567 stable_empty_observations = 0;
11568 }
11569 tokio::time::sleep(Duration::from_millis(50)).await;
11570 }
11571 })
11572 .await
11573 .expect("the restarted background recovery should replay the drained Preparing CAS");
11574 assert_eq!(tier_delete_journal_count(restarted_store.clone()).await, 0);
11575 assert_eq!(tier_delete_dispatch_manifest_count(restarted_store).await, 0);
11576 restarted_shutdown.cancel();
11577 }
11578
11579 #[cfg(feature = "test-util")]
11580 #[tokio::test]
11581 #[serial_test::serial(storage_class_env)]
11582 async fn dispatch_manifest_shutdown_drains_admitted_delete_batch_without_tail_admission() {
11583 const JOURNAL_COUNT: usize = 65;
11584 const ADMITTED_BATCH: usize = 32;
11585
11586 let temp_dir = tempfile::tempdir().expect("create delete-batch shutdown store dir");
11587 let (ctx, store, shutdown) =
11588 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "dispatch-delete-batch-shutdown", &[4])).await;
11589 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
11590 let bucket = "dispatch-delete-batch-shutdown-bucket";
11591 store
11592 .make_bucket(bucket, &MakeBucketOptions::default())
11593 .await
11594 .expect("delete-batch shutdown bucket should be created");
11595 let incarnation = store
11596 .bucket_incarnation_id(bucket)
11597 .await
11598 .expect("delete-batch shutdown bucket incarnation should resolve");
11599 let (manifest_name, entries) = install_aborting_dispatch_fixture(
11600 store.clone(),
11601 bucket,
11602 incarnation,
11603 "archive/",
11604 "DELETE-BATCH-SHUTDOWN",
11605 [32; 32],
11606 JOURNAL_COUNT,
11607 )
11608 .await;
11609 let mut journal_names = entries.iter().map(tier_delete_journal_object_name).collect::<Vec<_>>();
11610 journal_names.sort();
11611 let published_delete_name = journal_names[0].clone();
11612 let hook = TierDeleteDispatchRollbackTestHook::install_pause_all_except_delete(&published_delete_name);
11613 let barrier =
11614 crate::set_disk::DeleteObjectCommitBarrier::install_after_publish(RUSTFS_META_BUCKET, &published_delete_name);
11615 let worker_store = store.clone();
11616 let worker_manifest = manifest_name.clone();
11617 let worker = tokio::spawn(async move {
11618 recover_test_tier_delete_dispatch_manifest_with_page_budget(worker_store, &worker_manifest, Duration::from_secs(30))
11619 .await
11620 });
11621 barrier.wait_until_paused().await;
11622 tokio::time::timeout(Duration::from_secs(30), hook.wait_until_delete_pause_count(ADMITTED_BATCH - 1))
11623 .await
11624 .expect("the rest of the bounded delete batch should pause before mutation");
11625 assert_eq!(hook.delete_entry_count(), ADMITTED_BATCH);
11626
11627 shutdown.cancel();
11628 tokio::time::sleep(Duration::from_millis(50)).await;
11629 assert_eq!(tier_delete_dispatch_manifest_recovery_count_for_test(&store), 1);
11630 assert!(
11631 tier_delete_dispatch_manifest_operation_lock_held_for_test(store.clone(), &manifest_name).await,
11632 "the partially published DELETE must retain its operation lock"
11633 );
11634 assert!(
11635 crate::services::notification_sys::tier_delete_journal_fleet_proof_has_inflight_for_test(),
11636 "the partially published DELETE must retain its fleet-proof permit"
11637 );
11638 assert_eq!(
11639 hook.delete_entry_count(),
11640 ADMITTED_BATCH,
11641 "shutdown must stop admitting the tail behind the bounded batch"
11642 );
11643
11644 barrier.release();
11645 tokio::time::sleep(Duration::from_millis(50)).await;
11646 assert_eq!(
11647 tier_delete_dispatch_manifest_recovery_count_for_test(&store),
11648 1,
11649 "the worker must retain its guards until every admitted sibling drains"
11650 );
11651 hook.release_all_deletes();
11652 assert!(
11653 !worker.await.expect("delete-batch shutdown worker should join"),
11654 "cooperative shutdown should retain the partial rollback for restart"
11655 );
11656 assert_eq!(hook.delete_entry_count(), ADMITTED_BATCH);
11657 assert_eq!(tier_delete_dispatch_manifest_recovery_count_for_test(&store), 0);
11658 assert!(
11659 tier_delete_journal_count(store.clone()).await >= JOURNAL_COUNT - 1,
11660 "only the already-published DELETE may commit before restart"
11661 );
11662 drop(barrier);
11663 drop(hook);
11664 drop(store);
11665 drop(ctx);
11666
11667 let (_restarted_ctx, restarted_store, restarted_shutdown) =
11668 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "dispatch-delete-batch-shutdown-restart", &[4]))
11669 .await;
11670 crate::bucket::metadata_sys::init_bucket_metadata_sys(restarted_store.clone(), Vec::new()).await;
11671 tokio::time::timeout(Duration::from_secs(30), async {
11672 let mut stable_empty_observations = 0;
11673 loop {
11674 if tier_delete_journal_count(restarted_store.clone()).await == 0
11675 && tier_delete_dispatch_manifest_count(restarted_store.clone()).await == 0
11676 && tier_delete_dispatch_manifest_recovery_count_for_test(&restarted_store) == 0
11677 {
11678 stable_empty_observations += 1;
11679 if stable_empty_observations == 10 {
11680 break;
11681 }
11682 } else {
11683 stable_empty_observations = 0;
11684 }
11685 tokio::time::sleep(Duration::from_millis(50)).await;
11686 }
11687 })
11688 .await
11689 .expect("the restarted background recovery should replay the partially drained delete batch");
11690 assert_eq!(tier_delete_journal_count(restarted_store.clone()).await, 0);
11691 assert_eq!(tier_delete_dispatch_manifest_count(restarted_store).await, 0);
11692 restarted_shutdown.cancel();
11693 }
11694
11695 #[cfg(feature = "test-util")]
11696 #[tokio::test]
11697 #[serial_test::serial(storage_class_env)]
11698 async fn dispatch_manifest_rollback_retries_delete_and_confirmation_failures() {
11699 const JOURNAL_COUNT: usize = 40;
11700
11701 let temp_dir = tempfile::tempdir().expect("create rollback retry store dir");
11702 let mut instance_ctx = crate::runtime::instance::InstanceContext::new();
11704 instance_ctx.suppress_tier_delete_journal_recovery_for_test();
11705 let (ctx, store, shutdown) = without_storage_class_env(build_isolated_test_store_with_layout(
11706 temp_dir.path(),
11707 "dispatch-rollback-retry",
11708 &[(1, 4)],
11709 CancellationToken::new(),
11710 Some(Arc::new(instance_ctx)),
11711 ))
11712 .await;
11713 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
11714 let bucket = "dispatch-rollback-retry-bucket";
11715 store
11716 .make_bucket(bucket, &MakeBucketOptions::default())
11717 .await
11718 .expect("rollback retry bucket should be created");
11719 let incarnation = store
11720 .bucket_incarnation_id(bucket)
11721 .await
11722 .expect("rollback retry bucket incarnation should resolve");
11723 let tier_name = "DISPATCH-ROLLBACK-RETRY";
11724 let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
11725 let identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
11726 .await
11727 .expect("rollback retry tier lease should resolve")
11728 .backend_identity();
11729
11730 let (delete_manifest, delete_entries) = install_aborting_dispatch_fixture(
11731 store.clone(),
11732 bucket,
11733 incarnation,
11734 "delete-failure/",
11735 tier_name,
11736 identity,
11737 JOURNAL_COUNT,
11738 )
11739 .await;
11740 let delete_failure_name = tier_delete_journal_object_name(&delete_entries[10]);
11741 let delete_hook = TierDeleteDispatchRollbackTestHook::install_delete_failure(&delete_failure_name);
11742 let delete_error = recover_test_tier_delete_dispatch_manifest(store.clone(), &delete_manifest)
11743 .await
11744 .expect_err("an injected member delete failure must retain the manifest")
11745 .to_string();
11746 assert!(delete_error.contains("injected tier delete dispatch rollback delete failure"));
11747 assert!(
11748 tier_delete_journal_count(store.clone()).await > 0,
11749 "the failed member and all work not admitted after the first error must remain retryable"
11750 );
11751 drop(delete_hook);
11752 recover_test_tier_delete_dispatch_manifest(store.clone(), &delete_manifest)
11753 .await
11754 .expect("the delete-stage retry should converge");
11755
11756 let (confirmation_manifest, confirmation_entries) = install_aborting_dispatch_fixture(
11757 store.clone(),
11758 bucket,
11759 incarnation,
11760 "confirmation-failure/",
11761 tier_name,
11762 identity,
11763 JOURNAL_COUNT,
11764 )
11765 .await;
11766 let confirmation_failure_name = tier_delete_journal_object_name(&confirmation_entries[10]);
11767 let confirmation_hook = TierDeleteDispatchRollbackTestHook::install_confirmation_failure(&confirmation_failure_name);
11768 let confirmation_error = recover_test_tier_delete_dispatch_manifest(store.clone(), &confirmation_manifest)
11769 .await
11770 .expect_err("an injected confirmation failure must retain the manifest")
11771 .to_string();
11772 assert!(confirmation_error.contains("injected tier delete dispatch rollback confirmation failure"));
11773 assert_eq!(
11774 tier_delete_journal_count(store.clone()).await,
11775 0,
11776 "confirmation failure occurs only after the bounded delete phase drains"
11777 );
11778 assert_eq!(tier_delete_dispatch_manifest_count(store.clone()).await, 1);
11779 drop(confirmation_hook);
11780 recover_test_tier_delete_dispatch_manifest(store.clone(), &confirmation_manifest)
11781 .await
11782 .expect("the confirmation-stage retry should converge over missing members");
11783
11784 assert_eq!(tier_delete_dispatch_manifest_count(store.clone()).await, 0);
11785 assert_eq!(backend.remove_count().await, 0, "rollback retries must never call the remote tier");
11786 shutdown.cancel();
11787 }
11788
11789 #[cfg(feature = "test-util")]
11790 #[test]
11791 #[serial_test::serial(storage_class_env)]
11792 fn v6_decommission_checkpoint_no_lock_put_rejects_lost_publication_fence() {
11793 run_large_stack_async_test(
11794 "v6-checkpoint-fence-loss",
11795 v6_decommission_checkpoint_no_lock_put_rejects_lost_publication_fence_case,
11796 );
11797 }
11798
11799 #[cfg(feature = "test-util")]
11800 async fn v6_decommission_checkpoint_no_lock_put_rejects_lost_publication_fence_case() {
11801 let temp_dir = tempfile::tempdir().expect("create v6 checkpoint fence-loss store dir");
11802 let (_ctx, store, _shutdown) =
11803 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "v6-checkpoint-fence-loss", &[4, 4])).await;
11804 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
11805 let bucket = "v6-checkpoint-fence-loss-bucket";
11806 store
11807 .make_bucket(bucket, &MakeBucketOptions::default())
11808 .await
11809 .expect("checkpoint fence-loss bucket should be created");
11810 let incarnation = store
11811 .bucket_incarnation_id(bucket)
11812 .await
11813 .expect("checkpoint fence-loss bucket incarnation should resolve");
11814 let entry = synthetic_v6_dispatch_entry(
11815 bucket,
11816 "archive/fence-loss.bin",
11817 "FENCE-LOSS-TIER",
11818 [18; 32],
11819 &uuid::Uuid::new_v4().to_string(),
11820 );
11821 let (manifest_name, entries) = install_test_tier_delete_dispatch_fixture(
11822 store.clone(),
11823 bucket,
11824 incarnation,
11825 "archive/",
11826 vec![(entry, Some(TierDeleteJournalState::Prepared))],
11827 TierDeleteDispatchManifestState::Preparing,
11828 )
11829 .await
11830 .expect("checkpoint fence-loss fixture should persist");
11831 let journal_name = tier_delete_journal_object_name(&entries[0]);
11832 let manifest_data = com::read_config(store.clone(), &manifest_name)
11833 .await
11834 .expect("checkpoint fence-loss manifest should be readable");
11835 let journal_data = com::read_config(store.clone(), &journal_name)
11836 .await
11837 .expect("checkpoint fence-loss journal should be readable");
11838 for pool in &store.pools {
11839 com::save_config(pool.clone(), &manifest_name, manifest_data.clone())
11840 .await
11841 .expect("manifest copy should persist in each checkpoint fence-loss pool");
11842 com::save_config(pool.clone(), &journal_name, journal_data.clone())
11843 .await
11844 .expect("journal copy should persist in each checkpoint fence-loss pool");
11845 }
11846 mark_test_pool_decommissioning(&store, 0).await;
11847 for (path, data) in [
11848 (&manifest_name, manifest_data.as_slice()),
11849 (&journal_name, journal_data.as_slice()),
11850 ] {
11851 let record = validate_durable_ilm_record(path, data).expect("checkpoint fence-loss v6 record should validate");
11852 store
11853 .persist_decommission_durable_ilm_receipt_for_test(0, 1, path, &record, false)
11854 .await
11855 .expect("checkpoint fence-loss receipt should persist");
11856 }
11857
11858 let loss_hook = crate::store::object::DecommissionMutationFenceLossHook::install(
11859 RUSTFS_META_BUCKET,
11860 &manifest_name,
11861 crate::store::object::DecommissionMutationFenceTestPhase::Migration,
11862 );
11863 let barrier = crate::set_disk::PutObjectCommitBarrier::install(
11864 RUSTFS_META_BUCKET,
11865 &manifest_name,
11866 crate::set_disk::PutObjectCommitPause::BeforeQuotaRename,
11867 );
11868 let worker_store = store.clone();
11869 let worker_manifest = manifest_name.clone();
11870 let worker =
11871 tokio::spawn(async move { recover_test_tier_delete_dispatch_manifest(worker_store, &worker_manifest).await });
11872 barrier.wait_until_paused().await;
11873 loss_hook.mark_lost();
11874 barrier.release();
11875 drop(barrier);
11876 let error = worker
11877 .await
11878 .expect("checkpoint fence-loss recovery task should join")
11879 .expect_err("lost publication fence must reject the no-lock checkpoint PUT")
11880 .to_string();
11881 assert!(
11882 error.contains("lock") || error.contains("fence"),
11883 "unexpected checkpoint fence-loss error: {error}"
11884 );
11885 assert_eq!(
11886 com::read_config(store.pools[1].clone(), &manifest_name)
11887 .await
11888 .expect("fenced target manifest should remain readable"),
11889 manifest_data,
11890 "the target checkpoint must not commit after its publication fence is lost"
11891 );
11892
11893 drop(loss_hook);
11894 recover_test_tier_delete_dispatch_manifest(store.clone(), &manifest_name)
11895 .await
11896 .expect("checkpoint recovery should converge after a fresh fence is acquired");
11897 }
11898
11899 #[cfg(feature = "test-util")]
11900 #[tokio::test]
11901 #[serial_test::serial(storage_class_env)]
11902 async fn v6_decommission_checkpoint_partial_target_failure_retries_without_advancing_receipts() {
11903 let temp_dir = tempfile::tempdir().expect("create v6 partial checkpoint store dir");
11904 let (_ctx, store, _shutdown) =
11905 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "v6-partial-checkpoint", &[4, 4, 4])).await;
11906 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
11907 let bucket = "v6-partial-checkpoint-bucket";
11908 store
11909 .make_bucket(bucket, &MakeBucketOptions::default())
11910 .await
11911 .expect("partial checkpoint bucket should be created");
11912 let incarnation = store
11913 .bucket_incarnation_id(bucket)
11914 .await
11915 .expect("partial checkpoint bucket incarnation should resolve");
11916 let entry = synthetic_v6_dispatch_entry(
11917 bucket,
11918 "archive/partial.bin",
11919 "PARTIAL-CHECKPOINT-TIER",
11920 [19; 32],
11921 &uuid::Uuid::new_v4().to_string(),
11922 );
11923 let (manifest_name, entries) = install_test_tier_delete_dispatch_fixture(
11924 store.clone(),
11925 bucket,
11926 incarnation,
11927 "archive/",
11928 vec![(entry, Some(TierDeleteJournalState::Prepared))],
11929 TierDeleteDispatchManifestState::Preparing,
11930 )
11931 .await
11932 .expect("partial checkpoint fixture should persist");
11933 let journal_name = tier_delete_journal_object_name(&entries[0]);
11934 let manifest_data = com::read_config(store.clone(), &manifest_name)
11935 .await
11936 .expect("partial checkpoint manifest should be readable");
11937 let journal_data = com::read_config(store.clone(), &journal_name)
11938 .await
11939 .expect("partial checkpoint journal should be readable");
11940 for pool in &store.pools {
11941 com::save_config(pool.clone(), &manifest_name, manifest_data.clone())
11942 .await
11943 .expect("manifest copy should persist in each partial checkpoint pool");
11944 com::save_config(pool.clone(), &journal_name, journal_data.clone())
11945 .await
11946 .expect("journal copy should persist in each partial checkpoint pool");
11947 }
11948 mark_test_pool_decommissioning_with_split_targets(&store, 0).await;
11949 for target_pool_index in [1, 2] {
11950 for (path, data) in [
11951 (&manifest_name, manifest_data.as_slice()),
11952 (&journal_name, journal_data.as_slice()),
11953 ] {
11954 let record = validate_durable_ilm_record(path, data).expect("partial checkpoint v6 record should validate");
11955 store
11956 .persist_decommission_durable_ilm_receipt_for_test(0, target_pool_index, path, &record, false)
11957 .await
11958 .expect("partial checkpoint receipt should persist on each target");
11959 }
11960 }
11961 let (aborting_data, preparing_etag) = test_tier_delete_dispatch_manifest_checkpoint(
11962 store.clone(),
11963 &manifest_name,
11964 TierDeleteDispatchManifestState::Aborting,
11965 )
11966 .await
11967 .expect("partial checkpoint Aborting generation should encode");
11968 let targets = store
11969 .decommission_durable_ilm_checkpoint_targets(&manifest_name, &aborting_data, &preparing_etag)
11970 .await
11971 .expect("partial checkpoint targets should resolve")
11972 .expect("the active decommission should own the partial checkpoint");
11973 assert_eq!(
11974 targets.iter().map(|target| target.target_pool_index).collect::<Vec<_>>(),
11975 vec![1, 2],
11976 "the capacity reservation must exercise two independently confirmed targets"
11977 );
11978
11979 let failure_hook = DecommissionCheckpointTargetFailureHook::install(2);
11980 let error = recover_test_tier_delete_dispatch_manifest(store.clone(), &manifest_name)
11981 .await
11982 .expect_err("the injected second-target failure must reject the checkpoint");
11983 assert!(
11984 error.to_string().contains("injected decommission checkpoint target failure"),
11985 "unexpected partial checkpoint error: {error}"
11986 );
11987 let mut diagnostic = &error as &(dyn std::error::Error + 'static);
11988 let mut target_context_found = false;
11989 while let Some(source) = diagnostic.source() {
11990 if source.to_string().contains("target pool 2") {
11991 target_context_found = true;
11992 break;
11993 }
11994 diagnostic = source;
11995 }
11996 assert!(
11997 target_context_found,
11998 "the stable checkpoint error must retain the failed target in its diagnostic source chain: {error:?}"
11999 );
12000 assert_eq!(
12001 com::read_config(store.pools[1].clone(), &manifest_name)
12002 .await
12003 .expect("first target checkpoint should be readable"),
12004 aborting_data,
12005 "the first target should model a committed checkpoint before the later failure"
12006 );
12007 assert_eq!(
12008 com::read_config(store.pools[2].clone(), &manifest_name)
12009 .await
12010 .expect("second target checkpoint should be readable"),
12011 manifest_data,
12012 "the failed second target must retain the prior generation"
12013 );
12014 let retry_targets = store
12015 .decommission_durable_ilm_checkpoint_targets(&manifest_name, &aborting_data, &preparing_etag)
12016 .await
12017 .expect("unadvanced receipts must still authorize the exact partial checkpoint retry")
12018 .expect("the partial checkpoint retry should remain decommission-owned");
12019 assert!(retry_targets[0].already_committed);
12020 assert!(!retry_targets[1].already_committed);
12021
12022 drop(failure_hook);
12023 recover_test_tier_delete_dispatch_manifest(store.clone(), &manifest_name)
12024 .await
12025 .expect("the partial checkpoint should converge after the target recovers");
12026 }
12027
12028 #[cfg(feature = "test-util")]
12029 #[tokio::test]
12030 #[serial_test::serial(storage_class_env)]
12031 async fn v6_decommission_checkpoint_shutdown_drains_target_put_and_replays_capacity() {
12032 let temp_dir = tempfile::tempdir().expect("create checkpoint shutdown store dir");
12033 let (ctx, store, shutdown) =
12034 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "v6-checkpoint-shutdown", &[4, 4])).await;
12035 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
12036 let bucket = "v6-checkpoint-shutdown-bucket";
12037 store
12038 .make_bucket(bucket, &MakeBucketOptions::default())
12039 .await
12040 .expect("checkpoint shutdown bucket should be created");
12041 let incarnation = store
12042 .bucket_incarnation_id(bucket)
12043 .await
12044 .expect("checkpoint shutdown bucket incarnation should resolve");
12045 let entry = synthetic_v6_dispatch_entry(
12046 bucket,
12047 "archive/checkpoint-shutdown.bin",
12048 "CHECKPOINT-SHUTDOWN",
12049 [33; 32],
12050 &uuid::Uuid::new_v4().to_string(),
12051 );
12052 let (manifest_name, entries) = install_test_tier_delete_dispatch_fixture(
12053 store.clone(),
12054 bucket,
12055 incarnation,
12056 "archive/",
12057 vec![(entry, Some(TierDeleteJournalState::Prepared))],
12058 TierDeleteDispatchManifestState::Preparing,
12059 )
12060 .await
12061 .expect("checkpoint shutdown fixture should persist");
12062 let journal_name = tier_delete_journal_object_name(&entries[0]);
12063 let manifest_data = com::read_config(store.clone(), &manifest_name)
12064 .await
12065 .expect("checkpoint shutdown manifest should be readable");
12066 let journal_data = com::read_config(store.clone(), &journal_name)
12067 .await
12068 .expect("checkpoint shutdown journal should be readable");
12069 for pool in &store.pools {
12070 com::save_config(pool.clone(), &manifest_name, manifest_data.clone())
12071 .await
12072 .expect("manifest copy should persist in each checkpoint shutdown pool");
12073 com::save_config(pool.clone(), &journal_name, journal_data.clone())
12074 .await
12075 .expect("journal copy should persist in each checkpoint shutdown pool");
12076 }
12077 mark_test_pool_decommissioning(&store, 0).await;
12078 for (path, data) in [
12079 (&manifest_name, manifest_data.as_slice()),
12080 (&journal_name, journal_data.as_slice()),
12081 ] {
12082 let record = validate_durable_ilm_record(path, data).expect("checkpoint shutdown v6 record should validate");
12083 store
12084 .persist_decommission_durable_ilm_receipt_for_test(0, 1, path, &record, false)
12085 .await
12086 .expect("checkpoint shutdown receipt should persist");
12087 }
12088 let (aborting_data, preparing_etag) = test_tier_delete_dispatch_manifest_checkpoint(
12089 store.clone(),
12090 &manifest_name,
12091 TierDeleteDispatchManifestState::Aborting,
12092 )
12093 .await
12094 .expect("checkpoint shutdown Aborting generation should encode");
12095 let targets = store
12096 .decommission_durable_ilm_checkpoint_targets(&manifest_name, &aborting_data, &preparing_etag)
12097 .await
12098 .expect("checkpoint shutdown target should resolve")
12099 .expect("the active decommission should own the checkpoint target");
12100 assert_eq!(targets.len(), 1);
12101 let target = targets[0].clone();
12102 let consumed_before_checkpoint = store.pool_meta.read().await.pools[0]
12103 .decommission
12104 .as_ref()
12105 .and_then(|info| info.capacity_reservation.as_ref())
12106 .expect("checkpoint shutdown capacity reservation should exist")
12107 .consumed_target_physical_bytes;
12108 let precommit_error = store
12109 .run_decommission_capacity_non_growing_replacement_with_capacity_lease(
12110 target.target_pool_index,
12111 Some(target.capacity_owner),
12112 Some(aborting_data.len()),
12113 |_| async { Err::<(), Error>(Error::other("injected checkpoint failure before commit")) },
12114 )
12115 .await
12116 .expect_err("the first checkpoint attempt should fail before writing its target")
12117 .to_string();
12118 assert!(precommit_error.contains("injected checkpoint failure before commit"));
12119 assert!(
12120 store
12121 .has_decommission_capacity_temporary_mutation_state(target.target_pool_index, target.capacity_owner)
12122 .await,
12123 "a failed checkpoint attempt must retain exact retry state"
12124 );
12125 let barrier = crate::set_disk::PutObjectCommitBarrier::install(
12126 RUSTFS_META_BUCKET,
12127 &manifest_name,
12128 crate::set_disk::PutObjectCommitPause::BeforeQuotaRename,
12129 );
12130 let worker_store = store.clone();
12131 let worker_manifest = manifest_name.clone();
12132 let worker = tokio::spawn(async move {
12133 recover_test_tier_delete_dispatch_manifest_with_page_budget(worker_store, &worker_manifest, Duration::from_secs(30))
12134 .await
12135 });
12136 barrier.wait_until_paused().await;
12137 assert!(
12138 store
12139 .has_decommission_capacity_temporary_mutation_state(target.target_pool_index, target.capacity_owner)
12140 .await,
12141 "the target PUT must persist its capacity intent before commit"
12142 );
12143
12144 shutdown.cancel();
12145 tokio::time::sleep(Duration::from_millis(50)).await;
12146 assert_eq!(tier_delete_dispatch_manifest_recovery_count_for_test(&store), 1);
12147 assert!(
12148 tier_delete_dispatch_manifest_operation_lock_held_for_test(store.clone(), &manifest_name).await,
12149 "the target checkpoint PUT must retain its operation lock after shutdown"
12150 );
12151 assert!(
12152 crate::services::notification_sys::tier_delete_journal_fleet_proof_has_inflight_for_test(),
12153 "the target checkpoint PUT must retain its fleet-proof permit after shutdown"
12154 );
12155
12156 barrier.release();
12157 assert!(
12158 !worker.await.expect("checkpoint shutdown worker should join"),
12159 "cooperative shutdown should retain the partially committed checkpoint"
12160 );
12161 assert_eq!(tier_delete_dispatch_manifest_recovery_count_for_test(&store), 0);
12162 assert_eq!(
12163 com::read_config(store.pools[target.target_pool_index].clone(), &manifest_name)
12164 .await
12165 .expect("the committed target checkpoint should be readable"),
12166 aborting_data,
12167 "the admitted target PUT must finish before its fences release"
12168 );
12169 assert!(
12170 !store
12171 .has_decommission_capacity_temporary_mutation_state(target.target_pool_index, target.capacity_owner)
12172 .await,
12173 "the admitted target PUT must drain its capacity transaction before releasing the recovery fences"
12174 );
12175 {
12176 let pool_meta = store.pool_meta.read().await;
12177 let reservation = pool_meta.pools[0]
12178 .decommission
12179 .as_ref()
12180 .and_then(|info| info.capacity_reservation.as_ref())
12181 .expect("checkpoint shutdown capacity reservation should remain active");
12182 let capacity_target = reservation
12183 .targets
12184 .iter()
12185 .find(|candidate| candidate.pool_index == target.target_pool_index)
12186 .expect("checkpoint shutdown capacity target should remain allocated");
12187 assert_eq!(
12188 reservation.consumed_target_physical_bytes, consumed_before_checkpoint,
12189 "a non-growing checkpoint replacement must not consume durable migration capacity"
12190 );
12191 assert_eq!(reservation.pending_target_physical_bytes, 0);
12192 assert_eq!(reservation.inflight_target_physical_bytes, 0);
12193 assert_eq!(capacity_target.pending_physical_bytes, 0);
12194 assert!(capacity_target.temporary_mutations.is_empty());
12195 }
12196 drop(barrier);
12197 let mut reloaded_pool_meta = PoolMeta::default();
12198 reloaded_pool_meta
12199 .load(store.pools[0].clone(), store.pools.clone())
12200 .await
12201 .expect("checkpoint shutdown pool metadata should reload from disk");
12202 *store.pool_meta.write().await = reloaded_pool_meta;
12203 recover_test_tier_delete_dispatch_manifest(store.clone(), &manifest_name)
12204 .await
12205 .expect("a fresh recovery attempt should replay the committed checkpoint and receipt");
12206 assert!(
12207 !store
12208 .has_decommission_capacity_temporary_mutation_state(target.target_pool_index, target.capacity_owner)
12209 .await,
12210 "durable replay must reconcile the exact capacity transaction"
12211 );
12212 assert_eq!(
12213 store
12214 .decommission_durable_ilm_receipt_count_for_test(0)
12215 .await
12216 .expect("checkpoint shutdown receipts should be listable"),
12217 2,
12218 "durable replay must advance both the manifest and journal receipts"
12219 );
12220 drop(ctx);
12221 }
12222
12223 #[cfg(feature = "test-util")]
12224 #[tokio::test]
12225 #[serial_test::serial(storage_class_env)]
12226 async fn v6_dispatch_recovery_advances_decommission_receipts_and_cleans_target_copies() {
12227 let temp_dir = tempfile::tempdir().expect("create v6 decommission receipt store dir");
12228 let (_ctx, store, _shutdown) =
12229 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "v6-dispatch-decommission-receipts", &[4, 4]))
12230 .await;
12231 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
12232 let bucket = "v6-dispatch-decommission-receipts-bucket";
12233 store
12234 .make_bucket(bucket, &MakeBucketOptions::default())
12235 .await
12236 .expect("receipt bucket should be created");
12237 let incarnation = store
12238 .bucket_incarnation_id(bucket)
12239 .await
12240 .expect("receipt bucket incarnation should resolve");
12241 let entry = synthetic_v6_dispatch_entry(
12242 bucket,
12243 "archive/receipt.bin",
12244 "RECEIPT-TIER",
12245 [17; 32],
12246 &uuid::Uuid::new_v4().to_string(),
12247 );
12248 let (manifest_name, entries) = install_test_tier_delete_dispatch_fixture(
12249 store.clone(),
12250 bucket,
12251 incarnation,
12252 "archive/",
12253 vec![(entry, Some(TierDeleteJournalState::Prepared))],
12254 TierDeleteDispatchManifestState::Preparing,
12255 )
12256 .await
12257 .expect("Preparing v6 fixture should persist");
12258 let journal_name = tier_delete_journal_object_name(&entries[0]);
12259 let manifest_data = com::read_config(store.clone(), &manifest_name)
12260 .await
12261 .expect("manifest fixture should be readable");
12262 let journal_data = com::read_config(store.clone(), &journal_name)
12263 .await
12264 .expect("journal fixture should be readable");
12265
12266 for pool in &store.pools {
12269 com::save_config(pool.clone(), &manifest_name, manifest_data.clone())
12270 .await
12271 .expect("manifest copy should persist in each pool");
12272 com::save_config(pool.clone(), &journal_name, journal_data.clone())
12273 .await
12274 .expect("journal copy should persist in each pool");
12275 }
12276 mark_test_pool_decommissioning(&store, 0).await;
12281
12282 let missing_receipt_error = recover_test_tier_delete_dispatch_manifest(store.clone(), &manifest_name)
12283 .await
12284 .expect_err("an active decommission without receipt coverage must block v6 checkpoint mutation")
12285 .to_string();
12286 assert!(
12287 missing_receipt_error.contains("missing receipt coverage"),
12288 "unexpected missing-receipt error: {missing_receipt_error}"
12289 );
12290 assert_eq!(
12291 com::read_config(store.pools[1].clone(), &manifest_name)
12292 .await
12293 .expect("blocked checkpoint must retain the target manifest generation"),
12294 manifest_data,
12295 "missing receipt coverage must fail before mutating a target copy"
12296 );
12297
12298 for (path, data) in [
12299 (&manifest_name, manifest_data.as_slice()),
12300 (&journal_name, journal_data.as_slice()),
12301 ] {
12302 let record = validate_durable_ilm_record(path, data).expect("v6 durable record should validate");
12303 store
12304 .persist_decommission_durable_ilm_receipt_for_test(0, 1, path, &record, false)
12305 .await
12306 .expect("initial decommission receipt should persist");
12307 }
12308
12309 let (aborting_data, preparing_etag) = test_tier_delete_dispatch_manifest_checkpoint(
12314 store.clone(),
12315 &manifest_name,
12316 TierDeleteDispatchManifestState::Aborting,
12317 )
12318 .await
12319 .expect("the Aborting checkpoint should encode");
12320 let checkpoint_targets = store
12321 .decommission_durable_ilm_checkpoint_targets(&manifest_name, &aborting_data, &preparing_etag)
12322 .await
12323 .expect("receipt-bearing checkpoint targets should resolve")
12324 .expect("an active decommission should own the checkpoint");
12325 assert_eq!(checkpoint_targets.len(), 1);
12326 let checkpoint_target = checkpoint_targets
12327 .into_iter()
12328 .next()
12329 .expect("one checkpoint target should exist");
12330 let checkpoint_owner = checkpoint_target.capacity_owner;
12331 let target_pool_index = checkpoint_target.target_pool_index;
12332 let injected_write_error = store
12333 .run_decommission_capacity_temporary_mutation_with_capacity_lease(
12334 target_pool_index,
12335 Some(checkpoint_owner),
12336 Some(aborting_data.len()),
12337 |_| {
12338 let target_pool = store.pools[target_pool_index].clone();
12339 let manifest_name = manifest_name.clone();
12340 let aborting_data = aborting_data.clone();
12341 async move {
12342 com::save_config(target_pool, &manifest_name, aborting_data)
12343 .await
12344 .expect("the injected target checkpoint should commit");
12345 Err::<(), Error>(Error::other("injected crash after checkpoint target commit"))
12346 }
12347 },
12348 )
12349 .await
12350 .expect_err("the injected post-commit crash should leave a pending capacity intent")
12351 .to_string();
12352 assert!(injected_write_error.contains("injected crash after checkpoint target commit"));
12353 assert!(
12354 store
12355 .has_decommission_capacity_temporary_mutation_state(target_pool_index, checkpoint_owner)
12356 .await,
12357 "the committed target must retain its exact unresolved capacity intent"
12358 );
12359
12360 recover_test_tier_delete_dispatch_manifest(store.clone(), &manifest_name)
12361 .await
12362 .expect("Preparing rollback should advance receipts and finish terminal cleanup");
12363 assert!(
12364 !store
12365 .has_decommission_capacity_temporary_mutation_state(target_pool_index, checkpoint_owner)
12366 .await,
12367 "already-committed checkpoint recovery must resolve the pending capacity intent"
12368 );
12369 let stats = recover_tier_delete_dispatch_manifests(store.clone(), 100, None)
12370 .await
12371 .expect("the terminal source checkpoint should remain non-actionable");
12372 assert_eq!((stats.scanned, stats.deleted, stats.retained, stats.failed), (1, 0, 1, 0));
12373 assert_eq!(
12374 store
12375 .decommission_durable_ilm_receipt_count_for_test(0)
12376 .await
12377 .expect("v6 terminal receipts should be listable"),
12378 2
12379 );
12380
12381 {
12382 let _proof_guard = crate::services::notification_sys::without_cross_pool_fence_fleet_proof_for_test();
12383 let proof_error = store
12384 .verify_and_cleanup_decommissioned_durable_ilm_record_for_test(
12385 0,
12386 store.pools[0].get_disks_by_key(&manifest_name),
12387 &manifest_name,
12388 )
12389 .await
12390 .expect_err("revoked v6 fleet proof must block final source cleanup")
12391 .to_string();
12392 assert!(
12393 proof_error.contains("fleet capability is unavailable"),
12394 "unexpected revoked-proof cleanup error: {proof_error}"
12395 );
12396 assert!(
12397 com::read_config(store.pools[0].clone(), &manifest_name).await.is_ok(),
12398 "revoked fleet proof must retain the last source manifest copy"
12399 );
12400 }
12401
12402 for path in [&manifest_name, &journal_name] {
12403 store
12404 .verify_and_cleanup_decommissioned_durable_ilm_record_for_test(0, store.pools[0].get_disks_by_key(path), path)
12405 .await
12406 .expect("terminal receipt should authorize verified source cleanup");
12407 }
12408 let stats = recover_tier_delete_dispatch_manifests(store.clone(), 100, None)
12409 .await
12410 .expect("verified source cleanup should remove the last manifest copy");
12411 assert_eq!((stats.scanned, stats.deleted, stats.failed), (0, 0, 0));
12412 for path in [&manifest_name, &journal_name] {
12413 for pool in &store.pools {
12414 assert!(
12415 matches!(com::read_config(pool.clone(), path).await, Err(Error::ConfigNotFound)),
12416 "terminal receipt cleanup must remove `{path}` from every staged pool"
12417 );
12418 }
12419 }
12420 }
12421
12422 #[cfg(feature = "test-util")]
12423 #[tokio::test]
12424 #[serial_test::serial(storage_class_env)]
12425 async fn authorized_all_committed_manifest_recovers_after_last_commit_crash() {
12426 let temp_dir = tempfile::tempdir().expect("create authorized manifest recovery store dir");
12427 let (ctx, store, _shutdown) =
12428 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "authorized-manifest-last-commit", &[4])).await;
12429 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
12430 let bucket = "authorized-manifest-last-commit-bucket";
12431 store
12432 .make_bucket(bucket, &MakeBucketOptions::default())
12433 .await
12434 .expect("authorized recovery bucket should be created");
12435 let incarnation = store
12436 .bucket_incarnation_id(bucket)
12437 .await
12438 .expect("bucket incarnation should resolve");
12439 let tier_name = "AUTHORIZED-LAST-COMMIT";
12440 let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
12441 let lease = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
12442 .await
12443 .expect("tier lease should resolve");
12444 let mut entries = Vec::new();
12445 for index in 0..2 {
12446 let remote_version = uuid::Uuid::new_v4().to_string();
12447 let entry = synthetic_v6_dispatch_entry(
12448 bucket,
12449 &format!("archive/{index}.bin"),
12450 tier_name,
12451 lease.backend_identity(),
12452 &remote_version,
12453 );
12454 backend.set_put_remote_version(Some(remote_version)).await;
12455 lease
12456 .put(&entry.obj_name, ReaderImpl::Body(bytes::Bytes::from_static(b"remote candidate")), 16)
12457 .await
12458 .expect("remote candidate should be seeded");
12459 entries.push((entry, Some(TierDeleteJournalState::Committed)));
12460 }
12461 let (manifest_name, _) = install_test_tier_delete_dispatch_fixture(
12462 store.clone(),
12463 bucket,
12464 incarnation,
12465 "archive/",
12466 entries,
12467 TierDeleteDispatchManifestState::DispatchAuthorized,
12468 )
12469 .await
12470 .expect("last-commit crash fixture should persist");
12471
12472 let manifest_stats = recover_tier_delete_dispatch_manifests(store.clone(), 100, None)
12473 .await
12474 .expect("all-Committed Authorized manifest should advance");
12475 assert_eq!((manifest_stats.scanned, manifest_stats.advanced, manifest_stats.failed), (1, 1, 0));
12476 assert_eq!(
12477 test_tier_delete_dispatch_manifest_state(store.clone(), &manifest_name)
12478 .await
12479 .expect("advanced manifest should remain readable"),
12480 Some(TierDeleteDispatchManifestState::Completed)
12481 );
12482 let journal_stats = recover_tier_delete_journal_entries(store.clone(), 100, None)
12483 .await
12484 .expect("Completed journal set should recover remote candidates");
12485 assert_eq!((journal_stats.scanned, journal_stats.deleted, journal_stats.failed), (2, 2, 0));
12486 assert_eq!(tier_delete_journal_count(store.clone()).await, 0);
12487 assert_eq!(tier_delete_dispatch_manifest_count(store.clone()).await, 1);
12488 let gc_stats = recover_tier_delete_dispatch_manifests(store.clone(), 100, None)
12489 .await
12490 .expect("manifest coordinator should remove the empty Completed manifest");
12491 assert_eq!((gc_stats.scanned, gc_stats.deleted, gc_stats.failed), (1, 1, 0));
12492 assert_eq!(tier_delete_dispatch_manifest_count(store).await, 0);
12493 assert_eq!(backend.object_count().await, 0);
12494 assert_eq!(backend.remove_versions().await.len(), 2);
12495 }
12496
12497 #[cfg(feature = "test-util")]
12498 async fn transition_transaction_record_count(store: Arc<crate::store::ECStore>) -> usize {
12499 store
12500 .list_objects_v2(
12501 RUSTFS_META_BUCKET,
12502 TRANSITION_TRANSACTION_RECORD_PREFIX,
12503 None,
12504 None,
12505 100,
12506 false,
12507 None,
12508 false,
12509 )
12510 .await
12511 .expect("transition transaction records should be listable")
12512 .objects
12513 .len()
12514 }
12515
12516 #[cfg(feature = "test-util")]
12517 async fn only_transition_transaction(store: Arc<crate::store::ECStore>) -> TransitionTransaction {
12518 let records = store
12519 .clone()
12520 .list_objects_v2(
12521 RUSTFS_META_BUCKET,
12522 TRANSITION_TRANSACTION_RECORD_PREFIX,
12523 None,
12524 None,
12525 100,
12526 false,
12527 None,
12528 false,
12529 )
12530 .await
12531 .expect("transition transaction records should be listable")
12532 .objects;
12533 assert_eq!(records.len(), 1, "test fixture should have exactly one transition transaction");
12534 let transaction_id = records[0]
12535 .name
12536 .rsplit('/')
12537 .next()
12538 .and_then(|name| name.strip_suffix(".json"))
12539 .and_then(|name| uuid::Uuid::parse_str(name).ok())
12540 .expect("transition transaction path should end in its UUID");
12541 load_transition_transaction_record(store, transaction_id)
12542 .await
12543 .expect("transition transaction should load")
12544 }
12545
12546 #[cfg(feature = "test-util")]
12547 async fn register_transition_reconcile_test_tier(
12548 handle: &Arc<tokio::sync::RwLock<TierConfigMgr>>,
12549 tier_name: &str,
12550 ) -> MockWarmBackend {
12551 let backend = MockWarmBackend::new();
12552 let mut manager = handle.write().await;
12553 manager.tiers.insert(
12554 tier_name.to_string(),
12555 TierConfig {
12556 version: "v1".to_string(),
12557 tier_type: TierType::Wasabi,
12558 name: tier_name.to_string(),
12559 wasabi: Some(TierWasabi {
12560 name: tier_name.to_string(),
12561 endpoint: "https://s3.wasabisys.com".to_string(),
12562 access_key: "test-access-key".to_string(),
12563 secret_key: "test-secret-key".to_string(),
12564 bucket: "mock-tier".to_string(),
12565 prefix: format!("mock/{}/", uuid::Uuid::new_v4()),
12566 region: "us-east-1".to_string(),
12567 }),
12568 ..Default::default()
12569 },
12570 );
12571 manager
12572 .install_test_driver(tier_name, Box::new(backend.clone()))
12573 .expect("mock fallback tier driver should install");
12574 backend
12575 }
12576
12577 #[cfg(feature = "test-util")]
12578 async fn wait_for_tier_delete_journal_recovery(
12579 store: Arc<crate::store::ECStore>,
12580 backend: &MockWarmBackend,
12581 expected_removes: usize,
12582 ) {
12583 tokio::time::timeout(Duration::from_secs(30), async {
12584 loop {
12585 if backend.remove_versions().await.len() >= expected_removes
12586 && tier_delete_journal_count(store.clone()).await == 0
12587 {
12588 return;
12589 }
12590 tokio::time::sleep(Duration::from_millis(10)).await;
12591 }
12592 })
12593 .await
12594 .expect("tier delete journal recovery should complete");
12595 }
12596
12597 #[cfg(feature = "test-util")]
12598 async fn wait_for_tier_free_version_recovery(
12599 store: Arc<crate::store::ECStore>,
12600 backend: &MockWarmBackend,
12601 expected_removes: usize,
12602 ) {
12603 ExpiryState::resize_workers(1, store.clone()).await;
12604 let mut last_progress = None;
12605 let result = tokio::time::timeout(Duration::from_secs(30), async {
12606 loop {
12607 let stats = recover_tier_free_versions(store.clone(), 100, None, None)
12608 .await
12609 .expect("tier free-version recovery scan should succeed");
12610 let removes = backend.remove_versions().await.len();
12611 let recovered = removes >= expected_removes && stats.enqueued == 0 && stats.failed == 0;
12612 last_progress = Some((removes, stats));
12613 if recovered {
12614 return;
12615 }
12616 tokio::time::sleep(Duration::from_millis(50)).await;
12617 }
12618 })
12619 .await;
12620 if result.is_err() {
12621 let expiry_state = store.ctx.expiry_state();
12622 let workers = expiry_state
12623 .try_read()
12624 .ok()
12625 .map(|state| (state.pending_tasks(), state.active_tasks()));
12626 panic!(
12627 "tier free-version recovery should complete: expected_removes={expected_removes}, \
12628 last_progress={last_progress:?}, workers(pending, active)={workers:?}"
12629 );
12630 }
12631 wait_for_expiry_workers_idle(&store).await;
12632 }
12633
12634 #[cfg(feature = "test-util")]
12639 async fn exact_metadata_absent_during_cleanup(store: &crate::store::ECStore, bucket: &str, object: &str) -> bool {
12640 match store.pools[0]
12641 .get_disks_by_key(object)
12642 .load_file_info_versions_exact(bucket, object)
12643 .await
12644 {
12645 Ok(metadata) => metadata.is_none(),
12646 Err(StorageError::InsufficientReadQuorum(_, _)) => false,
12647 Err(error) => panic!("{object} cleanup metadata should remain readable: {error:?}"),
12648 }
12649 }
12650
12651 #[cfg(feature = "test-util")]
12652 async fn wait_for_expiry_workers_idle(store: &crate::store::ECStore) {
12653 let expiry_state = store.ctx.expiry_state();
12654 tokio::time::timeout(Duration::from_secs(30), async {
12655 loop {
12656 let idle = {
12657 let state = expiry_state.read().await;
12658 state.pending_tasks() == 0 && state.active_tasks() == 0
12659 };
12660 if idle {
12661 tokio::time::sleep(Duration::from_millis(10)).await;
12665 let state = expiry_state.read().await;
12666 if state.pending_tasks() == 0 && state.active_tasks() == 0 {
12667 return;
12668 }
12669 } else {
12670 tokio::time::sleep(Duration::from_millis(10)).await;
12671 }
12672 }
12673 })
12674 .await
12675 .expect("lifecycle expiry workers should become idle");
12676 }
12677
12678 #[cfg(feature = "test-util")]
12679 async fn run_transitioned_delete_free_version_owner_case(
12680 store_name: &str,
12681 tier_name: &str,
12682 bucket: &str,
12683 object: &str,
12684 restore_before_delete: bool,
12685 causal_enqueue: bool,
12686 delete_with_journal: bool,
12687 ) {
12688 let temp_dir = tempfile::tempdir().expect("create transitioned delete store dir");
12689 let (ctx, store, _shutdown) =
12690 without_storage_class_env(build_isolated_test_store(temp_dir.path(), store_name, &[4])).await;
12691 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
12692
12693 let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
12694 store
12695 .make_bucket(bucket, &MakeBucketOptions::default())
12696 .await
12697 .expect("source bucket should be created");
12698 let mut reader = PutObjReader::from_vec(vec![b'd'; 1024 * 1024]);
12699 let original = store
12700 .put_object(bucket, object, &mut reader, &ObjectOptions::default())
12701 .await
12702 .expect("source object should be written");
12703 store
12704 .transition_object(
12705 bucket,
12706 object,
12707 &ObjectOptions {
12708 transition: TransitionOptions {
12709 status: TRANSITION_PENDING.to_string(),
12710 tier: tier_name.to_string(),
12711 etag: original.etag.clone().expect("source should have an etag"),
12712 ..Default::default()
12713 },
12714 mod_time: original.mod_time,
12715 ..Default::default()
12716 },
12717 )
12718 .await
12719 .expect("source transition should commit");
12720 assert_eq!(backend.object_count().await, 1, "transition should create one remote object");
12721 if restore_before_delete {
12722 store
12723 .clone()
12724 .restore_transitioned_object(
12725 bucket,
12726 object,
12727 &ObjectOptions {
12728 transition: TransitionOptions {
12729 restore_request: s3s::dto::RestoreRequest {
12730 days: Some(1),
12731 ..Default::default()
12732 },
12733 ..Default::default()
12734 },
12735 ..Default::default()
12736 },
12737 )
12738 .await
12739 .expect("restore should complete before transitioned delete");
12740 let restored = store
12741 .get_object_info(bucket, object, &ObjectOptions::default())
12742 .await
12743 .expect("restored transitioned source should remain readable");
12744 assert!(
12745 restored.user_defined.contains_key(rustfs_filemeta::metadata_keys::RESTORE),
12746 "restore completion metadata must be present before the delete regression"
12747 );
12748 }
12749
12750 if causal_enqueue {
12751 ExpiryState::resize_workers(1, store.clone()).await;
12752 }
12753 backend.set_remove_failure(!causal_enqueue);
12754 if delete_with_journal {
12755 store
12756 .delete_object_with_tier_delete_journal(bucket, object, ObjectOptions::default())
12757 .await
12758 .expect("transitioned source journal-wrapper delete should commit");
12759 } else {
12760 store
12761 .delete_object(bucket, object, ObjectOptions::default())
12762 .await
12763 .expect("transitioned source plain object-layer delete should commit");
12764 }
12765
12766 if causal_enqueue {
12767 tokio::time::timeout(Duration::from_secs(30), async {
12768 loop {
12769 let metadata_absent = {
12770 let mut read_opts = ObjectOptions::default();
12772 let _guards = store
12773 .acquire_all_physical_object_read_locks(
12774 "transitioned_delete_cleanup_test",
12775 bucket,
12776 object,
12777 &mut read_opts,
12778 )
12779 .await
12780 .expect("causal free-version cleanup observation should acquire object read locks");
12781 store.pools[0]
12782 .get_disks_by_key(object)
12783 .load_file_info_versions_exact(bucket, object)
12784 .await
12785 .expect("causal free-version cleanup metadata should remain readable")
12786 .is_none()
12787 };
12788 if metadata_absent && backend.remove_versions().await.len() == 1 {
12789 return;
12790 }
12791 tokio::time::sleep(Duration::from_millis(10)).await;
12792 }
12793 })
12794 .await
12795 .expect("committed free-version should be cleaned without a recovery scan");
12796 assert_eq!(backend.object_count().await, 0, "causal cleanup should remove the remote object");
12797 assert_eq!(
12798 tier_delete_journal_count(store.clone()).await,
12799 0,
12800 "ordinary causal cleanup must not create a journal"
12801 );
12802 store
12803 .delete_bucket(bucket, &DeleteBucketOptions::default())
12804 .await
12805 .expect("bucket delete should succeed after causal free-version cleanup");
12806 return;
12807 }
12808
12809 let local_versions = store.pools[0]
12810 .get_disks_by_key(object)
12811 .load_file_info_versions_exact(bucket, object)
12812 .await
12813 .expect("post-delete metadata should remain decodable");
12814 let local_versions = local_versions.expect("ordinary delete must leave a durable free-version cleanup owner");
12815 assert_eq!(
12816 local_versions
12817 .versions
12818 .iter()
12819 .chain(local_versions.free_versions.iter())
12820 .filter(|version| version.tier_free_version())
12821 .count(),
12822 1,
12823 "ordinary delete must leave exactly one hidden free-version owner"
12824 );
12825 assert_eq!(
12826 local_versions
12827 .versions
12828 .iter()
12829 .filter(|version| !version.tier_free_version())
12830 .count(),
12831 0,
12832 "ordinary delete must remove the visible transitioned source"
12833 );
12834 let listed = store
12835 .clone()
12836 .list_object_versions(bucket, "", None, None, None, 100)
12837 .await
12838 .expect("source versions should be listable");
12839 assert!(listed.objects.is_empty(), "source must have no visible versions after delete");
12840
12841 assert_eq!(
12842 tier_delete_journal_count(store.clone()).await,
12843 0,
12844 "ordinary delete must not create a journal"
12845 );
12846
12847 backend.set_remove_failure(false);
12848 ExpiryState::resize_workers(1, store.clone()).await;
12849 let recovered = recover_tier_free_versions(store.clone(), 100, None, None)
12850 .await
12851 .expect("free-version recovery should scan the hidden owner");
12852 assert_eq!((recovered.enqueued, recovered.failed), (1, 0));
12853 tokio::time::timeout(Duration::from_secs(30), async {
12854 loop {
12855 let metadata_absent = {
12856 let mut read_opts = ObjectOptions::default();
12858 let _guards = store
12859 .acquire_all_physical_object_read_locks(
12860 "transitioned_delete_cleanup_test",
12861 bucket,
12862 object,
12863 &mut read_opts,
12864 )
12865 .await
12866 .expect("free-version cleanup observation should acquire object read locks");
12867 store.pools[0]
12868 .get_disks_by_key(object)
12869 .load_file_info_versions_exact(bucket, object)
12870 .await
12871 .expect("free-version cleanup metadata should remain readable")
12872 .is_none()
12873 };
12874 if metadata_absent && backend.remove_versions().await.len() == 1 {
12875 return;
12876 }
12877 tokio::time::sleep(Duration::from_millis(10)).await;
12878 }
12879 })
12880 .await
12881 .expect("free-version recovery should delete the remote and exact local owner");
12882 assert_eq!(backend.object_count().await, 0, "free-version recovery should remove the remote object");
12883 store
12884 .delete_bucket(bucket, &DeleteBucketOptions::default())
12885 .await
12886 .expect("ordinary bucket delete should succeed after the last transitioned object");
12887 }
12888
12889 #[cfg(feature = "test-util")]
12890 #[tokio::test]
12891 #[serial_test::serial(storage_class_env)]
12892 async fn transitioned_delete_uses_free_version_as_cleanup_owner() {
12893 run_transitioned_delete_free_version_owner_case(
12894 "transitioned-delete-journal-owner",
12895 "DELETEOWNER",
12896 "transitioned-delete-journal-owner-bucket",
12897 "transition/archive.bin",
12898 false,
12899 false,
12900 true,
12901 )
12902 .await;
12903 }
12904
12905 #[cfg(feature = "test-util")]
12906 #[tokio::test]
12907 #[serial_test::serial(storage_class_env)]
12908 async fn restored_transitioned_delete_uses_free_version_as_cleanup_owner() {
12909 run_transitioned_delete_free_version_owner_case(
12910 "restored-transitioned-delete-journal-owner",
12911 "RESTOREDELETEOWNER",
12912 "restored-transitioned-delete-journal-owner-bucket",
12913 "transition/archive.bin",
12914 true,
12915 false,
12916 true,
12917 )
12918 .await;
12919 }
12920
12921 #[cfg(feature = "test-util")]
12922 #[tokio::test]
12923 #[serial_test::serial(storage_class_env)]
12924 async fn transitioned_delete_causally_enqueues_free_version() {
12925 run_transitioned_delete_free_version_owner_case(
12926 "transitioned-delete-causal-enqueue",
12927 "DELETE-CAUSAL",
12928 "transitioned-delete-causal-enqueue-bucket",
12929 "transition/archive.bin",
12930 false,
12931 true,
12932 false,
12933 )
12934 .await;
12935 }
12936
12937 #[cfg(feature = "test-util")]
12938 #[tokio::test]
12939 #[serial_test::serial(storage_class_env)]
12940 async fn restored_transitioned_delete_causally_enqueues_free_version() {
12941 run_transitioned_delete_free_version_owner_case(
12942 "restored-transitioned-delete-causal-enqueue",
12943 "RESTORE-DELETE-CAUSAL",
12944 "restored-transitioned-delete-causal-enqueue-bucket",
12945 "transition/archive.bin",
12946 true,
12947 true,
12948 true,
12949 )
12950 .await;
12951 }
12952
12953 #[cfg(feature = "test-util")]
12954 #[tokio::test]
12955 #[serial_test::serial(storage_class_env)]
12956 async fn force_tier_remove_blocks_on_physical_free_version_hidden_by_other_pool() {
12957 let temp_dir = tempfile::tempdir().expect("create tier reference store dir");
12958 let (ctx, store, _shutdown) =
12959 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "tier-free-reference-proof", &[4, 4])).await;
12960 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
12961
12962 let tier_name = "FREE-PROOF";
12963 let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
12964 let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
12965 .await
12966 .expect("test tier identity should resolve")
12967 .backend_identity();
12968 let bucket = "tier-free-reference-proof-bucket";
12969 let object = "archive.bin";
12970 store
12971 .make_bucket(bucket, &MakeBucketOptions::default())
12972 .await
12973 .expect("source bucket should be created");
12974 let mut reader = PutObjReader::from_vec(vec![b'f'; 1024 * 1024]);
12975 let original = store.pools[0]
12976 .put_object(bucket, object, &mut reader, &ObjectOptions::default())
12977 .await
12978 .expect("source object should be written");
12979 store.pools[0]
12980 .transition_object(
12981 bucket,
12982 object,
12983 &ObjectOptions {
12984 transition: TransitionOptions {
12985 status: TRANSITION_PENDING.to_string(),
12986 tier: tier_name.to_string(),
12987 etag: original.etag.clone().expect("source should have an etag"),
12988 ..Default::default()
12989 },
12990 mod_time: original.mod_time,
12991 ..Default::default()
12992 },
12993 )
12994 .await
12995 .expect("source transition should commit");
12996 backend.set_remove_failure(true);
12997 store.pools[0]
12998 .delete_object(
12999 bucket,
13000 object,
13001 ObjectOptions {
13002 tier_delete_journal_api: Some(store.clone()),
13003 ..Default::default()
13004 },
13005 )
13006 .await
13007 .expect("journal-less compatibility delete should retain a free-version");
13008
13009 let exact = store.pools[0]
13010 .get_disks_by_key(object)
13011 .load_file_info_versions_exact(bucket, object)
13012 .await
13013 .expect("free-version metadata should remain decodable")
13014 .expect("free-version metadata should remain on disk");
13015 assert_eq!(
13016 exact
13017 .versions
13018 .iter()
13019 .chain(exact.free_versions.iter())
13020 .filter(|version| version.tier_free_version())
13021 .count(),
13022 1,
13023 "fixture must contain one real serialized hidden free-version"
13024 );
13025
13026 let mut hot_reader = PutObjReader::from_vec(vec![b'h'; 1024 * 1024]);
13027 store.pools[1]
13028 .put_object(
13029 bucket,
13030 object,
13031 &mut hot_reader,
13032 &ObjectOptions {
13033 mod_time: Some(OffsetDateTime::now_utc() + time::Duration::hours(1)),
13034 ..Default::default()
13035 },
13036 )
13037 .await
13038 .expect("second pool hot source should be written");
13039 let merged = store
13040 .get_object_info(bucket, object, &ObjectOptions::default())
13041 .await
13042 .expect("merged lookup should return the second-pool hot source");
13043 assert!(
13044 merged.transitioned_object.status.is_empty() && !merged.transitioned_object.free_version,
13045 "the ordinary second-pool source must hide the first-pool free-version from merged lookup"
13046 );
13047
13048 let err = ensure_no_authoritative_tier_references_for_test(store, tier_name, backend_identity, true)
13049 .await
13050 .expect_err("force tier removal must not bypass a physical free-version cleanup obligation");
13051 assert_eq!(err.code, ERR_TIER_BACKEND_IN_USE.code);
13052 }
13053
13054 #[cfg(feature = "test-util")]
13055 #[tokio::test]
13056 #[serial_test::serial(storage_class_env)]
13057 async fn tier_reference_proof_fails_closed_when_physical_bucket_topology_is_incomplete() {
13058 let temp_dir = tempfile::tempdir().expect("create incomplete tier reference store dir");
13059 let (ctx, store, _shutdown) =
13060 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "tier-reference-incomplete-topology", &[4]))
13061 .await;
13062 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
13063
13064 let tier_name = "INCOMPLETE-PROOF";
13065 register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
13066 let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
13067 .await
13068 .expect("test tier identity should resolve")
13069 .backend_identity();
13070 store.pools[0].disk_set[0].disks.write().await[0] = None;
13071
13072 let err = ensure_no_authoritative_tier_references_for_test(store, tier_name, backend_identity, true)
13073 .await
13074 .expect_err("an incomplete physical bucket enumeration must block tier mutation");
13075 assert_eq!(err.code, ERR_TIER_INVALID_CONFIG.code);
13076 assert!(
13077 err.message.contains("complete set quorum"),
13078 "unexpected reference proof error: {}",
13079 err.message
13080 );
13081 }
13082
13083 #[cfg(feature = "test-util")]
13084 async fn copy_test_xlmeta_between_pools(
13085 root: &std::path::Path,
13086 source_pool: usize,
13087 target_pool: usize,
13088 bucket: &str,
13089 object: &str,
13090 ) {
13091 for disk_index in 0..4 {
13092 let source = root.join(format!("pool{source_pool}/set0/disk{disk_index}/{bucket}/{object}/{STORAGE_FORMAT_FILE}"));
13093 let target = root.join(format!("pool{target_pool}/set0/disk{disk_index}/{bucket}/{object}/{STORAGE_FORMAT_FILE}"));
13094 tokio::fs::create_dir_all(target.parent().expect("target xl.meta should have a parent"))
13095 .await
13096 .expect("target object directory should be created");
13097 tokio::fs::copy(&source, &target)
13098 .await
13099 .expect("xl.meta should copy exactly between pools");
13100 }
13101 }
13102
13103 #[cfg(feature = "test-util")]
13104 async fn rewrite_transitioned_xlmeta_as_legacy_unknown(
13105 root: &std::path::Path,
13106 pool_index: usize,
13107 bucket: &str,
13108 object: &str,
13109 minio_unversioned: bool,
13110 ) {
13111 for disk_index in 0..4 {
13112 let metadata_path =
13113 root.join(format!("pool{pool_index}/set0/disk{disk_index}/{bucket}/{object}/{STORAGE_FORMAT_FILE}"));
13114 let encoded = tokio::fs::read(&metadata_path)
13115 .await
13116 .expect("transitioned xl.meta should be readable");
13117 let mut metadata = FileMeta::load(&encoded).expect("transitioned xl.meta should decode");
13118 let mut found = false;
13119 for shallow in &mut metadata.versions {
13120 let mut version = shallow
13121 .parse_version_meta()
13122 .expect("transitioned version record should decode");
13123 let Some(object_meta) = version.object.as_mut() else {
13124 continue;
13125 };
13126 if rustfs_utils::http::metadata_compat::get_bytes(
13127 &object_meta.meta_sys,
13128 rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_STATUS,
13129 )
13130 .as_deref()
13131 != Some(rustfs_filemeta::TRANSITION_COMPLETE.as_bytes())
13132 {
13133 continue;
13134 }
13135 found = true;
13136 for suffix in [
13137 rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
13138 rustfs_utils::http::metadata_compat::SUFFIX_TRANSITIONED_VERSION_STATE,
13139 rustfs_utils::http::metadata_compat::SUFFIX_TRANSITIONED_VERSION_ID,
13140 ] {
13141 rustfs_utils::http::metadata_compat::remove_bytes(&mut object_meta.meta_sys, suffix);
13142 }
13143 if minio_unversioned {
13144 object_meta
13145 .meta_sys
13146 .insert("x-minio-internal-transitioned-versionID".to_string(), Vec::new());
13147 }
13148 *shallow = rustfs_filemeta::FileMetaShallowVersion::try_from(version)
13149 .expect("legacy transitioned version should re-encode");
13150 }
13151 assert!(found, "transitioned source should exist in serialized xl.meta");
13152 tokio::fs::write(&metadata_path, metadata.marshal_msg().expect("legacy transitioned xl.meta should encode"))
13153 .await
13154 .expect("legacy transitioned xl.meta should persist");
13155 }
13156 }
13157
13158 #[cfg(feature = "test-util")]
13159 async fn read_store_body(
13160 store: &Arc<crate::store::ECStore>,
13161 bucket: &str,
13162 object: &str,
13163 range: Option<HTTPRangeSpec>,
13164 opts: &ObjectOptions,
13165 ) -> Vec<u8> {
13166 let mut reader = store
13167 .get_object_reader(bucket, object, range, HeaderMap::new(), opts)
13168 .await
13169 .expect("object reader should open");
13170 let mut body = Vec::new();
13171 reader.stream.read_to_end(&mut body).await.expect("object body should drain");
13172 body
13173 }
13174
13175 #[cfg(feature = "test-util")]
13176 #[test]
13177 #[serial_test::serial(storage_class_env)]
13178 fn legacy_transition_state_inspection_and_apply_keep_all_disk_copies_unchanged() {
13179 run_large_stack_async_test("legacy-state-reconcile-inspection", || {
13180 legacy_transition_state_inspection_and_apply_case(false)
13181 });
13182 }
13183
13184 #[cfg(feature = "test-util")]
13185 #[cfg(not(windows))]
13186 #[test]
13187 #[serial_test::serial(storage_class_env)]
13188 fn legacy_transition_state_backfill_retries_partial_commits_and_preserves_other_bytes() {
13189 run_large_stack_async_test("legacy-state-reconcile-backfill", || {
13190 legacy_transition_state_inspection_and_apply_case(true)
13191 });
13192 }
13193
13194 #[cfg(feature = "test-util")]
13195 async fn legacy_transition_state_inspection_and_apply_case(write_enabled: bool) {
13196 #[cfg(windows)]
13197 assert!(!write_enabled, "Windows supports inspection but cannot prove repair directory durability");
13198 use crate::bucket::lifecycle::legacy_transition_state_reconcile::{
13199 LegacyTransitionStateReconcileOutcome as Outcome, LegacyTransitionStateReconcileRequest,
13200 LegacyTransitionStateReconcileSelector,
13201 };
13202 for (remote_version, expected_state) in [
13203 ("", rustfs_filemeta::TransitionVersionState::KnownDisabled),
13204 ("null", rustfs_filemeta::TransitionVersionState::SuspendedNull),
13205 ("opaque-version", rustfs_filemeta::TransitionVersionState::Exact),
13206 ] {
13207 let temp_dir = tempfile::tempdir().expect("legacy reconcile store directory");
13208 let (ctx, store, _shutdown) =
13209 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "legacy-state-reconcile-inspect", &[4]))
13210 .await;
13211 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
13212 let tier_name = "LEGACY-RECONCILE";
13213 let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
13214 backend.set_put_remote_version(Some(remote_version.to_string())).await;
13215 let bucket = "legacy-state-reconcile-bucket";
13216 let object = "archive.bin";
13217 store
13218 .make_bucket(bucket, &MakeBucketOptions::default())
13219 .await
13220 .expect("create legacy fixture bucket");
13221 let mut reader = PutObjReader::from_vec(b"legacy reconcile body".repeat(1024));
13222 let source = store
13223 .put_object(bucket, object, &mut reader, &ObjectOptions::default())
13224 .await
13225 .expect("write source");
13226 {
13227 let _proof = crate::services::notification_sys::install_current_remote_version_state_fleet_proof_for_test();
13230 temp_env::async_with_vars(
13231 [
13232 (rustfs_config::ENV_TIER_REMOTE_VERSION_STATE_WRITE, Some("true")),
13233 (rustfs_config::ENV_TIER_REMOTE_VERSION_STATE_FLEET_CONFIRMED, Some("true")),
13234 ],
13235 store.transition_object(
13236 bucket,
13237 object,
13238 &ObjectOptions {
13239 transition: TransitionOptions {
13240 status: TRANSITION_PENDING.to_string(),
13241 tier: tier_name.to_string(),
13242 etag: source.etag.clone().expect("source ETag"),
13243 ..Default::default()
13244 },
13245 mod_time: source.mod_time,
13246 ..Default::default()
13247 },
13248 ),
13249 )
13250 .await
13251 .expect("transition source");
13252 }
13253 assert!(
13254 crate::services::notification_sys::acquire_legacy_transition_state_reconcile_fleet_proof()
13255 .await
13256 .is_none(),
13257 "fixture setup must not grant the missing reconciliation write capability"
13258 );
13259 let selector = LegacyTransitionStateReconcileSelector {
13260 bucket: bucket.to_string(),
13261 object: object.to_string(),
13262 version_id: "null".to_string(),
13263 };
13264 if expected_state == rustfs_filemeta::TransitionVersionState::Exact {
13265 backend
13266 .set_transition_candidate_probe_override(Some(
13267 crate::services::tier::warm_backend::TransitionCandidateProbe::Ambiguous,
13268 ))
13269 .await;
13270 }
13271 let converged = store
13272 .inspect_legacy_transition_state(selector.clone())
13273 .await
13274 .expect("inspect an already explicit transition");
13275 assert_eq!(converged.outcome, Outcome::Migrated, "{converged:?}");
13276 assert!(!converged.changed);
13277 backend.set_transition_candidate_probe_override(None).await;
13278 rewrite_transitioned_xlmeta_as_legacy_unknown(temp_dir.path(), 0, bucket, object, remote_version.is_empty()).await;
13279 let paths = (0..4)
13280 .map(|disk| {
13281 temp_dir
13282 .path()
13283 .join(format!("pool0/set0/disk{disk}/{bucket}/{object}/{STORAGE_FORMAT_FILE}"))
13284 })
13285 .collect::<Vec<_>>();
13286 let mut original = Vec::new();
13287 for path in &paths {
13288 original.push(tokio::fs::read(path).await.expect("original xl.meta"));
13289 }
13290 backend.clear_op_log().await;
13291 let inspection = store.inspect_legacy_transition_state(selector.clone());
13292 assert!(
13293 std::mem::size_of_val(&inspection) <= 4 * 1024,
13294 "admin inspection future must remain stack-bounded"
13295 );
13296 let inspected = inspection.await.expect("inspect legacy state");
13297 assert_eq!(inspected.outcome, Outcome::ReadyToMigrate, "{inspected:?}");
13298 assert!(!inspected.readiness.post_ready, "current fleet cannot authorize conditional writes");
13299 let target = inspected.target.expect("live probe should establish one model");
13300 assert_eq!(target.state, expected_state);
13301 let request = LegacyTransitionStateReconcileRequest {
13302 confirm: true,
13303 selector,
13304 source: inspected.source.expect("immutable source"),
13305 original_sets: inspected.original_sets,
13306 target,
13307 reconciliation_digest: inspected.reconciliation_digest.expect("expected tuple digest"),
13308 };
13309 #[cfg(not(windows))]
13310 if write_enabled {
13311 crate::services::notification_sys::with_legacy_transition_state_fleet_proof_for_test(async {
13312 crate::disk::local::bucket_durability::set(bucket, Some(crate::disk::local::DurabilityMode::None));
13313 let unsynced = store.reconcile_legacy_transition_state(request.clone()).await;
13314 crate::disk::local::bucket_durability::set(bucket, None);
13315 let unsynced = unsynced.expect("repair without metadata durability");
13316 assert_eq!(unsynced.outcome, Outcome::BackendUnavailable);
13317 assert!(!unsynced.changed);
13318 let rollback = paths[0].parent().expect("object directory").join(Uuid::new_v4().to_string());
13319 tokio::fs::create_dir(&rollback).await.expect("pending rollback directory");
13320 tokio::fs::write(rollback.join(crate::disk::STORAGE_FORMAT_FILE_BACKUP), &original[0])
13321 .await
13322 .expect("pending old metadata backup");
13323 let unsettled = store.reconcile_legacy_transition_state(request.clone()).await;
13324 tokio::fs::remove_dir_all(&rollback).await.expect("settle fixture rollback");
13325 let unsettled = unsettled.expect("repair must wait for rollback");
13326 assert_eq!(unsettled.outcome, Outcome::BackendUnavailable);
13327 assert!(!unsettled.changed);
13328 for (path, bytes) in paths.iter().zip(&original) {
13329 assert_eq!(tokio::fs::read(path).await.expect("blocked repair leaves original bytes"), *bytes);
13330 }
13331 let disks = store.all_set_disks()[0].disk_inventory().await;
13334 let first_disk = disks[0].as_ref().expect("first physical disk");
13335 let publication_path = first_disk
13336 .get_object_path_for_io_if_local(bucket, &format!("{object}/{STORAGE_FORMAT_FILE}"))
13337 .expect("local disk")
13338 .expect("publication path");
13339 let crash_key = format!("{object}/{STORAGE_FORMAT_FILE}");
13340 let _hook = crate::disk::os::prepared_publication_test_hooks::install_at(
13341 crate::disk::os::prepared_publication_test_hooks::Stage::Rename,
13342 &publication_path,
13343 move || {
13344 crate::crash_inject::arm(crate::crash_inject::CrashPoint::MetaWriteAfterTmpBeforeRename, &crash_key);
13345 },
13346 );
13347 let partial = store
13348 .reconcile_legacy_transition_state(request.clone())
13349 .await
13350 .expect("partial repair response");
13351 assert_eq!(partial.outcome, Outcome::BackendUnavailable, "{partial:?}");
13352 assert!(partial.changed, "first copy was committed: {partial:?}");
13353 assert!(partial.changes_indeterminate);
13354 assert_ne!(tokio::fs::read(&paths[0]).await.expect("first committed copy"), original[0]);
13355 for (path, bytes) in paths[1..].iter().zip(&original[1..]) {
13356 assert_eq!(tokio::fs::read(path).await.expect("uncommitted copy"), *bytes);
13357 }
13358 assert!(
13359 store.all_set_disks()[0]
13360 .load_file_info_versions_exact(bucket, object)
13361 .await
13362 .is_err(),
13363 "cleanup cannot select a partial repair subset"
13364 );
13365 let repaired = store
13366 .reconcile_legacy_transition_state(request.clone())
13367 .await
13368 .expect("retry original snapshot");
13369 assert_eq!(repaired.outcome, Outcome::Migrated, "{repaired:?}");
13370 assert!(repaired.changed);
13371 assert!(!repaired.changes_indeterminate);
13372 let mut committed = Vec::new();
13373 for (path, original) in paths.iter().zip(&original) {
13374 let raw = tokio::fs::read(path).await.expect("repaired copy");
13375 let metadata = FileMeta::load(&raw).expect("decode repaired copy");
13376 let previous = FileMeta::load(original).expect("decode original copy");
13377 assert_eq!(
13378 metadata.transition_reconcile_generation(None).unwrap(),
13379 previous.transition_reconcile_generation(None).unwrap()
13380 );
13381 let (_, version) = metadata.find_version(None).expect("selected version");
13382 let info = version.into_fileinfo(bucket, object, true).expect("repaired FileInfo");
13383 assert_eq!(info.transition_version_state, expected_state);
13384 assert_eq!(info.transition_version, request.target.remote_version);
13385 committed.push(raw);
13386 }
13387 assert!(
13388 store.all_set_disks()[0]
13389 .load_file_info_versions_exact(bucket, object)
13390 .await
13391 .expect("converged cleanup snapshot")
13392 .is_some()
13393 );
13394 let replay = store
13395 .reconcile_legacy_transition_state(request.clone())
13396 .await
13397 .expect("idempotent original request replay");
13398 assert_eq!(replay.outcome, Outcome::Migrated, "{replay:?}");
13399 assert!(!replay.changed);
13400 for (path, expected) in paths.iter().zip(&committed) {
13401 assert_eq!(
13402 tokio::fs::read(path).await.expect("replayed copy"),
13403 *expected,
13404 "idempotence preserves raw encoding"
13405 );
13406 }
13407 for (path, bytes) in paths.iter().zip(&original) {
13408 tokio::fs::write(path, bytes)
13409 .await
13410 .expect("reset independent cancellation fixture");
13411 }
13412 let (entered_tx, entered) = tokio::sync::oneshot::channel();
13413 let (release, released) = std::sync::mpsc::channel::<()>();
13414 let _pause = crate::disk::os::prepared_publication_test_hooks::install_at(
13415 crate::disk::os::prepared_publication_test_hooks::Stage::Rename,
13416 &publication_path,
13417 move || {
13418 let _ = entered_tx.send(());
13419 let _ = released.recv();
13420 },
13421 );
13422 let mut repair = Box::pin(store.reconcile_legacy_transition_state(request.clone()));
13423 tokio::select! {
13424 result = &mut repair => panic!("repair completed before publication pause: {result:?}"),
13425 result = entered => result.expect("publication executor entered"),
13426 }
13427 let update_options = crate::disk::UpdateMetadataOpts::default();
13428 let mut update = Box::pin(first_disk.update_metadata(
13429 bucket,
13430 object,
13431 FileInfo {
13432 metadata: HashMap::from([("x-amz-meta-concurrent".to_string(), "kept".to_string())]),
13433 ..Default::default()
13434 },
13435 &update_options,
13436 ));
13437 assert!(
13438 tokio::time::timeout(std::time::Duration::from_millis(25), update.as_mut())
13439 .await
13440 .is_err(),
13441 "another metadata RMW must wait for publication"
13442 );
13443 drop(repair);
13444 assert!(
13445 tokio::time::timeout(std::time::Duration::from_millis(25), update.as_mut())
13446 .await
13447 .is_err(),
13448 "cancelling the coordinator must not release an in-flight disk mutation"
13449 );
13450 release.send(()).expect("resume owned publication");
13451 update.await.expect("serialized metadata update");
13452 let raw = tokio::fs::read(&paths[0])
13453 .await
13454 .expect("cancelled repair and later metadata update");
13455 let (_, version) = FileMeta::load(&raw)
13456 .expect("metadata after cancellation")
13457 .find_version(None)
13458 .expect("selected version");
13459 let info = version
13460 .into_fileinfo(bucket, object, true)
13461 .expect("metadata after serialized update");
13462 assert_eq!(info.transition_version_state, expected_state);
13463 assert_eq!(info.metadata.get("x-amz-meta-concurrent").map(String::as_str), Some("kept"));
13464 let stale = store
13465 .reconcile_legacy_transition_state(request.clone())
13466 .await
13467 .expect("stale original request");
13468 assert_eq!(
13469 stale.outcome,
13470 Outcome::Corrupt,
13471 "unrelated metadata change invalidates the original generation: {stale:?}"
13472 );
13473 assert!(!stale.changed);
13474 assert_eq!(tokio::fs::read(&paths[0]).await.expect("stale write leaves bytes unchanged"), raw);
13475 assert_eq!(backend.remove_count().await, 0);
13476 let operations = backend.op_log().await;
13479 assert!(
13480 operations.iter().all(|operation| match operation {
13481 MockWarmOp::Probe { object } | MockWarmOp::Get { object } => object == &request.source.remote_object,
13482 _ => false,
13483 }),
13484 "unexpected backend effects: {operations:?}"
13485 );
13486 })
13487 .await;
13488 continue;
13489 }
13490 let mut tampered = request.clone();
13491 tampered.source.remote_object.push_str("-other");
13492 let probes_before = backend.op_log().await.len();
13493 let rejected = store
13494 .reconcile_legacy_transition_state(tampered)
13495 .await
13496 .expect("reject tampered tuple");
13497 assert_eq!(rejected.outcome, Outcome::Corrupt);
13498 assert_eq!(backend.op_log().await.len(), probes_before, "invalid digest must not probe the backend");
13499 let applied = store
13500 .reconcile_legacy_transition_state(request)
13501 .await
13502 .expect("apply must report unavailable write authority");
13503 assert_eq!(applied.outcome, Outcome::BackendUnavailable, "{applied:?}");
13504 assert_eq!(applied.reason_code, "write_fence_unavailable");
13505 assert!(!applied.changed);
13506 for (path, expected) in paths.iter().zip(&original) {
13507 assert_eq!(tokio::fs::read(path).await.expect("xl.meta after inspection"), *expected);
13508 }
13509 assert_eq!(backend.remove_count().await, 0);
13510 assert!(
13511 backend
13512 .op_log()
13513 .await
13514 .iter()
13515 .all(|operation| matches!(operation, MockWarmOp::Probe { .. }))
13516 );
13517
13518 backend.set_unreachable(true).await;
13519 let unavailable = store
13520 .inspect_legacy_transition_state(LegacyTransitionStateReconcileSelector {
13521 bucket: bucket.to_string(),
13522 object: object.to_string(),
13523 version_id: "null".to_string(),
13524 })
13525 .await
13526 .expect("unreachable tier is a diagnostic outcome");
13527 assert_eq!(unavailable.outcome, Outcome::BackendUnavailable);
13528 assert!(!unavailable.changed);
13529 backend.set_unreachable(false).await;
13530 for candidate in ["", "00000000-0000-0000-0000-000000000000", "bad\nversion"] {
13531 backend
13532 .set_transition_candidate_probe_override(Some(
13533 crate::services::tier::warm_backend::TransitionCandidateProbe::VersionedPresent(candidate.to_string()),
13534 ))
13535 .await;
13536 let invalid_proof = store
13537 .inspect_legacy_transition_state(LegacyTransitionStateReconcileSelector {
13538 bucket: bucket.to_string(),
13539 object: object.to_string(),
13540 version_id: "null".to_string(),
13541 })
13542 .await
13543 .expect("invalid backend proof is a diagnostic outcome");
13544 assert_eq!(invalid_proof.outcome, Outcome::BackendUnavailable, "{invalid_proof:?}");
13545 assert!(invalid_proof.target.is_none());
13546 }
13547 backend.set_transition_candidate_probe_override(None).await;
13548
13549 backend.clear_op_log().await;
13550 for path in &paths[1..] {
13551 tokio::fs::remove_file(path)
13552 .await
13553 .expect("hide majority metadata copies in fixture");
13554 }
13555 let minority = store
13556 .inspect_legacy_transition_state(LegacyTransitionStateReconcileSelector {
13557 bucket: bucket.to_string(),
13558 object: object.to_string(),
13559 version_id: "null".to_string(),
13560 })
13561 .await
13562 .expect("inspect minority legacy record");
13563 assert_eq!(
13564 minority.outcome,
13565 Outcome::BackendUnavailable,
13566 "a minority owner must remain visible: {minority:?}"
13567 );
13568 assert!(
13569 backend.op_log().await.is_empty(),
13570 "unproven metadata quorum cannot initiate a remote probe"
13571 );
13572 for (path, bytes) in paths.iter().zip(&original) {
13573 tokio::fs::write(path, bytes).await.expect("restore fixture copies");
13574 }
13575 tokio::fs::write(&paths[0], b"corrupt-xl-meta")
13576 .await
13577 .expect("inject corrupt copy");
13578 let corrupt = store
13579 .inspect_legacy_transition_state(LegacyTransitionStateReconcileSelector {
13580 bucket: bucket.to_string(),
13581 object: object.to_string(),
13582 version_id: "null".to_string(),
13583 })
13584 .await
13585 .expect("inspect corrupt legacy record");
13586 assert_eq!(corrupt.outcome, Outcome::Corrupt, "{corrupt:?}");
13587 assert!(backend.op_log().await.is_empty(), "corruption must fail before backend I/O");
13588 }
13589 }
13590
13591 #[cfg(feature = "test-util")]
13592 #[tokio::test]
13593 #[serial_test::serial(storage_class_env)]
13594 async fn legacy_unknown_unversioned_transition_supports_head_get_and_range_without_backfill() {
13595 let temp_dir = tempfile::tempdir().expect("create legacy unknown unversioned store dir");
13596 let (ctx, store, _shutdown) =
13597 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "legacy-unknown-unversioned-read", &[4])).await;
13598 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
13599 let tier_name = "LEGACY-UNKNOWN-UNVERSIONED-READ";
13600 let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
13601 backend.set_put_remote_version(Some(String::new())).await;
13602 let bucket = "legacy-unknown-unversioned-read-bucket";
13603 let object = "object.bin";
13604 let payload = b"legacy unversioned remote tier object remains readable".repeat(1024);
13605 store
13606 .make_bucket(bucket, &MakeBucketOptions::default())
13607 .await
13608 .expect("legacy source bucket should be created");
13609 let mut reader = PutObjReader::from_vec(payload.clone());
13610 let source = store
13611 .put_object(bucket, object, &mut reader, &ObjectOptions::default())
13612 .await
13613 .expect("legacy source should be written");
13614 store
13615 .transition_object(
13616 bucket,
13617 object,
13618 &ObjectOptions {
13619 transition: TransitionOptions {
13620 status: TRANSITION_PENDING.to_string(),
13621 tier: tier_name.to_string(),
13622 etag: source.etag.clone().expect("legacy source should have an etag"),
13623 ..Default::default()
13624 },
13625 mod_time: source.mod_time,
13626 ..Default::default()
13627 },
13628 )
13629 .await
13630 .expect("legacy source should transition");
13631 rewrite_transitioned_xlmeta_as_legacy_unknown(temp_dir.path(), 0, bucket, object, true).await;
13632 backend.clear_op_log().await;
13633
13634 let opts = ObjectOptions {
13635 metadata_cache_safe: false,
13636 ..Default::default()
13637 };
13638 let head = store
13639 .get_object_info(bucket, object, &opts)
13640 .await
13641 .expect("legacy transitioned HEAD should use local metadata");
13642 assert_eq!(head.transition_version_state, rustfs_filemeta::TransitionVersionState::Unknown);
13643 assert!(head.transitioned_object.version_id.is_empty());
13644 assert_eq!(
13645 head.user_defined
13646 .get("x-minio-internal-transitioned-versionID")
13647 .map(String::as_str),
13648 Some(""),
13649 "the MinIO empty version-key provenance must survive xl.meta decoding"
13650 );
13651 assert!(
13652 !rustfs_utils::http::metadata_compat::contains_key_str(
13653 &head.user_defined,
13654 rustfs_utils::http::metadata_compat::SUFFIX_TRANSITIONED_VERSION_STATE,
13655 ),
13656 "the compatibility read must not synthesize version-state metadata"
13657 );
13658
13659 let full_body = read_store_body(&store, bucket, object, None, &opts).await;
13660 assert_eq!(full_body, payload);
13661
13662 let range = HTTPRangeSpec {
13663 is_suffix_length: false,
13664 start: 7,
13665 end: 38,
13666 };
13667 let ranged_body = read_store_body(&store, bucket, object, Some(range), &opts).await;
13668 assert_eq!(ranged_body, &payload[7..=38]);
13669
13670 let after_read = store.pools[0]
13671 .get_disks_by_key(object)
13672 .load_file_info_versions_exact(bucket, object)
13673 .await
13674 .expect("legacy metadata should remain readable after GET")
13675 .expect("legacy object metadata should remain on disk")
13676 .versions
13677 .into_iter()
13678 .find(|version| version.transition_status == rustfs_filemeta::TRANSITION_COMPLETE)
13679 .expect("legacy transitioned source should remain visible after GET");
13680 assert_eq!(after_read.transition_version_state, rustfs_filemeta::TransitionVersionState::Unknown);
13681 assert!(after_read.transition_version.is_none());
13682 assert!(after_read.transition_version_id.is_none());
13683 assert_eq!(
13684 after_read
13685 .metadata
13686 .get("x-minio-internal-transitioned-versionID")
13687 .map(String::as_str),
13688 Some(""),
13689 "the MinIO empty version-key provenance must remain after GET and Range GET"
13690 );
13691 assert!(
13692 !rustfs_utils::http::metadata_compat::contains_key_str(
13693 &after_read.metadata,
13694 rustfs_utils::http::metadata_compat::SUFFIX_TRANSITIONED_VERSION_STATE,
13695 ),
13696 "the compatibility read must remain side-effect free"
13697 );
13698
13699 assert_eq!(
13700 backend.op_log().await,
13701 vec![
13702 MockWarmOp::Probe {
13703 object: after_read.transitioned_objname.clone(),
13704 },
13705 MockWarmOp::Get {
13706 object: after_read.transitioned_objname.clone(),
13707 },
13708 MockWarmOp::Probe {
13709 object: after_read.transitioned_objname.clone(),
13710 },
13711 MockWarmOp::Get {
13712 object: after_read.transitioned_objname,
13713 },
13714 ],
13715 "legacy reads should probe before each unversioned GET and never mutate local metadata"
13716 );
13717 assert_eq!(backend.remove_count().await, 0);
13718 }
13719
13720 #[cfg(feature = "test-util")]
13721 #[tokio::test]
13722 #[serial_test::serial(storage_class_env)]
13723 async fn legacy_unknown_transition_delete_falls_back_for_single_batch_and_blocks_prefix() {
13724 let temp_dir = tempfile::tempdir().expect("create legacy transitioned delete store dir");
13725 let (ctx, store, _shutdown) =
13726 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "legacy-unknown-transitioned-delete", &[4]))
13727 .await;
13728 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
13729 let tier_name = "LEGACY-UNKNOWN-DELETE";
13730 let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
13731 let bucket = "legacy-unknown-transitioned-delete-bucket";
13732 store
13733 .make_bucket(bucket, &MakeBucketOptions::default())
13734 .await
13735 .expect("legacy source bucket should be created");
13736
13737 for (index, object) in ["single.bin", "batch.bin", "prefix/legacy.bin"].into_iter().enumerate() {
13738 let mut reader = PutObjReader::from_vec(vec![b'l' + index as u8; 1024 * 1024]);
13739 let source = store
13740 .put_object(bucket, object, &mut reader, &ObjectOptions::default())
13741 .await
13742 .expect("legacy source should be written");
13743 store
13744 .transition_object(
13745 bucket,
13746 object,
13747 &ObjectOptions {
13748 transition: TransitionOptions {
13749 status: TRANSITION_PENDING.to_string(),
13750 tier: tier_name.to_string(),
13751 etag: source.etag.clone().expect("legacy source should have an etag"),
13752 ..Default::default()
13753 },
13754 mod_time: source.mod_time,
13755 ..Default::default()
13756 },
13757 )
13758 .await
13759 .expect("legacy source should transition");
13760 rewrite_transitioned_xlmeta_as_legacy_unknown(temp_dir.path(), 0, bucket, object, false).await;
13761 let legacy = store.pools[0]
13762 .get_disks_by_key(object)
13763 .load_file_info_versions_exact(bucket, object)
13764 .await
13765 .expect("legacy source metadata should remain decodable")
13766 .expect("legacy source should remain on disk")
13767 .versions
13768 .into_iter()
13769 .find(|version| version.transition_status == rustfs_filemeta::TRANSITION_COMPLETE)
13770 .expect("legacy transitioned source should remain visible before delete");
13771 assert_eq!(legacy.transition_version_state, rustfs_filemeta::TransitionVersionState::Unknown);
13772 assert_eq!(
13773 crate::services::tier::tier::tier_destination_id_from_metadata(&legacy.metadata)
13774 .expect("legacy metadata identity lookup should not fail"),
13775 None
13776 );
13777 }
13778
13779 backend.set_remove_failure(true);
13780 store
13781 .delete_object_with_tier_delete_journal(bucket, "single.bin", ObjectOptions::default())
13782 .await
13783 .expect("legacy single delete should retain its free-version fallback");
13784 let free_version_prefix_error = store
13785 .delete_object_with_tier_delete_journal(
13786 bucket,
13787 "single.bin",
13788 ObjectOptions {
13789 delete_prefix: true,
13790 ..Default::default()
13791 },
13792 )
13793 .await
13794 .expect_err("recursive prefix delete must not discard an existing hidden cleanup owner");
13795 assert!(
13796 free_version_prefix_error
13797 .to_string()
13798 .contains("tier free-version cleanup obligation"),
13799 "unexpected hidden-owner prefix failure: {free_version_prefix_error}"
13800 );
13801 let (_deleted, errors) = store
13802 .delete_objects_with_tier_delete_journal(
13803 bucket,
13804 vec![ObjectToDelete {
13805 object_name: "batch.bin".to_string(),
13806 ..Default::default()
13807 }],
13808 ObjectOptions::default(),
13809 )
13810 .await;
13811 assert!(errors.iter().all(Option::is_none), "legacy batch delete should succeed: {errors:?}");
13812 let prefix_error = store
13813 .delete_object_with_tier_delete_journal(
13814 bucket,
13815 "prefix/",
13816 ObjectOptions {
13817 delete_prefix: true,
13818 ..Default::default()
13819 },
13820 )
13821 .await
13822 .expect_err("legacy recursive prefix delete must fail closed before physical removal");
13823 assert!(
13824 prefix_error.to_string().contains("legacy Unknown"),
13825 "unexpected prefix failure: {prefix_error}"
13826 );
13827
13828 assert_eq!(
13829 tier_delete_journal_count(store.clone()).await,
13830 0,
13831 "legacy cleanup must not create an unsafe journal"
13832 );
13833 assert_eq!(
13834 backend.object_count().await,
13835 3,
13836 "failed free-version cleanup must retain all remote objects"
13837 );
13838 for object in ["single.bin", "batch.bin"] {
13839 let exact = store.pools[0]
13840 .get_disks_by_key(object)
13841 .load_file_info_versions_exact(bucket, object)
13842 .await
13843 .expect("legacy post-delete metadata should decode")
13844 .expect("legacy cleanup owner should remain on disk");
13845 assert_eq!(
13846 exact
13847 .versions
13848 .iter()
13849 .chain(exact.free_versions.iter())
13850 .filter(|version| version.tier_free_version())
13851 .count(),
13852 1,
13853 "legacy {object} must retain exactly one hidden cleanup owner"
13854 );
13855 assert_eq!(
13856 exact.versions.iter().filter(|version| !version.tier_free_version()).count(),
13857 0,
13858 "legacy {object} must have no visible source after delete"
13859 );
13860 }
13861 let retained_prefix = store.pools[0]
13862 .get_disks_by_key("prefix/legacy.bin")
13863 .load_file_info_versions_exact(bucket, "prefix/legacy.bin")
13864 .await
13865 .expect("rejected prefix metadata should decode")
13866 .expect("rejected prefix delete must retain its source");
13867 assert_eq!(
13868 retained_prefix
13869 .versions
13870 .iter()
13871 .filter(|version| !version.tier_free_version())
13872 .count(),
13873 1
13874 );
13875 }
13876
13877 #[cfg(feature = "test-util")]
13878 #[tokio::test]
13879 #[serial_test::serial(storage_class_env)]
13880 async fn stable_transitioned_recursive_prefix_delete_uses_journal_owners() {
13881 let temp_dir = tempfile::tempdir().expect("create stable transitioned prefix-delete store dir");
13882 let (ctx, store, _shutdown) =
13883 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "stable-transitioned-prefix-delete", &[4]))
13884 .await;
13885 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
13886 let tier_name = "STABLE-PREFIX-DELETE";
13887 let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
13888 let bucket = "stable-transitioned-prefix-delete-bucket";
13889 store
13890 .make_bucket(bucket, &MakeBucketOptions::default())
13891 .await
13892 .expect("stable prefix source bucket should be created");
13893
13894 let objects = ["prefix/first.bin", "prefix/second.bin"];
13895 for (index, object) in objects.into_iter().enumerate() {
13896 let mut reader = PutObjReader::from_vec(vec![b'a' + index as u8; 1024 * 1024]);
13897 let source = store
13898 .put_object(bucket, object, &mut reader, &ObjectOptions::default())
13899 .await
13900 .expect("stable prefix source should be written");
13901 store
13902 .transition_object(
13903 bucket,
13904 object,
13905 &ObjectOptions {
13906 transition: TransitionOptions {
13907 status: TRANSITION_PENDING.to_string(),
13908 tier: tier_name.to_string(),
13909 etag: source.etag.clone().expect("stable prefix source should have an etag"),
13910 ..Default::default()
13911 },
13912 mod_time: source.mod_time,
13913 ..Default::default()
13914 },
13915 )
13916 .await
13917 .expect("stable prefix source should transition");
13918 }
13919 assert_eq!(backend.object_count().await, objects.len());
13920 backend.set_remove_failure(true);
13921
13922 store
13923 .delete_object_with_tier_delete_journal(
13924 bucket,
13925 "prefix/",
13926 ObjectOptions {
13927 delete_prefix: true,
13928 ..Default::default()
13929 },
13930 )
13931 .await
13932 .expect("stable recursive prefix delete should use journal ownership");
13933
13934 assert_eq!(tier_delete_journal_count(store.clone()).await, objects.len());
13935 assert_eq!(backend.object_count().await, objects.len());
13936 for object in objects {
13937 assert!(
13938 store.pools[0]
13939 .get_disks_by_key(object)
13940 .load_file_info_versions_exact(bucket, object)
13941 .await
13942 .expect("stable prefix metadata lookup should succeed")
13943 .is_none(),
13944 "stable recursive delete must leave neither a visible source nor a free-version for {object}"
13945 );
13946 }
13947
13948 backend.set_remove_failure(false);
13949 let stats = recover_tier_delete_journal_entries(store.clone(), 100, None)
13950 .await
13951 .expect("stable prefix journals should recover");
13952 assert_eq!(stats.failed, 0);
13953 assert!(stats.scanned <= objects.len() && stats.deleted <= objects.len());
13954 assert_eq!(tier_delete_journal_count(store).await, 0);
13955 assert_eq!(backend.object_count().await, 0);
13956 assert_eq!(backend.remove_versions().await.len(), objects.len());
13957 }
13958
13959 #[cfg(feature = "test-util")]
13960 #[test]
13961 #[serial_test::serial(storage_class_env)]
13962 fn tier_delete_prefix_limit_and_multi_chunk_batches_converge() {
13963 run_large_stack_async_test(
13964 "tier-delete-prefix-limit-and-multi-chunk",
13965 tier_delete_prefix_limit_and_multi_chunk_batches_converge_case,
13966 );
13967 }
13968
13969 #[cfg(feature = "test-util")]
13970 async fn tier_delete_prefix_limit_and_multi_chunk_batches_converge_case() {
13971 let _batch_limit = TierDeleteDispatchBatchLimitGuard::install(2);
13972 let temp_dir = tempfile::tempdir().expect("create chunked prefix-delete store dir");
13973 let (ctx, store, shutdown) =
13974 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "chunked-prefix-delete", &[4])).await;
13975 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
13976 let tier_name = "CHUNKED-PREFIX-DELETE";
13977 let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
13978 let bucket = "chunked-prefix-delete-bucket";
13979 store
13980 .make_bucket(bucket, &MakeBucketOptions::default())
13981 .await
13982 .expect("chunked prefix source bucket should be created");
13983
13984 for (prefix, count) in [("at-limit", 2), ("limit-plus-one", 3), ("multi-chunk", 5)] {
13985 for index in 0..count {
13986 let object = format!("{prefix}/object-{index}.bin");
13987 let mut reader = PutObjReader::from_vec(vec![b'a' + index as u8; 1024 * 1024]);
13988 let source = store
13989 .put_object(bucket, &object, &mut reader, &ObjectOptions::default())
13990 .await
13991 .expect("chunked prefix source should be written");
13992 store
13993 .transition_object(
13994 bucket,
13995 &object,
13996 &ObjectOptions {
13997 transition: TransitionOptions {
13998 status: TRANSITION_PENDING.to_string(),
13999 tier: tier_name.to_string(),
14000 etag: source.etag.clone().expect("chunked prefix source should have an etag"),
14001 ..Default::default()
14002 },
14003 mod_time: source.mod_time,
14004 ..Default::default()
14005 },
14006 )
14007 .await
14008 .expect("chunked prefix source should transition");
14009 }
14010 }
14011 backend.set_remove_failure(true);
14012
14013 store
14014 .delete_object_with_tier_delete_journal(
14015 bucket,
14016 "at-limit/",
14017 ObjectOptions {
14018 delete_prefix: true,
14019 ..Default::default()
14020 },
14021 )
14022 .await
14023 .expect("exactly one batch must retain the v1 one-shot path");
14024
14025 let limit_plus_one_first = store
14026 .delete_object_with_tier_delete_journal(
14027 bucket,
14028 "limit-plus-one/",
14029 ObjectOptions {
14030 delete_prefix: true,
14031 ..Default::default()
14032 },
14033 )
14034 .await
14035 .expect_err("limit plus one must start a bounded parent transaction");
14036 assert!(
14037 limit_plus_one_first.to_string().contains("retry the next durable batch"),
14038 "unexpected first limit-plus-one result: {limit_plus_one_first}"
14039 );
14040 let active_limit_plus_one_records = store
14041 .clone()
14042 .list_objects_v2(
14043 RUSTFS_META_BUCKET,
14044 TIER_DELETE_DISPATCH_MANIFEST_PREFIX,
14045 None,
14046 None,
14047 10,
14048 false,
14049 None,
14050 false,
14051 )
14052 .await
14053 .expect("active limit-plus-one records should be listable");
14054 let mut active_limit_plus_one_parent_seen = false;
14055 for record in active_limit_plus_one_records
14056 .objects
14057 .iter()
14058 .filter(|record| !record.name.contains("/chunks/"))
14059 {
14060 let data = com::read_config(store.clone(), &record.name)
14061 .await
14062 .expect("an active dispatch root should be readable");
14063 let value: serde_json::Value = serde_json::from_slice(&data).expect("an active dispatch root should contain JSON");
14064 if value["prefix"] == "limit-plus-one/" {
14065 assert_eq!(value["record_type"], "chunked_parent");
14066 active_limit_plus_one_parent_seen = true;
14067 }
14068 }
14069 assert!(
14070 active_limit_plus_one_parent_seen,
14071 "limit plus one must establish the fail-closed parent at the legacy root"
14072 );
14073
14074 let mut limit_plus_one_completed = false;
14075 let mut limit_plus_one_retries = 1;
14076 for _ in 0..3 {
14077 match store
14078 .delete_object_with_tier_delete_journal(
14079 bucket,
14080 "limit-plus-one/",
14081 ObjectOptions {
14082 delete_prefix: true,
14083 ..Default::default()
14084 },
14085 )
14086 .await
14087 {
14088 Ok(_) => {
14089 limit_plus_one_completed = true;
14090 break;
14091 }
14092 Err(err) if err.to_string().contains("retry the next durable batch") => {
14093 limit_plus_one_retries += 1;
14094 }
14095 Err(err) => panic!("limit-plus-one delete returned an unexpected error: {err}"),
14096 }
14097 }
14098 assert!(limit_plus_one_completed, "limit plus one must converge through two bounded children");
14099 assert_eq!(limit_plus_one_retries, 2, "limit plus one must require exactly two child batches");
14100
14101 let first_batch = store
14102 .delete_object_with_tier_delete_journal(
14103 bucket,
14104 "multi-chunk/",
14105 ObjectOptions {
14106 delete_prefix: true,
14107 ..Default::default()
14108 },
14109 )
14110 .await
14111 .expect_err("the first bounded child must request a successor batch");
14112 assert!(
14113 first_batch.to_string().contains("retry the next durable batch"),
14114 "unexpected first child result: {first_batch}"
14115 );
14116
14117 let newcomer = "multi-chunk/zzz-newcomer.bin";
14118 let mut newcomer_reader = PutObjReader::from_vec(vec![b'n'; 1024 * 1024]);
14119 let newcomer_source = store
14120 .put_object(bucket, newcomer, &mut newcomer_reader, &ObjectOptions::default())
14121 .await
14122 .expect("a source created between chunks should be written");
14123 store
14124 .transition_object(
14125 bucket,
14126 newcomer,
14127 &ObjectOptions {
14128 transition: TransitionOptions {
14129 status: TRANSITION_PENDING.to_string(),
14130 tier: tier_name.to_string(),
14131 etag: newcomer_source
14132 .etag
14133 .clone()
14134 .expect("the between-chunks source should have an etag"),
14135 ..Default::default()
14136 },
14137 mod_time: newcomer_source.mod_time,
14138 ..Default::default()
14139 },
14140 )
14141 .await
14142 .expect("a source created between chunks should transition");
14143
14144 let mut completed = false;
14145 let mut durable_batch_retries = 1;
14146 for _ in 0..7 {
14147 match store
14148 .delete_object_with_tier_delete_journal(
14149 bucket,
14150 "multi-chunk/",
14151 ObjectOptions {
14152 delete_prefix: true,
14153 ..Default::default()
14154 },
14155 )
14156 .await
14157 {
14158 Ok(_) => {
14159 completed = true;
14160 break;
14161 }
14162 Err(err)
14163 if err.to_string().contains("retry the next durable batch")
14164 || err.to_string().contains("retry the next batch") =>
14165 {
14166 durable_batch_retries += 1;
14167 }
14168 Err(err) => panic!("chunked prefix delete returned an unexpected error: {err}"),
14169 }
14170 }
14171 assert!(completed, "six entries must converge through three bounded child batches");
14172 assert_eq!(durable_batch_retries, 3, "limit two should require exactly three child batches");
14173 assert_eq!(tier_delete_journal_count(store.clone()).await, 11);
14174 assert_eq!(
14175 backend.object_count().await,
14176 11,
14177 "remote cleanup must remain durable while the tier is unavailable"
14178 );
14179 let dispatch_records = store
14180 .clone()
14181 .list_objects_v2(
14182 RUSTFS_META_BUCKET,
14183 TIER_DELETE_DISPATCH_MANIFEST_PREFIX,
14184 None,
14185 None,
14186 20,
14187 false,
14188 None,
14189 false,
14190 )
14191 .await
14192 .expect("bounded dispatch records should be listable");
14193 let mut child_batches = 0;
14194 let mut legacy_at_limit_seen = false;
14195 for record in &dispatch_records.objects {
14196 let data = com::read_config(store.clone(), &record.name)
14197 .await
14198 .expect("a retained dispatch record should be readable");
14199 let value: serde_json::Value = serde_json::from_slice(&data).expect("a retained dispatch record should contain JSON");
14200 if record.name.contains("/chunks/") {
14201 let journal_count = value["journal_count"]
14202 .as_u64()
14203 .expect("a retained child should declare its journal count");
14204 assert!(journal_count <= 2, "a child batch exceeded the configured resource bound");
14205 child_batches += 1;
14206 } else if value["prefix"] == "at-limit/" {
14207 assert!(
14208 value.get("record_type").is_none(),
14209 "the exact-limit root must remain a legacy v1 manifest"
14210 );
14211 assert_eq!(value["journal_count"], 2);
14212 legacy_at_limit_seen = true;
14213 }
14214 }
14215 assert!(
14216 legacy_at_limit_seen,
14217 "the exact-limit dispatch must retain its byte-compatible root shape"
14218 );
14219 assert_eq!(child_batches, 5, "nine chunked sources should persist exactly five bounded children");
14220
14221 backend.set_remove_failure(false);
14222 drive_tier_delete_dispatch_restart_to_convergence(store.clone()).await;
14223 assert_eq!(tier_delete_journal_count(store.clone()).await, 0);
14224 assert_eq!(tier_delete_dispatch_manifest_count(store.clone()).await, 0);
14225 assert_eq!(backend.object_count().await, 0);
14226 assert_eq!(
14227 backend.remove_versions().await.len(),
14228 11,
14229 "each remote version must be removed exactly once"
14230 );
14231 shutdown.cancel();
14232 }
14233
14234 #[cfg(feature = "test-util")]
14235 #[test]
14236 #[serial_test::serial(storage_class_env)]
14237 fn tier_delete_chunk_crash_boundaries_resume_without_skipping_sources() {
14238 run_large_stack_async_test(
14239 "tier-delete-chunk-crash-boundaries",
14240 tier_delete_chunk_crash_boundaries_resume_without_skipping_sources_case,
14241 );
14242 }
14243
14244 #[cfg(feature = "test-util")]
14245 async fn tier_delete_chunk_crash_boundaries_resume_without_skipping_sources_case() {
14246 let _batch_limit = TierDeleteDispatchBatchLimitGuard::install(1);
14247 let stages = [
14248 TierDeleteChunkTestStage::ParentPersisted,
14249 TierDeleteChunkTestStage::ChildManifestPersisted,
14250 TierDeleteChunkTestStage::ParentBound,
14251 TierDeleteChunkTestStage::DispatchAuthorized,
14252 TierDeleteChunkTestStage::LocalReplayCompleted,
14253 TierDeleteChunkTestStage::ChildCompleted,
14254 TierDeleteChunkTestStage::ParentProgressed,
14255 TierDeleteChunkTestStage::FinalLocalDeletionCompleted,
14256 TierDeleteChunkTestStage::ParentCompleted,
14257 ];
14258 for (case, stage) in stages.into_iter().enumerate() {
14259 let temp_dir = tempfile::tempdir().expect("create chunk crash-boundary store dir");
14260 let initial_name = format!("chunk-crash-boundary-{case}");
14261 let (ctx, store, shutdown) =
14262 without_storage_class_env(build_isolated_test_store(temp_dir.path(), &initial_name, &[4])).await;
14263 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
14264 let tier_name = format!("CHUNK-CRASH-BOUNDARY-{case}");
14265 let backend = register_mock_tier(&ctx.tier_config_mgr(), &tier_name).await;
14266 backend.set_remove_failure(true);
14267 let bucket = format!("chunk-crash-boundary-{case}-bucket");
14268 let prefix = "prefix/";
14269 store
14270 .make_bucket(&bucket, &MakeBucketOptions::default())
14271 .await
14272 .expect("chunk crash-boundary bucket should be created");
14273 for index in 0..2 {
14274 let object = format!("{prefix}object-{index}.bin");
14275 let mut reader = PutObjReader::from_vec(vec![b'a' + index as u8; 1024 * 1024]);
14276 let source = store
14277 .put_object(&bucket, &object, &mut reader, &ObjectOptions::default())
14278 .await
14279 .expect("chunk crash-boundary source should be written");
14280 store
14281 .transition_object(
14282 &bucket,
14283 &object,
14284 &ObjectOptions {
14285 transition: TransitionOptions {
14286 status: TRANSITION_PENDING.to_string(),
14287 tier: tier_name.clone(),
14288 etag: source.etag.clone().expect("chunk crash-boundary source should have an etag"),
14289 ..Default::default()
14290 },
14291 mod_time: source.mod_time,
14292 ..Default::default()
14293 },
14294 )
14295 .await
14296 .expect("chunk crash-boundary source should transition");
14297 }
14298 let local_only = format!("{prefix}local-only.bin");
14299 let mut local_reader = PutObjReader::from_vec(vec![b'l'; 1024 * 1024]);
14300 store
14301 .put_object(&bucket, &local_only, &mut local_reader, &ObjectOptions::default())
14302 .await
14303 .expect("chunk crash-boundary local-only object should be written");
14304 let tier_config = ctx
14305 .tier_config_mgr()
14306 .read()
14307 .await
14308 .tiers
14309 .get(&tier_name)
14310 .expect("chunk crash-boundary tier config should remain available for restart")
14311 .clone_with_credentials();
14312
14313 if matches!(
14314 stage,
14315 TierDeleteChunkTestStage::FinalLocalDeletionCompleted | TierDeleteChunkTestStage::ParentCompleted
14316 ) {
14317 for _ in 0..2 {
14318 let retry = store
14319 .delete_object_with_tier_delete_journal(
14320 &bucket,
14321 prefix,
14322 ObjectOptions {
14323 delete_prefix: true,
14324 ..Default::default()
14325 },
14326 )
14327 .await
14328 .expect_err("each bounded child must complete before the final crash boundary");
14329 assert!(
14330 retry.to_string().contains("retry the next durable batch"),
14331 "unexpected bounded-child result before {stage:?}: {retry}"
14332 );
14333 }
14334 }
14335
14336 let barrier = TierDeleteChunkTestBarrier::install(stage);
14337 let worker_store = store.clone();
14338 let worker_bucket = bucket.clone();
14339 let worker = tokio::spawn(async move {
14340 worker_store
14341 .delete_object_with_tier_delete_journal(
14342 &worker_bucket,
14343 prefix,
14344 ObjectOptions {
14345 delete_prefix: true,
14346 ..Default::default()
14347 },
14348 )
14349 .await
14350 });
14351 tokio::time::timeout(Duration::from_secs(30), barrier.wait_until_paused())
14352 .await
14353 .unwrap_or_else(|_| panic!("chunk delete did not reach crash boundary {stage:?}"));
14354 worker.abort();
14355 let _ = worker.await;
14356 drop(barrier);
14357 let released_bucket_guard =
14358 tokio::time::timeout(Duration::from_secs(5), store.acquire_bucket_lifecycle_write_lock(&bucket))
14359 .await
14360 .unwrap_or_else(|_| panic!("canceling at {stage:?} did not release the bucket lifecycle lock"))
14361 .unwrap_or_else(|err| {
14362 panic!("bucket lifecycle lock reacquire failed after cancellation at {stage:?}: {err}")
14363 });
14364 drop(released_bucket_guard);
14365 shutdown.cancel();
14366 drop(store);
14367 drop(ctx);
14368
14369 let restarted_name = format!("chunk-crash-boundary-{case}-restart");
14370 let (restarted_ctx, restarted_store, restarted_shutdown) =
14371 without_storage_class_env(build_isolated_test_store(temp_dir.path(), &restarted_name, &[4])).await;
14372 {
14373 let tier_config_mgr = restarted_ctx.tier_config_mgr();
14374 let mut manager = tier_config_mgr.write().await;
14375 manager.tiers.insert(tier_name.clone(), tier_config);
14376 manager
14377 .install_test_driver(&tier_name, Box::new(backend.clone()))
14378 .expect("the exact chunk crash-boundary tier driver should reinstall after restart");
14379 }
14380 crate::bucket::metadata_sys::init_bucket_metadata_sys(restarted_store.clone(), Vec::new()).await;
14381
14382 let mut completed = false;
14383 for _ in 0..20 {
14384 let recovery = recover_tier_delete_dispatch_manifests(restarted_store.clone(), 100, None)
14385 .await
14386 .expect("chunk crash-boundary manifest recovery should remain readable");
14387 assert_eq!(recovery.failed, 0, "recovery must not quarantine a valid chunk boundary");
14388 match restarted_store
14389 .delete_object_with_tier_delete_journal(
14390 &bucket,
14391 prefix,
14392 ObjectOptions {
14393 delete_prefix: true,
14394 ..Default::default()
14395 },
14396 )
14397 .await
14398 {
14399 Ok(_) => {
14400 completed = true;
14401 break;
14402 }
14403 Err(err)
14404 if err.to_string().contains("retry")
14405 || err.to_string().contains("rollback")
14406 || err.to_string().contains("durable cleanup") => {}
14407 Err(err) => panic!("chunk crash-boundary retry returned an unexpected error at {stage:?}: {err}"),
14408 }
14409 }
14410 assert!(completed, "chunk state must converge after cancellation at {stage:?}");
14411 for index in 0..2 {
14412 let object = format!("{prefix}object-{index}.bin");
14413 assert!(
14414 restarted_store.pools[0]
14415 .get_disks_by_key(&object)
14416 .load_file_info_versions_exact(&bucket, &object)
14417 .await
14418 .expect("chunk crash-boundary source lookup should succeed")
14419 .is_none(),
14420 "source {object} must not be skipped after cancellation at {stage:?}"
14421 );
14422 }
14423 assert!(
14424 restarted_store.pools[0]
14425 .get_disks_by_key(&local_only)
14426 .load_file_info_versions_exact(&bucket, &local_only)
14427 .await
14428 .expect("chunk crash-boundary local-only source lookup should succeed")
14429 .is_none(),
14430 "the final raw delete must remove the local-only source after cancellation at {stage:?}"
14431 );
14432 assert_eq!(backend.object_count().await, 2);
14433 backend.set_remove_failure(false);
14434 drive_tier_delete_dispatch_restart_to_convergence(restarted_store.clone()).await;
14435 assert_eq!(tier_delete_journal_count(restarted_store.clone()).await, 0);
14436 assert_eq!(tier_delete_dispatch_manifest_count(restarted_store.clone()).await, 0);
14437 assert_eq!(backend.object_count().await, 0);
14438 assert_eq!(
14439 backend.remove_versions().await.len(),
14440 2,
14441 "each crash case must retain exactly one cleanup owner per remote version"
14442 );
14443 restarted_shutdown.cancel();
14444 }
14445 }
14446
14447 #[cfg(feature = "test-util")]
14448 #[test]
14449 #[serial_test::serial(storage_class_env)]
14450 fn tier_delete_chunk_missing_child_with_journals_fails_closed() {
14451 run_large_stack_async_test(
14452 "tier-delete-chunk-missing-child",
14453 tier_delete_chunk_missing_child_with_journals_fails_closed_case,
14454 );
14455 }
14456
14457 #[cfg(feature = "test-util")]
14458 async fn tier_delete_chunk_missing_child_with_journals_fails_closed_case() {
14459 let _batch_limit = TierDeleteDispatchBatchLimitGuard::install(1);
14460 let temp_dir = tempfile::tempdir().expect("create missing-child store dir");
14461 let (ctx, store, shutdown) =
14462 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "chunk-missing-child", &[4])).await;
14463 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
14464 let tier_name = "CHUNK-MISSING-CHILD";
14465 let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
14466 backend.set_remove_failure(true);
14467 let bucket = "chunk-missing-child-bucket";
14468 let prefix = "prefix/";
14469 store
14470 .make_bucket(bucket, &MakeBucketOptions::default())
14471 .await
14472 .expect("missing-child bucket should be created");
14473 for index in 0..2 {
14474 let object = format!("{prefix}object-{index}.bin");
14475 let mut reader = PutObjReader::from_vec(vec![b'm' + index as u8; 1024 * 1024]);
14476 let source = store
14477 .put_object(bucket, &object, &mut reader, &ObjectOptions::default())
14478 .await
14479 .expect("missing-child source should be written");
14480 store
14481 .transition_object(
14482 bucket,
14483 &object,
14484 &ObjectOptions {
14485 transition: TransitionOptions {
14486 status: TRANSITION_PENDING.to_string(),
14487 tier: tier_name.to_string(),
14488 etag: source.etag.clone().expect("missing-child source should have an etag"),
14489 ..Default::default()
14490 },
14491 mod_time: source.mod_time,
14492 ..Default::default()
14493 },
14494 )
14495 .await
14496 .expect("missing-child source should transition");
14497 }
14498
14499 let barrier = TierDeleteChunkTestBarrier::install(TierDeleteChunkTestStage::DispatchAuthorized);
14500 let worker_store = store.clone();
14501 let worker = tokio::spawn(async move {
14502 worker_store
14503 .delete_object_with_tier_delete_journal(
14504 bucket,
14505 prefix,
14506 ObjectOptions {
14507 delete_prefix: true,
14508 ..Default::default()
14509 },
14510 )
14511 .await
14512 });
14513 tokio::time::timeout(Duration::from_secs(30), barrier.wait_until_paused())
14514 .await
14515 .expect("chunk delete should persist its authorization before corruption injection");
14516 worker.abort();
14517 let _ = worker.await;
14518 drop(barrier);
14519
14520 let records = store
14521 .clone()
14522 .list_objects_v2(
14523 RUSTFS_META_BUCKET,
14524 TIER_DELETE_DISPATCH_MANIFEST_PREFIX,
14525 None,
14526 None,
14527 10,
14528 false,
14529 None,
14530 false,
14531 )
14532 .await
14533 .expect("parent and child records should be listable");
14534 let child = records
14535 .objects
14536 .iter()
14537 .find(|object| object.name.contains("/chunks/"))
14538 .expect("the bound child manifest should exist")
14539 .name
14540 .clone();
14541 com::delete_config(store.clone(), &child)
14542 .await
14543 .expect("the test should remove only the child manifest");
14544 assert_eq!(tier_delete_journal_count(store.clone()).await, 1);
14545
14546 let error = store
14547 .delete_object_with_tier_delete_journal(
14548 bucket,
14549 prefix,
14550 ObjectOptions {
14551 delete_prefix: true,
14552 ..Default::default()
14553 },
14554 )
14555 .await
14556 .expect_err("a missing child with retained journals must quarantine the parent");
14557 assert!(
14558 error.to_string().contains("missing child with retained journals"),
14559 "unexpected missing-child result: {error}"
14560 );
14561 assert_eq!(backend.object_count().await, 2, "fail-closed inspection must not remove a remote version");
14562 shutdown.cancel();
14563 }
14564
14565 #[cfg(feature = "test-util")]
14566 #[test]
14567 #[serial_test::serial(storage_class_env)]
14568 fn authorized_prefix_retry_replays_predecessor_before_newcomer() {
14569 let handle = std::thread::Builder::new()
14570 .name("authorized-prefix-predecessor-replay-test".to_string())
14571 .stack_size(32 * 1024 * 1024)
14572 .spawn(|| {
14573 let runtime = tokio::runtime::Builder::new_multi_thread()
14574 .enable_all()
14575 .worker_threads(2)
14576 .build()
14577 .expect("authorized predecessor replay test runtime should build");
14578 runtime.block_on(authorized_prefix_retry_replays_predecessor_before_newcomer_case());
14579 })
14580 .expect("authorized predecessor replay test thread should spawn");
14581 if let Err(payload) = handle.join() {
14582 std::panic::resume_unwind(payload);
14583 }
14584 }
14585
14586 #[cfg(feature = "test-util")]
14587 async fn authorized_prefix_retry_replays_predecessor_before_newcomer_case() {
14588 let _batch_limit = TierDeleteDispatchBatchLimitGuard::install(1);
14589 let temp_dir = tempfile::tempdir().expect("create authorized predecessor replay store dir");
14590 let (ctx, store, _shutdown) =
14591 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "authorized-prefix-predecessor-replay", &[4]))
14592 .await;
14593 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
14594 let tier_name = "AUTHORIZED-PREFIX-REPLAY";
14595 let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
14596 let lease = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
14597 .await
14598 .expect("authorized replay tier lease should resolve");
14599 let bucket = "authorized-prefix-predecessor-replay-bucket";
14600 let prefix = "prefix/";
14601 let predecessor = "prefix/000-predecessor.bin";
14602 let newcomer = "prefix/zzz-newcomer.bin";
14603 store
14604 .make_bucket(bucket, &MakeBucketOptions::default())
14605 .await
14606 .expect("authorized replay bucket should be created");
14607
14608 let mut predecessor_reader = PutObjReader::from_vec(vec![b'p'; 1024 * 1024]);
14609 let predecessor_source = store
14610 .put_object(bucket, predecessor, &mut predecessor_reader, &ObjectOptions::default())
14611 .await
14612 .expect("predecessor source should be written");
14613 store
14614 .transition_object(
14615 bucket,
14616 predecessor,
14617 &ObjectOptions {
14618 transition: TransitionOptions {
14619 status: TRANSITION_PENDING.to_string(),
14620 tier: tier_name.to_string(),
14621 etag: predecessor_source.etag.clone().expect("predecessor should have an etag"),
14622 ..Default::default()
14623 },
14624 mod_time: predecessor_source.mod_time,
14625 ..Default::default()
14626 },
14627 )
14628 .await
14629 .expect("predecessor should transition");
14630 let predecessor_info = store
14631 .get_object_info(bucket, predecessor, &ObjectOptions::default())
14632 .await
14633 .expect("transitioned predecessor should be readable");
14634 let mut predecessor_entry =
14635 transitioned_delete_journal_entry_for_source(None, false, false, bucket, predecessor, &predecessor_info)
14636 .expect("transitioned predecessor should produce a dispatch journal");
14637 predecessor_entry.backend_identity = Some(lease.backend_identity());
14638 let incarnation = store
14639 .bucket_incarnation_id(bucket)
14640 .await
14641 .expect("authorized replay bucket incarnation should resolve");
14642 let (manifest_name, _) = install_test_tier_delete_dispatch_fixture(
14643 store.clone(),
14644 bucket,
14645 incarnation,
14646 prefix,
14647 vec![(predecessor_entry, Some(TierDeleteJournalState::Dispatched))],
14648 TierDeleteDispatchManifestState::DispatchAuthorized,
14649 )
14650 .await
14651 .expect("authorized predecessor fixture should persist");
14652
14653 let mut newcomer_reader = PutObjReader::from_vec(vec![b'n'; 1024 * 1024]);
14654 let newcomer_source = store
14655 .put_object(bucket, newcomer, &mut newcomer_reader, &ObjectOptions::default())
14656 .await
14657 .expect("newcomer source should be written after authorization");
14658 store
14659 .transition_object(
14660 bucket,
14661 newcomer,
14662 &ObjectOptions {
14663 transition: TransitionOptions {
14664 status: TRANSITION_PENDING.to_string(),
14665 tier: tier_name.to_string(),
14666 etag: newcomer_source.etag.clone().expect("newcomer should have an etag"),
14667 ..Default::default()
14668 },
14669 mod_time: newcomer_source.mod_time,
14670 ..Default::default()
14671 },
14672 )
14673 .await
14674 .expect("newcomer should transition");
14675 backend.set_remove_failure(true);
14676
14677 let publication_epoch = store
14678 .scanner_data_usage_publication_epoch()
14679 .await
14680 .expect("authorized replay should observe the current scanner publication epoch");
14681 let publication_scope = store
14682 .scanner_data_usage_publication_commit_scope(
14683 publication_epoch,
14684 tokio::time::Instant::now() + Duration::from_secs(30),
14685 Vec::new(),
14686 )
14687 .await
14688 .expect("authorized replay should acquire a scanner publication scope");
14689
14690 let retry = store
14691 .delete_object_with_tier_delete_journal(
14692 bucket,
14693 prefix,
14694 ObjectOptions {
14695 delete_prefix: true,
14696 scanner_publication_commit_scope: Some(publication_scope.clone()),
14697 ..Default::default()
14698 },
14699 )
14700 .await
14701 .expect_err("the predecessor-only replay must request a successor retry");
14702 assert!(
14703 retry.to_string().contains("retry the successor dispatch"),
14704 "unexpected predecessor replay result: {retry}"
14705 );
14706 assert_eq!(
14707 publication_scope.state(),
14708 crate::object_api::ScannerPublicationCommitState::Committed,
14709 "an exact predecessor replay must report its local mutation as committed"
14710 );
14711 assert!(publication_scope.release_movement_permit().await);
14712 assert!(
14713 store.pools[0]
14714 .get_disks_by_key(predecessor)
14715 .load_file_info_versions_exact(bucket, predecessor)
14716 .await
14717 .expect("predecessor metadata should remain readable")
14718 .is_none(),
14719 "the authorized predecessor must be removed exactly"
14720 );
14721 assert!(
14722 store.pools[0]
14723 .get_disks_by_key(newcomer)
14724 .load_file_info_versions_exact(bucket, newcomer)
14725 .await
14726 .expect("newcomer metadata should remain readable")
14727 .is_some(),
14728 "the predecessor authorization must not delete the newcomer"
14729 );
14730 assert_eq!(
14731 test_tier_delete_dispatch_manifest_state(store.clone(), &manifest_name)
14732 .await
14733 .expect("completed predecessor manifest should remain readable"),
14734 Some(TierDeleteDispatchManifestState::Completed)
14735 );
14736
14737 backend.set_remove_failure(false);
14738 let predecessor_journals = recover_tier_delete_journal_entries(store.clone(), 100, None)
14739 .await
14740 .expect("completed predecessor journal should recover");
14741 assert_eq!(predecessor_journals.failed, 0);
14742 assert!(predecessor_journals.scanned <= 1 && predecessor_journals.deleted <= 1);
14743 assert_eq!(tier_delete_journal_count(store.clone()).await, 0);
14744 assert_eq!(backend.object_count().await, 1, "only the newcomer remote object should remain");
14745 assert_eq!(backend.remove_versions().await.len(), 1);
14746 let predecessor_manifest = recover_tier_delete_dispatch_manifests(store.clone(), 100, None)
14747 .await
14748 .expect("completed predecessor manifest should be collected");
14749 assert_eq!(predecessor_manifest.failed, 0);
14750 assert!(predecessor_manifest.deleted <= 1);
14751 assert_eq!(tier_delete_dispatch_manifest_count(store.clone()).await, 0);
14752
14753 store
14754 .delete_object_with_tier_delete_journal(
14755 bucket,
14756 prefix,
14757 ObjectOptions {
14758 delete_prefix: true,
14759 ..Default::default()
14760 },
14761 )
14762 .await
14763 .expect("a successor dispatch should delete the remaining newcomer");
14764 assert!(
14765 store.pools[0]
14766 .get_disks_by_key(newcomer)
14767 .load_file_info_versions_exact(bucket, newcomer)
14768 .await
14769 .expect("newcomer metadata lookup should succeed after successor")
14770 .is_none()
14771 );
14772 drive_tier_delete_dispatch_restart_to_convergence(store.clone()).await;
14773 assert_eq!(tier_delete_journal_count(store.clone()).await, 0);
14774 assert_eq!(tier_delete_dispatch_manifest_count(store.clone()).await, 0);
14775 assert_eq!(backend.object_count().await, 0);
14776 assert_eq!(backend.remove_versions().await.len(), 2);
14777 }
14778
14779 #[cfg(feature = "test-util")]
14780 #[tokio::test]
14781 #[serial_test::serial(storage_class_env)]
14782 async fn multi_pool_same_tuple_recursive_prefix_uses_one_journal_owner() {
14783 let temp_dir = tempfile::tempdir().expect("create shared-tuple prefix-delete store dir");
14784 let (ctx, store, _shutdown) = without_storage_class_env(build_isolated_test_store(
14785 temp_dir.path(),
14786 "multi-pool-shared-tuple-prefix-delete",
14787 &[4, 4],
14788 ))
14789 .await;
14790 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
14791 let tier_name = "SHARED-PREFIX-DELETE";
14792 let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
14793 let bucket = "multi-pool-shared-tuple-prefix-delete-bucket";
14794 let object = "prefix/shared.bin";
14795 store
14796 .make_bucket(bucket, &MakeBucketOptions::default())
14797 .await
14798 .expect("shared-tuple prefix bucket should be created");
14799 let mut reader = PutObjReader::from_vec(vec![b's'; 1024 * 1024]);
14800 let source = store.pools[0]
14801 .put_object(bucket, object, &mut reader, &ObjectOptions::default())
14802 .await
14803 .expect("shared-tuple prefix source should be written");
14804 store.pools[0]
14805 .transition_object(
14806 bucket,
14807 object,
14808 &ObjectOptions {
14809 transition: TransitionOptions {
14810 status: TRANSITION_PENDING.to_string(),
14811 tier: tier_name.to_string(),
14812 etag: source.etag.clone().expect("shared-tuple prefix source should have an etag"),
14813 ..Default::default()
14814 },
14815 mod_time: source.mod_time,
14816 ..Default::default()
14817 },
14818 )
14819 .await
14820 .expect("shared-tuple prefix source should transition");
14821 copy_test_xlmeta_between_pools(temp_dir.path(), 0, 1, bucket, object).await;
14822 assert_eq!(backend.object_count().await, 1);
14823 backend.set_remove_failure(true);
14824
14825 store
14826 .delete_object_with_tier_delete_journal(
14827 bucket,
14828 "prefix/",
14829 ObjectOptions {
14830 delete_prefix: true,
14831 ..Default::default()
14832 },
14833 )
14834 .await
14835 .expect("shared-tuple recursive prefix delete should commit");
14836
14837 assert_eq!(tier_delete_journal_count(store.clone()).await, 1);
14838 for pool_idx in 0..2 {
14839 assert!(
14840 store.pools[pool_idx]
14841 .get_disks_by_key(object)
14842 .load_file_info_versions_exact(bucket, object)
14843 .await
14844 .expect("shared-tuple post-delete metadata should decode")
14845 .is_none(),
14846 "pool {pool_idx} must be physically empty"
14847 );
14848 }
14849 backend.set_remove_failure(false);
14850 ctx.wake_tier_delete_journal_recovery();
14851 wait_for_tier_delete_journal_recovery(store.clone(), &backend, 1).await;
14852 assert_eq!(backend.object_count().await, 0);
14853 assert_eq!(backend.remove_versions().await.len(), 1);
14854 }
14855
14856 #[cfg(feature = "test-util")]
14857 #[tokio::test]
14858 #[serial_test::serial(storage_class_env)]
14859 async fn multi_pool_recursive_prefix_rejects_legacy_or_hidden_merge_loser_before_delete() {
14860 let temp_dir = tempfile::tempdir().expect("create merge-loser prefix-delete store dir");
14861 let (ctx, store, _shutdown) = without_storage_class_env(build_isolated_test_store(
14862 temp_dir.path(),
14863 "multi-pool-merge-loser-prefix-delete",
14864 &[4, 4],
14865 ))
14866 .await;
14867 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
14868 let tier_name = "MERGE-LOSER-PREFIX-DELETE";
14869 let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
14870 let bucket = "multi-pool-merge-loser-prefix-delete-bucket";
14871 store
14872 .make_bucket(bucket, &MakeBucketOptions::default())
14873 .await
14874 .expect("merge-loser prefix bucket should be created");
14875
14876 for (object, byte) in [("legacy/item.bin", b'l'), ("hidden/item.bin", b'h')] {
14877 let mut reader = PutObjReader::from_vec(vec![byte; 1024 * 1024]);
14878 let source = store.pools[0]
14879 .put_object(bucket, object, &mut reader, &ObjectOptions::default())
14880 .await
14881 .expect("merge-loser source should be written");
14882 store.pools[0]
14883 .transition_object(
14884 bucket,
14885 object,
14886 &ObjectOptions {
14887 transition: TransitionOptions {
14888 status: TRANSITION_PENDING.to_string(),
14889 tier: tier_name.to_string(),
14890 etag: source.etag.clone().expect("merge-loser source should have an etag"),
14891 ..Default::default()
14892 },
14893 mod_time: source.mod_time,
14894 ..Default::default()
14895 },
14896 )
14897 .await
14898 .expect("merge-loser source should transition");
14899 copy_test_xlmeta_between_pools(temp_dir.path(), 0, 1, bucket, object).await;
14900 }
14901 rewrite_transitioned_xlmeta_as_legacy_unknown(temp_dir.path(), 1, bucket, "legacy/item.bin", false).await;
14902 backend.set_remove_failure(true);
14903 store.pools[1]
14904 .delete_object(bucket, "hidden/item.bin", ObjectOptions::default())
14905 .await
14906 .expect("second-pool source delete should retain a hidden free-version");
14907
14908 let legacy_error = store
14909 .delete_object_with_tier_delete_journal(
14910 bucket,
14911 "legacy/",
14912 ObjectOptions {
14913 delete_prefix: true,
14914 ..Default::default()
14915 },
14916 )
14917 .await
14918 .expect_err("a legacy merge loser must reject recursive deletion");
14919 assert!(legacy_error.to_string().contains("legacy Unknown"), "unexpected error: {legacy_error}");
14920 let hidden_error = store
14921 .delete_object_with_tier_delete_journal(
14922 bucket,
14923 "hidden/",
14924 ObjectOptions {
14925 delete_prefix: true,
14926 ..Default::default()
14927 },
14928 )
14929 .await
14930 .expect_err("a hidden free-version merge loser must reject recursive deletion");
14931 assert!(
14932 hidden_error.to_string().contains("free-version cleanup obligation"),
14933 "unexpected error: {hidden_error}"
14934 );
14935
14936 assert_eq!(tier_delete_journal_count(store.clone()).await, 0);
14937 assert_eq!(backend.object_count().await, 2);
14938 assert_eq!(backend.remove_count().await, 0);
14939 for object in ["legacy/item.bin", "hidden/item.bin"] {
14940 let pool0 = store.pools[0]
14941 .get_disks_by_key(object)
14942 .load_file_info_versions_exact(bucket, object)
14943 .await
14944 .expect("winner metadata should decode")
14945 .expect("winner source must remain before destructive deletion");
14946 assert_eq!(pool0.versions.iter().filter(|version| !version.tier_free_version()).count(), 1);
14947 }
14948 let hidden_pool1 = store.pools[1]
14949 .get_disks_by_key("hidden/item.bin")
14950 .load_file_info_versions_exact(bucket, "hidden/item.bin")
14951 .await
14952 .expect("hidden loser metadata should decode")
14953 .expect("hidden loser cleanup owner must remain");
14954 assert_eq!(
14955 hidden_pool1
14956 .versions
14957 .iter()
14958 .chain(hidden_pool1.free_versions.iter())
14959 .filter(|version| version.tier_free_version())
14960 .count(),
14961 1
14962 );
14963 }
14964
14965 #[cfg(feature = "test-util")]
14966 #[tokio::test]
14967 #[serial_test::serial(storage_class_env)]
14968 async fn recursive_prefix_partial_pool_failure_keeps_prepared_cleanup_owners() {
14969 let temp_dir = tempfile::tempdir().expect("create partial prefix-delete store dir");
14970 let (ctx, store, _shutdown) =
14971 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "partial-pool-prefix-delete", &[4, 4])).await;
14972 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
14973 let tier_name = "PARTIAL-PREFIX-DELETE";
14974 let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
14975 let bucket = "partial-pool-prefix-delete-bucket";
14976 let object = "prefix/item.bin";
14977 store
14978 .make_bucket(bucket, &MakeBucketOptions::default())
14979 .await
14980 .expect("partial prefix bucket should be created");
14981 for (pool_idx, byte) in [(0, b'a'), (1, b'b')] {
14982 let mut reader = PutObjReader::from_vec(vec![byte; 1024 * 1024]);
14983 let source = store.pools[pool_idx]
14984 .put_object(bucket, object, &mut reader, &ObjectOptions::default())
14985 .await
14986 .expect("pool-local prefix source should be written");
14987 store.pools[pool_idx]
14988 .transition_object(
14989 bucket,
14990 object,
14991 &ObjectOptions {
14992 transition: TransitionOptions {
14993 status: TRANSITION_PENDING.to_string(),
14994 tier: tier_name.to_string(),
14995 etag: source.etag.clone().expect("pool-local source should have an etag"),
14996 ..Default::default()
14997 },
14998 mod_time: source.mod_time,
14999 ..Default::default()
15000 },
15001 )
15002 .await
15003 .expect("pool-local prefix source should transition");
15004 }
15005 assert_eq!(backend.object_count().await, 2);
15006 backend.set_remove_failure(true);
15007
15008 let barrier = crate::store::object::PrefixDeleteAfterJournalPrepareBarrier::install(bucket, "prefix/");
15009 let delete_store = store.clone();
15010 let delete = tokio::spawn(async move {
15011 delete_store
15012 .delete_object_with_tier_delete_journal(
15013 bucket,
15014 "prefix/",
15015 ObjectOptions {
15016 delete_prefix: true,
15017 ..Default::default()
15018 },
15019 )
15020 .await
15021 });
15022 tokio::time::timeout(Duration::from_secs(10), barrier.wait_until_paused())
15023 .await
15024 .expect("prefix delete should pause after every journal is Prepared");
15025 assert_eq!(tier_delete_journal_count(store.clone()).await, 2);
15026
15027 let failed_set = store.pools[1].get_disks_by_key(object);
15028 let offline_disks = force_set_disks_offline_for_test(&failed_set).await;
15029 barrier.release();
15030 let error = delete
15031 .await
15032 .expect("partial prefix delete task should join")
15033 .expect_err("an offline second pool must fail the recursive prefix delete");
15034 assert!(
15035 matches!(
15036 error,
15037 StorageError::InsufficientWriteQuorum(_, _) | StorageError::InsufficientReadQuorum(_, _)
15038 ),
15039 "unexpected error: {error:?}"
15040 );
15041 drop(offline_disks);
15042 drop(barrier);
15043
15044 assert_eq!(
15045 tier_delete_journal_count(store.clone()).await,
15046 2,
15047 "an ambiguous partial mutation must retain every Dispatched owner"
15048 );
15049 let first_pool_state = store.pools[0]
15050 .get_disks_by_key(object)
15051 .load_file_info_versions_exact(bucket, object)
15052 .await
15053 .expect("first-pool metadata should remain decodable after an ambiguous failure");
15054 if let Some(first_pool_state) = first_pool_state {
15055 assert_eq!(
15056 first_pool_state
15057 .versions
15058 .iter()
15059 .filter(|version| !version.tier_free_version())
15060 .count(),
15061 1,
15062 "a retained first-pool source must remain a visible source, not an unjournaled fallback owner"
15063 );
15064 }
15065 assert!(
15066 store.pools[1]
15067 .get_disks_by_key(object)
15068 .load_file_info_versions_exact(bucket, object)
15069 .await
15070 .expect("failed-pool metadata should decode")
15071 .is_some(),
15072 "the failed pool should retain its live source"
15073 );
15074
15075 let _ = recover_tier_delete_journal_entries(store.clone(), 100, None)
15076 .await
15077 .expect("Dispatched owners should reconcile after the failed pool returns");
15078 assert_eq!(
15079 tier_delete_journal_count(store.clone()).await,
15080 2,
15081 "recovery must never abort or delete a Dispatched journal while any source is still live"
15082 );
15083 store
15084 .delete_object_with_tier_delete_journal(
15085 bucket,
15086 "prefix/",
15087 ObjectOptions {
15088 delete_prefix: true,
15089 ..Default::default()
15090 },
15091 )
15092 .await
15093 .expect("retry should delete the remaining pool source");
15094 backend.set_remove_failure(false);
15095 for _ in 0..3 {
15096 let _ = recover_tier_delete_journal_entries(store.clone(), 100, None)
15097 .await
15098 .expect("remaining pool journals should settle after backend recovery");
15099 if tier_delete_journal_count(store.clone()).await == 0 {
15100 break;
15101 }
15102 }
15103 assert_eq!(tier_delete_journal_count(store.clone()).await, 0);
15104 assert_eq!(backend.object_count().await, 0);
15105 assert_eq!(backend.remove_versions().await.len(), 2);
15106 }
15107
15108 #[cfg(feature = "test-util")]
15109 #[tokio::test]
15110 #[serial_test::serial(storage_class_env)]
15111 async fn recursive_prefix_partial_set_failure_keeps_prepared_cleanup_owners() {
15112 let temp_dir = tempfile::tempdir().expect("create partial-set prefix-delete store dir");
15113 let (ctx, store, _shutdown) = without_storage_class_env(build_isolated_test_store_with_layout(
15114 temp_dir.path(),
15115 "partial-set-prefix-delete",
15116 &[(2, 4)],
15117 CancellationToken::new(),
15118 None,
15119 ))
15120 .await;
15121 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
15122 let tier_name = "PARTIAL-SET-PREFIX-DELETE";
15123 let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
15124 let bucket = "partial-set-prefix-delete-bucket";
15125 store
15126 .make_bucket(bucket, &MakeBucketOptions::default())
15127 .await
15128 .expect("partial-set prefix bucket should be created");
15129
15130 let objects = (0..2)
15131 .map(|target_set| {
15132 (0..512)
15133 .map(|index| format!("prefix/set-{target_set}-{index}.bin"))
15134 .find(|candidate| store.pools[0].get_disks_by_key(candidate).set_index == target_set)
15135 .expect("a deterministic key should map to each test set")
15136 })
15137 .collect::<Vec<_>>();
15138 for (index, object) in objects.iter().enumerate() {
15139 let mut reader = PutObjReader::from_vec(vec![b'0' + index as u8; 1024 * 1024]);
15140 let source = store.pools[0]
15141 .put_object(bucket, object, &mut reader, &ObjectOptions::default())
15142 .await
15143 .expect("set-local prefix source should be written");
15144 store.pools[0]
15145 .transition_object(
15146 bucket,
15147 object,
15148 &ObjectOptions {
15149 transition: TransitionOptions {
15150 status: TRANSITION_PENDING.to_string(),
15151 tier: tier_name.to_string(),
15152 etag: source.etag.clone().expect("set-local source should have an etag"),
15153 ..Default::default()
15154 },
15155 mod_time: source.mod_time,
15156 ..Default::default()
15157 },
15158 )
15159 .await
15160 .expect("set-local prefix source should transition");
15161 }
15162 assert_eq!(backend.object_count().await, 2);
15163 backend.set_remove_failure(true);
15164
15165 let barrier = crate::store::object::PrefixDeleteAfterJournalPrepareBarrier::install(bucket, "prefix/");
15166 let delete_store = store.clone();
15167 let delete = tokio::spawn(async move {
15168 delete_store
15169 .delete_object_with_tier_delete_journal(
15170 bucket,
15171 "prefix/",
15172 ObjectOptions {
15173 delete_prefix: true,
15174 ..Default::default()
15175 },
15176 )
15177 .await
15178 });
15179 tokio::time::timeout(Duration::from_secs(10), barrier.wait_until_paused())
15180 .await
15181 .expect("set-scoped prefix delete should pause after journal Prepare");
15182 assert_eq!(tier_delete_journal_count(store.clone()).await, 2);
15183
15184 let failed_set = store.pools[0].disk_set[1].clone();
15185 let offline_disks = force_set_disks_offline_for_test(&failed_set).await;
15186 barrier.release();
15187 let error = delete
15188 .await
15189 .expect("partial-set prefix delete task should join")
15190 .expect_err("an offline second set must fail recursive prefix deletion");
15191 assert!(
15192 matches!(
15193 error,
15194 StorageError::InsufficientWriteQuorum(_, _) | StorageError::InsufficientReadQuorum(_, _)
15195 ),
15196 "unexpected error: {error:?}"
15197 );
15198 drop(offline_disks);
15199 drop(barrier);
15200
15201 assert_eq!(tier_delete_journal_count(store.clone()).await, 2);
15202 let first_set_state = store.pools[0]
15203 .get_disks_by_key(&objects[0])
15204 .load_file_info_versions_exact(bucket, &objects[0])
15205 .await
15206 .expect("first-set metadata should remain decodable after an ambiguous failure");
15207 if let Some(first_set_state) = first_set_state {
15208 assert_eq!(
15209 first_set_state
15210 .versions
15211 .iter()
15212 .filter(|version| !version.tier_free_version())
15213 .count(),
15214 1,
15215 "a retained first-set source must remain visible under its Dispatched journal"
15216 );
15217 }
15218 assert!(
15219 store.pools[0]
15220 .get_disks_by_key(&objects[1])
15221 .load_file_info_versions_exact(bucket, &objects[1])
15222 .await
15223 .expect("failed-set metadata should decode")
15224 .is_some()
15225 );
15226
15227 let _ = recover_tier_delete_journal_entries(store.clone(), 100, None)
15228 .await
15229 .expect("set-scoped Dispatched owners should reconcile");
15230 assert_eq!(
15231 tier_delete_journal_count(store.clone()).await,
15232 2,
15233 "recovery must retain the complete Authorized dispatch until every source is absent"
15234 );
15235 store
15236 .delete_object_with_tier_delete_journal(
15237 bucket,
15238 "prefix/",
15239 ObjectOptions {
15240 delete_prefix: true,
15241 ..Default::default()
15242 },
15243 )
15244 .await
15245 .expect("set-scoped retry should delete the remaining source");
15246 backend.set_remove_failure(false);
15247 for _ in 0..3 {
15248 let _ = recover_tier_delete_journal_entries(store.clone(), 100, None)
15249 .await
15250 .expect("remaining set journals should settle after backend recovery");
15251 if tier_delete_journal_count(store.clone()).await == 0 {
15252 break;
15253 }
15254 }
15255 assert_eq!(tier_delete_journal_count(store.clone()).await, 0);
15256 assert_eq!(backend.object_count().await, 0);
15257 assert_eq!(backend.remove_versions().await.len(), 2);
15258 }
15259
15260 #[cfg(feature = "test-util")]
15261 #[tokio::test]
15262 #[serial_test::serial(storage_class_env)]
15263 async fn versioned_explicit_transition_delete_preserves_other_version_then_allows_bucket_delete() {
15264 let temp_dir = tempfile::tempdir().expect("create versioned transitioned delete store dir");
15265 let (ctx, store, _shutdown) =
15266 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "versioned-transitioned-delete-owner", &[4]))
15267 .await;
15268 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
15269 let tier_name = "VERSIONED-DELETE";
15270 let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
15271 let bucket = "versioned-transitioned-delete-owner-bucket";
15272 let object = "archive.bin";
15273 store
15274 .make_bucket(bucket, &MakeBucketOptions::default())
15275 .await
15276 .expect("versioned source bucket should be created");
15277 crate::bucket::metadata_sys::update_in(
15278 &ctx,
15279 bucket,
15280 BUCKET_VERSIONING_CONFIG,
15281 b"<VersioningConfiguration><Status>Enabled</Status></VersioningConfiguration>".to_vec(),
15282 )
15283 .await
15284 .expect("bucket versioning should be enabled");
15285
15286 let mut first_reader = PutObjReader::from_vec(vec![b'1'; 1024 * 1024]);
15287 let first = store
15288 .put_object(
15289 bucket,
15290 object,
15291 &mut first_reader,
15292 &ObjectOptions {
15293 versioned: true,
15294 ..Default::default()
15295 },
15296 )
15297 .await
15298 .expect("first version should be written");
15299 let first_version = first.version_id.expect("versioned PUT should return an identity");
15300 store
15301 .transition_object(
15302 bucket,
15303 object,
15304 &ObjectOptions {
15305 version_id: Some(first_version.to_string()),
15306 versioned: true,
15307 transition: TransitionOptions {
15308 status: TRANSITION_PENDING.to_string(),
15309 tier: tier_name.to_string(),
15310 etag: first.etag.clone().expect("first version should have an etag"),
15311 ..Default::default()
15312 },
15313 mod_time: first.mod_time,
15314 ..Default::default()
15315 },
15316 )
15317 .await
15318 .expect("first version should transition");
15319 let mut second_reader = PutObjReader::from_vec(vec![b'2'; 1024 * 1024]);
15320 let second = store
15321 .put_object(
15322 bucket,
15323 object,
15324 &mut second_reader,
15325 &ObjectOptions {
15326 versioned: true,
15327 ..Default::default()
15328 },
15329 )
15330 .await
15331 .expect("second version should be written");
15332 let second_version = second.version_id.expect("second PUT should return an identity");
15333
15334 backend.set_remove_failure(true);
15335 store
15336 .delete_object_with_tier_delete_journal(
15337 bucket,
15338 object,
15339 ObjectOptions {
15340 version_id: Some(first_version.to_string()),
15341 versioned: true,
15342 ..Default::default()
15343 },
15344 )
15345 .await
15346 .expect("explicit transitioned version delete should commit");
15347 store
15348 .get_object_info(
15349 bucket,
15350 object,
15351 &ObjectOptions {
15352 version_id: Some(second_version.to_string()),
15353 versioned: true,
15354 ..Default::default()
15355 },
15356 )
15357 .await
15358 .expect("the unrelated second version must remain readable");
15359 let exact = store.pools[0]
15360 .get_disks_by_key(object)
15361 .load_file_info_versions_exact(bucket, object)
15362 .await
15363 .expect("remaining version metadata should decode")
15364 .expect("the second version should remain");
15365 assert_eq!(
15366 exact
15367 .versions
15368 .iter()
15369 .chain(exact.free_versions.iter())
15370 .filter(|version| version.tier_free_version())
15371 .count(),
15372 1,
15373 "ordinary exact-version delete must leave one hidden free-version owner"
15374 );
15375 assert!(
15376 exact
15377 .versions
15378 .iter()
15379 .any(|version| version.version_id == Some(second_version)),
15380 "the explicit delete must not remove the other version"
15381 );
15382 assert_eq!(
15383 tier_delete_journal_count(store.clone()).await,
15384 0,
15385 "ordinary exact-version delete must not create a journal"
15386 );
15387 backend.set_remove_failure(false);
15388 wait_for_tier_free_version_recovery(store.clone(), &backend, 1).await;
15389 let exact = store.pools[0]
15390 .get_disks_by_key(object)
15391 .load_file_info_versions_exact(bucket, object)
15392 .await
15393 .expect("post-cleanup metadata should decode")
15394 .expect("the unrelated second version should remain after cleanup");
15395 assert_eq!(
15396 exact
15397 .versions
15398 .iter()
15399 .chain(exact.free_versions.iter())
15400 .filter(|version| version.tier_free_version())
15401 .count(),
15402 0,
15403 "free-version recovery must remove only its exact owner"
15404 );
15405 assert!(
15406 exact
15407 .versions
15408 .iter()
15409 .any(|version| version.version_id == Some(second_version)),
15410 "free-version recovery must preserve the unrelated version"
15411 );
15412
15413 store
15414 .delete_object_with_tier_delete_journal(
15415 bucket,
15416 object,
15417 ObjectOptions {
15418 version_id: Some(second_version.to_string()),
15419 versioned: true,
15420 ..Default::default()
15421 },
15422 )
15423 .await
15424 .expect("last ordinary version should be removed");
15425 store
15426 .delete_bucket(bucket, &DeleteBucketOptions::default())
15427 .await
15428 .expect("bucket delete should succeed after the last version is removed");
15429 }
15430
15431 #[cfg(feature = "test-util")]
15432 #[tokio::test]
15433 #[serial_test::serial(storage_class_env)]
15434 async fn suspended_null_transition_delete_uses_free_version_as_sole_owner() {
15435 let temp_dir = tempfile::tempdir().expect("create suspended transitioned delete store dir");
15436 let (ctx, store, _shutdown) =
15437 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "suspended-transitioned-delete-owner", &[4]))
15438 .await;
15439 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
15440 let tier_name = "SUSPENDED-DELETE";
15441 let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
15442 let bucket = "suspended-transitioned-delete-owner-bucket";
15443 let object = "archive.bin";
15444 store
15445 .make_bucket(bucket, &MakeBucketOptions::default())
15446 .await
15447 .expect("suspended source bucket should be created");
15448 crate::bucket::metadata_sys::update_in(
15449 &ctx,
15450 bucket,
15451 BUCKET_VERSIONING_CONFIG,
15452 b"<VersioningConfiguration><Status>Suspended</Status></VersioningConfiguration>".to_vec(),
15453 )
15454 .await
15455 .expect("bucket versioning should be suspended");
15456
15457 let mut reader = PutObjReader::from_vec(vec![b's'; 1024 * 1024]);
15458 let source = store
15459 .put_object(
15460 bucket,
15461 object,
15462 &mut reader,
15463 &ObjectOptions {
15464 version_suspended: true,
15465 ..Default::default()
15466 },
15467 )
15468 .await
15469 .expect("suspended null source should be written");
15470 let null_version = source.version_id.expect("suspended source should expose the null identity");
15471 assert!(null_version.is_nil(), "suspended source identity must be null");
15472 store
15473 .transition_object(
15474 bucket,
15475 object,
15476 &ObjectOptions {
15477 version_id: Some(null_version.to_string()),
15478 version_suspended: true,
15479 transition: TransitionOptions {
15480 status: TRANSITION_PENDING.to_string(),
15481 tier: tier_name.to_string(),
15482 etag: source.etag.clone().expect("suspended source should have an etag"),
15483 ..Default::default()
15484 },
15485 mod_time: source.mod_time,
15486 ..Default::default()
15487 },
15488 )
15489 .await
15490 .expect("suspended null source should transition");
15491
15492 backend.set_remove_failure(true);
15493 store
15494 .delete_object_with_tier_delete_journal(
15495 bucket,
15496 object,
15497 ObjectOptions {
15498 version_id: Some(null_version.to_string()),
15499 version_suspended: true,
15500 ..Default::default()
15501 },
15502 )
15503 .await
15504 .expect("explicit null transitioned version delete should commit");
15505 let exact = store.pools[0]
15506 .get_disks_by_key(object)
15507 .load_file_info_versions_exact(bucket, object)
15508 .await
15509 .expect("post-delete null metadata should decode")
15510 .expect("ordinary null-version delete must retain a cleanup owner");
15511 assert_eq!(
15512 exact
15513 .versions
15514 .iter()
15515 .chain(exact.free_versions.iter())
15516 .filter(|version| version.tier_free_version())
15517 .count(),
15518 1,
15519 "ordinary null-version delete must retain exactly one hidden free-version owner"
15520 );
15521 assert_eq!(
15522 exact.versions.iter().filter(|version| !version.tier_free_version()).count(),
15523 0,
15524 "ordinary null-version delete must remove the visible source"
15525 );
15526 assert_eq!(tier_delete_journal_count(store.clone()).await, 0);
15527 backend.set_remove_failure(false);
15528 wait_for_tier_free_version_recovery(store.clone(), &backend, 1).await;
15529 store
15530 .delete_bucket(bucket, &DeleteBucketOptions::default())
15531 .await
15532 .expect("suspended bucket should be empty after exact null-version cleanup");
15533 }
15534
15535 #[cfg(feature = "test-util")]
15536 #[tokio::test]
15537 #[serial_test::serial(storage_class_env)]
15538 async fn versioned_delete_marker_keeps_transitioned_source_and_remote_object() {
15539 let temp_dir = tempfile::tempdir().expect("create versioned delete-marker store dir");
15540 let (ctx, store, _shutdown) =
15541 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "transitioned-delete-marker", &[4])).await;
15542 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
15543 let tier_name = "DELETE-MARKER";
15544 let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
15545 let bucket = "transitioned-delete-marker-bucket";
15546 let object = "archive.bin";
15547 store
15548 .make_bucket(bucket, &MakeBucketOptions::default())
15549 .await
15550 .expect("versioned source bucket should be created");
15551 crate::bucket::metadata_sys::update_in(
15552 &ctx,
15553 bucket,
15554 BUCKET_VERSIONING_CONFIG,
15555 b"<VersioningConfiguration><Status>Enabled</Status></VersioningConfiguration>".to_vec(),
15556 )
15557 .await
15558 .expect("bucket versioning should be enabled");
15559
15560 let mut reader = PutObjReader::from_vec(vec![b'm'; 1024 * 1024]);
15561 let source = store
15562 .put_object(
15563 bucket,
15564 object,
15565 &mut reader,
15566 &ObjectOptions {
15567 versioned: true,
15568 ..Default::default()
15569 },
15570 )
15571 .await
15572 .expect("versioned source should be written");
15573 let source_version = source.version_id.expect("versioned source should have an identity");
15574 store
15575 .transition_object(
15576 bucket,
15577 object,
15578 &ObjectOptions {
15579 version_id: Some(source_version.to_string()),
15580 versioned: true,
15581 transition: TransitionOptions {
15582 status: TRANSITION_PENDING.to_string(),
15583 tier: tier_name.to_string(),
15584 etag: source.etag.clone().expect("versioned source should have an etag"),
15585 ..Default::default()
15586 },
15587 mod_time: source.mod_time,
15588 ..Default::default()
15589 },
15590 )
15591 .await
15592 .expect("versioned source should transition");
15593
15594 let marker = store
15595 .delete_object_with_tier_delete_journal(
15596 bucket,
15597 object,
15598 ObjectOptions {
15599 versioned: true,
15600 ..Default::default()
15601 },
15602 )
15603 .await
15604 .expect("versioned DELETE without versionId should create a marker");
15605 assert!(marker.delete_marker);
15606 let marker_version = marker.version_id.expect("delete marker should have an identity");
15607 assert_eq!(backend.object_count().await, 1, "delete-marker creation must retain the remote object");
15608 assert_eq!(
15609 tier_delete_journal_count(store.clone()).await,
15610 0,
15611 "delete-marker creation must not schedule cleanup"
15612 );
15613 let exact = store.pools[0]
15614 .get_disks_by_key(object)
15615 .load_file_info_versions_exact(bucket, object)
15616 .await
15617 .expect("versioned delete-marker metadata should decode")
15618 .expect("source and delete marker should remain on disk");
15619 assert!(
15620 exact.versions.iter().any(|version| {
15621 version.version_id == Some(source_version) && version.transition_status == rustfs_filemeta::TRANSITION_COMPLETE
15622 }),
15623 "the transitioned source must remain behind the delete marker"
15624 );
15625 assert_eq!(
15626 exact
15627 .versions
15628 .iter()
15629 .chain(exact.free_versions.iter())
15630 .filter(|version| version.tier_free_version())
15631 .count(),
15632 0,
15633 "delete-marker creation must not create a cleanup free-version"
15634 );
15635
15636 store
15637 .delete_object_with_tier_delete_journal(
15638 bucket,
15639 object,
15640 ObjectOptions {
15641 version_id: Some(marker_version.to_string()),
15642 versioned: true,
15643 ..Default::default()
15644 },
15645 )
15646 .await
15647 .expect("delete marker should be removable by exact identity");
15648 store
15649 .delete_object_with_tier_delete_journal(
15650 bucket,
15651 object,
15652 ObjectOptions {
15653 version_id: Some(source_version.to_string()),
15654 versioned: true,
15655 ..Default::default()
15656 },
15657 )
15658 .await
15659 .expect("transitioned source should be removable by exact identity");
15660 wait_for_tier_free_version_recovery(store.clone(), &backend, 1).await;
15661 if let Some(exact) = store.pools[0]
15662 .get_disks_by_key(object)
15663 .load_file_info_versions_exact(bucket, object)
15664 .await
15665 .expect("post-recovery version metadata should decode")
15666 {
15667 assert!(
15668 exact
15669 .versions
15670 .iter()
15671 .chain(exact.free_versions.iter())
15672 .all(|version| !version.tier_free_version()),
15673 "free-version recovery must remove the all-physical cleanup owner"
15674 );
15675 }
15676 store
15677 .delete_bucket(bucket, &DeleteBucketOptions::default())
15678 .await
15679 .expect("versioned source bucket should be empty after explicit cleanup");
15680 }
15681
15682 #[cfg(feature = "test-util")]
15683 #[tokio::test]
15684 #[serial_test::serial(storage_class_env)]
15685 async fn batch_transitioned_delete_uses_free_version_per_item() {
15686 let temp_dir = tempfile::tempdir().expect("create batch transitioned delete store dir");
15687 let (ctx, store, _shutdown) =
15688 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "batch-transitioned-delete-owner", &[4])).await;
15689 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
15690 let tier_name = "BATCH-DELETE";
15691 let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
15692 let bucket = "batch-transitioned-delete-owner-bucket";
15693 let transitioned = "transitioned.bin";
15694 let ordinary = "ordinary.bin";
15695 store
15696 .make_bucket(bucket, &MakeBucketOptions::default())
15697 .await
15698 .expect("batch source bucket should be created");
15699 let mut transitioned_reader = PutObjReader::from_vec(vec![b't'; 1024 * 1024]);
15700 let transitioned_source = store
15701 .put_object(bucket, transitioned, &mut transitioned_reader, &ObjectOptions::default())
15702 .await
15703 .expect("transitioned batch source should be written");
15704 store
15705 .transition_object(
15706 bucket,
15707 transitioned,
15708 &ObjectOptions {
15709 transition: TransitionOptions {
15710 status: TRANSITION_PENDING.to_string(),
15711 tier: tier_name.to_string(),
15712 etag: transitioned_source
15713 .etag
15714 .clone()
15715 .expect("transitioned batch source should have an etag"),
15716 ..Default::default()
15717 },
15718 mod_time: transitioned_source.mod_time,
15719 ..Default::default()
15720 },
15721 )
15722 .await
15723 .expect("batch source transition should commit");
15724 let mut ordinary_reader = PutObjReader::from_vec(vec![b'o'; 1024 * 1024]);
15725 store
15726 .put_object(bucket, ordinary, &mut ordinary_reader, &ObjectOptions::default())
15727 .await
15728 .expect("ordinary batch source should be written");
15729
15730 backend.set_remove_failure(true);
15731 let (_deleted, errors) = store
15732 .delete_objects_with_tier_delete_journal(
15733 bucket,
15734 vec![
15735 ObjectToDelete {
15736 object_name: transitioned.to_string(),
15737 ..Default::default()
15738 },
15739 ObjectToDelete {
15740 object_name: ordinary.to_string(),
15741 ..Default::default()
15742 },
15743 ObjectToDelete {
15744 object_name: "missing.bin".to_string(),
15745 ..Default::default()
15746 },
15747 ],
15748 ObjectOptions::default(),
15749 )
15750 .await;
15751 assert!(errors.iter().all(Option::is_none), "S3 batch delete should be idempotent: {errors:?}");
15752 let transitioned_versions = store.pools[0]
15753 .get_disks_by_key(transitioned)
15754 .load_file_info_versions_exact(bucket, transitioned)
15755 .await
15756 .expect("transitioned batch metadata should remain decodable")
15757 .expect("transitioned batch item must retain a cleanup owner");
15758 assert_eq!(
15759 transitioned_versions
15760 .versions
15761 .iter()
15762 .chain(transitioned_versions.free_versions.iter())
15763 .filter(|version| version.tier_free_version())
15764 .count(),
15765 1,
15766 "transitioned batch item must retain exactly one hidden free-version owner"
15767 );
15768 assert_eq!(
15769 transitioned_versions
15770 .versions
15771 .iter()
15772 .filter(|version| !version.tier_free_version())
15773 .count(),
15774 0,
15775 "transitioned batch item must have no visible source"
15776 );
15777 assert!(
15778 store.pools[0]
15779 .get_disks_by_key(ordinary)
15780 .load_file_info_versions_exact(bucket, ordinary)
15781 .await
15782 .expect("ordinary batch metadata should remain decodable")
15783 .is_none(),
15784 "ordinary batch item should have no local metadata"
15785 );
15786 assert_eq!(
15787 tier_delete_journal_count(store.clone()).await,
15788 0,
15789 "ordinary batch delete must not create a tier-delete journal"
15790 );
15791 assert_eq!(backend.object_count().await, 1, "failed remote cleanup must retain its object");
15792
15793 backend.set_remove_failure(false);
15794 wait_for_tier_free_version_recovery(store.clone(), &backend, 1).await;
15795 assert!(
15796 store.pools[0]
15797 .get_disks_by_key(transitioned)
15798 .load_file_info_versions_exact(bucket, transitioned)
15799 .await
15800 .expect("cleaned batch metadata should remain decodable")
15801 .is_none(),
15802 "free-version recovery must remove the exact cleanup owner"
15803 );
15804
15805 let causal = "causal.bin";
15806 let mut causal_reader = PutObjReader::from_vec(vec![b'c'; 1024 * 1024]);
15807 let causal_source = store
15808 .put_object(bucket, causal, &mut causal_reader, &ObjectOptions::default())
15809 .await
15810 .expect("causal batch source should be written");
15811 store
15812 .transition_object(
15813 bucket,
15814 causal,
15815 &ObjectOptions {
15816 transition: TransitionOptions {
15817 status: TRANSITION_PENDING.to_string(),
15818 tier: tier_name.to_string(),
15819 etag: causal_source.etag.clone().expect("causal batch source should have an etag"),
15820 ..Default::default()
15821 },
15822 mod_time: causal_source.mod_time,
15823 ..Default::default()
15824 },
15825 )
15826 .await
15827 .expect("causal batch source transition should commit");
15828 let (_deleted, errors) = store
15829 .delete_objects(
15830 bucket,
15831 vec![
15832 ObjectToDelete {
15833 object_name: causal.to_string(),
15834 ..Default::default()
15835 },
15836 ObjectToDelete {
15837 object_name: causal.to_string(),
15838 ..Default::default()
15839 },
15840 ],
15841 ObjectOptions::default(),
15842 )
15843 .await;
15844 assert!(
15845 errors.iter().all(Option::is_none),
15846 "duplicate causal batch deletes should remain idempotent: {errors:?}"
15847 );
15848 tokio::time::timeout(Duration::from_secs(30), async {
15849 loop {
15850 let metadata_absent = exact_metadata_absent_during_cleanup(&store, bucket, causal).await;
15851 if metadata_absent && backend.remove_versions().await.len() >= 2 {
15852 return;
15853 }
15854 tokio::time::sleep(Duration::from_millis(10)).await;
15855 }
15856 })
15857 .await
15858 .expect("batch free-version receipt should converge without another recovery scan");
15859 assert_eq!(
15860 backend.remove_versions().await.len(),
15861 2,
15862 "duplicate batch requests must cause only one remote delete for the causal object"
15863 );
15864
15865 crate::bucket::metadata_sys::update_in(
15866 &ctx,
15867 bucket,
15868 BUCKET_VERSIONING_CONFIG,
15869 b"<VersioningConfiguration><Status>Enabled</Status></VersioningConfiguration>".to_vec(),
15870 )
15871 .await
15872 .expect("causal batch bucket versioning should be enabled");
15873 let versioned_causal = "versioned-causal.bin";
15874 let mut versioned_reader = PutObjReader::from_vec(vec![b'v'; 1024 * 1024]);
15875 let versioned_source = store
15876 .put_object(
15877 bucket,
15878 versioned_causal,
15879 &mut versioned_reader,
15880 &ObjectOptions {
15881 versioned: true,
15882 ..Default::default()
15883 },
15884 )
15885 .await
15886 .expect("versioned causal batch source should be written");
15887 let versioned_source_id = versioned_source
15888 .version_id
15889 .expect("versioned causal batch source should have an identity");
15890 store
15891 .transition_object(
15892 bucket,
15893 versioned_causal,
15894 &ObjectOptions {
15895 version_id: Some(versioned_source_id.to_string()),
15896 versioned: true,
15897 transition: TransitionOptions {
15898 status: TRANSITION_PENDING.to_string(),
15899 tier: tier_name.to_string(),
15900 etag: versioned_source
15901 .etag
15902 .clone()
15903 .expect("versioned causal batch source should have an etag"),
15904 ..Default::default()
15905 },
15906 mod_time: versioned_source.mod_time,
15907 ..Default::default()
15908 },
15909 )
15910 .await
15911 .expect("versioned causal batch source transition should commit");
15912 let (_deleted, errors) = store
15913 .delete_objects(
15914 bucket,
15915 vec![ObjectToDelete {
15916 object_name: versioned_causal.to_string(),
15917 version_id: Some(versioned_source_id),
15918 ..Default::default()
15919 }],
15920 ObjectOptions::default(),
15921 )
15922 .await;
15923 assert!(
15924 errors.iter().all(Option::is_none),
15925 "explicit-version causal batch delete should commit: {errors:?}"
15926 );
15927 tokio::time::timeout(Duration::from_secs(30), async {
15928 loop {
15929 let metadata_absent = exact_metadata_absent_during_cleanup(&store, bucket, versioned_causal).await;
15930 if metadata_absent && backend.remove_versions().await.len() == 3 {
15931 return;
15932 }
15933 tokio::time::sleep(Duration::from_millis(10)).await;
15934 }
15935 })
15936 .await
15937 .expect("explicit-version batch receipt should converge without a recovery scan");
15938 store
15939 .delete_bucket(bucket, &DeleteBucketOptions::default())
15940 .await
15941 .expect("batch source bucket should be physically empty");
15942 }
15943
15944 #[cfg(feature = "test-util")]
15945 #[tokio::test]
15946 #[serial_test::serial(storage_class_env)]
15947 async fn batch_transitioned_delete_aggregate_error_still_enqueues_committed_free_version() {
15948 let temp_dir = tempfile::tempdir().expect("create aggregate-error batch delete store dir");
15949 let (ctx, store, shutdown) =
15950 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "batch-transitioned-aggregate-error", &[4, 4]))
15951 .await;
15952 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
15953 let tier_name = "BATCH-AGGREGATE-ERROR";
15954 let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
15955 let bucket = "batch-transitioned-aggregate-error-bucket";
15956 let object = "archive.bin";
15957 store
15958 .make_bucket(bucket, &MakeBucketOptions::default())
15959 .await
15960 .expect("aggregate-error source bucket should be created");
15961 let mut reader = PutObjReader::from_vec(vec![b'a'; 1024 * 1024]);
15962 let source = store.pools[0]
15963 .put_object(bucket, object, &mut reader, &ObjectOptions::default())
15964 .await
15965 .expect("aggregate-error source should be written");
15966 store.pools[0]
15967 .transition_object(
15968 bucket,
15969 object,
15970 &ObjectOptions {
15971 transition: TransitionOptions {
15972 status: TRANSITION_PENDING.to_string(),
15973 tier: tier_name.to_string(),
15974 etag: source.etag.clone().expect("aggregate-error source should have an etag"),
15975 ..Default::default()
15976 },
15977 mod_time: source.mod_time,
15978 ..Default::default()
15979 },
15980 )
15981 .await
15982 .expect("aggregate-error source should transition");
15983
15984 for disk_index in 0..4 {
15988 let source_meta = temp_dir
15989 .path()
15990 .join(format!("pool0/set0/disk{disk_index}/{bucket}/{object}/{STORAGE_FORMAT_FILE}"));
15991 let target_meta = temp_dir
15992 .path()
15993 .join(format!("pool1/set0/disk{disk_index}/{bucket}/{object}/{STORAGE_FORMAT_FILE}"));
15994 tokio::fs::create_dir_all(target_meta.parent().expect("target xl.meta should have a parent"))
15995 .await
15996 .expect("second-pool object directory should be created");
15997 tokio::fs::copy(&source_meta, &target_meta)
15998 .await
15999 .expect("transitioned xl.meta should copy exactly to the second pool");
16000 }
16001
16002 ExpiryState::resize_workers(1, store.clone()).await;
16003 let injection = crate::store::object::BatchDeletePoolErrorInjection::install(
16004 bucket,
16005 1,
16006 vec![(object.to_string(), StorageError::ErasureWriteQuorum)],
16007 );
16008 let (deleted, errors) = store
16009 .delete_objects(
16010 bucket,
16011 vec![ObjectToDelete {
16012 object_name: object.to_string(),
16013 ..Default::default()
16014 }],
16015 ObjectOptions::default(),
16016 )
16017 .await;
16018 assert_eq!(injection.observed(), 1, "the second pool should inject one post-commit aggregate error");
16019 assert_eq!(errors, vec![Some(StorageError::ErasureWriteQuorum)]);
16020 assert!(deleted[0].found, "the aggregate error must retain the committed pool result");
16021 drop(injection);
16022
16023 wait_for_expiry_workers_idle(&store).await;
16026 for pool in &store.pools {
16027 assert!(
16028 pool.get_disks_by_key(object)
16029 .load_file_info_versions_exact(bucket, object)
16030 .await
16031 .expect("aggregate-error cleanup metadata should remain readable")
16032 .is_none(),
16033 "aggregate failure must not suppress committed receipt cleanup"
16034 );
16035 }
16036 assert_eq!(
16037 backend.remove_count().await,
16038 1,
16039 "committed receipts must remove the shared remote object once"
16040 );
16041 assert_eq!(backend.object_count().await, 0, "the shared remote object should be removed exactly once");
16042 store
16043 .delete_bucket(bucket, &DeleteBucketOptions::default())
16044 .await
16045 .expect("aggregate-error bucket should be physically empty");
16046 shutdown.cancel();
16047 }
16048
16049 #[cfg(feature = "test-util")]
16050 #[tokio::test]
16051 #[serial_test::serial(storage_class_env)]
16052 async fn transitioned_delete_free_version_replays_after_store_restart() {
16053 let temp_dir = tempfile::tempdir().expect("create restarted free-version store dir");
16054 let (ctx, store, shutdown) =
16055 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "transitioned-delete-restart-before", &[4]))
16056 .await;
16057 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
16058 let tier_name = "DELETE-RESTART";
16059 let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
16060 let tier_config = ctx
16061 .tier_config_mgr()
16062 .read()
16063 .await
16064 .tiers
16065 .get(tier_name)
16066 .expect("mock tier config should exist")
16067 .clone_with_credentials();
16068 let bucket = "transitioned-delete-restart-bucket";
16069 let object = "archive.bin";
16070 store
16071 .make_bucket(bucket, &MakeBucketOptions::default())
16072 .await
16073 .expect("restart source bucket should be created");
16074 let mut reader = PutObjReader::from_vec(vec![b'r'; 1024 * 1024]);
16075 let original = store
16076 .put_object(bucket, object, &mut reader, &ObjectOptions::default())
16077 .await
16078 .expect("restart source object should be written");
16079 store
16080 .transition_object(
16081 bucket,
16082 object,
16083 &ObjectOptions {
16084 transition: TransitionOptions {
16085 status: TRANSITION_PENDING.to_string(),
16086 tier: tier_name.to_string(),
16087 etag: original.etag.clone().expect("restart source should have an etag"),
16088 ..Default::default()
16089 },
16090 mod_time: original.mod_time,
16091 ..Default::default()
16092 },
16093 )
16094 .await
16095 .expect("restart source transition should commit");
16096 backend.set_remove_failure(true);
16097 store
16098 .delete_object_with_tier_delete_journal(bucket, object, ObjectOptions::default())
16099 .await
16100 .expect("local source delete should commit while remote cleanup is retryable");
16101 let retained = store.pools[0]
16102 .get_disks_by_key(object)
16103 .load_file_info_versions_exact(bucket, object)
16104 .await
16105 .expect("post-delete local metadata should decode")
16106 .expect("ordinary delete must retain a free-version across restart");
16107 assert_eq!(
16108 retained
16109 .versions
16110 .iter()
16111 .chain(retained.free_versions.iter())
16112 .filter(|version| version.tier_free_version())
16113 .count(),
16114 1,
16115 "ordinary delete must retain exactly one free-version across restart"
16116 );
16117 assert_eq!(
16118 retained
16119 .versions
16120 .iter()
16121 .filter(|version| !version.tier_free_version())
16122 .count(),
16123 0,
16124 "ordinary delete must remove the visible source before restart"
16125 );
16126 assert_eq!(tier_delete_journal_count(store.clone()).await, 0);
16127 assert_eq!(backend.object_count().await, 1);
16128
16129 shutdown.cancel();
16130 drop(store);
16131 drop(ctx);
16132 let (restarted_ctx, restarted_store, restarted_shutdown) =
16133 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "transitioned-delete-restart-after", &[4]))
16134 .await;
16135 crate::bucket::metadata_sys::init_bucket_metadata_sys(restarted_store.clone(), Vec::new()).await;
16136 {
16137 let restarted_tier_config_mgr = restarted_ctx.tier_config_mgr();
16138 let mut manager = restarted_tier_config_mgr.write().await;
16139 manager.tiers.insert(tier_name.to_string(), tier_config);
16140 manager
16141 .install_test_driver(tier_name, Box::new(backend.clone()))
16142 .expect("restarted store should bind the original tier destination");
16143 }
16144 backend.set_remove_failure(false);
16145 wait_for_tier_free_version_recovery(restarted_store.clone(), &backend, 1).await;
16146 assert_eq!(tier_delete_journal_count(restarted_store.clone()).await, 0);
16147 assert_eq!(backend.object_count().await, 0);
16148 assert!(
16149 restarted_store.pools[0]
16150 .get_disks_by_key(object)
16151 .load_file_info_versions_exact(bucket, object)
16152 .await
16153 .expect("restarted cleanup metadata should decode")
16154 .is_none(),
16155 "restarted recovery must remove the exact free-version owner"
16156 );
16157 restarted_store
16158 .delete_bucket(bucket, &DeleteBucketOptions::default())
16159 .await
16160 .expect("source bucket should be empty after restarted free-version recovery");
16161 restarted_shutdown.cancel();
16162 }
16163
16164 #[cfg(feature = "test-util")]
16165 #[tokio::test]
16166 #[serial_test::serial(storage_class_env)]
16167 async fn transitioned_delete_local_quorum_failure_rolls_back_without_cleanup_owner() {
16168 let temp_dir = tempfile::tempdir().expect("create failed local delete store dir");
16169 let (ctx, store, _shutdown) =
16170 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "transitioned-delete-local-failure", &[4]))
16171 .await;
16172 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
16173 let tier_name = "DELETE-LOCAL-FAIL";
16174 let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
16175 let bucket = "transitioned-delete-local-failure-bucket";
16176 let object = "archive.bin";
16177 store
16178 .make_bucket(bucket, &MakeBucketOptions::default())
16179 .await
16180 .expect("failed-delete source bucket should be created");
16181 let mut reader = PutObjReader::from_vec(vec![b'q'; 1024 * 1024]);
16182 let source = store
16183 .put_object(bucket, object, &mut reader, &ObjectOptions::default())
16184 .await
16185 .expect("failed-delete source should be written");
16186 store
16187 .transition_object(
16188 bucket,
16189 object,
16190 &ObjectOptions {
16191 transition: TransitionOptions {
16192 status: TRANSITION_PENDING.to_string(),
16193 tier: tier_name.to_string(),
16194 etag: source.etag.clone().expect("failed-delete source should have an etag"),
16195 ..Default::default()
16196 },
16197 mod_time: source.mod_time,
16198 ..Default::default()
16199 },
16200 )
16201 .await
16202 .expect("failed-delete source should transition");
16203
16204 let set = store.pools[0].get_disks_by_key(object);
16205 let saved_disk = {
16206 let mut disks = set.disks.write().await;
16207 let first = disks[0].as_ref().expect("first disk should be online");
16208 crate::disk::local::set_delete_version_fail_after_commit(first.path().as_path(), object);
16209 disks[1].take().expect("second disk should be online before fault injection")
16210 };
16211 let err = store
16212 .delete_object_with_tier_delete_journal(bucket, object, ObjectOptions::default())
16213 .await
16214 .expect_err("one post-commit error plus one offline disk must miss delete quorum");
16215 assert!(matches!(err, StorageError::Io(_)), "unexpected local delete failure: {err:?}");
16216 assert!(
16217 err.to_string()
16218 .contains("Storage resources are insufficient for the write operation"),
16219 "the caller must observe the wrapped quorum failure: {err}"
16220 );
16221 set.disks.write().await[1] = Some(saved_disk);
16222
16223 let retained = store
16224 .get_object_info(bucket, object, &ObjectOptions::default())
16225 .await
16226 .expect("failed local delete must roll back the transitioned source");
16227 assert_eq!(retained.transitioned_object.status, rustfs_filemeta::TRANSITION_COMPLETE);
16228 let retained_versions = set
16229 .load_file_info_versions_exact(bucket, object)
16230 .await
16231 .expect("rolled-back metadata should decode")
16232 .expect("rolled-back source should remain on disk");
16233 assert_eq!(
16234 retained_versions
16235 .versions
16236 .iter()
16237 .chain(retained_versions.free_versions.iter())
16238 .filter(|version| version.tier_free_version())
16239 .count(),
16240 0,
16241 "failed local quorum must not create a cleanup owner"
16242 );
16243 assert_eq!(
16244 tier_delete_journal_count(store.clone()).await,
16245 0,
16246 "ordinary delete must not create a journal, including on rollback"
16247 );
16248 assert_eq!(backend.object_count().await, 1, "failed local commit must retain the remote object");
16249 assert_eq!(backend.remove_count().await, 0, "failed local commit must not dispatch remote cleanup");
16250
16251 backend.set_remove_failure(true);
16252 store
16253 .delete_object_with_tier_delete_journal(bucket, object, ObjectOptions::default())
16254 .await
16255 .expect("retry after disk recovery should commit");
16256 let cleanup_owner = set
16257 .load_file_info_versions_exact(bucket, object)
16258 .await
16259 .expect("retry metadata should decode")
16260 .expect("successful retry must leave a free-version owner");
16261 assert_eq!(
16262 cleanup_owner
16263 .versions
16264 .iter()
16265 .chain(cleanup_owner.free_versions.iter())
16266 .filter(|version| version.tier_free_version())
16267 .count(),
16268 1,
16269 "successful retry must leave exactly one free-version owner"
16270 );
16271 assert_eq!(tier_delete_journal_count(store.clone()).await, 0);
16272 backend.set_remove_failure(false);
16273 wait_for_tier_free_version_recovery(store.clone(), &backend, 1).await;
16274 store
16275 .delete_bucket(bucket, &DeleteBucketOptions::default())
16276 .await
16277 .expect("retry should leave the source bucket empty");
16278 }
16279
16280 #[cfg(feature = "test-util")]
16281 #[tokio::test]
16282 #[serial_test::serial(storage_class_env)]
16283 async fn batch_transitioned_delete_post_commit_failures_roll_back_without_free_version_receipt() {
16284 let temp_dir = tempfile::tempdir().expect("create failed batch delete store dir");
16285 let (ctx, store, shutdown) =
16286 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "batch-delete-local-failure", &[4])).await;
16287 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
16288 let tier_name = "BATCH-DELETE-LOCAL-FAIL";
16289 let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
16290 let bucket = "batch-delete-local-failure-bucket";
16291 let object = "archive.bin";
16292 store
16293 .make_bucket(bucket, &MakeBucketOptions::default())
16294 .await
16295 .expect("failed batch source bucket should be created");
16296 crate::bucket::metadata_sys::update_in(
16297 &ctx,
16298 bucket,
16299 BUCKET_VERSIONING_CONFIG,
16300 b"<VersioningConfiguration><Status>Enabled</Status></VersioningConfiguration>".to_vec(),
16301 )
16302 .await
16303 .expect("failed batch bucket versioning should be enabled");
16304 let mut transitioned_reader = PutObjReader::from_vec(vec![b't'; 1024 * 1024]);
16305 let transitioned_source = store
16306 .put_object(
16307 bucket,
16308 object,
16309 &mut transitioned_reader,
16310 &ObjectOptions {
16311 versioned: true,
16312 ..Default::default()
16313 },
16314 )
16315 .await
16316 .expect("failed batch transitioned version should be written");
16317 let transitioned_version_id = transitioned_source
16318 .version_id
16319 .expect("failed batch transitioned source should have a version identity");
16320 store
16321 .transition_object(
16322 bucket,
16323 object,
16324 &ObjectOptions {
16325 version_id: Some(transitioned_version_id.to_string()),
16326 versioned: true,
16327 transition: TransitionOptions {
16328 status: TRANSITION_PENDING.to_string(),
16329 tier: tier_name.to_string(),
16330 etag: transitioned_source
16331 .etag
16332 .clone()
16333 .expect("failed batch transitioned source should have an etag"),
16334 ..Default::default()
16335 },
16336 mod_time: transitioned_source.mod_time,
16337 ..Default::default()
16338 },
16339 )
16340 .await
16341 .expect("failed batch source version should transition");
16342 let mut ordinary_reader = PutObjReader::from_vec(vec![b'o'; 1024 * 1024]);
16343 let ordinary_source = store
16344 .put_object(
16345 bucket,
16346 object,
16347 &mut ordinary_reader,
16348 &ObjectOptions {
16349 versioned: true,
16350 ..Default::default()
16351 },
16352 )
16353 .await
16354 .expect("failed batch ordinary sibling should be written");
16355 let ordinary_version_id = ordinary_source
16356 .version_id
16357 .expect("failed batch ordinary sibling should have a version identity");
16358 let delete_requests = || {
16359 vec![
16360 ObjectToDelete {
16361 object_name: object.to_string(),
16362 version_id: Some(transitioned_version_id),
16363 ..Default::default()
16364 },
16365 ObjectToDelete {
16366 object_name: object.to_string(),
16367 version_id: Some(ordinary_version_id),
16368 ..Default::default()
16369 },
16370 ]
16371 };
16372
16373 let set = store.pools[0].get_disks_by_key(object);
16374 let disks = set.disks.read().await;
16375 assert_eq!(disks.len(), 4, "the rollback fixture must use four disks");
16376 for disk in disks.iter().take(3) {
16379 let disk = disk.as_ref().expect("injected rollback disks should be online");
16380 crate::disk::local::set_delete_version_fail_after_commit(disk.path().as_path(), object);
16381 }
16382 drop(disks);
16383 let receipt_sink = crate::object_api::TierFreeVersionReceiptSink::new();
16384 let (_deleted, errors) = store
16385 .delete_objects(
16386 bucket,
16387 delete_requests(),
16388 ObjectOptions {
16389 tier_free_version_receipt_sink: Some(receipt_sink.clone()),
16390 ..Default::default()
16391 },
16392 )
16393 .await;
16394 assert_eq!(
16395 errors,
16396 vec![Some(StorageError::Unexpected), Some(StorageError::Unexpected),],
16397 "three post-commit disk errors must fail batch delete before receipts publish"
16398 );
16399
16400 assert!(
16401 receipt_sink
16402 .drain()
16403 .expect("the test-owned failed-batch sink should drain exactly once")
16404 .is_empty(),
16405 "a rolled-back physical group must publish no cleanup receipt"
16406 );
16407 let retained_transitioned = store
16408 .get_object_info(
16409 bucket,
16410 object,
16411 &ObjectOptions {
16412 version_id: Some(transitioned_version_id.to_string()),
16413 versioned: true,
16414 ..Default::default()
16415 },
16416 )
16417 .await
16418 .expect("failed batch delete must restore the transitioned sibling");
16419 assert_eq!(retained_transitioned.transitioned_object.status, rustfs_filemeta::TRANSITION_COMPLETE);
16420 let retained_ordinary = store
16421 .get_object_info(
16422 bucket,
16423 object,
16424 &ObjectOptions {
16425 version_id: Some(ordinary_version_id.to_string()),
16426 versioned: true,
16427 ..Default::default()
16428 },
16429 )
16430 .await
16431 .expect("failed batch delete must restore the ordinary sibling");
16432 assert_ne!(retained_ordinary.transitioned_object.status, rustfs_filemeta::TRANSITION_COMPLETE);
16433 let retained_versions = set
16434 .load_file_info_versions_exact(bucket, object)
16435 .await
16436 .expect("rolled-back batch metadata should decode")
16437 .expect("rolled-back batch source should remain on disk");
16438 assert_eq!(
16439 retained_versions
16440 .versions
16441 .iter()
16442 .chain(retained_versions.free_versions.iter())
16443 .filter(|version| version.tier_free_version())
16444 .count(),
16445 0,
16446 "failed batch quorum must not retain a free-version owner"
16447 );
16448 let retained_version_ids = retained_versions
16449 .versions
16450 .iter()
16451 .filter_map(|version| version.version_id)
16452 .collect::<std::collections::HashSet<_>>();
16453 assert_eq!(
16454 retained_version_ids,
16455 std::collections::HashSet::from([transitioned_version_id, ordinary_version_id]),
16456 "the physical-group rollback must restore both explicit siblings"
16457 );
16458 assert_eq!(backend.object_count().await, 1, "failed batch commit must retain the remote object");
16459 assert_eq!(backend.remove_count().await, 0, "failed batch commit must not dispatch remote cleanup");
16460
16461 ExpiryState::resize_workers(1, store.clone()).await;
16462 let (_deleted, retry_errors) = store
16463 .delete_objects(bucket, delete_requests(), ObjectOptions::default())
16464 .await;
16465 assert!(
16466 retry_errors.iter().all(Option::is_none),
16467 "retry after disk recovery should commit: {retry_errors:?}"
16468 );
16469 tokio::time::timeout(Duration::from_secs(30), async {
16470 loop {
16471 let metadata_absent = match set.load_file_info_versions_exact(bucket, object).await {
16474 Ok(versions) => versions.is_none(),
16475 Err(StorageError::InsufficientReadQuorum(..)) => false,
16476 Err(err) => panic!("retry cleanup metadata should remain readable: {err:?}"),
16477 };
16478 if metadata_absent && backend.remove_count().await == 1 {
16479 return;
16480 }
16481 tokio::time::sleep(Duration::from_millis(10)).await;
16482 }
16483 })
16484 .await
16485 .expect("successful batch retry should converge without a recovery scan");
16486 store
16487 .delete_bucket(bucket, &DeleteBucketOptions::default())
16488 .await
16489 .expect("successful batch retry should leave the bucket empty");
16490 shutdown.cancel();
16491 }
16492
16493 #[cfg(feature = "test-util")]
16494 async fn run_multi_pool_same_remote_tuple_delete_case(batch: bool) {
16495 let temp_dir = tempfile::tempdir().expect("create shared-tuple multi-pool store dir");
16496 let case = if batch { "batch" } else { "single" };
16497 let (ctx, store, _shutdown) = without_storage_class_env(build_isolated_test_store(
16498 temp_dir.path(),
16499 &format!("multi-pool-same-tuple-{case}"),
16500 &[4, 4],
16501 ))
16502 .await;
16503 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
16504 let tier_name = if batch { "SAME-TUPLE-BATCH" } else { "SAME-TUPLE-SINGLE" };
16505 let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
16506 let bucket = format!("multi-pool-same-tuple-{case}-bucket");
16507 let object = "archive.bin";
16508 store
16509 .make_bucket(&bucket, &MakeBucketOptions::default())
16510 .await
16511 .expect("shared-tuple bucket should be created");
16512 let mut reader = PutObjReader::from_vec(vec![b's'; 1024 * 1024]);
16513 let source = store.pools[0]
16514 .put_object(&bucket, object, &mut reader, &ObjectOptions::default())
16515 .await
16516 .expect("first pool source should be written");
16517 store.pools[0]
16518 .transition_object(
16519 &bucket,
16520 object,
16521 &ObjectOptions {
16522 transition: TransitionOptions {
16523 status: TRANSITION_PENDING.to_string(),
16524 tier: tier_name.to_string(),
16525 etag: source.etag.clone().expect("shared source should have an etag"),
16526 ..Default::default()
16527 },
16528 mod_time: source.mod_time,
16529 ..Default::default()
16530 },
16531 )
16532 .await
16533 .expect("first pool source should transition");
16534 let first = store.pools[0]
16535 .get_object_info(&bucket, object, &ObjectOptions::default())
16536 .await
16537 .expect("first pool transitioned source should be readable");
16538
16539 for disk_index in 0..4 {
16543 let source_meta = temp_dir
16544 .path()
16545 .join(format!("pool0/set0/disk{disk_index}/{bucket}/{object}/{STORAGE_FORMAT_FILE}"));
16546 let target_meta = temp_dir
16547 .path()
16548 .join(format!("pool1/set0/disk{disk_index}/{bucket}/{object}/{STORAGE_FORMAT_FILE}"));
16549 tokio::fs::create_dir_all(target_meta.parent().expect("target xl.meta should have a parent"))
16550 .await
16551 .expect("second pool object directory should be created");
16552 tokio::fs::copy(&source_meta, &target_meta)
16553 .await
16554 .expect("transitioned xl.meta should copy exactly to the second pool");
16555 }
16556 let second = store.pools[1]
16557 .get_object_info(&bucket, object, &ObjectOptions::default())
16558 .await
16559 .expect("copied second-pool source should be readable");
16560 assert_eq!(
16561 (first.transitioned_object.name.clone(), first.transitioned_object.version_id.clone()),
16562 (second.transitioned_object.name.clone(), second.transitioned_object.version_id.clone())
16563 );
16564 assert_eq!(backend.object_count().await, 1);
16565
16566 let receipt_sink = crate::object_api::TierFreeVersionReceiptSink::new();
16567 let mut delete_opts = ObjectOptions {
16568 tier_delete_journal_api: Some(store.clone()),
16569 tier_free_version_receipt_sink: Some(receipt_sink.clone()),
16570 ..Default::default()
16571 };
16572 if batch {
16573 delete_opts.delete_replication_config_snapshot = Some(Arc::new(
16574 ReplicationObjectBridge::delete_request_config_in(&store.ctx, &bucket)
16575 .await
16576 .expect("batch delete replication snapshot should load"),
16577 ));
16578 let (_deleted, errors, _accounting) = store.pools[0]
16579 .delete_objects_with_accounting(
16580 &bucket,
16581 vec![ObjectToDelete {
16582 object_name: object.to_string(),
16583 ..Default::default()
16584 }],
16585 delete_opts.clone(),
16586 )
16587 .await;
16588 assert!(errors.iter().all(Option::is_none), "first pool batch delete should commit: {errors:?}");
16589 } else {
16590 store.pools[0]
16591 .delete_object(&bucket, object, delete_opts.clone())
16592 .await
16593 .expect("first pool single delete should commit");
16594 }
16595
16596 let first_pool_versions = store.pools[0]
16597 .get_disks_by_key(object)
16598 .load_file_info_versions_exact(&bucket, object)
16599 .await
16600 .expect("first-pool metadata should decode")
16601 .expect("first pool must retain a free-version owner");
16602 assert_eq!(
16603 first_pool_versions
16604 .versions
16605 .iter()
16606 .chain(first_pool_versions.free_versions.iter())
16607 .filter(|version| version.tier_free_version())
16608 .count(),
16609 1,
16610 "first pool must retain exactly one free-version owner"
16611 );
16612 assert_eq!(
16613 first_pool_versions
16614 .versions
16615 .iter()
16616 .filter(|version| !version.tier_free_version())
16617 .count(),
16618 0,
16619 "first pool must remove its visible source"
16620 );
16621 assert_eq!(
16622 tier_delete_journal_count(store.clone()).await,
16623 0,
16624 "ordinary pool-local delete must not create a journal"
16625 );
16626
16627 ExpiryState::resize_workers(1, store.clone()).await;
16628 let first_recovery = recover_tier_free_versions(store.clone(), 100, None, None)
16629 .await
16630 .expect("first-pool free-version should be recoverable");
16631 assert_eq!(
16632 (first_recovery.enqueued, first_recovery.failed),
16633 (0, 0),
16634 "the live second-pool source is the merged walk winner, so its hidden loser must not dispatch cleanup"
16635 );
16636 wait_for_expiry_workers_idle(&store).await;
16637 assert_eq!(backend.remove_count().await, 0, "live second-pool source must gate remote DELETE");
16638 assert_eq!(backend.object_count().await, 1, "live source must retain its remote body");
16639 assert!(
16640 store.pools[0]
16641 .get_disks_by_key(object)
16642 .load_file_info_versions_exact(&bucket, object)
16643 .await
16644 .expect("blocked cleanup metadata should decode")
16645 .is_some(),
16646 "blocked cleanup must retain the first-pool free-version"
16647 );
16648 let surviving = store.pools[1]
16649 .get_object_info(&bucket, object, &ObjectOptions::default())
16650 .await
16651 .expect("second-pool source must remain GET-addressable");
16652 assert_eq!(surviving.transitioned_object.name, first.transitioned_object.name);
16653
16654 backend.set_remove_failure(true);
16655 if batch {
16656 let (_deleted, errors, _accounting) = store.pools[1]
16657 .delete_objects_with_accounting(
16658 &bucket,
16659 vec![ObjectToDelete {
16660 object_name: object.to_string(),
16661 ..Default::default()
16662 }],
16663 delete_opts,
16664 )
16665 .await;
16666 assert!(errors.iter().all(Option::is_none), "second pool batch retry should commit: {errors:?}");
16667 } else {
16668 store.pools[1]
16669 .delete_object(&bucket, object, delete_opts)
16670 .await
16671 .expect("second pool single retry should commit");
16672 }
16673 wait_for_expiry_workers_idle(&store).await;
16674 let mut free_version_ids = Vec::new();
16675 for pool_idx in 0..2 {
16676 let retained = store.pools[pool_idx]
16677 .get_disks_by_key(object)
16678 .load_file_info_versions_exact(&bucket, object)
16679 .await
16680 .expect("shared-tuple free-version metadata should decode")
16681 .expect("remote failure must retain every shared-tuple cleanup owner");
16682 let owners = retained
16683 .versions
16684 .iter()
16685 .chain(retained.free_versions.iter())
16686 .filter(|version| version.tier_free_version())
16687 .collect::<Vec<_>>();
16688 assert_eq!(owners.len(), 1, "pool {pool_idx} must retain one free-version owner");
16689 free_version_ids.push(owners[0].version_id);
16690 }
16691 assert_ne!(
16692 free_version_ids[0], free_version_ids[1],
16693 "ordinary pool-local deletes must model distinct local free-version identities"
16694 );
16695
16696 backend.set_remove_failure(false);
16697 let receipts = receipt_sink
16698 .drain()
16699 .expect("the simulated outer multi-pool wrapper should drain exactly once");
16700 assert_eq!(
16701 receipts.len(),
16702 1,
16703 "the same physical key and remote tuple must collapse to one causal task"
16704 );
16705 assert_eq!(
16706 crate::bucket::lifecycle::bucket_lifecycle_ops::enqueue_committed_free_versions(&store, receipts).await,
16707 1,
16708 "the committed shared-tuple task should enter the running worker"
16709 );
16710 wait_for_expiry_workers_idle(&store).await;
16711 assert_eq!(backend.remove_count().await, 1, "shared remote tuple should be deleted exactly once");
16712 for pool_idx in 0..2 {
16713 assert!(
16714 store.pools[pool_idx]
16715 .get_disks_by_key(object)
16716 .load_file_info_versions_exact(&bucket, object)
16717 .await
16718 .expect("post-delete metadata should decode")
16719 .is_none(),
16720 "pool {pool_idx} must not retain source or free-version metadata"
16721 );
16722 }
16723 store
16724 .delete_bucket(&bucket, &DeleteBucketOptions::default())
16725 .await
16726 .expect("shared-tuple bucket should be physically empty");
16727 }
16728
16729 #[cfg(feature = "test-util")]
16730 #[tokio::test]
16731 #[serial_test::serial(storage_class_env)]
16732 async fn multi_pool_same_remote_tuple_single_delete_waits_for_all_sources() {
16733 run_multi_pool_same_remote_tuple_delete_case(false).await;
16734 }
16735
16736 #[cfg(feature = "test-util")]
16737 #[tokio::test]
16738 #[serial_test::serial(storage_class_env)]
16739 async fn multi_pool_same_remote_tuple_batch_delete_waits_for_all_sources() {
16740 run_multi_pool_same_remote_tuple_delete_case(true).await;
16741 }
16742
16743 #[cfg(feature = "test-util")]
16744 #[tokio::test]
16745 #[serial_test::serial(storage_class_env)]
16746 async fn multi_pool_transitioned_delete_persists_one_free_version_per_remote_tuple() {
16747 let temp_dir = tempfile::tempdir().expect("create multi-pool transitioned delete store dir");
16748 let (ctx, store, _shutdown) = without_storage_class_env(build_isolated_test_store(
16749 temp_dir.path(),
16750 "multi-pool-transitioned-delete-owner",
16751 &[4, 4],
16752 ))
16753 .await;
16754 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
16755 let tier_name = "MULTIPOOL-DELETE";
16756 let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
16757 let bucket = "multi-pool-transitioned-delete-owner-bucket";
16758 let object = "archive.bin";
16759 store
16760 .make_bucket(bucket, &MakeBucketOptions::default())
16761 .await
16762 .expect("multi-pool source bucket should be created");
16763
16764 let mut remote_tuples = Vec::new();
16765 for pool_idx in 0..2 {
16766 let mut reader = PutObjReader::from_vec(vec![b'a' + pool_idx as u8; 1024 * 1024]);
16767 let source = store.pools[pool_idx]
16768 .put_object(
16769 bucket,
16770 object,
16771 &mut reader,
16772 &ObjectOptions {
16773 mod_time: Some(OffsetDateTime::now_utc() + time::Duration::seconds(pool_idx as i64)),
16774 ..Default::default()
16775 },
16776 )
16777 .await
16778 .expect("pool-local source should be written");
16779 store.pools[pool_idx]
16780 .transition_object(
16781 bucket,
16782 object,
16783 &ObjectOptions {
16784 transition: TransitionOptions {
16785 status: TRANSITION_PENDING.to_string(),
16786 tier: tier_name.to_string(),
16787 etag: source.etag.clone().expect("pool-local source should have an etag"),
16788 ..Default::default()
16789 },
16790 mod_time: source.mod_time,
16791 ..Default::default()
16792 },
16793 )
16794 .await
16795 .expect("pool-local source should transition");
16796 let transitioned = store.pools[pool_idx]
16797 .get_object_info(bucket, object, &ObjectOptions::default())
16798 .await
16799 .expect("pool-local transitioned source should be readable");
16800 remote_tuples.push((
16801 transitioned.transitioned_object.name.clone(),
16802 transitioned.transitioned_object.version_id.clone(),
16803 ));
16804 }
16805 assert_ne!(remote_tuples[0], remote_tuples[1], "fixture must use distinct remote cleanup tuples");
16806 assert_eq!(backend.object_count().await, 2);
16807
16808 backend.set_remove_failure(true);
16809 store
16810 .delete_object_with_tier_delete_journal(bucket, object, ObjectOptions::default())
16811 .await
16812 .expect("store-level delete should remove both pool copies");
16813 for pool_idx in 0..2 {
16814 let retained = store.pools[pool_idx]
16815 .get_disks_by_key(object)
16816 .load_file_info_versions_exact(bucket, object)
16817 .await
16818 .expect("pool-local post-delete metadata should decode")
16819 .expect("each pool-specific remote tuple must retain a cleanup owner");
16820 assert_eq!(
16821 retained
16822 .versions
16823 .iter()
16824 .chain(retained.free_versions.iter())
16825 .filter(|version| version.tier_free_version())
16826 .count(),
16827 1,
16828 "pool {pool_idx} must retain exactly one free-version owner"
16829 );
16830 assert_eq!(
16831 retained
16832 .versions
16833 .iter()
16834 .filter(|version| !version.tier_free_version())
16835 .count(),
16836 0,
16837 "pool {pool_idx} must not retain a visible source"
16838 );
16839 }
16840 assert_eq!(
16841 tier_delete_journal_count(store.clone()).await,
16842 0,
16843 "ordinary multi-pool delete must not create tier-delete journals"
16844 );
16845 assert_eq!(backend.object_count().await, 2, "failed remote cleanup must retain both remote objects");
16846
16847 backend.set_remove_failure(false);
16848 wait_for_tier_free_version_recovery(store.clone(), &backend, 2).await;
16849 let mut removed = backend.remove_versions().await;
16850 removed.sort();
16851 remote_tuples.sort();
16852 assert_eq!(removed, remote_tuples, "recovery must use each pool's exact remote tuple");
16853 for pool_idx in 0..2 {
16854 assert!(
16855 store.pools[pool_idx]
16856 .get_disks_by_key(object)
16857 .load_file_info_versions_exact(bucket, object)
16858 .await
16859 .expect("post-recovery pool metadata should decode")
16860 .is_none(),
16861 "pool {pool_idx} must not retain source or free-version metadata"
16862 );
16863 }
16864 store
16865 .delete_bucket(bucket, &DeleteBucketOptions::default())
16866 .await
16867 .expect("multi-pool source bucket should be physically empty");
16868 }
16869
16870 #[tokio::test]
16876 #[serial_test::serial(storage_class_env)]
16877 async fn new_with_instance_ctx_threads_context_through_store_graph() {
16878 let temp_dir = tempfile::tempdir().expect("create temp store dir");
16879 let (instance_ctx, store, _shutdown) =
16880 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "instance-ctx-store-graph-test", &[4])).await;
16881
16882 assert!(
16883 Arc::ptr_eq(&store.ctx, &instance_ctx),
16884 "the store must adopt the explicitly passed instance context"
16885 );
16886 for sets in &store.pools {
16887 assert!(
16888 Arc::ptr_eq(sets.instance_ctx(), &instance_ctx),
16889 "every pool's Sets must carry the passed instance context"
16890 );
16891 }
16892 assert_eq!(
16893 instance_ctx.deployment_id(),
16894 Some(store.id),
16895 "the deployment id must land on the passed context and mirror the store id"
16896 );
16897
16898 let registered: Vec<String> = instance_ctx.local_disk_map().read().await.keys().cloned().collect();
16899 assert_eq!(registered.len(), 4, "the passed context must register all four local disks");
16900 let registered_disk_ids = instance_ctx.local_disk_id_map();
16901 let registered_disk_ids = registered_disk_ids.read().await;
16902 assert_eq!(registered_disk_ids.len(), 4, "the passed context must publish all four disk IDs");
16903 for endpoint in registered_disk_ids.values() {
16904 assert!(
16905 registered.contains(endpoint),
16906 "every disk ID in the passed context must resolve to one of its registered endpoints"
16907 );
16908 }
16909 drop(registered_disk_ids);
16910 let bootstrap = crate::runtime::instance::bootstrap_ctx();
16911 assert_ne!(
16912 bootstrap.deployment_id(),
16913 Some(store.id),
16914 "the bootstrap context must not absorb the fresh store's deployment id"
16915 );
16916 let bootstrap_map = bootstrap.local_disk_map();
16917 let bootstrap_map = bootstrap_map.read().await;
16918 for key in ®istered {
16919 assert!(
16920 !bootstrap_map.contains_key(key),
16921 "the bootstrap context must not absorb the fresh store's disks"
16922 );
16923 }
16924 drop(bootstrap_map);
16925 let bootstrap_disk_ids = bootstrap.local_disk_id_map();
16926 let bootstrap_disk_ids = bootstrap_disk_ids.read().await;
16927 for endpoint in bootstrap_disk_ids.values() {
16928 assert!(
16929 !registered.contains(endpoint),
16930 "the bootstrap context must not absorb the fresh store's disk IDs"
16931 );
16932 }
16933 }
16934
16935 #[tokio::test]
16936 #[serial_test::serial(storage_class_env)]
16937 async fn new_with_instance_ctx_applies_default_parity_to_each_real_pool() {
16938 let temp_dir = tempfile::tempdir().expect("create multi-pool store dir");
16939 let (_, store, _shutdown) =
16940 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "pool-parity-regression", &[4, 2])).await;
16941
16942 assert_eq!(store.pools.len(), 2);
16943 assert_eq!(store.pools[0].default_parity_count, 2);
16944 assert_eq!(store.pools[0].disk_set[0].default_parity_count, 2);
16945 assert_eq!(store.pools[1].default_parity_count, 1);
16946 assert_eq!(store.pools[1].disk_set[0].default_parity_count, 1);
16947 }
16948
16949 #[tokio::test]
16954 #[serial_test::serial(storage_class_env)]
16955 async fn two_stores_initialize_their_own_bucket_metadata_sys() {
16956 let temp_a = tempfile::tempdir().expect("create temp store dir a");
16957 let temp_b = tempfile::tempdir().expect("create temp store dir b");
16958 let (ctx_a, store_a, _shutdown_a) =
16959 without_storage_class_env(build_isolated_test_store(temp_a.path(), "bucket-metadata-isolation-a", &[4])).await;
16960 let (ctx_b, store_b, _shutdown_b) =
16961 without_storage_class_env(build_isolated_test_store(temp_b.path(), "bucket-metadata-isolation-b", &[4])).await;
16962
16963 crate::bucket::metadata_sys::init_bucket_metadata_sys(store_a.clone(), Vec::new()).await;
16964 crate::bucket::metadata_sys::init_bucket_metadata_sys(store_b.clone(), Vec::new()).await;
16966
16967 let sys_a = ctx_a
16968 .bucket_metadata_sys()
16969 .expect("store A's context must hold its metadata system");
16970 let sys_b = ctx_b
16971 .bucket_metadata_sys()
16972 .expect("store B's context must hold its metadata system");
16973 assert!(!Arc::ptr_eq(&sys_a, &sys_b), "each store must own a distinct bucket metadata system");
16974 }
16975
16976 #[tokio::test]
16977 #[serial_test::serial(storage_class_env)]
16978 async fn same_name_bucket_recursive_delete_uses_owning_instance_object_lock_state() {
16979 let temp_a = tempfile::tempdir().expect("create temp store dir a");
16980 let temp_b = tempfile::tempdir().expect("create temp store dir b");
16981 let (_ctx_a, store_a, _shutdown_a) =
16982 without_storage_class_env(build_isolated_test_store(temp_a.path(), "delete-scope-a", &[4])).await;
16983 let (_ctx_b, store_b, _shutdown_b) =
16984 without_storage_class_env(build_isolated_test_store(temp_b.path(), "delete-scope-b", &[4])).await;
16985 crate::bucket::metadata_sys::init_bucket_metadata_sys(store_a.clone(), Vec::new()).await;
16986 crate::bucket::metadata_sys::init_bucket_metadata_sys(store_b.clone(), Vec::new()).await;
16987
16988 let bucket = format!("same-name-delete-scope-{}", uuid::Uuid::new_v4());
16989 store_a
16990 .make_bucket(&bucket, &MakeBucketOptions::default())
16991 .await
16992 .expect("create plain bucket in store A");
16993 store_b
16994 .make_bucket(
16995 &bucket,
16996 &MakeBucketOptions {
16997 lock_enabled: true,
16998 ..Default::default()
16999 },
17000 )
17001 .await
17002 .expect("create Object Lock bucket in store B");
17003
17004 for store in [&store_a, &store_b] {
17005 let mut reader = PutObjReader::from_vec(b"payload".to_vec());
17006 store
17007 .put_object(
17008 &bucket,
17009 "prefix/object.bin",
17010 &mut reader,
17011 &ObjectOptions {
17012 versioned: Arc::ptr_eq(store, &store_b),
17013 ..Default::default()
17014 },
17015 )
17016 .await
17017 .expect("put same-name object");
17018 }
17019
17020 store_a
17021 .delete_object(
17022 &bucket,
17023 "prefix",
17024 ObjectOptions {
17025 delete_prefix: true,
17026 ..Default::default()
17027 },
17028 )
17029 .await
17030 .expect("plain store prefix delete should remain allowed");
17031
17032 let err = store_b
17033 .delete_object(
17034 &bucket,
17035 "prefix",
17036 ObjectOptions {
17037 delete_prefix: true,
17038 ..Default::default()
17039 },
17040 )
17041 .await
17042 .expect_err("store B must not consult store A's plain bucket metadata");
17043 assert!(matches!(err, StorageError::InvalidArgument(_, _, _)));
17044 store_b
17045 .get_object_info(&bucket, "prefix/object.bin", &ObjectOptions::default())
17046 .await
17047 .expect("locked store object must survive the rejected prefix delete");
17048 }
17049
17050 #[tokio::test]
17051 #[serial_test::serial(storage_class_env)]
17052 async fn recursive_delete_checks_object_lock_on_every_version_page() {
17053 let temp = tempfile::tempdir().expect("create temp store dir");
17054 let (_ctx, store, _shutdown) =
17055 without_storage_class_env(build_isolated_test_store(temp.path(), "recursive-delete-pagination", &[4])).await;
17056 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
17057
17058 let bucket = format!("recursive-page-{}", uuid::Uuid::new_v4());
17059 store
17060 .make_bucket(&bucket, &MakeBucketOptions::default())
17061 .await
17062 .expect("create plain bucket");
17063 for object in ["prefix/a.bin", "prefix/b.bin"] {
17064 let mut reader = PutObjReader::from_vec(b"plain".to_vec());
17065 store
17066 .put_object(&bucket, object, &mut reader, &ObjectOptions::default())
17067 .await
17068 .expect("write first-page object");
17069 }
17070 let locked_object = "prefix/z-locked.bin";
17071 let mut reader = PutObjReader::from_vec(b"locked".to_vec());
17072 store
17073 .put_object(
17074 &bucket,
17075 locked_object,
17076 &mut reader,
17077 &ObjectOptions {
17078 user_defined: HashMap::from([
17079 (
17080 rustfs_filemeta::metadata_keys::OBJECT_LOCK_MODE.to_string(),
17081 s3s::dto::ObjectLockRetentionMode::COMPLIANCE.to_string(),
17082 ),
17083 (
17084 rustfs_filemeta::metadata_keys::OBJECT_LOCK_RETAIN_UNTIL_DATE.to_string(),
17085 "2099-01-01T00:00:00Z".to_string(),
17086 ),
17087 ]),
17088 ..Default::default()
17089 },
17090 )
17091 .await
17092 .expect("write second-page locked object");
17093
17094 let first_page = store.pools[0].disk_set[0]
17095 .clone()
17096 .inner_list_object_versions_for_recursive_delete(&bucket, "prefix/", None, None, 2)
17097 .await
17098 .expect("list the first recursive-delete validation page");
17099 assert!(first_page.is_truncated, "the fixture must require a continuation page");
17100 assert!(
17101 first_page.objects.iter().all(|object| object.name != locked_object),
17102 "the locked object must not be visible on the first page"
17103 );
17104 let second_page = store.pools[0].disk_set[0]
17105 .clone()
17106 .inner_list_object_versions_for_recursive_delete(
17107 &bucket,
17108 "prefix/",
17109 first_page.next_marker,
17110 first_page.next_version_idmarker,
17111 2,
17112 )
17113 .await
17114 .expect("list the recursive-delete continuation page");
17115 assert!(
17116 second_page.objects.iter().any(|object| object.name == locked_object),
17117 "the continuation page must contain the locked object"
17118 );
17119
17120 let err = store
17121 .delete_object(
17122 &bucket,
17123 "prefix/",
17124 ObjectOptions {
17125 delete_prefix: true,
17126 ..Default::default()
17127 },
17128 )
17129 .await
17130 .expect_err("recursive delete must scan the continuation page and find the locked object");
17131 assert!(
17132 matches!(err, StorageError::PrefixAccessDenied(ref name, ref object) if name == &bucket && object == locked_object),
17133 "unexpected recursive delete error: {err:?}"
17134 );
17135 for object in ["prefix/a.bin", "prefix/b.bin", locked_object] {
17136 store
17137 .get_object_info(&bucket, object, &ObjectOptions::default())
17138 .await
17139 .expect("a rejected recursive delete must leave every page intact");
17140 }
17141 }
17142
17143 #[tokio::test]
17144 #[serial_test::serial(storage_class_env)]
17145 async fn delete_holds_bucket_incarnation_sentinel_through_object_commit() {
17146 let temp = tempfile::tempdir().expect("create temp store dir");
17147 let (_ctx, store, _shutdown) =
17148 without_storage_class_env(build_isolated_test_store(temp.path(), "delete-bucket-incarnation", &[4])).await;
17149 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
17150
17151 let bucket = format!("delete-bucket-incarnation-{}", uuid::Uuid::new_v4());
17152 let object = "same-key.bin";
17153 store
17154 .make_bucket(&bucket, &MakeBucketOptions::default())
17155 .await
17156 .expect("create initial plain bucket");
17157 let old_incarnation = store
17158 .bucket_incarnation_id(&bucket)
17159 .await
17160 .expect("read initial bucket incarnation");
17161 let mut reader = PutObjReader::from_vec(b"old generation".to_vec());
17162 store
17163 .put_object(&bucket, object, &mut reader, &ObjectOptions::default())
17164 .await
17165 .expect("put old-generation object");
17166
17167 let barrier = crate::store::object::DeleteAfterObjectLockSnapshotBarrier::install(&bucket);
17168 let delete_store = Arc::clone(&store);
17169 let delete_bucket = bucket.clone();
17170 let delete = tokio::spawn(async move {
17171 delete_store
17172 .delete_object(&delete_bucket, object, ObjectOptions::default())
17173 .await
17174 });
17175 barrier.wait_until_paused().await;
17176
17177 let recreate_store = Arc::clone(&store);
17178 let recreate_bucket = bucket.clone();
17179 let mut recreate = tokio::spawn(async move {
17180 recreate_store
17181 .delete_bucket(
17182 &recreate_bucket,
17183 &crate::storage_api_contracts::bucket::DeleteBucketOptions {
17184 force: true,
17185 ..Default::default()
17186 },
17187 )
17188 .await?;
17189 recreate_store
17190 .make_bucket(
17191 &recreate_bucket,
17192 &MakeBucketOptions {
17193 lock_enabled: true,
17194 ..Default::default()
17195 },
17196 )
17197 .await?;
17198 let mut reader = PutObjReader::from_vec(b"new protected generation".to_vec());
17199 recreate_store
17200 .put_object(
17201 &recreate_bucket,
17202 object,
17203 &mut reader,
17204 &ObjectOptions {
17205 versioned: true,
17206 ..Default::default()
17207 },
17208 )
17209 .await?;
17210 Ok::<_, Error>(())
17211 });
17212
17213 assert!(
17214 tokio::time::timeout(Duration::from_millis(100), &mut recreate).await.is_err(),
17215 "bucket delete and same-name recreation must wait for the in-flight object delete"
17216 );
17217 barrier.release();
17218 delete
17219 .await
17220 .expect("delete task should join")
17221 .expect("old-generation delete should complete");
17222 tokio::time::timeout(Duration::from_secs(10), recreate)
17223 .await
17224 .expect("bucket recreation should finish after delete releases the sentinel")
17225 .expect("bucket recreation task should join")
17226 .expect("bucket recreation should succeed");
17227
17228 let stale_delete_err = store
17229 .delete_object(
17230 &bucket,
17231 object,
17232 ObjectOptions {
17233 expected_bucket_incarnation_id: Some(old_incarnation),
17234 versioned: true,
17235 ..Default::default()
17236 },
17237 )
17238 .await
17239 .expect_err("an authorization snapshot from the old bucket incarnation must be rejected");
17240 assert!(matches!(stale_delete_err, StorageError::BucketNotFound(name) if name == bucket));
17241
17242 let (_deleted, stale_batch_errors) = store
17243 .delete_objects(
17244 &bucket,
17245 vec![crate::storage_api_contracts::object::ObjectToDelete {
17246 object_name: object.to_string(),
17247 ..Default::default()
17248 }],
17249 ObjectOptions {
17250 expected_bucket_incarnation_id: Some(old_incarnation),
17251 versioned: true,
17252 ..Default::default()
17253 },
17254 )
17255 .await;
17256 assert!(
17257 matches!(stale_batch_errors.as_slice(), [Some(StorageError::BucketNotFound(name))] if name.as_str() == bucket),
17258 "batch delete must reject an authorization snapshot from the old bucket incarnation"
17259 );
17260
17261 store
17262 .get_object_info(
17263 &bucket,
17264 object,
17265 &ObjectOptions {
17266 versioned: true,
17267 ..Default::default()
17268 },
17269 )
17270 .await
17271 .expect("the new locked-bucket object must survive the old-generation delete");
17272 }
17273
17274 #[tokio::test]
17275 #[serial_test::serial(storage_class_env)]
17276 async fn delete_objects_holds_bucket_incarnation_sentinel_through_commit() {
17277 let temp = tempfile::tempdir().expect("create temp store dir");
17278 let (_ctx, store, _shutdown) =
17279 without_storage_class_env(build_isolated_test_store(temp.path(), "batch-delete-bucket-incarnation", &[4])).await;
17280 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
17281
17282 let bucket = format!("batch-incarnation-{}", uuid::Uuid::new_v4());
17283 let object = "same-key.bin";
17284 store
17285 .make_bucket(&bucket, &MakeBucketOptions::default())
17286 .await
17287 .expect("create initial bucket");
17288 let mut reader = PutObjReader::from_vec(b"old generation".to_vec());
17289 store
17290 .put_object(&bucket, object, &mut reader, &ObjectOptions::default())
17291 .await
17292 .expect("put old-generation object");
17293
17294 let barrier = crate::store::object::DeleteAfterObjectLockSnapshotBarrier::install(&bucket);
17295 let delete_store = Arc::clone(&store);
17296 let delete_bucket = bucket.clone();
17297 let delete = tokio::spawn(async move {
17298 delete_store
17299 .delete_objects(
17300 &delete_bucket,
17301 vec![crate::storage_api_contracts::object::ObjectToDelete {
17302 object_name: object.to_string(),
17303 ..Default::default()
17304 }],
17305 ObjectOptions::default(),
17306 )
17307 .await
17308 });
17309 barrier.wait_until_paused().await;
17310
17311 let recreate_store = Arc::clone(&store);
17312 let recreate_bucket = bucket.clone();
17313 let mut recreate = tokio::spawn(async move {
17314 recreate_store
17315 .delete_bucket(
17316 &recreate_bucket,
17317 &crate::storage_api_contracts::bucket::DeleteBucketOptions {
17318 force: true,
17319 ..Default::default()
17320 },
17321 )
17322 .await?;
17323 recreate_store
17324 .make_bucket(&recreate_bucket, &MakeBucketOptions::default())
17325 .await?;
17326 let mut reader = PutObjReader::from_vec(b"new generation".to_vec());
17327 recreate_store
17328 .put_object(&recreate_bucket, object, &mut reader, &ObjectOptions::default())
17329 .await?;
17330 Ok::<_, Error>(())
17331 });
17332
17333 assert!(
17334 tokio::time::timeout(Duration::from_millis(100), &mut recreate).await.is_err(),
17335 "bucket recreation must wait for the in-flight batch delete"
17336 );
17337 barrier.release();
17338 let (_deleted, errors) = delete.await.expect("batch delete task should join");
17339 assert!(
17340 errors.iter().all(Option::is_none),
17341 "old-generation batch delete should complete: {errors:?}"
17342 );
17343 tokio::time::timeout(Duration::from_secs(10), recreate)
17344 .await
17345 .expect("bucket recreation should finish after batch delete releases the sentinel")
17346 .expect("bucket recreation task should join")
17347 .expect("bucket recreation should succeed");
17348 store
17349 .get_object_info(&bucket, object, &ObjectOptions::default())
17350 .await
17351 .expect("new-generation object must survive the old batch delete");
17352 }
17353
17354 #[tokio::test]
17355 #[serial_test::serial(storage_class_env)]
17356 async fn prefix_delete_serializes_with_concurrent_object_lock_metadata_updates() {
17357 let temp = tempfile::tempdir().expect("create temp store dir");
17358 let (_ctx, store, _shutdown) =
17359 without_storage_class_env(build_isolated_test_store(temp.path(), "prefix-delete-lock-metadata", &[4])).await;
17360 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
17361
17362 let cases = [
17363 (
17364 "compliance",
17365 HashMap::from([
17366 ("x-amz-object-lock-mode".to_string(), "COMPLIANCE".to_string()),
17367 ("x-amz-object-lock-retain-until-date".to_string(), "2099-01-01T00:00:00Z".to_string()),
17368 ]),
17369 ),
17370 (
17371 "legal-hold",
17372 HashMap::from([("x-amz-object-lock-legal-hold".to_string(), "ON".to_string())]),
17373 ),
17374 ];
17375
17376 for (case, eval_metadata) in cases {
17377 let bucket = format!("prefix-lock-{case}-{}", uuid::Uuid::new_v4());
17378 let object = "protected-after-scan.bin";
17379 store
17380 .make_bucket(
17381 &bucket,
17382 &MakeBucketOptions {
17383 lock_enabled: true,
17384 ..Default::default()
17385 },
17386 )
17387 .await
17388 .expect("create Object Lock bucket");
17389 let mut reader = PutObjReader::from_vec(b"body".to_vec());
17390 let object_info = store
17391 .put_object(
17392 &bucket,
17393 object,
17394 &mut reader,
17395 &ObjectOptions {
17396 versioned: true,
17397 ..Default::default()
17398 },
17399 )
17400 .await
17401 .expect("write unprotected version");
17402 let version_id = object_info.version_id.expect("versioned PUT should return an ID").to_string();
17403
17404 let barrier = crate::store::object::DeleteAfterObjectLockSnapshotBarrier::install(&bucket);
17405 let delete_store = Arc::clone(&store);
17406 let delete_bucket = bucket.clone();
17407 let delete = tokio::spawn(async move {
17408 delete_store
17409 .delete_object(
17410 &delete_bucket,
17411 object,
17412 ObjectOptions {
17413 delete_prefix: true,
17414 delete_prefix_object: true,
17415 versioned: true,
17416 ..Default::default()
17417 },
17418 )
17419 .await
17420 });
17421 barrier.wait_until_paused().await;
17422
17423 let update_store = Arc::clone(&store);
17424 let update_bucket = bucket.clone();
17425 let mut update = tokio::spawn(async move {
17426 update_store
17427 .put_object_metadata(
17428 &update_bucket,
17429 object,
17430 &ObjectOptions {
17431 version_id: Some(version_id),
17432 versioned: true,
17433 eval_metadata: Some(eval_metadata),
17434 ..Default::default()
17435 },
17436 )
17437 .await
17438 });
17439 assert!(
17440 tokio::time::timeout(Duration::from_millis(100), &mut update).await.is_err(),
17441 "{case} metadata update must wait behind the prefix-delete lifecycle write fence"
17442 );
17443
17444 barrier.release();
17445 delete
17446 .await
17447 .expect("prefix delete task should join")
17448 .expect("prefix delete should remove the unprotected version");
17449 update
17450 .await
17451 .expect("metadata update task should join")
17452 .expect_err("metadata update must not succeed after the version was deleted");
17453 assert!(
17454 store
17455 .get_object_info(
17456 &bucket,
17457 object,
17458 &ObjectOptions {
17459 versioned: true,
17460 ..Default::default()
17461 },
17462 )
17463 .await
17464 .is_err(),
17465 "{case} case must not report a successful protected write followed by deletion"
17466 );
17467 }
17468 }
17469
17470 #[tokio::test]
17471 #[serial_test::serial(storage_class_env)]
17472 async fn put_rejects_same_name_bucket_recreated_after_request_snapshot() {
17473 let temp = tempfile::tempdir().expect("create temp store dir");
17474 let (_ctx, store, _shutdown) =
17475 without_storage_class_env(build_isolated_test_store(temp.path(), "put-bucket-incarnation", &[4])).await;
17476 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
17477
17478 let bucket = format!("put-bucket-incarnation-{}", uuid::Uuid::new_v4());
17479 let object = "object.bin";
17480 store
17481 .make_bucket(&bucket, &MakeBucketOptions::default())
17482 .await
17483 .expect("create initial bucket");
17484 let old_incarnation = store
17485 .bucket_incarnation_id(&bucket)
17486 .await
17487 .expect("read initial bucket incarnation");
17488
17489 let barrier = crate::set_disk::PutObjectCommitBarrier::install(
17490 &bucket,
17491 object,
17492 crate::set_disk::PutObjectCommitPause::BeforeNamespace,
17493 );
17494 let put_store = Arc::clone(&store);
17495 let put_bucket = bucket.clone();
17496 let put = tokio::spawn(async move {
17497 let mut reader = PutObjReader::from_vec(b"old generation".to_vec());
17498 put_store
17499 .put_object(
17500 &put_bucket,
17501 object,
17502 &mut reader,
17503 &ObjectOptions {
17504 versioned: true,
17505 ..Default::default()
17506 },
17507 )
17508 .await
17509 });
17510 barrier.wait_until_paused().await;
17511
17512 store
17513 .delete_bucket(&bucket, &crate::storage_api_contracts::bucket::DeleteBucketOptions::default())
17514 .await
17515 .expect("delete initial empty bucket while PUT is staged");
17516 store
17517 .make_bucket(&bucket, &MakeBucketOptions::default())
17518 .await
17519 .expect("recreate same-name bucket");
17520 let new_incarnation = store
17521 .bucket_incarnation_id(&bucket)
17522 .await
17523 .expect("read recreated bucket incarnation");
17524 assert_ne!(old_incarnation, new_incarnation);
17525
17526 barrier.release();
17527 let err = put
17528 .await
17529 .expect("PUT task should join")
17530 .expect_err("old-generation PUT must not commit into the recreated bucket");
17531 assert!(
17532 matches!(err, StorageError::BucketNotFound(ref name) if name == &bucket),
17533 "unexpected stale PUT error: {err:?}"
17534 );
17535 assert!(
17536 store
17537 .get_object_info(&bucket, object, &ObjectOptions::default())
17538 .await
17539 .is_err(),
17540 "the recreated bucket must not contain the staged old-generation object"
17541 );
17542 }
17543
17544 #[tokio::test]
17545 #[serial_test::serial(storage_class_env)]
17546 async fn exact_version_put_uses_only_its_own_bucket_object_lock_snapshot() {
17547 let target_temp = tempfile::tempdir().expect("create target store dir");
17548 let foreign_temp = tempfile::tempdir().expect("create foreign store dir");
17549 let (target_ctx, target_store, _target_shutdown) =
17550 without_storage_class_env(build_isolated_test_store(target_temp.path(), "put-object-lock-target", &[4])).await;
17551 let (_foreign_ctx, foreign_store, _foreign_shutdown) =
17552 without_storage_class_env(build_isolated_test_store(foreign_temp.path(), "put-object-lock-foreign", &[4])).await;
17553 crate::bucket::metadata_sys::init_bucket_metadata_sys(target_store.clone(), Vec::new()).await;
17554 crate::bucket::metadata_sys::init_bucket_metadata_sys(foreign_store.clone(), Vec::new()).await;
17555
17556 let bucket = format!("put-object-lock-snapshot-{}", uuid::Uuid::new_v4());
17557 let object = "object.bin";
17558 target_store
17559 .make_bucket(
17560 &bucket,
17561 &MakeBucketOptions {
17562 lock_enabled: true,
17563 ..Default::default()
17564 },
17565 )
17566 .await
17567 .expect("create target Object Lock bucket");
17568 foreign_store
17569 .make_bucket(&bucket, &MakeBucketOptions::default())
17570 .await
17571 .expect("create same-name plain bucket in foreign store");
17572 let stale_snapshot = target_store
17573 .object_lock_config_snapshot(&bucket)
17574 .await
17575 .expect("load target snapshot before changing its default retention");
17576
17577 let mut metadata = crate::bucket::metadata_sys::get_in(&target_ctx, &bucket)
17578 .await
17579 .expect("load target bucket metadata")
17580 .as_ref()
17581 .clone();
17582 metadata
17583 .update_config(
17584 crate::bucket::metadata::OBJECT_LOCK_CONFIG,
17585 b"<ObjectLockConfiguration><ObjectLockEnabled>Enabled</ObjectLockEnabled><Rule><DefaultRetention><Mode>COMPLIANCE</Mode><Days>1</Days></DefaultRetention></Rule></ObjectLockConfiguration>".to_vec(),
17586 )
17587 .expect("set default COMPLIANCE metadata");
17588 let update_started = Arc::new(tokio::sync::Barrier::new(2));
17589 let update_ctx = Arc::clone(&target_ctx);
17590 let update_started_task = Arc::clone(&update_started);
17591 let mut update = tokio::spawn(async move {
17592 update_started_task.wait().await;
17593 let _transaction_guard =
17594 crate::bucket::metadata_sys::acquire_bucket_metadata_transaction_lock_in(&update_ctx, &metadata.name).await?;
17595 crate::bucket::metadata_sys::set_bucket_metadata_in(&update_ctx, metadata).await
17596 });
17597 update_started.wait().await;
17598 assert!(
17599 tokio::time::timeout(Duration::from_millis(100), &mut update).await.is_err(),
17600 "an active Object Lock snapshot must hold the metadata revision read fence"
17601 );
17602 drop(stale_snapshot);
17603 update
17604 .await
17605 .expect("Object Lock metadata update task should join")
17606 .expect("persist target Object Lock metadata after snapshot release");
17607
17608 let original_body = b"protected body".to_vec();
17609 let mut original_reader = PutObjReader::from_vec(original_body.clone());
17610 let original = target_store
17611 .put_object(
17612 &bucket,
17613 object,
17614 &mut original_reader,
17615 &ObjectOptions {
17616 versioned: true,
17617 ..Default::default()
17618 },
17619 )
17620 .await
17621 .expect("write protected target version");
17622 let version_id = original
17623 .version_id
17624 .expect("versioned PUT should return a version ID")
17625 .to_string();
17626
17627 let mut replacement = PutObjReader::from_vec(b"replacement".to_vec());
17628 let err = target_store
17629 .put_object(
17630 &bucket,
17631 object,
17632 &mut replacement,
17633 &ObjectOptions {
17634 version_id: Some(version_id.clone()),
17635 versioned: true,
17636 ..Default::default()
17637 },
17638 )
17639 .await
17640 .expect_err("ECStore must load default COMPLIANCE before an exact-version overwrite");
17641 assert!(matches!(err, StorageError::PrefixAccessDenied(_, _)));
17642
17643 let foreign_snapshot = foreign_store
17644 .object_lock_config_snapshot(&bucket)
17645 .await
17646 .expect("load same-name foreign bucket snapshot");
17647 let mut foreign_replacement = PutObjReader::from_vec(b"foreign replacement".to_vec());
17648 target_store
17649 .put_object(
17650 &bucket,
17651 object,
17652 &mut foreign_replacement,
17653 &ObjectOptions {
17654 version_id: Some(version_id.clone()),
17655 versioned: true,
17656 object_lock_config_snapshot: Some(foreign_snapshot),
17657 ..Default::default()
17658 },
17659 )
17660 .await
17661 .expect_err("a snapshot from another store must fail closed");
17662
17663 let mut reader = target_store
17664 .get_object_reader(
17665 &bucket,
17666 object,
17667 None,
17668 HeaderMap::new(),
17669 &ObjectOptions {
17670 version_id: Some(version_id),
17671 versioned: true,
17672 ..Default::default()
17673 },
17674 )
17675 .await
17676 .expect("protected target version should remain readable");
17677 let mut body = Vec::new();
17678 reader
17679 .stream
17680 .read_to_end(&mut body)
17681 .await
17682 .expect("protected body should drain");
17683 assert_eq!(body, original_body);
17684 }
17685
17686 #[cfg(feature = "test-util")]
17687 #[tokio::test]
17688 #[serial_test::serial(storage_class_env)]
17689 async fn object_lock_snapshot_uses_read_quorum_bucket_existence_probe() {
17690 let temp = tempfile::tempdir().expect("create degraded snapshot store dir");
17691 let (ctx, store, _shutdown) = without_storage_class_env(build_isolated_test_store_with_layout(
17692 temp.path(),
17693 "degraded-object-lock-snapshot",
17694 &[(2, 12)],
17695 CancellationToken::new(),
17696 None,
17697 ))
17698 .await;
17699 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
17700
17701 let bucket = format!("degraded-ol-{}", uuid::Uuid::new_v4());
17702 store
17703 .make_bucket(&bucket, &MakeBucketOptions::default())
17704 .await
17705 .expect("create snapshot bucket");
17706 let expected_incarnation = store
17707 .bucket_incarnation_id(&bucket)
17708 .await
17709 .expect("read bucket incarnation before degrading sets");
17710
17711 let mut offline_disks = Vec::new();
17712 for set in store.all_set_disks() {
17713 offline_disks.push(force_set_disk_range_offline_for_test(&set, 6..12).await);
17714 }
17715
17716 let snapshot = store
17717 .object_lock_config_snapshot(&bucket)
17718 .await
17719 .expect("read-quorum bucket existence should admit guarded Object Lock snapshot");
17720 assert!(matches!(
17721 snapshot.state(),
17722 crate::bucket::metadata_sys::ObjectLockConfigState::ConfirmedAbsent
17723 ));
17724 assert!(snapshot.is_valid_for_destructive_put(store.id, &bucket, expected_incarnation));
17725
17726 let current_incarnation = crate::bucket::metadata_sys::get_object_lock_config_and_incarnation_from_disk_in(&ctx, &bucket)
17727 .await
17728 .expect("authoritative metadata read should also survive at read quorum")
17729 .1;
17730 assert_eq!(current_incarnation, expected_incarnation);
17731
17732 drop(offline_disks);
17733 }
17734
17735 #[tokio::test]
17736 #[serial_test::serial(storage_class_env)]
17737 async fn force_create_existing_bucket_preserves_incarnation_and_inflight_request() {
17738 let temp = tempfile::tempdir().expect("create temp store dir");
17739 let (_ctx, store, _shutdown) =
17740 without_storage_class_env(build_isolated_test_store(temp.path(), "force-create-incarnation", &[4])).await;
17741 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
17742
17743 let bucket = format!("force-create-incarnation-{}", uuid::Uuid::new_v4());
17744 store
17745 .make_bucket(&bucket, &MakeBucketOptions::default())
17746 .await
17747 .expect("create initial bucket");
17748 let incarnation = store.bucket_incarnation_id(&bucket).await.expect("read initial incarnation");
17749
17750 store
17751 .make_bucket(
17752 &bucket,
17753 &MakeBucketOptions {
17754 force_create: true,
17755 ..Default::default()
17756 },
17757 )
17758 .await
17759 .expect("idempotent force-create should succeed");
17760 assert_eq!(
17761 store.bucket_incarnation_id(&bucket).await.unwrap(),
17762 incarnation,
17763 "idempotent force-create must not rotate the bucket generation"
17764 );
17765
17766 let mut reader = PutObjReader::from_vec(b"authorized before force-create".to_vec());
17767 store
17768 .put_object(
17769 &bucket,
17770 "inflight.bin",
17771 &mut reader,
17772 &ObjectOptions {
17773 expected_bucket_incarnation_id: Some(incarnation),
17774 ..Default::default()
17775 },
17776 )
17777 .await
17778 .expect("a request authorized before idempotent force-create must still commit");
17779 }
17780
17781 #[tokio::test]
17782 #[serial_test::serial(storage_class_env)]
17783 async fn lock_enabled_create_retries_metadata_intent_without_unlocking() {
17784 let temp = tempfile::tempdir().expect("create temp store dir");
17785 let (ctx, store, _shutdown) =
17786 without_storage_class_env(build_isolated_test_store(temp.path(), "lock-create-intent-retry", &[4])).await;
17787 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
17788
17789 let bucket = format!("lock-create-intent-retry-{}", uuid::Uuid::new_v4());
17790 let mut intent = crate::bucket::metadata::BucketMetadata::new(&bucket);
17791 intent.lock_enabled = true;
17792 crate::bucket::metadata_sys::set_new_bucket_metadata_in(&ctx, intent)
17793 .await
17794 .expect("persist pre-visibility lock intent");
17795 assert!(
17796 store
17797 .peer_sys
17798 .get_bucket_info(&bucket, &crate::storage_api_contracts::bucket::BucketOptions::default())
17799 .await
17800 .is_err(),
17801 "the intent must not make the physical bucket visible"
17802 );
17803
17804 store
17805 .make_bucket(
17806 &bucket,
17807 &MakeBucketOptions {
17808 lock_enabled: true,
17809 ..Default::default()
17810 },
17811 )
17812 .await
17813 .expect("retry lock-enabled creation");
17814 assert!(matches!(
17815 crate::bucket::metadata_sys::get_object_lock_config_state_in(&ctx, &bucket)
17816 .await
17817 .expect("read retried lock state"),
17818 crate::bucket::metadata_sys::ObjectLockConfigState::Configured { .. }
17819 ));
17820 }
17821
17822 #[tokio::test]
17823 #[serial_test::serial(storage_class_env)]
17824 async fn physical_lock_bucket_with_incomplete_metadata_fails_closed() {
17825 let temp = tempfile::tempdir().expect("create temp store dir");
17826 let (ctx, store, _shutdown) =
17827 without_storage_class_env(build_isolated_test_store(temp.path(), "lock-create-incomplete-metadata", &[4])).await;
17828 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
17829
17830 let bucket = format!("lock-create-incomplete-metadata-{}", uuid::Uuid::new_v4());
17831 let mut metadata = crate::bucket::metadata::BucketMetadata::new(&bucket);
17832 metadata.lock_enabled = true;
17833 metadata
17834 .save_with_store(store.clone())
17835 .await
17836 .expect("persist lock metadata before sidecar");
17837 store
17838 .peer_sys
17839 .make_bucket(&bucket, &MakeBucketOptions::default())
17840 .await
17841 .expect("expose physical bucket after metadata phase");
17842
17843 let err = crate::bucket::metadata_sys::get_object_lock_config_state_in(&ctx, &bucket)
17844 .await
17845 .expect_err("missing sidecar must not fabricate an unlocked bucket");
17846 assert!(err.to_string().contains("sidecar is missing"), "unexpected error: {err}");
17847 }
17848
17849 #[tokio::test]
17850 #[serial_test::serial(storage_class_env)]
17851 async fn successful_lock_enabled_create_has_complete_metadata_intent() {
17852 let temp = tempfile::tempdir().expect("create temp store dir");
17853 let (ctx, store, _shutdown) =
17854 without_storage_class_env(build_isolated_test_store(temp.path(), "lock-create-complete-metadata", &[4])).await;
17855 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
17856
17857 let bucket = format!("lock-create-{}", uuid::Uuid::new_v4());
17858 store
17859 .make_bucket(
17860 &bucket,
17861 &MakeBucketOptions {
17862 lock_enabled: true,
17863 ..Default::default()
17864 },
17865 )
17866 .await
17867 .expect("create lock-enabled bucket");
17868 let (metadata, persisted) = crate::bucket::metadata_sys::get_config_from_disk_with_presence_in(&ctx, &bucket)
17869 .await
17870 .expect("read completed metadata intent");
17871 assert!(persisted && metadata.bucket_incarnation_sidecar && metadata.lock_enabled);
17872 assert!(matches!(
17873 crate::bucket::metadata_sys::get_object_lock_config_state_in(&ctx, &bucket)
17874 .await
17875 .expect("read completed lock state"),
17876 crate::bucket::metadata_sys::ObjectLockConfigState::Configured { .. }
17877 ));
17878 }
17879
17880 #[tokio::test]
17881 #[serial_test::serial(storage_class_env)]
17882 async fn lifecycle_expiry_rejects_task_from_recreated_bucket_incarnation() {
17883 let temp = tempfile::tempdir().expect("create temp store dir");
17884 let (_ctx, store, _shutdown) =
17885 without_storage_class_env(build_isolated_test_store(temp.path(), "lifecycle-expiry-incarnation", &[4])).await;
17886 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
17887
17888 let bucket = format!("lifecycle-incarnation-{}", uuid::Uuid::new_v4());
17889 let object = "same-key.bin";
17890 store
17891 .make_bucket(&bucket, &MakeBucketOptions::default())
17892 .await
17893 .expect("create initial bucket");
17894 let old_incarnation = store.bucket_incarnation_id(&bucket).await.expect("read old incarnation");
17895 let mut old_reader = PutObjReader::from_vec(b"old-generation".to_vec());
17896 store
17897 .put_object(&bucket, object, &mut old_reader, &ObjectOptions::default())
17898 .await
17899 .expect("put old-generation object");
17900 let old_object = store
17901 .get_object_info(&bucket, object, &ObjectOptions::default())
17902 .await
17903 .expect("snapshot old-generation object");
17904
17905 store
17906 .delete_object(&bucket, object, ObjectOptions::default())
17907 .await
17908 .expect("delete old-generation object");
17909 store
17910 .delete_bucket(&bucket, &crate::storage_api_contracts::bucket::DeleteBucketOptions::default())
17911 .await
17912 .expect("delete old bucket");
17913 store
17914 .make_bucket(&bucket, &MakeBucketOptions::default())
17915 .await
17916 .expect("recreate bucket");
17917 let mut new_reader = PutObjReader::from_vec(b"new-generation".to_vec());
17918 store
17919 .put_object(&bucket, object, &mut new_reader, &ObjectOptions::default())
17920 .await
17921 .expect("put replacement object");
17922
17923 let applied = crate::bucket::lifecycle::bucket_lifecycle_ops::apply_expiry_on_non_transitioned_objects(
17924 store.clone(),
17925 &old_object,
17926 &crate::bucket::lifecycle::lifecycle::Event {
17927 action: crate::bucket::lifecycle::lifecycle::IlmAction::DeleteAction,
17928 ..Default::default()
17929 },
17930 &crate::bucket::lifecycle::bucket_lifecycle_audit::LcEventSrc::Scanner,
17931 old_incarnation,
17932 )
17933 .await;
17934 assert!(!applied, "an expiry task from the old bucket incarnation must fail closed");
17935
17936 let mut reader = store
17937 .get_object_reader(&bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
17938 .await
17939 .expect("new-generation object must survive stale expiry");
17940 let mut body = Vec::new();
17941 reader.stream.read_to_end(&mut body).await.expect("read replacement object");
17942 assert_eq!(body, b"new-generation");
17943 }
17944
17945 #[tokio::test]
17946 #[serial_test::serial(storage_class_env)]
17947 async fn pre_upgrade_multipart_upload_can_complete_and_abort_in_the_same_bucket_lifetime() {
17948 let temp = tempfile::tempdir().expect("create temp store dir");
17949 let (_ctx, store, _shutdown) =
17950 without_storage_class_env(build_isolated_test_store(temp.path(), "multipart-upgrade-compat", &[4])).await;
17951 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
17952
17953 let bucket = format!("multipart-upgrade-compat-{}", uuid::Uuid::new_v4());
17954 let created = OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("fixed timestamp should be valid");
17955 store
17956 .make_bucket(
17957 &bucket,
17958 &MakeBucketOptions {
17959 created_at: Some(created),
17960 ..Default::default()
17961 },
17962 )
17963 .await
17964 .expect("create bucket");
17965
17966 let complete_object = "complete.bin";
17967 let complete_upload = store.pools[0]
17968 .new_multipart_upload(
17969 &bucket,
17970 complete_object,
17971 &ObjectOptions {
17972 mod_time: Some(created + time::Duration::seconds(1)),
17973 ..Default::default()
17974 },
17975 )
17976 .await
17977 .expect("create a pre-upgrade multipart upload without an incarnation stamp");
17978 let mut part_reader = PutObjReader::from_vec(b"pre-upgrade multipart body".to_vec());
17979 let part = store.pools[0]
17980 .put_object_part(
17981 &bucket,
17982 complete_object,
17983 &complete_upload.upload_id,
17984 1,
17985 &mut part_reader,
17986 &ObjectOptions::default(),
17987 )
17988 .await
17989 .expect("stage a pre-upgrade multipart part");
17990 store
17991 .clone()
17992 .complete_multipart_upload(
17993 &bucket,
17994 complete_object,
17995 &complete_upload.upload_id,
17996 vec![crate::storage_api_contracts::multipart::CompletePart {
17997 part_num: part.part_num,
17998 etag: part.etag,
17999 ..Default::default()
18000 }],
18001 &ObjectOptions::default(),
18002 )
18003 .await
18004 .expect("a pre-upgrade upload from the current bucket lifetime should complete");
18005 store
18006 .get_object_info(&bucket, complete_object, &ObjectOptions::default())
18007 .await
18008 .expect("completed pre-upgrade upload should be visible");
18009
18010 let abort_object = "abort.bin";
18011 let abort_upload = store.pools[0]
18012 .new_multipart_upload(
18013 &bucket,
18014 abort_object,
18015 &ObjectOptions {
18016 mod_time: Some(created + time::Duration::seconds(2)),
18017 ..Default::default()
18018 },
18019 )
18020 .await
18021 .expect("create another pre-upgrade multipart upload");
18022 store
18023 .abort_multipart_upload(&bucket, abort_object, &abort_upload.upload_id, &ObjectOptions::default())
18024 .await
18025 .expect("a pre-upgrade upload from the current bucket lifetime should abort");
18026 let err = store
18027 .get_multipart_info(&bucket, abort_object, &abort_upload.upload_id, &ObjectOptions::default())
18028 .await
18029 .expect_err("aborted pre-upgrade upload should be removed");
18030 assert!(matches!(err, StorageError::InvalidUploadID(..)));
18031 }
18032
18033 #[tokio::test]
18034 #[serial_test::serial(storage_class_env)]
18035 async fn complete_multipart_rejects_legacy_upload_from_recreated_bucket_incarnation() {
18036 let temp = tempfile::tempdir().expect("create temp store dir");
18037 let (_ctx, store, _shutdown) =
18038 without_storage_class_env(build_isolated_test_store(temp.path(), "multipart-bucket-incarnation", &[4])).await;
18039 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
18040
18041 let bucket = format!("multipart-incarnation-{}", uuid::Uuid::new_v4());
18042 let object = "object.bin";
18043 let created = OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("fixed timestamp should be valid");
18044 store
18045 .make_bucket(
18046 &bucket,
18047 &MakeBucketOptions {
18048 created_at: Some(created),
18049 ..Default::default()
18050 },
18051 )
18052 .await
18053 .expect("create initial bucket");
18054 let old_incarnation = store
18055 .bucket_incarnation_id(&bucket)
18056 .await
18057 .expect("read initial bucket incarnation");
18058 let upload = store.pools[0]
18059 .new_multipart_upload(
18060 &bucket,
18061 object,
18062 &ObjectOptions {
18063 mod_time: Some(created + time::Duration::seconds(1)),
18064 ..Default::default()
18065 },
18066 )
18067 .await
18068 .expect("create old-generation legacy multipart upload");
18069
18070 store
18071 .delete_bucket(&bucket, &crate::storage_api_contracts::bucket::DeleteBucketOptions::default())
18072 .await
18073 .expect("delete bucket with an incomplete multipart upload");
18074 store
18075 .make_bucket(
18076 &bucket,
18077 &MakeBucketOptions {
18078 created_at: Some(created + time::Duration::seconds(2)),
18079 ..Default::default()
18080 },
18081 )
18082 .await
18083 .expect("recreate same-name bucket");
18084 let new_incarnation = store
18085 .bucket_incarnation_id(&bucket)
18086 .await
18087 .expect("read recreated bucket incarnation");
18088 assert_ne!(old_incarnation, new_incarnation);
18089
18090 let err = store
18091 .clone()
18092 .complete_multipart_upload(&bucket, object, &upload.upload_id, Vec::new(), &ObjectOptions::default())
18093 .await
18094 .expect_err("an upload initiated in the old bucket incarnation must not commit into the replacement");
18095 assert!(
18096 match &err {
18097 StorageError::BucketNotFound(name) => name == &bucket,
18098 StorageError::InvalidUploadID(name, key, _) => name == &bucket && key == object,
18099 _ => false,
18100 },
18101 "unexpected stale multipart completion error: {err:?}"
18102 );
18103 assert!(
18104 store
18105 .get_object_info(&bucket, object, &ObjectOptions::default())
18106 .await
18107 .is_err(),
18108 "the recreated bucket must not contain the old-generation multipart upload"
18109 );
18110 }
18111
18112 #[tokio::test]
18113 #[serial_test::serial(storage_class_env)]
18114 async fn delete_after_bucket_validation_preserves_bucket_not_found() {
18115 let temp = tempfile::tempdir().expect("create temp store dir");
18116 let (_ctx, store, _shutdown) =
18117 without_storage_class_env(build_isolated_test_store(temp.path(), "delete-missing-bucket", &[4])).await;
18118 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
18119
18120 let bucket = format!("delete-missing-bucket-{}", uuid::Uuid::new_v4());
18121 store
18122 .make_bucket(&bucket, &MakeBucketOptions::default())
18123 .await
18124 .expect("create bucket before validation");
18125 store
18126 .get_bucket_info(&bucket, &crate::storage_api_contracts::bucket::BucketOptions::default())
18127 .await
18128 .expect("request-level bucket validation should succeed");
18129 store
18130 .delete_bucket(&bucket, &crate::storage_api_contracts::bucket::DeleteBucketOptions::default())
18131 .await
18132 .expect("delete bucket after validation");
18133
18134 let err = store
18135 .delete_object(&bucket, "missing.bin", ObjectOptions::default())
18136 .await
18137 .expect_err("post-validation bucket deletion must not become an object-not-found error");
18138
18139 assert!(matches!(err, StorageError::BucketNotFound(name) if name == bucket));
18140 }
18141
18142 #[tokio::test]
18143 #[serial_test::serial(storage_class_env)]
18144 async fn object_lock_metadata_read_error_blocks_every_destructive_delete_shape() {
18145 let temp = tempfile::tempdir().expect("create temp store dir");
18146 let (ctx, store, _shutdown) =
18147 without_storage_class_env(build_isolated_test_store(temp.path(), "delete-config-read-error", &[4])).await;
18148 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
18149
18150 let bucket = format!("delete-config-read-error-{}", uuid::Uuid::new_v4());
18151 store
18152 .make_bucket(&bucket, &MakeBucketOptions::default())
18153 .await
18154 .expect("create bucket");
18155 for object in ["single.bin", "batch.bin", "prefix/child.bin", "lifecycle.bin"] {
18156 let mut reader = PutObjReader::from_vec(b"must survive".to_vec());
18157 store
18158 .put_object(&bucket, object, &mut reader, &ObjectOptions::default())
18159 .await
18160 .expect("put object before metadata corruption");
18161 }
18162
18163 crate::bucket::metadata_sys::inject_object_lock_disk_read_error_in(&ctx, &bucket)
18164 .await
18165 .expect("inject single-delete metadata read failure");
18166
18167 let single_err = store
18168 .delete_object(&bucket, "single.bin", ObjectOptions::default())
18169 .await
18170 .expect_err("single delete must propagate the metadata read failure");
18171 assert!(
18172 single_err
18173 .to_string()
18174 .contains("injected Object Lock metadata disk read failure")
18175 );
18176
18177 crate::bucket::metadata_sys::inject_object_lock_disk_read_error_in(&ctx, &bucket)
18178 .await
18179 .expect("inject batch-delete metadata read failure");
18180 let (_deleted, batch_errs) = store
18181 .delete_objects(
18182 &bucket,
18183 vec![crate::storage_api_contracts::object::ObjectToDelete {
18184 object_name: "batch.bin".to_string(),
18185 ..Default::default()
18186 }],
18187 ObjectOptions::default(),
18188 )
18189 .await;
18190 assert!(
18191 batch_errs[0]
18192 .as_ref()
18193 .is_some_and(|err| err.to_string().contains("injected Object Lock metadata disk read failure"))
18194 );
18195
18196 crate::bucket::metadata_sys::inject_object_lock_disk_read_error_in(&ctx, &bucket)
18197 .await
18198 .expect("inject prefix-delete metadata read failure");
18199 let prefix_err = store
18200 .delete_object(
18201 &bucket,
18202 "prefix",
18203 ObjectOptions {
18204 delete_prefix: true,
18205 ..Default::default()
18206 },
18207 )
18208 .await
18209 .expect_err("prefix delete must propagate the metadata read failure");
18210 assert!(
18211 prefix_err
18212 .to_string()
18213 .contains("injected Object Lock metadata disk read failure")
18214 );
18215
18216 crate::bucket::metadata_sys::inject_object_lock_disk_read_error_in(&ctx, &bucket)
18217 .await
18218 .expect("inject lifecycle-style delete-all metadata read failure");
18219 let lifecycle_err = store
18220 .delete_object(
18221 &bucket,
18222 "lifecycle.bin",
18223 ObjectOptions {
18224 delete_prefix: true,
18225 delete_prefix_object: true,
18226 ..Default::default()
18227 },
18228 )
18229 .await
18230 .expect_err("lifecycle-style delete-all must propagate the metadata read failure");
18231 assert!(
18232 lifecycle_err
18233 .to_string()
18234 .contains("injected Object Lock metadata disk read failure")
18235 );
18236
18237 for object in ["single.bin", "batch.bin", "prefix/child.bin", "lifecycle.bin"] {
18238 store
18239 .get_object_info(&bucket, object, &ObjectOptions::default())
18240 .await
18241 .expect("metadata failure must preserve every object");
18242 }
18243 }
18244
18245 #[cfg(feature = "test-util")]
18246 #[tokio::test]
18247 #[serial_test::serial(storage_class_env)]
18248 async fn legacy_tier_delete_journals_create_redacted_recovery_controls_without_remote_calls() {
18249 let temp_dir = tempfile::tempdir().expect("create legacy journal recovery store dir");
18250 let (ctx, store, _shutdown) =
18251 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "legacy-tier-journal-recovery", &[4])).await;
18252 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
18253
18254 let tier_name = "LEGACY-RECOVERY";
18255 let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
18256 let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
18257 .await
18258 .expect("legacy recovery tier lease should resolve")
18259 .backend_identity();
18260 let fixtures = [
18261 serde_json::json!({
18262 "version": 1,
18263 "obj_name": "legacy/remote-v1",
18264 "version_id": "opaque-v1",
18265 "tier_name": tier_name,
18266 }),
18267 serde_json::json!({
18268 "version": 2,
18269 "obj_name": "legacy/remote-v2",
18270 "version_id": "opaque-v2",
18271 "tier_name": tier_name,
18272 "backend_identity": backend_identity,
18273 }),
18274 ];
18275 let mut journal_paths = Vec::new();
18276 for fixture in &fixtures {
18277 let data = serde_json::to_vec(&fixture).expect("legacy journal fixture should encode");
18278 let entry = crate::bucket::lifecycle::tier_delete_journal::decode_tier_delete_journal_entry(&data)
18279 .expect("legacy journal fixture should decode");
18280 let path = tier_delete_journal_object_name(&entry);
18281 com::save_config(store.clone(), &path, data)
18282 .await
18283 .expect("legacy journal fixture should persist");
18284 journal_paths.push(path);
18285 }
18286 let corrupt_path = format!(
18287 "{TIER_DELETE_JOURNAL_PREFIX}/{}.json",
18288 rustfs_utils::crypto::hex_sha256(b"corrupt legacy tier journal", ToOwned::to_owned)
18289 );
18290 com::save_config(store.clone(), &corrupt_path, b"{corrupt".to_vec())
18291 .await
18292 .expect("corrupt legacy journal fixture should persist");
18293
18294 let (first, concurrent) = tokio::join!(
18295 recover_tier_delete_journal_entries(store.clone(), 100, None),
18296 recover_tier_delete_journal_entries(store.clone(), 100, None),
18297 );
18298 for stats in [first, concurrent] {
18299 let stats = stats.expect("concurrent legacy journal recovery scan should finish");
18300 assert_eq!((stats.scanned, stats.deleted, stats.failed), (3, 0, 0));
18301 }
18302 assert_eq!(tier_delete_journal_count(store.clone()).await, 3);
18303 assert_eq!(backend.remove_count().await, 0, "legacy recovery must not call the remote tier");
18304 assert_eq!(backend.exact_remove_count(), 0, "legacy recovery must not issue exact remote DELETE");
18305 assert!(backend.op_log().await.is_empty(), "legacy recovery must not invoke any backend operation");
18306
18307 let mut first_controls = list_recovery_controls(store.clone(), IlmRecoveryProtocol::TierDeleteJournal, None, 100, None)
18308 .await
18309 .expect("legacy recovery controls should be listable")
18310 .records;
18311 first_controls.sort_by(|left, right| left.control_id.cmp(&right.control_id));
18312 assert_eq!(first_controls.len(), 3);
18313 assert_eq!(
18314 first_controls
18315 .iter()
18316 .filter(|control| control.classification == IlmRecoveryClassification::RetainedAmbiguous)
18317 .count(),
18318 2
18319 );
18320 assert_eq!(
18321 first_controls
18322 .iter()
18323 .filter(|control| control.classification == IlmRecoveryClassification::Corrupt)
18324 .count(),
18325 1
18326 );
18327 for view in &first_controls {
18328 assert_eq!(view.protocol, IlmRecoveryProtocol::TierDeleteJournal);
18329 assert_eq!(view.revision, 1);
18330 assert_eq!(view.attempt_count, 0);
18331 let encoded = serde_json::to_string(view).expect("recovery control view should encode");
18332 for secret in ["legacy/remote-v1", "legacy/remote-v2", "opaque-v1", "opaque-v2", tier_name] {
18333 assert!(!encoded.contains(secret), "recovery control view must redact `{secret}`");
18334 }
18335 let persisted = load_recovery_control(store.clone(), IlmRecoveryProtocol::TierDeleteJournal, &view.control_id)
18336 .await
18337 .expect("legacy recovery control should load");
18338 match view.source_schema.as_str() {
18339 "rustfs-tier-delete-journal-v1" => {
18340 assert_eq!(persisted.control.identity.record_class, "tier_delete_journal_v1");
18341 assert_eq!(view.last_error_code, IlmRecoveryErrorCode::RemoteVersionUnknown);
18342 }
18343 "rustfs-tier-delete-journal-v2" => {
18344 assert_eq!(persisted.control.identity.record_class, "tier_delete_journal_v2");
18345 assert_eq!(view.last_error_code, IlmRecoveryErrorCode::RemoteVersionUnknown);
18346 }
18347 "rustfs-tier-delete-journal-unknown" => {
18348 assert_eq!(persisted.control.identity.record_class, "tier_delete_journal_corrupt");
18349 assert_eq!(view.last_error_code, IlmRecoveryErrorCode::SourceCorrupt);
18350 }
18351 schema => panic!("unexpected legacy recovery source schema: {schema}"),
18352 }
18353 }
18354
18355 let creator_sha256 = rustfs_utils::crypto::hex_sha256(b"legacy-export-actor", ToOwned::to_owned);
18356 let mut created_exports = Vec::new();
18357 for exportable in first_controls
18358 .iter()
18359 .filter(|control| control.classification == IlmRecoveryClassification::RetainedAmbiguous)
18360 {
18361 let observation = inspect_recovery_export_observation(store.clone(), &exportable.control_id)
18362 .await
18363 .expect("fresh legacy recovery observation should be exportable");
18364 let created = create_recovery_export(store.clone(), &observation, &creator_sha256)
18365 .await
18366 .expect("legacy recovery export should be created exactly once");
18367 assert!(!created.replayed);
18368 let loaded = load_recovery_export(store.clone(), &created.export_id)
18369 .await
18370 .expect("created legacy recovery export should load");
18371 assert_eq!(loaded.encoded, created.encoded, "export readback must preserve the exact committed bytes");
18372 let replayed = create_recovery_export(store.clone(), &observation, &creator_sha256)
18373 .await
18374 .expect("the same observed generation should replay its immutable export");
18375 assert!(replayed.replayed);
18376 assert_eq!(replayed.encoded, created.encoded);
18377 created_exports.push(created);
18378 }
18379 assert_eq!(created_exports.len(), 2, "both v1 and v2 legacy journals must have an export path");
18380
18381 let corrupt_export_id = &created_exports[0].export_id;
18382 let export_path = recovery_export_record_object_name(IlmRecoveryProtocol::TierDeleteJournal, corrupt_export_id)
18383 .expect("export path should build");
18384 com::save_config(store.clone(), &export_path, Vec::new())
18385 .await
18386 .expect("zero-byte corruption fixture should persist");
18387 let corrupt_export = load_recovery_export(store.clone(), corrupt_export_id)
18388 .await
18389 .expect_err("an existing zero-byte export must fail closed");
18390 assert!(!matches!(corrupt_export, Error::ConfigNotFound));
18391
18392 com::save_config(
18393 store.clone(),
18394 &journal_paths[0],
18395 serde_json::to_vec_pretty(&fixtures[0]).expect("rewritten legacy journal fixture should encode"),
18396 )
18397 .await
18398 .expect("equivalent legacy journal rewrite should persist");
18399 let second = recover_tier_delete_journal_entries(store.clone(), 100, None)
18400 .await
18401 .expect("repeated legacy journal recovery scan should finish");
18402 assert_eq!((second.scanned, second.deleted, second.failed), (3, 0, 0));
18403 let mut second_controls = list_recovery_controls(store, IlmRecoveryProtocol::TierDeleteJournal, None, 100, None)
18404 .await
18405 .expect("repeated legacy recovery controls should remain listable")
18406 .records;
18407 second_controls.sort_by(|left, right| left.control_id.cmp(&right.control_id));
18408 assert_eq!(second_controls, first_controls, "repeated scans must not reset durable controls");
18409 assert_eq!(backend.remove_count().await, 0, "repeated recovery must remain remote-call free");
18410 assert_eq!(backend.exact_remove_count(), 0);
18411 assert!(
18412 backend.op_log().await.is_empty(),
18413 "repeated recovery must not invoke any backend operation"
18414 );
18415 }
18416
18417 #[cfg(feature = "test-util")]
18418 #[tokio::test]
18419 #[serial_test::serial(storage_class_env)]
18420 async fn legacy_recovery_disposition_removes_only_local_journals_and_replays() {
18421 Box::pin(legacy_recovery_disposition_removes_only_local_journals_and_replays_case()).await;
18422 }
18423
18424 #[cfg(feature = "test-util")]
18425 async fn legacy_recovery_disposition_removes_only_local_journals_and_replays_case() {
18426 let temp_dir = tempfile::tempdir().expect("create legacy disposition store dir");
18427 let (ctx, store, _shutdown) =
18428 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "legacy-recovery-disposition", &[4])).await;
18429 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
18430
18431 let tier_name = "LEGACY-DISPOSITION";
18432 let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
18433 let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
18434 .await
18435 .expect("legacy disposition tier lease should resolve")
18436 .backend_identity();
18437 let fixtures = [
18438 serde_json::json!({
18439 "version": 1,
18440 "obj_name": "legacy/disposition-v1",
18441 "version_id": "opaque-disposition-v1",
18442 "tier_name": tier_name,
18443 }),
18444 serde_json::json!({
18445 "version": 2,
18446 "obj_name": "legacy/disposition-v2",
18447 "version_id": "opaque-disposition-v2",
18448 "tier_name": tier_name,
18449 "backend_identity": backend_identity,
18450 }),
18451 ];
18452 let mut journal_paths = Vec::new();
18453 for fixture in &fixtures {
18454 let data = serde_json::to_vec(fixture).expect("legacy disposition fixture should encode");
18455 let entry = crate::bucket::lifecycle::tier_delete_journal::decode_tier_delete_journal_entry(&data)
18456 .expect("legacy disposition fixture should decode");
18457 let path = tier_delete_journal_object_name(&entry);
18458 com::save_config(store.clone(), &path, data)
18459 .await
18460 .expect("legacy disposition fixture should persist");
18461 journal_paths.push(path);
18462 }
18463
18464 let recovered = recover_tier_delete_journal_entries(store.clone(), 100, None)
18465 .await
18466 .expect("legacy disposition recovery scan should finish");
18467 assert_eq!((recovered.scanned, recovered.deleted, recovered.failed), (2, 0, 0));
18468 assert_eq!(tier_delete_journal_count(store.clone()).await, 2);
18469
18470 let mut controls = list_recovery_controls(
18471 store.clone(),
18472 IlmRecoveryProtocol::TierDeleteJournal,
18473 Some(IlmRecoveryClassification::RetainedAmbiguous),
18474 100,
18475 None,
18476 )
18477 .await
18478 .expect("legacy disposition controls should be listable")
18479 .records;
18480 controls.sort_by(|left, right| left.control_id.cmp(&right.control_id));
18481 assert_eq!(controls.len(), 2, "both legacy schemas must support disposition");
18482
18483 let actor_sha256 = rustfs_utils::crypto::hex_sha256(b"legacy-disposition-actor", ToOwned::to_owned);
18484 let wrong_actor_sha256 = rustfs_utils::crypto::hex_sha256(b"different-disposition-actor", ToOwned::to_owned);
18485 let wrong_export_sha256 = "ff".repeat(32);
18486 for (index, control) in controls.iter().enumerate() {
18487 let observation = inspect_recovery_export_observation(store.clone(), &control.control_id)
18488 .await
18489 .expect("legacy disposition source should be observable");
18490 let export = create_recovery_export(store.clone(), &observation, &actor_sha256)
18491 .await
18492 .expect("legacy disposition export should persist");
18493 let confirmed_at_unix_nanos = i64::try_from(OffsetDateTime::now_utc().unix_timestamp_nanos())
18494 .expect("legacy disposition timestamp should fit i64");
18495
18496 if index == 0 {
18497 let wrong_hash = Box::pin(execute_recovery_disposition(
18498 store.clone(),
18499 &observation,
18500 &export.export_id,
18501 &wrong_export_sha256,
18502 &actor_sha256,
18503 confirmed_at_unix_nanos,
18504 ))
18505 .await
18506 .expect_err("a mismatched export checksum must fail before local deletion");
18507 assert_eq!(wrong_hash, Error::PreconditionFailed);
18508 assert_eq!(tier_delete_journal_count(store.clone()).await, 2);
18509 }
18510
18511 let dry_run = dry_run_recovery_disposition(
18512 store.clone(),
18513 &observation,
18514 &export.export_id,
18515 &export.content_sha256,
18516 &actor_sha256,
18517 confirmed_at_unix_nanos,
18518 )
18519 .await
18520 .expect("legacy disposition dry-run should validate exact local state");
18521 assert_eq!(dry_run.source_copy_count, observation.source_generation.copies.len());
18522 assert_eq!(
18523 tier_delete_journal_count(store.clone()).await,
18524 fixtures.len() - index,
18525 "dry-run must not delete a legacy journal"
18526 );
18527 assert!(
18528 matches!(
18529 load_recovery_disposition(store.clone(), IlmRecoveryProtocol::TierDeleteJournal, &dry_run.disposition_id,)
18530 .await,
18531 Err(Error::ConfigNotFound)
18532 ),
18533 "dry-run must not persist a disposition record"
18534 );
18535
18536 if index == 0 {
18537 inject_recovery_disposition_crash_once(RecoveryDispositionCrashStage::AfterLocalDelete);
18538 Box::pin(execute_recovery_disposition(
18539 store.clone(),
18540 &observation,
18541 &export.export_id,
18542 &export.content_sha256,
18543 &actor_sha256,
18544 confirmed_at_unix_nanos,
18545 ))
18546 .await
18547 .expect_err("the injected crash must stop after local delete commits");
18548 assert_eq!(tier_delete_journal_count(store.clone()).await, 1);
18549 let interrupted =
18550 load_recovery_disposition(store.clone(), IlmRecoveryProtocol::TierDeleteJournal, &dry_run.disposition_id)
18551 .await
18552 .expect("the applying disposition must survive the post-delete crash");
18553 assert_eq!(interrupted.disposition.state, IlmRecoveryDispositionState::Applying);
18554 assert!(
18555 interrupted.disposition.confirmed_absent.is_empty(),
18556 "the crash must occur before absence progress is persisted"
18557 );
18558 } else {
18559 inject_recovery_disposition_crash_once(RecoveryDispositionCrashStage::AfterControlAbandon);
18560 Box::pin(execute_recovery_disposition(
18561 store.clone(),
18562 &observation,
18563 &export.export_id,
18564 &export.content_sha256,
18565 &actor_sha256,
18566 confirmed_at_unix_nanos,
18567 ))
18568 .await
18569 .expect_err("the injected crash must stop after control abandonment commits");
18570 let interrupted =
18571 load_recovery_disposition(store.clone(), IlmRecoveryProtocol::TierDeleteJournal, &dry_run.disposition_id)
18572 .await
18573 .expect("the applying disposition must survive the post-control crash");
18574 assert_eq!(interrupted.disposition.state, IlmRecoveryDispositionState::Applying);
18575 assert_eq!(
18576 interrupted.disposition.confirmed_absent.len(),
18577 interrupted.disposition.identity.source_generation.copies.len()
18578 );
18579
18580 let abandoned =
18581 load_recovery_control(store.clone(), IlmRecoveryProtocol::TierDeleteJournal, &observation.control_id)
18582 .await
18583 .expect("the abandoned control must survive the injected crash");
18584 let exact_abandoned = abandoned.control.encode().expect("the exact abandoned control should encode");
18585 let mut wrong_history = abandoned.control;
18586 wrong_history.last_error_code = IlmRecoveryErrorCode::CleanupFailed;
18587 let control_path = recovery_control_record_object_name(observation.protocol, &observation.control_id)
18588 .expect("control path should remain canonical");
18589 com::save_config(
18590 store.clone(),
18591 &control_path,
18592 wrong_history
18593 .encode()
18594 .expect("the alternate valid control history should encode"),
18595 )
18596 .await
18597 .expect("the alternate control history fixture should persist");
18598 let wrong_history_err = Box::pin(execute_recovery_disposition(
18599 store.clone(),
18600 &observation,
18601 &export.export_id,
18602 &export.content_sha256,
18603 &actor_sha256,
18604 confirmed_at_unix_nanos + 1,
18605 ))
18606 .await
18607 .expect_err("a different abandoned control history must not bridge to completion");
18608 assert_eq!(wrong_history_err, Error::PreconditionFailed);
18609 com::save_config(store.clone(), &control_path, exact_abandoned)
18610 .await
18611 .expect("the exact abandoned control fixture should be restored");
18612 }
18613
18614 let replay_confirmed_at_unix_nanos = confirmed_at_unix_nanos + 2;
18615 let executed = Box::pin(execute_recovery_disposition(
18616 store.clone(),
18617 &observation,
18618 &export.export_id,
18619 &export.content_sha256,
18620 &actor_sha256,
18621 replay_confirmed_at_unix_nanos,
18622 ))
18623 .await
18624 .expect("a later request must resume and complete the interrupted disposition");
18625 assert_eq!(executed.state, IlmRecoveryDispositionState::Completed);
18626 assert_eq!(executed.outcome, IlmRecoveryDispositionExecutionOutcome::Completed);
18627 assert_eq!(executed.confirmed_absent_copy_count, executed.source_copy_count);
18628 assert_eq!(tier_delete_journal_count(store.clone()).await, fixtures.len() - index - 1);
18629 assert!(matches!(
18630 com::read_config(store.clone(), &observation.canonical_source_path).await,
18631 Err(Error::ConfigNotFound)
18632 ));
18633 if index == 0 {
18634 let untouched = journal_paths
18635 .iter()
18636 .find(|path| *path != &observation.canonical_source_path)
18637 .expect("the other legacy journal should remain");
18638 com::read_config(store.clone(), untouched)
18639 .await
18640 .expect("disposition must not remove a different legacy journal");
18641 }
18642
18643 let abandoned = load_recovery_control(store.clone(), IlmRecoveryProtocol::TierDeleteJournal, &observation.control_id)
18644 .await
18645 .expect("abandoned recovery control should remain inspectable");
18646 assert_eq!(abandoned.control.classification, IlmRecoveryClassification::Abandoned);
18647 assert_eq!(abandoned.control.revision, observation.control_revision + 1);
18648 assert_eq!(abandoned.control.observed_source_generation, observation.source_generation);
18649
18650 let persisted =
18651 load_recovery_disposition(store.clone(), IlmRecoveryProtocol::TierDeleteJournal, &executed.disposition_id)
18652 .await
18653 .expect("completed disposition should remain durable");
18654 assert_eq!(persisted.disposition.state, IlmRecoveryDispositionState::Completed);
18655
18656 let replayed = Box::pin(execute_recovery_disposition(
18657 store.clone(),
18658 &observation,
18659 &export.export_id,
18660 &export.content_sha256,
18661 &actor_sha256,
18662 replay_confirmed_at_unix_nanos + 1,
18663 ))
18664 .await
18665 .expect("same actor should replay the completed disposition");
18666 assert_eq!(replayed.state, IlmRecoveryDispositionState::Completed);
18667 assert_eq!(replayed.outcome, IlmRecoveryDispositionExecutionOutcome::Replayed);
18668
18669 let wrong_actor = Box::pin(execute_recovery_disposition(
18670 store.clone(),
18671 &observation,
18672 &export.export_id,
18673 &export.content_sha256,
18674 &wrong_actor_sha256,
18675 replay_confirmed_at_unix_nanos + 2,
18676 ))
18677 .await
18678 .expect_err("a different actor must not replay a completed disposition");
18679 assert_eq!(wrong_actor, Error::PreconditionFailed);
18680
18681 assert!(
18682 !Box::pin(garbage_collect_completed_recovery_disposition(
18683 store.clone(),
18684 &persisted,
18685 persisted.disposition.retain_until_unix_nanos - 1,
18686 ))
18687 .await
18688 .expect("completed disposition should remain before retention expires")
18689 );
18690 assert!(
18691 Box::pin(garbage_collect_completed_recovery_disposition(
18692 store.clone(),
18693 &persisted,
18694 persisted.disposition.retain_until_unix_nanos,
18695 ))
18696 .await
18697 .expect("expired completed disposition should be garbage collected")
18698 );
18699 assert!(matches!(
18700 load_recovery_disposition(store.clone(), IlmRecoveryProtocol::TierDeleteJournal, &executed.disposition_id).await,
18701 Err(Error::ConfigNotFound)
18702 ));
18703 assert_eq!(backend.remove_count().await, 0, "legacy disposition must not call the remote tier");
18704 assert_eq!(backend.exact_remove_count(), 0, "legacy disposition must not issue exact remote DELETE");
18705 assert!(
18706 backend.op_log().await.is_empty(),
18707 "legacy disposition must not invoke any backend operation"
18708 );
18709 }
18710 }
18711
18712 #[cfg(feature = "test-util")]
18713 #[tokio::test]
18714 #[serial_test::serial(storage_class_env)]
18715 async fn v4_tier_delete_journal_recovery_is_exact_and_store_scoped() {
18716 let temp_a = tempfile::tempdir().expect("create temp store dir a");
18717 let temp_b = tempfile::tempdir().expect("create temp store dir b");
18718 let (ctx_a, store_a, _shutdown_a) =
18719 without_storage_class_env(build_isolated_test_store(temp_a.path(), "tier-journal-recovery-a", &[4])).await;
18720 let (ctx_b, store_b, _shutdown_b) =
18721 without_storage_class_env(build_isolated_test_store(temp_b.path(), "tier-journal-recovery-b", &[4])).await;
18722 crate::bucket::metadata_sys::init_bucket_metadata_sys(store_a.clone(), Vec::new()).await;
18723 crate::bucket::metadata_sys::init_bucket_metadata_sys(store_b.clone(), Vec::new()).await;
18724
18725 let tier_a = "JOURNAL-A";
18726 let tier_b = "JOURNAL-B";
18727 let backend_a = register_mock_tier(&ctx_a.tier_config_mgr(), tier_a).await;
18728 let backend_b = register_mock_tier(&ctx_b.tier_config_mgr(), tier_b).await;
18729 let identity_a = TierConfigMgr::acquire_operation_lease(&ctx_a.tier_config_mgr(), tier_a)
18730 .await
18731 .expect("store A tier lease should resolve")
18732 .backend_identity();
18733 let identity_b = TierConfigMgr::acquire_operation_lease(&ctx_b.tier_config_mgr(), tier_b)
18734 .await
18735 .expect("store B tier lease should resolve")
18736 .backend_identity();
18737 let entry_a = Jentry {
18738 persisted_version: 0,
18739 obj_name: "remote-a".to_string(),
18740 version_id: "version-a".to_string(),
18741 tier_name: tier_a.to_string(),
18742 backend_identity: Some(identity_a),
18743 version_id_exact: true,
18744 version_state: rustfs_filemeta::TransitionVersionState::Exact,
18745 state: crate::bucket::lifecycle::tier_sweeper::TierDeleteJournalState::Committed,
18746 source: None,
18747 dispatch: None,
18748 };
18749 let entry_b = Jentry {
18750 persisted_version: 0,
18751 obj_name: "remote-b".to_string(),
18752 version_id: "version-b".to_string(),
18753 tier_name: tier_b.to_string(),
18754 backend_identity: Some(identity_b),
18755 version_id_exact: true,
18756 version_state: rustfs_filemeta::TransitionVersionState::Exact,
18757 state: crate::bucket::lifecycle::tier_sweeper::TierDeleteJournalState::Committed,
18758 source: None,
18759 dispatch: None,
18760 };
18761 persist_tier_delete_journal_entry(store_a.clone(), &entry_a)
18762 .await
18763 .expect("store A v4 compatibility fixture should persist");
18764 persist_tier_delete_journal_entry(store_b.clone(), &entry_b)
18765 .await
18766 .expect("store B v4 compatibility fixture should persist");
18767 let entry_a_v3 = Jentry {
18768 obj_name: "remote-a-v3".to_string(),
18769 version_id: "version-a-v3".to_string(),
18770 ..entry_a.clone()
18771 };
18772 let mut raw_v3: serde_json::Value = serde_json::from_slice(
18773 &encode_tier_delete_journal_entry(&entry_a_v3).expect("store A v3 fixture should encode from the v4 shape"),
18774 )
18775 .expect("store A v3 fixture should be JSON");
18776 raw_v3["version"] = serde_json::json!(3);
18777 raw_v3
18778 .as_object_mut()
18779 .expect("store A v3 fixture should be an object")
18780 .remove("version_state");
18781 let path_a_v3 = tier_delete_journal_object_name(&entry_a_v3);
18782 com::save_config(
18783 store_a.clone(),
18784 &path_a_v3,
18785 serde_json::to_vec(&raw_v3).expect("store A raw v3 fixture should encode"),
18786 )
18787 .await
18788 .expect("store A raw v3 compatibility fixture should persist");
18789 let path_a = tier_delete_journal_object_name(&entry_a);
18790 let path_b = tier_delete_journal_object_name(&entry_b);
18791 let recovered_a = recover_tier_delete_journal_entries(store_a.clone(), 100, None)
18792 .await
18793 .expect("store A recovery scan should finish");
18794 let recovered_b = recover_tier_delete_journal_entries(store_b.clone(), 100, None)
18795 .await
18796 .expect("store B recovery scan should finish");
18797 assert_eq!((recovered_a.scanned, recovered_a.deleted, recovered_a.failed), (2, 2, 0));
18798 assert_eq!((recovered_b.scanned, recovered_b.deleted, recovered_b.failed), (1, 1, 0));
18799 let mut removed_a = backend_a.remove_versions().await;
18800 removed_a.sort();
18801 assert_eq!(
18802 removed_a,
18803 vec![
18804 ("remote-a".to_string(), "version-a".to_string()),
18805 ("remote-a-v3".to_string(), "version-a-v3".to_string()),
18806 ]
18807 );
18808 assert_eq!(backend_b.remove_versions().await, vec![("remote-b".to_string(), "version-b".to_string())]);
18809 assert!(matches!(com::read_config(store_a.clone(), &path_a).await, Err(Error::ConfigNotFound)));
18810 assert!(matches!(com::read_config(store_a.clone(), &path_a_v3).await, Err(Error::ConfigNotFound)));
18811 assert!(matches!(com::read_config(store_b.clone(), &path_b).await, Err(Error::ConfigNotFound)));
18812 assert_eq!(tier_delete_journal_count(store_a).await, 0);
18813 assert_eq!(tier_delete_journal_count(store_b).await, 0);
18814 }
18815
18816 #[cfg(feature = "test-util")]
18817 #[tokio::test]
18818 #[serial_test::serial(storage_class_env)]
18819 async fn v5_tier_delete_recovery_retains_live_sources_and_aborts_only_prepared() {
18820 let temp_dir = tempfile::tempdir().expect("create v5 live-source store dir");
18821 let (ctx, store, _shutdown) =
18822 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "v5-live-source-recovery", &[4])).await;
18823 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
18824
18825 let tier_name = "V5-LIVE-SOURCE";
18826 let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
18827 let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
18828 .await
18829 .expect("v5 live-source tier lease should resolve")
18830 .backend_identity();
18831 let bucket = "v5-live-source-recovery-bucket";
18832 store
18833 .make_bucket(bucket, &MakeBucketOptions::default())
18834 .await
18835 .expect("v5 live-source bucket should be created");
18836
18837 let mut paths = Vec::new();
18838 for (index, state) in [
18839 TierDeleteJournalState::Prepared,
18840 TierDeleteJournalState::Dispatched,
18841 TierDeleteJournalState::Committed,
18842 ]
18843 .into_iter()
18844 .enumerate()
18845 {
18846 let object = format!("live-{index}.bin");
18847 let remote_version = uuid::Uuid::new_v4().to_string();
18848 backend.set_put_remote_version(Some(remote_version)).await;
18849 let mut reader = PutObjReader::from_vec(format!("v5 live source {index}").into_bytes());
18850 let source = store
18851 .put_object(bucket, &object, &mut reader, &ObjectOptions::default())
18852 .await
18853 .expect("v5 live source should be written");
18854 store
18855 .transition_object(
18856 bucket,
18857 &object,
18858 &ObjectOptions {
18859 transition: TransitionOptions {
18860 status: TRANSITION_PENDING.to_string(),
18861 tier: tier_name.to_string(),
18862 etag: source.etag.clone().expect("v5 live source should have an ETag"),
18863 ..Default::default()
18864 },
18865 version_id: source.version_id.map(|version| version.to_string()),
18866 mod_time: source.mod_time,
18867 ..Default::default()
18868 },
18869 )
18870 .await
18871 .expect("v5 live source should transition");
18872 let transitioned = store
18873 .get_object_info(
18874 bucket,
18875 &object,
18876 &ObjectOptions {
18877 no_lock: true,
18878 metadata_cache_safe: false,
18879 ..Default::default()
18880 },
18881 )
18882 .await
18883 .expect("v5 transitioned source should be readable");
18884 let mut entry = transitioned_delete_journal_entry_for_source(None, false, false, bucket, &object, &transitioned)
18885 .expect("v5 transitioned source should produce a stable journal identity");
18886 entry.persisted_version = 5;
18887 entry.backend_identity = Some(backend_identity);
18888 entry.state = state;
18889 entry.dispatch = None;
18890 let path = tier_delete_journal_object_name(&entry);
18891 com::save_config(
18892 store.clone(),
18893 &path,
18894 encode_tier_delete_journal_entry(&entry).expect("raw v5 live-source journal should encode"),
18895 )
18896 .await
18897 .expect("raw v5 live-source journal should persist");
18898 paths.push((state, path));
18899 }
18900
18901 let stats = recover_tier_delete_journal_entries(store.clone(), 100, None)
18902 .await
18903 .expect("v5 live-source recovery should finish");
18904 assert_eq!((stats.scanned, stats.deleted, stats.failed), (3, 1, 2));
18905 assert_eq!(backend.remove_count().await, 0, "a live source must block every remote DELETE");
18906 for (state, path) in paths {
18907 let observed = com::read_config(store.clone(), &path).await;
18908 if state == TierDeleteJournalState::Prepared {
18909 assert!(matches!(observed, Err(Error::ConfigNotFound)), "Prepared v5 evidence should abort");
18910 } else {
18911 assert!(observed.is_ok(), "{state:?} v5 evidence must remain while its exact source exists");
18912 }
18913 }
18914 }
18915
18916 #[cfg(feature = "test-util")]
18917 #[tokio::test]
18918 #[serial_test::serial(storage_class_env)]
18919 async fn v5_tier_delete_recovery_replays_all_crash_states_without_sources() {
18920 let temp_dir = tempfile::tempdir().expect("create v5 crash-replay store dir");
18921 let (ctx, store, _shutdown) =
18922 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "v5-crash-replay", &[4])).await;
18923 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
18924
18925 let tier_name = "V5-CRASH-REPLAY";
18926 let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
18927 let lease = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
18928 .await
18929 .expect("v5 crash-replay tier lease should resolve");
18930 let backend_identity = lease.backend_identity();
18931 let bucket = "v5-crash-replay-bucket";
18932 store
18933 .make_bucket(bucket, &MakeBucketOptions::default())
18934 .await
18935 .expect("v5 crash-replay bucket should be created");
18936
18937 for (index, state, seed_remote) in [
18938 (0, TierDeleteJournalState::Prepared, true),
18939 (1, TierDeleteJournalState::Dispatched, true),
18940 (2, TierDeleteJournalState::Committed, true),
18941 (3, TierDeleteJournalState::Committed, false),
18944 ] {
18945 let remote_version = uuid::Uuid::new_v4().to_string();
18946 let mut entry = synthetic_v6_dispatch_entry(
18947 bucket,
18948 &format!("missing-{index}.bin"),
18949 tier_name,
18950 backend_identity,
18951 &remote_version,
18952 );
18953 entry.persisted_version = 5;
18954 entry.state = state;
18955 entry.dispatch = None;
18956 if seed_remote {
18957 backend.set_put_remote_version(Some(remote_version)).await;
18958 let body = bytes::Bytes::from(format!("v5 remote candidate {index}"));
18959 lease
18960 .put(
18961 &entry.obj_name,
18962 ReaderImpl::Body(body.clone()),
18963 i64::try_from(body.len()).expect("v5 candidate length should fit i64"),
18964 )
18965 .await
18966 .expect("v5 remote candidate should be seeded");
18967 }
18968 let path = tier_delete_journal_object_name(&entry);
18969 com::save_config(
18970 store.clone(),
18971 &path,
18972 encode_tier_delete_journal_entry(&entry).expect("raw v5 crash journal should encode"),
18973 )
18974 .await
18975 .expect("raw v5 crash journal should persist");
18976 }
18977
18978 let stats = recover_tier_delete_journal_entries(store.clone(), 100, None)
18979 .await
18980 .expect("v5 crash-state recovery should finish");
18981 assert_eq!((stats.scanned, stats.deleted, stats.failed), (4, 4, 0));
18982 assert_eq!(tier_delete_journal_count(store).await, 0);
18983 assert_eq!(backend.object_count().await, 0);
18984 assert_eq!(backend.exact_remove_count(), 4, "every v5 state must use exact-version cleanup");
18985 }
18986
18987 #[cfg(feature = "test-util")]
18988 #[tokio::test]
18989 #[serial_test::serial(storage_class_env)]
18990 async fn dispatched_tier_delete_recovery_finds_directory_source_on_encoded_set() {
18991 let temp_dir = tempfile::tempdir().expect("create temp store dir");
18992 let shutdown = CancellationToken::new();
18993 let (ctx, store, _shutdown) = without_storage_class_env(build_isolated_test_store_with_layout(
18994 temp_dir.path(),
18995 "prepared-directory-recovery",
18996 &[(2, 4)],
18997 shutdown,
18998 None,
18999 ))
19000 .await;
19001 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
19002
19003 let pool = &store.pools[0];
19004 let object = (0..10_000)
19005 .map(|index| format!("directory-{index}/"))
19006 .find(|candidate| {
19007 let encoded = rustfs_utils::path::encode_dir_object(candidate);
19008 !Arc::ptr_eq(&pool.get_disks_by_key(candidate), &pool.get_disks_by_key(&encoded))
19009 })
19010 .expect("test topology should have a directory key whose encoded form hashes to another set");
19011 let encoded = rustfs_utils::path::encode_dir_object(&object);
19012 assert!(!Arc::ptr_eq(&pool.get_disks_by_key(&object), &pool.get_disks_by_key(&encoded)));
19013
19014 let tier_name = "PREPAREDDIRECTORY";
19015 let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
19016 let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
19017 .await
19018 .expect("tier lease should resolve")
19019 .backend_identity();
19020 let bucket = "prepared-directory-recovery-bucket";
19021 store
19022 .make_bucket(bucket, &MakeBucketOptions::default())
19023 .await
19024 .expect("bucket should be created");
19025 let mut reader = PutObjReader::from_vec(b"directory source".to_vec());
19026 let original = store
19027 .put_object(bucket, &object, &mut reader, &ObjectOptions::default())
19028 .await
19029 .expect("directory source should be written");
19030 store
19031 .transition_object(
19032 bucket,
19033 &object,
19034 &ObjectOptions {
19035 transition: TransitionOptions {
19036 status: TRANSITION_PENDING.to_string(),
19037 tier: tier_name.to_string(),
19038 etag: original.etag.clone().expect("source should have an ETag"),
19039 ..Default::default()
19040 },
19041 mod_time: original.mod_time,
19042 ..Default::default()
19043 },
19044 )
19045 .await
19046 .expect("directory transition should commit");
19047 let committed = store
19048 .get_object_info(
19049 bucket,
19050 &object,
19051 &ObjectOptions {
19052 no_lock: true,
19053 metadata_cache_safe: false,
19054 ..Default::default()
19055 },
19056 )
19057 .await
19058 .expect("transitioned directory source should be readable");
19059 let mut entry = transitioned_delete_journal_entry_for_source(None, false, false, bucket, &object, &committed)
19060 .expect("transitioned source should produce a prepared journal entry");
19061 entry.backend_identity = Some(backend_identity);
19062 let incarnation = store
19063 .bucket_incarnation_id(bucket)
19064 .await
19065 .expect("bucket incarnation should resolve");
19066 install_test_tier_delete_dispatch_fixture(
19067 store.clone(),
19068 bucket,
19069 incarnation,
19070 &object,
19071 vec![(entry, Some(TierDeleteJournalState::Dispatched))],
19072 TierDeleteDispatchManifestState::DispatchAuthorized,
19073 )
19074 .await
19075 .expect("authorized dispatched journal should persist");
19076
19077 let stats = recover_tier_delete_journal_entries(store.clone(), 100, None)
19078 .await
19079 .expect("prepared recovery should complete");
19080
19081 assert_eq!((stats.scanned, stats.deleted, stats.failed), (1, 0, 1));
19082 assert_eq!(tier_delete_journal_count(store).await, 1);
19083 assert_eq!(backend.remove_count().await, 0);
19084 assert_eq!(backend.object_count().await, 1, "live directory source must retain its remote object");
19085 }
19086
19087 #[cfg(feature = "test-util")]
19088 #[tokio::test]
19089 #[serial_test::serial(storage_class_env)]
19090 async fn dispatched_tier_delete_recovery_checks_later_pool_then_commits_after_source_removal() {
19091 let temp_dir = tempfile::tempdir().expect("create cross-pool recovery store dir");
19092 let (ctx, store, _shutdown) =
19093 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "prepared-cross-pool-recovery", &[4, 4])).await;
19094 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
19095 let tier_name = "PREPAREDCROSSPOOL";
19096 let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
19097 let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
19098 .await
19099 .expect("tier lease should resolve")
19100 .backend_identity();
19101 let bucket = "prepared-cross-pool-recovery-bucket";
19102 let object = "object";
19103 store
19104 .make_bucket(bucket, &MakeBucketOptions::default())
19105 .await
19106 .expect("bucket should be created");
19107 let mut reader = PutObjReader::from_vec(b"later pool source".to_vec());
19108 let original = store.pools[1]
19109 .put_object(bucket, object, &mut reader, &ObjectOptions::default())
19110 .await
19111 .expect("source should be written only to the later pool");
19112 store.pools[1]
19113 .transition_object(
19114 bucket,
19115 object,
19116 &ObjectOptions {
19117 transition: TransitionOptions {
19118 status: TRANSITION_PENDING.to_string(),
19119 tier: tier_name.to_string(),
19120 etag: original.etag.clone().expect("source should have an ETag"),
19121 ..Default::default()
19122 },
19123 mod_time: original.mod_time,
19124 ..Default::default()
19125 },
19126 )
19127 .await
19128 .expect("later-pool transition should commit");
19129 let committed = store.pools[1]
19130 .get_object_info(bucket, object, &ObjectOptions::default())
19131 .await
19132 .expect("transitioned source should be readable");
19133 let mut entry = transitioned_delete_journal_entry_for_source(None, false, false, bucket, object, &committed)
19134 .expect("transitioned source should produce a prepared journal");
19135 entry.backend_identity = Some(backend_identity);
19136 let incarnation = store
19137 .bucket_incarnation_id(bucket)
19138 .await
19139 .expect("bucket incarnation should resolve");
19140 install_test_tier_delete_dispatch_fixture(
19141 store.clone(),
19142 bucket,
19143 incarnation,
19144 object,
19145 vec![(entry, Some(TierDeleteJournalState::Dispatched))],
19146 TierDeleteDispatchManifestState::DispatchAuthorized,
19147 )
19148 .await
19149 .expect("authorized dispatched journal should persist");
19150
19151 let retained = recover_tier_delete_journal_entries(store.clone(), 100, None)
19152 .await
19153 .expect("recovery should scan the later pool");
19154 assert_eq!((retained.scanned, retained.deleted, retained.failed), (1, 0, 1));
19155 assert_eq!(backend.remove_count().await, 0);
19156 assert_eq!(backend.object_count().await, 1);
19157 assert_eq!(
19158 tier_delete_journal_count(store.clone()).await,
19159 1,
19160 "an authorized journal must remain while its live source is durable"
19161 );
19162
19163 store.pools[1]
19164 .delete_object(
19165 bucket,
19166 object,
19167 ObjectOptions {
19168 version_id: committed.version_id.map(|version| version.to_string()),
19169 expiration: crate::storage_api_contracts::lifecycle::ExpirationOptions { expire: true },
19170 skip_free_version: true,
19171 ..Default::default()
19172 },
19173 )
19174 .await
19175 .expect("source version should be removed before recovery retry");
19176 let committed_stats = recover_tier_delete_journal_entries(store.clone(), 100, None)
19177 .await
19178 .expect("journal recovery should commit an absent stable source");
19179 assert_eq!((committed_stats.scanned, committed_stats.deleted, committed_stats.failed), (1, 0, 1));
19180 assert_eq!(backend.remove_count().await, 0);
19181
19182 let manifest_stats = recover_tier_delete_dispatch_manifests(store.clone(), 100, None)
19183 .await
19184 .expect("manifest recovery should authorize committed remote cleanup");
19185 assert_eq!((manifest_stats.advanced, manifest_stats.failed), (1, 0));
19186
19187 let deleted = recover_tier_delete_journal_entries(store.clone(), 100, None)
19188 .await
19189 .expect("completed journal should delete the exact remote version");
19190 assert_eq!((deleted.scanned, deleted.deleted, deleted.failed), (1, 1, 0));
19191 assert_eq!(tier_delete_journal_count(store).await, 0);
19192 assert_eq!(backend.remove_count().await, 1);
19193 assert_eq!(backend.object_count().await, 0);
19194 }
19195
19196 #[cfg(feature = "test-util")]
19197 #[tokio::test]
19198 #[serial_test::serial(storage_class_env)]
19199 async fn dispatched_tier_delete_recovery_retains_journal_on_source_metadata_error() {
19200 let temp_dir = tempfile::tempdir().expect("create metadata-error recovery store dir");
19201 let (ctx, store, _shutdown) =
19202 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "prepared-metadata-error-recovery", &[4])).await;
19203 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
19204 let tier_name = "PREPAREDMETADATAERROR";
19205 let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
19206 let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
19207 .await
19208 .expect("tier lease should resolve")
19209 .backend_identity();
19210 let bucket = "prepared-metadata-error-recovery-bucket";
19211 let object = "object";
19212 store
19213 .make_bucket(bucket, &MakeBucketOptions::default())
19214 .await
19215 .expect("bucket should be created");
19216 let mut reader = PutObjReader::from_vec(b"source with unreadable metadata".to_vec());
19217 let original = store
19218 .put_object(bucket, object, &mut reader, &ObjectOptions::default())
19219 .await
19220 .expect("source should be written");
19221 store
19222 .transition_object(
19223 bucket,
19224 object,
19225 &ObjectOptions {
19226 transition: TransitionOptions {
19227 status: TRANSITION_PENDING.to_string(),
19228 tier: tier_name.to_string(),
19229 etag: original.etag.clone().expect("source should have an ETag"),
19230 ..Default::default()
19231 },
19232 mod_time: original.mod_time,
19233 ..Default::default()
19234 },
19235 )
19236 .await
19237 .expect("source transition should commit");
19238 let committed = store
19239 .get_object_info(bucket, object, &ObjectOptions::default())
19240 .await
19241 .expect("transitioned source should be readable");
19242 let mut entry = transitioned_delete_journal_entry_for_source(None, false, false, bucket, object, &committed)
19243 .expect("transitioned source should produce a prepared journal");
19244 entry.backend_identity = Some(backend_identity);
19245 let incarnation = store
19246 .bucket_incarnation_id(bucket)
19247 .await
19248 .expect("bucket incarnation should resolve");
19249 install_test_tier_delete_dispatch_fixture(
19250 store.clone(),
19251 bucket,
19252 incarnation,
19253 object,
19254 vec![(entry, Some(TierDeleteJournalState::Dispatched))],
19255 TierDeleteDispatchManifestState::DispatchAuthorized,
19256 )
19257 .await
19258 .expect("authorized dispatched journal should persist");
19259
19260 for disk_index in 0..4 {
19261 let metadata_path = temp_dir
19262 .path()
19263 .join(format!("pool0/set0/disk{disk_index}/{bucket}/{object}/{STORAGE_FORMAT_FILE}"));
19264 tokio::fs::write(metadata_path, b"not-xl-meta")
19265 .await
19266 .expect("source metadata should be corrupted");
19267 }
19268
19269 let stats = recover_tier_delete_journal_entries(store.clone(), 100, None)
19270 .await
19271 .expect("recovery scan should complete despite the entry failure");
19272
19273 assert_eq!((stats.scanned, stats.deleted, stats.failed), (1, 0, 1));
19274 assert_eq!(tier_delete_journal_count(store).await, 1, "journal must remain dispatched for retry");
19275 assert_eq!(backend.remove_count().await, 0, "unreadable source metadata must block remote deletion");
19276 assert_eq!(backend.object_count().await, 1, "remote source must remain intact");
19277 }
19278
19279 #[cfg(feature = "test-util")]
19280 #[tokio::test]
19281 #[serial_test::serial(storage_class_env)]
19282 async fn tier_delete_recovery_retains_content_that_does_not_match_its_object_name() {
19283 let temp_dir = tempfile::tempdir().expect("create mismatched journal store dir");
19284 let (ctx, store, _shutdown) =
19285 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "mismatched-tier-journal", &[4])).await;
19286 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
19287 let tier_name = "MISMATCHEDJOURNAL";
19288 let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
19289 let lease = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
19290 .await
19291 .expect("tier lease should resolve");
19292 let remote_version = uuid::Uuid::new_v4().to_string();
19293 backend.set_put_remote_version(Some(remote_version.clone())).await;
19294 lease
19295 .put("remote/original", ReaderImpl::Body(bytes::Bytes::from_static(b"remote body")), 11)
19296 .await
19297 .expect("remote body should be seeded");
19298 let entry = Jentry {
19299 persisted_version: 0,
19300 obj_name: "remote/original".to_string(),
19301 version_id: remote_version,
19302 tier_name: tier_name.to_string(),
19303 backend_identity: Some(lease.backend_identity()),
19304 version_id_exact: true,
19305 version_state: rustfs_filemeta::TransitionVersionState::Exact,
19306 state: TierDeleteJournalState::Prepared,
19307 source: Some(TierDeleteSourceIdentity {
19308 bucket: "absent-source-bucket".to_string(),
19309 object: "absent-source-object".to_string(),
19310 version_id: Some(uuid::Uuid::new_v4().to_string()),
19311 versioned: true,
19312 version_suspended: false,
19313 data_dir: Some(uuid::Uuid::new_v4().to_string()),
19314 etag: Some("etag".to_string()),
19315 mod_time: Some(OffsetDateTime::UNIX_EPOCH.to_string()),
19316 }),
19317 dispatch: None,
19318 };
19319 let (_manifest_name, bound) = install_test_tier_delete_dispatch_fixture(
19320 store.clone(),
19321 "absent-source-bucket",
19322 uuid::Uuid::new_v4(),
19323 "absent-source-object",
19324 vec![(entry, Some(TierDeleteJournalState::Dispatched))],
19325 TierDeleteDispatchManifestState::DispatchAuthorized,
19326 )
19327 .await
19328 .expect("authorized dispatched journal should persist");
19329 let journal_name = tier_delete_journal_object_name(&bound[0]);
19330 let data = com::read_config(store.clone(), &journal_name)
19331 .await
19332 .expect("prepared journal should be readable");
19333 let mut value: serde_json::Value = serde_json::from_slice(&data).expect("journal should contain JSON");
19334 value["obj_name"] = serde_json::json!("remote/replaced");
19335 com::save_config(
19336 store.clone(),
19337 &journal_name,
19338 serde_json::to_vec(&value).expect("mismatched journal should encode"),
19339 )
19340 .await
19341 .expect("mismatched journal content should be written under the original name");
19342
19343 let stats = recover_tier_delete_journal_entries(store.clone(), 100, None)
19344 .await
19345 .expect("recovery scan should complete");
19346
19347 assert_eq!((stats.scanned, stats.deleted, stats.failed), (1, 0, 1));
19348 assert_eq!(tier_delete_journal_count(store).await, 1, "mismatched journal must be retained");
19349 assert_eq!(backend.remove_count().await, 0);
19350 assert_eq!(backend.object_count().await, 1);
19351 }
19352
19353 #[cfg(feature = "test-util")]
19354 #[test]
19355 #[serial_test::serial(storage_class_env)]
19356 fn transitioned_history_expiry_uses_journal_or_legacy_free_version() {
19357 std::thread::Builder::new()
19358 .name("transitioned-delete-all-test".to_string())
19359 .stack_size(32 * 1024 * 1024)
19360 .spawn(|| {
19361 let runtime = tokio::runtime::Builder::new_multi_thread()
19362 .enable_all()
19363 .worker_threads(2)
19364 .build()
19365 .expect("test runtime should build");
19366 runtime.block_on(async {
19367 let temp_dir = tempfile::tempdir().expect("create transitioned delete-all store dir");
19368 let (ctx, store, _shutdown) =
19369 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "transitioned-delete-all", &[4]))
19370 .await;
19371 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
19372 let tier_name = "DELETEALLTRANSITIONED";
19373 let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
19374 let bucket = "transitioned-delete-all-bucket";
19375 let object = "object";
19376 store
19377 .make_bucket(bucket, &MakeBucketOptions::default())
19378 .await
19379 .expect("bucket should be created");
19380 crate::bucket::metadata_sys::update_in(
19381 &ctx,
19382 bucket,
19383 BUCKET_VERSIONING_CONFIG,
19384 b"<VersioningConfiguration><Status>Enabled</Status></VersioningConfiguration>".to_vec(),
19385 )
19386 .await
19387 .expect("bucket versioning should be enabled");
19388 crate::bucket::metadata_sys::update_in(
19389 &ctx,
19390 bucket,
19391 BUCKET_LIFECYCLE_CONFIG,
19392 br#"<LifecycleConfiguration>
19393 <Rule>
19394 <ID>delete-all-versions</ID>
19395 <Status>Enabled</Status>
19396 <Filter><Prefix></Prefix></Filter>
19397 <Expiration><Days>1</Days><ExpiredObjectAllVersions>true</ExpiredObjectAllVersions></Expiration>
19398 </Rule>
19399</LifecycleConfiguration>"#
19400 .to_vec(),
19401 )
19402 .await
19403 .expect("delete-all lifecycle should be configured");
19404
19405 let old_time = OffsetDateTime::now_utc() - time::Duration::days(3);
19406 let mut history_reader = PutObjReader::from_vec(b"transitioned history".to_vec());
19407 let history = store
19408 .put_object(
19409 bucket,
19410 object,
19411 &mut history_reader,
19412 &ObjectOptions {
19413 versioned: true,
19414 mod_time: Some(old_time - time::Duration::hours(1)),
19415 ..Default::default()
19416 },
19417 )
19418 .await
19419 .expect("historical version should be written");
19420 let mut current_reader = PutObjReader::from_vec(b"current version".to_vec());
19421 let current = store
19422 .put_object(
19423 bucket,
19424 object,
19425 &mut current_reader,
19426 &ObjectOptions {
19427 versioned: true,
19428 mod_time: Some(old_time),
19429 ..Default::default()
19430 },
19431 )
19432 .await
19433 .expect("current version should be written");
19434 store
19435 .transition_object(
19436 bucket,
19437 object,
19438 &ObjectOptions {
19439 versioned: true,
19440 version_id: history.version_id.map(|version_id| version_id.to_string()),
19441 transition: TransitionOptions {
19442 status: TRANSITION_PENDING.to_string(),
19443 tier: tier_name.to_string(),
19444 etag: history.etag.clone().expect("history should have an ETag"),
19445 ..Default::default()
19446 },
19447 mod_time: history.mod_time,
19448 ..Default::default()
19449 },
19450 )
19451 .await
19452 .expect("historical version should transition");
19453 assert_eq!(backend.object_count().await, 1);
19454 assert_eq!(backend.put_versions().await.len(), 1);
19455 backend.set_remove_failure(true);
19456
19457 let incarnation = store
19458 .bucket_incarnation_id_from_disk(bucket)
19459 .await
19460 .expect("bucket incarnation should be available");
19461 for disk_index in 0..4 {
19462 let metadata_path = temp_dir
19463 .path()
19464 .join(format!("pool0/set0/disk{disk_index}/{bucket}/{object}/{STORAGE_FORMAT_FILE}"));
19465 let encoded = tokio::fs::read(&metadata_path)
19466 .await
19467 .expect("transition metadata should be readable");
19468 let mut metadata = FileMeta::load(&encoded).expect("transition metadata should decode");
19469 let (version_index, mut transitioned) = metadata
19470 .find_version(history.version_id)
19471 .expect("transitioned history should exist");
19472 rustfs_utils::http::metadata_compat::remove_bytes(
19475 &mut transitioned.object.as_mut().expect("history should be an object").meta_sys,
19476 rustfs_utils::http::metadata_compat::SUFFIX_TRANSITIONED_VERSION_STATE,
19477 );
19478 metadata.versions[version_index] = rustfs_filemeta::FileMetaShallowVersion::try_from(transitioned)
19479 .expect("legacy history should re-encode");
19480 tokio::fs::write(
19481 &metadata_path,
19482 metadata.marshal_msg().expect("unknown transition metadata should encode"),
19483 )
19484 .await
19485 .expect("unknown transition metadata should be written");
19486 let encoded = tokio::fs::read(&metadata_path)
19487 .await
19488 .expect("legacy transition metadata should be readable");
19489 let legacy = FileMeta::load(&encoded)
19490 .expect("legacy transition metadata should decode")
19491 .find_version(history.version_id)
19492 .expect("legacy history should exist")
19493 .1
19494 .into_fileinfo(bucket, object, true)
19495 .expect("legacy history should decode");
19496 assert_eq!(legacy.transition_version_state, rustfs_filemeta::TransitionVersionState::Unknown);
19497 }
19498 let lifecycle_event = crate::bucket::lifecycle::lifecycle::Event {
19499 action: rustfs_scanner_metrics::metrics::IlmAction::DeleteAllVersionsAction,
19500 rule_id: "delete-all-versions".to_string(),
19501 ..Default::default()
19502 };
19503 let legacy_applied =
19504 crate::bucket::lifecycle::bucket_lifecycle_ops::apply_expiry_on_non_transitioned_objects(
19505 store.clone(),
19506 ¤t,
19507 &lifecycle_event,
19508 &crate::bucket::lifecycle::bucket_lifecycle_audit::LcEventSrc::Scanner,
19509 incarnation,
19510 )
19511 .await;
19512 assert!(legacy_applied, "legacy Unknown transition metadata should use the free-version fallback");
19513 assert_eq!(tier_delete_journal_count(store.clone()).await, 0);
19514 assert_eq!(backend.remove_count().await, 0);
19515 let retained = store.pools[0].disk_set[0]
19516 .load_file_info_versions_exact(bucket, object)
19517 .await
19518 .expect("fallback delete-all metadata should remain readable")
19519 .expect("fallback delete-all should retain a hidden cleanup owner");
19520 assert_eq!(
19521 retained
19522 .versions
19523 .iter()
19524 .filter(|version| !version.tier_free_version())
19525 .count(),
19526 0
19527 );
19528 assert_eq!(
19529 retained.versions.iter().filter(|version| version.tier_free_version()).count(),
19530 1,
19531 "legacy cleanup must retain exactly one hidden free-version"
19532 );
19533 assert_eq!(backend.object_count().await, 1, "legacy remote cleanup must remain pending");
19534
19535 let exact_tier_name = "DELETEALLTRANSITIONEDEXACT";
19539 let exact_backend = register_mock_tier(&ctx.tier_config_mgr(), exact_tier_name).await;
19540 let exact_object = "exact-object";
19541 let mut exact_history_reader = PutObjReader::from_vec(b"exact transitioned history".to_vec());
19542 let exact_history = store
19543 .put_object(
19544 bucket,
19545 exact_object,
19546 &mut exact_history_reader,
19547 &ObjectOptions {
19548 versioned: true,
19549 mod_time: Some(old_time - time::Duration::hours(1)),
19550 ..Default::default()
19551 },
19552 )
19553 .await
19554 .expect("second historical version should be written");
19555 let mut exact_current_reader = PutObjReader::from_vec(b"exact current version".to_vec());
19556 let exact_current = store
19557 .put_object(
19558 bucket,
19559 exact_object,
19560 &mut exact_current_reader,
19561 &ObjectOptions {
19562 versioned: true,
19563 mod_time: Some(old_time),
19564 ..Default::default()
19565 },
19566 )
19567 .await
19568 .expect("second current version should be written");
19569 store
19570 .transition_object(
19571 bucket,
19572 exact_object,
19573 &ObjectOptions {
19574 versioned: true,
19575 version_id: exact_history.version_id.map(|version_id| version_id.to_string()),
19576 transition: TransitionOptions {
19577 status: TRANSITION_PENDING.to_string(),
19578 tier: exact_tier_name.to_string(),
19579 etag: exact_history.etag.clone().expect("second history should have an ETag"),
19580 ..Default::default()
19581 },
19582 mod_time: exact_history.mod_time,
19583 ..Default::default()
19584 },
19585 )
19586 .await
19587 .expect("second historical version should transition exactly");
19588 let exact_remote_version = exact_backend
19589 .put_versions()
19590 .await
19591 .last()
19592 .cloned()
19593 .expect("second transition should publish a remote version");
19594 exact_backend.set_remove_failure(true);
19595 let applied = crate::bucket::lifecycle::bucket_lifecycle_ops::apply_expiry_on_non_transitioned_objects(
19596 store.clone(),
19597 &exact_current,
19598 &lifecycle_event,
19599 &crate::bucket::lifecycle::bucket_lifecycle_audit::LcEventSrc::Scanner,
19600 incarnation,
19601 )
19602 .await;
19603
19604 assert!(applied, "delete-all should remove current and transitioned history");
19605 assert!(
19606 store.pools[0].disk_set[0]
19607 .load_file_info_versions_exact(bucket, exact_object)
19608 .await
19609 .expect("exact journal-owned metadata lookup should succeed")
19610 .is_none(),
19611 "exact journal ownership must remove the source key without leaving a free-version"
19612 );
19613 let versions = store.pools[0].disk_set[0]
19614 .load_file_info_versions_exact(bucket, object)
19615 .await
19616 .expect("remaining fallback metadata should be readable")
19617 .expect("the prior legacy cleanup owner should remain");
19618 assert_eq!(
19619 versions
19620 .versions
19621 .iter()
19622 .filter(|version| !version.tier_free_version())
19623 .count(),
19624 0,
19625 "exact delete-all must remove every visible version"
19626 );
19627 assert_eq!(
19628 versions.versions.iter().filter(|version| version.tier_free_version()).count(),
19629 1,
19630 "the exact v6 journal must not add another free-version"
19631 );
19632 assert_eq!(tier_delete_journal_count(store.clone()).await, 1);
19633 assert_eq!(backend.object_count().await, 1, "legacy remote cleanup must remain pending");
19634 assert_eq!(exact_backend.object_count().await, 1, "exact remote cleanup must remain journal-owned");
19635
19636 exact_backend.set_remove_failure(false);
19637 let stats = recover_tier_delete_journal_entries(store.clone(), 100, None)
19638 .await
19639 .expect("committed journal should recover");
19640 assert_eq!(stats.failed, 0);
19641 assert!(stats.scanned <= 1 && stats.deleted <= 1);
19642 assert_eq!(tier_delete_journal_count(store).await, 0);
19643 assert_eq!(
19644 backend.object_count().await,
19645 1,
19646 "journal recovery must leave the legacy free-version remote"
19647 );
19648 assert_eq!(exact_backend.object_count().await, 0);
19649 assert!(exact_backend.exact_remove_count() >= 1);
19650 assert_eq!(exact_backend.remove_versions().await, vec![exact_remote_version]);
19651 });
19652 })
19653 .expect("test thread should spawn")
19654 .join()
19655 .expect("test thread should complete");
19656 }
19657
19658 #[cfg(feature = "test-util")]
19659 #[tokio::test]
19660 #[serial_test::serial(storage_class_env)]
19661 async fn odm_write_back_requires_one_set_and_enabled_namespace_locking() {
19662 for (layout, locking, supported) in [
19663 (&[(1, 4)][..], true, true),
19664 (&[(1, 4), (1, 4)][..], true, false),
19665 (&[(2, 4)][..], true, false),
19666 (&[(1, 4)][..], false, false),
19667 ] {
19668 temp_env::async_with_vars([("RUSTFS_LOCK_ENABLED", Some(if locking { "true" } else { "false" }))], async {
19669 let dir = tempfile::tempdir().expect("isolated topology");
19670 let shutdown = CancellationToken::new();
19671 let (_ctx, store, _) = without_storage_class_env(build_isolated_test_store_with_layout(
19672 dir.path(),
19673 "odm-topology",
19674 layout,
19675 shutdown.clone(),
19676 None,
19677 ))
19678 .await;
19679 assert_eq!(
19680 store.supports_atomic_create_only_write_back(),
19681 supported,
19682 "layout={layout:?}, locking={locking}"
19683 );
19684 shutdown.cancel();
19685 })
19686 .await;
19687 }
19688 }
19689
19690 #[cfg(feature = "test-util")]
19691 #[tokio::test]
19692 #[serial_test::serial(storage_class_env)]
19693 async fn dispatched_tier_delete_recovery_requires_namespace_locking() {
19694 temp_env::async_with_vars([("RUSTFS_LOCK_ENABLED", Some("false"))], async {
19695 let temp_dir = tempfile::tempdir().expect("create lock-disabled store dir");
19696 let (ctx, store, _shutdown) =
19697 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "prepared-recovery-lock-disabled", &[4]))
19698 .await;
19699 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
19700 assert!(ctx.lock_manager().is_disabled());
19701
19702 let tier_name = "PREPAREDLOCKDISABLED";
19703 let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
19704 let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
19705 .await
19706 .expect("tier lease should resolve")
19707 .backend_identity();
19708 let entry = Jentry {
19709 persisted_version: 0,
19710 obj_name: "remote/lock-disabled".to_string(),
19711 version_id: uuid::Uuid::new_v4().to_string(),
19712 tier_name: tier_name.to_string(),
19713 backend_identity: Some(backend_identity),
19714 version_id_exact: true,
19715 version_state: rustfs_filemeta::TransitionVersionState::Exact,
19716 state: TierDeleteJournalState::Prepared,
19717 source: Some(TierDeleteSourceIdentity {
19718 bucket: "absent-source-bucket".to_string(),
19719 object: "absent-source-object".to_string(),
19720 version_id: Some(uuid::Uuid::new_v4().to_string()),
19721 versioned: true,
19722 version_suspended: false,
19723 data_dir: Some(uuid::Uuid::new_v4().to_string()),
19724 etag: Some("etag".to_string()),
19725 mod_time: Some(OffsetDateTime::UNIX_EPOCH.to_string()),
19726 }),
19727 dispatch: None,
19728 };
19729 install_test_tier_delete_dispatch_fixture(
19730 store.clone(),
19731 "absent-source-bucket",
19732 uuid::Uuid::new_v4(),
19733 "absent-source-object",
19734 vec![(entry, Some(TierDeleteJournalState::Dispatched))],
19735 TierDeleteDispatchManifestState::DispatchAuthorized,
19736 )
19737 .await
19738 .expect("authorized dispatched journal should persist");
19739
19740 let stats = recover_tier_delete_journal_entries(store.clone(), 100, None)
19741 .await
19742 .expect("recovery scan should complete");
19743
19744 assert_eq!((stats.scanned, stats.deleted, stats.failed), (1, 0, 1));
19745 assert_eq!(tier_delete_journal_count(store).await, 1, "journal must remain dispatched for retry");
19746 assert_eq!(backend.remove_count().await, 0, "lock-disabled recovery must not delete remotely");
19747 })
19748 .await;
19749 }
19750
19751 #[cfg(feature = "test-util")]
19752 #[tokio::test]
19753 #[serial_test::serial(storage_class_env)]
19754 async fn lifecycle_delete_all_requires_namespace_locking_before_mutation() {
19755 temp_env::async_with_vars([("RUSTFS_LOCK_ENABLED", Some("false"))], async {
19756 let temp_dir = tempfile::tempdir().expect("create lock-disabled delete-all store dir");
19757 let (ctx, store, _shutdown) =
19758 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "delete-all-lock-disabled", &[4])).await;
19759 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
19760 assert!(ctx.lock_manager().is_disabled());
19761 let bucket = "delete-all-lock-disabled-bucket";
19762 let object = "object";
19763 store
19764 .make_bucket(bucket, &MakeBucketOptions::default())
19765 .await
19766 .expect("bucket should be created");
19767 let mut reader = PutObjReader::from_vec(b"must survive".to_vec());
19768 let no_lock_opts = ObjectOptions {
19769 no_lock: true,
19770 ..Default::default()
19771 };
19772 let original = store.pools[0]
19773 .put_object(bucket, object, &mut reader, &no_lock_opts)
19774 .await
19775 .expect("source should be written");
19776
19777 let mut delete_opts = ObjectOptions {
19778 delete_prefix: true,
19779 delete_prefix_object: true,
19780 lifecycle_delete_all: Some(crate::object_api::LifecycleDeleteAllRequest {
19781 version_id: original.version_id,
19782 delete_marker: false,
19783 action: rustfs_scanner_metrics::metrics::IlmAction::DeleteAllVersionsAction,
19784 rule_id: "rule".to_string(),
19785 phase: crate::object_api::LifecycleDeleteAllPhase::Preflight,
19786 }),
19787 delete_replication_config_snapshot: Some(Arc::new(
19788 crate::bucket::replication::DeleteReplicationConfigSnapshot::default(),
19789 )),
19790 ..Default::default()
19791 };
19792 delete_opts.ensure_lifecycle_delete_all_journal();
19793
19794 let err = store
19795 .delete_object_with_tier_delete_journal(bucket, object, delete_opts)
19796 .await
19797 .expect_err("delete-all must reject disabled namespace locking");
19798
19799 assert!(err.to_string().contains("requires namespace locking"));
19800 let retained = store.pools[0]
19801 .get_object_info(bucket, object, &no_lock_opts)
19802 .await
19803 .expect("rejected delete-all must retain the source");
19804 assert_eq!(retained.etag, original.etag);
19805 assert_eq!(tier_delete_journal_count(store).await, 0, "rejected delete-all must not prepare journals");
19806 })
19807 .await;
19808 }
19809
19810 #[cfg(feature = "test-util")]
19811 #[tokio::test]
19812 #[serial_test::serial(storage_class_env)]
19813 async fn tier_mutation_intent_record_round_trips_through_config_store() {
19814 let temp_dir = tempfile::tempdir().expect("create temp store dir");
19815 let (_ctx, store, _shutdown) =
19816 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "tier-mutation-intent-record", &[4])).await;
19817 let mutation_id = uuid::Uuid::new_v4();
19818 let intent = TierMutationIntent {
19819 mutation_id,
19820 revision: 1,
19821 kind: TierMutationIntentKind::Edit,
19822 state: TierMutationIntentState::Prepared,
19823 old_config_etag: Some("old-etag".to_string()),
19824 committed_config_etag: None,
19825 candidate_digest: [3; 32],
19826 affected_targets: vec![TierMutationIntentTarget {
19827 tier_name: "COLD-A".to_string(),
19828 old_backend_identity: Some([1; 32]),
19829 new_backend_identity: Some([2; 32]),
19830 }],
19831 expires_at_unix_nanos: 1_780_000_000_000_000_000,
19832 };
19833
19834 save_tier_mutation_intent_record(store.clone(), &intent)
19835 .await
19836 .expect("tier mutation intent record should persist");
19837 let loaded = load_tier_mutation_intent_record(store.clone(), mutation_id)
19838 .await
19839 .expect("tier mutation intent record should load");
19840
19841 assert_eq!(loaded, intent);
19842
19843 delete_tier_mutation_intent_record(store.clone(), mutation_id)
19844 .await
19845 .expect("tier mutation intent record delete should be idempotent");
19846 delete_tier_mutation_intent_record(store.clone(), mutation_id)
19847 .await
19848 .expect("tier mutation intent record delete should tolerate missing records");
19849 let err = load_tier_mutation_intent_record(store, mutation_id)
19850 .await
19851 .expect_err("deleted tier mutation intent record should not load");
19852 assert!(matches!(err, Error::ConfigNotFound));
19853 }
19854
19855 #[cfg(feature = "test-util")]
19856 #[tokio::test]
19857 #[serial_test::serial(storage_class_env)]
19858 async fn tier_probe_intent_store_enforces_create_cas_and_terminal_delete_preconditions() {
19859 let temp_dir = tempfile::tempdir().expect("create temp store dir");
19860 let (_ctx, store, _shutdown) =
19861 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "tier-probe-intent-cas", &[4])).await;
19862 let probe_id = uuid::Uuid::new_v4();
19863 let creator_epoch = uuid::Uuid::new_v4();
19864 let initial = TierProbeIntent {
19865 probe_id,
19866 revision: 1,
19867 state: TierProbeIntentState::UploadOutcomeUnknown,
19868 operation: TierProbeOperationIdentity::Verify {
19869 config_etag: "config-etag".to_string(),
19870 backend_identity: [1; 32],
19871 },
19872 tier_name: "COLD-A".to_string(),
19873 destination_id: [1; 32],
19874 probe_object: format!("rustfs-tier-probe-{probe_id}"),
19875 creator_id: "node-a".to_string(),
19876 creator_epoch,
19877 created_at_unix_nanos: 1_780_000_000_000_000_000,
19878 owner: TierProbeOwnerFence {
19879 owner_id: "node-a".to_string(),
19880 owner_epoch: creator_epoch,
19881 not_after_unix_nanos: 1_780_000_900_000_000_000,
19882 },
19883 remote_version: TierProbeRemoteVersion::default(),
19884 };
19885
19886 save_tier_probe_intent_record_if_absent(store.clone(), &initial)
19887 .await
19888 .expect("initial probe intent should persist with create-only semantics");
19889 let duplicate = save_tier_probe_intent_record_if_absent(store.clone(), &initial)
19890 .await
19891 .expect_err("duplicate create must fail closed");
19892 assert!(matches!(duplicate, Error::PreconditionFailed));
19893
19894 let observed_initial = load_tier_probe_intent_record(store.clone(), probe_id)
19895 .await
19896 .expect("initial probe intent should load with an ETag");
19897 assert_eq!(observed_initial.intent(), &initial);
19898
19899 let nonterminal_delete = delete_tier_probe_intent_record_if_current(store.clone(), &observed_initial)
19900 .await
19901 .expect_err("nonterminal evidence must not be deleted");
19902 assert!(nonterminal_delete.to_string().contains("must be terminal"));
19903
19904 let mut fabricated_current_intent = initial.clone();
19905 fabricated_current_intent.tier_name = "COLD-B".to_string();
19906 let mut fabricated_successor = fabricated_current_intent.clone();
19907 fabricated_successor
19908 .advance(
19909 TierProbeIntentState::Uploaded,
19910 TierProbeRemoteVersion::versioned(uuid::Uuid::new_v4().to_string()),
19911 )
19912 .expect("fabricated successor should be internally valid");
19913 let fabricated_current = observed_initial.with_intent_for_test(fabricated_current_intent.clone());
19914 let crossed_cas = save_tier_probe_intent_record_if_current(store.clone(), &fabricated_current, &fabricated_successor)
19915 .await
19916 .expect_err("a live ETag must not authorize a different caller record");
19917 assert!(matches!(crossed_cas, Error::PreconditionFailed));
19918 assert_eq!(
19919 load_tier_probe_intent_record(store.clone(), probe_id)
19920 .await
19921 .expect("crossed CAS must retain the authoritative record")
19922 .intent(),
19923 &initial
19924 );
19925
19926 let mut fabricated_terminal_intent = fabricated_current_intent;
19927 fabricated_terminal_intent
19928 .advance(TierProbeIntentState::AbortedNoRemote, TierProbeRemoteVersion::default())
19929 .expect("fabricated terminal should be internally valid");
19930 let fabricated_terminal = observed_initial.with_intent_for_test(fabricated_terminal_intent);
19931 let crossed_delete = delete_tier_probe_intent_record_if_current(store.clone(), &fabricated_terminal)
19932 .await
19933 .expect_err("a live ETag must not delete for a different caller record");
19934 assert!(matches!(crossed_delete, Error::PreconditionFailed));
19935 assert_eq!(
19936 load_tier_probe_intent_record(store.clone(), probe_id)
19937 .await
19938 .expect("crossed delete must retain the authoritative record")
19939 .intent(),
19940 &initial
19941 );
19942
19943 let remote_version = TierProbeRemoteVersion::versioned(uuid::Uuid::new_v4().to_string());
19944 let mut uploaded = observed_initial.intent().clone();
19945 uploaded
19946 .advance(TierProbeIntentState::Uploaded, remote_version.clone())
19947 .expect("known PUT result should advance");
19948 save_tier_probe_intent_record_if_current(store.clone(), &observed_initial, &uploaded)
19949 .await
19950 .expect("the matching initial ETag should admit one successor");
19951
19952 let stale_cas = save_tier_probe_intent_record_if_current(store.clone(), &observed_initial, &uploaded)
19953 .await
19954 .expect_err("a consumed ETag must not overwrite the current generation");
19955 assert!(matches!(stale_cas, Error::PreconditionFailed));
19956
19957 let observed_uploaded = load_tier_probe_intent_record(store.clone(), probe_id)
19958 .await
19959 .expect("uploaded generation should load");
19960 assert_eq!(observed_uploaded.intent(), &uploaded);
19961 let mut cleanup = observed_uploaded.intent().clone();
19962 cleanup
19963 .advance(TierProbeIntentState::CleanupPending, remote_version.clone())
19964 .expect("known candidate should become cleanup-pending");
19965 save_tier_probe_intent_record_if_current(store.clone(), &observed_uploaded, &cleanup)
19966 .await
19967 .expect("cleanup generation should persist by exact ETag");
19968
19969 let observed_cleanup = load_tier_probe_intent_record(store.clone(), probe_id)
19970 .await
19971 .expect("cleanup generation should load");
19972 let mut completed = observed_cleanup.intent().clone();
19973 completed
19974 .advance(TierProbeIntentState::Completed, remote_version)
19975 .expect("exact cleanup should become terminal");
19976 save_tier_probe_intent_record_if_current(store.clone(), &observed_cleanup, &completed)
19977 .await
19978 .expect("terminal generation should persist by exact ETag");
19979
19980 let stale_terminal = observed_cleanup.with_intent_for_test(completed.clone());
19981 let stale_delete = delete_tier_probe_intent_record_if_current(store.clone(), &stale_terminal)
19982 .await
19983 .expect_err("a stale ETag must not delete terminal evidence");
19984 assert!(matches!(stale_delete, Error::PreconditionFailed));
19985
19986 let observed_completed = load_tier_probe_intent_record(store.clone(), probe_id)
19987 .await
19988 .expect("terminal generation should remain after stale delete");
19989 assert_eq!(observed_completed.intent(), &completed);
19990 delete_tier_probe_intent_record_if_current(store.clone(), &observed_completed)
19991 .await
19992 .expect("the exact terminal ETag should delete the record");
19993 assert!(matches!(load_tier_probe_intent_record(store, probe_id).await, Err(Error::ConfigNotFound)));
19994 }
19995
19996 #[cfg(feature = "test-util")]
19997 #[tokio::test]
19998 #[serial_test::serial(storage_class_env)]
19999 async fn tier_mutation_intent_record_scan_retains_good_records_and_counts_bad_records() {
20000 let temp_dir = tempfile::tempdir().expect("create temp store dir");
20001 let (_ctx, store, _shutdown) =
20002 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "tier-mutation-intent-scan", &[4])).await;
20003 let build_intent = |mutation_id: uuid::Uuid, tier_name: &str| TierMutationIntent {
20004 mutation_id,
20005 revision: 1,
20006 kind: TierMutationIntentKind::Edit,
20007 state: TierMutationIntentState::Prepared,
20008 old_config_etag: Some("old-etag".to_string()),
20009 committed_config_etag: None,
20010 candidate_digest: [3; 32],
20011 affected_targets: vec![TierMutationIntentTarget {
20012 tier_name: tier_name.to_string(),
20013 old_backend_identity: Some([1; 32]),
20014 new_backend_identity: Some([2; 32]),
20015 }],
20016 expires_at_unix_nanos: 1_780_000_000_000_000_000,
20017 };
20018 let first_id = uuid::Uuid::parse_str("12345678-1234-5678-9abc-def012345678").expect("first uuid should parse");
20019 let second_id = uuid::Uuid::parse_str("22345678-1234-5678-9abc-def012345678").expect("second uuid should parse");
20020 let first = build_intent(first_id, "COLD-A");
20021 let second = build_intent(second_id, "COLD-B");
20022 save_tier_mutation_intent_record(store.clone(), &first)
20023 .await
20024 .expect("first tier mutation intent record should persist");
20025 save_tier_mutation_intent_record(store.clone(), &second)
20026 .await
20027 .expect("second tier mutation intent record should persist");
20028 com::save_config(
20029 store.clone(),
20030 &format!("{TIER_MUTATION_INTENT_RECORD_PREFIX}/00/00/33345678123456789abcdef012345678.json"),
20031 b"{}".to_vec(),
20032 )
20033 .await
20034 .expect("malformed-shard intent record should persist");
20035 com::save_config(
20036 store.clone(),
20037 &format!("{TIER_MUTATION_INTENT_RECORD_PREFIX}/44/44/44445678123456789abcdef012345678.json"),
20038 b"{".to_vec(),
20039 )
20040 .await
20041 .expect("corrupt-json intent record should persist");
20042
20043 let scan = list_tier_mutation_intent_records(store, 100, None)
20044 .await
20045 .expect("tier mutation intent records should scan");
20046 let mut loaded_ids: Vec<_> = scan.intents.into_iter().map(|intent| intent.mutation_id).collect();
20047 loaded_ids.sort();
20048
20049 assert_eq!(scan.scanned, 4);
20050 assert_eq!(scan.failed, 2);
20051 assert_eq!(loaded_ids, vec![first_id, second_id]);
20052 assert!(!scan.truncated);
20053 assert_eq!(scan.next_marker, None);
20054 }
20055
20056 #[cfg(feature = "test-util")]
20057 #[tokio::test]
20058 #[serial_test::serial(storage_class_env)]
20059 async fn tier_mutation_intent_record_advance_is_idempotent_in_config_store() {
20060 let temp_dir = tempfile::tempdir().expect("create temp store dir");
20061 let (_ctx, store, _shutdown) =
20062 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "tier-mutation-intent-advance", &[4])).await;
20063 let mutation_id = uuid::Uuid::new_v4();
20064 let intent = TierMutationIntent {
20065 mutation_id,
20066 revision: 1,
20067 kind: TierMutationIntentKind::Edit,
20068 state: TierMutationIntentState::Prepared,
20069 old_config_etag: Some("old-etag".to_string()),
20070 committed_config_etag: None,
20071 candidate_digest: [3; 32],
20072 affected_targets: vec![TierMutationIntentTarget {
20073 tier_name: "COLD-A".to_string(),
20074 old_backend_identity: Some([1; 32]),
20075 new_backend_identity: Some([2; 32]),
20076 }],
20077 expires_at_unix_nanos: 1_780_000_000_000_000_000,
20078 };
20079 save_tier_mutation_intent_record(store.clone(), &intent)
20080 .await
20081 .expect("prepared tier mutation intent record should persist");
20082 let (loaded_before_advance, stale_etag) = load_tier_mutation_intent_record_with_etag(store.clone(), mutation_id)
20083 .await
20084 .expect("prepared tier mutation intent record should load with etag");
20085 assert_eq!(loaded_before_advance, intent);
20086 assert!(!stale_etag.is_empty());
20087
20088 let (committed, first_advanced) = advance_tier_mutation_intent_record_idempotent(
20089 store.clone(),
20090 mutation_id,
20091 TierMutationIntentState::Committed,
20092 Some("new-etag".to_string()),
20093 )
20094 .await
20095 .expect("first commit should advance the record");
20096 assert!(first_advanced);
20097 assert_eq!(committed.state, TierMutationIntentState::Committed);
20098 assert_eq!(committed.revision, 2);
20099 assert_eq!(committed.committed_config_etag.as_deref(), Some("new-etag"));
20100
20101 let (retried, retry_advanced) = advance_tier_mutation_intent_record_idempotent(
20102 store.clone(),
20103 mutation_id,
20104 TierMutationIntentState::Committed,
20105 Some("new-etag".to_string()),
20106 )
20107 .await
20108 .expect("same commit retry should be idempotent");
20109 assert!(!retry_advanced);
20110 assert_eq!(retried, committed);
20111
20112 let conflict = advance_tier_mutation_intent_record_idempotent(
20113 store.clone(),
20114 mutation_id,
20115 TierMutationIntentState::Committed,
20116 Some("other-etag".to_string()),
20117 )
20118 .await
20119 .expect_err("conflicting commit retry should fail closed");
20120 assert!(matches!(conflict, Error::Io(_)));
20121 assert!(conflict.to_string().contains("committed config etag does not match"));
20122
20123 let mut stale_conflict = intent;
20124 stale_conflict
20125 .advance_idempotent(TierMutationIntentState::Committed, Some("other-etag".to_string()))
20126 .expect("stale conflicting intent should advance locally");
20127 let stale_save = save_tier_mutation_intent_record_if_current(store.clone(), &stale_conflict, &stale_etag)
20128 .await
20129 .expect_err("stale etag must fail closed instead of overwriting the committed record");
20130 assert!(matches!(stale_save, Error::PreconditionFailed));
20131
20132 let loaded = load_tier_mutation_intent_record(store.clone(), mutation_id)
20133 .await
20134 .expect("conflicting retry must not overwrite the durable record");
20135 assert_eq!(loaded, committed);
20136
20137 let abort_id = uuid::Uuid::new_v4();
20138 let abort_intent = TierMutationIntent {
20139 mutation_id: abort_id,
20140 revision: 1,
20141 kind: TierMutationIntentKind::Edit,
20142 state: TierMutationIntentState::Prepared,
20143 old_config_etag: Some("old-etag".to_string()),
20144 committed_config_etag: None,
20145 candidate_digest: [4; 32],
20146 affected_targets: vec![TierMutationIntentTarget {
20147 tier_name: "COLD-B".to_string(),
20148 old_backend_identity: Some([1; 32]),
20149 new_backend_identity: Some([2; 32]),
20150 }],
20151 expires_at_unix_nanos: 1_780_000_000_000_000_000,
20152 };
20153 save_tier_mutation_intent_record(store.clone(), &abort_intent)
20154 .await
20155 .expect("prepared abort intent record should persist");
20156
20157 let (aborted, first_abort_advanced) =
20158 advance_tier_mutation_intent_record_idempotent(store.clone(), abort_id, TierMutationIntentState::Aborted, None)
20159 .await
20160 .expect("first abort should advance the record");
20161 assert!(first_abort_advanced);
20162 assert_eq!(aborted.state, TierMutationIntentState::Aborted);
20163 assert_eq!(aborted.revision, 2);
20164 assert_eq!(aborted.committed_config_etag, None);
20165
20166 let (aborted_retry, retry_abort_advanced) =
20167 advance_tier_mutation_intent_record_idempotent(store, abort_id, TierMutationIntentState::Aborted, None)
20168 .await
20169 .expect("same abort retry should be idempotent");
20170 assert!(!retry_abort_advanced);
20171 assert_eq!(aborted_retry, aborted);
20172 }
20173
20174 #[cfg(feature = "test-util")]
20175 fn tier_mutation_peer_test_intent(
20176 mutation_id: uuid::Uuid,
20177 tier_name: &str,
20178 candidate_digest: [u8; 32],
20179 ) -> TierMutationIntent {
20180 TierMutationIntent {
20181 mutation_id,
20182 revision: 1,
20183 kind: TierMutationIntentKind::Edit,
20184 state: TierMutationIntentState::Prepared,
20185 old_config_etag: Some("old-etag".to_string()),
20186 committed_config_etag: None,
20187 candidate_digest,
20188 affected_targets: vec![TierMutationIntentTarget {
20189 tier_name: tier_name.to_string(),
20190 old_backend_identity: Some([1; 32]),
20191 new_backend_identity: Some([2; 32]),
20192 }],
20193 expires_at_unix_nanos: i64::MAX,
20194 }
20195 }
20196
20197 #[cfg(feature = "test-util")]
20198 #[tokio::test]
20199 #[serial_test::serial(storage_class_env)]
20200 async fn tier_mutation_abort_tombstone_rejects_delayed_prepare() {
20201 let temp_dir = tempfile::tempdir().expect("create temp store dir");
20202 let (_ctx, store, _shutdown) =
20203 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "tier-mutation-abort-tombstone", &[4])).await;
20204 register_mock_tier(&store.tier_config_mgr(), "COLD-A").await;
20205 store
20206 .tier_config_mgr()
20207 .read()
20208 .await
20209 .save_tiering_config(store.clone())
20210 .await
20211 .expect("base tier config should persist");
20212 let config_info = store
20213 .get_object_info(
20214 RUSTFS_META_BUCKET,
20215 &format!("{}/{}", com::CONFIG_PREFIX, TIER_CONFIG_FILE),
20216 &ObjectOptions::default(),
20217 )
20218 .await
20219 .expect("base tier config metadata should load");
20220 let mutation_id = uuid::Uuid::new_v4();
20221 let mut intent = tier_mutation_peer_test_intent(mutation_id, "COLD-A", [7; 32]);
20222 intent.old_config_etag = config_info.etag;
20223 intent.expires_at_unix_nanos =
20224 i64::try_from((OffsetDateTime::now_utc() + time::Duration::hours(1)).unix_timestamp_nanos())
20225 .expect("future tombstone expiry should fit i64");
20226 let payload = intent.encode().expect("prepare identity should encode");
20227
20228 let mismatch_id = uuid::Uuid::new_v4();
20229 let mut mismatch = intent.clone();
20230 mismatch.mutation_id = mismatch_id;
20231 mismatch.old_config_etag = Some("not-the-current-config-etag".to_string());
20232 let mismatch_error = handle_tier_mutation_peer_request(
20233 store.clone(),
20234 TIER_MUTATION_RPC_PROTOCOL_VERSION,
20235 TierMutationRpcPhase::Abort,
20236 mismatch_id,
20237 &mismatch.encode().expect("proof-mismatch Abort should encode"),
20238 )
20239 .await
20240 .expect_err("a missing-record Abort with the wrong config proof must fail closed");
20241 assert!(matches!(mismatch_error, TierMutationPeerError::AbortProofMismatch));
20242 assert!(matches!(
20243 load_tier_mutation_intent_record(store.clone(), mismatch_id).await,
20244 Err(Error::ConfigNotFound)
20245 ));
20246
20247 let aborted = handle_tier_mutation_peer_request(
20248 store.clone(),
20249 TIER_MUTATION_RPC_PROTOCOL_VERSION,
20250 TierMutationRpcPhase::Abort,
20251 mutation_id,
20252 &payload,
20253 )
20254 .await
20255 .expect("missing-record abort should persist a tombstone");
20256 assert!(aborted.applied);
20257 assert_eq!(aborted.state, TierMutationPeerState::Aborted);
20258 let tombstone = load_tier_mutation_intent_record(store.clone(), mutation_id)
20259 .await
20260 .expect("abort tombstone should be durable");
20261 assert_eq!(tombstone.state, TierMutationIntentState::Aborted);
20262 assert!(tombstone.same_identity_as(&intent));
20263
20264 let manager = store.tier_config_mgr();
20265 TierConfigMgr::reload_handle(&manager, store.clone())
20266 .await
20267 .expect("reload should retain an unexpired abort tombstone");
20268 let tombstone_after_reload = load_tier_mutation_intent_record(store.clone(), mutation_id)
20269 .await
20270 .expect("reload must not clean an unexpired abort tombstone");
20271 assert_eq!(tombstone_after_reload.state, TierMutationIntentState::Aborted);
20272 assert!(tombstone_after_reload.same_identity_as(&intent));
20273
20274 register_mock_tier(&store.tier_config_mgr(), "COLD-B").await;
20275 store
20276 .tier_config_mgr()
20277 .read()
20278 .await
20279 .save_tiering_config(store.clone())
20280 .await
20281 .expect("later config change should persist before idempotent Abort replay");
20282 let retried_abort = handle_tier_mutation_peer_request(
20283 store.clone(),
20284 TIER_MUTATION_RPC_PROTOCOL_VERSION,
20285 TierMutationRpcPhase::Abort,
20286 mutation_id,
20287 &payload,
20288 )
20289 .await
20290 .expect("terminal Abort retry must not depend on the mutable current config proof");
20291 assert!(!retried_abort.applied);
20292 assert_eq!(retried_abort.state, TierMutationPeerState::Aborted);
20293
20294 let delayed = handle_tier_mutation_peer_request(
20295 store.clone(),
20296 TIER_MUTATION_RPC_PROTOCOL_VERSION,
20297 TierMutationRpcPhase::Prepare,
20298 mutation_id,
20299 &payload,
20300 )
20301 .await
20302 .expect("delayed matching prepare should converge on the tombstone");
20303 assert!(!delayed.applied);
20304 assert_eq!(delayed.state, TierMutationPeerState::Aborted);
20305 TierConfigMgr::acquire_operation_lease(&store.tier_config_mgr(), "COLD-A")
20306 .await
20307 .expect("delayed prepare must not reinstall the runtime block");
20308
20309 let mut conflicting = intent.clone();
20310 conflicting.candidate_digest = [8; 32];
20311 let conflict = handle_tier_mutation_peer_request(
20312 store,
20313 TIER_MUTATION_RPC_PROTOCOL_VERSION,
20314 TierMutationRpcPhase::Prepare,
20315 mutation_id,
20316 &conflicting.encode().expect("conflicting prepare should encode"),
20317 )
20318 .await
20319 .expect_err("different identity must not reuse an abort tombstone");
20320 assert!(matches!(conflict, TierMutationPeerError::ConflictingIntent));
20321 }
20322
20323 #[cfg(feature = "test-util")]
20324 #[tokio::test]
20325 #[serial_test::serial(storage_class_env)]
20326 async fn tier_mutation_expired_prepare_cannot_recreate_a_cleaned_tombstone() {
20327 let temp_dir = tempfile::tempdir().expect("create temp store dir");
20328 let (_ctx, store, _shutdown) =
20329 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "tier-mutation-expired-replay", &[4])).await;
20330 register_mock_tier(&store.tier_config_mgr(), "COLD-A").await;
20331 let mutation_id = uuid::Uuid::new_v4();
20332 let mut intent = tier_mutation_peer_test_intent(mutation_id, "COLD-A", [11; 32]);
20333 intent.expires_at_unix_nanos =
20334 i64::try_from((OffsetDateTime::now_utc() - time::Duration::seconds(1)).unix_timestamp_nanos())
20335 .expect("expired timestamp should fit i64");
20336 let payload = intent.encode().expect("expired signed Prepare should still encode");
20337
20338 let error = handle_tier_mutation_peer_request(
20339 store.clone(),
20340 TIER_MUTATION_RPC_PROTOCOL_VERSION,
20341 TierMutationRpcPhase::Prepare,
20342 mutation_id,
20343 &payload,
20344 )
20345 .await
20346 .expect_err("a missing-record expired Prepare must fail closed");
20347 assert!(matches!(error, TierMutationPeerError::ExpiredIntent));
20348 assert!(matches!(
20349 load_tier_mutation_intent_record(store.clone(), mutation_id).await,
20350 Err(Error::ConfigNotFound)
20351 ));
20352 TierConfigMgr::acquire_operation_lease(&store.tier_config_mgr(), "COLD-A")
20353 .await
20354 .expect("expired replay must not install a peer-only runtime fence");
20355 }
20356
20357 #[cfg(feature = "test-util")]
20358 #[tokio::test]
20359 #[serial_test::serial(storage_class_env)]
20360 async fn tier_mutation_peer_prepare_waits_for_inflight_reference_lease() {
20361 let temp_dir = tempfile::tempdir().expect("create temp store dir");
20362 let (_ctx, store, _shutdown) =
20363 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "tier-mutation-peer-drain", &[4])).await;
20364 register_mock_tier(&store.tier_config_mgr(), "COLD-A").await;
20365 let old_lease = TierConfigMgr::acquire_operation_lease(&store.tier_config_mgr(), "COLD-A")
20366 .await
20367 .expect("old tier operation lease should be available");
20368 let mutation_id = uuid::Uuid::new_v4();
20369 let mut intent = tier_mutation_peer_test_intent(mutation_id, "COLD-A", [9; 32]);
20370 intent.affected_targets[0].old_backend_identity = Some(old_lease.backend_identity());
20371 let payload = intent.encode().expect("prepare intent should encode");
20372 let prepare_store = store.clone();
20373 let prepare = tokio::spawn(async move {
20374 handle_tier_mutation_peer_request(
20375 prepare_store,
20376 TIER_MUTATION_RPC_PROTOCOL_VERSION,
20377 TierMutationRpcPhase::Prepare,
20378 mutation_id,
20379 &payload,
20380 )
20381 .await
20382 });
20383
20384 tokio::time::timeout(Duration::from_secs(5), async {
20385 loop {
20386 if TierConfigMgr::acquire_operation_lease(&store.tier_config_mgr(), "COLD-A")
20387 .await
20388 .is_err()
20389 {
20390 return;
20391 }
20392 tokio::task::yield_now().await;
20393 }
20394 })
20395 .await
20396 .expect("peer Prepare should install its runtime block");
20397 assert!(
20398 !prepare.is_finished(),
20399 "peer Prepare must not acknowledge while a pre-existing reference creator still holds a lease"
20400 );
20401
20402 drop(old_lease);
20403 let outcome = prepare
20404 .await
20405 .expect("peer Prepare task should join")
20406 .expect("peer Prepare should finish after the lease drains");
20407 assert_eq!(outcome.state, TierMutationPeerState::Prepared);
20408 TierConfigMgr::clear_prepared_mutation_intent_block(&store.tier_config_mgr(), mutation_id)
20409 .await
20410 .expect("test should clear the prepared runtime block");
20411 }
20412
20413 #[cfg(feature = "test-util")]
20414 #[test]
20415 #[serial_test::serial(storage_class_env)]
20416 fn tier_mutation_peer_handler_applies_prepare_commit_and_abort_idempotently() {
20417 std::thread::Builder::new()
20422 .name("tier-mutation-peer-handler-test".to_string())
20423 .stack_size(32 * 1024 * 1024)
20424 .spawn(|| {
20425 let runtime = tokio::runtime::Builder::new_multi_thread()
20426 .enable_all()
20427 .worker_threads(2)
20428 .build()
20429 .expect("tier mutation peer handler test runtime should build");
20430 runtime.block_on(tier_mutation_peer_handler_case());
20431 })
20432 .expect("tier mutation peer handler test thread should spawn")
20433 .join()
20434 .expect("tier mutation peer handler test thread should complete");
20435 }
20436
20437 #[cfg(feature = "test-util")]
20438 async fn tier_mutation_peer_handler_case() {
20439 let temp_dir = tempfile::tempdir().expect("create temp store dir");
20440 let (_ctx, store, _shutdown) =
20441 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "tier-mutation-peer-handler", &[4])).await;
20442 register_mock_tier(&store.tier_config_mgr(), "COLD-A").await;
20443 let (candidate_digest, config_etag) = {
20444 let tier_config_mgr = store.tier_config_mgr();
20445 let manager = tier_config_mgr.read().await;
20446 let candidate_digest = tier_config_candidate_digest(&manager).expect("peer commit candidate digest should build");
20447 manager
20448 .save_tiering_config(store.clone())
20449 .await
20450 .expect("peer commit config fixture should persist");
20451 let config_info = store
20452 .get_object_info(
20453 RUSTFS_META_BUCKET,
20454 &format!("{}/{}", com::CONFIG_PREFIX, TIER_CONFIG_FILE),
20455 &ObjectOptions::default(),
20456 )
20457 .await
20458 .expect("peer commit config fixture should load");
20459 (candidate_digest, config_info.etag.expect("peer commit config should carry an ETag"))
20460 };
20461 let mutation_id = uuid::Uuid::new_v4();
20462 let mut intent = tier_mutation_peer_test_intent(mutation_id, "COLD-A", candidate_digest);
20463 intent.old_config_etag = Some(config_etag.clone());
20464 let prepare_payload = intent.encode().expect("prepare intent should encode");
20465
20466 let prepared = handle_tier_mutation_peer_request(
20467 store.clone(),
20468 TIER_MUTATION_RPC_PROTOCOL_VERSION,
20469 TierMutationRpcPhase::Prepare,
20470 mutation_id,
20471 &prepare_payload,
20472 )
20473 .await
20474 .expect("first prepare should create the peer intent");
20475 assert!(prepared.applied);
20476 assert_eq!(prepared.state, TierMutationPeerState::Prepared);
20477 let blocked = match TierConfigMgr::acquire_operation_lease(&store.tier_config_mgr(), "COLD-A").await {
20478 Ok(_) => panic!("prepared peer mutation should block new tier operation leases"),
20479 Err(err) => err,
20480 };
20481 assert!(
20482 blocked.message.contains("being replaced"),
20483 "prepared peer mutation should reuse the existing blocked-tier error: {blocked}"
20484 );
20485
20486 let retried_prepare = handle_tier_mutation_peer_request(
20487 store.clone(),
20488 TIER_MUTATION_RPC_PROTOCOL_VERSION,
20489 TierMutationRpcPhase::Prepare,
20490 mutation_id,
20491 &prepare_payload,
20492 )
20493 .await
20494 .expect("same prepare retry should be idempotent");
20495 assert!(!retried_prepare.applied);
20496 assert_eq!(retried_prepare.state, TierMutationPeerState::Prepared);
20497 let retried_blocked = match TierConfigMgr::acquire_operation_lease(&store.tier_config_mgr(), "COLD-A").await {
20498 Ok(_) => panic!("prepared retry should keep blocking new tier operation leases"),
20499 Err(err) => err,
20500 };
20501 assert!(
20502 retried_blocked.message.contains("being replaced"),
20503 "prepared retry should keep the existing blocked-tier error: {retried_blocked}"
20504 );
20505
20506 let mismatched_commit = handle_tier_mutation_peer_request(
20507 store.clone(),
20508 TIER_MUTATION_RPC_PROTOCOL_VERSION,
20509 TierMutationRpcPhase::Commit,
20510 mutation_id,
20511 b"not-the-current-etag",
20512 )
20513 .await
20514 .expect_err("commit with a mismatched config proof must fail closed");
20515 assert!(matches!(mismatched_commit, TierMutationPeerError::CommitProofMismatch));
20516
20517 register_mock_tier(&store.tier_config_mgr(), "COLD-C").await;
20518 let bad_digest_id = uuid::Uuid::new_v4();
20519 let mut bad_digest_intent = tier_mutation_peer_test_intent(bad_digest_id, "COLD-C", [9; 32]);
20520 bad_digest_intent.old_config_etag = Some(config_etag.clone());
20521 handle_tier_mutation_peer_request(
20522 store.clone(),
20523 TIER_MUTATION_RPC_PROTOCOL_VERSION,
20524 TierMutationRpcPhase::Prepare,
20525 bad_digest_id,
20526 &bad_digest_intent.encode().expect("bad digest prepare intent should encode"),
20527 )
20528 .await
20529 .expect("bad digest prepare should install a prepared intent");
20530 let mismatched_digest_commit = handle_tier_mutation_peer_request(
20531 store.clone(),
20532 TIER_MUTATION_RPC_PROTOCOL_VERSION,
20533 TierMutationRpcPhase::Commit,
20534 bad_digest_id,
20535 config_etag.as_bytes(),
20536 )
20537 .await
20538 .expect_err("a correct ETag with a mismatched candidate digest must fail closed");
20539 assert!(matches!(mismatched_digest_commit, TierMutationPeerError::CommitProofMismatch));
20540 let bad_digest_loaded = load_tier_mutation_intent_record(store.clone(), bad_digest_id)
20541 .await
20542 .expect("mismatched digest must leave the prepared intent durable");
20543 assert_eq!(bad_digest_loaded.state, TierMutationIntentState::Prepared);
20544 let bad_digest_blocked = match TierConfigMgr::acquire_operation_lease(&store.tier_config_mgr(), "COLD-C").await {
20545 Ok(_) => panic!("mismatched digest must retain the prepared runtime fence"),
20546 Err(err) => err,
20547 };
20548 assert!(bad_digest_blocked.message.contains("being replaced"), "{bad_digest_blocked}");
20549 handle_tier_mutation_peer_request(
20550 store.clone(),
20551 TIER_MUTATION_RPC_PROTOCOL_VERSION,
20552 TierMutationRpcPhase::Abort,
20553 bad_digest_id,
20554 &bad_digest_intent.encode().expect("bad digest abort identity should encode"),
20555 )
20556 .await
20557 .expect("the negative digest proof fixture should clean up through abort");
20558
20559 let committed = handle_tier_mutation_peer_request(
20560 store.clone(),
20561 TIER_MUTATION_RPC_PROTOCOL_VERSION,
20562 TierMutationRpcPhase::Commit,
20563 mutation_id,
20564 config_etag.as_bytes(),
20565 )
20566 .await
20567 .expect("commit should advance the prepared peer intent");
20568 assert!(committed.applied);
20569 assert_eq!(committed.state, TierMutationPeerState::Committed);
20570 let committed_blocked = match TierConfigMgr::acquire_operation_lease(&store.tier_config_mgr(), "COLD-A").await {
20571 Ok(_) => panic!("committed peer mutation must remain blocked until local reload publishes the config"),
20572 Err(err) => err,
20573 };
20574 assert!(
20575 committed_blocked.message.contains("being replaced"),
20576 "committed peer mutation should keep the existing blocked-tier error: {committed_blocked}"
20577 );
20578
20579 let retried_commit = handle_tier_mutation_peer_request(
20580 store.clone(),
20581 TIER_MUTATION_RPC_PROTOCOL_VERSION,
20582 TierMutationRpcPhase::Commit,
20583 mutation_id,
20584 config_etag.as_bytes(),
20585 )
20586 .await
20587 .expect("same commit retry should be idempotent");
20588 assert!(!retried_commit.applied);
20589 assert_eq!(retried_commit.state, TierMutationPeerState::Committed);
20590 let retried_commit_blocked = match TierConfigMgr::acquire_operation_lease(&store.tier_config_mgr(), "COLD-A").await {
20591 Ok(_) => panic!("committed retry must keep the tier blocked until local reload"),
20592 Err(err) => err,
20593 };
20594 assert!(
20595 retried_commit_blocked.message.contains("being replaced"),
20596 "committed retry should keep the existing blocked-tier error: {retried_commit_blocked}"
20597 );
20598
20599 let delayed_prepare_retry = handle_tier_mutation_peer_request(
20600 store.clone(),
20601 TIER_MUTATION_RPC_PROTOCOL_VERSION,
20602 TierMutationRpcPhase::Prepare,
20603 mutation_id,
20604 &prepare_payload,
20605 )
20606 .await
20607 .expect("delayed duplicate prepare should report the durable committed state");
20608 assert!(!delayed_prepare_retry.applied);
20609 assert_eq!(delayed_prepare_retry.state, TierMutationPeerState::Committed);
20610 let delayed_prepare_blocked = match TierConfigMgr::acquire_operation_lease(&store.tier_config_mgr(), "COLD-A").await {
20611 Ok(_) => panic!("delayed committed prepare retry must preserve the committed runtime block"),
20612 Err(err) => err,
20613 };
20614 assert!(
20615 delayed_prepare_blocked.message.contains("being replaced"),
20616 "delayed committed prepare retry should keep the existing blocked-tier error: {delayed_prepare_blocked}"
20617 );
20618
20619 let loaded = load_tier_mutation_intent_record(store.clone(), mutation_id)
20620 .await
20621 .expect("committed peer intent should remain durable");
20622 assert_eq!(loaded.state, TierMutationIntentState::Committed);
20623 assert_eq!(loaded.committed_config_etag.as_deref(), Some(config_etag.as_str()));
20624
20625 store
20626 .tier_config_mgr()
20627 .read()
20628 .await
20629 .save_tiering_config(store.clone())
20630 .await
20631 .expect("tier config should persist for cleaned intent commit proof");
20632 let tier_config_info = store
20633 .get_object_info(
20634 RUSTFS_META_BUCKET,
20635 &format!("{}/{}", com::CONFIG_PREFIX, TIER_CONFIG_FILE),
20636 &ObjectOptions::default(),
20637 )
20638 .await
20639 .expect("tier config object info should load");
20640 let tier_config_etag = tier_config_info.etag.expect("tier config should carry an ETag");
20641 delete_tier_mutation_intent_record(store.clone(), mutation_id)
20642 .await
20643 .expect("simulate another node cleaning the shared committed peer intent");
20644 let cleaned_commit_retry = handle_tier_mutation_peer_request(
20645 store.clone(),
20646 TIER_MUTATION_RPC_PROTOCOL_VERSION,
20647 TierMutationRpcPhase::Commit,
20648 mutation_id,
20649 tier_config_etag.as_bytes(),
20650 )
20651 .await
20652 .expect("commit retry after durable cleanup should be idempotently terminal");
20653 assert!(!cleaned_commit_retry.applied);
20654 assert_eq!(cleaned_commit_retry.state, TierMutationPeerState::Committed);
20655 let cleaned_commit_blocked = match TierConfigMgr::acquire_operation_lease(&store.tier_config_mgr(), "COLD-A").await {
20656 Ok(_) => panic!("shared intent cleanup must not clear this node's committed runtime block"),
20657 Err(err) => err,
20658 };
20659 assert!(
20660 cleaned_commit_blocked.message.contains("being replaced"),
20661 "commit retry after shared cleanup should keep the existing blocked-tier error: {cleaned_commit_blocked}"
20662 );
20663 let mismatched_cleaned_commit = handle_tier_mutation_peer_request(
20664 store.clone(),
20665 TIER_MUTATION_RPC_PROTOCOL_VERSION,
20666 TierMutationRpcPhase::Commit,
20667 mutation_id,
20668 b"not-the-current-etag",
20669 )
20670 .await
20671 .expect_err("missing intent without a matching committed config ETag must fail closed");
20672 assert!(matches!(mismatched_cleaned_commit, TierMutationPeerError::Store(Error::ConfigNotFound)));
20673 let refresh_store = store.clone();
20674 let refresh_manager = store.tier_config_mgr();
20675 let refresh_worker = tokio::spawn(async move {
20676 TierConfigMgr::refresh_tier_config_handle_with(refresh_manager, refresh_store).await;
20677 });
20678 tokio::time::timeout(Duration::from_secs(5), async {
20679 loop {
20680 if let Ok(lease) = TierConfigMgr::acquire_operation_lease(&store.tier_config_mgr(), "COLD-A").await {
20681 drop(lease);
20682 break;
20683 }
20684 tokio::task::yield_now().await;
20685 }
20686 })
20687 .await
20688 .expect("matching cleaned commit should wake the refresh worker and clear the committed fence");
20689 refresh_worker.abort();
20690 let _ = refresh_worker.await;
20691
20692 let abort_id = uuid::Uuid::new_v4();
20693 register_mock_tier(&store.tier_config_mgr(), "COLD-B").await;
20694 store
20695 .tier_config_mgr()
20696 .read()
20697 .await
20698 .save_tiering_config(store.clone())
20699 .await
20700 .expect("abort target tier config should persist");
20701 let abort_config_info = store
20702 .get_object_info(
20703 RUSTFS_META_BUCKET,
20704 &format!("{}/{}", com::CONFIG_PREFIX, TIER_CONFIG_FILE),
20705 &ObjectOptions::default(),
20706 )
20707 .await
20708 .expect("abort target config metadata should load");
20709 let abort_config_etag = abort_config_info.etag.expect("abort target config should carry an ETag");
20710 let mut abort_intent = tier_mutation_peer_test_intent(abort_id, "COLD-B", [4; 32]);
20711 abort_intent.old_config_etag = Some(abort_config_etag);
20712 let abort_prepare_payload = abort_intent.encode().expect("abort prepare intent should encode");
20713 handle_tier_mutation_peer_request(
20714 store.clone(),
20715 TIER_MUTATION_RPC_PROTOCOL_VERSION,
20716 TierMutationRpcPhase::Prepare,
20717 abort_id,
20718 &abort_prepare_payload,
20719 )
20720 .await
20721 .expect("abort target prepare should create the peer intent");
20722 let abort_blocked = match TierConfigMgr::acquire_operation_lease(&store.tier_config_mgr(), "COLD-B").await {
20723 Ok(_) => panic!("abort target prepare should block new tier operation leases"),
20724 Err(err) => err,
20725 };
20726 assert!(
20727 abort_blocked.message.contains("being replaced"),
20728 "abort target prepare should reuse the existing blocked-tier error: {abort_blocked}"
20729 );
20730
20731 let aborted = handle_tier_mutation_peer_request(
20732 store.clone(),
20733 TIER_MUTATION_RPC_PROTOCOL_VERSION,
20734 TierMutationRpcPhase::Abort,
20735 abort_id,
20736 &abort_prepare_payload,
20737 )
20738 .await
20739 .expect("abort should advance the prepared peer intent");
20740 assert!(aborted.applied);
20741 assert_eq!(aborted.state, TierMutationPeerState::Aborted);
20742 TierConfigMgr::acquire_operation_lease(&store.tier_config_mgr(), "COLD-B")
20743 .await
20744 .expect("durable abort must clear the peer runtime fence immediately");
20745
20746 let retried_abort = handle_tier_mutation_peer_request(
20747 store.clone(),
20748 TIER_MUTATION_RPC_PROTOCOL_VERSION,
20749 TierMutationRpcPhase::Abort,
20750 abort_id,
20751 &abort_prepare_payload,
20752 )
20753 .await
20754 .expect("same abort retry should be idempotent before recovery cleanup");
20755 assert!(!retried_abort.applied);
20756 assert_eq!(retried_abort.state, TierMutationPeerState::Aborted);
20757 }
20758
20759 #[cfg(feature = "test-util")]
20760 #[tokio::test]
20761 #[serial_test::serial(storage_class_env)]
20762 async fn late_abort_after_config_commit_keeps_fence_until_commit_recovery() {
20763 let temp_dir = tempfile::tempdir().expect("create temp store dir");
20764 let (_ctx, store, _shutdown) =
20765 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "tier-mutation-late-abort", &[4])).await;
20766 register_mock_tier(&store.tier_config_mgr(), "COLD-A").await;
20767 store
20768 .tier_config_mgr()
20769 .read()
20770 .await
20771 .save_tiering_config(store.clone())
20772 .await
20773 .expect("base tier config should persist");
20774 let base_config_info = store
20775 .get_object_info(
20776 RUSTFS_META_BUCKET,
20777 &format!("{}/{}", com::CONFIG_PREFIX, TIER_CONFIG_FILE),
20778 &ObjectOptions::default(),
20779 )
20780 .await
20781 .expect("base tier config metadata should load");
20782 let base_etag = base_config_info.etag.expect("base tier config should carry an ETag");
20783
20784 register_mock_tier(&store.tier_config_mgr(), "COLD-B").await;
20785 let candidate_digest = {
20786 let manager = store.tier_config_mgr();
20787 let manager = manager.read().await;
20788 tier_config_candidate_digest(&manager).expect("candidate digest should build")
20789 };
20790 let mutation_id = uuid::Uuid::new_v4();
20791 let mut intent = tier_mutation_peer_test_intent(mutation_id, "COLD-B", candidate_digest);
20792 intent.old_config_etag = Some(base_etag.clone());
20793 let prepare_payload = intent.encode().expect("late abort prepare intent should encode");
20794 handle_tier_mutation_peer_request(
20795 store.clone(),
20796 TIER_MUTATION_RPC_PROTOCOL_VERSION,
20797 TierMutationRpcPhase::Prepare,
20798 mutation_id,
20799 &prepare_payload,
20800 )
20801 .await
20802 .expect("prepare should install the runtime fence");
20803
20804 store
20805 .tier_config_mgr()
20806 .read()
20807 .await
20808 .save_tiering_config(store.clone())
20809 .await
20810 .expect("candidate tier config should persist before the late abort");
20811 let committed_config_info = store
20812 .get_object_info(
20813 RUSTFS_META_BUCKET,
20814 &format!("{}/{}", com::CONFIG_PREFIX, TIER_CONFIG_FILE),
20815 &ObjectOptions::default(),
20816 )
20817 .await
20818 .expect("committed tier config metadata should load");
20819 let committed_etag = committed_config_info
20820 .etag
20821 .expect("committed tier config should carry an ETag");
20822 assert_ne!(committed_etag, base_etag);
20823
20824 let late_abort = handle_tier_mutation_peer_request(
20825 store.clone(),
20826 TIER_MUTATION_RPC_PROTOCOL_VERSION,
20827 TierMutationRpcPhase::Abort,
20828 mutation_id,
20829 &prepare_payload,
20830 )
20831 .await
20832 .expect_err("an abort after the candidate config commit must fail closed");
20833 assert!(matches!(late_abort, TierMutationPeerError::AbortProofMismatch));
20834 let prepared = load_tier_mutation_intent_record(store.clone(), mutation_id)
20835 .await
20836 .expect("rejected late abort must retain the prepared intent");
20837 assert_eq!(prepared.state, TierMutationIntentState::Prepared);
20838 let blocked = match TierConfigMgr::acquire_operation_lease(&store.tier_config_mgr(), "COLD-B").await {
20839 Ok(_) => panic!("rejected late abort must retain the runtime fence"),
20840 Err(err) => err,
20841 };
20842 assert!(blocked.message.contains("being replaced"), "{blocked}");
20843
20844 let committed = handle_tier_mutation_peer_request(
20845 store.clone(),
20846 TIER_MUTATION_RPC_PROTOCOL_VERSION,
20847 TierMutationRpcPhase::Commit,
20848 mutation_id,
20849 committed_etag.as_bytes(),
20850 )
20851 .await
20852 .expect("matching commit should converge the rejected late abort fixture");
20853 assert!(committed.applied);
20854 assert_eq!(committed.state, TierMutationPeerState::Committed);
20855
20856 let refresh_store = store.clone();
20857 let refresh_manager = store.tier_config_mgr();
20858 let refresh_worker = tokio::spawn(async move {
20859 TierConfigMgr::refresh_tier_config_handle_with(refresh_manager, refresh_store).await;
20860 });
20861 tokio::time::timeout(Duration::from_secs(5), async {
20862 loop {
20863 if let Ok(lease) = TierConfigMgr::acquire_operation_lease(&store.tier_config_mgr(), "COLD-B").await {
20864 drop(lease);
20865 break;
20866 }
20867 tokio::task::yield_now().await;
20868 }
20869 })
20870 .await
20871 .expect("commit recovery should publish before clearing the late-abort fence");
20872 refresh_worker.abort();
20873 let _ = refresh_worker.await;
20874 }
20875
20876 #[cfg(feature = "test-util")]
20877 #[tokio::test]
20878 #[serial_test::serial(storage_class_env)]
20879 async fn missing_record_commit_promotes_prepared_fence_until_worker_publish() {
20880 let temp_dir = tempfile::tempdir().expect("create temp store dir");
20881 let (_ctx, store, _shutdown) =
20882 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "tier-mutation-missing-record-commit", &[4]))
20883 .await;
20884 register_mock_tier(&store.tier_config_mgr(), "COLD-A").await;
20885 let tier_config_mgr = store.tier_config_mgr();
20886 let candidate_digest = {
20887 let manager = tier_config_mgr.read().await;
20888 let digest = tier_config_candidate_digest(&manager).expect("candidate digest should build");
20889 manager
20890 .save_tiering_config(store.clone())
20891 .await
20892 .expect("candidate config should persist");
20893 digest
20894 };
20895 let config_info = store
20896 .get_object_info(
20897 RUSTFS_META_BUCKET,
20898 &format!("{}/{}", com::CONFIG_PREFIX, TIER_CONFIG_FILE),
20899 &ObjectOptions::default(),
20900 )
20901 .await
20902 .expect("candidate config metadata should load");
20903 let config_etag = config_info.etag.expect("candidate config should carry an ETag");
20904 let mutation_id = uuid::Uuid::new_v4();
20905 let intent = tier_mutation_peer_test_intent(mutation_id, "COLD-A", candidate_digest);
20906 handle_tier_mutation_peer_request(
20907 store.clone(),
20908 TIER_MUTATION_RPC_PROTOCOL_VERSION,
20909 TierMutationRpcPhase::Prepare,
20910 mutation_id,
20911 &intent.encode().expect("prepare intent should encode"),
20912 )
20913 .await
20914 .expect("prepare should install the runtime fence");
20915 delete_tier_mutation_intent_record(store.clone(), mutation_id)
20916 .await
20917 .expect("simulate shared intent cleanup before the local commit arrives");
20918
20919 let committed = handle_tier_mutation_peer_request(
20920 store.clone(),
20921 TIER_MUTATION_RPC_PROTOCOL_VERSION,
20922 TierMutationRpcPhase::Commit,
20923 mutation_id,
20924 config_etag.as_bytes(),
20925 )
20926 .await
20927 .expect("matching commit after shared cleanup should be terminal");
20928 assert!(!committed.applied);
20929 assert_eq!(committed.state, TierMutationPeerState::Committed);
20930 let blocked = match TierConfigMgr::acquire_operation_lease(&tier_config_mgr, "COLD-A").await {
20931 Ok(_) => panic!("the promoted committed fence must block old-generation leases"),
20932 Err(err) => err,
20933 };
20934 assert!(blocked.message.contains("being replaced"), "{blocked}");
20935
20936 let refresh_store = store.clone();
20937 let refresh_manager = tier_config_mgr.clone();
20938 let refresh_worker = tokio::spawn(async move {
20939 TierConfigMgr::refresh_tier_config_handle_with(refresh_manager, refresh_store).await;
20940 });
20941 tokio::time::timeout(Duration::from_secs(5), async {
20942 loop {
20943 if let Ok(lease) = TierConfigMgr::acquire_operation_lease(&tier_config_mgr, "COLD-A").await {
20944 drop(lease);
20945 break;
20946 }
20947 tokio::task::yield_now().await;
20948 }
20949 })
20950 .await
20951 .expect("the commit notification should drive publish and clear the promoted fence");
20952 refresh_worker.abort();
20953 let _ = refresh_worker.await;
20954 }
20955
20956 #[cfg(feature = "test-util")]
20957 #[tokio::test]
20958 #[serial_test::serial(storage_class_env)]
20959 async fn tier_mutation_peer_handler_rejects_conflicting_prepare_without_overwrite() {
20960 let temp_dir = tempfile::tempdir().expect("create temp store dir");
20961 let (_ctx, store, _shutdown) =
20962 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "tier-mutation-peer-conflict", &[4])).await;
20963 let mutation_id = uuid::Uuid::new_v4();
20964 let intent = tier_mutation_peer_test_intent(mutation_id, "COLD-A", [3; 32]);
20965 let prepare_payload = intent.encode().expect("prepare intent should encode");
20966 handle_tier_mutation_peer_request(
20967 store.clone(),
20968 TIER_MUTATION_RPC_PROTOCOL_VERSION,
20969 TierMutationRpcPhase::Prepare,
20970 mutation_id,
20971 &prepare_payload,
20972 )
20973 .await
20974 .expect("first prepare should create the peer intent");
20975
20976 let conflicting = tier_mutation_peer_test_intent(mutation_id, "COLD-A", [4; 32]);
20977 let conflicting_payload = conflicting.encode().expect("conflicting intent should encode");
20978 let conflict = handle_tier_mutation_peer_request(
20979 store.clone(),
20980 TIER_MUTATION_RPC_PROTOCOL_VERSION,
20981 TierMutationRpcPhase::Prepare,
20982 mutation_id,
20983 &conflicting_payload,
20984 )
20985 .await
20986 .expect_err("conflicting prepare must fail closed");
20987 assert!(matches!(conflict, TierMutationPeerError::ConflictingIntent));
20988
20989 let loaded = load_tier_mutation_intent_record(store, mutation_id)
20990 .await
20991 .expect("conflicting prepare must not overwrite the first record");
20992 assert_eq!(loaded.candidate_digest, [3; 32]);
20993 assert_eq!(loaded.state, TierMutationIntentState::Prepared);
20994 }
20995
20996 #[cfg(feature = "test-util")]
20997 #[tokio::test]
20998 #[serial_test::serial(storage_class_env)]
20999 async fn tier_mutation_intent_record_scan_paginates_exact_limit() {
21000 let temp_dir = tempfile::tempdir().expect("create temp store dir");
21001 let (_ctx, store, _shutdown) =
21002 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "tier-mutation-intent-scan-page", &[4])).await;
21003 let build_intent = |mutation_id: uuid::Uuid, tier_name: &str| TierMutationIntent {
21004 mutation_id,
21005 revision: 1,
21006 kind: TierMutationIntentKind::Edit,
21007 state: TierMutationIntentState::Prepared,
21008 old_config_etag: Some("old-etag".to_string()),
21009 committed_config_etag: None,
21010 candidate_digest: [3; 32],
21011 affected_targets: vec![TierMutationIntentTarget {
21012 tier_name: tier_name.to_string(),
21013 old_backend_identity: Some([1; 32]),
21014 new_backend_identity: Some([2; 32]),
21015 }],
21016 expires_at_unix_nanos: 1_780_000_000_000_000_000,
21017 };
21018 let intent_ids = vec![
21019 uuid::Uuid::parse_str("11345678-1234-5678-9abc-def012345678").expect("first uuid should parse"),
21020 uuid::Uuid::parse_str("22345678-1234-5678-9abc-def012345678").expect("second uuid should parse"),
21021 uuid::Uuid::parse_str("33345678-1234-5678-9abc-def012345678").expect("third uuid should parse"),
21022 ];
21023 for (index, mutation_id) in intent_ids.iter().copied().enumerate() {
21024 let intent = build_intent(mutation_id, &format!("COLD-{index}"));
21025 save_tier_mutation_intent_record(store.clone(), &intent)
21026 .await
21027 .expect("tier mutation intent record should persist");
21028 }
21029
21030 let exact_page = list_tier_mutation_intent_records(store.clone(), 3, None)
21031 .await
21032 .expect("exact full page should scan");
21033 assert_eq!(exact_page.scanned, 3);
21034 assert_eq!(exact_page.intents.len(), 3);
21035 assert_eq!(exact_page.failed, 0);
21036 assert!(!exact_page.truncated);
21037 assert_eq!(exact_page.next_marker, None);
21038
21039 let first_page = list_tier_mutation_intent_records(store.clone(), 2, None)
21040 .await
21041 .expect("first page should scan");
21042 assert_eq!(first_page.scanned, 2);
21043 assert_eq!(first_page.intents.len(), 2);
21044 assert_eq!(first_page.failed, 0);
21045 assert!(first_page.truncated);
21046 assert!(first_page.next_marker.is_some());
21047
21048 let second_page = list_tier_mutation_intent_records(store, 2, first_page.next_marker)
21049 .await
21050 .expect("second page should scan");
21051 let mut loaded_ids: Vec<_> = first_page
21052 .intents
21053 .into_iter()
21054 .chain(second_page.intents.into_iter())
21055 .map(|intent| intent.mutation_id)
21056 .collect();
21057 loaded_ids.sort();
21058
21059 assert_eq!(second_page.scanned, 1);
21060 assert_eq!(second_page.failed, 0);
21061 assert!(!second_page.truncated);
21062 assert_eq!(second_page.next_marker, None);
21063 assert_eq!(loaded_ids, intent_ids);
21064 }
21065
21066 #[cfg(feature = "test-util")]
21067 fn transition_mutation_measurement(observations: &[TransitionTransactionMutationObservation]) -> (usize, usize, u128) {
21068 (
21069 observations.len(),
21070 observations.iter().map(|observation| observation.encoded_bytes).sum(),
21071 observations.iter().map(|observation| observation.elapsed.as_micros()).sum(),
21072 )
21073 }
21074
21075 #[cfg(feature = "test-util")]
21076 #[tokio::test]
21077 #[serial_test::serial(storage_class_env)]
21078 async fn compact_transition_transactions_halve_success_path_quorum_mutations() {
21079 let temp_dir = tempfile::tempdir().expect("create transition mutation measurement dir");
21080 let (ctx, store, shutdown) =
21081 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "transition-mutation-measurement", &[4])).await;
21082 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
21083 let tier_name = "MUTATION-MEASURE";
21084 register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
21085 let bucket = "transition-mutation-measurement";
21086 store
21087 .make_bucket(bucket, &MakeBucketOptions::default())
21088 .await
21089 .expect("measurement bucket should be created");
21090
21091 let run_case = |profile: &'static str, size: usize| {
21092 let store = store.clone();
21093 async move {
21094 let object = format!("{profile}-{size}.bin");
21095 let mut reader = PutObjReader::from_vec(vec![b'm'; size]);
21096 let original = store
21097 .put_object(bucket, &object, &mut reader, &ObjectOptions::default())
21098 .await
21099 .expect("measurement source should be written");
21100 let probe = TransitionTransactionMutationProbe::install(bucket, &object);
21101 store
21102 .transition_object(
21103 bucket,
21104 &object,
21105 &ObjectOptions {
21106 transition: TransitionOptions {
21107 status: TRANSITION_PENDING.to_string(),
21108 tier: tier_name.to_string(),
21109 etag: original.etag.clone().expect("measurement source should have an ETag"),
21110 ..Default::default()
21111 },
21112 version_id: original.version_id.map(|version| version.to_string()),
21113 mod_time: original.mod_time,
21114 ..Default::default()
21115 },
21116 )
21117 .await
21118 .expect("measurement transition should commit");
21119 probe.observations()
21120 }
21121 };
21122
21123 let sizes = [4 * 1024, 1024 * 1024];
21124 let mut legacy = Vec::with_capacity(sizes.len());
21125 for size in sizes {
21126 legacy.push((size, run_case("legacy", size).await));
21127 }
21128
21129 let compaction_proof = install_transition_transaction_compaction_fleet_proof_for_test("object-transaction-fencing-test");
21130 for ((size, legacy_observations), compact_size) in legacy.into_iter().zip(sizes) {
21131 assert_eq!(size, compact_size);
21132 let compact_observations = run_case("compact", size).await;
21133 let legacy_measurement = transition_mutation_measurement(&legacy_observations);
21134 let compact_measurement = transition_mutation_measurement(&compact_observations);
21135 assert_eq!(legacy_measurement.0, 6, "legacy success should use five saves and one delete");
21136 assert_eq!(compact_measurement.0, 3, "compact success should use two saves and one delete");
21137 assert!(
21138 compact_measurement.1 < legacy_measurement.1,
21139 "compact transaction bodies should write fewer aggregate bytes"
21140 );
21141 assert_eq!(
21142 legacy_observations
21143 .iter()
21144 .map(|observation| (observation.kind, observation.previous_state, observation.state))
21145 .collect::<Vec<_>>(),
21146 vec![
21147 (TransitionTransactionMutationKind::Create, None, TransitionTransactionState::UploadStarted),
21148 (
21149 TransitionTransactionMutationKind::CompareAndSave,
21150 Some(TransitionTransactionState::UploadStarted),
21151 TransitionTransactionState::UploadOutcomeUnknown,
21152 ),
21153 (
21154 TransitionTransactionMutationKind::CompareAndSave,
21155 Some(TransitionTransactionState::UploadOutcomeUnknown),
21156 TransitionTransactionState::Uploaded,
21157 ),
21158 (
21159 TransitionTransactionMutationKind::CompareAndSave,
21160 Some(TransitionTransactionState::Uploaded),
21161 TransitionTransactionState::LocalCommitStarted,
21162 ),
21163 (
21164 TransitionTransactionMutationKind::CompareAndSave,
21165 Some(TransitionTransactionState::LocalCommitStarted),
21166 TransitionTransactionState::Committed,
21167 ),
21168 (
21169 TransitionTransactionMutationKind::Delete,
21170 Some(TransitionTransactionState::Committed),
21171 TransitionTransactionState::Committed,
21172 ),
21173 ]
21174 );
21175 assert_eq!(
21176 compact_observations
21177 .iter()
21178 .map(|observation| (observation.kind, observation.previous_state, observation.state))
21179 .collect::<Vec<_>>(),
21180 vec![
21181 (
21182 TransitionTransactionMutationKind::Create,
21183 None,
21184 TransitionTransactionState::UploadOutcomeUnknown,
21185 ),
21186 (
21187 TransitionTransactionMutationKind::CompareAndSave,
21188 Some(TransitionTransactionState::UploadOutcomeUnknown),
21189 TransitionTransactionState::LocalCommitStarted,
21190 ),
21191 (
21192 TransitionTransactionMutationKind::Delete,
21193 Some(TransitionTransactionState::LocalCommitStarted),
21194 TransitionTransactionState::LocalCommitStarted,
21195 ),
21196 ]
21197 );
21198 assert!(legacy_observations.iter().all(|observation| observation.succeeded));
21199 assert!(compact_observations.iter().all(|observation| observation.succeeded));
21200 println!(
21201 "transition_mutation_measurement,profile=legacy,size={size},mutations={},encoded_bytes={},latency_us={},quorum_operations={}",
21202 legacy_measurement.0, legacy_measurement.1, legacy_measurement.2, legacy_measurement.0
21203 );
21204 println!(
21205 "transition_mutation_measurement,profile=compact,size={size},mutations={},encoded_bytes={},latency_us={},quorum_operations={}",
21206 compact_measurement.0, compact_measurement.1, compact_measurement.2, compact_measurement.0
21207 );
21208 }
21209 assert_eq!(transition_transaction_record_count(store.clone()).await, 0);
21210 let admitted = acquire_transition_transaction_compaction_fleet_proof()
21211 .expect("published homogeneous proof should admit one compact writer");
21212 assert!(transition_transaction_compaction_fleet_proof_matches(&admitted));
21213 drop(compaction_proof);
21214 assert!(
21215 !transition_transaction_compaction_fleet_proof_matches(&admitted),
21216 "revocation must fence a writer admitted by the previous process-epoch snapshot"
21217 );
21218 drop(admitted);
21219 assert!(
21220 acquire_transition_transaction_compaction_fleet_proof().is_none(),
21221 "revoking the homogeneous proof must restore the legacy writer profile"
21222 );
21223 shutdown.cancel();
21224 }
21225
21226 #[cfg(feature = "test-util")]
21227 #[tokio::test]
21228 #[serial_test::serial(storage_class_env)]
21229 async fn compact_transition_kill_points_preserve_the_only_remote_owner() {
21230 let temp_dir = tempfile::tempdir().expect("create compact transition kill-point dir");
21231 let (ctx, store, shutdown) =
21232 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "compact-transition-kill-points", &[4])).await;
21233 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
21234 let tier_name = "COMPACT-KILL";
21235 let backend = register_transition_reconcile_test_tier(&ctx.tier_config_mgr(), tier_name).await;
21236 let _tier_lease = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
21237 .await
21238 .expect("mock tier lease should remain available during recovery");
21239 let bucket = "compact-transition-kill-points";
21240 store
21241 .make_bucket(bucket, &MakeBucketOptions::default())
21242 .await
21243 .expect("kill-point bucket should be created");
21244 let _compaction_proof = install_transition_transaction_compaction_fleet_proof_for_test("object-transaction-fencing-test");
21245
21246 for (index, point, expected_state, expected_revision, expected_committed, expected_recovered) in [
21247 (
21248 0,
21249 TransitionTransactionKillPoint::PrePutFence,
21250 TransitionTransactionState::UploadOutcomeUnknown,
21251 1,
21252 false,
21253 true,
21254 ),
21255 (
21256 1,
21257 TransitionTransactionKillPoint::UploadBeforeCommitFence,
21258 TransitionTransactionState::UploadOutcomeUnknown,
21259 1,
21260 false,
21261 true,
21262 ),
21263 (
21264 2,
21265 TransitionTransactionKillPoint::LocalCommitBeforeDelete,
21266 TransitionTransactionState::LocalCommitStarted,
21267 2,
21268 true,
21269 true,
21270 ),
21271 (
21272 3,
21273 TransitionTransactionKillPoint::CommitFenceBeforeLocalCommit,
21274 TransitionTransactionState::LocalCommitStarted,
21275 2,
21276 false,
21277 false,
21278 ),
21279 ] {
21280 let object = format!("kill-point-{index}.bin");
21281 let payload = vec![b'k' + u8::try_from(index).expect("small case index should fit u8"); 64 * 1024];
21282 let mut reader = PutObjReader::from_vec(payload.clone());
21283 let source = store
21284 .put_object(bucket, &object, &mut reader, &ObjectOptions::default())
21285 .await
21286 .expect("kill-point source should be written");
21287 let remote_before = backend.object_count().await;
21288 let barrier = TransitionTransactionKillPointBarrier::install(bucket, &object, point);
21289 let transition_store = store.clone();
21290 let transition_object = object.clone();
21291 let transition = tokio::spawn(async move {
21292 transition_store
21293 .transition_object(
21294 bucket,
21295 &transition_object,
21296 &ObjectOptions {
21297 transition: TransitionOptions {
21298 status: TRANSITION_PENDING.to_string(),
21299 tier: tier_name.to_string(),
21300 etag: source.etag.clone().expect("kill-point source should have an ETag"),
21301 ..Default::default()
21302 },
21303 version_id: source.version_id.map(|version| version.to_string()),
21304 mod_time: source.mod_time,
21305 ..Default::default()
21306 },
21307 )
21308 .await
21309 });
21310 barrier.wait_until_paused().await;
21311 transition.abort();
21312 assert!(
21313 transition
21314 .await
21315 .expect_err("kill-point transition should be cancelled")
21316 .is_cancelled(),
21317 "kill-point transition should stop without unwinding"
21318 );
21319 drop(barrier);
21320
21321 let transaction = only_transition_transaction(store.clone()).await;
21322 assert_eq!((transaction.state, transaction.revision), (expected_state, expected_revision));
21323 let paused_source = store
21324 .get_object_info(
21325 bucket,
21326 &object,
21327 &ObjectOptions {
21328 metadata_cache_safe: false,
21329 ..Default::default()
21330 },
21331 )
21332 .await
21333 .expect("kill-point source metadata should remain readable");
21334 assert_eq!(
21335 paused_source.transitioned_object.status == rustfs_filemeta::TRANSITION_COMPLETE,
21336 expected_committed
21337 );
21338 let expected_remote_at_pause = remote_before + usize::from(point != TransitionTransactionKillPoint::PrePutFence);
21339 assert_eq!(backend.object_count().await, expected_remote_at_pause);
21340
21341 let stats = recover_transition_transaction_records_at(
21342 store.clone(),
21343 100,
21344 None,
21345 i128::from(transaction.not_after_unix_nanos) + 1,
21346 )
21347 .await
21348 .expect("kill-point transaction recovery should complete");
21349 if expected_recovered {
21350 assert_eq!((stats.scanned, stats.recovered, stats.retained, stats.failed), (1, 1, 0, 0));
21351 assert_eq!(transition_transaction_record_count(store.clone()).await, 0);
21352 } else {
21353 assert_eq!((stats.scanned, stats.recovered, stats.retained, stats.failed), (1, 0, 1, 0));
21354 assert_eq!(
21355 transition_transaction_record_count(store.clone()).await,
21356 1,
21357 "an uncommitted local-commit fence must retain its exact remote owner"
21358 );
21359 }
21360 let expected_remote_after_recovery = if matches!(
21361 point,
21362 TransitionTransactionKillPoint::LocalCommitBeforeDelete
21363 | TransitionTransactionKillPoint::CommitFenceBeforeLocalCommit
21364 ) {
21365 remote_before + 1
21366 } else {
21367 remote_before
21368 };
21369 assert_eq!(backend.object_count().await, expected_remote_after_recovery);
21370 assert_eq!(backend.remove_count().await, usize::from(index >= 1));
21371
21372 let mut restored = Vec::new();
21373 store
21374 .get_object_reader(bucket, &object, None, HeaderMap::new(), &ObjectOptions::default())
21375 .await
21376 .expect("kill-point source should remain readable through its authoritative location")
21377 .stream
21378 .read_to_end(&mut restored)
21379 .await
21380 .expect("kill-point source body should drain");
21381 assert_eq!(restored, payload);
21382
21383 if !expected_recovered {
21384 break;
21385 }
21386 }
21387 shutdown.cancel();
21388 }
21389
21390 #[cfg(feature = "test-util")]
21391 #[tokio::test]
21392 #[serial_test::serial(storage_class_env)]
21393 async fn cancelled_transition_cleanup_recovers_from_its_own_instance_transaction() {
21394 struct ResolverReset(Arc<std::sync::Mutex<Option<std::sync::Weak<crate::store::ECStore>>>>);
21395
21396 impl Drop for ResolverReset {
21397 fn drop(&mut self) {
21398 *self.0.lock().unwrap_or_else(std::sync::PoisonError::into_inner) = None;
21399 }
21400 }
21401
21402 let temp_a = tempfile::tempdir().expect("create transition store dir a");
21403 let temp_b = tempfile::tempdir().expect("create transition store dir b");
21404 let shutdown_a = CancellationToken::new();
21405 let shutdown_b = CancellationToken::new();
21406 shutdown_a.cancel();
21407 shutdown_b.cancel();
21408 let (ctx_a, store_a, shutdown_a) = without_storage_class_env(build_isolated_test_store_with_shutdown(
21409 temp_a.path(),
21410 "transition-cleanup-context-a",
21411 &[4],
21412 shutdown_a,
21413 ))
21414 .await;
21415 let (ctx_b, store_b, shutdown_b) = without_storage_class_env(build_isolated_test_store_with_shutdown(
21416 temp_b.path(),
21417 "transition-cleanup-context-b",
21418 &[4],
21419 shutdown_b,
21420 ))
21421 .await;
21422 assert!(shutdown_a.is_cancelled());
21423 assert!(shutdown_b.is_cancelled());
21424 crate::bucket::metadata_sys::init_bucket_metadata_sys(store_a.clone(), Vec::new()).await;
21425
21426 let resolver_target = Arc::new(std::sync::Mutex::new(Some(Arc::downgrade(&store_b))));
21427 let resolver_store = resolver_target.clone();
21428 assert!(
21429 set_object_store_resolver(Arc::new(move || {
21430 resolver_store
21431 .lock()
21432 .unwrap_or_else(std::sync::PoisonError::into_inner)
21433 .as_ref()
21434 .and_then(std::sync::Weak::upgrade)
21435 })),
21436 "the cross-context regression test must install the only process object-store resolver"
21437 );
21438 let _resolver_reset = ResolverReset(resolver_target);
21439 assert!(
21440 runtime_sources::object_store_handle().is_some_and(|store| Arc::ptr_eq(&store, &store_b)),
21441 "the process resolver must deliberately point at store B"
21442 );
21443
21444 let tier_name = "CROSSCTXA";
21445 let backend = register_transition_reconcile_test_tier(&ctx_a.tier_config_mgr(), tier_name).await;
21446 backend.set_put_remote_version(Some(uuid::Uuid::new_v4().to_string())).await;
21447 backend.reject_next_non_empty_remote_version_validation();
21448 let remove_barrier = backend.arm_failing_remove_barrier().await;
21449
21450 let bucket = "transition-cleanup-context-a";
21451 let object = "rejected-candidate.bin";
21452 store_a
21453 .make_bucket(bucket, &MakeBucketOptions::default())
21454 .await
21455 .expect("store A bucket should be created");
21456 let mut reader = PutObjReader::from_vec(b"cross-context rejected transition cleanup".repeat(1024));
21457 let original = store_a
21458 .put_object(bucket, object, &mut reader, &ObjectOptions::default())
21459 .await
21460 .expect("store A source object should be written");
21461 let opts = ObjectOptions {
21462 no_lock: true,
21463 transition: TransitionOptions {
21464 status: TRANSITION_PENDING.to_string(),
21465 tier: tier_name.to_string(),
21466 etag: original.etag.clone().expect("the source object should have an ETag"),
21467 ..Default::default()
21468 },
21469 version_id: original.version_id.map(|version| version.to_string()),
21470 mod_time: original.mod_time,
21471 ..Default::default()
21472 };
21473
21474 let cleanup_store_barrier = TransitionCleanupStoreBarrier::install();
21475 let transition_store = store_a.clone();
21476 let transition = tokio::spawn(async move { transition_store.transition_object(bucket, object, &opts).await });
21477 cleanup_store_barrier.wait_until_paused().await;
21478 transition.abort();
21479 assert!(
21480 transition
21481 .await
21482 .expect_err("the transition task should observe cancellation")
21483 .is_cancelled()
21484 );
21485
21486 remove_barrier.wait_until_paused().await;
21487 let transaction_counts = (
21488 transition_transaction_record_count(store_a.clone()).await,
21489 transition_transaction_record_count(store_b.clone()).await,
21490 );
21491 assert_eq!(
21492 transaction_counts,
21493 (1, 0),
21494 "the transition transaction must remain only on store A even while the process resolver points at store B"
21495 );
21496 let journal_counts = (
21497 tier_delete_journal_count(store_a.clone()).await,
21498 tier_delete_journal_count(store_b.clone()).await,
21499 );
21500 assert_eq!(
21501 journal_counts,
21502 (0, 0),
21503 "rejected transition uploads must not create a tier-delete journal"
21504 );
21505 assert_eq!(backend.object_count().await, 1, "failed cleanup should retain the remote candidate");
21506 remove_barrier.release();
21507 remove_barrier.wait_until_operation_dropped().await;
21508
21509 let recovered = recover_transition_transaction_records(store_a.clone(), 100, None)
21510 .await
21511 .expect("store A should recover its own cancelled-transition transaction");
21512 assert_eq!(
21513 (recovered.scanned, recovered.recovered, recovered.retained, recovered.failed),
21514 (1, 1, 0, 0)
21515 );
21516 assert_eq!(transition_transaction_record_count(store_a.clone()).await, 0);
21517 assert_eq!(transition_transaction_record_count(store_b.clone()).await, 0);
21518 assert_eq!(tier_delete_journal_count(store_a.clone()).await, 0);
21519 assert_eq!(tier_delete_journal_count(store_b.clone()).await, 0);
21520 assert_eq!(
21521 backend.object_count().await,
21522 0,
21523 "store A recovery should delete the exact remote candidate"
21524 );
21525 assert!(!Arc::ptr_eq(&ctx_a, &ctx_b), "the regression requires two distinct instance contexts");
21526 }
21527
21528 #[cfg(feature = "test-util")]
21529 #[tokio::test]
21530 #[serial_test::serial(storage_class_env)]
21531 async fn transition_transaction_recovery_expires_abandoned_attempt_at_budget_bound() {
21532 let temp_dir = tempfile::tempdir().expect("create temp store dir");
21533 let (ctx, store, _shutdown) = without_storage_class_env(build_isolated_test_store(
21534 temp_dir.path(),
21535 "transition-transaction-expired-attempt-budget",
21536 &[4],
21537 ))
21538 .await;
21539 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
21540
21541 let transaction = TransitionTransaction::new(TransitionTransactionInit {
21542 deployment_id: ctx.deployment_id().expect("test store should initialize deployment id"),
21543 transaction_id: uuid::Uuid::new_v4(),
21544 owner_epoch: uuid::Uuid::new_v4(),
21545 write_id: uuid::Uuid::new_v4(),
21546 source: TransitionSourceIdentity {
21547 bucket: "source-bucket".to_string(),
21548 object: "source-object".to_string(),
21549 version_id: Some(uuid::Uuid::new_v4()),
21550 data_dir: uuid::Uuid::new_v4(),
21551 mod_time_unix_nanos: 1_770_000_000_000_000_000,
21552 size: 42,
21553 etag: "source-etag".to_string(),
21554 version_mode: TransitionSourceVersionMode::Versioned,
21555 },
21556 tier_name: "UNUSEDABANDONEDTIER".to_string(),
21557 backend_fingerprint: [7; 32],
21558 not_after_unix_nanos: 1,
21559 })
21560 .expect("transaction should build");
21561 save_transition_transaction_record(store.clone(), &transaction)
21562 .await
21563 .expect("transaction record should persist");
21564
21565 let record_name =
21566 transition_transaction_record_object_name(transaction.transaction_id).expect("transaction record name should derive");
21567 let source = observe_recovery_source(
21568 store.clone(),
21569 &record_name,
21570 crate::bucket::lifecycle::transition_transaction::TRANSITION_TRANSACTION_SCHEMA,
21571 )
21572 .await
21573 .expect("transaction source generation should be observable");
21574 let mut control = IlmRecoveryControl::new(
21575 IlmRecoveryControlIdentity {
21576 protocol: IlmRecoveryProtocol::TransitionTransaction,
21577 canonical_source_path: record_name,
21578 stable_operation_identity: transaction.transaction_id.to_string(),
21579 record_class: "transition_transaction_v1".to_string(),
21580 },
21581 source.generation,
21582 IlmRecoveryClassification::Retrying,
21583 2_000_000_000,
21584 IlmRecoveryErrorCode::None,
21585 )
21586 .expect("recovery control should build");
21587 let mut now = 3_000_000_000;
21588 for _ in 1..MAX_RECOVERY_ATTEMPTS {
21589 now = now.max(control.next_attempt_at_unix_nanos.unwrap_or(now));
21590 control
21591 .claim("cancelled-or-timed-out-owner", uuid::Uuid::new_v4(), now, 1)
21592 .expect("abandoned attempt should claim");
21593 control
21594 .record_expired_attempt(now + 1)
21595 .expect("expired attempt should consume retry budget");
21596 now += 2;
21597 }
21598 now = now.max(control.next_attempt_at_unix_nanos.expect("last retry should have a backoff"));
21599 control
21600 .claim("cancelled-or-timed-out-owner", uuid::Uuid::new_v4(), now, 1)
21601 .expect("final abandoned attempt should claim");
21602 save_recovery_control_if_absent(store.clone(), &control)
21603 .await
21604 .expect("claimed recovery control should persist");
21605
21606 let stats = recover_transition_transaction_records_at(store.clone(), 100, None, i128::from(now + 1))
21607 .await
21608 .expect("recovery should account for the expired attempt");
21609 assert_eq!((stats.scanned, stats.recovered, stats.retained, stats.failed), (1, 0, 1, 0));
21610
21611 let control_id = transition_recovery_control_id(&transaction).expect("control id should derive");
21612 let persisted = load_recovery_control(store.clone(), IlmRecoveryProtocol::TransitionTransaction, &control_id)
21613 .await
21614 .expect("expired recovery control should remain inspectable");
21615 assert_eq!(persisted.control.classification, IlmRecoveryClassification::OperatorRequired);
21616 assert_eq!(persisted.control.attempt_count, u64::from(MAX_RECOVERY_ATTEMPTS));
21617 assert_eq!(persisted.control.consecutive_failure_count, MAX_RECOVERY_ATTEMPTS);
21618 assert_eq!(persisted.control.last_error_code, IlmRecoveryErrorCode::AttemptLeaseExpired);
21619 assert!(persisted.control.owner.is_none());
21620 assert_eq!(
21621 transition_transaction_record_count(store).await,
21622 1,
21623 "budget exhaustion must retain the source record"
21624 );
21625 }
21626
21627 #[cfg(feature = "test-util")]
21628 #[tokio::test]
21629 #[serial_test::serial(storage_class_env)]
21630 async fn transition_transaction_recovery_retains_active_uploaded_candidate_until_commit() {
21631 let temp_dir = tempfile::tempdir().expect("create temp store dir");
21632 let (ctx, store, _shutdown) =
21633 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "transition-transaction-active-uploaded", &[4]))
21634 .await;
21635 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
21636
21637 let tier_name = "TXACTIVEUPLOADED";
21638 let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
21639 let bucket = "transition-transaction-active-uploaded-bucket";
21640 let object = "object.bin";
21641 let payload = b"active Uploaded ownership must survive recovery until local commit".repeat(1024);
21642 store
21643 .make_bucket(bucket, &MakeBucketOptions::default())
21644 .await
21645 .expect("source bucket should be created");
21646 let mut reader = PutObjReader::from_vec(payload.clone());
21647 let source = store
21648 .put_object(bucket, object, &mut reader, &ObjectOptions::default())
21649 .await
21650 .expect("source object should be written");
21651 let barrier = TransitionUploadedCommitBarrier::install(bucket, object);
21652 let transition_store = store.clone();
21653 let transition = tokio::spawn(async move {
21654 transition_store
21655 .transition_object(
21656 bucket,
21657 object,
21658 &ObjectOptions {
21659 transition: TransitionOptions {
21660 status: TRANSITION_PENDING.to_string(),
21661 tier: tier_name.to_string(),
21662 etag: source.etag.clone().expect("source object should have an ETag"),
21663 ..Default::default()
21664 },
21665 version_id: source.version_id.map(|version_id| version_id.to_string()),
21666 mod_time: source.mod_time,
21667 ..Default::default()
21668 },
21669 )
21670 .await
21671 });
21672 barrier.wait_until_paused().await;
21673
21674 let stats = recover_transition_transaction_records(store.clone(), 100, None)
21675 .await
21676 .expect("recovery should inspect the active Uploaded transaction");
21677 assert_eq!((stats.scanned, stats.recovered, stats.retained, stats.failed), (1, 0, 1, 0));
21678 assert_eq!(backend.object_count().await, 1, "active ownership must retain the remote candidate");
21679 assert_eq!(backend.remove_count().await, 0, "active ownership must fence remote DELETE");
21680 assert_eq!(transition_transaction_record_count(store.clone()).await, 1);
21681
21682 barrier.release();
21683 transition
21684 .await
21685 .expect("transition task should join")
21686 .expect("transition should commit after the barrier is released");
21687 assert_eq!(transition_transaction_record_count(store.clone()).await, 0);
21688 assert_eq!(backend.remove_count().await, 0);
21689
21690 let mut body = Vec::new();
21691 store
21692 .get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
21693 .await
21694 .expect("committed tier object should remain readable")
21695 .stream
21696 .read_to_end(&mut body)
21697 .await
21698 .expect("tier object body should drain");
21699 assert_eq!(body, payload);
21700 }
21701
21702 #[cfg(feature = "test-util")]
21703 #[tokio::test]
21704 #[serial_test::serial(storage_class_env)]
21705 async fn transition_transaction_recovery_deletes_expired_uploaded_remote_candidate() {
21706 let temp_dir = tempfile::tempdir().expect("create temp store dir");
21707 let (ctx, store, _shutdown) =
21708 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "transition-transaction-recovery", &[4])).await;
21709 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
21710
21711 let tier_name = "TXRECOVERY";
21712 let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
21713 let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
21714 .await
21715 .expect("tier lease should resolve")
21716 .backend_identity();
21717 let exact_version = uuid::Uuid::new_v4().to_string();
21718 let cases = [
21719 (
21720 "exact",
21721 exact_version.clone(),
21722 TransitionRemoteVersion::versioned(exact_version),
21723 TransitionSourceVersionMode::Versioned,
21724 ),
21725 (
21726 "suspended-null",
21727 "null".to_string(),
21728 TransitionRemoteVersion::versioned("null"),
21729 TransitionSourceVersionMode::VersionSuspended,
21730 ),
21731 (
21732 "known-unversioned",
21733 String::new(),
21734 TransitionRemoteVersion::unversioned(),
21735 TransitionSourceVersionMode::Unversioned,
21736 ),
21737 ];
21738 let mut expected_removes = Vec::new();
21739 let mut recovery_control_ids = Vec::new();
21740 for (case, put_version, remote_version, source_mode) in cases {
21741 let mut transaction = TransitionTransaction::new(TransitionTransactionInit {
21742 deployment_id: ctx.deployment_id().expect("test store should initialize deployment id"),
21743 transaction_id: uuid::Uuid::new_v4(),
21744 owner_epoch: uuid::Uuid::new_v4(),
21745 write_id: uuid::Uuid::new_v4(),
21746 source: TransitionSourceIdentity {
21747 bucket: "source-bucket".to_string(),
21748 object: format!("source-{case}"),
21749 version_id: (source_mode == TransitionSourceVersionMode::Versioned).then(uuid::Uuid::new_v4),
21750 data_dir: uuid::Uuid::new_v4(),
21751 mod_time_unix_nanos: 1_770_000_000_000_000_000,
21752 size: 42,
21753 etag: "source-etag".to_string(),
21754 version_mode: source_mode,
21755 },
21756 tier_name: tier_name.to_string(),
21757 backend_fingerprint: backend_identity,
21758 not_after_unix_nanos: 1,
21759 })
21760 .expect("transaction should build");
21761 transaction
21762 .advance(transaction.fence(), TransitionTransactionState::Uploaded, Some(remote_version))
21763 .expect("transaction should enter uploaded state");
21764 backend.set_put_remote_version(Some(put_version.clone())).await;
21765 let candidate = bytes::Bytes::from_static(b"orphan candidate");
21766 backend
21767 .put(
21768 &transaction.remote_object,
21769 ReaderImpl::Body(candidate.clone()),
21770 i64::try_from(candidate.len()).expect("test candidate length should fit i64"),
21771 )
21772 .await
21773 .expect("mock backend should accept candidate");
21774 save_transition_transaction_record(store.clone(), &transaction)
21775 .await
21776 .expect("transaction record should persist");
21777 recovery_control_ids
21778 .push(transition_recovery_control_id(&transaction).expect("transition recovery control id should derive"));
21779 expected_removes.push((transaction.remote_object, put_version));
21780 }
21781
21782 let stats = recover_transition_transaction_records(store.clone(), 100, None)
21783 .await
21784 .expect("transition transaction recovery should run");
21785
21786 assert_eq!((stats.scanned, stats.recovered, stats.retained, stats.failed), (3, 3, 0, 0));
21787 assert_eq!(transition_transaction_record_count(store.clone()).await, 0);
21788 let mut actual_removes = backend.remove_versions().await;
21789 actual_removes.sort();
21790 expected_removes.sort();
21791 assert_eq!(actual_removes, expected_removes, "recovery must preserve each remote version shape");
21792 assert_eq!(backend.exact_remove_count(), 2);
21793 assert_eq!(backend.object_count().await, 0);
21794 for control_id in recovery_control_ids {
21795 let control = load_recovery_control(store.clone(), IlmRecoveryProtocol::TransitionTransaction, &control_id)
21796 .await
21797 .expect("completed recovery control should remain inspectable");
21798 assert_eq!(control.control.classification, IlmRecoveryClassification::Terminal);
21799 assert_eq!(control.control.attempt_count, 1);
21800 assert!(control.control.owner.is_none());
21801 }
21802
21803 let replay = recover_transition_transaction_records(store, 100, None)
21804 .await
21805 .expect("replayed recovery should remain idempotent");
21806 assert_eq!((replay.scanned, replay.recovered, replay.retained, replay.failed), (0, 0, 0, 0));
21807 assert_eq!(
21808 backend.remove_versions().await.len(),
21809 3,
21810 "each expired candidate must be deleted only once"
21811 );
21812 }
21813
21814 #[cfg(feature = "test-util")]
21815 #[tokio::test]
21816 #[serial_test::serial(storage_class_env)]
21817 async fn transition_transaction_recovery_resumes_source_cleanup_after_terminal_crash() {
21818 let temp_dir = tempfile::tempdir().expect("create temp store dir");
21819 let (ctx, store, _shutdown) =
21820 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "transition-transaction-terminal-crash", &[4]))
21821 .await;
21822 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
21823
21824 let tier_name = "TXTERMINALCRASH";
21825 let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
21826 let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
21827 .await
21828 .expect("tier lease should resolve")
21829 .backend_identity();
21830 let remote_version = uuid::Uuid::new_v4().to_string();
21831 let mut transaction = TransitionTransaction::new(TransitionTransactionInit {
21832 deployment_id: ctx.deployment_id().expect("test store should initialize deployment id"),
21833 transaction_id: uuid::Uuid::new_v4(),
21834 owner_epoch: uuid::Uuid::new_v4(),
21835 write_id: uuid::Uuid::new_v4(),
21836 source: TransitionSourceIdentity {
21837 bucket: "source-bucket".to_string(),
21838 object: "source-object".to_string(),
21839 version_id: Some(uuid::Uuid::new_v4()),
21840 data_dir: uuid::Uuid::new_v4(),
21841 mod_time_unix_nanos: 1_770_000_000_000_000_000,
21842 size: 42,
21843 etag: "source-etag".to_string(),
21844 version_mode: TransitionSourceVersionMode::Versioned,
21845 },
21846 tier_name: tier_name.to_string(),
21847 backend_fingerprint: backend_identity,
21848 not_after_unix_nanos: 1,
21849 })
21850 .expect("transaction should build");
21851 transaction
21852 .advance(
21853 transaction.fence(),
21854 TransitionTransactionState::Uploaded,
21855 Some(TransitionRemoteVersion::versioned(remote_version.clone())),
21856 )
21857 .expect("transaction should enter uploaded state");
21858 backend.set_put_remote_version(Some(remote_version)).await;
21859 let candidate = bytes::Bytes::from_static(b"terminal crash candidate");
21860 backend
21861 .put(
21862 &transaction.remote_object,
21863 ReaderImpl::Body(candidate.clone()),
21864 i64::try_from(candidate.len()).expect("test candidate length should fit i64"),
21865 )
21866 .await
21867 .expect("mock backend should accept candidate");
21868 save_transition_transaction_record(store.clone(), &transaction)
21869 .await
21870 .expect("transaction record should persist");
21871 let control_id = transition_recovery_control_id(&transaction).expect("control id should derive");
21872
21873 let barrier = TransitionRecoveryTerminalBarrier::install(transaction.transaction_id);
21874 let recovery_store = store.clone();
21875 let recovery = tokio::spawn(async move { recover_transition_transaction_records(recovery_store, 100, None).await });
21876 barrier.wait_until_paused().await;
21877 let terminal = load_recovery_control(store.clone(), IlmRecoveryProtocol::TransitionTransaction, &control_id)
21878 .await
21879 .expect("terminal control should persist before source cleanup");
21880 assert_eq!(terminal.control.classification, IlmRecoveryClassification::Terminal);
21881 assert_eq!(transition_transaction_record_count(store.clone()).await, 1);
21882 assert_eq!(backend.object_count().await, 0);
21883 assert_eq!(backend.exact_remove_count(), 1);
21884
21885 recovery.abort();
21886 assert!(
21887 recovery
21888 .await
21889 .expect_err("recovery should be cancelled at the crash boundary")
21890 .is_cancelled()
21891 );
21892 drop(barrier);
21893
21894 let replay = recover_transition_transaction_records(store.clone(), 100, None)
21895 .await
21896 .expect("terminal control should resume source cleanup without another remote delete");
21897 assert_eq!((replay.scanned, replay.recovered, replay.retained, replay.failed), (1, 1, 0, 0));
21898 assert_eq!(transition_transaction_record_count(store).await, 0);
21899 assert_eq!(backend.exact_remove_count(), 1, "terminal replay must not repeat the remote delete");
21900 }
21901
21902 #[cfg(feature = "test-util")]
21903 #[tokio::test]
21904 #[serial_test::serial(storage_class_env)]
21905 async fn transition_transaction_recovery_cas_loses_to_active_state_advance() {
21906 let temp_dir = tempfile::tempdir().expect("create temp store dir");
21907 let (ctx, store, _shutdown) =
21908 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "transition-transaction-recovery-cas", &[4]))
21909 .await;
21910 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
21911
21912 let tier_name = "TXRECOVERYCAS";
21913 let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
21914 let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
21915 .await
21916 .expect("tier lease should resolve")
21917 .backend_identity();
21918 let remote_version = uuid::Uuid::new_v4().to_string();
21919 let mut uploaded = TransitionTransaction::new(TransitionTransactionInit {
21920 deployment_id: ctx.deployment_id().expect("test store should initialize deployment id"),
21921 transaction_id: uuid::Uuid::new_v4(),
21922 owner_epoch: uuid::Uuid::new_v4(),
21923 write_id: uuid::Uuid::new_v4(),
21924 source: TransitionSourceIdentity {
21925 bucket: "source-bucket".to_string(),
21926 object: "source-object".to_string(),
21927 version_id: Some(uuid::Uuid::new_v4()),
21928 data_dir: uuid::Uuid::new_v4(),
21929 mod_time_unix_nanos: 1_770_000_000_000_000_000,
21930 size: 42,
21931 etag: "source-etag".to_string(),
21932 version_mode: TransitionSourceVersionMode::Versioned,
21933 },
21934 tier_name: tier_name.to_string(),
21935 backend_fingerprint: backend_identity,
21936 not_after_unix_nanos: 1,
21937 })
21938 .expect("transaction should build");
21939 uploaded
21940 .advance(
21941 uploaded.fence(),
21942 TransitionTransactionState::Uploaded,
21943 Some(TransitionRemoteVersion::versioned(remote_version.clone())),
21944 )
21945 .expect("transaction should enter uploaded state");
21946 backend.set_put_remote_version(Some(remote_version)).await;
21947 let candidate = bytes::Bytes::from_static(b"CAS-owned transition remote candidate");
21948 backend
21949 .put(
21950 &uploaded.remote_object,
21951 ReaderImpl::Body(candidate.clone()),
21952 i64::try_from(candidate.len()).expect("test candidate length should fit i64"),
21953 )
21954 .await
21955 .expect("mock backend should accept candidate");
21956 save_transition_transaction_record(store.clone(), &uploaded)
21957 .await
21958 .expect("transaction record should persist");
21959 let recovery_control_id =
21960 transition_recovery_control_id(&uploaded).expect("transition recovery control id should derive");
21961
21962 let barrier = TransitionRecoveryClaimBarrier::install(uploaded.transaction_id);
21963 let recovery_store = store.clone();
21964 let recovery = tokio::spawn(async move { recover_transition_transaction_records(recovery_store, 100, None).await });
21965 barrier.wait_until_paused().await;
21966
21967 let mut active = uploaded.clone();
21968 active
21969 .advance(active.fence(), TransitionTransactionState::LocalCommitStarted, None)
21970 .expect("active owner should advance to local commit");
21971 save_transition_transaction_record_if_current(store.clone(), &uploaded, &active)
21972 .await
21973 .expect("active owner should win the persisted CAS");
21974 barrier.release();
21975
21976 let stats = recovery
21977 .await
21978 .expect("recovery task should join")
21979 .expect("recovery should treat the lost CAS as a retained transaction");
21980 assert_eq!((stats.scanned, stats.recovered, stats.retained, stats.failed), (1, 0, 1, 0));
21981 assert_eq!(
21982 load_transition_transaction_record(store.clone(), uploaded.transaction_id)
21983 .await
21984 .expect("newer transaction revision must remain"),
21985 active
21986 );
21987 let control = load_recovery_control(store, IlmRecoveryProtocol::TransitionTransaction, &recovery_control_id)
21988 .await
21989 .expect("lost source CAS should retain a retryable recovery control");
21990 assert_eq!(control.control.classification, IlmRecoveryClassification::Retrying);
21991 assert_eq!(control.control.consecutive_failure_count, 1);
21992 assert_eq!(control.control.last_error_code, IlmRecoveryErrorCode::SourceGenerationChanged);
21993 assert_eq!(backend.object_count().await, 1, "a stale recovery must not delete the candidate");
21994 assert_eq!(backend.remove_count().await, 0);
21995 }
21996
21997 #[cfg(feature = "test-util")]
21998 #[tokio::test]
21999 #[serial_test::serial(storage_class_env)]
22000 async fn transition_transaction_recovery_quarantines_missing_required_fields() {
22001 let temp_dir = tempfile::tempdir().expect("create temp store dir");
22002 let (ctx, store, _shutdown) = without_storage_class_env(build_isolated_test_store(
22003 temp_dir.path(),
22004 "transition-transaction-recovery-corrupt",
22005 &[4],
22006 ))
22007 .await;
22008 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
22009
22010 let tier_name = "TXRECOVERYCORRUPT";
22011 let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
22012 let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
22013 .await
22014 .expect("tier lease should resolve")
22015 .backend_identity();
22016 let remote_version = uuid::Uuid::new_v4().to_string();
22017 let mut transaction = TransitionTransaction::new(TransitionTransactionInit {
22018 deployment_id: ctx.deployment_id().expect("test store should initialize deployment id"),
22019 transaction_id: uuid::Uuid::new_v4(),
22020 owner_epoch: uuid::Uuid::new_v4(),
22021 write_id: uuid::Uuid::new_v4(),
22022 source: TransitionSourceIdentity {
22023 bucket: "source-bucket".to_string(),
22024 object: "source-object".to_string(),
22025 version_id: None,
22026 data_dir: uuid::Uuid::new_v4(),
22027 mod_time_unix_nanos: 1_770_000_000_000_000_000,
22028 size: 42,
22029 etag: "source-etag".to_string(),
22030 version_mode: TransitionSourceVersionMode::Unversioned,
22031 },
22032 tier_name: tier_name.to_string(),
22033 backend_fingerprint: backend_identity,
22034 not_after_unix_nanos: 1,
22035 })
22036 .expect("transaction should build");
22037 transaction
22038 .advance(
22039 transaction.fence(),
22040 TransitionTransactionState::Uploaded,
22041 Some(TransitionRemoteVersion::versioned(remote_version.clone())),
22042 )
22043 .expect("transaction should enter uploaded state");
22044 backend.set_put_remote_version(Some(remote_version)).await;
22045 let candidate = bytes::Bytes::from_static(b"corrupt journal candidate");
22046 backend
22047 .put(
22048 &transaction.remote_object,
22049 ReaderImpl::Body(candidate.clone()),
22050 i64::try_from(candidate.len()).expect("test candidate length should fit i64"),
22051 )
22052 .await
22053 .expect("mock backend should accept candidate");
22054
22055 let mut persisted: serde_json::Value = serde_json::from_slice(&transaction.encode().expect("transaction should encode"))
22056 .expect("transaction should be JSON");
22057 persisted["transaction"]
22058 .as_object_mut()
22059 .expect("transaction payload should be an object")
22060 .remove("not_after_unix_nanos");
22061 let path =
22062 transition_transaction_record_object_name(transaction.transaction_id).expect("transaction path should be canonical");
22063 com::save_config(
22064 store.clone(),
22065 &path,
22066 serde_json::to_vec(&persisted).expect("corrupt fixture should encode"),
22067 )
22068 .await
22069 .expect("corrupt transaction fixture should persist");
22070
22071 let stats = recover_transition_transaction_records(store.clone(), 100, None)
22072 .await
22073 .expect("recovery scan should isolate a corrupt record");
22074 assert_eq!((stats.scanned, stats.recovered, stats.retained, stats.failed), (1, 0, 0, 1));
22075 assert_eq!(
22076 transition_transaction_record_count(store).await,
22077 1,
22078 "corrupt evidence must remain quarantined"
22079 );
22080 assert_eq!(backend.object_count().await, 1, "corrupt evidence must never authorize remote DELETE");
22081 assert_eq!(backend.remove_count().await, 0);
22082 }
22083
22084 #[cfg(feature = "test-util")]
22085 #[tokio::test]
22086 #[serial_test::serial(storage_class_env)]
22087 async fn transition_transaction_recovery_retries_cleanup_pending_candidate() {
22088 let temp_dir = tempfile::tempdir().expect("create temp store dir");
22089 let (ctx, store, _shutdown) =
22090 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "transition-transaction-cleanup", &[4])).await;
22091 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
22092
22093 let tier_name = "TXCLEANUP";
22094 let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
22095 let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
22096 .await
22097 .expect("tier lease should resolve")
22098 .backend_identity();
22099 let remote_version = uuid::Uuid::new_v4().to_string();
22100 let mut transaction = TransitionTransaction::new(TransitionTransactionInit {
22101 deployment_id: ctx.deployment_id().expect("test store should initialize deployment id"),
22102 transaction_id: uuid::Uuid::new_v4(),
22103 owner_epoch: uuid::Uuid::new_v4(),
22104 write_id: uuid::Uuid::new_v4(),
22105 source: TransitionSourceIdentity {
22106 bucket: "source-bucket".to_string(),
22107 object: "source-object".to_string(),
22108 version_id: Some(uuid::Uuid::new_v4()),
22109 data_dir: uuid::Uuid::new_v4(),
22110 mod_time_unix_nanos: 1_770_000_000_000_000_000,
22111 size: 42,
22112 etag: "source-etag".to_string(),
22113 version_mode: TransitionSourceVersionMode::Versioned,
22114 },
22115 tier_name: tier_name.to_string(),
22116 backend_fingerprint: backend_identity,
22117 not_after_unix_nanos: 1_780_000_000_000_000_000,
22118 })
22119 .expect("transaction should build");
22120 let uploaded_fence = transaction
22121 .advance(
22122 transaction.fence(),
22123 TransitionTransactionState::Uploaded,
22124 Some(TransitionRemoteVersion::versioned(remote_version.clone())),
22125 )
22126 .expect("transaction should enter uploaded state");
22127 transaction
22128 .mark_cleanup_pending(
22129 uploaded_fence,
22130 TransitionCleanupProof {
22131 transaction_id: transaction.transaction_id,
22132 write_id: transaction.write_id,
22133 remote_object: transaction.remote_object.clone(),
22134 remote_version: transaction.remote_version.clone(),
22135 backend_fingerprint: transaction.backend_fingerprint,
22136 decision: TransitionCleanupDecision::UploadAbortedBeforeLocalCommit,
22137 },
22138 )
22139 .expect("transaction should enter cleanup pending state");
22140 backend.set_put_remote_version(Some(remote_version.clone())).await;
22141 let candidate = bytes::Bytes::from_static(b"cleanup pending candidate");
22142 backend
22143 .put(
22144 &transaction.remote_object,
22145 ReaderImpl::Body(candidate.clone()),
22146 i64::try_from(candidate.len()).expect("test candidate length should fit i64"),
22147 )
22148 .await
22149 .expect("mock backend should accept candidate");
22150 save_transition_transaction_record(store.clone(), &transaction)
22151 .await
22152 .expect("transaction record should persist");
22153
22154 let stats = recover_transition_transaction_records(store.clone(), 100, None)
22155 .await
22156 .expect("transition transaction recovery should run");
22157
22158 assert_eq!((stats.scanned, stats.recovered, stats.retained, stats.failed), (1, 1, 0, 0));
22159 assert_eq!(transition_transaction_record_count(store.clone()).await, 0);
22160 assert_eq!(
22161 backend.remove_versions().await,
22162 vec![(transaction.remote_object.clone(), remote_version)],
22163 "cleanup pending recovery must retry the exact remote candidate delete"
22164 );
22165 assert_eq!(backend.exact_remove_count(), 1);
22166 assert_eq!(backend.object_count().await, 0);
22167 }
22168
22169 #[cfg(feature = "test-util")]
22170 #[tokio::test]
22171 #[serial_test::serial(storage_class_env)]
22172 async fn transition_transaction_recovery_retries_cleanup_pending_unversioned_candidate() {
22173 let temp_dir = tempfile::tempdir().expect("create temp store dir");
22174 let (ctx, store, _shutdown) = without_storage_class_env(build_isolated_test_store(
22175 temp_dir.path(),
22176 "transition-transaction-cleanup-unversioned",
22177 &[4],
22178 ))
22179 .await;
22180 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
22181
22182 let tier_name = "TXCLEANUPUNVERSIONED";
22183 let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
22184 let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
22185 .await
22186 .expect("tier lease should resolve")
22187 .backend_identity();
22188 let mut transaction = TransitionTransaction::new(TransitionTransactionInit {
22189 deployment_id: ctx.deployment_id().expect("test store should initialize deployment id"),
22190 transaction_id: uuid::Uuid::new_v4(),
22191 owner_epoch: uuid::Uuid::new_v4(),
22192 write_id: uuid::Uuid::new_v4(),
22193 source: TransitionSourceIdentity {
22194 bucket: "source-bucket".to_string(),
22195 object: "source-object".to_string(),
22196 version_id: None,
22197 data_dir: uuid::Uuid::new_v4(),
22198 mod_time_unix_nanos: 1_770_000_000_000_000_000,
22199 size: 42,
22200 etag: "source-etag".to_string(),
22201 version_mode: TransitionSourceVersionMode::Unversioned,
22202 },
22203 tier_name: tier_name.to_string(),
22204 backend_fingerprint: backend_identity,
22205 not_after_unix_nanos: 1_780_000_000_000_000_000,
22206 })
22207 .expect("transaction should build");
22208 let uploaded_fence = transaction
22209 .advance(
22210 transaction.fence(),
22211 TransitionTransactionState::Uploaded,
22212 Some(TransitionRemoteVersion::unversioned()),
22213 )
22214 .expect("transaction should enter uploaded state");
22215 transaction
22216 .mark_cleanup_pending(
22217 uploaded_fence,
22218 TransitionCleanupProof {
22219 transaction_id: transaction.transaction_id,
22220 write_id: transaction.write_id,
22221 remote_object: transaction.remote_object.clone(),
22222 remote_version: transaction.remote_version.clone(),
22223 backend_fingerprint: transaction.backend_fingerprint,
22224 decision: TransitionCleanupDecision::UploadAbortedBeforeLocalCommit,
22225 },
22226 )
22227 .expect("transaction should enter cleanup pending state");
22228 backend.set_put_remote_version(Some(String::new())).await;
22229 let candidate = bytes::Bytes::from_static(b"cleanup pending unversioned candidate");
22230 backend
22231 .put(
22232 &transaction.remote_object,
22233 ReaderImpl::Body(candidate.clone()),
22234 i64::try_from(candidate.len()).expect("test candidate length should fit i64"),
22235 )
22236 .await
22237 .expect("mock backend should accept candidate");
22238 save_transition_transaction_record(store.clone(), &transaction)
22239 .await
22240 .expect("transaction record should persist");
22241
22242 let stats = recover_transition_transaction_records(store.clone(), 100, None)
22243 .await
22244 .expect("transition transaction recovery should run");
22245
22246 assert_eq!((stats.scanned, stats.recovered, stats.retained, stats.failed), (1, 1, 0, 0));
22247 assert_eq!(transition_transaction_record_count(store.clone()).await, 0);
22248 assert_eq!(
22249 backend.remove_versions().await,
22250 vec![(transaction.remote_object.clone(), String::new())],
22251 "unversioned cleanup must not send a synthetic remote version"
22252 );
22253 assert_eq!(backend.exact_remove_count(), 0);
22254 assert_eq!(backend.object_count().await, 0);
22255 }
22256
22257 #[cfg(feature = "test-util")]
22258 #[tokio::test]
22259 #[serial_test::serial(storage_class_env)]
22260 async fn transition_transaction_recovery_keeps_cleanup_pending_record_when_delete_fails() {
22261 let temp_dir = tempfile::tempdir().expect("create temp store dir");
22262 let (ctx, store, _shutdown) =
22263 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "transition-transaction-cleanup-fail", &[4]))
22264 .await;
22265 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
22266
22267 let tier_name = "TXCLEANUPFAIL";
22268 let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
22269 let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
22270 .await
22271 .expect("tier lease should resolve")
22272 .backend_identity();
22273 let remote_version = uuid::Uuid::new_v4().to_string();
22274 let mut transaction = TransitionTransaction::new(TransitionTransactionInit {
22275 deployment_id: ctx.deployment_id().expect("test store should initialize deployment id"),
22276 transaction_id: uuid::Uuid::new_v4(),
22277 owner_epoch: uuid::Uuid::new_v4(),
22278 write_id: uuid::Uuid::new_v4(),
22279 source: TransitionSourceIdentity {
22280 bucket: "source-bucket".to_string(),
22281 object: "source-object".to_string(),
22282 version_id: Some(uuid::Uuid::new_v4()),
22283 data_dir: uuid::Uuid::new_v4(),
22284 mod_time_unix_nanos: 1_770_000_000_000_000_000,
22285 size: 42,
22286 etag: "source-etag".to_string(),
22287 version_mode: TransitionSourceVersionMode::Versioned,
22288 },
22289 tier_name: tier_name.to_string(),
22290 backend_fingerprint: backend_identity,
22291 not_after_unix_nanos: 1_780_000_000_000_000_000,
22292 })
22293 .expect("transaction should build");
22294 transaction
22295 .advance(
22296 transaction.fence(),
22297 TransitionTransactionState::Uploaded,
22298 Some(TransitionRemoteVersion::versioned(remote_version.clone())),
22299 )
22300 .expect("transaction should enter uploaded state");
22301 let candidate = bytes::Bytes::from_static(b"cleanup pending candidate retained after failure");
22302 backend.set_put_remote_version(Some(remote_version)).await;
22303 backend
22304 .put(
22305 &transaction.remote_object,
22306 ReaderImpl::Body(candidate.clone()),
22307 i64::try_from(candidate.len()).expect("test candidate length should fit i64"),
22308 )
22309 .await
22310 .expect("mock backend should accept candidate");
22311 save_transition_transaction_record(store.clone(), &transaction)
22312 .await
22313 .expect("transaction record should persist");
22314 let recovery_control_id =
22315 transition_recovery_control_id(&transaction).expect("transition recovery control id should derive");
22316
22317 backend.set_remove_failure(true);
22318 let stats = recover_transition_transaction_records(store.clone(), 100, None)
22319 .await
22320 .expect("transition transaction recovery should keep scanning after cleanup failure");
22321
22322 assert_eq!((stats.scanned, stats.recovered, stats.retained, stats.failed), (1, 0, 0, 1));
22323 assert_eq!(
22324 transition_transaction_record_count(store.clone()).await,
22325 1,
22326 "failed cleanup must keep the cleanup-pending transaction for retry"
22327 );
22328 assert_eq!(backend.remove_versions().await, Vec::<(String, String)>::new());
22329 assert_eq!(backend.exact_remove_count(), 1);
22330 assert_eq!(backend.object_count().await, 1);
22331 let control = load_recovery_control(store.clone(), IlmRecoveryProtocol::TransitionTransaction, &recovery_control_id)
22332 .await
22333 .expect("failed recovery control should persist");
22334 assert_eq!(control.control.classification, IlmRecoveryClassification::Retrying);
22335 assert_eq!(control.control.attempt_count, 1);
22336 assert_eq!(control.control.consecutive_failure_count, 1);
22337 assert!(
22338 control
22339 .control
22340 .next_attempt_at_unix_nanos
22341 .is_some_and(|next| next > OffsetDateTime::now_utc().unix_timestamp_nanos() as i64)
22342 );
22343
22344 backend.set_remove_failure(false);
22345 let replay = recover_transition_transaction_records(store.clone(), 100, None)
22346 .await
22347 .expect("recovery before the persisted deadline should be skipped");
22348 assert_eq!((replay.scanned, replay.recovered, replay.retained, replay.failed), (1, 0, 1, 0));
22349 assert_eq!(backend.exact_remove_count(), 1, "persisted backoff must prevent an immediate retry");
22350
22351 let retry_at = control
22352 .control
22353 .next_attempt_at_unix_nanos
22354 .expect("retry deadline should persist");
22355 let retried = recover_transition_transaction_records_at(store.clone(), 100, None, i128::from(retry_at) + 1)
22356 .await
22357 .expect("recovery at the persisted deadline should retry the advanced source generation");
22358 assert_eq!((retried.scanned, retried.recovered, retried.retained, retried.failed), (1, 1, 0, 0));
22359 assert_eq!(backend.exact_remove_count(), 2);
22360 assert_eq!(backend.object_count().await, 0);
22361 assert_eq!(transition_transaction_record_count(store.clone()).await, 0);
22362 let terminal = load_recovery_control(store, IlmRecoveryProtocol::TransitionTransaction, &recovery_control_id)
22363 .await
22364 .expect("completed retry control should remain inspectable");
22365 assert_eq!(terminal.control.classification, IlmRecoveryClassification::Terminal);
22366 assert_eq!(terminal.control.attempt_count, 2);
22367 }
22368
22369 #[cfg(feature = "test-util")]
22370 #[tokio::test]
22371 #[serial_test::serial(storage_class_env)]
22372 async fn transition_transaction_recovery_keeps_cleanup_pending_local_commit() {
22373 let temp_dir = tempfile::tempdir().expect("create temp store dir");
22374 let (ctx, store, _shutdown) = without_storage_class_env(build_isolated_test_store(
22375 temp_dir.path(),
22376 "transition-transaction-cleanup-committed",
22377 &[4],
22378 ))
22379 .await;
22380 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
22381
22382 let tier_name = "TXCLEANUPCOMMITTED";
22383 let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
22384 let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
22385 .await
22386 .expect("tier lease should resolve")
22387 .backend_identity();
22388 let bucket = "transition-transaction-cleanup-committed-bucket";
22389 let object = "object.bin";
22390 store
22391 .make_bucket(bucket, &MakeBucketOptions::default())
22392 .await
22393 .expect("bucket should be created");
22394 let mut reader = PutObjReader::from_vec(b"cleanup pending local commit record cleanup".repeat(1024));
22395 let original = store
22396 .put_object(bucket, object, &mut reader, &ObjectOptions::default())
22397 .await
22398 .expect("source object should be written");
22399 let opts = ObjectOptions {
22400 no_lock: true,
22401 transition: TransitionOptions {
22402 status: TRANSITION_PENDING.to_string(),
22403 tier: tier_name.to_string(),
22404 etag: original.etag.clone().expect("source object should have etag"),
22405 ..Default::default()
22406 },
22407 mod_time: original.mod_time,
22408 ..Default::default()
22409 };
22410 store
22411 .transition_object(bucket, object, &opts)
22412 .await
22413 .expect("transition should commit");
22414 let committed = store
22415 .get_object_info(
22416 bucket,
22417 object,
22418 &ObjectOptions {
22419 no_lock: true,
22420 metadata_cache_safe: false,
22421 ..Default::default()
22422 },
22423 )
22424 .await
22425 .expect("committed object info should be readable");
22426 let mut remote_parts = committed.transitioned_object.name.rsplit('/');
22427 let write_id = uuid::Uuid::parse_str(remote_parts.next().expect("remote object should contain write id"))
22428 .expect("write id should parse");
22429 let transaction_id = uuid::Uuid::parse_str(remote_parts.next().expect("remote object should contain transaction id"))
22430 .expect("transaction id should parse");
22431 let source = TransitionSourceIdentity {
22432 bucket: bucket.to_string(),
22433 object: object.to_string(),
22434 version_id: None,
22435 data_dir: original.data_dir.expect("source object should have data_dir"),
22436 mod_time_unix_nanos: original
22437 .mod_time
22438 .expect("source object should have mod_time")
22439 .unix_timestamp_nanos()
22440 .try_into()
22441 .expect("test timestamp should fit i64"),
22442 size: original.size,
22443 etag: original.etag.expect("source object should have etag"),
22444 version_mode: TransitionSourceVersionMode::Unversioned,
22445 };
22446 let mut transaction = TransitionTransaction::new(TransitionTransactionInit {
22447 deployment_id: ctx.deployment_id().expect("test store should initialize deployment id"),
22448 transaction_id,
22449 owner_epoch: uuid::Uuid::new_v4(),
22450 write_id,
22451 source,
22452 tier_name: tier_name.to_string(),
22453 backend_fingerprint: backend_identity,
22454 not_after_unix_nanos: 1_780_000_000_000_000_000,
22455 })
22456 .expect("transaction should build");
22457 transaction
22458 .advance(
22459 transaction.fence(),
22460 TransitionTransactionState::Uploaded,
22461 Some(TransitionRemoteVersion::known_from_put_response(
22462 committed.transitioned_object.version_id.clone(),
22463 )),
22464 )
22465 .expect("transaction should enter uploaded state");
22466 let local_commit_fence = transaction
22467 .advance(transaction.fence(), TransitionTransactionState::LocalCommitStarted, None)
22468 .expect("transaction should enter local commit state");
22469 transaction
22470 .mark_cleanup_pending(
22471 local_commit_fence,
22472 TransitionCleanupProof {
22473 transaction_id: transaction.transaction_id,
22474 write_id: transaction.write_id,
22475 remote_object: transaction.remote_object.clone(),
22476 remote_version: transaction.remote_version.clone(),
22477 backend_fingerprint: transaction.backend_fingerprint,
22478 decision: TransitionCleanupDecision::SourceReconciledUnchanged {
22479 observed_source: transaction.source.clone(),
22480 },
22481 },
22482 )
22483 .expect("transaction should enter cleanup pending state");
22484 save_transition_transaction_record(store.clone(), &transaction)
22485 .await
22486 .expect("transaction record should persist");
22487
22488 let stats = recover_transition_transaction_records(store.clone(), 100, None)
22489 .await
22490 .expect("transition transaction recovery should run");
22491
22492 assert_eq!((stats.scanned, stats.recovered, stats.retained, stats.failed), (1, 1, 0, 0));
22493 assert_eq!(transition_transaction_record_count(store.clone()).await, 0);
22494 assert_eq!(
22495 backend.object_count().await,
22496 1,
22497 "cleanup pending recovery must keep the remote body once local metadata references it"
22498 );
22499 assert_eq!(backend.remove_count().await, 0);
22500 }
22501
22502 #[cfg(feature = "test-util")]
22503 #[tokio::test]
22504 #[serial_test::serial(storage_class_env)]
22505 async fn transition_transaction_recovery_drops_record_after_confirmed_local_commit() {
22506 let temp_dir = tempfile::tempdir().expect("create temp store dir");
22507 let (ctx, store, _shutdown) =
22508 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "transition-transaction-committed", &[4])).await;
22509 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
22510
22511 let tier_name = "TXCOMMITTED";
22512 let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
22513 let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
22514 .await
22515 .expect("tier lease should resolve")
22516 .backend_identity();
22517 let bucket = "transition-transaction-committed-bucket";
22518 let object = "object.bin";
22519 store
22520 .make_bucket(bucket, &MakeBucketOptions::default())
22521 .await
22522 .expect("bucket should be created");
22523 let mut reader = PutObjReader::from_vec(b"confirmed local commit record cleanup".repeat(1024));
22524 let original = store
22525 .put_object(bucket, object, &mut reader, &ObjectOptions::default())
22526 .await
22527 .expect("source object should be written");
22528 let opts = ObjectOptions {
22529 no_lock: true,
22530 transition: TransitionOptions {
22531 status: TRANSITION_PENDING.to_string(),
22532 tier: tier_name.to_string(),
22533 etag: original.etag.clone().expect("source object should have etag"),
22534 ..Default::default()
22535 },
22536 mod_time: original.mod_time,
22537 ..Default::default()
22538 };
22539 store
22540 .transition_object(bucket, object, &opts)
22541 .await
22542 .expect("transition should commit");
22543 let committed = store
22544 .get_object_info(
22545 bucket,
22546 object,
22547 &ObjectOptions {
22548 no_lock: true,
22549 metadata_cache_safe: false,
22550 ..Default::default()
22551 },
22552 )
22553 .await
22554 .expect("committed object info should be readable");
22555 let mut remote_parts = committed.transitioned_object.name.rsplit('/');
22556 let write_id = uuid::Uuid::parse_str(remote_parts.next().expect("remote object should contain write id"))
22557 .expect("write id should parse");
22558 let transaction_id = uuid::Uuid::parse_str(remote_parts.next().expect("remote object should contain transaction id"))
22559 .expect("transaction id should parse");
22560 let mut transaction = TransitionTransaction::new(TransitionTransactionInit {
22561 deployment_id: ctx.deployment_id().expect("test store should initialize deployment id"),
22562 transaction_id,
22563 owner_epoch: uuid::Uuid::new_v4(),
22564 write_id,
22565 source: TransitionSourceIdentity {
22566 bucket: bucket.to_string(),
22567 object: object.to_string(),
22568 version_id: None,
22569 data_dir: original.data_dir.expect("source object should have data_dir"),
22570 mod_time_unix_nanos: original
22571 .mod_time
22572 .expect("source object should have mod_time")
22573 .unix_timestamp_nanos()
22574 .try_into()
22575 .expect("test timestamp should fit i64"),
22576 size: original.size,
22577 etag: original.etag.expect("source object should have etag"),
22578 version_mode: TransitionSourceVersionMode::Unversioned,
22579 },
22580 tier_name: tier_name.to_string(),
22581 backend_fingerprint: backend_identity,
22582 not_after_unix_nanos: 1_780_000_000_000_000_000,
22583 })
22584 .expect("transaction should build");
22585 transaction
22586 .advance(
22587 transaction.fence(),
22588 TransitionTransactionState::Uploaded,
22589 Some(TransitionRemoteVersion::known_from_put_response(
22590 committed.transitioned_object.version_id.clone(),
22591 )),
22592 )
22593 .expect("transaction should enter uploaded state");
22594 transaction
22595 .advance(transaction.fence(), TransitionTransactionState::LocalCommitStarted, None)
22596 .expect("transaction should enter local commit state");
22597 save_transition_transaction_record(store.clone(), &transaction)
22598 .await
22599 .expect("transaction record should persist");
22600 assert_eq!(transition_transaction_record_count(store.clone()).await, 1);
22601
22602 let stats = recover_transition_transaction_records(store.clone(), 100, None)
22603 .await
22604 .expect("transition transaction recovery should run");
22605
22606 assert_eq!((stats.scanned, stats.recovered, stats.retained, stats.failed), (1, 1, 0, 0));
22607 assert_eq!(transition_transaction_record_count(store.clone()).await, 0);
22608 assert_eq!(
22609 backend.object_count().await,
22610 1,
22611 "confirmed local commit recovery must not delete the committed remote body"
22612 );
22613 assert_eq!(backend.remove_count().await, 0);
22614 }
22615
22616 #[cfg(feature = "test-util")]
22617 #[tokio::test]
22618 #[serial_test::serial(storage_class_env)]
22619 async fn transition_transaction_recovery_retains_unproven_remote_candidates() {
22620 let temp_dir = tempfile::tempdir().expect("create temp store dir");
22621 let (ctx, store, _shutdown) =
22622 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "transition-transaction-unproven", &[4])).await;
22623 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
22624
22625 let tier_name = "TXUNPROVEN";
22626 let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
22627 let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
22628 .await
22629 .expect("tier lease should resolve")
22630 .backend_identity();
22631 let remote_version = uuid::Uuid::new_v4().to_string();
22632 let new_transaction = || {
22633 TransitionTransaction::new(TransitionTransactionInit {
22634 deployment_id: ctx.deployment_id().expect("test store should initialize deployment id"),
22635 transaction_id: uuid::Uuid::new_v4(),
22636 owner_epoch: uuid::Uuid::new_v4(),
22637 write_id: uuid::Uuid::new_v4(),
22638 source: TransitionSourceIdentity {
22639 bucket: "absent-source-bucket".to_string(),
22640 object: "source-object".to_string(),
22641 version_id: None,
22642 data_dir: uuid::Uuid::new_v4(),
22643 mod_time_unix_nanos: 1_770_000_000_000_000_000,
22644 size: 42,
22645 etag: "source-etag".to_string(),
22646 version_mode: TransitionSourceVersionMode::Unversioned,
22647 },
22648 tier_name: tier_name.to_string(),
22649 backend_fingerprint: backend_identity,
22650 not_after_unix_nanos: 1_780_000_000_000_000_000,
22651 })
22652 .expect("transaction should build")
22653 };
22654
22655 let upload_started = new_transaction();
22656 let mut local_commit_started = new_transaction();
22657 local_commit_started
22658 .advance(
22659 local_commit_started.fence(),
22660 TransitionTransactionState::Uploaded,
22661 Some(TransitionRemoteVersion::versioned(remote_version.clone())),
22662 )
22663 .expect("transaction should enter uploaded state");
22664 local_commit_started
22665 .advance(local_commit_started.fence(), TransitionTransactionState::LocalCommitStarted, None)
22666 .expect("transaction should enter local commit state");
22667 let upload_started_control_id =
22668 transition_recovery_control_id(&upload_started).expect("upload-started control id should derive");
22669 let local_commit_control_id =
22670 transition_recovery_control_id(&local_commit_started).expect("local-commit control id should derive");
22671
22672 backend.set_put_remote_version(Some(remote_version)).await;
22673 for transaction in [&upload_started, &local_commit_started] {
22674 let candidate = bytes::Bytes::from_static(b"unproven transition remote candidate");
22675 backend
22676 .put(
22677 &transaction.remote_object,
22678 ReaderImpl::Body(candidate.clone()),
22679 i64::try_from(candidate.len()).expect("test candidate length should fit i64"),
22680 )
22681 .await
22682 .expect("mock backend should accept candidate");
22683 save_transition_transaction_record(store.clone(), transaction)
22684 .await
22685 .expect("transaction record should persist");
22686 }
22687
22688 let stats = recover_transition_transaction_records(store.clone(), 100, None)
22689 .await
22690 .expect("transition transaction recovery should run");
22691
22692 assert_eq!((stats.scanned, stats.recovered, stats.retained, stats.failed), (2, 0, 2, 0));
22693 assert_eq!(
22694 transition_transaction_record_count(store.clone()).await,
22695 2,
22696 "an upload without completion proof or unproven local commit must remain for authoritative reconcile"
22697 );
22698 assert_eq!(backend.object_count().await, 2, "recovery must not delete an unproven remote candidate");
22699 assert_eq!(backend.remove_count().await, 0);
22700 assert_eq!(backend.exact_remove_count(), 0);
22701 let upload_started_control =
22702 load_recovery_control(store.clone(), IlmRecoveryProtocol::TransitionTransaction, &upload_started_control_id)
22703 .await
22704 .expect("upload-started control should persist");
22705 assert_eq!(
22706 upload_started_control.control.classification,
22707 IlmRecoveryClassification::RetainedAmbiguous
22708 );
22709 let local_commit_control =
22710 load_recovery_control(store.clone(), IlmRecoveryProtocol::TransitionTransaction, &local_commit_control_id)
22711 .await
22712 .expect("local-commit control should persist");
22713 assert_eq!(local_commit_control.control.classification, IlmRecoveryClassification::OperatorRequired);
22714
22715 let upload_status = inspect_transition_recovery_retry_for_operator(store.clone(), &upload_started_control_id)
22716 .await
22717 .expect("retained upload should be inspectable for a bounded retry");
22718 let local_status = inspect_transition_recovery_retry_for_operator(store.clone(), &local_commit_control_id)
22719 .await
22720 .expect("operator-required local commit should be inspectable for a bounded retry");
22721 assert!(upload_status.retry_ready);
22722 assert!(local_status.retry_ready);
22723 assert!(matches!(
22724 retry_transition_recovery_for_operator(
22725 store.clone(),
22726 &upload_started_control_id,
22727 upload_status.control_revision + 1,
22728 &upload_status.source_generation_sha256,
22729 )
22730 .await,
22731 Err(TransitionOperatorError::StaleRecoveryControl)
22732 ));
22733
22734 let put_count_before_retry = backend.put_count().await;
22735 let get_count_before_retry = backend.get_count().await;
22736 let remove_count_before_retry = backend.remove_count().await;
22737 let upload_retry = retry_transition_recovery_for_operator(
22738 store.clone(),
22739 &upload_started_control_id,
22740 upload_status.control_revision,
22741 &upload_status.source_generation_sha256,
22742 )
22743 .await
22744 .expect("exact retained upload generation should be rearmed");
22745 let local_retry = retry_transition_recovery_for_operator(
22746 store.clone(),
22747 &local_commit_control_id,
22748 local_status.control_revision,
22749 &local_status.source_generation_sha256,
22750 )
22751 .await
22752 .expect("exact operator-required local commit generation should be rearmed");
22753 assert_eq!(upload_retry.classification, IlmRecoveryClassification::Retrying);
22754 assert_eq!(local_retry.classification, IlmRecoveryClassification::Retrying);
22755 assert_eq!(upload_retry.attempt_count, upload_status.attempt_count);
22756 assert_eq!(local_retry.attempt_count, local_status.attempt_count);
22757 assert_eq!(backend.put_count().await, put_count_before_retry);
22758 assert_eq!(backend.get_count().await, get_count_before_retry);
22759 assert_eq!(backend.remove_count().await, remove_count_before_retry);
22760 assert_eq!(backend.exact_remove_count(), 0, "operator retry must not directly issue remote DELETE");
22761
22762 let retried = recover_transition_transaction_records(store.clone(), 100, None)
22763 .await
22764 .expect("rearmed records should be re-evaluated through normal recovery");
22765 assert_eq!((retried.scanned, retried.recovered, retried.retained, retried.failed), (2, 0, 2, 0));
22766 let upload_retained =
22767 load_recovery_control(store.clone(), IlmRecoveryProtocol::TransitionTransaction, &upload_started_control_id)
22768 .await
22769 .expect("upload retry result should persist");
22770 let local_retained = load_recovery_control(store, IlmRecoveryProtocol::TransitionTransaction, &local_commit_control_id)
22771 .await
22772 .expect("local commit retry result should persist");
22773 assert_eq!(upload_retained.control.classification, IlmRecoveryClassification::RetainedAmbiguous);
22774 assert_eq!(local_retained.control.classification, IlmRecoveryClassification::OperatorRequired);
22775 assert_eq!(upload_retained.control.attempt_count, upload_status.attempt_count + 1);
22776 assert_eq!(local_retained.control.attempt_count, local_status.attempt_count + 1);
22777 assert_eq!(backend.put_count().await, put_count_before_retry);
22778 assert_eq!(backend.get_count().await, get_count_before_retry);
22779 assert_eq!(backend.remove_count().await, remove_count_before_retry);
22780 assert_eq!(backend.exact_remove_count(), 0);
22781 }
22782
22783 #[cfg(feature = "test-util")]
22784 #[tokio::test]
22785 #[serial_test::serial(storage_class_env)]
22786 async fn transition_transaction_recovery_handles_provider_recovered_unknown_upload() {
22787 let versioned_remote = uuid::Uuid::new_v4().to_string();
22788 let nil_remote = uuid::Uuid::nil().to_string();
22789 for (case, tier_name, remote_version, should_recover) in [
22790 ("missing", "TXPROBEMISSING", None, true),
22791 ("unversioned", "TXPROBEUNVERSIONED", Some(String::new()), true),
22792 ("versioned", "TXPROBEVERSIONED", Some(versioned_remote), true),
22793 ("nil-version", "TXPROBENILVERSION", Some(nil_remote), false),
22794 ] {
22795 let temp_dir = tempfile::tempdir().expect("create temp store dir");
22796 let (ctx, store, _shutdown) = without_storage_class_env(build_isolated_test_store(
22797 temp_dir.path(),
22798 &format!("transition-transaction-probe-{case}"),
22799 &[4],
22800 ))
22801 .await;
22802 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
22803
22804 let backend = register_transition_reconcile_test_tier(&ctx.tier_config_mgr(), tier_name).await;
22805 let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
22806 .await
22807 .expect("tier lease should resolve")
22808 .backend_identity();
22809 let mut transaction = TransitionTransaction::new(TransitionTransactionInit {
22810 deployment_id: ctx.deployment_id().expect("test store should initialize deployment id"),
22811 transaction_id: uuid::Uuid::new_v4(),
22812 owner_epoch: uuid::Uuid::new_v4(),
22813 write_id: uuid::Uuid::new_v4(),
22814 source: TransitionSourceIdentity {
22815 bucket: "source-bucket".to_string(),
22816 object: "source-object".to_string(),
22817 version_id: None,
22818 data_dir: uuid::Uuid::new_v4(),
22819 mod_time_unix_nanos: 1_770_000_000_000_000_000,
22820 size: 42,
22821 etag: "source-etag".to_string(),
22822 version_mode: TransitionSourceVersionMode::Unversioned,
22823 },
22824 tier_name: tier_name.to_string(),
22825 backend_fingerprint: backend_identity,
22826 not_after_unix_nanos: 1,
22827 })
22828 .expect("transaction should build");
22829 transaction
22830 .advance(transaction.fence(), TransitionTransactionState::UploadOutcomeUnknown, None)
22831 .expect("transaction should enter unknown upload outcome state");
22832
22833 if let Some(version) = &remote_version {
22834 backend.set_put_remote_version(Some(version.clone())).await;
22835 let candidate = bytes::Bytes::from_static(b"provider-recovered transition remote candidate");
22836 backend
22837 .put(
22838 &transaction.remote_object,
22839 ReaderImpl::Body(candidate.clone()),
22840 i64::try_from(candidate.len()).expect("test candidate length should fit i64"),
22841 )
22842 .await
22843 .expect("mock backend should accept candidate");
22844 }
22845 save_transition_transaction_record(store.clone(), &transaction)
22846 .await
22847 .expect("transaction record should persist");
22848
22849 let stats = recover_transition_transaction_records(store.clone(), 100, None)
22850 .await
22851 .expect("transition transaction recovery should run");
22852
22853 assert_eq!(
22854 (stats.scanned, stats.recovered, stats.retained, stats.failed),
22855 if should_recover { (1, 1, 0, 0) } else { (1, 0, 1, 0) }
22856 );
22857 assert_eq!(transition_transaction_record_count(store.clone()).await, usize::from(!should_recover));
22858 assert_eq!(
22859 backend.object_count().await,
22860 usize::from(!should_recover),
22861 "case {case}: only a valid provider version state may be recovered destructively"
22862 );
22863 let removed = remote_version
22864 .filter(|_| should_recover)
22865 .map(|version| vec![(transaction.remote_object.clone(), version)])
22866 .unwrap_or_default();
22867 assert_eq!(
22868 backend.remove_versions().await,
22869 removed,
22870 "case {case}: recovery must delete only provider-recovered candidates"
22871 );
22872 assert_eq!(
22873 backend.exact_remove_count(),
22874 usize::from(removed.first().is_some_and(|(_, version)| !version.is_empty()))
22875 );
22876 }
22877 }
22878
22879 #[cfg(feature = "test-util")]
22880 #[tokio::test]
22881 #[serial_test::serial(storage_class_env)]
22882 async fn operator_reconcile_deletes_exact_candidate_before_finalizing_record() {
22883 let temp_dir = tempfile::tempdir().expect("create temp store dir");
22884 let (ctx, store, _shutdown) =
22885 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "transition-operator-reconcile", &[4])).await;
22886 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
22887
22888 let tier_name = "TXOPERATOR";
22889 let backend = register_transition_reconcile_test_tier(&ctx.tier_config_mgr(), tier_name).await;
22890 let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
22891 .await
22892 .expect("tier lease should resolve")
22893 .backend_identity();
22894 let mut transaction = TransitionTransaction::new(TransitionTransactionInit {
22895 deployment_id: ctx.deployment_id().expect("test store should initialize deployment id"),
22896 transaction_id: uuid::Uuid::new_v4(),
22897 owner_epoch: uuid::Uuid::new_v4(),
22898 write_id: uuid::Uuid::new_v4(),
22899 source: TransitionSourceIdentity {
22900 bucket: "source-bucket".to_string(),
22901 object: "source-object".to_string(),
22902 version_id: None,
22903 data_dir: uuid::Uuid::new_v4(),
22904 mod_time_unix_nanos: 1_770_000_000_000_000_000,
22905 size: 42,
22906 etag: "source-etag".to_string(),
22907 version_mode: TransitionSourceVersionMode::Unversioned,
22908 },
22909 tier_name: tier_name.to_string(),
22910 backend_fingerprint: backend_identity,
22911 not_after_unix_nanos: 1,
22912 })
22913 .expect("transaction should build");
22914 transaction
22915 .advance(transaction.fence(), TransitionTransactionState::UploadOutcomeUnknown, None)
22916 .expect("transaction should enter unknown upload outcome state");
22917
22918 let remote_version = uuid::Uuid::new_v4().to_string();
22919 backend.set_put_remote_version(Some(remote_version.clone())).await;
22920 let candidate = bytes::Bytes::from_static(b"operator-confirmed transition candidate");
22921 backend
22922 .put(
22923 &transaction.remote_object,
22924 ReaderImpl::Body(candidate.clone()),
22925 i64::try_from(candidate.len()).expect("test candidate length should fit i64"),
22926 )
22927 .await
22928 .expect("mock backend should accept candidate");
22929 save_transition_transaction_record(store.clone(), &transaction)
22930 .await
22931 .expect("transaction record should persist");
22932
22933 let status = inspect_transition_transaction_for_operator(store.clone(), transaction.transaction_id)
22934 .await
22935 .expect("operator inspection should probe the candidate");
22936 assert_eq!(status.probe, TransitionOperatorProbe::VersionedPresent(remote_version.clone()));
22937
22938 let wrong_version = uuid::Uuid::new_v4().to_string();
22939 let err = delete_transition_candidate_for_operator(store.clone(), transaction.transaction_id, &wrong_version)
22940 .await
22941 .expect_err("a mismatched exact version must fail before deleting a candidate");
22942 assert!(matches!(
22943 err,
22944 TransitionOperatorError::CandidateVersionMismatch {
22945 expected,
22946 actual: TransitionOperatorProbe::VersionedPresent(ref observed),
22947 } if expected == wrong_version && observed == &remote_version
22948 ));
22949 assert!(backend.contains(&transaction.remote_object).await);
22950 assert_eq!(backend.exact_remove_count(), 0);
22951 load_transition_transaction_record(store.clone(), transaction.transaction_id)
22952 .await
22953 .expect("an incorrect exact version must retain the transaction journal");
22954
22955 let result = delete_transition_candidate_for_operator(store.clone(), transaction.transaction_id, &remote_version)
22956 .await
22957 .expect("operator-confirmed exact candidate should be deleted");
22958 assert_eq!(result.status.probe, TransitionOperatorProbe::Missing);
22959 assert!(result.journal_observed_after_delete);
22960 assert_eq!(backend.exact_remove_count(), 1);
22961 assert_eq!(backend.remove_versions().await, vec![(transaction.remote_object.clone(), remote_version)]);
22962 load_transition_transaction_record(store.clone(), transaction.transaction_id)
22963 .await
22964 .expect("candidate deletion must retain the transaction journal");
22965
22966 finalize_missing_transition_transaction_for_operator(store.clone(), transaction.transaction_id)
22967 .await
22968 .expect("a separately confirmed missing candidate should permit finalization");
22969 assert!(matches!(
22970 load_transition_transaction_record(store, transaction.transaction_id).await,
22971 Err(Error::ConfigNotFound)
22972 ));
22973 }
22974
22975 #[cfg(feature = "test-util")]
22976 #[tokio::test]
22977 #[serial_test::serial(storage_class_env)]
22978 async fn operator_finalize_retains_record_without_missing_proof() {
22979 let temp_dir = tempfile::tempdir().expect("create temp store dir");
22980 let (ctx, store, _shutdown) =
22981 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "transition-operator-fail-closed", &[4])).await;
22982 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
22983
22984 let tier_name = "TXOPERATORFAIL";
22985 let backend = register_transition_reconcile_test_tier(&ctx.tier_config_mgr(), tier_name).await;
22986 let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
22987 .await
22988 .expect("tier lease should resolve")
22989 .backend_identity();
22990 let mut transaction = TransitionTransaction::new(TransitionTransactionInit {
22991 deployment_id: ctx.deployment_id().expect("test store should initialize deployment id"),
22992 transaction_id: uuid::Uuid::new_v4(),
22993 owner_epoch: uuid::Uuid::new_v4(),
22994 write_id: uuid::Uuid::new_v4(),
22995 source: TransitionSourceIdentity {
22996 bucket: "source-bucket".to_string(),
22997 object: "source-object".to_string(),
22998 version_id: None,
22999 data_dir: uuid::Uuid::new_v4(),
23000 mod_time_unix_nanos: 1_770_000_000_000_000_000,
23001 size: 42,
23002 etag: "source-etag".to_string(),
23003 version_mode: TransitionSourceVersionMode::Unversioned,
23004 },
23005 tier_name: tier_name.to_string(),
23006 backend_fingerprint: backend_identity,
23007 not_after_unix_nanos: 1,
23008 })
23009 .expect("transaction should build");
23010 transaction
23011 .advance(transaction.fence(), TransitionTransactionState::UploadOutcomeUnknown, None)
23012 .expect("transaction should enter unknown upload outcome state");
23013 save_transition_transaction_record(store.clone(), &transaction)
23014 .await
23015 .expect("transaction record should persist");
23016 backend
23017 .set_transition_candidate_probe_override(Some(TransitionCandidateProbe::Unsupported))
23018 .await;
23019
23020 assert!(matches!(
23021 finalize_missing_transition_transaction_for_operator(store.clone(), transaction.transaction_id).await,
23022 Err(TransitionOperatorError::CandidateNotMissing(TransitionOperatorProbe::Unsupported))
23023 ));
23024 load_transition_transaction_record(store, transaction.transaction_id)
23025 .await
23026 .expect("an unsupported probe must retain the transaction journal");
23027 assert_eq!(backend.remove_count().await, 0);
23028 }
23029
23030 #[cfg(feature = "test-util")]
23031 #[tokio::test]
23032 #[serial_test::serial(storage_class_env)]
23033 async fn transition_response_loss_persists_unknown_outcome_for_provider_recovery() {
23034 let temp_dir = tempfile::tempdir().expect("create temp store dir");
23035 let (ctx, store, _shutdown) =
23036 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "transition-response-loss", &[4])).await;
23037 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
23038
23039 let tier_name = "TXRESPONSELOSS";
23040 let backend = register_transition_reconcile_test_tier(&ctx.tier_config_mgr(), tier_name).await;
23041 let _compaction_proof = install_transition_transaction_compaction_fleet_proof_for_test("object-transaction-fencing-test");
23042 let bucket = "transition-response-loss-bucket";
23043 let object = "source.bin";
23044 store
23045 .make_bucket(bucket, &MakeBucketOptions::default())
23046 .await
23047 .expect("source bucket should be created");
23048 let payload = b"a response-lost tier PUT must remain recoverable".repeat(1024);
23049 let mut reader = PutObjReader::from_vec(payload.clone());
23050 let source = store
23051 .put_object(bucket, object, &mut reader, &ObjectOptions::default())
23052 .await
23053 .expect("source object should be written");
23054 backend.lose_next_put_response();
23055
23056 let error = store
23057 .transition_object(
23058 bucket,
23059 object,
23060 &ObjectOptions {
23061 no_lock: true,
23062 transition: TransitionOptions {
23063 status: TRANSITION_PENDING.to_string(),
23064 tier: tier_name.to_string(),
23065 etag: source.etag.clone().expect("source object should have an ETag"),
23066 ..Default::default()
23067 },
23068 version_id: source.version_id.map(|version| version.to_string()),
23069 mod_time: source.mod_time,
23070 ..Default::default()
23071 },
23072 )
23073 .await
23074 .expect_err("a lost tier PUT response must fail the transition request");
23075 assert!(
23076 matches!(error, StorageError::Io(ref err) if err.kind() == std::io::ErrorKind::ConnectionReset),
23077 "the response-loss error must remain visible to the caller: {error:?}"
23078 );
23079
23080 let records = store
23081 .clone()
23082 .list_objects_v2(
23083 RUSTFS_META_BUCKET,
23084 TRANSITION_TRANSACTION_RECORD_PREFIX,
23085 None,
23086 None,
23087 10,
23088 false,
23089 None,
23090 false,
23091 )
23092 .await
23093 .expect("transition transaction records should be listable");
23094 assert_eq!(records.objects.len(), 1, "response loss must leave one durable transaction record");
23095 let transaction_id = records.objects[0]
23096 .name
23097 .rsplit('/')
23098 .next()
23099 .and_then(|name| name.strip_suffix(".json"))
23100 .and_then(|name| uuid::Uuid::parse_str(name).ok())
23101 .expect("transaction record name should contain a UUID");
23102 let transaction = load_transition_transaction_record(store.clone(), transaction_id)
23103 .await
23104 .expect("response loss transaction record should load");
23105 assert_eq!(
23106 transaction.state,
23107 TransitionTransactionState::UploadOutcomeUnknown,
23108 "a response-lost PUT must not remain in UploadStarted"
23109 );
23110 assert_eq!(transaction.revision, 1, "compact response loss must retain the pre-PUT fence generation");
23111 assert!(
23112 backend.contains(&transaction.remote_object).await,
23113 "the test backend must retain the remote candidate"
23114 );
23115
23116 backend
23117 .set_transition_candidate_probe_override(Some(TransitionCandidateProbe::Unsupported))
23118 .await;
23119 let unsupported_stats = recover_transition_transaction_records(store.clone(), 100, None)
23120 .await
23121 .expect("active unknown ownership should remain fenced before provider recovery");
23122 assert_eq!(
23123 (
23124 unsupported_stats.scanned,
23125 unsupported_stats.recovered,
23126 unsupported_stats.retained,
23127 unsupported_stats.failed
23128 ),
23129 (1, 0, 1, 0),
23130 "active unknown ownership must retain the upload before the recovery deadline"
23131 );
23132 assert_eq!(transition_transaction_record_count(store.clone()).await, 1);
23133 let recovery_control_id =
23134 transition_recovery_control_id(&transaction).expect("transition recovery control id should derive");
23135 assert!(matches!(
23136 load_recovery_control(store.clone(), IlmRecoveryProtocol::TransitionTransaction, &recovery_control_id).await,
23137 Err(Error::ConfigNotFound)
23138 ));
23139 assert!(
23140 backend.contains(&transaction.remote_object).await,
23141 "active ownership must not delete the candidate"
23142 );
23143 assert_eq!(backend.remove_count().await, 0, "active ownership must not attempt cleanup");
23144 assert!(
23145 !backend
23146 .op_log()
23147 .await
23148 .iter()
23149 .any(|operation| matches!(operation, MockWarmOp::Probe { .. })),
23150 "active ownership must not probe the provider"
23151 );
23152
23153 backend.set_transition_candidate_probe_override(None).await;
23154 let stats =
23155 recover_transition_transaction_records_at(store.clone(), 100, None, i128::from(transaction.not_after_unix_nanos) + 1)
23156 .await
23157 .expect("provider-authoritative recovery should run");
23158 assert_eq!((stats.scanned, stats.recovered, stats.retained, stats.failed), (1, 1, 0, 0));
23159 assert_eq!(transition_transaction_record_count(store.clone()).await, 0);
23160 assert_eq!(backend.object_count().await, 0, "recovery must delete the provider-confirmed candidate");
23161 let op_log = backend.op_log().await;
23162 assert!(
23163 op_log.iter().any(|operation| matches!(operation, MockWarmOp::Probe { .. })),
23164 "response-loss recovery must enter the provider probe branch"
23165 );
23166 assert!(
23167 op_log.iter().any(|operation| matches!(operation, MockWarmOp::Put { .. })),
23168 "response-loss fixture must record that the remote PUT reached the backend"
23169 );
23170 let source_after = store
23171 .get_object_info(bucket, object, &ObjectOptions::default())
23172 .await
23173 .expect("recovery must preserve the local source object");
23174 assert_eq!(source_after.size, i64::try_from(payload.len()).expect("payload length should fit i64"));
23175 }
23176}