1use super::*;
16use crate::core::pools::{
17 PoolMetaReplicaState, PoolMetaWriteState, local_decommission_queue_prefix, persist_pool_meta_identity_for_attested_pools,
18 persist_pool_meta_identity_for_startup, pool_meta_has_active_decommission,
19};
20use crate::runtime::instance::InstanceContext;
21use crate::runtime::sources as runtime_sources;
22use crate::storage_api_contracts::object::EcstoreObjectIO;
23use rustfs_config::server_config::KVS;
24use rustfs_credentials::{RPC_SECRET_REQUIRED_OPERATOR_MESSAGE, try_get_rpc_token};
25use std::future::Future;
26use tracing::{debug, error, info, warn};
27
28const LOG_COMPONENT_ECSTORE: &str = "ecstore";
29const LOG_SUBSYSTEM_STORE_INIT: &str = "store_init";
30const EVENT_DECOMMISSION_RESUME_RETRY: &str = "decommission_resume_retry";
31const EVENT_STORE_FORMAT_RETRY: &str = "store_format_retry";
32const EVENT_ECSTORE_INIT_STATUS: &str = "ecstore_init_status";
33const EVENT_STORE_RPC_SECRET_PREFLIGHT_FAILED: &str = "store_rpc_secret_preflight_failed";
34
35fn pool_first_endpoint_is_local(pool: &crate::layout::endpoints::PoolEndpoints) -> bool {
36 pool.endpoints.as_ref().first().is_some_and(|endpoint| endpoint.is_local)
37}
38
39fn startup_pool_drive_counts(endpoint_pools: &EndpointServerPools) -> Vec<usize> {
40 endpoint_pools.as_ref().iter().map(|pool| pool.drives_per_set).collect()
41}
42
43fn resolve_startup_pool_defaults(endpoint_pools: &EndpointServerPools) -> Result<Vec<usize>> {
44 resolve_startup_pool_defaults_with(endpoint_pools, ECStore::validate_startup_storage_class)
45}
46
47fn resolve_startup_pool_defaults_with(
48 endpoint_pools: &EndpointServerPools,
49 validate: impl FnOnce(&EndpointServerPools) -> Result<()>,
50) -> Result<Vec<usize>> {
51 validate(endpoint_pools)?;
52 let drive_counts = startup_pool_drive_counts(endpoint_pools);
53 drive_counts.into_iter().map(ec_drives_no_config).collect()
54}
55
56fn preflight_startup_rpc_secret(endpoint_pools: &EndpointServerPools) -> Result<()> {
61 preflight_startup_rpc_secret_with(endpoint_pools, try_get_rpc_token)
62}
63
64fn preflight_startup_rpc_secret_with(
65 endpoint_pools: &EndpointServerPools,
66 resolve_rpc_token: impl FnOnce() -> std::io::Result<String>,
67) -> Result<()> {
68 let has_remote_endpoint = endpoint_pools
69 .as_ref()
70 .iter()
71 .flat_map(|pool| pool.endpoints.as_ref().iter())
72 .any(|endpoint| !endpoint.is_local);
73
74 if !has_remote_endpoint {
75 return Ok(());
76 }
77
78 match resolve_rpc_token() {
79 Ok(_) => Ok(()),
80 Err(err) => {
81 error!(
86 event = EVENT_STORE_RPC_SECRET_PREFLIGHT_FAILED,
87 component = LOG_COMPONENT_ECSTORE,
88 subsystem = LOG_SUBSYSTEM_STORE_INIT,
89 "RPC auth secret resolution failed: {err}; {RPC_SECRET_REQUIRED_OPERATOR_MESSAGE}"
90 );
91 Err(Error::other(format!(
92 "store init aborted: endpoints include remote nodes but {err}; {RPC_SECRET_REQUIRED_OPERATOR_MESSAGE}"
93 )))
94 }
95 }
96}
97
98const LOCAL_DECOMMISSION_RESUME_RETRY_DELAY: Duration = Duration::from_secs(30);
99const LOCAL_DECOMMISSION_WATCHDOG_INTERVAL: Duration = Duration::from_secs(30);
100const LOCAL_DECOMMISSION_WATCHDOG_MAX_RETRY_DELAY: Duration = Duration::from_secs(60 * 5);
101const REBALANCE_INITIAL_RESUME_DELAY: Duration = Duration::from_secs(10);
102const REBALANCE_RESUME_RETRY_DELAY: Duration = Duration::from_secs(10);
103
104fn should_retry_format_load(err: &Error) -> bool {
105 !matches!(
106 err,
107 Error::CorruptedFormat | Error::UnsupportedSnsdExpansion { .. } | Error::PoolTopologyMismatch { .. }
108 )
109}
110
111fn should_auto_start_rebalance_after_init(decommission_running: bool, rebalance_resume_required: bool) -> bool {
112 rebalance_resume_required && !decommission_running
113}
114
115fn should_defer_rebalance_auto_start(distributed: bool, fleet_proof_available: bool) -> bool {
116 distributed && !fleet_proof_available
117}
118
119async fn wait_for_local_decommission_resume_delay(rx: &CancellationToken, delay: Duration) -> bool {
120 tokio::select! {
121 _ = rx.cancelled() => false,
122 _ = tokio::time::sleep(delay) => true,
123 }
124}
125
126fn local_decommission_watchdog_retry_delay(consecutive_failures: u32) -> Duration {
127 let exponent = consecutive_failures.saturating_sub(1).min(4);
128 LOCAL_DECOMMISSION_RESUME_RETRY_DELAY
129 .saturating_mul(1_u32 << exponent)
130 .min(LOCAL_DECOMMISSION_WATCHDOG_MAX_RETRY_DELAY)
131}
132
133async fn wait_for_rebalance_resume_delay(rx: &CancellationToken, delay: Duration) -> bool {
134 tokio::select! {
135 _ = rx.cancelled() => false,
136 _ = tokio::time::sleep(delay) => true,
137 }
138}
139
140async fn wait_for_rebalance_resume_retry(rx: &CancellationToken) -> bool {
141 wait_for_rebalance_resume_delay(rx, REBALANCE_RESUME_RETRY_DELAY).await
142}
143
144fn resolve_store_init_stage_result(result: Result<()>, stage: &str) -> Result<()> {
145 result.map_err(|err| Error::other(format!("store init failed during {stage}: {err}")))
146}
147
148async fn load_pool_meta_for_startup<S>(
149 pools: Vec<Arc<S>>,
150 write_state: &mut PoolMetaWriteState,
151) -> Result<(PoolMeta, PoolMetaReplicaState)>
152where
153 S: EcstoreObjectIO,
154{
155 let mut meta = PoolMeta::default();
156 let replica_state = meta
157 .load_for_startup_observing(pools, write_state)
158 .await
159 .map_err(|err| Error::other_with_context("store init failed during load_pool_meta", err))?;
160 write_state.observe_replicas(replica_state);
161 write_state
162 .ensure_missing_metadata_can_initialize()
163 .map_err(|err| Error::other(format!("store init failed during classify_pool_meta_absence: {err}")))?;
164 Ok((meta, replica_state))
165}
166
167async fn establish_pool_meta_bootstrap_identity_if_proven<S>(
168 pools: Vec<Arc<S>>,
169 write_state: &mut PoolMetaWriteState,
170 elected_writer: bool,
171) -> Result<()>
172where
173 S: EcstoreObjectIO,
174{
175 if elected_writer && write_state.bootstrap_identity_proven() {
176 return persist_pool_meta_identity_for_startup(pools, write_state, false).await;
177 }
178 if write_state.bootstrap_identity_proven() {
179 return Ok(());
180 }
181 let attested = write_state.attested_pool_indices();
186 if attested.is_empty() {
187 return Ok(());
188 }
189 persist_pool_meta_identity_for_attested_pools(pools, write_state, &attested).await
190}
191
192async fn save_validated_pool_meta_for_startup<S>(
193 meta: &PoolMeta,
194 pools: Vec<Arc<S>>,
195 write_state: &mut PoolMetaWriteState,
196) -> Result<PoolMeta>
197where
198 S: EcstoreObjectIO,
199{
200 meta.save_for_startup_observing(pools, write_state)
201 .await
202 .map_err(|err| Error::other(format!("store init failed during save_validated_pool_meta: {err}")))
203}
204
205async fn persist_pool_meta_for_startup_if_safe<S>(
206 meta: &PoolMeta,
207 pools: Vec<Arc<S>>,
208 replica_state: PoolMetaReplicaState,
209 write_state: &mut PoolMetaWriteState,
210 topology_update: bool,
211 elected_writer: bool,
212) -> Result<PoolMeta>
213where
214 S: EcstoreObjectIO,
215{
216 if !elected_writer {
217 return Ok(meta.clone());
218 }
219 let should_write = topology_update || (replica_state.needs_repair && replica_state.repair_write_safe);
220 if topology_update {
221 replica_state.ensure_write_safe("store init failed during save_validated_pool_meta")?;
222 }
223 if should_write || write_state.identity_requires_repair() {
224 write_state.ensure_write_safe("store init failed during save_validated_pool_meta")?;
225 }
226 let mut committed = meta.clone();
227 if should_write {
228 if write_state.bootstrap_identity_proven() || write_state.identity_is_pending() {
229 persist_pool_meta_identity_for_startup(pools.clone(), write_state, false)
230 .await
231 .map_err(|err| Error::other(format!("store init failed during prepare_pool_meta_identity: {err}")))?;
232 }
233 committed = save_validated_pool_meta_for_startup(meta, pools.clone(), write_state).await?;
234 }
235 if should_write || write_state.identity_requires_repair() {
236 persist_pool_meta_identity_for_startup(pools, write_state, true)
237 .await
238 .map_err(|err| Error::other(format!("store init failed during commit_pool_meta_identity: {err}")))?;
239 }
240 Ok(committed)
241}
242
243async fn run_local_decommission_watchdog<F, Fut>(rx: CancellationToken, mut reconcile: F)
244where
245 F: FnMut() -> Fut,
246 Fut: Future<Output = Result<()>>,
247{
248 let mut consecutive_failures = 0_u32;
249
250 loop {
251 if rx.is_cancelled() {
252 return;
253 }
254
255 let delay = match reconcile().await {
256 Ok(()) => {
257 consecutive_failures = 0;
258 LOCAL_DECOMMISSION_WATCHDOG_INTERVAL
259 }
260 Err(err) => {
261 consecutive_failures = consecutive_failures.saturating_add(1);
262 let retry_delay = local_decommission_watchdog_retry_delay(consecutive_failures);
263 warn!(
264 event = EVENT_DECOMMISSION_RESUME_RETRY,
265 component = LOG_COMPONENT_ECSTORE,
266 subsystem = LOG_SUBSYSTEM_STORE_INIT,
267 consecutive_failures,
268 retry_delay_secs = retry_delay.as_secs(),
269 error = %err,
270 "Retrying decommission worker recovery"
271 );
272 retry_delay
273 }
274 };
275
276 if !wait_for_local_decommission_resume_delay(&rx, delay).await {
277 return;
278 }
279 }
280}
281
282async fn reconcile_local_decommission_after_init(store: &Arc<ECStore>, rx: CancellationToken) -> Result<()> {
283 store
284 .ensure_pool_meta_side_effects_safe("decommission worker recovery blocked while pool metadata requires recovery")
285 .await?;
286 if store.has_active_local_decommission_worker().await {
287 return Ok(());
288 }
289 store.refresh_pool_status_meta().await?;
290 let resume_required = pool_meta_has_active_decommission(&*store.pool_meta.read().await);
291 if resume_required {
292 crate::core::pools::acquire_pool_activation_fleet_proof(&store.ctx).await?;
293 }
294 store.spawn_missing_local_decommission_routines_with_token(rx).await
295}
296
297async fn supervise_local_decommission_after_init(store: Arc<ECStore>, rx: CancellationToken) {
298 run_local_decommission_watchdog(rx.clone(), || {
299 let store = store.clone();
300 let worker_rx = rx.clone();
301 async move { reconcile_local_decommission_after_init(&store, worker_rx).await }
302 })
303 .await;
304}
305
306async fn resume_rebalance_after_init(store: Arc<ECStore>, rx: CancellationToken) {
307 if !wait_for_rebalance_resume_delay(&rx, REBALANCE_INITIAL_RESUME_DELAY).await {
308 return;
309 }
310
311 loop {
312 if rx.is_cancelled() {
313 return;
314 }
315
316 let resume_required = store
317 .rebalance_meta
318 .read()
319 .await
320 .as_ref()
321 .is_some_and(crate::services::rebalance::rebalance_requires_worker_activation);
322 if !resume_required {
323 return;
324 }
325
326 if should_defer_rebalance_auto_start(
327 store.ctx.is_dist_erasure().await,
328 crate::services::notification_sys::acquire_cross_pool_fence_fleet_proof().is_some(),
329 ) {
330 if !wait_for_rebalance_resume_retry(&rx).await {
331 return;
332 }
333 continue;
334 }
335
336 match store.start_rebalance().await {
337 Ok(()) => return,
338 Err(err) if crate::core::pools::is_pool_activation_fleet_proof_error(&err) => {
339 warn!(
340 event = EVENT_ECSTORE_INIT_STATUS,
341 component = LOG_COMPONENT_ECSTORE,
342 subsystem = LOG_SUBSYSTEM_STORE_INIT,
343 stage = "start_rebalance",
344 state = "retrying",
345 reason = "fleet_capability_proof_unavailable",
346 error = %err,
347 retry_delay_secs = REBALANCE_RESUME_RETRY_DELAY.as_secs(),
348 "Retrying deferred rebalance auto-start"
349 );
350 if !wait_for_rebalance_resume_retry(&rx).await {
351 return;
352 }
353 }
354 Err(err) => {
355 error!(
356 event = EVENT_ECSTORE_INIT_STATUS,
357 component = LOG_COMPONENT_ECSTORE,
358 subsystem = LOG_SUBSYSTEM_STORE_INIT,
359 stage = "start_rebalance",
360 state = "failed",
361 reason = "deferred_resume_failed",
362 error = %err,
363 "Failed to resume rebalance after store initialization"
364 );
365 return;
366 }
367 }
368 }
369}
370
371impl ECStore {
372 pub fn background_cancel_token(&self) -> Option<CancellationToken> {
374 self.ctx.background_cancel_token()
375 }
376
377 pub fn validate_startup_storage_class(endpoint_pools: &EndpointServerPools) -> Result<()> {
379 let drive_counts = startup_pool_drive_counts(endpoint_pools);
380 storageclass::lookup_config_for_pools(&KVS::new(), &drive_counts).map(|_| ())
381 }
382
383 #[allow(clippy::new_ret_no_self)]
384 #[instrument(level = "debug", skip(endpoint_pools))]
385 pub async fn new(address: SocketAddr, endpoint_pools: EndpointServerPools, ctx: CancellationToken) -> Result<Arc<Self>> {
386 Self::new_with_instance_ctx(address, endpoint_pools, ctx, crate::runtime::instance::bootstrap_ctx()).await
387 }
388
389 #[allow(clippy::new_ret_no_self)]
397 #[instrument(level = "debug", skip(endpoint_pools, instance_ctx))]
398 pub async fn new_with_instance_ctx(
399 address: SocketAddr,
400 endpoint_pools: EndpointServerPools,
401 ctx: CancellationToken,
402 instance_ctx: Arc<InstanceContext>,
403 ) -> Result<Arc<Self>> {
404 Box::pin(Self::new_with_instance_ctx_inner(address, endpoint_pools, ctx, instance_ctx)).await
405 }
406
407 async fn new_with_instance_ctx_inner(
408 address: SocketAddr,
409 endpoint_pools: EndpointServerPools,
410 ctx: CancellationToken,
411 instance_ctx: Arc<InstanceContext>,
412 ) -> Result<Arc<Self>> {
413 instance_ctx.bind_background_cancel_token(ctx.clone());
414
415 let default_pool_parities = resolve_startup_pool_defaults(&endpoint_pools)?;
422
423 preflight_startup_rpc_secret(&endpoint_pools)?;
424
425 let mut deployment_id = None;
426 let mut pool_meta_bootstrap_authorities = Vec::new();
427
428 let mut pools = Vec::with_capacity(endpoint_pools.as_ref().len());
431 let mut disk_map = HashMap::with_capacity(endpoint_pools.as_ref().len());
432
433 let mut local_disks = Vec::new();
434
435 debug!(
436 event = EVENT_ECSTORE_INIT_STATUS,
437 component = LOG_COMPONENT_ECSTORE,
438 subsystem = LOG_SUBSYSTEM_STORE_INIT,
439 address = %address,
440 "Initializing ECStore address"
441 );
442 let mut host = address.ip().to_string();
443 if host.is_empty() {
444 host = runtime_sources::rustfs_host().await
445 }
446 let mut port = address.port().to_string();
447 if port.is_empty() {
448 port = runtime_sources::rustfs_port().to_string()
449 }
450 debug!(
451 event = EVENT_ECSTORE_INIT_STATUS,
452 component = LOG_COMPONENT_ECSTORE,
453 subsystem = LOG_SUBSYSTEM_STORE_INIT,
454 host = %host,
455 port = %port,
456 "Initializing ECStore host"
457 );
458 init_local_peer(&endpoint_pools, &host, &port).await;
459
460 for (i, pool_eps) in endpoint_pools.as_ref().iter().enumerate() {
463 let pool_first_is_local = pool_first_endpoint_is_local(pool_eps);
464 let parity_drives = default_pool_parities
465 .get(i)
466 .copied()
467 .ok_or_else(|| Error::other(format!("store init failed to resolve default parity for pool {i}")))?;
468
469 let (mut disks, errs) = init_format::init_disks(
476 &pool_eps.endpoints,
477 &DiskOption {
478 cleanup: true,
479 health_check: true,
480 },
481 )
482 .await;
483
484 check_disk_fatal_errs(&errs)?;
485
486 let loaded_format = {
487 let mut times = 0;
488 let mut interval = 1;
489 loop {
490 match init_format::connect_load_init_formats_with_instance_ctx(
491 &instance_ctx,
492 pool_first_is_local,
493 &mut disks,
494 pool_eps.set_count,
495 pool_eps.drives_per_set,
496 deployment_id,
497 )
498 .await
499 {
500 Ok(fm) => break Ok(fm),
501 Err(e) if !should_retry_format_load(&e) => break Err(e),
502 Err(e) if times >= 10 => {
504 break Err(Error::other(format!("store init failed to load formats after {times} retries: {e}")));
505 }
506 Err(_) => {}
508 }
509 times += 1;
510 if interval < 16 {
511 interval *= 2;
512 }
513 debug!(
514 event = EVENT_STORE_FORMAT_RETRY,
515 component = LOG_COMPONENT_ECSTORE,
516 subsystem = LOG_SUBSYSTEM_STORE_INIT,
517 retry_count = times,
518 retry_delay_secs = interval,
519 "Retrying storage format load"
520 );
521 select! {
522 _ = tokio::signal::ctrl_c() => {
523 info!(
524 event = EVENT_STORE_FORMAT_RETRY,
525 component = LOG_COMPONENT_ECSTORE,
526 subsystem = LOG_SUBSYSTEM_STORE_INIT,
527 reason = "ctrl_c",
528 "Interrupted storage format retry loop"
529 );
530 exit(0);
531 }
532 _ = sleep(Duration::from_secs(interval)) => {
533 }
534 }
535 for disk in disks.iter().flatten() {
538 disk.reset_health_for_store_init_retry();
539 }
540 }
541 }?;
542 pool_meta_bootstrap_authorities.push(loaded_format.pool_meta_bootstrap_authority);
546 let fm = loaded_format.format;
547
548 for disk in disks.iter().flatten() {
550 disk.enable_health_check();
551 }
552
553 if deployment_id.is_none() {
554 deployment_id = Some(fm.id);
555 }
556
557 if deployment_id != Some(fm.id) {
558 return Err(Error::other("store init failed: deployment IDs do not match across pools"));
559 }
560
561 if deployment_id.is_some_and(|id| id.is_nil()) {
562 deployment_id = Some(Uuid::new_v4());
563 }
564
565 for disk in disks.iter() {
566 if disk.is_some() && disk.as_ref().expect("operation should succeed").is_local() {
567 local_disks.push(disk.as_ref().expect("operation should succeed").clone());
568 }
569 }
570
571 let sets = Sets::new_with_instance_ctx(disks.clone(), pool_eps, &fm, i, parity_drives, instance_ctx.clone()).await?;
572 pools.push(sets);
573
574 disk_map.insert(i, disks);
575 }
576
577 if !instance_ctx.is_dist_erasure().await {
579 runtime_sources::record_local_disks(&instance_ctx, local_disks).await;
580 }
581
582 let deployment_id = deployment_id.ok_or_else(|| Error::other("store init failed: deployment id is not initialized"))?;
583 let peer_sys = S3PeerSys::new_with_instance_ctx(&endpoint_pools, instance_ctx.clone());
584 let mut pool_meta = PoolMeta::new(&pools, &PoolMeta::default());
585 pool_meta.dont_save = true;
586 let elected_bootstrap_writer = pools
587 .first()
588 .is_some_and(|pool| pool_first_endpoint_is_local(&pool.endpoints));
589 let pool_meta_write_state = PoolMetaWriteState::for_startup_with_pool_bootstrap_authorities(
590 deployment_id,
591 pool_meta_bootstrap_authorities,
592 elected_bootstrap_writer,
593 );
594
595 let decommission_cancelers = RwLock::new(vec![None; pools.len()]);
596 let ec = Arc::new(ECStore {
597 id: deployment_id,
598 disk_map,
599 pools,
600 peer_sys,
601 pool_meta: RwLock::new(pool_meta),
602 rebalance_meta: RwLock::new(None),
603 decommission_cancelers,
604 start_gate: Mutex::new(()),
605 pool_meta_save_gate: Mutex::new(pool_meta_write_state),
606 ctx: instance_ctx.clone(),
610 bucket_fence_registry: std::sync::Arc::default(),
611 });
612
613 if instance_ctx.deployment_id().is_none() {
615 instance_ctx.set_deployment_id(deployment_id);
616 }
617
618 let wait_sec = 5;
619 let mut exit_count = 0;
620 loop {
621 if let Err(err) = ec.init(ctx.clone()).await {
622 error!("init err: {}", err);
623 error!("retry after {} second", wait_sec);
624 sleep(Duration::from_secs(wait_sec)).await;
625
626 if exit_count > 10 {
627 return Err(Error::other("store init failed: init retry budget exhausted"));
628 }
629
630 exit_count += 1;
631
632 continue;
633 }
634
635 break;
636 }
637
638 runtime_sources::publish_object_store(ec.clone()).await;
639
640 Ok(ec)
641 }
642
643 #[instrument(level = "debug", skip(self, rx))]
644 pub async fn init(self: &Arc<Self>, rx: CancellationToken) -> Result<()> {
645 runtime_sources::ensure_boot_time().await;
646
647 let should_persist_pool_meta = self
648 .pools
649 .first()
650 .is_some_and(|pool| pool_first_endpoint_is_local(&pool.endpoints));
651 #[cfg(feature = "e2e-test-hooks")]
652 let startup_attempt = uuid::Uuid::new_v4();
653 let (meta, pool_meta_replica_state) = {
654 let mut write_state = self.pool_meta_save_gate.lock().await;
655 establish_pool_meta_bootstrap_identity_if_proven(self.pools.clone(), &mut write_state, should_persist_pool_meta)
656 .await
657 .map_err(|err| Error::other(format!("store init failed during establish_pool_meta_bootstrap_identity: {err}")))?;
658 let load = load_pool_meta_for_startup(self.pools.clone(), &mut write_state);
659 #[cfg(feature = "e2e-test-hooks")]
660 let load = crate::core::pools::startup_cas_test_scope(startup_attempt, "load", &self.pools, load);
661 load.await?
662 };
663 let update = meta.validate(self.pools.clone())?;
664 #[cfg(feature = "e2e-test-hooks")]
665 crate::core::pools::startup_cas_test_observe(serde_json::json!({
666 "kind": "startup-classifier", "attempt": startup_attempt,
667 "elected_writer": should_persist_pool_meta,
668 "needs_repair": pool_meta_replica_state.needs_repair,
669 "repair_write_safe": pool_meta_replica_state.repair_write_safe,
670 "topology_update": update,
671 }));
672 let endpoints = runtime_sources::endpoint_pools_or_default();
673
674 let mut installed_pool_meta = if update {
675 PoolMeta::new(&self.pools, &meta)
676 } else {
677 meta.clone()
678 };
679 {
682 let mut write_state = self.pool_meta_save_gate.lock().await;
683 let persist = persist_pool_meta_for_startup_if_safe(
684 &installed_pool_meta,
685 self.pools.clone(),
686 pool_meta_replica_state,
687 &mut write_state,
688 update,
689 should_persist_pool_meta,
690 );
691 #[cfg(feature = "e2e-test-hooks")]
692 let persist = crate::core::pools::startup_cas_test_scope(startup_attempt, "persist", &self.pools, persist);
693 installed_pool_meta = persist.await?;
694 }
695
696 {
697 let mut pool_meta = self.pool_meta.write().await;
698 *pool_meta = installed_pool_meta.clone();
699 }
700
701 resolve_store_init_stage_result(self.load_rebalance_meta().await, "load_rebalance_meta")?;
702 let rebalance_resume_required = {
703 let rebalance_meta = self.rebalance_meta.read().await;
704 rebalance_meta
705 .as_ref()
706 .is_some_and(crate::services::rebalance::rebalance_requires_worker_activation)
707 };
708 let decommission_running =
709 pool_meta_has_active_decommission(&installed_pool_meta) || self.is_decommission_running().await;
710 let distributed = self.ctx.is_dist_erasure().await;
711 let fleet_proof_available = crate::services::notification_sys::acquire_cross_pool_fence_fleet_proof().is_some();
712 let mut rebalance_auto_start_deferred = false;
713 if should_auto_start_rebalance_after_init(decommission_running, rebalance_resume_required) {
714 if should_defer_rebalance_auto_start(distributed, fleet_proof_available) {
715 rebalance_auto_start_deferred = true;
716 warn!(
717 event = EVENT_ECSTORE_INIT_STATUS,
718 component = LOG_COMPONENT_ECSTORE,
719 subsystem = LOG_SUBSYSTEM_STORE_INIT,
720 stage = "start_rebalance",
721 state = "deferred",
722 reason = "fleet_capability_proof_unavailable",
723 retry_delay_secs = REBALANCE_RESUME_RETRY_DELAY.as_secs(),
724 "Deferred rebalance auto-start until a live fleet capability proof is available"
725 );
726 } else if let Err(err) = self.start_rebalance().await {
727 if crate::core::pools::is_pool_activation_fleet_proof_error(&err) {
728 rebalance_auto_start_deferred = true;
729 warn!(
730 event = EVENT_ECSTORE_INIT_STATUS,
731 component = LOG_COMPONENT_ECSTORE,
732 subsystem = LOG_SUBSYSTEM_STORE_INIT,
733 stage = "start_rebalance",
734 state = "deferred",
735 reason = "fleet_capability_proof_changed",
736 error = %err,
737 retry_delay_secs = REBALANCE_RESUME_RETRY_DELAY.as_secs(),
738 "Deferred rebalance auto-start after the fleet capability proof changed"
739 );
740 } else {
741 return resolve_store_init_stage_result(Err(err), "start_rebalance");
742 }
743 }
744 } else if decommission_running && rebalance_resume_required {
745 warn!(
746 event = EVENT_ECSTORE_INIT_STATUS,
747 component = LOG_COMPONENT_ECSTORE,
748 subsystem = LOG_SUBSYSTEM_STORE_INIT,
749 stage = "start_rebalance",
750 reason = "active_decommission",
751 "Deferred rebalance auto-start during store init because decommission is active"
752 );
753 }
754
755 let _ = self.scanner_data_usage_publication_blocked().await;
759
760 let pools = installed_pool_meta.return_resumable_pools();
761 let mut pool_indices = Vec::with_capacity(pools.len());
762
763 for p in pools.iter() {
764 if let Some(idx) = endpoints.get_pool_idx(&p.cmd_line) {
765 pool_indices.push(idx);
766 } else {
767 return Err(Error::other(format!(
768 "store init failed to resolve resumable decommission pool `{}` from current endpoints",
769 p.cmd_line
770 )));
771 }
772 }
773
774 let local_pool_indices = local_decommission_queue_prefix(&endpoints, &pool_indices)?;
775 let has_local_decommission_leadership = endpoints.as_ref().iter().any(pool_first_endpoint_is_local);
776 let pool_meta_write_safe = self
777 .ensure_pool_meta_side_effects_safe("decommission resume blocked while pool metadata requires recovery")
778 .await
779 .is_ok();
780 if !pool_meta_replica_state.repair_write_safe || !pool_meta_write_safe {
781 warn!(
782 event = EVENT_DECOMMISSION_RESUME_RETRY,
783 component = LOG_COMPONENT_ECSTORE,
784 subsystem = LOG_SUBSYSTEM_STORE_INIT,
785 state = "blocked",
786 pool_indices = ?local_pool_indices,
787 reason = "pool_meta_write_blocked",
788 "Decommission watchdog waiting for pool metadata recovery"
789 );
790 }
791 if has_local_decommission_leadership {
792 tokio::spawn(supervise_local_decommission_after_init(self.clone(), rx.clone()));
795 }
796
797 let recovery_store = self.clone();
798 let recovery_rx = rx.clone();
799 tokio::spawn(async move {
800 let mut delay = std::time::Duration::from_secs(5);
801 loop {
802 tokio::select! {
803 _ = recovery_rx.cancelled() => return,
804 _ = tokio::time::sleep(delay) => {}
805 }
806 let result = tokio::select! {
807 _ = recovery_rx.cancelled() => return,
808 result = tokio::time::timeout(std::time::Duration::from_secs(30), recovery_store.recover_pool_meta_transaction()) => result,
809 };
810 delay = match result {
811 Ok(Ok(_)) => std::time::Duration::from_secs(5),
812 failure => {
813 let error = match failure {
814 Ok(Err(error)) => error,
815 _ => Error::Timeout,
816 };
817 recovery_store.record_pool_meta_recovery_failure(error);
818 (delay * 2).min(std::time::Duration::from_secs(60))
819 }
820 };
821 }
822 });
823
824 runtime_sources::init_bucket_monitor_for_current_endpoints();
825 crate::bucket::bucket_target_sys::BucketTargetSys::get().start_heartbeat();
826
827 init_background_expiry(self.clone()).await;
828 crate::bucket::lifecycle::bucket_lifecycle_ops::init_background_stale_multipart_upload_cleanup(self.clone());
829
830 TransitionState::init(self.clone()).await;
831 crate::services::tier::tier::try_migrate_tiering_config(self.clone()).await;
832
833 if let Err(err) = runtime_sources::init_tier_config_mgr(self.clone()).await {
834 info!("TierConfigMgr init error: {}", err);
835 }
836
837 if rebalance_auto_start_deferred {
838 let store = self.clone();
839 tokio::spawn(resume_rebalance_after_init(store, rx));
840 }
841
842 Ok(())
843 }
844
845 pub fn init_local_disks() {}
846
847 pub fn single_pool(&self) -> bool {
848 self.pools.len() == 1
849 }
850
851 pub fn supports_atomic_create_only_write_back(&self) -> bool {
854 !self.ctx.lock_manager().is_disabled() && self.pools.len() == 1 && self.pools[0].disk_set.len() == 1
855 }
856}
857
858#[cfg(test)]
859mod tests {
860 use super::{
861 LOCAL_DECOMMISSION_RESUME_RETRY_DELAY, LOCAL_DECOMMISSION_WATCHDOG_MAX_RETRY_DELAY, PoolMetaWriteState,
862 establish_pool_meta_bootstrap_identity_if_proven, load_pool_meta_for_startup, local_decommission_watchdog_retry_delay,
863 persist_pool_meta_for_startup_if_safe, pool_first_endpoint_is_local, pool_meta_has_active_decommission,
864 preflight_startup_rpc_secret_with, resolve_startup_pool_defaults_with, resolve_store_init_stage_result,
865 run_local_decommission_watchdog, save_validated_pool_meta_for_startup, should_auto_start_rebalance_after_init,
866 should_defer_rebalance_auto_start, should_retry_format_load, wait_for_local_decommission_resume_delay,
867 };
868 use crate::core::pools::PoolMetaBootstrapAuthority;
869 #[cfg(feature = "test-util")]
870 use crate::disk::DiskAPI;
871 #[cfg(feature = "test-util")]
872 use crate::{
873 bucket::lifecycle::{
874 DurableIlmRecordCheckpoint, ILM_META_PREFIX, ValidatedDurableIlmRecord,
875 bucket_lifecycle_ops::{ExpiryState, ManualTransitionRunOptions, recover_manual_transition_jobs_once},
876 lifecycle::{TRANSITION_PENDING, TransitionOptions},
877 manual_transition_job::{
878 ManualTransitionJobRecord, ManualTransitionScopeAdmission, ManualTransitionTaskRecord,
879 ManualTransitionWorkerResult, ManualTransitionWorkerResultRecord, manual_transition_job_record_object_name,
880 manual_transition_scope_record_object_name, manual_transition_task_object_name,
881 manual_transition_worker_result_object_name, manual_transition_worker_result_task_key,
882 },
883 recovery_control::{
884 IlmRecoveryClassification, IlmRecoveryControl, IlmRecoveryControlIdentity, IlmRecoveryErrorCode,
885 IlmRecoveryProtocol, MAX_RECOVERY_ATTEMPTS, list_recovery_controls, load_recovery_control,
886 observe_recovery_source, recovery_control_record_object_name, save_recovery_control_if_absent,
887 },
888 recovery_disposition::{
889 IlmRecoveryDispositionExecutionOutcome, IlmRecoveryDispositionState, RecoveryDispositionCrashStage,
890 dry_run_recovery_disposition, execute_recovery_disposition, inject_recovery_disposition_crash_once,
891 load_recovery_disposition,
892 },
893 recovery_disposition_runtime::garbage_collect_completed_recovery_disposition,
894 recovery_export::{
895 create_recovery_export, inspect_recovery_export_observation, load_recovery_export,
896 recovery_export_record_object_name,
897 },
898 tier_delete_journal::{
899 DecommissionCheckpointTargetFailureHook, TIER_DELETE_DISPATCH_MANIFEST_PREFIX, TIER_DELETE_JOURNAL_PREFIX,
900 TierDeleteChunkTestBarrier, TierDeleteChunkTestStage, TierDeleteDispatchBatchLimitGuard,
901 TierDeleteDispatchManifestState, TierDeleteDispatchMemberReadTestHook, TierDeleteDispatchMemberReadTestStage,
902 TierDeleteDispatchRollbackTestHook, complete_tier_delete_dispatch, encode_tier_delete_journal_entry,
903 install_test_tier_delete_dispatch_fixture, persist_tier_delete_journal_entry, prepare_tier_delete_dispatch,
904 recover_test_tier_delete_dispatch_manifest, recover_test_tier_delete_dispatch_manifest_with_page_budget,
905 recover_test_tier_delete_journal_pass_with_budget, recover_tier_delete_dispatch_manifests,
906 recover_tier_delete_journal_entries, test_tier_delete_dispatch_manifest_checkpoint,
907 test_tier_delete_dispatch_manifest_state, tier_delete_dispatch_manifest_operation_lock_held_for_test,
908 tier_delete_dispatch_manifest_recovery_count_for_test, tier_delete_dispatch_manifest_recovery_inflight_for_test,
909 tier_delete_journal_object_name,
910 },
911 tier_free_version_recovery::recover_tier_free_versions,
912 tier_sweeper::{
913 Jentry, TierDeleteJournalState, TierDeleteSourceIdentity, transitioned_delete_journal_entry_for_source,
914 },
915 transition_transaction::{
916 TRANSITION_TRANSACTION_RECORD_PREFIX, TransitionCleanupDecision, TransitionCleanupProof, TransitionOperatorError,
917 TransitionOperatorProbe, TransitionRecoveryClaimBarrier, TransitionRecoveryTerminalBarrier,
918 TransitionRemoteVersion, TransitionSourceIdentity, TransitionSourceVersionMode, TransitionTransaction,
919 TransitionTransactionInit, TransitionTransactionState, delete_transition_candidate_for_operator,
920 finalize_missing_transition_transaction_for_operator, inspect_transition_recovery_retry_for_operator,
921 inspect_transition_transaction_for_operator, load_transition_transaction_record,
922 recover_transition_transaction_records, recover_transition_transaction_records_at,
923 retry_transition_recovery_for_operator, save_transition_transaction_record,
924 save_transition_transaction_record_if_current, transition_recovery_control_id,
925 transition_transaction_record_object_name,
926 },
927 validate_durable_ilm_record,
928 },
929 bucket::metadata::{BUCKET_LIFECYCLE_CONFIG, BUCKET_VERSIONING_CONFIG},
930 config::com,
931 core::pools::DecomBucketInfo,
932 data_movement::SourceCleanupDeleteBarrier,
933 disk::{BUCKET_META_PREFIX, RUSTFS_META_BUCKET, STORAGE_FORMAT_FILE},
934 runtime::{global::set_object_store_resolver, sources as runtime_sources},
935 services::notification_sys::{
936 acquire_tier_delete_journal_fleet_proof, acquire_transition_transaction_compaction_fleet_proof,
937 install_transition_transaction_compaction_fleet_proof_for_test,
938 transition_transaction_compaction_fleet_proof_matches,
939 },
940 services::tier::{
941 test_util::{MockWarmBackend, MockWarmOp, TransitionCleanupStoreBarrier, register_mock_tier},
942 tier::{
943 ERR_TIER_BACKEND_IN_USE, ERR_TIER_INVALID_CONFIG, TIER_CONFIG_FILE, TierConfigMgr,
944 ensure_no_authoritative_tier_references_for_test, tier_config_candidate_digest,
945 },
946 tier_config::{TierConfig, TierType, TierWasabi},
947 tier_mutation_intent::{
948 TIER_MUTATION_INTENT_RECORD_PREFIX, TierMutationIntent, TierMutationIntentKind, TierMutationIntentState,
949 TierMutationIntentTarget, advance_tier_mutation_intent_record_idempotent, delete_tier_mutation_intent_record,
950 list_tier_mutation_intent_records, load_tier_mutation_intent_record, load_tier_mutation_intent_record_with_etag,
951 save_tier_mutation_intent_record, save_tier_mutation_intent_record_if_current,
952 },
953 tier_mutation_peer::{TierMutationPeerError, TierMutationPeerState, handle_tier_mutation_peer_request},
954 tier_probe_intent::{
955 TierProbeIntent, TierProbeIntentState, TierProbeOperationIdentity, TierProbeOwnerFence, TierProbeRemoteVersion,
956 delete_tier_probe_intent_record_if_current, load_tier_probe_intent_record,
957 save_tier_probe_intent_record_if_absent, save_tier_probe_intent_record_if_current,
958 },
959 warm_backend::{TransitionCandidateProbe, WarmBackend},
960 },
961 set_disk::{
962 SetDiskTransitionTransactionKillPoint as TransitionTransactionKillPoint,
963 SetDiskTransitionTransactionKillPointBarrier as TransitionTransactionKillPointBarrier,
964 SetDiskTransitionTransactionMutationKind as TransitionTransactionMutationKind,
965 SetDiskTransitionTransactionMutationObservation as TransitionTransactionMutationObservation,
966 SetDiskTransitionTransactionMutationProbe as TransitionTransactionMutationProbe,
967 SetDiskTransitionUploadedCommitBarrier as TransitionUploadedCommitBarrier,
968 },
969 storage_api_contracts::list::ListOperations as _,
970 };
971 #[cfg(feature = "test-util")]
972 use crate::{bucket::replication::ReplicationObjectBridge, storage_api_contracts::bucket::DeleteBucketOptions};
973 use crate::{
974 bucket::replication::{ReplicationState, ReplicationStatusType, replication_statuses_map},
975 core::pools::{
976 DecommissionErasureLayout, DecommissionPoolCapacityInfo, POOL_META_IDENTITY_NAME, POOL_META_NAME, POOL_META_VERSION,
977 PoolDecommissionInfo, PoolMeta, PoolStatus, pending_pool_meta_identity_for_test,
978 pool_meta_identity_initialized_for_test, pool_meta_v3_commit_state_for_test,
979 set_decommission_capacity_info_overrides_for_test,
980 },
981 disk::endpoint::Endpoint,
982 error::{Error, Result, StorageError},
983 io_support::rio::{WritePlan, compression_metadata_value},
984 layout::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints},
985 object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader},
986 services::rebalance::{RebalStatus, RebalanceInfo, RebalanceMeta, RebalanceStats},
987 storage_api_contracts::{
988 bucket::{BucketOperations as _, MakeBucketOptions},
989 multipart::MultipartOperations as _,
990 object::{ObjectIO, ObjectOperations as _, ObjectToDelete},
991 range::HTTPRangeSpec,
992 },
993 };
994 #[cfg(feature = "test-util")]
995 use futures::{StreamExt as _, TryStreamExt as _};
996 use http::HeaderMap;
997 use rustfs_config::server_config::KVS;
998 #[cfg(feature = "test-util")]
999 use rustfs_filemeta::{FileInfo, FileMeta};
1000 use rustfs_filemeta::{FileInfoVersions, MetaCacheEntry};
1001 #[cfg(feature = "test-util")]
1002 use rustfs_protos::{TIER_MUTATION_RPC_PROTOCOL_VERSION, TierMutationRpcPhase};
1003 use rustfs_rio::{Checksum, ChecksumType};
1004 #[cfg(feature = "test-util")]
1005 use rustfs_s3_client::transition_api::ReaderImpl;
1006 use rustfs_utils::{
1007 CompressionAlgorithm,
1008 http::{SUFFIX_COMPRESSION, insert_str},
1009 };
1010 use std::{
1011 collections::HashMap,
1012 future::Future,
1013 io::Cursor,
1014 sync::{
1015 Arc, Mutex,
1016 atomic::{AtomicBool, AtomicUsize, Ordering},
1017 },
1018 time::Duration,
1019 };
1020 use time::OffsetDateTime;
1021 use tokio::io::AsyncReadExt;
1022 use tokio_util::sync::CancellationToken;
1023
1024 fn run_large_stack_async_test<C, F>(name: &str, case: C)
1025 where
1026 C: FnOnce() -> F + Send + 'static,
1027 F: Future<Output = ()> + 'static,
1028 {
1029 const STACK_SIZE: usize = if cfg!(debug_assertions) {
1030 8 * rustfs_config::DEFAULT_THREAD_STACK_SIZE
1031 } else if cfg!(target_os = "macos") {
1032 2 * rustfs_config::DEFAULT_THREAD_STACK_SIZE
1033 } else {
1034 rustfs_config::DEFAULT_THREAD_STACK_SIZE
1035 };
1036 std::thread::Builder::new()
1037 .name(name.to_string())
1038 .stack_size(STACK_SIZE)
1039 .spawn(move || {
1040 let runtime = tokio::runtime::Builder::new_multi_thread()
1041 .enable_all()
1042 .worker_threads(2)
1043 .thread_stack_size(STACK_SIZE)
1044 .build()
1045 .expect("large-stack store test runtime should build");
1046 runtime.block_on(case());
1047 })
1048 .expect("large-stack store test thread should spawn")
1049 .join()
1050 .expect("large-stack store test thread should complete");
1051 }
1052 use uuid::Uuid;
1053
1054 #[test]
1055 fn new_with_instance_ctx_future_remains_stack_bounded() {
1056 let future = crate::store::ECStore::new_with_instance_ctx(
1057 std::net::SocketAddr::from(([127, 0, 0, 1], 0)),
1058 EndpointServerPools(Vec::new()),
1059 CancellationToken::new(),
1060 Arc::new(crate::runtime::instance::InstanceContext::new()),
1061 );
1062 let future_size = std::mem::size_of_val(&future);
1063
1064 assert!(
1065 future_size <= 4 * 1024,
1066 "ECStore constructor future must remain stack-bounded; measured {future_size} bytes"
1067 );
1068 }
1069
1070 fn startup_pool_meta_payload(meta: &PoolMeta) -> Vec<u8> {
1071 meta.encode_config_data_for_test().expect("pool metadata should encode")
1072 }
1073
1074 #[derive(Debug)]
1075 struct StartupPoolMetaStorage {
1076 read_payload: Vec<u8>,
1077 read_error: bool,
1078 read_without_lock: AtomicBool,
1079 wrote_without_lock: AtomicBool,
1080 wrote_with_max_parity: AtomicBool,
1081 written_payload: Mutex<Option<Vec<u8>>>,
1082 revision: AtomicUsize,
1083 pool_meta_write_attempts: AtomicUsize,
1084 fail_pool_meta_write_at: AtomicUsize,
1085 pool_meta_written_versions: Mutex<Vec<u16>>,
1086 objects: Mutex<HashMap<String, (Vec<u8>, String)>>,
1087 }
1088
1089 impl StartupPoolMetaStorage {
1090 fn new(read_payload: Vec<u8>) -> Self {
1091 let mut objects = HashMap::new();
1092 if !read_payload.is_empty() {
1093 objects.insert(POOL_META_NAME.to_string(), (read_payload.clone(), "startup-pool-meta-0".to_string()));
1094 }
1095 Self {
1096 read_payload,
1097 read_error: false,
1098 read_without_lock: AtomicBool::new(false),
1099 wrote_without_lock: AtomicBool::new(false),
1100 wrote_with_max_parity: AtomicBool::new(false),
1101 written_payload: Mutex::new(None),
1102 revision: AtomicUsize::new(0),
1103 pool_meta_write_attempts: AtomicUsize::new(0),
1104 fail_pool_meta_write_at: AtomicUsize::new(0),
1105 pool_meta_written_versions: Mutex::new(Vec::new()),
1106 objects: Mutex::new(objects),
1107 }
1108 }
1109
1110 fn unreadable() -> Self {
1111 Self {
1112 read_payload: Vec::new(),
1113 read_error: true,
1114 read_without_lock: AtomicBool::new(false),
1115 wrote_without_lock: AtomicBool::new(false),
1116 wrote_with_max_parity: AtomicBool::new(false),
1117 written_payload: Mutex::new(None),
1118 revision: AtomicUsize::new(0),
1119 pool_meta_write_attempts: AtomicUsize::new(0),
1120 fail_pool_meta_write_at: AtomicUsize::new(0),
1121 pool_meta_written_versions: Mutex::new(Vec::new()),
1122 objects: Mutex::new(HashMap::new()),
1123 }
1124 }
1125
1126 fn object_info(&self, bucket: &str, object: &str, size: usize, etag: String) -> ObjectInfo {
1127 ObjectInfo {
1128 bucket: bucket.to_string(),
1129 name: object.to_string(),
1130 size: size as i64,
1131 actual_size: size as i64,
1132 etag: Some(etag),
1133 ..Default::default()
1134 }
1135 }
1136 }
1137
1138 #[async_trait::async_trait]
1139 impl ObjectIO for StartupPoolMetaStorage {
1140 type Error = Error;
1141 type RangeSpec = HTTPRangeSpec;
1142 type HeaderMap = HeaderMap;
1143 type ObjectOptions = ObjectOptions;
1144 type ObjectInfo = ObjectInfo;
1145 type GetObjectReader = GetObjectReader;
1146 type PutObjectReader = PutObjReader;
1147
1148 async fn get_object_reader(
1149 &self,
1150 bucket: &str,
1151 object: &str,
1152 _range: Option<HTTPRangeSpec>,
1153 _h: HeaderMap,
1154 opts: &ObjectOptions,
1155 ) -> Result<GetObjectReader> {
1156 assert!(opts.no_lock, "store init pool metadata load must not require namespace locks");
1157 self.read_without_lock.store(true, Ordering::SeqCst);
1158 if self.read_error {
1159 return Err(Error::other("pool metadata read quorum unavailable"));
1160 }
1161 let Some((payload, etag)) = self
1162 .objects
1163 .lock()
1164 .unwrap_or_else(std::sync::PoisonError::into_inner)
1165 .get(object)
1166 .cloned()
1167 else {
1168 return Err(Error::FileNotFound);
1169 };
1170
1171 Ok(GetObjectReader {
1172 stream: Box::new(Cursor::new(payload.clone())),
1173 object_info: self.object_info(bucket, object, payload.len(), etag),
1174 buffered_body: None,
1175 body_source: Default::default(),
1176 })
1177 }
1178
1179 async fn put_object(
1180 &self,
1181 bucket: &str,
1182 object: &str,
1183 data: &mut PutObjReader,
1184 opts: &ObjectOptions,
1185 ) -> Result<ObjectInfo> {
1186 assert!(opts.no_lock, "store init pool metadata save must not require namespace locks");
1187 self.wrote_without_lock.store(true, Ordering::SeqCst);
1188 self.wrote_with_max_parity.store(opts.max_parity, Ordering::SeqCst);
1189 if object == POOL_META_NAME {
1190 let attempt = self.pool_meta_write_attempts.fetch_add(1, Ordering::SeqCst) + 1;
1191 if self.fail_pool_meta_write_at.load(Ordering::SeqCst) == attempt {
1192 return Err(Error::Timeout);
1193 }
1194 }
1195 let current_etag = self
1196 .objects
1197 .lock()
1198 .unwrap_or_else(std::sync::PoisonError::into_inner)
1199 .get(object)
1200 .map(|(_, etag)| etag.clone());
1201 if opts
1202 .http_preconditions
1203 .as_ref()
1204 .and_then(|preconditions| preconditions.if_none_match_value())
1205 == Some("*")
1206 && current_etag.is_some()
1207 {
1208 return Err(Error::PreconditionFailed);
1209 }
1210 if let Some(expected) = opts
1211 .http_preconditions
1212 .as_ref()
1213 .and_then(|preconditions| preconditions.if_match_value())
1214 && current_etag.as_deref() != Some(expected)
1215 {
1216 return Err(Error::PreconditionFailed);
1217 }
1218 let mut payload = Vec::new();
1219 data.stream.read_to_end(&mut payload).await?;
1220 let size = payload.len();
1221 if object == POOL_META_NAME {
1222 *self.written_payload.lock().unwrap_or_else(std::sync::PoisonError::into_inner) = Some(payload.clone());
1223 if payload.len() >= 4 {
1224 self.pool_meta_written_versions
1225 .lock()
1226 .unwrap_or_else(std::sync::PoisonError::into_inner)
1227 .push(u16::from_le_bytes([payload[2], payload[3]]));
1228 }
1229 }
1230 let etag = format!("startup-pool-meta-{}", self.revision.fetch_add(1, Ordering::SeqCst) + 1);
1231 self.objects
1232 .lock()
1233 .unwrap_or_else(std::sync::PoisonError::into_inner)
1234 .insert(object.to_string(), (payload, etag.clone()));
1235 Ok(self.object_info(bucket, object, size, etag))
1236 }
1237 }
1238
1239 fn init_test_pool_meta(decommission: Option<PoolDecommissionInfo>) -> PoolMeta {
1240 PoolMeta {
1241 version: POOL_META_VERSION,
1242 pools: vec![PoolStatus {
1243 id: 0,
1244 cmd_line: "pool-0".to_string(),
1245 last_update: OffsetDateTime::UNIX_EPOCH,
1246 decommission,
1247 }],
1248 dont_save: false,
1249 }
1250 }
1251
1252 #[tokio::test]
1253 async fn test_fresh_bootstrap_pool_meta_save_reaches_v3_cas_without_blocking_itself() {
1254 let storage = Arc::new(StartupPoolMetaStorage::new(Vec::new()));
1255 let mut write_state = PoolMetaWriteState::for_startup(Uuid::new_v4(), true);
1256 establish_pool_meta_bootstrap_identity_if_proven(vec![storage.clone()], &mut write_state, true)
1257 .await
1258 .expect("the elected fresh bootstrap should persist its identity before loading pool metadata");
1259
1260 let (loaded, replica_state) = load_pool_meta_for_startup(vec![storage.clone()], &mut write_state)
1261 .await
1262 .expect("startup pool metadata load should tolerate missing metadata without locks");
1263 assert!(loaded.pools.is_empty());
1264 assert!(!replica_state.needs_repair);
1265 assert!(replica_state.repair_write_safe);
1266 assert!(storage.read_without_lock.load(Ordering::SeqCst));
1267
1268 let meta = PoolMeta {
1269 version: POOL_META_VERSION,
1270 pools: Vec::new(),
1271 dont_save: false,
1272 };
1273 save_validated_pool_meta_for_startup(&meta, vec![storage.clone()], &mut write_state)
1274 .await
1275 .expect("startup pool metadata save should bypass locks");
1276 assert!(storage.wrote_without_lock.load(Ordering::SeqCst));
1277 assert!(storage.wrote_with_max_parity.load(Ordering::SeqCst));
1278 assert_eq!(
1279 storage.pool_meta_write_attempts.load(Ordering::SeqCst),
1280 2,
1281 "fresh bootstrap should reach both V3 prepare and commit CAS writes"
1282 );
1283 assert_eq!(
1284 *storage
1285 .pool_meta_written_versions
1286 .lock()
1287 .unwrap_or_else(std::sync::PoisonError::into_inner),
1288 vec![3, 3]
1289 );
1290 write_state
1291 .ensure_write_safe("fresh bootstrap publication")
1292 .expect("successful startup publication must disarm the transaction guard");
1293 }
1294
1295 #[tokio::test]
1296 async fn test_legacy_adoption_pool_meta_bootstrap_reaches_v3_cas() {
1297 let deployment_id = Uuid::new_v4();
1298 let storage = Arc::new(StartupPoolMetaStorage::new(Vec::new()));
1299 let mut write_state =
1300 PoolMetaWriteState::for_startup_with_bootstrap_authority(deployment_id, PoolMetaBootstrapAuthority::LegacyAdoption);
1301
1302 establish_pool_meta_bootstrap_identity_if_proven(vec![storage.clone()], &mut write_state, true)
1303 .await
1304 .expect("verified legacy adoption should persist a nonce-bound identity before loading pool metadata");
1305 let (loaded, replica_state) = load_pool_meta_for_startup(vec![storage.clone()], &mut write_state)
1306 .await
1307 .expect("verified legacy adoption should authorize initially missing pool metadata");
1308 assert!(loaded.pools.is_empty());
1309
1310 let committed = persist_pool_meta_for_startup_if_safe(
1311 &init_test_pool_meta(None),
1312 vec![storage.clone()],
1313 replica_state,
1314 &mut write_state,
1315 true,
1316 true,
1317 )
1318 .await
1319 .expect("verified legacy adoption should publish initial pool metadata");
1320 assert_eq!(committed.pools[0].cmd_line, "pool-0");
1321
1322 let objects = storage.objects.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
1323 assert!(objects.contains_key(POOL_META_NAME));
1324 let identity = objects
1325 .get(POOL_META_IDENTITY_NAME)
1326 .map(|(payload, _)| payload.clone())
1327 .expect("legacy adoption should commit the bootstrap identity");
1328 assert!(pool_meta_identity_initialized_for_test(&identity).expect("decode committed identity"));
1329 }
1330
1331 #[tokio::test]
1332 async fn test_non_elected_legacy_adoption_cannot_initialize_pool_meta() {
1333 let storage = Arc::new(StartupPoolMetaStorage::new(Vec::new()));
1334 let mut write_state =
1335 PoolMetaWriteState::for_startup_with_bootstrap_authority(Uuid::new_v4(), PoolMetaBootstrapAuthority::LegacyAdoption);
1336
1337 establish_pool_meta_bootstrap_identity_if_proven(vec![storage.clone()], &mut write_state, false)
1338 .await
1339 .expect("a non-elected distributed node must not create legacy adoption authority");
1340 let err = load_pool_meta_for_startup(vec![storage.clone()], &mut write_state)
1341 .await
1342 .expect_err("legacy adoption still requires the elected writer to create a durable identity");
1343 assert!(err.to_string().contains("no durable bootstrap identity"));
1344 assert!(
1345 !storage
1346 .objects
1347 .lock()
1348 .unwrap_or_else(std::sync::PoisonError::into_inner)
1349 .contains_key(POOL_META_NAME),
1350 "classification must not create pool metadata on non-elected nodes"
1351 );
1352 }
1353
1354 #[tokio::test]
1355 async fn test_unproven_pending_identity_cannot_authorize_all_missing_pool_meta() {
1356 let deployment_id = Uuid::new_v4();
1357 let storage = Arc::new(StartupPoolMetaStorage::new(Vec::new()));
1358 let mut proven_bootstrap = PoolMetaWriteState::for_startup(deployment_id, true);
1359 establish_pool_meta_bootstrap_identity_if_proven(vec![storage.clone()], &mut proven_bootstrap, true)
1360 .await
1361 .expect("fresh topology proof should persist a nonce-bound pending identity");
1362
1363 let identity = storage
1364 .objects
1365 .lock()
1366 .unwrap_or_else(std::sync::PoisonError::into_inner)
1367 .get(POOL_META_IDENTITY_NAME)
1368 .map(|(payload, _)| payload.clone())
1369 .expect("pending identity should be durable");
1370 assert!(!pool_meta_identity_initialized_for_test(&identity).expect("decode pending identity"));
1371
1372 let mut unproven_restart = PoolMetaWriteState::for_startup(deployment_id, false);
1373 let err = load_pool_meta_for_startup(vec![storage.clone()], &mut unproven_restart)
1374 .await
1375 .expect_err("a pending identity alone must not authorize an all-missing restart");
1376 assert!(err.to_string().contains("no verified fresh-bootstrap proof"));
1377 unproven_restart
1378 .ensure_write_safe("unproven pending identity restart")
1379 .expect_err("the rejected restart must latch the write gate");
1380 assert!(
1381 !storage
1382 .objects
1383 .lock()
1384 .unwrap_or_else(std::sync::PoisonError::into_inner)
1385 .contains_key(POOL_META_NAME),
1386 "classification must not create pool metadata"
1387 );
1388 }
1389
1390 #[tokio::test]
1391 async fn test_nonfresh_identity_repair_crash_never_persists_pending_bootstrap_authority() {
1392 let deployment_id = Uuid::new_v4();
1393 let initial = init_test_pool_meta(None);
1394 let storage = Arc::new(StartupPoolMetaStorage::new(startup_pool_meta_payload(&initial)));
1395 let mut write_state = PoolMetaWriteState::for_startup(deployment_id, false);
1396 let (loaded, replica_state) = load_pool_meta_for_startup(vec![storage.clone()], &mut write_state)
1397 .await
1398 .expect("an existing pool metadata snapshot may repair its missing identity");
1399
1400 storage.fail_pool_meta_write_at.store(1, Ordering::SeqCst);
1401 persist_pool_meta_for_startup_if_safe(&loaded, vec![storage.clone()], replica_state, &mut write_state, true, true)
1402 .await
1403 .expect_err("the injected crash boundary should stop before the pool metadata replacement");
1404
1405 let identity = storage
1406 .objects
1407 .lock()
1408 .unwrap_or_else(std::sync::PoisonError::into_inner)
1409 .get(POOL_META_IDENTITY_NAME)
1410 .map(|(payload, _)| payload.clone())
1411 .expect("identity repair should be durable before the failed topology write");
1412 assert!(
1413 pool_meta_identity_initialized_for_test(&identity).expect("decode repaired identity"),
1414 "an existing cluster must never leave pending bootstrap authority at this crash boundary"
1415 );
1416
1417 storage
1418 .objects
1419 .lock()
1420 .unwrap_or_else(std::sync::PoisonError::into_inner)
1421 .remove(POOL_META_NAME);
1422 let mut restarted = PoolMetaWriteState::for_startup(deployment_id, false);
1423 let err = load_pool_meta_for_startup(vec![storage], &mut restarted)
1424 .await
1425 .expect_err("wiping pool metadata after the crash must require recovery");
1426 assert!(err.to_string().contains("initialized cluster identity exists"));
1427 }
1428
1429 #[tokio::test]
1430 #[serial_test::serial(pool_meta_version_env)]
1431 async fn test_existing_legacy_identity_and_replica_repair_respects_disabled_v3_gates() {
1432 temp_env::async_with_vars(
1433 [
1434 (rustfs_config::ENV_POOL_META_V3_WRITE, None::<&str>),
1435 (rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, None::<&str>),
1436 ],
1437 async {
1438 let deployment_id = Uuid::new_v4();
1439 let corrupt = Arc::new(StartupPoolMetaStorage::new(vec![0, 1, 2]));
1440 let legacy = init_test_pool_meta(None);
1441 let valid = Arc::new(StartupPoolMetaStorage::new(startup_pool_meta_payload(&legacy)));
1442 let mut write_state = PoolMetaWriteState::for_startup(deployment_id, false);
1443 let (loaded, replica_state) = load_pool_meta_for_startup(vec![corrupt.clone(), valid.clone()], &mut write_state)
1444 .await
1445 .expect("existing V2 metadata should remain readable while its identity is missing");
1446 assert!(replica_state.needs_repair);
1447
1448 let committed = persist_pool_meta_for_startup_if_safe(
1449 &loaded,
1450 vec![corrupt.clone(), valid.clone()],
1451 replica_state,
1452 &mut write_state,
1453 true,
1454 true,
1455 )
1456 .await
1457 .expect("legacy replica and identity repair should succeed without crossing the V3 gate");
1458 assert_eq!(committed.version, POOL_META_VERSION);
1459 for storage in [corrupt, valid] {
1460 assert_eq!(
1461 *storage
1462 .pool_meta_written_versions
1463 .lock()
1464 .unwrap_or_else(std::sync::PoisonError::into_inner),
1465 vec![POOL_META_VERSION],
1466 "legacy repair must write exactly one V2 snapshot"
1467 );
1468 let identity = storage
1469 .objects
1470 .lock()
1471 .unwrap_or_else(std::sync::PoisonError::into_inner)
1472 .get(POOL_META_IDENTITY_NAME)
1473 .map(|(payload, _)| payload.clone())
1474 .expect("identity repair should persist on every pool");
1475 assert!(pool_meta_identity_initialized_for_test(&identity).expect("decode repaired identity"));
1476 }
1477 },
1478 )
1479 .await;
1480 }
1481
1482 fn startup_object(storage: &StartupPoolMetaStorage, object: &str) -> Option<Vec<u8>> {
1483 storage
1484 .objects
1485 .lock()
1486 .unwrap_or_else(std::sync::PoisonError::into_inner)
1487 .get(object)
1488 .map(|(payload, _)| payload.clone())
1489 }
1490
1491 fn error_chain_text(err: &Error) -> String {
1495 let mut parts = vec![err.to_string()];
1496 let mut current: Option<&(dyn std::error::Error + 'static)> = Some(err);
1497 while let Some(error) = current {
1498 current = if let Some(io) = error.downcast_ref::<std::io::Error>() {
1499 io.get_ref().map(|inner| inner as &(dyn std::error::Error + 'static))
1500 } else {
1501 error.source()
1502 };
1503 if let Some(next) = current {
1504 parts.push(next.to_string());
1505 }
1506 }
1507 parts.join(" <- ")
1508 }
1509
1510 fn inject_startup_object(storage: &StartupPoolMetaStorage, object: &str, payload: Vec<u8>) {
1511 storage
1512 .objects
1513 .lock()
1514 .unwrap_or_else(std::sync::PoisonError::into_inner)
1515 .insert(object.to_string(), (payload, format!("injected-{object}")));
1516 }
1517
1518 fn init_test_pool_meta_with_pools(pool_count: usize) -> PoolMeta {
1519 PoolMeta {
1520 version: POOL_META_VERSION,
1521 pools: (0..pool_count)
1522 .map(|id| PoolStatus {
1523 id,
1524 cmd_line: format!("pool-{id}"),
1525 last_update: OffsetDateTime::UNIX_EPOCH,
1526 decommission: None,
1527 })
1528 .collect(),
1529 dont_save: false,
1530 }
1531 }
1532
1533 fn two_pool_creator_states(deployment_id: Uuid) -> (PoolMetaWriteState, PoolMetaWriteState) {
1536 let node0 = PoolMetaWriteState::for_startup_with_pool_bootstrap_authorities(
1537 deployment_id,
1538 vec![PoolMetaBootstrapAuthority::Fresh, PoolMetaBootstrapAuthority::None],
1539 true,
1540 );
1541 let node1 = PoolMetaWriteState::for_startup_with_pool_bootstrap_authorities(
1542 deployment_id,
1543 vec![PoolMetaBootstrapAuthority::None, PoolMetaBootstrapAuthority::Fresh],
1544 false,
1545 );
1546 (node0, node1)
1547 }
1548
1549 #[tokio::test]
1550 async fn test_two_pool_bootstrap_with_distinct_format_creators_converges_through_creator_attestation() {
1551 let deployment_id = Uuid::new_v4();
1552 let pool0 = Arc::new(StartupPoolMetaStorage::new(Vec::new()));
1553 let pool1 = Arc::new(StartupPoolMetaStorage::new(Vec::new()));
1554 let pools = vec![pool0.clone(), pool1.clone()];
1555 let (mut node0, mut node1) = two_pool_creator_states(deployment_id);
1556 assert!(!node0.bootstrap_identity_proven(), "reading pool1's format is not deployment-wide proof");
1557 assert!(!node1.bootstrap_identity_proven());
1558
1559 establish_pool_meta_bootstrap_identity_if_proven(pools.clone(), &mut node1, false)
1562 .await
1563 .expect("a non-first creator without a durable nonce writes nothing");
1564 assert!(startup_object(&pool0, POOL_META_IDENTITY_NAME).is_none());
1565 assert!(startup_object(&pool1, POOL_META_IDENTITY_NAME).is_none());
1566 let err = load_pool_meta_for_startup(pools.clone(), &mut node1)
1567 .await
1568 .expect_err("nothing durable authorizes a non-elected node");
1569 assert!(err.to_string().contains("bootstrap pending"), "{err}");
1570 node1
1571 .ensure_write_safe("waiting non-elected creator")
1572 .expect("waiting for the elected writer must not latch the write gate");
1573
1574 establish_pool_meta_bootstrap_identity_if_proven(pools.clone(), &mut node0, true)
1577 .await
1578 .expect("the first pool's creator mints the pending identity");
1579 let minted = startup_object(&pool0, POOL_META_IDENTITY_NAME).expect("pool0 pending identity");
1580 assert!(!pool_meta_identity_initialized_for_test(&minted).expect("decode pending identity"));
1581 assert!(
1582 startup_object(&pool1, POOL_META_IDENTITY_NAME).is_none(),
1583 "node0 holds no first-hand proof for pool1 and must not attest it"
1584 );
1585 let err = load_pool_meta_for_startup(pools.clone(), &mut node0)
1586 .await
1587 .expect_err("an unattested pool keeps the elected writer from publishing");
1588 assert!(err.to_string().contains("waiting for every pool creator"), "{err}");
1589 node0
1590 .ensure_write_safe("waiting elected writer")
1591 .expect("waiting for creators must not latch the write gate");
1592 assert!(startup_object(&pool0, POOL_META_NAME).is_none());
1593
1594 establish_pool_meta_bootstrap_identity_if_proven(pools.clone(), &mut node1, false)
1597 .await
1598 .expect("the pool1 creator attests with the durable nonce");
1599 assert_eq!(startup_object(&pool1, POOL_META_IDENTITY_NAME).as_deref(), Some(minted.as_slice()));
1600 let err = load_pool_meta_for_startup(pools.clone(), &mut node1)
1601 .await
1602 .expect_err("a complete pending set never unlocks a non-elected node");
1603 assert!(err.to_string().contains("waiting for the elected writer to publish"), "{err}");
1604 node1
1605 .ensure_write_safe("attested non-elected creator")
1606 .expect("waiting for pool.bin must not latch the write gate");
1607 assert!(startup_object(&pool0, POOL_META_NAME).is_none());
1608
1609 establish_pool_meta_bootstrap_identity_if_proven(pools.clone(), &mut node0, true)
1612 .await
1613 .expect("re-establishing an already minted identity is idempotent");
1614 let (_, replica_state) = load_pool_meta_for_startup(pools.clone(), &mut node0)
1615 .await
1616 .expect("complete creator attestation authorizes the initial pool metadata write");
1617 persist_pool_meta_for_startup_if_safe(
1618 &init_test_pool_meta_with_pools(2),
1619 pools.clone(),
1620 replica_state,
1621 &mut node0,
1622 true,
1623 true,
1624 )
1625 .await
1626 .expect("the elected writer publishes pool.bin and commits the identity");
1627 for pool in [&pool0, &pool1] {
1628 assert!(startup_object(pool, POOL_META_NAME).is_some());
1629 let identity = startup_object(pool, POOL_META_IDENTITY_NAME).expect("committed identity");
1630 assert!(pool_meta_identity_initialized_for_test(&identity).expect("decode committed identity"));
1631 }
1632
1633 let before = startup_object(&pool1, POOL_META_IDENTITY_NAME);
1635 establish_pool_meta_bootstrap_identity_if_proven(pools.clone(), &mut node1, false)
1636 .await
1637 .expect("an initialized deployment never reopens bootstrap");
1638 assert_eq!(startup_object(&pool1, POOL_META_IDENTITY_NAME), before);
1639 load_pool_meta_for_startup(pools, &mut node1)
1640 .await
1641 .expect("published pool metadata admits the non-elected node");
1642 node1
1643 .ensure_write_safe("converged non-elected creator")
1644 .expect("no latch remains after convergence");
1645 }
1646
1647 #[tokio::test]
1648 async fn test_two_pool_bootstrap_rejects_pending_replicas_from_different_bootstraps() {
1649 let deployment_id = Uuid::new_v4();
1650 let pool0 = Arc::new(StartupPoolMetaStorage::new(Vec::new()));
1651 let pool1 = Arc::new(StartupPoolMetaStorage::new(Vec::new()));
1652 let pools = vec![pool0.clone(), pool1.clone()];
1653 let (mut node0, _) = two_pool_creator_states(deployment_id);
1654 establish_pool_meta_bootstrap_identity_if_proven(pools.clone(), &mut node0, true)
1655 .await
1656 .expect("the first pool's creator mints the pending identity");
1657 inject_startup_object(
1658 &pool1,
1659 POOL_META_IDENTITY_NAME,
1660 pending_pool_meta_identity_for_test(deployment_id, 1, Uuid::new_v4()).expect("encode foreign pending identity"),
1661 );
1662
1663 let err = load_pool_meta_for_startup(pools.clone(), &mut node0)
1664 .await
1665 .expect_err("a pending replica bound to another bootstrap nonce must fail closed");
1666 let chain = error_chain_text(&err);
1667 assert!(chain.contains("disagree on fresh-bootstrap proof"), "{chain}");
1668 node0
1669 .ensure_write_safe("split bootstrap")
1670 .expect_err("a split bootstrap latches the write gate");
1671 assert!(startup_object(&pool0, POOL_META_NAME).is_none());
1672 }
1673
1674 #[tokio::test]
1675 async fn test_two_pool_bootstrap_treats_corrupt_creator_replica_as_recovery_not_waiting() {
1676 let deployment_id = Uuid::new_v4();
1677 let pool0 = Arc::new(StartupPoolMetaStorage::new(Vec::new()));
1678 let pool1 = Arc::new(StartupPoolMetaStorage::new(Vec::new()));
1679 let pools = vec![pool0.clone(), pool1.clone()];
1680 let (mut node0, _) = two_pool_creator_states(deployment_id);
1681 establish_pool_meta_bootstrap_identity_if_proven(pools.clone(), &mut node0, true)
1682 .await
1683 .expect("the first pool's creator mints the pending identity");
1684 let mut corrupt = pending_pool_meta_identity_for_test(deployment_id, 1, Uuid::new_v4()).expect("encode identity");
1687 corrupt.truncate(4);
1688 corrupt.extend_from_slice(b"not a cluster identity");
1689 inject_startup_object(&pool1, POOL_META_IDENTITY_NAME, corrupt);
1690
1691 let err = load_pool_meta_for_startup(pools.clone(), &mut node0)
1692 .await
1693 .expect_err("a corrupt replica is not a creator that is still catching up");
1694 let chain = error_chain_text(&err);
1695 assert!(chain.contains("no verified fresh-bootstrap proof"), "{chain}");
1696 node0
1697 .ensure_write_safe("corrupt attestation")
1698 .expect_err("a corrupt attestation latches the write gate");
1699 assert!(startup_object(&pool0, POOL_META_NAME).is_none());
1700 }
1701
1702 #[tokio::test]
1703 async fn test_elected_restart_without_first_hand_proof_cannot_reuse_a_complete_pending_set() {
1704 let deployment_id = Uuid::new_v4();
1705 let pool0 = Arc::new(StartupPoolMetaStorage::new(Vec::new()));
1706 let pool1 = Arc::new(StartupPoolMetaStorage::new(Vec::new()));
1707 let pools = vec![pool0.clone(), pool1.clone()];
1708 let (mut node0, mut node1) = two_pool_creator_states(deployment_id);
1709 establish_pool_meta_bootstrap_identity_if_proven(pools.clone(), &mut node0, true)
1710 .await
1711 .expect("mint");
1712 establish_pool_meta_bootstrap_identity_if_proven(pools.clone(), &mut node1, false)
1713 .await
1714 .expect("attest");
1715 assert_eq!(
1716 startup_object(&pool0, POOL_META_IDENTITY_NAME),
1717 startup_object(&pool1, POOL_META_IDENTITY_NAME),
1718 "both creators attested the same pending identity"
1719 );
1720
1721 let mut restarted = PoolMetaWriteState::for_startup_with_pool_bootstrap_authorities(
1724 deployment_id,
1725 vec![PoolMetaBootstrapAuthority::None, PoolMetaBootstrapAuthority::None],
1726 true,
1727 );
1728 establish_pool_meta_bootstrap_identity_if_proven(pools.clone(), &mut restarted, true)
1729 .await
1730 .expect("a restart without first-hand proof writes nothing");
1731 let err = load_pool_meta_for_startup(pools.clone(), &mut restarted)
1732 .await
1733 .expect_err("a pending set alone never authorizes a writer without first-hand proof");
1734 assert!(err.to_string().contains("no verified fresh-bootstrap proof"), "{err}");
1735 restarted
1736 .ensure_write_safe("unproven restart")
1737 .expect_err("the rejected restart latches the write gate");
1738 assert!(startup_object(&pool0, POOL_META_NAME).is_none());
1739 }
1740
1741 #[tokio::test]
1742 async fn test_fresh_pool_joining_an_initialized_deployment_never_reopens_bootstrap() {
1743 let deployment_id = Uuid::new_v4();
1744 let pool0 = Arc::new(StartupPoolMetaStorage::new(Vec::new()));
1745 let mut founder = PoolMetaWriteState::for_startup(deployment_id, true);
1746 establish_pool_meta_bootstrap_identity_if_proven(vec![pool0.clone()], &mut founder, true)
1747 .await
1748 .expect("the founder mints");
1749 let (_, replica_state) = load_pool_meta_for_startup(vec![pool0.clone()], &mut founder)
1750 .await
1751 .expect("the founder may initialize");
1752 persist_pool_meta_for_startup_if_safe(
1753 &init_test_pool_meta(None),
1754 vec![pool0.clone()],
1755 replica_state,
1756 &mut founder,
1757 true,
1758 true,
1759 )
1760 .await
1761 .expect("the founder commits");
1762 let founded = startup_object(&pool0, POOL_META_IDENTITY_NAME).expect("committed identity");
1763 assert!(pool_meta_identity_initialized_for_test(&founded).expect("decode committed identity"));
1764
1765 let pool1 = Arc::new(StartupPoolMetaStorage::new(Vec::new()));
1768 let pools = vec![pool0.clone(), pool1.clone()];
1769 for elected in [false, true] {
1770 let mut joiner = PoolMetaWriteState::for_startup_with_pool_bootstrap_authorities(
1771 deployment_id,
1772 vec![PoolMetaBootstrapAuthority::None, PoolMetaBootstrapAuthority::Fresh],
1773 elected,
1774 );
1775 establish_pool_meta_bootstrap_identity_if_proven(pools.clone(), &mut joiner, elected)
1776 .await
1777 .expect("an initialized deployment ignores first-hand proof for a new pool");
1778 assert!(
1779 startup_object(&pool1, POOL_META_IDENTITY_NAME).is_none(),
1780 "no pending identity may be written to an expansion pool"
1781 );
1782 assert_eq!(startup_object(&pool0, POOL_META_IDENTITY_NAME).as_deref(), Some(founded.as_slice()));
1783 let (_, replica_state) = load_pool_meta_for_startup(pools.clone(), &mut joiner)
1784 .await
1785 .expect("published pool metadata admits the joiner");
1786 joiner
1787 .ensure_write_safe("expansion joiner")
1788 .expect("joining never latches the write gate");
1789 if elected {
1790 persist_pool_meta_for_startup_if_safe(
1791 &init_test_pool_meta_with_pools(2),
1792 pools.clone(),
1793 replica_state,
1794 &mut joiner,
1795 true,
1796 true,
1797 )
1798 .await
1799 .expect("the topology update repairs the new pool's replicas");
1800 let identity = startup_object(&pool1, POOL_META_IDENTITY_NAME).expect("expansion pool identity");
1801 assert!(pool_meta_identity_initialized_for_test(&identity).expect("decode repaired identity"));
1802 assert!(startup_object(&pool1, POOL_META_NAME).is_some());
1803 }
1804 }
1805 }
1806
1807 #[tokio::test]
1808 async fn test_store_init_distinguishes_fresh_deployment_from_wiped_lagging_node() {
1809 let deployment_id = Uuid::new_v4();
1810 let storage = Arc::new(StartupPoolMetaStorage::new(Vec::new()));
1811 let mut fresh_state = PoolMetaWriteState::for_startup(deployment_id, true);
1812 establish_pool_meta_bootstrap_identity_if_proven(vec![storage.clone()], &mut fresh_state, true)
1813 .await
1814 .expect("fresh topology proof should become a durable pending identity");
1815 let (_, replica_state) = load_pool_meta_for_startup(vec![storage.clone()], &mut fresh_state)
1816 .await
1817 .expect("new formats with no identity may initialize pool metadata exactly once");
1818 let committed = persist_pool_meta_for_startup_if_safe(
1819 &init_test_pool_meta(None),
1820 vec![storage.clone()],
1821 replica_state,
1822 &mut fresh_state,
1823 true,
1824 true,
1825 )
1826 .await
1827 .expect("fresh deployment should durably commit identity and pool metadata");
1828 assert_eq!(committed.pools[0].cmd_line, "pool-0");
1829 {
1830 let objects = storage.objects.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
1831 assert!(objects.contains_key(POOL_META_NAME));
1832 assert!(objects.contains_key(POOL_META_IDENTITY_NAME));
1833 }
1834
1835 storage
1836 .objects
1837 .lock()
1838 .unwrap_or_else(std::sync::PoisonError::into_inner)
1839 .remove(POOL_META_NAME);
1840 let mut wiped_pool_state = PoolMetaWriteState::for_startup(deployment_id, false);
1841 let err = load_pool_meta_for_startup(vec![storage.clone()], &mut wiped_pool_state)
1842 .await
1843 .expect_err("an initialized identity must prevent an empty pool metadata rebuild");
1844 assert!(err.to_string().contains("initialized cluster identity exists"));
1845
1846 storage
1847 .objects
1848 .lock()
1849 .unwrap_or_else(std::sync::PoisonError::into_inner)
1850 .remove(POOL_META_IDENTITY_NAME);
1851 let mut wiped_all_state = PoolMetaWriteState::for_startup(deployment_id, false);
1852 let err = load_pool_meta_for_startup(vec![storage], &mut wiped_all_state)
1853 .await
1854 .expect_err("existing formats without identity or pool metadata must require recovery");
1855 assert!(err.to_string().contains("no durable bootstrap identity"));
1856
1857 let distributed_node = Arc::new(StartupPoolMetaStorage::new(Vec::new()));
1858 let mut non_elected_state = PoolMetaWriteState::for_startup(Uuid::new_v4(), true);
1859 establish_pool_meta_bootstrap_identity_if_proven(vec![distributed_node.clone()], &mut non_elected_state, false)
1860 .await
1861 .expect("a non-elected distributed node must not create bootstrap authority");
1862 let err = load_pool_meta_for_startup(vec![distributed_node], &mut non_elected_state)
1863 .await
1864 .expect_err("a distributed node without durable identity or pool metadata must require recovery");
1865 assert!(err.to_string().contains("no durable bootstrap identity"));
1866 }
1867
1868 #[tokio::test]
1869 async fn test_store_init_resumes_pending_v3_bootstrap_without_legacy_overwrite() {
1870 let deployment_id = Uuid::new_v4();
1871 let storage = Arc::new(StartupPoolMetaStorage::new(Vec::new()));
1872 let mut first_bootstrap = PoolMetaWriteState::for_startup(deployment_id, true);
1873 establish_pool_meta_bootstrap_identity_if_proven(vec![storage.clone()], &mut first_bootstrap, true)
1874 .await
1875 .expect("fresh bootstrap should persist a pending identity");
1876 let (_, replica_state) = load_pool_meta_for_startup(vec![storage.clone()], &mut first_bootstrap)
1877 .await
1878 .expect("the durable pending identity should authorize the initial pool metadata write");
1879
1880 storage.fail_pool_meta_write_at.store(2, Ordering::SeqCst);
1881 persist_pool_meta_for_startup_if_safe(
1882 &init_test_pool_meta(None),
1883 vec![storage.clone()],
1884 replica_state,
1885 &mut first_bootstrap,
1886 true,
1887 true,
1888 )
1889 .await
1890 .expect_err("the injected crash boundary should leave only the V3 prepare record");
1891
1892 let (pending_pool, pending_identity) = {
1893 let objects = storage.objects.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
1894 (
1895 objects
1896 .get(POOL_META_NAME)
1897 .map(|(payload, _)| payload.clone())
1898 .expect("the V3 prepare record should be durable"),
1899 objects
1900 .get(POOL_META_IDENTITY_NAME)
1901 .map(|(payload, _)| payload.clone())
1902 .expect("the bootstrap identity should be durable"),
1903 )
1904 };
1905 assert_eq!(pool_meta_v3_commit_state_for_test(pending_pool).expect("decode pending V3"), (1, false));
1906 assert!(!pool_meta_identity_initialized_for_test(&pending_identity).expect("decode pending identity"));
1907
1908 storage.fail_pool_meta_write_at.store(0, Ordering::SeqCst);
1909 let mut restarted = PoolMetaWriteState::for_startup(deployment_id, false);
1910 let (loaded, replica_state) = load_pool_meta_for_startup(vec![storage.clone()], &mut restarted)
1911 .await
1912 .expect("restart should recover the predecessor embedded in the pending V3 record");
1913 assert!(loaded.pools.is_empty());
1914 assert!(replica_state.needs_repair);
1915 let committed = persist_pool_meta_for_startup_if_safe(
1916 &init_test_pool_meta(None),
1917 vec![storage.clone()],
1918 replica_state,
1919 &mut restarted,
1920 true,
1921 true,
1922 )
1923 .await
1924 .expect("restart should finish generation 1 before promoting the identity");
1925 assert_eq!(committed.version, 3);
1926
1927 let (committed_pool, committed_identity) = {
1928 let objects = storage.objects.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
1929 (
1930 objects
1931 .get(POOL_META_NAME)
1932 .map(|(payload, _)| payload.clone())
1933 .expect("the committed V3 record should be durable"),
1934 objects
1935 .get(POOL_META_IDENTITY_NAME)
1936 .map(|(payload, _)| payload.clone())
1937 .expect("the committed identity should be durable"),
1938 )
1939 };
1940 assert_eq!(
1941 pool_meta_v3_commit_state_for_test(committed_pool).expect("decode committed V3"),
1942 (1, true)
1943 );
1944 assert!(pool_meta_identity_initialized_for_test(&committed_identity).expect("decode committed identity"));
1945 assert!(
1946 storage
1947 .pool_meta_written_versions
1948 .lock()
1949 .unwrap_or_else(std::sync::PoisonError::into_inner)
1950 .iter()
1951 .all(|version| *version == 3),
1952 "bootstrap recovery must never overwrite a pending V3 record with legacy metadata"
1953 );
1954 }
1955
1956 #[tokio::test]
1957 async fn test_store_init_pool_meta_falls_back_from_corrupt_first_replica() {
1958 let corrupt = Arc::new(StartupPoolMetaStorage::new(vec![0, 1, 2]));
1959 let expected = init_test_pool_meta(None);
1960 let backup = Arc::new(StartupPoolMetaStorage::new(startup_pool_meta_payload(&expected)));
1961 let mut write_state = PoolMetaWriteState::default();
1962
1963 let (loaded, replica_state) = load_pool_meta_for_startup(vec![corrupt.clone(), backup.clone()], &mut write_state)
1964 .await
1965 .expect("startup should select the validated backup replica");
1966
1967 assert!(replica_state.needs_repair);
1968 assert!(replica_state.repair_write_safe);
1969 assert_eq!(loaded.pools.len(), 1);
1970 assert_eq!(loaded.pools[0].cmd_line, expected.pools[0].cmd_line);
1971 assert!(corrupt.read_without_lock.load(Ordering::SeqCst));
1972 assert!(backup.read_without_lock.load(Ordering::SeqCst));
1973
1974 persist_pool_meta_for_startup_if_safe(
1975 &loaded,
1976 vec![corrupt.clone(), backup.clone()],
1977 replica_state,
1978 &mut write_state,
1979 false,
1980 true,
1981 )
1982 .await
1983 .expect("the elected startup writer should repair validated corrupt replicas");
1984
1985 let corrupt_write = corrupt
1986 .written_payload
1987 .lock()
1988 .unwrap_or_else(std::sync::PoisonError::into_inner)
1989 .clone()
1990 .expect("corrupt replica should be repaired");
1991 let backup_write = backup
1992 .written_payload
1993 .lock()
1994 .unwrap_or_else(std::sync::PoisonError::into_inner)
1995 .clone()
1996 .expect("backup replica should receive the same canonical snapshot");
1997 assert_eq!(corrupt_write, backup_write);
1998 assert_ne!(corrupt_write, corrupt.read_payload);
1999 }
2000
2001 #[tokio::test]
2002 async fn test_store_init_pool_meta_does_not_repair_unreadable_replica() {
2003 let valid = Arc::new(StartupPoolMetaStorage::new(startup_pool_meta_payload(&init_test_pool_meta(None))));
2004 let unreadable = Arc::new(StartupPoolMetaStorage::unreadable());
2005 let mut write_state = PoolMetaWriteState::default();
2006
2007 let (loaded, replica_state) = load_pool_meta_for_startup(vec![valid.clone(), unreadable.clone()], &mut write_state)
2008 .await
2009 .expect("startup should use a validated replica without overwriting an unreadable copy");
2010 assert!(replica_state.needs_repair);
2011 assert!(!replica_state.repair_write_safe);
2012
2013 persist_pool_meta_for_startup_if_safe(
2014 &loaded,
2015 vec![valid.clone(), unreadable.clone()],
2016 replica_state,
2017 &mut write_state,
2018 false,
2019 true,
2020 )
2021 .await
2022 .expect("an unreadable copy should defer repair when no topology write is needed");
2023 assert!(!valid.wrote_without_lock.load(Ordering::SeqCst));
2024 assert!(!unreadable.wrote_without_lock.load(Ordering::SeqCst));
2025
2026 let err = persist_pool_meta_for_startup_if_safe(
2027 &loaded,
2028 vec![valid.clone(), unreadable.clone()],
2029 replica_state,
2030 &mut write_state,
2031 true,
2032 true,
2033 )
2034 .await
2035 .expect_err("a topology update must not overwrite an unreadable replica");
2036 assert!(err.to_string().contains("cannot overwrite an unreadable replica"));
2037 assert!(!valid.wrote_without_lock.load(Ordering::SeqCst));
2038 assert!(!unreadable.wrote_without_lock.load(Ordering::SeqCst));
2039 }
2040
2041 #[tokio::test]
2042 async fn test_store_init_pool_meta_stays_blocked_after_all_replicas_were_unreadable() {
2043 let unreadable_a = Arc::new(StartupPoolMetaStorage::unreadable());
2044 let unreadable_b = Arc::new(StartupPoolMetaStorage::unreadable());
2045 let mut write_state = PoolMetaWriteState::default();
2046
2047 load_pool_meta_for_startup(vec![unreadable_a, unreadable_b], &mut write_state)
2048 .await
2049 .expect_err("startup must fail when no readable pool metadata replica exists");
2050
2051 let repaired = Arc::new(StartupPoolMetaStorage::new(startup_pool_meta_payload(&init_test_pool_meta(None))));
2052 let (loaded, replica_state) = load_pool_meta_for_startup(vec![repaired.clone()], &mut write_state)
2053 .await
2054 .expect("a later startup retry may read the repaired replica");
2055 assert!(replica_state.repair_write_safe);
2056
2057 let err =
2058 persist_pool_meta_for_startup_if_safe(&loaded, vec![repaired.clone()], replica_state, &mut write_state, true, true)
2059 .await
2060 .expect_err("the same store instance must not write after observing unreadable replicas");
2061 assert!(
2062 err.to_string()
2063 .contains("restart after all replicas are readable and consistent")
2064 );
2065 assert!(!repaired.wrote_without_lock.load(Ordering::SeqCst));
2066 }
2067
2068 #[test]
2069 fn test_pool_first_endpoint_is_local_respects_local_flag() {
2070 let mut local_endpoint = Endpoint::try_from("http://127.0.0.1:9000/data").expect("endpoint should parse");
2071 local_endpoint.is_local = true;
2072 let pool = PoolEndpoints {
2073 legacy: false,
2074 set_count: 1,
2075 drives_per_set: 1,
2076 endpoints: Endpoints::from(vec![local_endpoint]),
2077 cmd_line: "pool-0".to_string(),
2078 platform: String::new(),
2079 };
2080
2081 assert!(pool_first_endpoint_is_local(&pool));
2082 }
2083
2084 #[test]
2085 fn test_pool_first_endpoint_is_local_rejects_missing_endpoint() {
2086 let pool = PoolEndpoints {
2087 legacy: false,
2088 set_count: 1,
2089 drives_per_set: 1,
2090 endpoints: Endpoints::from(Vec::<Endpoint>::new()),
2091 cmd_line: "pool-0".to_string(),
2092 platform: String::new(),
2093 };
2094
2095 assert!(!pool_first_endpoint_is_local(&pool));
2096 }
2097
2098 #[test]
2099 fn test_local_decommission_watchdog_retry_delay_is_bounded() {
2100 assert_eq!(local_decommission_watchdog_retry_delay(1), LOCAL_DECOMMISSION_RESUME_RETRY_DELAY);
2101 assert_eq!(
2102 local_decommission_watchdog_retry_delay(u32::MAX),
2103 LOCAL_DECOMMISSION_WATCHDOG_MAX_RETRY_DELAY
2104 );
2105 }
2106
2107 #[tokio::test(start_paused = true)]
2108 async fn test_local_decommission_watchdog_cancelled_start_does_not_reconcile() {
2109 let rx = CancellationToken::new();
2110 rx.cancel();
2111 run_local_decommission_watchdog(rx, || async {
2112 panic!("cancelled startup must not schedule persisted work");
2113 })
2114 .await;
2115 }
2116
2117 #[tokio::test]
2118 #[serial_test::serial]
2119 async fn test_local_decommission_recovery_waits_for_live_fleet_proof_before_reserving_worker() {
2120 let (_temp_dirs, store, _other_store) = crate::services::rebalance::test_two_pool_stores(None).await;
2121 mark_test_pool_decommissioning(&store, 0).await;
2122 assert!(store.ctx.is_dist_erasure().await);
2123 let worker_rx = CancellationToken::new();
2124
2125 {
2126 let _proof_guard = crate::services::notification_sys::without_cross_pool_fence_fleet_proof_for_test();
2127 let err = super::reconcile_local_decommission_after_init(&store, worker_rx.clone())
2128 .await
2129 .expect_err("cold distributed recovery must wait for live fleet proof");
2130 assert!(
2131 crate::core::pools::is_pool_activation_fleet_proof_error(&err),
2132 "recovery must reach the live fleet proof gate: {err:?}"
2133 );
2134 assert!(store.decommission_cancelers.read().await.iter().all(Option::is_none));
2135 assert!(pool_meta_has_active_decommission(&*store.pool_meta.read().await));
2136 }
2137
2138 super::reconcile_local_decommission_after_init(&store, worker_rx.clone())
2139 .await
2140 .expect("restored fleet proof should admit the persisted worker");
2141 assert!(store.has_active_local_decommission_worker().await);
2142 worker_rx.cancel();
2143 }
2144
2145 #[tokio::test(start_paused = true)]
2146 async fn test_local_decommission_watchdog_retries_general_failures_until_cancelled() {
2147 let rx = CancellationToken::new();
2148 let attempts = Arc::new(AtomicUsize::new(0));
2149 let task = tokio::spawn(run_local_decommission_watchdog(rx.clone(), {
2150 let attempts = attempts.clone();
2151 let rx = rx.clone();
2152 move || {
2153 let attempts = attempts.clone();
2154 let rx = rx.clone();
2155 async move {
2156 match attempts.fetch_add(1, Ordering::SeqCst) {
2157 0 => Err(StorageError::other("pool activation requires a live fleet capability proof")),
2158 1 => Err(StorageError::SlowDown),
2159 _ => {
2160 rx.cancel();
2161 Ok(())
2162 }
2163 }
2164 }
2165 }
2166 }));
2167
2168 tokio::task::yield_now().await;
2169 assert_eq!(attempts.load(Ordering::SeqCst), 1);
2170 tokio::time::advance(LOCAL_DECOMMISSION_RESUME_RETRY_DELAY).await;
2171 tokio::task::yield_now().await;
2172 assert_eq!(attempts.load(Ordering::SeqCst), 2);
2173 tokio::time::advance(local_decommission_watchdog_retry_delay(2)).await;
2174 task.await.expect("watchdog task should exit after cancellation");
2175 assert_eq!(attempts.load(Ordering::SeqCst), 3);
2176 }
2177
2178 #[tokio::test(start_paused = true)]
2179 async fn test_local_decommission_watchdog_rescans_after_success() {
2180 let rx = CancellationToken::new();
2181 let attempts = Arc::new(AtomicUsize::new(0));
2182 let task = tokio::spawn(run_local_decommission_watchdog(rx.clone(), {
2183 let attempts = attempts.clone();
2184 let rx = rx.clone();
2185 move || {
2186 let attempts = attempts.clone();
2187 let rx = rx.clone();
2188 async move {
2189 if attempts.fetch_add(1, Ordering::SeqCst) == 1 {
2190 rx.cancel();
2191 }
2192 Ok(())
2193 }
2194 }
2195 }));
2196
2197 tokio::task::yield_now().await;
2198 assert_eq!(attempts.load(Ordering::SeqCst), 1);
2199 tokio::time::advance(super::LOCAL_DECOMMISSION_WATCHDOG_INTERVAL).await;
2200 task.await.expect("watchdog task should exit after cancellation");
2201 assert_eq!(attempts.load(Ordering::SeqCst), 2);
2202 }
2203
2204 #[test]
2205 fn test_should_retry_format_load_rejects_permanent_corruption() {
2206 assert!(!should_retry_format_load(&StorageError::CorruptedFormat));
2207 assert!(should_retry_format_load(&StorageError::ErasureReadQuorum));
2208 assert!(should_retry_format_load(&StorageError::FirstDiskWait));
2209 }
2210
2211 #[test]
2212 fn test_should_retry_format_load_rejects_permanent_topology_errors() {
2213 for error in [
2214 StorageError::UnsupportedSnsdExpansion { configured_drives: 4 },
2215 StorageError::PoolTopologyMismatch {
2216 stored_drives: 4,
2217 stored_set_drive_count: 4,
2218 configured_drives: 8,
2219 configured_set_drive_count: 8,
2220 },
2221 ] {
2222 assert!(!should_retry_format_load(&error), "topology errors require operator action: {error}");
2223 }
2224 for error in [
2225 StorageError::DiskNotFound,
2226 StorageError::Timeout,
2227 StorageError::RemoteNotInitialized,
2228 StorageError::NotFirstDisk,
2229 StorageError::other(std::io::Error::from(std::io::ErrorKind::ConnectionRefused)),
2230 ] {
2231 assert!(
2232 should_retry_format_load(&error),
2233 "transient failures retain their existing retry path: {error}"
2234 );
2235 }
2236 }
2237
2238 #[test]
2239 fn test_should_auto_start_rebalance_after_init_allows_active_rebalance_without_decommission() {
2240 assert!(should_auto_start_rebalance_after_init(false, true));
2241 }
2242
2243 #[test]
2244 fn test_should_auto_start_rebalance_after_init_rejects_active_decommission() {
2245 assert!(!should_auto_start_rebalance_after_init(true, true));
2246 }
2247
2248 #[test]
2249 fn test_should_auto_start_rebalance_after_init_rejects_terminal_or_missing_rebalance() {
2250 assert!(!should_auto_start_rebalance_after_init(false, false));
2251 }
2252
2253 #[test]
2254 fn test_should_defer_rebalance_auto_start_only_without_distributed_fleet_proof() {
2255 assert!(should_defer_rebalance_auto_start(true, false));
2256 assert!(!should_defer_rebalance_auto_start(true, true));
2257 assert!(!should_defer_rebalance_auto_start(false, false));
2258 }
2259
2260 #[test]
2261 fn test_store_init_recovery_skips_rebalance_when_decommission_metadata_is_active() {
2262 let pool_meta = init_test_pool_meta(Some(PoolDecommissionInfo {
2263 start_time: Some(OffsetDateTime::UNIX_EPOCH),
2264 complete: false,
2265 failed: false,
2266 canceled: false,
2267 ..Default::default()
2268 }));
2269 let rebalance_meta = Some(RebalanceMeta {
2270 pool_stats: vec![RebalanceStats {
2271 participating: true,
2272 info: RebalanceInfo {
2273 status: RebalStatus::Started,
2274 ..Default::default()
2275 },
2276 ..Default::default()
2277 }],
2278 ..Default::default()
2279 });
2280
2281 assert!(!should_auto_start_rebalance_after_init(
2282 pool_meta_has_active_decommission(&pool_meta),
2283 rebalance_meta
2284 .as_ref()
2285 .is_some_and(crate::services::rebalance::rebalance_requires_worker_activation)
2286 ));
2287 }
2288
2289 #[test]
2290 fn test_store_init_recovery_allows_active_rebalance_without_decommission() {
2291 let pool_meta = init_test_pool_meta(None);
2292 let rebalance_meta = Some(RebalanceMeta {
2293 pool_stats: vec![RebalanceStats {
2294 participating: true,
2295 info: RebalanceInfo {
2296 status: RebalStatus::Started,
2297 ..Default::default()
2298 },
2299 ..Default::default()
2300 }],
2301 ..Default::default()
2302 });
2303
2304 assert!(should_auto_start_rebalance_after_init(
2305 pool_meta_has_active_decommission(&pool_meta),
2306 rebalance_meta
2307 .as_ref()
2308 .is_some_and(crate::services::rebalance::rebalance_requires_worker_activation)
2309 ));
2310 }
2311
2312 #[test]
2313 fn test_store_init_recovery_skips_completed_rebalance_metadata() {
2314 let pool_meta = init_test_pool_meta(None);
2315 let rebalance_meta = Some(RebalanceMeta {
2316 pool_stats: vec![RebalanceStats {
2317 participating: true,
2318 info: RebalanceInfo {
2319 status: RebalStatus::Completed,
2320 ..Default::default()
2321 },
2322 ..Default::default()
2323 }],
2324 ..Default::default()
2325 });
2326
2327 assert!(!should_auto_start_rebalance_after_init(
2328 pool_meta_has_active_decommission(&pool_meta),
2329 rebalance_meta
2330 .as_ref()
2331 .is_some_and(crate::services::rebalance::rebalance_requires_worker_activation)
2332 ));
2333 }
2334
2335 #[test]
2336 fn test_resolve_store_init_stage_result_passthrough_ok() {
2337 resolve_store_init_stage_result(Ok(()), "load_rebalance_meta").expect("successful stage should pass through");
2338 }
2339
2340 #[test]
2341 fn test_resolve_store_init_stage_result_wraps_error_context() {
2342 let err = resolve_store_init_stage_result(Err(StorageError::SlowDown), "start_rebalance")
2343 .expect_err("failed stage should be wrapped");
2344 let err_message = err.to_string();
2345 assert!(err_message.contains("store init failed during start_rebalance"));
2346 assert!(err_message.contains(&StorageError::SlowDown.to_string()));
2347 }
2348
2349 #[tokio::test]
2350 async fn test_wait_for_local_decommission_resume_delay_returns_true_after_delay() {
2351 let rx = CancellationToken::new();
2352 assert!(wait_for_local_decommission_resume_delay(&rx, Duration::from_millis(1)).await);
2353 }
2354
2355 #[tokio::test]
2356 async fn test_wait_for_local_decommission_resume_delay_returns_false_when_cancelled() {
2357 let rx = CancellationToken::new();
2358 rx.cancel();
2359 assert!(!wait_for_local_decommission_resume_delay(&rx, Duration::from_secs(1)).await);
2360 }
2361
2362 #[test]
2363 fn test_pool_first_endpoint_is_local_uses_pool_scope_for_expansion() {
2364 let mut remote_endpoint = Endpoint::try_from("http://127.0.0.2:9000/data1").expect("remote endpoint should parse");
2365 remote_endpoint.is_local = false;
2366
2367 let mut local_endpoint = Endpoint::try_from("http://127.0.0.1:9000/data1").expect("local endpoint should parse");
2368 local_endpoint.is_local = true;
2369
2370 let endpoints = EndpointServerPools::from(vec![
2371 PoolEndpoints {
2372 legacy: false,
2373 set_count: 1,
2374 drives_per_set: 1,
2375 endpoints: Endpoints::from(vec![remote_endpoint]),
2376 cmd_line: "pool-0".to_string(),
2377 platform: String::new(),
2378 },
2379 PoolEndpoints {
2380 legacy: false,
2381 set_count: 1,
2382 drives_per_set: 1,
2383 endpoints: Endpoints::from(vec![local_endpoint]),
2384 cmd_line: "pool-1".to_string(),
2385 platform: String::new(),
2386 },
2387 ]);
2388
2389 assert!(!endpoints.first_local(), "cluster first endpoint is intentionally remote");
2390 assert!(
2391 pool_first_endpoint_is_local(endpoints.as_ref().get(1).expect("second pool should exist")),
2392 "the expanded pool should be initialized by its own first local endpoint"
2393 );
2394 }
2395
2396 fn rpc_preflight_pools(pools: Vec<Vec<Endpoint>>) -> EndpointServerPools {
2397 EndpointServerPools::from(
2398 pools
2399 .into_iter()
2400 .enumerate()
2401 .map(|(pool_index, endpoints)| PoolEndpoints {
2402 legacy: false,
2403 set_count: 1,
2404 drives_per_set: endpoints.len(),
2405 endpoints: Endpoints::from(endpoints),
2406 cmd_line: format!("pool-{pool_index}"),
2407 platform: String::new(),
2408 })
2409 .collect::<Vec<_>>(),
2410 )
2411 }
2412
2413 fn parsed_local_endpoint() -> Endpoint {
2414 let mut endpoint = Endpoint::try_from("http://127.0.0.1:9000/data").expect("local endpoint should parse");
2415 endpoint.is_local = true;
2416 endpoint
2417 }
2418
2419 fn parsed_remote_endpoint() -> Endpoint {
2420 let mut endpoint = Endpoint::try_from("http://10.0.0.2:9000/data").expect("remote endpoint should parse");
2421 endpoint.is_local = false;
2422 endpoint
2423 }
2424
2425 #[test]
2426 fn test_rpc_secret_preflight_aborts_for_remote_endpoints_without_secret() {
2427 let pools = rpc_preflight_pools(vec![vec![parsed_local_endpoint(), parsed_remote_endpoint()]]);
2430
2431 let err = preflight_startup_rpc_secret_with(&pools, || {
2432 Err(std::io::Error::other(rustfs_credentials::RPC_SECRET_REQUIRED_MESSAGE))
2433 })
2434 .expect_err("remote endpoints without an RPC secret must abort startup before the format retry loop");
2435
2436 let message = err.to_string();
2437 assert!(message.contains("store init aborted"), "unexpected error: {message}");
2438 assert!(
2439 message.contains(rustfs_credentials::RPC_SECRET_REQUIRED_MESSAGE),
2440 "error must state the resolution failure: {message}"
2441 );
2442 assert!(
2443 message.contains(rustfs_credentials::RPC_SECRET_REQUIRED_OPERATOR_MESSAGE),
2444 "error must carry the operator remediation guidance: {message}"
2445 );
2446 }
2447
2448 #[test]
2449 fn test_rpc_secret_preflight_aborts_for_remote_endpoint_in_later_pool() {
2450 let pools = rpc_preflight_pools(vec![vec![parsed_local_endpoint()], vec![parsed_remote_endpoint()]]);
2453
2454 let err = preflight_startup_rpc_secret_with(&pools, || {
2455 Err(std::io::Error::other(rustfs_credentials::RPC_SECRET_REQUIRED_MESSAGE))
2456 })
2457 .expect_err("a remote endpoint in a later pool must abort startup without an RPC secret");
2458
2459 let message = err.to_string();
2460 assert!(message.contains("store init aborted"), "unexpected error: {message}");
2461 }
2462
2463 #[test]
2464 fn test_rpc_secret_preflight_skips_resolution_for_local_only_endpoints() {
2465 preflight_startup_rpc_secret_with(&rpc_preflight_pools(vec![vec![parsed_local_endpoint()]]), || {
2466 panic!("single-node topology must not resolve an RPC secret")
2467 })
2468 .expect("local-only topology must start without an RPC secret");
2469 }
2470
2471 #[test]
2472 fn test_rpc_secret_preflight_accepts_remote_endpoints_with_resolved_secret() {
2473 preflight_startup_rpc_secret_with(&rpc_preflight_pools(vec![vec![parsed_remote_endpoint()]]), || {
2474 Ok("resolved-secret".to_string())
2475 })
2476 .expect("a resolvable RPC secret must not block startup");
2477 }
2478
2479 #[tokio::test]
2480 #[serial_test::serial(storage_class_env)]
2481 async fn test_new_with_instance_ctx_aborts_before_format_retry_loop_without_rpc_secret() {
2482 if rustfs_credentials::try_get_rpc_token().is_ok() {
2488 return;
2489 }
2490
2491 let mut remote_endpoint = parsed_remote_endpoint();
2492 remote_endpoint.set_pool_index(0);
2493 remote_endpoint.set_set_index(0);
2494 remote_endpoint.set_disk_index(0);
2495
2496 let err = crate::store::ECStore::new_with_instance_ctx(
2497 "127.0.0.1:0".parse().expect("test address"),
2498 rpc_preflight_pools(vec![vec![remote_endpoint]]),
2499 CancellationToken::new(),
2500 Arc::new(crate::runtime::instance::InstanceContext::new()),
2501 )
2502 .await
2503 .expect_err("distributed init without an RPC secret must abort before the format retry loop");
2504
2505 let message = err.to_string();
2506 assert!(message.contains("store init aborted"), "unexpected error: {message}");
2507 assert!(
2508 message.contains(rustfs_credentials::RPC_SECRET_REQUIRED_OPERATOR_MESSAGE),
2509 "abort must carry the operator remediation guidance: {message}"
2510 );
2511 }
2512
2513 fn endpoint_pools_with_drive_counts(counts: &[usize]) -> EndpointServerPools {
2514 EndpointServerPools::from(
2515 counts
2516 .iter()
2517 .enumerate()
2518 .map(|(pool_index, &drives_per_set)| PoolEndpoints {
2519 legacy: false,
2520 set_count: 1,
2521 drives_per_set,
2522 endpoints: Endpoints::from(Vec::new()),
2523 cmd_line: format!("pool-{pool_index}"),
2524 platform: String::new(),
2525 })
2526 .collect::<Vec<_>>(),
2527 )
2528 }
2529
2530 #[test]
2531 fn startup_pool_defaults_are_resolved_per_pool() {
2532 let validate = |pools: &EndpointServerPools| {
2533 let drive_counts: Vec<_> = pools.as_ref().iter().map(|pool| pool.drives_per_set).collect();
2534 crate::config::storageclass::lookup_config_for_pools_without_env(&KVS::new(), &drive_counts).map(|_| ())
2535 };
2536 let defaults = resolve_startup_pool_defaults_with(&endpoint_pools_with_drive_counts(&[4, 2]), validate)
2537 .expect("heterogeneous topology should resolve");
2538 assert_eq!(defaults, vec![2, 1]);
2539
2540 let defaults = resolve_startup_pool_defaults_with(&endpoint_pools_with_drive_counts(&[4, 6]), validate)
2541 .expect("heterogeneous topology should resolve");
2542 assert_eq!(defaults, vec![2, 3]);
2543 }
2544
2545 #[test]
2546 fn startup_pool_defaults_validate_explicit_environment_for_every_pool() {
2547 let validate = |pools: &EndpointServerPools| {
2548 let drive_counts: Vec<_> = pools.as_ref().iter().map(|pool| pool.drives_per_set).collect();
2549 let mut kvs = KVS::new();
2550 kvs.insert(crate::config::storageclass::CLASS_STANDARD.to_string(), "EC:2".to_string());
2551 crate::config::storageclass::lookup_config_for_pools_without_env(&kvs, &drive_counts).map(|_| ())
2552 };
2553 let err = resolve_startup_pool_defaults_with(&endpoint_pools_with_drive_counts(&[4, 2]), validate)
2554 .expect_err("explicit EC:2 must fail before any two-drive pool I/O");
2555 assert!(err.to_string().contains("pool 1") && err.to_string().contains("2 drives"));
2556 }
2557
2558 #[test]
2559 #[serial_test::serial(storage_class_env)]
2560 fn startup_pool_defaults_validate_environment_without_changing_metadata_fallback() {
2561 temp_env::with_vars(
2562 [
2563 (crate::config::storageclass::STANDARD_ENV, Some("EC:1")),
2564 (crate::config::storageclass::RRS_ENV, None),
2565 (crate::config::storageclass::OPTIMIZE_ENV, None),
2566 (crate::config::storageclass::INLINE_BLOCK_ENV, None),
2567 ],
2568 || {
2569 let runtime = crate::config::storageclass::lookup_config_for_pools(&KVS::new(), &[6, 4])
2570 .expect("explicit standard parity must resolve the runtime candidate");
2571 assert_eq!(runtime.parities_for_sc(crate::config::storageclass::STANDARD), Some(vec![1, 1]));
2572
2573 let defaults = super::resolve_startup_pool_defaults(&endpoint_pools_with_drive_counts(&[6, 4]))
2574 .expect("explicit standard parity must validate for every pool");
2575 assert_eq!(defaults, vec![3, 2]);
2576 },
2577 );
2578 }
2579
2580 async fn without_storage_class_env<F: Future>(future: F) -> F::Output {
2581 temp_env::async_with_vars(
2582 [
2583 (crate::config::storageclass::STANDARD_ENV, None::<&str>),
2584 (crate::config::storageclass::RRS_ENV, None::<&str>),
2585 (crate::config::storageclass::OPTIMIZE_ENV, None::<&str>),
2586 (crate::config::storageclass::INLINE_BLOCK_ENV, None::<&str>),
2587 ],
2588 future,
2589 )
2590 .await
2591 }
2592
2593 async fn build_isolated_test_store(
2595 temp_dir: &std::path::Path,
2596 cmd_line: &str,
2597 pool_drive_counts: &[usize],
2598 ) -> (
2599 Arc<crate::runtime::instance::InstanceContext>,
2600 Arc<crate::store::ECStore>,
2601 CancellationToken,
2602 ) {
2603 build_isolated_test_store_with_shutdown(temp_dir, cmd_line, pool_drive_counts, CancellationToken::new()).await
2604 }
2605
2606 async fn build_isolated_test_store_with_shutdown(
2607 temp_dir: &std::path::Path,
2608 cmd_line: &str,
2609 pool_drive_counts: &[usize],
2610 shutdown: CancellationToken,
2611 ) -> (
2612 Arc<crate::runtime::instance::InstanceContext>,
2613 Arc<crate::store::ECStore>,
2614 CancellationToken,
2615 ) {
2616 let pool_layouts = pool_drive_counts
2617 .iter()
2618 .map(|&drives_per_set| (1, drives_per_set))
2619 .collect::<Vec<_>>();
2620 build_isolated_test_store_with_layout(temp_dir, cmd_line, &pool_layouts, shutdown, None).await
2621 }
2622
2623 async fn build_isolated_test_store_with_layout(
2624 temp_dir: &std::path::Path,
2625 cmd_line: &str,
2626 pool_layouts: &[(usize, usize)],
2627 shutdown: CancellationToken,
2628 instance_ctx: Option<Arc<crate::runtime::instance::InstanceContext>>,
2629 ) -> (
2630 Arc<crate::runtime::instance::InstanceContext>,
2631 Arc<crate::store::ECStore>,
2632 CancellationToken,
2633 ) {
2634 let mut pools = Vec::with_capacity(pool_layouts.len());
2635 for (pool_index, &(set_count, drives_per_set)) in pool_layouts.iter().enumerate() {
2636 let mut endpoints = Vec::with_capacity(set_count * drives_per_set);
2637 for set_index in 0..set_count {
2638 for disk_index in 0..drives_per_set {
2639 let path = temp_dir.join(format!("pool{pool_index}/set{set_index}/disk{disk_index}"));
2640 tokio::fs::create_dir_all(&path).await.expect("create disk dir");
2641 let mut endpoint =
2642 Endpoint::try_from(path.to_str().expect("disk path should be utf-8")).expect("local endpoint");
2643 endpoint.set_pool_index(pool_index);
2644 endpoint.set_set_index(set_index);
2645 endpoint.set_disk_index(disk_index);
2646 endpoints.push(endpoint);
2647 }
2648 }
2649 pools.push(PoolEndpoints {
2650 legacy: false,
2651 set_count,
2652 drives_per_set,
2653 endpoints: Endpoints::from(endpoints),
2654 cmd_line: format!("{cmd_line}-pool-{pool_index}"),
2655 platform: "test".to_string(),
2656 });
2657 }
2658 let endpoint_pools = EndpointServerPools(pools);
2659 crate::services::notification_sys::install_cross_pool_fence_fleet_proof_for_test();
2660
2661 let instance_ctx = instance_ctx.unwrap_or_else(|| Arc::new(crate::runtime::instance::InstanceContext::new()));
2662 crate::store::init_local_disks_with_instance_ctx(&instance_ctx, endpoint_pools.clone())
2663 .await
2664 .expect("register local disks into the fresh context");
2665
2666 let store = crate::store::ECStore::new_with_instance_ctx(
2667 "127.0.0.1:0".parse().expect("test address"),
2668 endpoint_pools,
2669 shutdown.clone(),
2670 instance_ctx.clone(),
2671 )
2672 .await
2673 .expect("store should build around the fresh context");
2674
2675 let layout = DecommissionErasureLayout { data: 2, parity: 2 };
2679 let snapshot: Vec<DecommissionPoolCapacityInfo> = store
2680 .pools
2681 .iter()
2682 .enumerate()
2683 .map(|(pool_index, _)| DecommissionPoolCapacityInfo::for_test(pool_index, layout, 1 << 40, 1 << 40, 1 << 30))
2684 .collect();
2685 set_decommission_capacity_info_overrides_for_test(store.id, (0..128).map(|_| snapshot.clone()).collect());
2686
2687 (instance_ctx, store, shutdown)
2688 }
2689
2690 #[cfg(feature = "test-util")]
2691 #[tokio::test]
2692 #[serial_test::serial(storage_class_env)]
2693 async fn cross_key_materialized_copy_does_not_publish_remote_tuple_ownership() {
2694 let temp_dir = tempfile::tempdir().expect("create materialized-copy store dir");
2695 let (ctx, store, shutdown) =
2696 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "copy-materialized", &[4])).await;
2697 crate::bucket::metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
2698
2699 let bucket = format!("copy-materialized-{}", Uuid::new_v4());
2700 let source_object = "source-restored.bin";
2701 let target_object = "target-local.bin";
2702 let payload = b"materialized copy must own only its new local bytes".to_vec();
2703 store
2704 .make_bucket(&bucket, &MakeBucketOptions::default())
2705 .await
2706 .expect("copy test bucket should be created");
2707
2708 let tier_name = "COPY-MATERIALIZED-TIER";
2709 register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
2710 let lease = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
2711 .await
2712 .expect("mock tier generation should be available");
2713 let backend_identity = lease.backend_identity();
2714 drop(lease);
2715
2716 let mut source_metadata = HashMap::from([
2717 ("content-type".to_string(), "application/octet-stream".to_string()),
2718 (
2719 "x-amz-restore".to_string(),
2720 "ongoing-request=\"false\", expiry-date=\"2099-01-01T00:00:00Z\"".to_string(),
2721 ),
2722 ]);
2723 for (suffix, value) in [
2724 (
2725 rustfs_utils::http::SUFFIX_TRANSITION_STATUS,
2726 crate::bucket::lifecycle::core::TRANSITION_COMPLETE.to_string(),
2727 ),
2728 (
2729 rustfs_utils::http::SUFFIX_TRANSITIONED_OBJECTNAME,
2730 "remote/source-restored.bin".to_string(),
2731 ),
2732 (rustfs_utils::http::SUFFIX_TRANSITION_TIER, tier_name.to_string()),
2733 (rustfs_utils::http::SUFFIX_TRANSITIONED_VERSION_ID, "remote-source-version".to_string()),
2734 (
2735 rustfs_utils::http::SUFFIX_TRANSITIONED_VERSION_STATE,
2736 rustfs_filemeta::TransitionVersionState::Exact.as_str().to_string(),
2737 ),
2738 (
2739 rustfs_utils::http::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
2740 rustfs_utils::crypto::hex(backend_identity),
2741 ),
2742 (rustfs_utils::http::SUFFIX_TRANSITION_TRANSACTION_ID, Uuid::new_v4().to_string()),
2743 ] {
2744 rustfs_utils::http::metadata_compat::insert_str(&mut source_metadata, suffix, value);
2745 }
2746
2747 let mut source_body = PutObjReader::from_vec(payload.clone());
2748 store
2749 .put_object(
2750 &bucket,
2751 source_object,
2752 &mut source_body,
2753 &ObjectOptions {
2754 user_defined: source_metadata,
2755 ..Default::default()
2756 },
2757 )
2758 .await
2759 .expect("restored transitioned source should be written");
2760
2761 let source_opts = ObjectOptions::default();
2762 let mut source_info = store
2763 .get_object_info(&bucket, source_object, &source_opts)
2764 .await
2765 .expect("source metadata should be readable");
2766 assert_eq!(
2767 source_info.transitioned_object.status,
2768 crate::bucket::lifecycle::core::TRANSITION_COMPLETE
2769 );
2770 assert_eq!(source_info.transitioned_object.tier, tier_name);
2771 assert!(source_info.data_dir.is_some(), "restored source must retain local data");
2772 source_info.put_object_reader = Some(PutObjReader::from_vec(payload.clone()));
2773
2774 let copied = store
2775 .copy_object(
2776 &bucket,
2777 source_object,
2778 &bucket,
2779 target_object,
2780 &mut source_info,
2781 &source_opts,
2782 &ObjectOptions::default(),
2783 )
2784 .await
2785 .expect("cross-key materialized copy should succeed");
2786 assert!(copied.transitioned_object.status.is_empty());
2787 assert!(copied.transitioned_object.name.is_empty());
2788 assert!(copied.transitioned_object.version_id.is_empty());
2789 assert!(copied.transitioned_object.tier.is_empty());
2790 assert!(!copied.transitioned_object.free_version);
2791 for suffix in [
2792 rustfs_utils::http::SUFFIX_TRANSITION_STATUS,
2793 rustfs_utils::http::SUFFIX_TRANSITIONED_OBJECTNAME,
2794 rustfs_utils::http::SUFFIX_TRANSITIONED_VERSION_ID,
2795 rustfs_utils::http::SUFFIX_TRANSITIONED_VERSION_STATE,
2796 rustfs_utils::http::SUFFIX_TRANSITION_TIER,
2797 rustfs_utils::http::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
2798 rustfs_utils::http::SUFFIX_TRANSITION_TRANSACTION_ID,
2799 ] {
2800 assert!(
2801 !rustfs_utils::http::metadata_compat::contains_key_str(copied.user_defined.as_ref(), suffix),
2802 "target retained protected source suffix {suffix}"
2803 );
2804 }
2805 assert_eq!(
2806 copied.user_defined.get("content-type").map(String::as_str),
2807 Some("application/octet-stream")
2808 );
2809
2810 let mut target_reader = store
2811 .get_object_reader(&bucket, target_object, None, HeaderMap::new(), &ObjectOptions::default())
2812 .await
2813 .expect("target object should be readable");
2814 let mut target_body = Vec::new();
2815 target_reader
2816 .stream
2817 .read_to_end(&mut target_body)
2818 .await
2819 .expect("target body should stream");
2820 assert_eq!(target_body, payload);
2821
2822 let source_after = store
2823 .get_object_info(&bucket, source_object, &source_opts)
2824 .await
2825 .expect("source metadata should remain readable after copy");
2826 assert_eq!(
2827 source_after.transitioned_object.status,
2828 crate::bucket::lifecycle::core::TRANSITION_COMPLETE
2829 );
2830 assert_eq!(source_after.transitioned_object.tier, tier_name);
2831 assert_eq!(source_after.transitioned_object.name, "remote/source-restored.bin");
2832 shutdown.cancel();
2833 }
2834
2835 #[cfg(feature = "test-util")]
2836 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2837 #[serial_test::serial(storage_class_env)]
2838 async fn tier_overwrite_put_and_self_copy_recover_persisted_cleanup_owners() {
2839 use crate::bucket::lifecycle::bucket_lifecycle_ops::ExpiryState;
2840 use crate::bucket::lifecycle::tier_free_version_recovery::recover_tier_free_versions;
2841 use rustfs_filemeta::TransitionVersionState::{Exact, KnownDisabled, SuspendedNull};
2842 use rustfs_s3_client::transition_api::ReaderImpl;
2843 use rustfs_utils::http::{
2844 SUFFIX_TRANSITION_STATUS, SUFFIX_TRANSITION_TIER, SUFFIX_TRANSITION_TIER_DESTINATION_ID,
2845 SUFFIX_TRANSITIONED_OBJECTNAME, SUFFIX_TRANSITIONED_VERSION_ID, SUFFIX_TRANSITIONED_VERSION_STATE, insert_str,
2846 };
2847
2848 let temp_dir = tempfile::tempdir().expect("create tier overwrite store");
2849 let (mut ctx, mut store, mut shutdown) =
2850 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "tier-overwrite", &[4])).await;
2851 crate::bucket::metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
2852 let tier = "OVERWRITE-TIER";
2853 let backend = register_mock_tier(&ctx.tier_config_mgr(), tier).await;
2854 let lease = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier)
2855 .await
2856 .expect("tier identity");
2857 let identity = rustfs_utils::crypto::hex(lease.backend_identity());
2858 drop(lease);
2859
2860 for state in [Exact, KnownDisabled, SuspendedNull] {
2861 for suspended in [false, true] {
2862 for self_copy in [false, true] {
2863 let bucket = format!("tier-overwrite-{}", Uuid::new_v4());
2864 let object = "object";
2865 let remote = format!("remote/{bucket}");
2866 let version = match state {
2867 Exact => "opaque-overwrite-version",
2868 SuspendedNull => "null",
2869 _ => "",
2870 };
2871 let payload = vec![0x5b; if suspended { 512 * 1024 } else { 257 }];
2872 store
2873 .make_bucket(&bucket, &MakeBucketOptions::default())
2874 .await
2875 .expect("create bucket");
2876 backend.set_put_remote_version(Some(version.to_string())).await;
2877 let lease = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier)
2878 .await
2879 .expect("seed tier lease");
2880 lease
2881 .put(
2882 &remote,
2883 ReaderImpl::Body(bytes::Bytes::from(payload.clone())),
2884 payload.len().try_into().expect("payload size"),
2885 )
2886 .await
2887 .expect("seed remote bytes");
2888 drop(lease);
2889 let mut metadata = HashMap::from([
2890 ("content-type".to_string(), "application/octet-stream".to_string()),
2891 (
2892 "x-amz-restore".to_string(),
2893 "ongoing-request=\"false\", expiry-date=\"2099-01-01T00:00:00Z\"".to_string(),
2894 ),
2895 ]);
2896 for (suffix, value) in [
2897 (SUFFIX_TRANSITION_STATUS, "complete"),
2898 (SUFFIX_TRANSITION_TIER, tier),
2899 (SUFFIX_TRANSITION_TIER_DESTINATION_ID, identity.as_str()),
2900 (SUFFIX_TRANSITIONED_OBJECTNAME, remote.as_str()),
2901 (SUFFIX_TRANSITIONED_VERSION_STATE, state.as_str()),
2902 ] {
2903 insert_str(&mut metadata, suffix, value.to_string());
2904 }
2905 if !version.is_empty() {
2906 insert_str(&mut metadata, SUFFIX_TRANSITIONED_VERSION_ID, version.to_string());
2907 }
2908 let options = ObjectOptions {
2917 version_suspended: suspended,
2918 write_completion: crate::object_api::WriteCompletion::TailDrained,
2919 ..Default::default()
2920 };
2921 store
2922 .put_object(
2923 &bucket,
2924 object,
2925 &mut PutObjReader::from_vec(payload.clone()),
2926 &ObjectOptions {
2927 user_defined: metadata,
2928 ..options.clone()
2929 },
2930 )
2931 .await
2932 .expect("seed transitioned source with locally restored bytes");
2933 backend.set_remove_failure(true);
2939 let expected = if self_copy {
2940 payload.clone()
2941 } else {
2942 vec![0x73; payload.len()]
2943 };
2944 let new_metadata = HashMap::from([
2945 ("content-type".to_string(), "text/plain".to_string()),
2946 ("x-amz-meta-replacement".to_string(), "kept".to_string()),
2947 ]);
2948 if self_copy {
2949 let mut source = store
2950 .get_object_info(&bucket, object, &options)
2951 .await
2952 .expect("self-copy source");
2953 source.metadata_only = false;
2954 source.user_defined = Arc::new(new_metadata);
2955 source.put_object_reader = Some(PutObjReader::from_vec(expected.clone()));
2956 store
2957 .copy_object(&bucket, object, &bucket, object, &mut source, &options, &options)
2958 .await
2959 .expect("materialized self-copy");
2960 } else {
2961 store
2962 .put_object(
2963 &bucket,
2964 object,
2965 &mut PutObjReader::from_vec(expected.clone()),
2966 &ObjectOptions {
2967 user_defined: new_metadata,
2968 ..options.clone()
2969 },
2970 )
2971 .await
2972 .expect("overwrite transitioned null version");
2973 }
2974
2975 let set = store.pools[0].get_disks_by_key(object);
2976 let versions = set
2977 .load_file_info_versions_exact(&bucket, object)
2978 .await
2979 .expect("read committed disk metadata")
2980 .expect("replacement metadata exists");
2981 let free: Vec<_> = versions
2982 .versions
2983 .iter()
2984 .chain(versions.free_versions.iter())
2985 .filter(|fi| fi.tier_free_version())
2986 .collect();
2987 assert_eq!(free.len(), 1, "{state:?}, suspended={suspended}, copy={self_copy}");
2988 assert_eq!(free[0].transitioned_objname, remote);
2989 assert_eq!(free[0].transition_version_state, state);
2990 assert_every_disk_holds_only_the_cleanup_owner(&set, &bucket, object, &remote).await;
2991 assert!(backend.contains(&remote).await, "commit must not delete remote bytes before cleanup");
2992 let removed_before = backend.remove_count().await;
2993
2994 wait_for_expiry_workers_idle(&store).await;
2999
3000 let tier_config = ctx
3004 .tier_config_mgr()
3005 .read()
3006 .await
3007 .tiers
3008 .get(tier)
3009 .expect("tier configuration survives restart")
3010 .clone_with_credentials();
3011 drop(set);
3012 shutdown.cancel();
3013 drop(store);
3014 drop(ctx);
3015 (ctx, store, shutdown) =
3016 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "tier-overwrite-restart", &[4]))
3017 .await;
3018 crate::bucket::metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
3019 {
3020 let manager = ctx.tier_config_mgr();
3021 let mut manager = manager.write().await;
3022 manager.tiers.insert(tier.to_string(), tier_config);
3023 manager
3024 .install_test_driver(tier, Box::new(backend.clone()))
3025 .expect("rebind the same remote destination after restart");
3026 }
3027 let set = store.pools[0].get_disks_by_key(object);
3028 backend.set_remove_failure(true);
3031 ExpiryState::resize_workers(1, Arc::clone(&store)).await;
3032 let recovered = recover_tier_free_versions(Arc::clone(&store), 100, None, None)
3033 .await
3034 .expect("recover persisted cleanup owner");
3035 assert!(recovered.enqueued >= 1);
3036 wait_for_expiry_workers_idle(&store).await;
3037 assert!(backend.contains(&remote).await, "failed cleanup must retain remote bytes");
3038 assert_eq!(backend.remove_count().await, removed_before);
3039 backend.set_remove_failure(false);
3040 eprintln!("Recovering overwrite cleanup: state={state:?}, suspended={suspended}, copy={self_copy}");
3043 wait_for_tier_free_version_recovery(Arc::clone(&store), &backend, removed_before + 1).await;
3044 let versions = set
3045 .load_file_info_versions_exact(&bucket, object)
3046 .await
3047 .expect("read cleanup progress")
3048 .expect("new object must survive cleanup");
3049 assert!(
3050 versions
3051 .versions
3052 .iter()
3053 .chain(versions.free_versions.iter())
3054 .all(|fi| !fi.tier_free_version()),
3055 "cleanup must remove its owner: {state:?}, suspended={suspended}, copy={self_copy}"
3056 );
3057 assert!(!backend.contains(&remote).await);
3058 assert_eq!(backend.remove_count().await, removed_before + 1, "one successful remote DELETE per owner");
3059 assert_eq!(backend.remove_versions().await.last(), Some(&(remote.clone(), version.to_string())));
3060 let mut reader = store
3061 .get_object_reader(&bucket, object, None, HeaderMap::new(), &options)
3062 .await
3063 .expect("replacement remains readable");
3064 let mut actual = Vec::new();
3065 reader.stream.read_to_end(&mut actual).await.expect("read replacement bytes");
3066 assert_eq!(actual, expected);
3067 let current = store
3068 .get_object_info(&bucket, object, &options)
3069 .await
3070 .expect("replacement metadata");
3071 assert_eq!(current.user_defined.get("content-type").map(String::as_str), Some("text/plain"));
3072 assert_eq!(current.user_defined.get("x-amz-meta-replacement").map(String::as_str), Some("kept"));
3073 assert!(current.transitioned_object.status.is_empty());
3074 }
3075 }
3076 }
3077 shutdown.cancel();
3078 }
3079
3080 #[cfg(feature = "test-util")]
3081 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
3082 #[serial_test::serial(storage_class_env)]
3083 async fn copy_object_immediately_reads_small_completed_multipart_source() {
3084 let temp_dir = tempfile::tempdir().expect("create small multipart copy store dir");
3085 let (_ctx, store, shutdown) =
3086 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "small-multipart-copy", &[1])).await;
3087 crate::bucket::metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
3088
3089 let bucket = format!("small-multipart-copy-{}", Uuid::new_v4());
3090 let source_object = "docker/registry/v2/repositories/example/_uploads/upload-id/data";
3091 let target_object = "docker/registry/v2/blobs/sha256/c0/digest/data";
3092 let payload = vec![0xAB; 273];
3093
3094 store
3095 .make_bucket(&bucket, &MakeBucketOptions::default())
3096 .await
3097 .expect("create bucket for small multipart copy");
3098 let upload = store
3099 .new_multipart_upload(&bucket, source_object, &ObjectOptions::default())
3100 .await
3101 .expect("create source multipart upload");
3102 let mut part_reader = PutObjReader::from_vec(payload.clone());
3103 let part = store
3104 .put_object_part(&bucket, source_object, &upload.upload_id, 1, &mut part_reader, &ObjectOptions::default())
3105 .await
3106 .expect("stage small multipart source part");
3107 let completed = store
3108 .clone()
3109 .complete_multipart_upload(
3110 &bucket,
3111 source_object,
3112 &upload.upload_id,
3113 vec![crate::storage_api_contracts::multipart::CompletePart {
3114 part_num: part.part_num,
3115 etag: part.etag,
3116 ..Default::default()
3117 }],
3118 &ObjectOptions::default(),
3119 )
3120 .await
3121 .expect("complete the small multipart source");
3122 assert_eq!(completed.get_actual_size().expect("completed object logical size"), payload.len() as i64);
3123
3124 let source_reader = store
3125 .get_object_reader(&bucket, source_object, None, HeaderMap::new(), &ObjectOptions::default())
3126 .await
3127 .expect("completed multipart source should be immediately readable");
3128 let mut copy_info = source_reader.object_info.clone();
3129 let actual_size = copy_info.get_actual_size().expect("copy source logical size should resolve");
3130 assert_eq!(actual_size, payload.len() as i64);
3131 let copy_reader = rustfs_rio::HashReader::from_stream(source_reader.stream, actual_size, actual_size, None, None, false)
3132 .expect("copy source hash reader should build");
3133 copy_info.put_object_reader = Some(PutObjReader::new(copy_reader));
3134
3135 store
3136 .copy_object(
3137 &bucket,
3138 source_object,
3139 &bucket,
3140 target_object,
3141 &mut copy_info,
3142 &ObjectOptions::default(),
3143 &ObjectOptions::default(),
3144 )
3145 .await
3146 .expect("CopyObject should accept a freshly completed multipart source");
3147
3148 let mut target_reader = store
3149 .get_object_reader(&bucket, target_object, None, HeaderMap::new(), &ObjectOptions::default())
3150 .await
3151 .expect("copied target should be readable");
3152 let mut target_body = Vec::new();
3153 target_reader
3154 .stream
3155 .read_to_end(&mut target_body)
3156 .await
3157 .expect("target body should stream");
3158 assert_eq!(target_body, payload);
3159 shutdown.cancel();
3160 }
3161
3162 #[cfg(feature = "test-util")]
3172 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
3173 #[serial_test::serial(storage_class_env)]
3174 async fn copy_object_forwards_quota_admission_on_legacy_snapshot_quota_bucket() {
3175 let temp_dir = tempfile::tempdir().expect("create legacy quota copy store dir");
3176 let (ctx, store, shutdown) =
3177 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "legacy-quota-copy", &[1])).await;
3178 crate::bucket::metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
3179
3180 let bucket = format!("legacy-quota-copy-{}", Uuid::new_v4());
3181 let source_object = "docker/registry/v2/repositories/example/_uploads/upload-id/data";
3182 let target_object = "docker/registry/v2/blobs/sha256/a5/digest/data";
3183 let payload = vec![0x5A; 8178];
3184 let quota_limit = 1u64 << 30;
3185
3186 store
3187 .make_bucket(&bucket, &MakeBucketOptions::default())
3188 .await
3189 .expect("create bucket for legacy quota copy");
3190 let legacy_quota = format!(r#"{{"quota":{quota_limit},"quota_type":"Hard"}}"#);
3193 crate::bucket::metadata_sys::update_in(
3194 &ctx,
3195 &bucket,
3196 crate::bucket::metadata::BUCKET_QUOTA_CONFIG_FILE,
3197 legacy_quota.into_bytes(),
3198 )
3199 .await
3200 .expect("persist legacy snapshot quota");
3201 let (quota, _, _) = crate::bucket::metadata_sys::get_quota_config_and_incarnation_from_disk_in(&ctx, &bucket)
3202 .await
3203 .expect("legacy quota should load");
3204 let quota = quota.expect("legacy quota must be persisted");
3205 assert_eq!(quota.quota, Some(quota_limit));
3206 assert!(!quota.uses_durable_reservations(), "fixture must stay on the snapshot protocol");
3207
3208 let mut write_opts = ObjectOptions::default();
3209 assert!(write_opts.set_quota_admission(0, quota_limit));
3210
3211 let upload = store
3212 .new_multipart_upload(&bucket, source_object, &write_opts)
3213 .await
3214 .expect("create source multipart upload");
3215 let mut part_reader = PutObjReader::from_vec(payload.clone());
3216 let part = store
3217 .put_object_part(&bucket, source_object, &upload.upload_id, 1, &mut part_reader, &write_opts)
3218 .await
3219 .expect("stage multipart source part");
3220 store
3221 .clone()
3222 .complete_multipart_upload(
3223 &bucket,
3224 source_object,
3225 &upload.upload_id,
3226 vec![crate::storage_api_contracts::multipart::CompletePart {
3227 part_num: part.part_num,
3228 etag: part.etag,
3229 ..Default::default()
3230 }],
3231 &write_opts,
3232 )
3233 .await
3234 .expect("complete the multipart source under the legacy quota");
3235
3236 let source_reader = store
3237 .get_object_reader(&bucket, source_object, None, HeaderMap::new(), &ObjectOptions::default())
3238 .await
3239 .expect("completed multipart source should be readable");
3240 let mut copy_info = source_reader.object_info.clone();
3241 let actual_size = copy_info.get_actual_size().expect("copy source logical size should resolve");
3242 assert_eq!(actual_size, payload.len() as i64);
3243 let copy_reader = rustfs_rio::HashReader::from_stream(source_reader.stream, actual_size, actual_size, None, None, false)
3244 .expect("copy source hash reader should build");
3245 copy_info.put_object_reader = Some(PutObjReader::new(copy_reader));
3246
3247 let mut dst_opts = ObjectOptions::default();
3248 assert!(dst_opts.set_quota_admission(payload.len() as u64, quota_limit));
3249 store
3250 .copy_object(
3251 &bucket,
3252 source_object,
3253 &bucket,
3254 target_object,
3255 &mut copy_info,
3256 &ObjectOptions::default(),
3257 &dst_opts,
3258 )
3259 .await
3260 .expect("CopyObject must forward the handler quota admission to the destination write");
3261
3262 let mut target_reader = store
3263 .get_object_reader(&bucket, target_object, None, HeaderMap::new(), &ObjectOptions::default())
3264 .await
3265 .expect("copied target should be readable");
3266 let mut target_body = Vec::new();
3267 target_reader
3268 .stream
3269 .read_to_end(&mut target_body)
3270 .await
3271 .expect("target body should stream");
3272 assert_eq!(target_body, payload);
3273 shutdown.cancel();
3274 }
3275
3276 #[cfg(feature = "test-util")]
3277 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
3278 #[serial_test::serial(storage_class_env)]
3279 async fn unfinished_multipart_upload_is_not_copy_source_readable() {
3280 let temp_dir = tempfile::tempdir().expect("create unfinished multipart copy store dir");
3281 let (_ctx, store, shutdown) =
3282 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "unfinished-multipart-copy", &[1])).await;
3283 crate::bucket::metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
3284
3285 let bucket = format!("unfinished-multipart-copy-{}", Uuid::new_v4());
3286 let source_object = "docker/registry/v2/repositories/example/_uploads/upload-id/data";
3287 let payload = vec![0xCD; 273];
3288
3289 store
3290 .make_bucket(&bucket, &MakeBucketOptions::default())
3291 .await
3292 .expect("create bucket for unfinished multipart copy source");
3293 let upload = store
3294 .new_multipart_upload(&bucket, source_object, &ObjectOptions::default())
3295 .await
3296 .expect("create source multipart upload");
3297 let mut part_reader = PutObjReader::from_vec(payload);
3298 store
3299 .put_object_part(&bucket, source_object, &upload.upload_id, 1, &mut part_reader, &ObjectOptions::default())
3300 .await
3301 .expect("stage unfinished multipart source part");
3302
3303 let source_err = match store
3304 .get_object_reader(&bucket, source_object, None, HeaderMap::new(), &ObjectOptions::default())
3305 .await
3306 {
3307 Ok(_) => panic!("an uncompleted multipart upload must not be readable as a CopyObject source"),
3308 Err(err) => err,
3309 };
3310 assert!(
3311 matches!(
3312 source_err,
3313 StorageError::ObjectNotFound(_, _) | StorageError::FileNotFound | StorageError::VersionNotFound(_, _, _)
3314 ),
3315 "unexpected unfinished multipart source error: {source_err:?}"
3316 );
3317
3318 shutdown.cancel();
3319 }
3320
3321 #[tokio::test]
3322 #[serial_test::serial(storage_class_env)]
3323 async fn pool_metadata_preflight_recovery_preserves_single_and_multi_pool_public_mutations() {
3324 for layout in [vec![4], vec![4, 4]] {
3325 let temp_dir = tempfile::tempdir().unwrap();
3326 let (_ctx, store, shutdown) =
3327 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "pool-meta-retry", &layout)).await;
3328 crate::bucket::metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
3329 let bucket = format!("pool-meta-retry-{}", Uuid::new_v4());
3330 store.make_bucket(&bucket, &MakeBucketOptions::default()).await.unwrap();
3331 let mut saved_disks = Vec::new();
3332 for set in &store.pools[0].disk_set {
3333 let mut disks = set.disks.write().await;
3334 let count = disks.len();
3335 saved_disks.push((set.clone(), std::mem::replace(&mut *disks, vec![None; count])));
3336 }
3337 let indices = (0..layout.len()).collect::<Vec<_>>();
3338 let err = store.save_current_pool_meta_for_test(&indices).await.unwrap_err();
3339 assert_eq!(
3340 err.pool_metadata_failure().unwrap().kind,
3341 crate::error::PoolMetadataFailure::ReadUnavailable
3342 );
3343 for (set, disks) in saved_disks {
3344 *set.disks.write().await = disks;
3345 }
3346 store.save_current_pool_meta_for_test(&indices).await.unwrap();
3347 assert!(store.pool_meta_writes_ready().await);
3348
3349 let payload = b"pool metadata recovery payload".to_vec();
3350 store
3351 .put_object(&bucket, "put", &mut PutObjReader::from_vec(payload.clone()), &ObjectOptions::default())
3352 .await
3353 .unwrap();
3354 let mut reader = store
3355 .get_object_reader(&bucket, "put", None, HeaderMap::new(), &ObjectOptions::default())
3356 .await
3357 .unwrap();
3358 let mut actual = Vec::new();
3359 reader.stream.read_to_end(&mut actual).await.unwrap();
3360 assert_eq!(actual, payload);
3361 drop(reader);
3362 store.delete_object(&bucket, "put", ObjectOptions::default()).await.unwrap();
3363 assert!(crate::error::is_err_object_not_found(
3364 &store
3365 .get_object_info(&bucket, "put", &ObjectOptions::default())
3366 .await
3367 .unwrap_err()
3368 ));
3369
3370 let upload = store
3371 .new_multipart_upload(&bucket, "multipart", &ObjectOptions::default())
3372 .await
3373 .unwrap();
3374 let part = store
3375 .put_object_part(
3376 &bucket,
3377 "multipart",
3378 &upload.upload_id,
3379 1,
3380 &mut PutObjReader::from_vec(payload.clone()),
3381 &ObjectOptions::default(),
3382 )
3383 .await
3384 .unwrap();
3385 store
3386 .clone()
3387 .complete_multipart_upload(
3388 &bucket,
3389 "multipart",
3390 &upload.upload_id,
3391 vec![crate::storage_api_contracts::multipart::CompletePart {
3392 part_num: part.part_num,
3393 etag: part.etag,
3394 ..Default::default()
3395 }],
3396 &ObjectOptions::default(),
3397 )
3398 .await
3399 .unwrap();
3400 let mut reader = store
3401 .get_object_reader(&bucket, "multipart", None, HeaderMap::new(), &ObjectOptions::default())
3402 .await
3403 .unwrap();
3404 actual.clear();
3405 reader.stream.read_to_end(&mut actual).await.unwrap();
3406 assert_eq!(actual, payload);
3407 drop(reader);
3408 let upload = store
3409 .new_multipart_upload(&bucket, "abort", &ObjectOptions::default())
3410 .await
3411 .unwrap();
3412 store
3413 .abort_multipart_upload(&bucket, "abort", &upload.upload_id, &ObjectOptions::default())
3414 .await
3415 .unwrap();
3416 assert!(store.pool_meta_writes_ready().await);
3417 shutdown.cancel();
3418 }
3419 }
3420
3421 #[cfg(feature = "test-util")]
3422 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
3423 #[serial_test::serial(storage_class_env)]
3424 async fn complete_multipart_waits_for_tail_rename_before_copy_source_visibility() {
3425 let temp_dir = tempfile::tempdir().expect("create early-ack multipart copy store dir");
3426 let (_ctx, store, shutdown) =
3427 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "early-ack-multipart-copy", &[4])).await;
3428 crate::bucket::metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
3429
3430 let bucket = format!("early-ack-multipart-copy-{}", Uuid::new_v4());
3431 let source_object = "docker/registry/v2/repositories/example/_uploads/upload-id/data";
3432 let target_object = "docker/registry/v2/blobs/sha256/c0/digest/data";
3433 let payload = vec![0xCD; 273];
3434
3435 store
3436 .make_bucket(&bucket, &MakeBucketOptions::default())
3437 .await
3438 .expect("create bucket for early-ack multipart copy");
3439 let upload = store
3440 .new_multipart_upload(&bucket, source_object, &ObjectOptions::default())
3441 .await
3442 .expect("create source multipart upload");
3443 let mut part_reader = PutObjReader::from_vec(payload.clone());
3444 let part = store
3445 .put_object_part(&bucket, source_object, &upload.upload_id, 1, &mut part_reader, &ObjectOptions::default())
3446 .await
3447 .expect("stage small multipart source part");
3448 let completed_parts = vec![crate::storage_api_contracts::multipart::CompletePart {
3449 part_num: part.part_num,
3450 etag: part.etag,
3451 ..Default::default()
3452 }];
3453
3454 temp_env::async_with_vars([(crate::set_disk::ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async {
3455 let rename_tasks = crate::set_disk::rename_fanout_barrier::observe_tasks(source_object);
3456 let rename_barrier = crate::set_disk::rename_fanout_barrier::arm(
3457 source_object,
3458 0,
3459 crate::set_disk::rename_fanout_barrier::PHASE_RENAME,
3460 );
3461 let complete_store = Arc::clone(&store);
3462 let complete_bucket = bucket.clone();
3463 let complete_upload_id = upload.upload_id.clone();
3464 let mut complete = tokio::spawn(async move {
3465 complete_store
3466 .complete_multipart_upload(
3467 &complete_bucket,
3468 source_object,
3469 &complete_upload_id,
3470 completed_parts,
3471 &ObjectOptions::default(),
3472 )
3473 .await
3474 });
3475 tokio::time::timeout(Duration::from_secs(30), rename_barrier.wait_until_paused())
3476 .await
3477 .expect("multipart completion should pause one tail disk during rename");
3478 assert!(
3479 rename_tasks.running() >= 1,
3480 "the paused multipart tail disk must remain in flight before completion returns"
3481 );
3482 assert!(
3483 tokio::time::timeout(Duration::from_millis(100), &mut complete).await.is_err(),
3484 "CompleteMultipartUpload must not return while a copy source rename tail is still pending"
3485 );
3486 rename_barrier.release();
3487 complete
3488 .await
3489 .expect("multipart completion task should join")
3490 .expect("multipart completion should return after every rename tail finishes");
3491
3492 let source_reader = store
3493 .get_object_reader(&bucket, source_object, None, HeaderMap::new(), &ObjectOptions::default())
3494 .await
3495 .expect("completed multipart source should be immediately readable after success");
3496 let mut copy_info = source_reader.object_info.clone();
3497 let actual_size = copy_info.get_actual_size().expect("copy source logical size should resolve");
3498 assert_eq!(actual_size, payload.len() as i64);
3499 let copy_reader =
3500 rustfs_rio::HashReader::from_stream(source_reader.stream, actual_size, actual_size, None, None, false)
3501 .expect("copy source hash reader should build");
3502 copy_info.put_object_reader = Some(PutObjReader::new(copy_reader));
3503
3504 store
3505 .copy_object(
3506 &bucket,
3507 source_object,
3508 &bucket,
3509 target_object,
3510 &mut copy_info,
3511 &ObjectOptions::default(),
3512 &ObjectOptions::default(),
3513 )
3514 .await
3515 .expect("CopyObject should accept a freshly completed multipart source");
3516 })
3517 .await;
3518
3519 let mut target_reader = store
3520 .get_object_reader(&bucket, target_object, None, HeaderMap::new(), &ObjectOptions::default())
3521 .await
3522 .expect("copied target should be readable after tail release");
3523 let mut target_body = Vec::new();
3524 target_reader
3525 .stream
3526 .read_to_end(&mut target_body)
3527 .await
3528 .expect("target body should stream");
3529 assert_eq!(target_body, payload);
3530 shutdown.cancel();
3531 }
3532
3533 #[cfg(feature = "test-util")]
3534 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
3535 #[serial_test::serial(storage_class_env)]
3536 async fn early_ack_put_tails_block_scanner_publication_until_all_renames_finish() {
3537 use crate::storage_api_contracts::namespace::NamespaceLocking as _;
3538
3539 let temp_dir = tempfile::tempdir().expect("create scanner PUT tail store dir");
3540 let (ctx, store, shutdown) =
3541 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "scanner-put-tails", &[4])).await;
3542 crate::bucket::metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
3543 let bucket = format!("scanner-put-tails-{}", Uuid::new_v4());
3544 store
3545 .make_bucket(&bucket, &MakeBucketOptions::default())
3546 .await
3547 .expect("create scanner PUT tail bucket");
3548 let set = &store.pools[0].disk_set[0];
3549 let objects = [("scanner-tail-a", vec![0xA1; 273]), ("scanner-tail-b", vec![0xB2; 379])];
3550
3551 temp_env::async_with_vars([(crate::set_disk::ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async {
3552 let (active, blocked, movement_generation) = store.scanner_data_movement_activity().await;
3553 assert!(!active && !blocked);
3554 assert!(ctx.scanner_publication_state_allowed(), "the set admission cache should start allowed");
3555 let (old_lease, _) = store
3556 .acquire_scanner_publication_lease(movement_generation, crate::runtime::instance::SCANNER_PUBLICATION_LEASE_TTL)
3557 .await
3558 .expect("publication lease should be admitted before either PUT starts");
3559
3560 let barriers: Vec<_> = objects
3561 .iter()
3562 .map(|(object, _)| {
3563 crate::set_disk::rename_fanout_barrier::arm(object, 0, crate::set_disk::rename_fanout_barrier::PHASE_RENAME)
3564 })
3565 .collect();
3566 let trackers: Vec<_> = objects
3567 .iter()
3568 .map(|(object, _)| crate::set_disk::rename_fanout_barrier::observe_tasks(object))
3569 .collect();
3570 let puts: Vec<_> = objects
3571 .iter()
3572 .map(|(object, body)| {
3573 let put_store = Arc::clone(&store);
3574 let put_bucket = bucket.clone();
3575 let object = *object;
3576 let body = body.clone();
3577 tokio::spawn(async move {
3578 let mut reader = PutObjReader::from_vec(body);
3579 put_store
3580 .put_object(&put_bucket, object, &mut reader, &ObjectOptions::default())
3581 .await
3582 })
3583 })
3584 .collect();
3585 let committed = tokio::time::timeout(Duration::from_secs(30), async {
3586 for barrier in &barriers {
3587 barrier.wait_until_paused().await;
3588 }
3589 let mut committed = Vec::with_capacity(puts.len());
3590 for put in puts {
3591 committed.push(
3592 put.await
3593 .expect("early-ACK PUT task should join while its tail is paused")
3594 .expect("root PUT should return after quorum without waiting for its tail"),
3595 );
3596 }
3597 committed
3598 })
3599 .await
3600 .expect("both root PUTs must quorum-ACK while their tail disks remain paused");
3601
3602 assert!(trackers.iter().all(|tracker| tracker.running() >= 1));
3603 assert!(ctx.namespace_commits_pending());
3604 assert!(
3605 ctx.scanner_publication_state_allowed(),
3606 "pending PUT tails must not disable scanner namespace walks"
3607 );
3608 let (active, blocked, observed_movement_generation) = store.scanner_data_movement_activity().await;
3609 assert!(!active, "ordinary PUT tails are not decommission or rebalance work");
3610 assert!(!blocked, "ordinary PUT tails must not block the movement-only scan baseline");
3611 assert_eq!(observed_movement_generation, movement_generation);
3612 assert!(store.scanner_data_usage_publication_blocked().await);
3613 assert!(store.scanner_data_usage_publication_admission_guard().await.is_some());
3614 assert!(set.scanner_data_usage_publication_admission_guard().await.is_some());
3615 for error in [
3616 store
3617 .acquire_scanner_publication_lease(
3618 movement_generation,
3619 crate::runtime::instance::SCANNER_PUBLICATION_LEASE_TTL,
3620 )
3621 .await
3622 .expect_err("a new remote publication lease must reject pending PUT tails"),
3623 store
3624 .validate_scanner_publication_lease(old_lease, movement_generation)
3625 .await
3626 .expect_err("an existing remote lease must not bypass pending PUT tails"),
3627 store
3628 .acquire_scanner_publication_lease_guard(old_lease)
3629 .await
3630 .expect_err("target-side publication admission must reject pending PUT tails"),
3631 ] {
3632 assert!(
3633 error.to_string().contains("blocked"),
3634 "publication must fail because of active tails: {error}"
3635 );
3636 }
3637 store.release_scanner_publication_lease(old_lease).await;
3638
3639 for (index, barrier) in barriers.iter().enumerate() {
3640 let commit_generation = ctx.namespace_commit_generation();
3641 let namespace_generation = store.scanner_namespace_mutation_generation();
3642 barrier.release();
3643 tokio::time::timeout(Duration::from_secs(30), async {
3644 while trackers[index].running() != 0 || ctx.namespace_commit_generation() <= commit_generation {
3645 tokio::task::yield_now().await;
3646 }
3647 if index + 1 == barriers.len() {
3648 while ctx.namespace_commits_pending() {
3649 tokio::task::yield_now().await;
3650 }
3651 }
3652 })
3653 .await
3654 .expect("released tail must drain and publish its terminal namespace generation");
3655 assert!(store.scanner_namespace_mutation_generation() > namespace_generation);
3656 let pending = index + 1 < barriers.len();
3657 assert_eq!(ctx.namespace_commits_pending(), pending);
3658 assert_eq!(store.scanner_data_usage_publication_blocked().await, pending);
3659 assert!(!store.scanner_data_movement_activity().await.1);
3660 assert!(store.scanner_data_usage_publication_admission_guard().await.is_some());
3661 assert!(set.scanner_data_usage_publication_admission_guard().await.is_some());
3662 }
3663
3664 let (lease, generation) = store
3665 .acquire_scanner_publication_lease(movement_generation, crate::runtime::instance::SCANNER_PUBLICATION_LEASE_TTL)
3666 .await
3667 .expect("remote publication lease should resume after both tails drain");
3668 store
3669 .validate_scanner_publication_lease(lease, generation)
3670 .await
3671 .expect("a resumed remote publication lease should validate");
3672 drop(
3673 store
3674 .acquire_scanner_publication_lease_guard(lease)
3675 .await
3676 .expect("target-side publication admission should resume after both tails drain"),
3677 );
3678 assert!(store.release_scanner_publication_lease(lease).await);
3679
3680 let disks = set.disk_inventory().await;
3681 assert_eq!(disks.len(), 4);
3682 for ((object, body), committed) in objects.iter().zip(&committed) {
3683 let logical_size = i64::try_from(body.len()).expect("fixture payload size should fit i64");
3684 let etag = committed.etag.as_ref().expect("root PUT should return a committed ETag");
3685 for (disk_index, disk) in disks.iter().enumerate() {
3686 let file_info = disk
3687 .as_ref()
3688 .expect("every fixture disk should remain online")
3689 .read_version(
3690 "",
3691 &bucket,
3692 object,
3693 "",
3694 &crate::disk::ReadOptions {
3695 read_data: true,
3696 ..Default::default()
3697 },
3698 )
3699 .await
3700 .unwrap_or_else(|err| panic!("disk {disk_index} should publish {object} after its tail finishes: {err}"));
3701 assert_eq!(file_info.size, logical_size);
3702 assert_eq!(file_info.metadata.get(http::header::ETAG.as_str()), Some(etag));
3703 assert!(
3704 file_info.inline_data(),
3705 "small fixture payloads should have an inline shard on every disk"
3706 );
3707 let inline_data = file_info.data.as_ref().expect("every disk should retain its inline shard");
3708 let erasure = crate::erasure::coding::Erasure::try_new_with_options(
3709 file_info.erasure.data_blocks,
3710 file_info.erasure.parity_blocks,
3711 file_info.erasure.block_size,
3712 file_info.uses_legacy_checksum,
3713 )
3714 .expect("persisted erasure geometry should be valid");
3715 let shard_size =
3716 usize::try_from(erasure.shard_file_size(logical_size)).expect("fixture shard size should fit usize");
3717 crate::erasure::coding::bitrot_verify(
3718 Cursor::new(inline_data.clone()),
3719 inline_data.len(),
3720 shard_size,
3721 rustfs_utils::HashAlgorithm::HighwayHash256S,
3722 erasure.shard_size(),
3723 )
3724 .await
3725 .unwrap_or_else(|err| panic!("disk {disk_index} should retain a complete valid shard for {object}: {err}"));
3726 }
3727 let mut reader = store
3728 .get_object_reader(&bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
3729 .await
3730 .expect("fully drained PUT should be readable");
3731 let mut actual = Vec::new();
3732 reader.stream.read_to_end(&mut actual).await.expect("PUT body should drain");
3733 assert_eq!(&actual, body);
3734 }
3735
3736 let generation_before_internal_put = ctx.namespace_commit_generation();
3737 let internal_object = "scanner-tail-regression/internal-metadata";
3738 let internal_body = b"scanner metadata must not invalidate its own publication";
3739 let mut internal_reader = PutObjReader::from_vec(internal_body.to_vec());
3740 store
3741 .put_object(RUSTFS_META_BUCKET, internal_object, &mut internal_reader, &ObjectOptions::default())
3742 .await
3743 .expect("internal metadata PUT should commit without scanner self-invalidation");
3744 let internal_lock = set
3745 .new_ns_lock(RUSTFS_META_BUCKET, internal_object)
3746 .await
3747 .expect("internal metadata tail lock should be available");
3748 drop(
3749 internal_lock
3750 .get_write_lock(Duration::from_secs(30))
3751 .await
3752 .expect("internal metadata tail should drain"),
3753 );
3754 assert_eq!(ctx.namespace_commit_generation(), generation_before_internal_put);
3755 assert!(!ctx.namespace_commits_pending());
3756 assert!(store.scanner_data_usage_publication_admission_guard().await.is_some());
3757 assert!(set.scanner_data_usage_publication_admission_guard().await.is_some());
3758 let mut internal_reader = store
3759 .get_object_reader(RUSTFS_META_BUCKET, internal_object, None, HeaderMap::new(), &ObjectOptions::default())
3760 .await
3761 .expect("internal metadata should remain readable");
3762 let mut actual = Vec::new();
3763 internal_reader
3764 .stream
3765 .read_to_end(&mut actual)
3766 .await
3767 .expect("internal metadata body should drain");
3768 assert_eq!(actual, internal_body);
3769 })
3770 .await;
3771 shutdown.cancel();
3772 }
3773
3774 #[cfg(feature = "test-util")]
3775 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
3776 #[serial_test::serial(storage_class_env)]
3777 async fn cancelled_early_ack_put_keeps_scanner_publication_blocked_until_tail_finishes() {
3778 let temp_dir = tempfile::tempdir().expect("create cancelled scanner PUT tail store dir");
3779 let (ctx, store, shutdown) =
3780 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "scanner-cancelled-put-tail", &[4])).await;
3781 crate::bucket::metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
3782 let bucket = format!("scanner-cancelled-put-tail-{}", Uuid::new_v4());
3783 let object = "scanner-cancelled-tail";
3784 let body = vec![0xC3; 273];
3785 store
3786 .make_bucket(&bucket, &MakeBucketOptions::default())
3787 .await
3788 .expect("create cancelled scanner PUT tail bucket");
3789
3790 temp_env::async_with_vars([(crate::set_disk::ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async {
3791 let tracker = crate::set_disk::rename_fanout_barrier::observe_tasks(object);
3792 let tail =
3793 crate::set_disk::rename_fanout_barrier::arm(object, 0, crate::set_disk::rename_fanout_barrier::PHASE_RENAME);
3794 let quorum = crate::set_disk::PutObjectCommitBarrier::install(
3795 &bucket,
3796 object,
3797 crate::set_disk::PutObjectCommitPause::AfterRenameQuorum,
3798 );
3799 let handoff = crate::set_disk::PutObjectCommitBarrier::install(
3800 &bucket,
3801 object,
3802 crate::set_disk::PutObjectCommitPause::AfterRenameHandoff,
3803 );
3804 let put_store = Arc::clone(&store);
3805 let put_bucket = bucket.clone();
3806 let put_body = body.clone();
3807 let put = tokio::spawn(async move {
3808 let mut reader = PutObjReader::from_vec(put_body);
3809 put_store
3810 .put_object(&put_bucket, object, &mut reader, &ObjectOptions::default())
3811 .await
3812 });
3813 tokio::time::timeout(Duration::from_secs(30), tail.wait_until_paused())
3814 .await
3815 .expect("cancelled PUT should pause one disk before rename");
3816 quorum.wait_until_paused().await;
3817 put.abort();
3818 assert!(
3819 put.await
3820 .expect_err("caller should be cancelled after rename quorum")
3821 .is_cancelled()
3822 );
3823 quorum.release();
3824 handoff.wait_until_paused().await;
3825 assert!(tracker.running() >= 1);
3826 assert!(ctx.namespace_commits_pending());
3827 assert!(!store.scanner_data_movement_activity().await.1);
3828 assert!(store.scanner_data_usage_publication_blocked().await);
3829 assert!(store.scanner_data_usage_publication_admission_guard().await.is_some());
3830 assert!(
3831 store.pools[0].disk_set[0]
3832 .scanner_data_usage_publication_admission_guard()
3833 .await
3834 .is_some()
3835 );
3836 let generation = store.scanner_namespace_mutation_generation();
3837
3838 handoff.release();
3839 tail.release();
3840 tokio::time::timeout(Duration::from_secs(30), async {
3841 while tracker.running() != 0 || ctx.namespace_commits_pending() {
3842 tokio::task::yield_now().await;
3843 }
3844 })
3845 .await
3846 .expect("cancelled request's detached fanout must release scanner admission after finishing");
3847 assert!(store.scanner_namespace_mutation_generation() > generation);
3848 assert!(!store.scanner_data_usage_publication_blocked().await);
3849 assert!(store.scanner_data_usage_publication_admission_guard().await.is_some());
3850 for (disk_index, disk) in store.pools[0].disk_set[0].disk_inventory().await.iter().enumerate() {
3851 let file_info = disk
3852 .as_ref()
3853 .expect("cancelled PUT fixture disk should remain online")
3854 .read_version("", &bucket, object, "", &crate::disk::ReadOptions::default())
3855 .await
3856 .unwrap_or_else(|err| panic!("cancelled PUT must still publish on disk {disk_index}: {err}"));
3857 assert_eq!(file_info.size, i64::try_from(body.len()).expect("fixture body size should fit i64"));
3858 }
3859 let mut reader = store
3860 .get_object_reader(&bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
3861 .await
3862 .expect("a cancelled caller must not discard its quorum-committed object");
3863 let mut actual = Vec::new();
3864 reader
3865 .stream
3866 .read_to_end(&mut actual)
3867 .await
3868 .expect("cancelled PUT body should drain");
3869 assert_eq!(actual, body);
3870 })
3871 .await;
3872 shutdown.cancel();
3873 }
3874
3875 #[cfg(feature = "test-util")]
3876 #[test]
3877 #[serial_test::serial(storage_class_env)]
3878 fn data_movement_multipart_part_staging_holds_no_publication_lock_or_tier_lease() {
3879 run_large_stack_async_test(
3880 "multipart-part-staging-publication-fence",
3881 data_movement_multipart_part_staging_holds_no_publication_lock_or_tier_lease_case,
3882 );
3883 }
3884
3885 #[cfg(feature = "test-util")]
3886 async fn data_movement_multipart_part_staging_holds_no_publication_lock_or_tier_lease_case() {
3887 let temp_dir = tempfile::tempdir().expect("create multipart staging-fence store dir");
3888 let (ctx, store, shutdown) =
3889 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "mpu-staging-publication", &[4, 4])).await;
3890 crate::bucket::metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
3891
3892 let bucket = format!("mpu-staging-publication-{}", Uuid::new_v4());
3893 let object = "remote-owned.bin";
3894 let payload = b"multipart part staging must not publish an object".to_vec();
3895 store
3896 .make_bucket(&bucket, &MakeBucketOptions::default())
3897 .await
3898 .expect("multipart staging test bucket should be created");
3899
3900 let tier_name = "MPU-STAGING-TIER";
3901 register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
3902 let lease = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
3903 .await
3904 .expect("mock tier generation should be available");
3905 let backend_identity = lease.backend_identity();
3906 drop(lease);
3907
3908 let mut source_metadata = HashMap::from([(
3909 "x-amz-restore".to_string(),
3910 "ongoing-request=\"false\", expiry-date=\"2099-01-01T00:00:00Z\"".to_string(),
3911 )]);
3912 for (suffix, value) in [
3913 (
3914 rustfs_utils::http::SUFFIX_TRANSITION_STATUS,
3915 crate::bucket::lifecycle::core::TRANSITION_COMPLETE.to_string(),
3916 ),
3917 (rustfs_utils::http::SUFFIX_TRANSITIONED_OBJECTNAME, "remote/mpu.bin".to_string()),
3918 (rustfs_utils::http::SUFFIX_TRANSITION_TIER, tier_name.to_string()),
3919 (rustfs_utils::http::SUFFIX_TRANSITIONED_VERSION_ID, "remote-mpu-version".to_string()),
3920 (
3921 rustfs_utils::http::SUFFIX_TRANSITIONED_VERSION_STATE,
3922 rustfs_filemeta::TransitionVersionState::Exact.as_str().to_string(),
3923 ),
3924 (
3925 rustfs_utils::http::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
3926 rustfs_utils::crypto::hex(backend_identity),
3927 ),
3928 ] {
3929 rustfs_utils::http::metadata_compat::insert_str(&mut source_metadata, suffix, value);
3930 }
3931 let mut source_body = PutObjReader::from_vec(payload.clone());
3932 store.pools[0]
3933 .put_object(
3934 &bucket,
3935 object,
3936 &mut source_body,
3937 &ObjectOptions {
3938 user_defined: source_metadata,
3939 ..Default::default()
3940 },
3941 )
3942 .await
3943 .expect("remote-owned source should be written to the source pool");
3944 let source = store.pools[0]
3945 .get_object_info(&bucket, object, &ObjectOptions::default())
3946 .await
3947 .expect("remote-owned source metadata should be readable");
3948 let publication = store
3949 .acquire_remote_tuple_publication_fence(&bucket, 0, &source, true)
3950 .await
3951 .expect("multipart migration should capture its publication capability");
3952 assert_eq!(
3953 TierConfigMgr::active_operation_lease_count(&ctx.tier_config_mgr(), tier_name).await,
3954 0,
3955 "capturing the commit-late capability must not pin the tier generation"
3956 );
3957
3958 let bucket_incarnation_id = store
3959 .bucket_incarnation_id(&bucket)
3960 .await
3961 .expect("bucket incarnation should resolve");
3962 mark_test_pool_decommissioning(&store, 0).await;
3963 let capacity_owner = test_decommission_capacity_owner(store.as_ref(), 0)
3964 .await
3965 .with_mutation_id(Uuid::new_v4());
3966 let upload_metadata = source.user_defined.as_ref().clone();
3967 let mut staging_opts = ObjectOptions {
3968 data_movement: true,
3969 src_pool_idx: 0,
3970 versioned: source.version_id.is_some(),
3971 version_id: source.version_id.map(|version_id| version_id.to_string()),
3972 mod_time: source.mod_time,
3973 user_defined: upload_metadata,
3974 expected_bucket_incarnation_id: Some(bucket_incarnation_id),
3975 ..Default::default()
3976 };
3977 let upload_identity = crate::data_movement::data_movement_upload_identity_from_options(&staging_opts);
3978 rustfs_utils::http::insert_str(
3979 &mut staging_opts.user_defined,
3980 rustfs_utils::http::SUFFIX_DATA_MOVEMENT_UPLOAD,
3981 upload_identity,
3982 );
3983 capacity_owner.apply_to(&mut staging_opts);
3984 let (upload, target_pool_idx, staged_incarnation_id) = store
3985 .handle_new_multipart_upload_with_pool_idx(&bucket, object, &staging_opts, None)
3986 .await
3987 .expect("target multipart upload should be staged");
3988 assert_eq!(target_pool_idx, 1, "the active pool must receive the staged upload");
3989 assert_eq!(staged_incarnation_id, Some(bucket_incarnation_id));
3990
3991 let barrier = crate::set_disk::MultipartCommitBarrier::install(
3992 &bucket,
3993 object,
3994 crate::set_disk::MultipartCommitPause::PutPartBeforeLockLost,
3995 );
3996 let part_store = Arc::clone(&store);
3997 let part_bucket = bucket.clone();
3998 let upload_id = upload.upload_id.clone();
3999 let part_payload = payload.clone();
4000 let part = tokio::spawn(async move {
4001 let mut reader = PutObjReader::from_vec(part_payload);
4002 let mut part_opts = ObjectOptions {
4003 data_movement: true,
4004 src_pool_idx: 0,
4005 part_number: Some(1),
4006 expected_bucket_incarnation_id: Some(bucket_incarnation_id),
4007 ..Default::default()
4008 };
4009 capacity_owner.apply_to(&mut part_opts);
4010 part_store
4011 .put_object_part_for_data_movement(1, &part_bucket, object, &upload_id, &mut reader, &part_opts)
4012 .await
4013 });
4014 barrier.wait_until_paused().await;
4015
4016 assert_eq!(
4017 TierConfigMgr::active_operation_lease_count(&ctx.tier_config_mgr(), tier_name).await,
4018 0,
4019 "UploadPart staging must not pin the tier generation"
4020 );
4021 let encoded = rustfs_utils::path::encode_dir_object(object);
4022 let mut proof_opts = ObjectOptions::default();
4023 let proof = tokio::time::timeout(
4024 Duration::from_millis(250),
4025 store.acquire_all_physical_object_read_locks("tier_delete_journal_recovery", &bucket, &encoded, &mut proof_opts),
4026 )
4027 .await
4028 .expect("UploadPart staging must not hold any object publication write lock")
4029 .expect("the all-physical recovery proof should acquire every object namespace");
4030 let source_head = store.pools[0]
4031 .get_object_info(&bucket, object, &ObjectOptions::default())
4032 .await
4033 .expect("source HEAD should remain available while the part commit is paused");
4034 assert_eq!(source_head.etag, source.etag);
4035 assert!(
4036 store.pools[1]
4037 .get_object_info(&bucket, object, &ObjectOptions::default())
4038 .await
4039 .is_err(),
4040 "UploadPart staging must not publish target object metadata"
4041 );
4042 drop(proof);
4043
4044 barrier.release();
4045 drop(barrier);
4046 part.await
4047 .expect("UploadPart staging task should join")
4048 .expect("UploadPart staging should commit after the observation releases");
4049 let mut abort_opts = ObjectOptions {
4050 data_movement: true,
4051 src_pool_idx: 0,
4052 versioned: source.version_id.is_some(),
4053 version_id: source.version_id.map(|version_id| version_id.to_string()),
4054 mod_time: source.mod_time,
4055 expected_bucket_incarnation_id: Some(bucket_incarnation_id),
4056 ..Default::default()
4057 };
4058 capacity_owner.apply_to(&mut abort_opts);
4059 store
4060 .abort_multipart_upload_for_data_movement(1, &bucket, object, &upload.upload_id, &abort_opts)
4061 .await
4062 .expect("staged target upload should be removable without publishing");
4063 drop(publication);
4064 assert_eq!(TierConfigMgr::active_operation_lease_count(&ctx.tier_config_mgr(), tier_name).await, 0);
4065 shutdown.cancel();
4066 }
4067
4068 #[cfg(feature = "test-util")]
4069 struct OfflineTestDisks {
4070 disks: Vec<crate::disk::DiskStore>,
4071 }
4072
4073 #[cfg(feature = "test-util")]
4074 impl Drop for OfflineTestDisks {
4075 fn drop(&mut self) {
4076 for disk in &self.disks {
4077 disk.reset_health_for_store_init_retry();
4078 }
4079 }
4080 }
4081
4082 #[cfg(feature = "test-util")]
4083 async fn force_set_disks_offline_for_test(set: &Arc<crate::set_disk::SetDisks>) -> OfflineTestDisks {
4084 let disks = set
4085 .disks
4086 .read()
4087 .await
4088 .iter()
4089 .map(|disk| disk.clone().expect("fault-injection set should start fully online"))
4090 .collect::<Vec<_>>();
4091 for disk in &disks {
4092 disk.close().await.expect("fault injection should stop per-disk monitoring");
4093 disk.force_offline_for_test();
4094 }
4095
4096 set.connect_disks().await;
4100 let current = set.disks.read().await.clone();
4101 assert_eq!(current.len(), disks.len());
4102 for (slot, expected) in current.iter().zip(&disks) {
4103 let disk = slot.as_ref().expect("offline test slot must remain populated");
4104 assert!(Arc::ptr_eq(disk, expected), "endpoint monitor must not replace an offline test handle");
4105 assert_eq!(disk.runtime_state(), crate::disk::health_state::RuntimeDriveHealthState::Offline);
4106 assert!(
4107 disk.is_online().await,
4108 "the valid disk identity must prevent endpoint renewal while the IO health gate remains offline"
4109 );
4110 }
4111 OfflineTestDisks { disks }
4112 }
4113
4114 #[cfg(feature = "test-util")]
4115 async fn force_set_disk_range_offline_for_test(
4116 set: &Arc<crate::set_disk::SetDisks>,
4117 range: std::ops::Range<usize>,
4118 ) -> OfflineTestDisks {
4119 let disks = set
4120 .disks
4121 .read()
4122 .await
4123 .get(range)
4124 .expect("offline test range must fit the set")
4125 .iter()
4126 .map(|disk| disk.clone().expect("fault-injection disk should start online"))
4127 .collect::<Vec<_>>();
4128 for disk in &disks {
4129 disk.close().await.expect("fault injection should stop per-disk monitoring");
4130 disk.force_offline_for_test();
4131 }
4132 set.connect_disks().await;
4133 for disk in &disks {
4134 assert_eq!(disk.runtime_state(), crate::disk::health_state::RuntimeDriveHealthState::Offline);
4135 }
4136 OfflineTestDisks { disks }
4137 }
4138
4139 fn active_rebalance_meta_for_pool(pool_count: usize, active_pool_idx: usize) -> RebalanceMeta {
4140 let now = OffsetDateTime::now_utc();
4141 let mut pool_stats = vec![RebalanceStats::default(); pool_count];
4142 pool_stats[active_pool_idx] = RebalanceStats {
4143 participating: true,
4144 info: RebalanceInfo {
4145 start_time: Some(now),
4146 status: RebalStatus::Started,
4147 ..Default::default()
4148 },
4149 ..Default::default()
4150 };
4151
4152 RebalanceMeta {
4153 id: uuid::Uuid::new_v4().to_string(),
4154 pool_stats,
4155 ..Default::default()
4156 }
4157 }
4158
4159 #[tokio::test]
4160 #[serial_test::serial(storage_class_env)]
4161 async fn quota_object_fence_ignores_an_unrelated_offline_pool() {
4162 let temp_dir = tempfile::tempdir().expect("create quota fence store dir");
4163 let (_ctx, store, shutdown) =
4164 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "quota-object-fence", &[4, 4])).await;
4165 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
4166 let bucket = format!("quota-object-fence-{}", uuid::Uuid::new_v4());
4167 let object = "object.bin";
4168 store
4169 .make_bucket(&bucket, &MakeBucketOptions::default())
4170 .await
4171 .expect("create quota fence bucket");
4172 store.pools[1].disk_set[0].disks.write().await.fill(None);
4173
4174 crate::bucket::quota::reservation::fence_namespace_mutations_for_test(&store, &bucket, object, Some((0, 0)))
4175 .await
4176 .expect("the selected pool fence should ignore an unrelated offline pool");
4177 let err = crate::bucket::quota::reservation::fence_namespace_mutations_for_test(&store, &bucket, object, None)
4178 .await
4179 .expect_err("legacy reservations must conservatively fence every pool");
4180 assert!(matches!(err, StorageError::ErasureWriteQuorum));
4181
4182 shutdown.cancel();
4183 }
4184
4185 async fn migrate_versioned_decommission_test_object(
4186 store: &Arc<crate::store::ECStore>,
4187 bucket: &str,
4188 object: &str,
4189 payload: &[u8],
4190 op_label: &'static str,
4191 ) -> (uuid::Uuid, FileInfoVersions) {
4192 let mut source = PutObjReader::from_vec(payload.to_vec());
4193 let source_info = store.pools[0]
4194 .put_object(
4195 bucket,
4196 object,
4197 &mut source,
4198 &ObjectOptions {
4199 versioned: true,
4200 ..Default::default()
4201 },
4202 )
4203 .await
4204 .expect("write versioned source to the pool being decommissioned");
4205 let source_version = source_info.version_id.expect("versioned source must have a version ID");
4206 let expected_source_versions = store.pools[0]
4207 .get_disks_by_key(object)
4208 .load_file_info_versions_exact(bucket, object)
4209 .await
4210 .expect("source versions should be readable before migration")
4211 .expect("source versions should exist before migration");
4212 {
4213 let mut pool_meta = store.pool_meta.write().await;
4214 pool_meta.pools[0].decommission = Some(PoolDecommissionInfo {
4215 start_time: Some(OffsetDateTime::now_utc()),
4216 ..Default::default()
4217 });
4218 }
4219
4220 let barrier = crate::set_disk::PutObjectCommitBarrier::install(
4221 bucket,
4222 object,
4223 crate::set_disk::PutObjectCommitPause::AfterNamespace,
4224 );
4225 let migration_store = Arc::clone(store);
4226 let migration_bucket = bucket.to_string();
4227 let migration_object = object.to_string();
4228 let migration = tokio::spawn(async move {
4229 let source_reader = migration_store.pools[0]
4230 .get_object_reader(
4231 &migration_bucket,
4232 &migration_object,
4233 None,
4234 HeaderMap::new(),
4235 &ObjectOptions {
4236 versioned: true,
4237 version_id: Some(source_version.to_string()),
4238 no_lock: true,
4239 data_movement: true,
4240 raw_data_movement_read: true,
4241 ..Default::default()
4242 },
4243 )
4244 .await?;
4245 crate::data_movement::migrate_decommission_object(
4246 migration_store,
4247 0,
4248 migration_bucket,
4249 source_reader,
4250 None,
4251 op_label,
4252 None,
4253 )
4254 .await
4255 });
4256 barrier.wait_until_paused().await;
4257 barrier.release();
4258 migration
4259 .await
4260 .expect("versioned decommission migration task should join")
4261 .expect("versioned decommission migration should commit");
4262
4263 (source_version, expected_source_versions)
4264 }
4265
4266 fn set_test_decommission_capacity_override(store: &Arc<crate::store::ECStore>, pool_idx: usize) {
4267 let layout = DecommissionErasureLayout { data: 2, parity: 2 };
4268 let source_physical_bytes = 1024 * 1024 * 1024;
4269 let target_physical_bytes = source_physical_bytes * 8;
4270 let capacity: Vec<DecommissionPoolCapacityInfo> = store
4271 .pools
4272 .iter()
4273 .enumerate()
4274 .map(|(index, _)| {
4275 if index == pool_idx {
4276 DecommissionPoolCapacityInfo::for_test(index, layout, 0, source_physical_bytes, source_physical_bytes)
4277 } else {
4278 DecommissionPoolCapacityInfo::for_test(index, layout, target_physical_bytes, target_physical_bytes, 0)
4279 }
4280 })
4281 .collect();
4282 set_decommission_capacity_info_overrides_for_test(store.id, (0..128).map(|_| capacity.clone()).collect());
4283 }
4284
4285 async fn mark_test_pool_decommissioning(store: &Arc<crate::store::ECStore>, pool_idx: usize) {
4286 set_test_decommission_capacity_override(store, pool_idx);
4287 store
4288 .save_current_pool_meta_for_decommission_start(&[pool_idx], Vec::new())
4289 .await
4290 .expect("test decommission capacity reservation should activate");
4291 }
4292
4293 #[cfg(feature = "test-util")]
4294 async fn mark_test_pool_decommissioning_with_split_targets(store: &Arc<crate::store::ECStore>, pool_idx: usize) {
4295 let layout = DecommissionErasureLayout { data: 2, parity: 2 };
4296 let source_physical_bytes = 1024 * 1024 * 1024;
4297 let capacity = store
4298 .pools
4299 .iter()
4300 .enumerate()
4301 .map(|(index, _)| {
4302 if index == pool_idx {
4303 DecommissionPoolCapacityInfo::for_test(index, layout, 0, source_physical_bytes, source_physical_bytes)
4304 } else {
4305 DecommissionPoolCapacityInfo::for_test(index, layout, source_physical_bytes, source_physical_bytes, 0)
4309 }
4310 })
4311 .collect::<Vec<_>>();
4312 set_decommission_capacity_info_overrides_for_test(store.id, (0..128).map(|_| capacity.clone()).collect());
4313 store
4314 .save_current_pool_meta_for_decommission_start(&[pool_idx], Vec::new())
4315 .await
4316 .expect("split-target decommission capacity reservation should activate");
4317 }
4318
4319 #[cfg(feature = "test-util")]
4320 async fn test_decommission_capacity_owner(
4321 store: &crate::store::ECStore,
4322 source_pool_index: usize,
4323 ) -> crate::core::pools::DecommissionCapacityOwner {
4324 let pool_meta = store.pool_meta.read().await;
4325 let reservation = pool_meta.pools[source_pool_index]
4326 .decommission
4327 .as_ref()
4328 .and_then(|info| info.capacity_reservation.as_ref())
4329 .expect("test decommission capacity reservation should exist");
4330 crate::core::pools::DecommissionCapacityOwner {
4331 source_pool_index,
4332 operation_id: reservation.operation_id,
4333 generation: reservation.generation,
4334 owner_nonce: reservation.owner_nonce,
4335 mutation_id: None,
4336 }
4337 }
4338
4339 const DECOMMISSION_TEST_FAULT_STAGE_DELETE_MARKER: &str = "delete_marker_copy";
4340 const DECOMMISSION_TEST_FAULT_STAGE_MIGRATE_OBJECT: &str = "migrate_object";
4341 #[cfg(feature = "test-util")]
4342 const DECOMMISSION_TEST_FAULT_STAGE_TIERED: &str = "decommission_tiered_object";
4343
4344 fn decommission_retry_fault_hook(
4345 bucket: &str,
4346 object: &str,
4347 faults: Arc<AtomicUsize>,
4348 ) -> crate::core::pools::DecommissionTestFaultDecision {
4349 let target_bucket = bucket.to_string();
4350 let target_object = object.to_string();
4351 Arc::new(move |stage, bucket, object, attempt, succeeded| {
4352 if !succeeded
4353 || attempt >= crate::core::pools::DECOMMISSION_VERSION_COPY_ATTEMPTS
4354 || stage != DECOMMISSION_TEST_FAULT_STAGE_MIGRATE_OBJECT
4355 || bucket != target_bucket
4356 || object != target_object
4357 {
4358 return false;
4359 }
4360
4361 faults
4365 .try_update(Ordering::SeqCst, Ordering::SeqCst, |faults| {
4366 (faults < crate::core::pools::DECOMMISSION_VERSION_COPY_ATTEMPTS.saturating_sub(1))
4367 .then_some(faults.saturating_add(1))
4368 })
4369 .is_ok()
4370 })
4371 }
4372
4373 async fn seed_decommission_source(
4374 store: &Arc<crate::store::ECStore>,
4375 bucket: &str,
4376 object: &str,
4377 body: Vec<u8>,
4378 opts: &ObjectOptions,
4379 ) {
4380 let mut reader = PutObjReader::from_vec(body);
4381 store.pools[0]
4382 .put_object(bucket, object, &mut reader, opts)
4383 .await
4384 .expect("seed decommission source object");
4385 }
4386
4387 async fn run_decommission_entry_retry_test(
4388 store: &Arc<crate::store::ECStore>,
4389 rx: CancellationToken,
4390 bucket: &str,
4391 object: &str,
4392 expected_bucket_incarnation_id: Option<uuid::Uuid>,
4393 source_changed_exhaustions: Arc<AtomicUsize>,
4394 ) -> crate::error::Result<()> {
4395 let source_set = store.pools[0].get_disks_by_key(object);
4396 store
4397 .decommission_entry_with_retry_state_for_test(
4398 rx,
4399 0,
4400 MetaCacheEntry {
4401 name: object.to_string(),
4402 ..Default::default()
4403 },
4404 bucket.to_string(),
4405 source_set,
4406 expected_bucket_incarnation_id,
4407 source_changed_exhaustions,
4408 )
4409 .await
4410 }
4411
4412 async fn read_decommission_target_body(
4413 store: &Arc<crate::store::ECStore>,
4414 bucket: &str,
4415 object: &str,
4416 opts: &ObjectOptions,
4417 ) -> Vec<u8> {
4418 let mut reader = store.pools[1]
4419 .get_object_reader(bucket, object, None, HeaderMap::new(), opts)
4420 .await
4421 .expect("read decommission target object");
4422 let mut body = Vec::new();
4423 reader
4424 .stream
4425 .read_to_end(&mut body)
4426 .await
4427 .expect("drain decommission target body");
4428 body
4429 }
4430
4431 async fn assert_decommission_source_absent(
4432 store: &Arc<crate::store::ECStore>,
4433 bucket: &str,
4434 object: &str,
4435 opts: &ObjectOptions,
4436 ) {
4437 let err = store.pools[0]
4438 .get_object_info(bucket, object, opts)
4439 .await
4440 .expect_err("decommission source must be retained until target commit, then removed");
4441 assert!(
4442 matches!(err, StorageError::ObjectNotFound(_, _) | StorageError::VersionNotFound(_, _, _)),
4443 "unexpected decommission source result: {err:?}"
4444 );
4445 }
4446
4447 #[cfg(feature = "test-util")]
4448 async fn seed_transitioned_free_version(
4449 ctx: &Arc<crate::runtime::instance::InstanceContext>,
4450 store: &Arc<crate::store::ECStore>,
4451 bucket: &str,
4452 object: &str,
4453 ) -> (uuid::Uuid, uuid::Uuid) {
4454 let tier_name = format!("DECOMFREE{}", uuid::Uuid::new_v4().simple());
4455 register_mock_tier(&ctx.tier_config_mgr(), &tier_name).await;
4456
4457 let mut reader = PutObjReader::from_vec(b"transitioned source bytes".to_vec());
4458 let source = store.pools[0]
4459 .put_object(
4460 bucket,
4461 object,
4462 &mut reader,
4463 &ObjectOptions {
4464 versioned: true,
4465 ..Default::default()
4466 },
4467 )
4468 .await
4469 .expect("write transitioned decommission source");
4470 let source_version = source.version_id.expect("transitioned source must be versioned");
4471 store.pools[0]
4472 .transition_object(
4473 bucket,
4474 object,
4475 &ObjectOptions {
4476 versioned: true,
4477 version_id: Some(source_version.to_string()),
4478 transition: TransitionOptions {
4479 status: TRANSITION_PENDING.to_string(),
4480 tier: tier_name,
4481 etag: source.etag.clone().expect("transitioned source must have an ETag"),
4482 ..Default::default()
4483 },
4484 mod_time: source.mod_time,
4485 ..Default::default()
4486 },
4487 )
4488 .await
4489 .expect("transition source before decommission");
4490 store.pools[0]
4491 .delete_object(
4492 bucket,
4493 object,
4494 ObjectOptions {
4495 versioned: true,
4496 version_id: Some(source_version.to_string()),
4497 ..Default::default()
4498 },
4499 )
4500 .await
4501 .expect("delete transitioned source version");
4502
4503 let versions = store.pools[0]
4504 .get_disks_by_key(object)
4505 .load_file_info_versions_exact(bucket, object)
4506 .await
4507 .expect("source versions should decode after transition delete")
4508 .expect("source free version should remain after transition delete");
4509 let free_version = versions
4510 .versions
4511 .iter()
4512 .find(|version| version.tier_free_version())
4513 .and_then(|version| version.version_id)
4514 .expect("transition delete should create a free version");
4515 (source_version, free_version)
4516 }
4517
4518 async fn write_decommission_test_multipart_source(
4519 store: &Arc<crate::store::ECStore>,
4520 pool_idx: usize,
4521 bucket: &str,
4522 object: &str,
4523 ) {
4524 let pool = &store.pools[pool_idx];
4525 let upload = pool
4526 .new_multipart_upload(bucket, object, &ObjectOptions::default())
4527 .await
4528 .expect("create decommission multipart source upload");
4529 let first_part = vec![b'm'; 5 * 1024 * 1024];
4530 let second_part = b"decommission multipart tail".to_vec();
4531 let mut completed_parts = Vec::with_capacity(2);
4532 for (part_number, body) in [(1, first_part), (2, second_part)] {
4533 let mut reader = PutObjReader::from_vec(body);
4534 let part = pool
4535 .put_object_part(bucket, object, &upload.upload_id, part_number, &mut reader, &ObjectOptions::default())
4536 .await
4537 .expect("write decommission multipart source part");
4538 completed_parts.push(crate::storage_api_contracts::multipart::CompletePart {
4539 part_num: part.part_num,
4540 etag: part.etag,
4541 ..Default::default()
4542 });
4543 }
4544 pool.clone()
4545 .complete_multipart_upload(bucket, object, &upload.upload_id, completed_parts, &ObjectOptions::default())
4546 .await
4547 .expect("complete decommission multipart source object");
4548 }
4549
4550 struct DataMovementSourceSeed {
4551 pool_idx: usize,
4552 version_id: Option<Uuid>,
4553 body: Vec<u8>,
4554 multipart_split: Option<usize>,
4555 mod_time: OffsetDateTime,
4556 user_defined: HashMap<String, String>,
4557 }
4558
4559 async fn seed_data_movement_source_reader(
4560 store: &Arc<crate::store::ECStore>,
4561 bucket: &str,
4562 object: &str,
4563 seed: DataMovementSourceSeed,
4564 ) -> GetObjectReader {
4565 let DataMovementSourceSeed {
4566 pool_idx,
4567 version_id,
4568 body,
4569 multipart_split,
4570 mod_time,
4571 user_defined,
4572 } = seed;
4573 let pool = &store.pools[pool_idx];
4574 let write_opts = ObjectOptions {
4575 versioned: version_id.is_some(),
4576 version_id: version_id.map(|version_id| version_id.to_string()),
4577 mod_time: Some(mod_time),
4578 user_defined,
4579 ..Default::default()
4580 };
4581 if let Some(split) = multipart_split {
4582 assert!(split > 0 && split < body.len(), "multipart source split must create two non-empty parts");
4583 let upload = pool
4584 .new_multipart_upload(bucket, object, &write_opts)
4585 .await
4586 .expect("create physical data-movement source upload");
4587 let mut completed_parts = Vec::with_capacity(2);
4588 for (part_number, bytes) in [(1, &body[..split]), (2, &body[split..])] {
4589 let mut reader = PutObjReader::from_vec(bytes.to_vec());
4590 let part = pool
4591 .put_object_part(bucket, object, &upload.upload_id, part_number, &mut reader, &ObjectOptions::default())
4592 .await
4593 .expect("write physical data-movement source part");
4594 completed_parts.push(crate::storage_api_contracts::multipart::CompletePart {
4595 part_num: part.part_num,
4596 etag: part.etag,
4597 ..Default::default()
4598 });
4599 }
4600 pool.clone()
4601 .complete_multipart_upload(bucket, object, &upload.upload_id, completed_parts, &write_opts)
4602 .await
4603 .expect("complete physical data-movement source object");
4604 } else {
4605 let mut reader = PutObjReader::from_vec(body);
4606 pool.put_object(bucket, object, &mut reader, &write_opts)
4607 .await
4608 .expect("write physical data-movement source object");
4609 }
4610
4611 pool.get_object_reader(
4612 bucket,
4613 object,
4614 None,
4615 HeaderMap::new(),
4616 &ObjectOptions {
4617 versioned: version_id.is_some(),
4618 version_id: version_id.map(|version_id| version_id.to_string()),
4619 raw_data_movement_read: true,
4620 include_part_checksums: true,
4621 ..Default::default()
4622 },
4623 )
4624 .await
4625 .expect("read the physical data-movement source snapshot")
4626 }
4627
4628 async fn assert_pool_object_present(pool: &Arc<crate::core::sets::Sets>, bucket: &str, object: &str) {
4629 pool.get_object_info(bucket, object, &ObjectOptions::default())
4630 .await
4631 .expect("expected object generation must remain present");
4632 }
4633
4634 async fn assert_pool_object_absent(pool: &Arc<crate::core::sets::Sets>, bucket: &str, object: &str) {
4635 let err = pool
4636 .get_object_info(bucket, object, &ObjectOptions::default())
4637 .await
4638 .expect_err("fenced decommission target must remain absent");
4639 assert!(
4640 matches!(err, StorageError::ObjectNotFound(_, _) | StorageError::VersionNotFound(_, _, _)),
4641 "unexpected fenced target result: {err:?}"
4642 );
4643 }
4644
4645 async fn write_suspended_decommission_source(store: &Arc<crate::store::ECStore>, bucket: &str, object: &str) {
4646 let mut reader = PutObjReader::from_vec(b"suspended source generation".to_vec());
4647 let source = store.pools[0]
4648 .put_object(
4649 bucket,
4650 object,
4651 &mut reader,
4652 &ObjectOptions {
4653 version_suspended: true,
4654 mod_time: Some(OffsetDateTime::UNIX_EPOCH + time::Duration::SECOND),
4655 ..Default::default()
4656 },
4657 )
4658 .await
4659 .expect("write suspended null source version");
4660 assert!(
4661 source.version_id.is_none_or(|version_id| version_id.is_nil()),
4662 "suspended source must use the null version identity"
4663 );
4664 }
4665
4666 async fn assert_suspended_null_source_present(store: &Arc<crate::store::ECStore>, bucket: &str, object: &str) {
4667 let versions = store.pools[0]
4668 .get_disks_by_key(object)
4669 .load_file_info_versions_exact(bucket, object)
4670 .await
4671 .expect("suspended source versions should be readable")
4672 .expect("suspended source must exist before worker convergence");
4673 assert_eq!(versions.versions.len(), 1, "DELETE must not add a marker to the retiring source");
4674 let source = &versions.versions[0];
4675 assert!(
4676 !source.deleted && source.version_id.is_none_or(|version_id| version_id.is_nil()),
4677 "the source pool must retain its null data version until worker convergence"
4678 );
4679 assert_eq!(source.mod_time, Some(OffsetDateTime::UNIX_EPOCH + time::Duration::SECOND));
4680
4681 let mut reader = store.pools[0]
4682 .get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
4683 .await
4684 .expect("the retiring source must remain directly readable before worker convergence");
4685 let mut body = Vec::new();
4686 reader
4687 .stream
4688 .read_to_end(&mut body)
4689 .await
4690 .expect("read retained source bytes");
4691 assert_eq!(body, b"suspended source generation");
4692 }
4693
4694 async fn assert_suspended_null_delete_marker_visible(
4695 store: &Arc<crate::store::ECStore>,
4696 bucket: &str,
4697 object: &str,
4698 marker_mod_time: OffsetDateTime,
4699 ) {
4700 let versions = store.pools[1]
4701 .get_disks_by_key(object)
4702 .load_file_info_versions_exact(bucket, object)
4703 .await
4704 .expect("healthy target versions should be readable")
4705 .expect("the healthy target must retain the DELETE marker");
4706 assert_eq!(versions.versions.len(), 1, "the target must contain only the null delete marker");
4707 let marker = &versions.versions[0];
4708 assert!(marker.deleted, "migration must not replace the DELETE marker with source data");
4709 assert!(marker.version_id.is_none_or(|version_id| version_id.is_nil()));
4710 assert_eq!(marker.size, 0);
4711 assert_eq!(marker.mod_time, Some(marker_mod_time), "migration must preserve the marker generation");
4712 assert!(marker_mod_time > OffsetDateTime::UNIX_EPOCH + time::Duration::SECOND);
4713
4714 let head_err = store
4715 .get_object_info(bucket, object, &ObjectOptions::default())
4716 .await
4717 .expect_err("HEAD must observe the DELETE marker instead of the old null source");
4718 assert!(matches!(head_err, Error::ObjectNotFound(_, _)), "unexpected HEAD result: {head_err:?}");
4719 let get_err = match store
4720 .get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
4721 .await
4722 {
4723 Ok(_) => panic!("GET must not resurrect the deleted null source"),
4724 Err(err) => err,
4725 };
4726 assert!(matches!(get_err, Error::ObjectNotFound(_, _)), "unexpected GET result: {get_err:?}");
4727 }
4728
4729 #[tokio::test]
4730 #[serial_test::serial(storage_class_env)]
4731 async fn tag_updates_skip_active_rebalance_source_pool() {
4732 let temp_dir = tempfile::tempdir().expect("create writer-fencing store dir");
4733 let (_ctx, store, shutdown) =
4734 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "writer-fencing-tags", &[4, 4])).await;
4735 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
4736
4737 let bucket = format!("writer-fencing-tags-{}", uuid::Uuid::new_v4());
4738 let object = "tagged-object.bin";
4739 store
4740 .make_bucket(&bucket, &MakeBucketOptions::default())
4741 .await
4742 .expect("create writer fencing bucket");
4743
4744 let old_time = OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("fixed timestamp should be valid");
4745 let newer_time = old_time + time::Duration::seconds(10);
4746 let mut source_reader = PutObjReader::from_vec(b"source-body".to_vec());
4747 store.pools[0]
4748 .put_object(
4749 &bucket,
4750 object,
4751 &mut source_reader,
4752 &ObjectOptions {
4753 mod_time: Some(newer_time),
4754 ..Default::default()
4755 },
4756 )
4757 .await
4758 .expect("write newer source object");
4759 let mut target_reader = PutObjReader::from_vec(b"target-body".to_vec());
4760 store.pools[1]
4761 .put_object(
4762 &bucket,
4763 object,
4764 &mut target_reader,
4765 &ObjectOptions {
4766 mod_time: Some(old_time),
4767 ..Default::default()
4768 },
4769 )
4770 .await
4771 .expect("write older target object");
4772
4773 *store.rebalance_meta.write().await = Some(active_rebalance_meta_for_pool(store.pools.len(), 0));
4774 assert!(store.is_pool_rebalancing(0).await, "pool 0 must be marked as an active rebalance source");
4775
4776 let tags = "rebalance=target";
4777 assert_ne!(
4778 store.pools[0]
4779 .get_object_tags(&bucket, object, &ObjectOptions::default())
4780 .await
4781 .expect("source object tags should be readable before update"),
4782 tags,
4783 "source object must start without the target tag"
4784 );
4785 assert_ne!(
4786 store.pools[1]
4787 .get_object_tags(&bucket, object, &ObjectOptions::default())
4788 .await
4789 .expect("target object tags should be readable before update"),
4790 tags,
4791 "target object must start without the target tag"
4792 );
4793 let selected_pool = store
4794 .get_pool_idx_existing_with_opts(
4795 &bucket,
4796 object,
4797 &ObjectOptions {
4798 no_lock: true,
4799 metadata_chg: true,
4800 skip_decommissioned: true,
4801 skip_rebalancing: true,
4802 ..Default::default()
4803 },
4804 )
4805 .await
4806 .expect("writer lookup should select an existing non-rebalancing pool");
4807 assert_eq!(selected_pool, 1, "writer lookup must skip active rebalance pool 0");
4808
4809 let updated = store
4810 .put_object_tags(&bucket, object, tags, &ObjectOptions::default())
4811 .await
4812 .expect("tag update should use the non-rebalancing target pool");
4813 assert_eq!(
4814 updated.mod_time,
4815 Some(old_time),
4816 "tag update must return the non-rebalancing pool object rather than the newer active source"
4817 );
4818
4819 let target_tags = store.pools[1]
4820 .get_object_tags(&bucket, object, &ObjectOptions::default())
4821 .await
4822 .expect("target object tags should be readable");
4823 assert_eq!(target_tags, tags, "non-rebalancing pool must receive writer tag updates");
4824
4825 shutdown.cancel();
4826 }
4827
4828 #[tokio::test]
4829 #[serial_test::serial(storage_class_env)]
4830 async fn multipart_listing_skips_active_rebalance_source_pool() {
4831 let temp_dir = tempfile::tempdir().expect("create multipart writer-fencing store dir");
4832 let (_ctx, store, shutdown) =
4833 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "writer-fencing-multipart", &[4, 4])).await;
4834 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
4835
4836 let bucket = format!("writer-fencing-multipart-{}", uuid::Uuid::new_v4());
4837 store
4838 .make_bucket(&bucket, &MakeBucketOptions::default())
4839 .await
4840 .expect("create multipart writer fencing bucket");
4841
4842 let incarnation = store.bucket_incarnation_id(&bucket).await.expect("read bucket incarnation");
4843 let lifecycle_guard = store
4844 .acquire_bucket_lifecycle_read_lock(&bucket)
4845 .await
4846 .expect("acquire multipart test lifecycle fence");
4847 let mut upload_opts = ObjectOptions {
4848 expected_bucket_incarnation_id: Some(incarnation),
4849 ..Default::default()
4850 };
4851 upload_opts.add_bucket_lifecycle_lock_guard(&lifecycle_guard);
4852 let source_upload = store.pools[0]
4853 .new_multipart_upload(&bucket, "source-only.bin", &upload_opts)
4854 .await
4855 .expect("create source upload");
4856 let target_upload = store.pools[1]
4857 .new_multipart_upload(&bucket, "target-visible.bin", &upload_opts)
4858 .await
4859 .expect("create target upload");
4860
4861 *store.rebalance_meta.write().await = Some(active_rebalance_meta_for_pool(store.pools.len(), 0));
4862 assert!(store.is_pool_rebalancing(0).await, "pool 0 must be marked as an active rebalance source");
4863
4864 let listed = store
4865 .list_multipart_uploads(&bucket, "", None, None, None, 100)
4866 .await
4867 .expect("list multipart uploads");
4868 let listed_uploads: Vec<(&str, &str)> = listed
4869 .uploads
4870 .iter()
4871 .map(|upload| (upload.object.as_str(), upload.upload_id.as_str()))
4872 .collect();
4873
4874 assert!(
4875 !listed_uploads.contains(&("source-only.bin", source_upload.upload_id.as_str())),
4876 "active source pool upload must be hidden from multipart listing"
4877 );
4878 assert!(
4879 listed_uploads.contains(&("target-visible.bin", target_upload.upload_id.as_str())),
4880 "non-rebalancing pool upload must remain visible"
4881 );
4882
4883 shutdown.cancel();
4884 }
4885
4886 #[tokio::test]
4887 #[serial_test::serial(storage_class_env)]
4888 async fn suspended_decommission_source_multipart_remains_operable_until_drained() {
4889 let temp_dir = tempfile::tempdir().expect("create decommission multipart drain store dir");
4890 let (_ctx, store, shutdown) =
4891 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "decommission-multipart-drain", &[4, 4])).await;
4892 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
4893
4894 let bucket = format!("decommission-mp-drain-{}", uuid::Uuid::new_v4());
4895 let complete_object = "complete.bin";
4896 let abort_object = "abort.bin";
4897 store
4898 .make_bucket(&bucket, &MakeBucketOptions::default())
4899 .await
4900 .expect("create decommission multipart drain bucket");
4901
4902 let incarnation = store.bucket_incarnation_id(&bucket).await.expect("read bucket incarnation");
4903 let lifecycle_guard = store
4904 .acquire_bucket_lifecycle_read_lock(&bucket)
4905 .await
4906 .expect("acquire multipart creation lifecycle fence");
4907 let mut upload_opts = ObjectOptions {
4908 expected_bucket_incarnation_id: Some(incarnation),
4909 ..Default::default()
4910 };
4911 upload_opts.add_bucket_lifecycle_lock_guard(&lifecycle_guard);
4912 let complete_upload = store.pools[0]
4913 .new_multipart_upload(&bucket, complete_object, &upload_opts)
4914 .await
4915 .expect("create source upload to complete");
4916 let abort_upload = store.pools[0]
4917 .new_multipart_upload(&bucket, abort_object, &upload_opts)
4918 .await
4919 .expect("create source upload to abort");
4920 drop(lifecycle_guard);
4921
4922 mark_test_pool_decommissioning(&store, 0).await;
4923 let err = store
4924 .ensure_decommission_multipart_uploads_drained_for_test(0)
4925 .await
4926 .expect_err("an unresolved source multipart upload must block final decommission");
4927 let drain_error = err.to_string();
4928 assert!(
4929 drain_error.contains("still contains multipart upload") && drain_error.contains(&bucket),
4930 "the drain error must identify both the upload path and user bucket: {drain_error}"
4931 );
4932
4933 let listed = store
4934 .list_multipart_uploads(&bucket, "", None, None, None, 100)
4935 .await
4936 .expect("list uploads from suspended decommission source");
4937 assert!(
4938 listed
4939 .uploads
4940 .iter()
4941 .any(|upload| upload.upload_id.as_str() == complete_upload.upload_id.as_str()),
4942 "the upload selected before suspension must remain visible"
4943 );
4944 store
4945 .get_multipart_info(&bucket, complete_object, &complete_upload.upload_id, &ObjectOptions::default())
4946 .await
4947 .expect("read upload metadata from suspended decommission source");
4948
4949 let mut part_reader = PutObjReader::from_vec(b"multipart body".to_vec());
4950 let part = store
4951 .put_object_part(
4952 &bucket,
4953 complete_object,
4954 &complete_upload.upload_id,
4955 1,
4956 &mut part_reader,
4957 &ObjectOptions::default(),
4958 )
4959 .await
4960 .expect("write part to suspended decommission source");
4961 let parts = store
4962 .list_object_parts(&bucket, complete_object, &complete_upload.upload_id, None, 100, &ObjectOptions::default())
4963 .await
4964 .expect("list parts from suspended decommission source");
4965 assert_eq!(parts.parts.len(), 1);
4966 assert_eq!(parts.parts[0].etag.as_deref(), part.etag.as_deref());
4967
4968 store
4969 .clone()
4970 .complete_multipart_upload(
4971 &bucket,
4972 complete_object,
4973 &complete_upload.upload_id,
4974 vec![crate::storage_api_contracts::multipart::CompletePart {
4975 part_num: part.part_num,
4976 etag: part.etag,
4977 ..Default::default()
4978 }],
4979 &ObjectOptions::default(),
4980 )
4981 .await
4982 .expect("complete upload on suspended decommission source");
4983 store
4984 .abort_multipart_upload(&bucket, abort_object, &abort_upload.upload_id, &ObjectOptions::default())
4985 .await
4986 .expect("abort upload on suspended decommission source");
4987
4988 tokio::time::timeout(Duration::from_secs(5), async {
4989 loop {
4990 match store.ensure_decommission_multipart_uploads_drained_for_test(0).await {
4991 Ok(()) => break,
4992 Err(err) if err.to_string().contains("still contains multipart upload") => {
4993 tokio::time::sleep(Duration::from_millis(10)).await;
4994 }
4995 Err(err) => panic!("unexpected final decommission drain error: {err:?}"),
4996 }
4997 }
4998 })
4999 .await
5000 .expect("final decommission gate should open after upload cleanup converges");
5001 assert_pool_object_present(&store.pools[0], &bucket, complete_object).await;
5002
5003 shutdown.cancel();
5004 }
5005
5006 #[tokio::test]
5007 #[serial_test::serial(storage_class_env)]
5008 async fn active_multipart_upload_routes_before_faulted_suspended_source() {
5009 use sha2::Digest;
5010
5011 let temp_dir = tempfile::tempdir().expect("create active-first multipart routing store dir");
5012 let (_ctx, store, shutdown) =
5013 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "active-first-multipart-routing", &[4, 4]))
5014 .await;
5015 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
5016
5017 let bucket = format!("active-first-mp-route-{}", uuid::Uuid::new_v4());
5018 let object = "target-upload.bin";
5019 store
5020 .make_bucket(&bucket, &MakeBucketOptions::default())
5021 .await
5022 .expect("create active-first multipart routing bucket");
5023
5024 let incarnation = store.bucket_incarnation_id(&bucket).await.expect("read bucket incarnation");
5025 let lifecycle_guard = store
5026 .acquire_bucket_lifecycle_read_lock(&bucket)
5027 .await
5028 .expect("acquire multipart creation lifecycle fence");
5029 let mut upload_opts = ObjectOptions {
5030 expected_bucket_incarnation_id: Some(incarnation),
5031 ..Default::default()
5032 };
5033 upload_opts.add_bucket_lifecycle_lock_guard(&lifecycle_guard);
5034 let upload = store.pools[1]
5035 .new_multipart_upload(&bucket, object, &upload_opts)
5036 .await
5037 .expect("create upload in active target pool");
5038 drop(lifecycle_guard);
5039
5040 mark_test_pool_decommissioning(&store, 0).await;
5041 let upload_sha =
5045 hex_simd::encode_to_string(sha2::Sha256::digest(format!("{bucket}/{object}").as_bytes()), hex_simd::AsciiCase::Lower);
5046 let upload_uuid = crate::runtime::sources::upload_uuid_suffix(&upload.upload_id);
5047 for disk_index in 0..4 {
5048 let metadata_path = temp_dir
5049 .path()
5050 .join(format!("pool0/set0/disk{disk_index}"))
5051 .join(crate::disk::RUSTFS_META_MULTIPART_BUCKET)
5052 .join(&upload_sha)
5053 .join(&upload_uuid)
5054 .join(crate::disk::STORAGE_FORMAT_FILE);
5055 tokio::fs::create_dir_all(metadata_path.parent().expect("multipart metadata path should have a parent"))
5056 .await
5057 .expect("create corrupt source upload directory");
5058 tokio::fs::write(metadata_path, b"not-xl-meta")
5059 .await
5060 .expect("inject source upload metadata read failure");
5061 }
5062
5063 let source_result = store.pools[0]
5064 .get_multipart_info(&bucket, object, &upload.upload_id, &ObjectOptions::default())
5065 .await;
5066 let routed_result = store
5067 .get_multipart_info(&bucket, object, &upload.upload_id, &ObjectOptions::default())
5068 .await;
5069
5070 assert!(
5071 source_result
5072 .as_ref()
5073 .is_err_and(|err| !crate::error::is_err_invalid_upload_id(err)),
5074 "the suspended source must expose the injected hard read failure: {source_result:?}"
5075 );
5076 let routed = routed_result.expect("the active target UploadID must be resolved before the faulted suspended source");
5077 assert_eq!(routed.upload_id, upload.upload_id);
5078
5079 shutdown.cancel();
5080 }
5081
5082 #[tokio::test]
5083 #[serial_test::serial(storage_class_env)]
5084 async fn delete_objects_skips_active_rebalance_source_pool() {
5085 let temp_dir = tempfile::tempdir().expect("create batch-delete writer-fencing store dir");
5086 let (_ctx, store, shutdown) =
5087 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "batch-delete-rebalance", &[4, 4])).await;
5088 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
5089
5090 let bucket = format!("batch-delete-rebalance-{}", uuid::Uuid::new_v4());
5091 let object = "delete-me.bin";
5092 store
5093 .make_bucket(&bucket, &MakeBucketOptions::default())
5094 .await
5095 .expect("create batch delete rebalance bucket");
5096
5097 let mut source_reader = PutObjReader::from_vec(b"source-body".to_vec());
5098 store.pools[0]
5099 .put_object(&bucket, object, &mut source_reader, &ObjectOptions::default())
5100 .await
5101 .expect("write object on active source pool");
5102 let mut target_reader = PutObjReader::from_vec(b"target-body".to_vec());
5103 store.pools[1]
5104 .put_object(&bucket, object, &mut target_reader, &ObjectOptions::default())
5105 .await
5106 .expect("write object on non-rebalancing target pool");
5107
5108 let mut pool_stats = vec![RebalanceStats::default(); store.pools.len()];
5109 pool_stats[0] = RebalanceStats {
5110 participating: true,
5111 info: RebalanceInfo {
5112 start_time: Some(OffsetDateTime::now_utc()),
5113 status: RebalStatus::Started,
5114 ..Default::default()
5115 },
5116 ..Default::default()
5117 };
5118 *store.rebalance_meta.write().await = Some(RebalanceMeta {
5119 id: uuid::Uuid::new_v4().to_string(),
5120 pool_stats,
5121 ..Default::default()
5122 });
5123 assert!(store.is_pool_rebalancing(0).await, "pool 0 must be marked as an active rebalance source");
5124
5125 let (deleted, errs) = store
5126 .delete_objects(
5127 &bucket,
5128 vec![crate::storage_api_contracts::object::ObjectToDelete {
5129 object_name: object.to_string(),
5130 ..Default::default()
5131 }],
5132 ObjectOptions::default(),
5133 )
5134 .await;
5135 assert!(matches!(errs.as_slice(), [None]), "batch delete must not fail: {errs:?}");
5136 assert!(
5137 matches!(deleted.as_slice(), [deleted] if deleted.found && deleted.object_name == object),
5138 "batch delete must report the non-rebalancing pool deletion"
5139 );
5140
5141 store.pools[0]
5142 .get_object_info(&bucket, object, &ObjectOptions::default())
5143 .await
5144 .expect("active source pool object must not be deleted by DeleteObjects");
5145 let target_err = store.pools[1]
5146 .get_object_info(&bucket, object, &ObjectOptions::default())
5147 .await
5148 .expect_err("non-rebalancing target pool object must be deleted");
5149 assert!(
5150 matches!(target_err, StorageError::ObjectNotFound(_, _)),
5151 "target pool should report object not found after DeleteObjects, got {target_err:?}"
5152 );
5153
5154 shutdown.cancel();
5155 }
5156
5157 #[test]
5158 #[serial_test::serial(storage_class_env)]
5159 fn data_movement_conflicts_and_multipart_retries_converge_safely() {
5160 std::thread::Builder::new()
5163 .stack_size(16 * 1024 * 1024)
5164 .spawn(|| {
5165 let runtime = tokio::runtime::Builder::new_multi_thread()
5166 .enable_all()
5167 .worker_threads(2)
5168 .build()
5169 .expect("build data movement test runtime");
5170 runtime.block_on(async move {
5171 let temp_dir = tempfile::tempdir().expect("create data movement store dir");
5172 let (_ctx, store, _shutdown) = without_storage_class_env(build_isolated_test_store(
5173 temp_dir.path(),
5174 "data-movement-conflict-convergence",
5175 &[4, 4],
5176 ))
5177 .await;
5178 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
5179
5180 let bucket = format!("data-movement-conflict-{}", uuid::Uuid::new_v4());
5181 store
5182 .make_bucket(&bucket, &MakeBucketOptions::default())
5183 .await
5184 .expect("create data movement bucket");
5185 let source_mod_time = OffsetDateTime::UNIX_EPOCH + time::Duration::SECOND;
5186 let target_mod_time = source_mod_time + time::Duration::SECOND;
5187
5188 let object = "single-object";
5189 let target_body = b"newer client body".to_vec();
5190 let mut target_reader = PutObjReader::from_vec(target_body.clone());
5191 store.pools[1]
5192 .put_object(
5193 &bucket,
5194 object,
5195 &mut target_reader,
5196 &ObjectOptions {
5197 mod_time: Some(target_mod_time),
5198 ..Default::default()
5199 },
5200 )
5201 .await
5202 .expect("write newer single-part target");
5203
5204 let source_body = b"stale migration body".to_vec();
5205 let source_reader = seed_data_movement_source_reader(
5206 &store,
5207 &bucket,
5208 object,
5209 DataMovementSourceSeed {
5210 pool_idx: 0,
5211 version_id: None,
5212 body: source_body,
5213 multipart_split: None,
5214 mod_time: source_mod_time,
5215 user_defined: HashMap::new(),
5216 },
5217 )
5218 .await;
5219 crate::data_movement::migrate_object(
5220 store.clone(),
5221 0,
5222 bucket.clone(),
5223 source_reader,
5224 None,
5225 "test_data_movement",
5226 )
5227 .await
5228 .expect("newer single-part target should converge migration");
5229
5230 let mut reader = store.pools[1]
5231 .get_object_reader(&bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
5232 .await
5233 .expect("read converged single-part target");
5234 let mut body = Vec::new();
5235 reader.stream.read_to_end(&mut body).await.expect("drain single-part target");
5236 assert_eq!(body, target_body);
5237
5238 let multipart_object = "multipart-object";
5239 let multipart_target_body = b"newer multipart client body".to_vec();
5240 let mut multipart_target_reader = PutObjReader::from_vec(multipart_target_body.clone());
5241 store.pools[1]
5242 .put_object(
5243 &bucket,
5244 multipart_object,
5245 &mut multipart_target_reader,
5246 &ObjectOptions {
5247 mod_time: Some(target_mod_time),
5248 ..Default::default()
5249 },
5250 )
5251 .await
5252 .expect("write newer multipart target");
5253
5254 let first_part_size = 5 * 1024 * 1024;
5255 let mut multipart_source_body = vec![b'a'; first_part_size];
5256 multipart_source_body.push(b'b');
5257 let multipart_source_reader = seed_data_movement_source_reader(
5258 &store,
5259 &bucket,
5260 multipart_object,
5261 DataMovementSourceSeed {
5262 pool_idx: 0,
5263 version_id: None,
5264 body: multipart_source_body,
5265 multipart_split: Some(first_part_size),
5266 mod_time: source_mod_time,
5267 user_defined: HashMap::new(),
5268 },
5269 )
5270 .await;
5271 crate::data_movement::migrate_object(
5272 store.clone(),
5273 0,
5274 bucket.clone(),
5275 multipart_source_reader,
5276 None,
5277 "test_data_movement",
5278 )
5279 .await
5280 .expect("newer multipart target should converge migration");
5281
5282 let uploads = store.pools[1]
5283 .list_multipart_uploads(&bucket, multipart_object, None, None, None, 100)
5284 .await
5285 .expect("list target pool multipart uploads");
5286 assert!(uploads.uploads.is_empty(), "superseded migration staging must be aborted");
5287
5288 let mut reader = store.pools[1]
5289 .get_object_reader(&bucket, multipart_object, None, HeaderMap::new(), &ObjectOptions::default())
5290 .await
5291 .expect("read converged multipart target");
5292 let mut body = Vec::new();
5293 reader.stream.read_to_end(&mut body).await.expect("drain multipart target");
5294 assert_eq!(body, multipart_target_body);
5295
5296 let retry_object = "multipart-retry-object";
5297 let retry_object_etag = "0123456789abcdef0123456789abcdef".to_string();
5298 let retry_first_part_size = 5 * 1024 * 1024;
5299 let retry_object_mod_time =
5300 OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("fixed retry timestamp should be valid");
5301 let mut retry_source_body = vec![b'c'; retry_first_part_size];
5302 retry_source_body.push(b'd');
5303 let retry_first_checksum =
5304 Checksum::new_from_data(ChecksumType::CRC32C, &retry_source_body[..retry_first_part_size])
5305 .expect("first source part checksum should be created");
5306 let retry_second_checksum =
5307 Checksum::new_from_data(ChecksumType::CRC32C, &retry_source_body[retry_first_part_size..])
5308 .expect("second source part checksum should be created");
5309 let mut retry_checksum_parts = retry_first_checksum.raw.clone();
5310 retry_checksum_parts.extend_from_slice(&retry_second_checksum.raw);
5311 let mut retry_checksum_type = ChecksumType::CRC32C;
5312 retry_checksum_type
5313 .merge(ChecksumType::MULTIPART)
5314 .merge(ChecksumType::INCLUDES_MULTIPART);
5315 let retry_object_checksum = Checksum::new_from_data(retry_checksum_type, &retry_checksum_parts)
5316 .expect("source multipart checksum should be created");
5317 let retry_object_checksum_bytes = retry_object_checksum.to_bytes(&retry_checksum_parts);
5318 let mut retry_metadata = HashMap::from([("x-amz-meta-retry".to_string(), "stable".to_string())]);
5319 insert_str(
5320 &mut retry_metadata,
5321 SUFFIX_COMPRESSION,
5322 compression_metadata_value(CompressionAlgorithm::default()),
5323 );
5324 let retry_upload = store.pools[0]
5325 .new_multipart_upload(
5326 &bucket,
5327 retry_object,
5328 &ObjectOptions {
5329 user_defined: retry_metadata,
5330 want_checksum: Some(retry_object_checksum.clone()),
5331 ..Default::default()
5332 },
5333 )
5334 .await
5335 .expect("create source multipart upload");
5336 let mut retry_parts = Vec::with_capacity(2);
5337 for (part_number, plaintext, checksum) in [
5338 (1, &retry_source_body[..retry_first_part_size], &retry_first_checksum),
5339 (2, &retry_source_body[retry_first_part_size..], &retry_second_checksum),
5340 ] {
5341 let part_size = i64::try_from(plaintext.len()).expect("source part size should fit i64");
5342 let mut plaintext_reader = rustfs_rio::HashReader::from_stream(
5343 Cursor::new(plaintext.to_vec()),
5344 part_size,
5345 part_size,
5346 None,
5347 None,
5348 false,
5349 )
5350 .expect("create source part plaintext reader");
5351 plaintext_reader
5352 .add_non_trailing_checksum(Some(checksum.clone()), false)
5353 .expect("set source part checksum");
5354 let compressed = WritePlan::new()
5355 .with_compression(CompressionAlgorithm::default())
5356 .apply(plaintext_reader, part_size)
5357 .expect("compress source part");
5358 let mut reader = PutObjReader::new(compressed);
5359 let part = store.pools[0]
5360 .put_object_part(
5361 &bucket,
5362 retry_object,
5363 &retry_upload.upload_id,
5364 part_number,
5365 &mut reader,
5366 &ObjectOptions::default(),
5367 )
5368 .await
5369 .expect("write source multipart part");
5370 retry_parts.push(crate::storage_api_contracts::multipart::CompletePart {
5371 part_num: part.part_num,
5372 etag: part.etag,
5373 checksum_crc32c: Some(checksum.encoded.clone()),
5374 ..Default::default()
5375 });
5376 }
5377 store.pools[0]
5378 .clone()
5379 .complete_multipart_upload(
5380 &bucket,
5381 retry_object,
5382 &retry_upload.upload_id,
5383 retry_parts,
5384 &ObjectOptions {
5385 mod_time: Some(retry_object_mod_time),
5386 want_checksum: Some(retry_object_checksum),
5387 preserve_etag: Some(retry_object_etag.clone()),
5388 ..Default::default()
5389 },
5390 )
5391 .await
5392 .expect("complete source multipart object");
5393
5394 *store.rebalance_meta.write().await = Some(active_rebalance_meta_for_pool(store.pools.len(), 0));
5395 assert!(store.is_pool_rebalancing(0).await, "pool 0 must be marked as an active rebalance source");
5396
5397 let mut retry_reader = store.pools[0]
5398 .get_object_reader(
5399 &bucket,
5400 retry_object,
5401 None,
5402 HeaderMap::new(),
5403 &ObjectOptions {
5404 raw_data_movement_read: true,
5405 ..Default::default()
5406 },
5407 )
5408 .await
5409 .expect("read source multipart object for first migration");
5410 let mut retry_source_info = retry_reader.object_info.clone();
5411 let mut retry_source_parts = retry_source_info.parts.as_ref().clone();
5412 for (part, checksum) in retry_source_parts
5413 .iter_mut()
5414 .zip([&retry_first_checksum, &retry_second_checksum])
5415 {
5416 part.mod_time = Some(OffsetDateTime::UNIX_EPOCH);
5417 part.checksums = Some(HashMap::from([(ChecksumType::CRC32C.to_string(), checksum.encoded.clone())]));
5418 }
5419 retry_source_info.parts = Arc::new(retry_source_parts);
5420 assert_eq!(retry_source_info.etag.as_deref(), Some(retry_object_etag.as_str()));
5421 assert!(retry_source_info.is_multipart());
5422 assert!(retry_source_info.parts.iter().all(|part| part.checksums.is_some()));
5423 assert_eq!(retry_source_info.checksum.as_deref(), Some(retry_object_checksum_bytes.as_ref()));
5424 assert!(
5425 !retry_source_info
5426 .user_defined
5427 .contains_key(rustfs_rio::RUSTFS_MULTIPART_CHECKSUM)
5428 );
5429 assert!(
5430 !retry_source_info
5431 .user_defined
5432 .contains_key(rustfs_rio::RUSTFS_MULTIPART_CHECKSUM_TYPE)
5433 );
5434 retry_reader.object_info = retry_source_info.clone();
5435 temp_env::async_with_vars(
5436 [
5437 (rustfs_config::ENV_DATA_MOVEMENT_PART_CHECKSUMS_WRITE, None::<&str>),
5438 (rustfs_config::ENV_DATA_MOVEMENT_PART_CHECKSUMS_FLEET_CONFIRMED, None::<&str>),
5439 ],
5440 crate::data_movement::migrate_object(
5441 store.clone(),
5442 0,
5443 bucket.clone(),
5444 retry_reader,
5445 None,
5446 "test_data_movement_compatible_retry",
5447 ),
5448 )
5449 .await
5450 .expect("default multipart migration should not require the checksum sidecar capability");
5451
5452 let compatible_target = store.pools[1]
5453 .get_object_info(
5454 &bucket,
5455 retry_object,
5456 &ObjectOptions {
5457 include_part_checksums: true,
5458 ..Default::default()
5459 },
5460 )
5461 .await
5462 .expect("read compatible multipart target metadata");
5463 assert_eq!(compatible_target.checksum, retry_source_info.checksum);
5464 assert!(compatible_target.parts.iter().all(|part| part.checksums.is_none()));
5465 assert_eq!(
5466 rustfs_utils::http::get_consistent_str(
5467 &compatible_target.user_defined,
5468 rustfs_utils::http::SUFFIX_DATA_MOVED
5469 ),
5470 Some("true")
5471 );
5472 assert!(!rustfs_utils::http::contains_key_str(
5473 &compatible_target.user_defined,
5474 rustfs_utils::http::SUFFIX_PART_CHECKSUMS
5475 ));
5476 let mut compatible_reader = store.pools[1]
5477 .get_object_reader(&bucket, retry_object, None, HeaderMap::new(), &ObjectOptions::default())
5478 .await
5479 .expect("read compatible multipart target body");
5480 let mut compatible_body = Vec::new();
5481 compatible_reader
5482 .stream
5483 .read_to_end(&mut compatible_body)
5484 .await
5485 .expect("drain compatible multipart target body");
5486 assert_eq!(compatible_body, retry_source_body);
5487
5488 let mut compatible_retry_reader = store.pools[0]
5489 .get_object_reader(
5490 &bucket,
5491 retry_object,
5492 None,
5493 HeaderMap::new(),
5494 &ObjectOptions {
5495 raw_data_movement_read: true,
5496 ..Default::default()
5497 },
5498 )
5499 .await
5500 .expect("read source multipart object for compatible retry");
5501 compatible_retry_reader.object_info = retry_source_info.clone();
5502 temp_env::async_with_vars(
5503 [
5504 (rustfs_config::ENV_DATA_MOVEMENT_PART_CHECKSUMS_WRITE, Some("true")),
5505 (rustfs_config::ENV_DATA_MOVEMENT_PART_CHECKSUMS_FLEET_CONFIRMED, Some("true")),
5506 ],
5507 crate::data_movement::migrate_object(
5508 store.clone(),
5509 0,
5510 bucket.clone(),
5511 compatible_retry_reader,
5512 None,
5513 "test_data_movement_compatible_retry",
5514 ),
5515 )
5516 .await
5517 .expect("pre-gate multipart target should converge after enabling checksum persistence");
5518 let compatible_target_after_retry = store.pools[1]
5519 .get_object_info(
5520 &bucket,
5521 retry_object,
5522 &ObjectOptions {
5523 include_part_checksums: true,
5524 ..Default::default()
5525 },
5526 )
5527 .await
5528 .expect("read pre-gate target after fleet-confirmed retry");
5529 assert_eq!(compatible_target_after_retry.data_dir, compatible_target.data_dir);
5530 assert_eq!(compatible_target_after_retry.parts, compatible_target.parts);
5531 let compatible_uploads = store.pools[1]
5532 .list_multipart_uploads(&bucket, retry_object, None, None, None, 100)
5533 .await
5534 .expect("list compatible target multipart uploads");
5535 assert!(compatible_uploads.uploads.is_empty(), "compatible retry staging must be aborted");
5536
5537 store.pools[1]
5538 .delete_object(&bucket, retry_object, ObjectOptions::default())
5539 .await
5540 .expect("remove compatible target before fleet-confirmed migration");
5541 let mut retry_reader = store.pools[0]
5542 .get_object_reader(
5543 &bucket,
5544 retry_object,
5545 None,
5546 HeaderMap::new(),
5547 &ObjectOptions {
5548 raw_data_movement_read: true,
5549 ..Default::default()
5550 },
5551 )
5552 .await
5553 .expect("read source multipart object for fleet-confirmed migration");
5554 retry_reader.object_info = retry_source_info.clone();
5555 temp_env::async_with_vars(
5556 [
5557 (rustfs_config::ENV_DATA_MOVEMENT_PART_CHECKSUMS_WRITE, Some("true")),
5558 (rustfs_config::ENV_DATA_MOVEMENT_PART_CHECKSUMS_FLEET_CONFIRMED, Some("true")),
5559 ],
5560 crate::data_movement::migrate_object(
5561 store.clone(),
5562 0,
5563 bucket.clone(),
5564 retry_reader,
5565 None,
5566 "test_data_movement_retry",
5567 ),
5568 )
5569 .await
5570 .expect("first multipart migration should succeed");
5571
5572 let target_before_retry = store.pools[1]
5573 .get_object_info(
5574 &bucket,
5575 retry_object,
5576 &ObjectOptions {
5577 include_part_checksums: true,
5578 ..Default::default()
5579 },
5580 )
5581 .await
5582 .expect("read target metadata before retry");
5583 assert_eq!(target_before_retry.checksum, retry_source_info.checksum);
5584 assert_eq!(
5585 target_before_retry
5586 .parts
5587 .iter()
5588 .map(|part| (&part.number, &part.checksums))
5589 .collect::<Vec<_>>(),
5590 retry_source_info
5591 .parts
5592 .iter()
5593 .map(|part| (&part.number, &part.checksums))
5594 .collect::<Vec<_>>()
5595 );
5596 assert_eq!(retry_source_info.mod_time, target_before_retry.mod_time);
5597 assert!(
5598 retry_source_info.parts.iter().any(|source_part| {
5599 target_before_retry
5600 .parts
5601 .iter()
5602 .find(|target_part| target_part.number == source_part.number)
5603 .is_some_and(|target_part| target_part.mod_time != source_part.mod_time)
5604 }),
5605 "multipart migration must rewrite at least one target part timestamp"
5606 );
5607 let mut retry_reader = store.pools[0]
5608 .get_object_reader(
5609 &bucket,
5610 retry_object,
5611 None,
5612 HeaderMap::new(),
5613 &ObjectOptions {
5614 raw_data_movement_read: true,
5615 ..Default::default()
5616 },
5617 )
5618 .await
5619 .expect("read source multipart object for retry");
5620 retry_reader.object_info = retry_source_info;
5621 temp_env::async_with_vars(
5622 [
5623 (rustfs_config::ENV_DATA_MOVEMENT_PART_CHECKSUMS_WRITE, Some("true")),
5624 (rustfs_config::ENV_DATA_MOVEMENT_PART_CHECKSUMS_FLEET_CONFIRMED, Some("true")),
5625 ],
5626 crate::data_movement::migrate_object(
5627 store.clone(),
5628 0,
5629 bucket.clone(),
5630 retry_reader,
5631 None,
5632 "test_data_movement_retry",
5633 ),
5634 )
5635 .await
5636 .expect("equivalent multipart migration retry should converge");
5637
5638 let target_after_retry = store.pools[1]
5639 .get_object_info(&bucket, retry_object, &ObjectOptions::default())
5640 .await
5641 .expect("read target metadata after retry");
5642 assert_eq!(target_after_retry.data_dir, target_before_retry.data_dir);
5643 assert_eq!(target_after_retry.etag, target_before_retry.etag);
5644 assert_eq!(target_after_retry.checksum, target_before_retry.checksum);
5645 assert_eq!(target_after_retry.mod_time, target_before_retry.mod_time);
5646 assert!(!rustfs_utils::http::contains_key_str(
5647 &target_after_retry.user_defined,
5648 rustfs_utils::http::SUFFIX_PART_CHECKSUMS
5649 ));
5650 assert!(target_after_retry.parts.iter().all(|part| part.checksums.is_none()));
5651 let cached_target_after_retry = store.pools[1]
5652 .get_object_info(&bucket, retry_object, &ObjectOptions::default())
5653 .await
5654 .expect("read target metadata from the ordinary cache path");
5655 assert!(!rustfs_utils::http::contains_key_str(
5656 &cached_target_after_retry.user_defined,
5657 rustfs_utils::http::SUFFIX_PART_CHECKSUMS
5658 ));
5659 assert!(cached_target_after_retry.parts.iter().all(|part| part.checksums.is_none()));
5660 let hydrated_target_after_retry = store.pools[1]
5661 .get_object_info(
5662 &bucket,
5663 retry_object,
5664 &ObjectOptions {
5665 include_part_checksums: true,
5666 ..Default::default()
5667 },
5668 )
5669 .await
5670 .expect("hydrate target metadata after ordinary cache reads");
5671 assert_eq!(hydrated_target_after_retry.user_defined, target_before_retry.user_defined);
5672 assert_eq!(hydrated_target_after_retry.parts, target_before_retry.parts);
5673
5674 let mut target_reader = store.pools[1]
5675 .get_object_reader(&bucket, retry_object, None, HeaderMap::new(), &ObjectOptions::default())
5676 .await
5677 .expect("read target multipart body after retry");
5678 let mut target_body = Vec::new();
5679 target_reader
5680 .stream
5681 .read_to_end(&mut target_body)
5682 .await
5683 .expect("drain target multipart body after retry");
5684 assert_eq!(target_body, retry_source_body);
5685
5686 let uploads = store.pools[1]
5687 .list_multipart_uploads(&bucket, retry_object, None, None, None, 100)
5688 .await
5689 .expect("list retry target multipart uploads");
5690 assert!(uploads.uploads.is_empty(), "retry migration staging must be aborted");
5691
5692 Box::pin(async {
5693 store.pools[0]
5694 .delete_object(&bucket, retry_object, ObjectOptions::default())
5695 .await
5696 .expect("remove first-hop source before second migration");
5697 *store.rebalance_meta.write().await = Some(active_rebalance_meta_for_pool(store.pools.len(), 1));
5698 let second_hop_reader = store.pools[1]
5699 .get_object_reader(
5700 &bucket,
5701 retry_object,
5702 None,
5703 HeaderMap::new(),
5704 &ObjectOptions {
5705 data_movement: true,
5706 raw_data_movement_read: true,
5707 ..Default::default()
5708 },
5709 )
5710 .await
5711 .expect("read first target as the second migration source");
5712 assert!(
5713 second_hop_reader
5714 .object_info
5715 .parts
5716 .iter()
5717 .all(|part| part.checksums.is_some()),
5718 "data-movement source reads must hydrate persisted part checksums"
5719 );
5720 temp_env::async_with_vars(
5721 [
5722 (rustfs_config::ENV_DATA_MOVEMENT_PART_CHECKSUMS_WRITE, Some("true")),
5723 (rustfs_config::ENV_DATA_MOVEMENT_PART_CHECKSUMS_FLEET_CONFIRMED, Some("true")),
5724 ],
5725 crate::data_movement::migrate_object(
5726 store.clone(),
5727 1,
5728 bucket.clone(),
5729 second_hop_reader,
5730 None,
5731 "test_data_movement_second_hop",
5732 ),
5733 )
5734 .await
5735 .expect("second multipart migration should preserve part checksums");
5736
5737 let second_hop_target = store.pools[0]
5738 .get_object_info(
5739 &bucket,
5740 retry_object,
5741 &ObjectOptions {
5742 include_part_checksums: true,
5743 ..Default::default()
5744 },
5745 )
5746 .await
5747 .expect("read second-hop target metadata");
5748 assert_eq!(second_hop_target.checksum, target_before_retry.checksum);
5749 assert_eq!(
5750 second_hop_target
5751 .parts
5752 .iter()
5753 .map(|part| (&part.number, &part.checksums))
5754 .collect::<Vec<_>>(),
5755 target_before_retry
5756 .parts
5757 .iter()
5758 .map(|part| (&part.number, &part.checksums))
5759 .collect::<Vec<_>>()
5760 );
5761 })
5762 .await;
5763 });
5764 })
5765 .expect("spawn data movement test thread")
5766 .join()
5767 .expect("join data movement test thread");
5768 }
5769
5770 #[test]
5771 #[serial_test::serial(storage_class_env)]
5772 fn data_movement_multipart_replaces_only_unlocked_owned_generation() {
5773 std::thread::Builder::new()
5774 .stack_size(16 * 1024 * 1024)
5775 .spawn(|| {
5776 let runtime = tokio::runtime::Builder::new_multi_thread()
5777 .enable_all()
5778 .worker_threads(2)
5779 .build()
5780 .expect("build data movement replacement test runtime");
5781 runtime.block_on(async move {
5782 let temp_dir = tempfile::tempdir().expect("create data movement replacement store dir");
5783 let (_ctx, store, _shutdown) = without_storage_class_env(build_isolated_test_store(
5784 temp_dir.path(),
5785 "data-movement-stale-replacement",
5786 &[4, 4],
5787 ))
5788 .await;
5789 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
5790
5791 let bucket = format!("dm-stale-replacement-{}", uuid::Uuid::new_v4());
5792 store
5793 .make_bucket(
5794 &bucket,
5795 &MakeBucketOptions {
5796 lock_enabled: true,
5797 ..Default::default()
5798 },
5799 )
5800 .await
5801 .expect("create data movement replacement bucket");
5802 *store.rebalance_meta.write().await = Some(active_rebalance_meta_for_pool(store.pools.len(), 0));
5803
5804 let version_id = uuid::Uuid::new_v4();
5805 let old_time = OffsetDateTime::UNIX_EPOCH + time::Duration::SECOND;
5806 let new_time = old_time + time::Duration::SECOND;
5807 let first_part_size = 5 * 1024 * 1024;
5808 let mut old_body = vec![b'a'; first_part_size];
5809 old_body.push(b'b');
5810 let mut new_body = vec![b'c'; first_part_size];
5811 new_body.push(b'd');
5812
5813 let source_reader = |name: &'static str, body: Vec<u8>, mod_time, metadata: HashMap<String, String>| {
5814 seed_data_movement_source_reader(
5815 &store,
5816 &bucket,
5817 name,
5818 DataMovementSourceSeed {
5819 pool_idx: 0,
5820 version_id: Some(version_id),
5821 body,
5822 multipart_split: Some(first_part_size),
5823 mod_time,
5824 user_defined: metadata,
5825 },
5826 )
5827 };
5828
5829 let replaceable = "replaceable.bin";
5830 crate::data_movement::migrate_object(
5831 store.clone(),
5832 0,
5833 bucket.clone(),
5834 source_reader(
5835 replaceable,
5836 old_body.clone(),
5837 old_time,
5838 HashMap::from([("x-amz-meta-generation".to_string(), "old".to_string())]),
5839 )
5840 .await,
5841 None,
5842 "test_stale_target_seed",
5843 )
5844 .await
5845 .expect("seed old migrated target");
5846 let seeded_target = store.pools[1]
5847 .get_object_info(
5848 &bucket,
5849 replaceable,
5850 &ObjectOptions {
5851 versioned: true,
5852 version_id: Some(version_id.to_string()),
5853 ..Default::default()
5854 },
5855 )
5856 .await
5857 .expect("read old migrated target");
5858 let replacement_opts = ObjectOptions {
5859 data_movement: true,
5860 versioned: true,
5861 version_id: Some(version_id.to_string()),
5862 mod_time: Some(new_time),
5863 http_preconditions: Some(crate::data_movement::data_movement_target_precondition()),
5864 ..Default::default()
5865 };
5866 assert!(
5867 crate::data_movement::can_replace_stale_data_movement_target(&seeded_target, &replacement_opts),
5868 "seeded target should be replaceable: {seeded_target:?}"
5869 );
5870 crate::data_movement::migrate_object(
5871 store.clone(),
5872 0,
5873 bucket.clone(),
5874 source_reader(
5875 replaceable,
5876 new_body.clone(),
5877 new_time,
5878 HashMap::from([("x-amz-meta-generation".to_string(), "new".to_string())]),
5879 )
5880 .await,
5881 None,
5882 "test_stale_target_replace",
5883 )
5884 .await
5885 .expect("newer source generation should replace the old migrated target");
5886
5887 let replacement = store.pools[1]
5888 .get_object_info(
5889 &bucket,
5890 replaceable,
5891 &ObjectOptions {
5892 versioned: true,
5893 version_id: Some(version_id.to_string()),
5894 ..Default::default()
5895 },
5896 )
5897 .await
5898 .expect("read replaced target metadata");
5899 assert_eq!(replacement.mod_time, Some(new_time));
5900 assert_eq!(replacement.user_defined.get("x-amz-meta-generation").map(String::as_str), Some("new"));
5901 let mut replacement_reader = store.pools[1]
5902 .get_object_reader(
5903 &bucket,
5904 replaceable,
5905 None,
5906 HeaderMap::new(),
5907 &ObjectOptions {
5908 versioned: true,
5909 version_id: Some(version_id.to_string()),
5910 ..Default::default()
5911 },
5912 )
5913 .await
5914 .expect("read replaced target body");
5915 let mut replacement_body = Vec::new();
5916 replacement_reader
5917 .stream
5918 .read_to_end(&mut replacement_body)
5919 .await
5920 .expect("drain replaced target body");
5921 assert_eq!(replacement_body, new_body);
5922
5923 let client_target = "client-target.bin";
5924 let client_body = b"client-owned exact version".to_vec();
5925 let mut client_reader = PutObjReader::from_vec(client_body.clone());
5926 store.pools[1]
5927 .put_object(
5928 &bucket,
5929 client_target,
5930 &mut client_reader,
5931 &ObjectOptions {
5932 versioned: true,
5933 version_id: Some(version_id.to_string()),
5934 mod_time: Some(old_time),
5935 user_defined: HashMap::from([("x-amz-meta-owner".to_string(), "client".to_string())]),
5936 ..Default::default()
5937 },
5938 )
5939 .await
5940 .expect("seed client-owned exact version");
5941 let err = crate::data_movement::migrate_object(
5942 store.clone(),
5943 0,
5944 bucket.clone(),
5945 source_reader(client_target, new_body.clone(), new_time, HashMap::new()).await,
5946 None,
5947 "test_client_target_reject",
5948 )
5949 .await
5950 .expect_err("data movement must not replace a client-owned exact version");
5951 assert!(err.to_string().contains("complete_multipart_upload"), "unexpected migration error: {err}");
5952 let mut preserved_reader = store.pools[1]
5953 .get_object_reader(
5954 &bucket,
5955 client_target,
5956 None,
5957 HeaderMap::new(),
5958 &ObjectOptions {
5959 versioned: true,
5960 version_id: Some(version_id.to_string()),
5961 ..Default::default()
5962 },
5963 )
5964 .await
5965 .expect("read preserved client target");
5966 let mut preserved_body = Vec::new();
5967 preserved_reader
5968 .stream
5969 .read_to_end(&mut preserved_body)
5970 .await
5971 .expect("drain preserved client target");
5972 assert_eq!(preserved_body, client_body);
5973
5974 let acknowledged_target = "acknowledged-target.bin";
5975 crate::data_movement::migrate_object(
5976 store.clone(),
5977 0,
5978 bucket.clone(),
5979 source_reader(
5980 acknowledged_target,
5981 old_body.clone(),
5982 old_time,
5983 HashMap::from([("x-amz-meta-generation".to_string(), "old".to_string())]),
5984 )
5985 .await,
5986 None,
5987 "test_acknowledged_target_seed",
5988 )
5989 .await
5990 .expect("seed migrated target before tag acknowledgement");
5991 let target_version_opts = ObjectOptions {
5992 versioned: true,
5993 version_id: Some(version_id.to_string()),
5994 ..Default::default()
5995 };
5996 store
5997 .put_object_tags(&bucket, acknowledged_target, "acknowledged=true", &target_version_opts)
5998 .await
5999 .expect("tag update should acknowledge the migrated target");
6000 let acknowledged = store.pools[1]
6001 .get_object_info(&bucket, acknowledged_target, &target_version_opts)
6002 .await
6003 .expect("read acknowledged migrated target");
6004 let rustfs_data_moved_key = rustfs_utils::http::internal_key_rustfs(rustfs_utils::http::SUFFIX_DATA_MOVED);
6005 let minio_data_moved_key =
6006 format!("{}{}", rustfs_utils::http::MINIO_INTERNAL_PREFIX, rustfs_utils::http::SUFFIX_DATA_MOVED);
6007 assert_eq!(acknowledged.user_defined.get(&rustfs_data_moved_key).map(String::as_str), Some(""));
6008 assert_eq!(acknowledged.user_defined.get(&minio_data_moved_key).map(String::as_str), Some(""));
6009 assert_eq!(
6010 rustfs_utils::http::get_consistent_str(&acknowledged.user_defined, rustfs_utils::http::SUFFIX_DATA_MOVED),
6011 None
6012 );
6013
6014 let err = crate::data_movement::migrate_object(
6015 store.clone(),
6016 0,
6017 bucket.clone(),
6018 source_reader(
6019 acknowledged_target,
6020 new_body.clone(),
6021 new_time,
6022 HashMap::from([("x-amz-meta-generation".to_string(), "new".to_string())]),
6023 )
6024 .await,
6025 None,
6026 "test_acknowledged_target_reject",
6027 )
6028 .await
6029 .expect_err("data movement must not replace a target acknowledged by a tag update");
6030 assert!(err.to_string().contains("complete_multipart_upload"), "unexpected migration error: {err}");
6031
6032 let acknowledged_tags = store.pools[1]
6033 .get_object_tags(&bucket, acknowledged_target, &target_version_opts)
6034 .await
6035 .expect("read preserved acknowledged target tags");
6036 assert_eq!(acknowledged_tags, "acknowledged=true");
6037 let mut acknowledged_reader = store.pools[1]
6038 .get_object_reader(&bucket, acknowledged_target, None, HeaderMap::new(), &target_version_opts)
6039 .await
6040 .expect("read preserved acknowledged target body");
6041 let mut acknowledged_body = Vec::new();
6042 acknowledged_reader
6043 .stream
6044 .read_to_end(&mut acknowledged_body)
6045 .await
6046 .expect("drain preserved acknowledged target body");
6047 assert_eq!(acknowledged_body, old_body);
6048
6049 let metadata_acknowledged_target = "metadata-acknowledged-target.bin";
6050 crate::data_movement::migrate_object(
6051 store.clone(),
6052 0,
6053 bucket.clone(),
6054 source_reader(
6055 metadata_acknowledged_target,
6056 old_body.clone(),
6057 old_time,
6058 HashMap::from([("x-amz-meta-generation".to_string(), "old".to_string())]),
6059 )
6060 .await,
6061 None,
6062 "test_metadata_acknowledged_target_seed",
6063 )
6064 .await
6065 .expect("seed migrated target before metadata acknowledgement");
6066 store
6067 .put_object_metadata(
6068 &bucket,
6069 metadata_acknowledged_target,
6070 &ObjectOptions {
6071 eval_metadata: Some(HashMap::from([(
6072 rustfs_filemeta::metadata_keys::OBJECT_LOCK_LEGAL_HOLD.to_string(),
6073 s3s::dto::ObjectLockLegalHoldStatus::OFF.to_string(),
6074 )])),
6075 ..target_version_opts.clone()
6076 },
6077 )
6078 .await
6079 .expect("legal hold metadata update should acknowledge the migrated target");
6080 let metadata_acknowledged = store.pools[1]
6081 .get_object_info(&bucket, metadata_acknowledged_target, &target_version_opts)
6082 .await
6083 .expect("read metadata-acknowledged migrated target");
6084 assert_eq!(
6085 metadata_acknowledged
6086 .user_defined
6087 .get(&rustfs_data_moved_key)
6088 .map(String::as_str),
6089 Some("")
6090 );
6091 assert_eq!(
6092 metadata_acknowledged
6093 .user_defined
6094 .get(&minio_data_moved_key)
6095 .map(String::as_str),
6096 Some("")
6097 );
6098 assert_eq!(
6099 metadata_acknowledged
6100 .user_defined
6101 .get(rustfs_filemeta::metadata_keys::OBJECT_LOCK_LEGAL_HOLD)
6102 .map(String::as_str),
6103 Some("OFF")
6104 );
6105 let err = crate::data_movement::migrate_object(
6106 store.clone(),
6107 0,
6108 bucket.clone(),
6109 source_reader(
6110 metadata_acknowledged_target,
6111 new_body.clone(),
6112 new_time,
6113 HashMap::from([("x-amz-meta-generation".to_string(), "new".to_string())]),
6114 )
6115 .await,
6116 None,
6117 "test_metadata_acknowledged_target_reject",
6118 )
6119 .await
6120 .expect_err("data movement must not replace a target acknowledged by a metadata update");
6121 assert!(err.to_string().contains("complete_multipart_upload"), "unexpected migration error: {err}");
6122
6123 let retain_until = (OffsetDateTime::now_utc() + time::Duration::days(1))
6124 .format(&time::format_description::well_known::Rfc3339)
6125 .expect("retain-until date should format");
6126 for (object, mode) in [
6127 ("compliance-target.bin", s3s::dto::ObjectLockRetentionMode::COMPLIANCE),
6128 ("governance-target.bin", s3s::dto::ObjectLockRetentionMode::GOVERNANCE),
6129 ] {
6130 let retained_metadata = HashMap::from([
6131 (rustfs_filemeta::metadata_keys::OBJECT_LOCK_MODE.to_string(), mode.to_string()),
6132 (
6133 rustfs_filemeta::metadata_keys::OBJECT_LOCK_RETAIN_UNTIL_DATE.to_string(),
6134 retain_until.clone(),
6135 ),
6136 ]);
6137 crate::data_movement::migrate_object(
6138 store.clone(),
6139 0,
6140 bucket.clone(),
6141 source_reader(object, old_body.clone(), old_time, retained_metadata.clone()).await,
6142 None,
6143 "test_retained_target_seed",
6144 )
6145 .await
6146 .expect("seed retained migrated target");
6147 let err = crate::data_movement::migrate_object(
6148 store.clone(),
6149 0,
6150 bucket.clone(),
6151 source_reader(object, new_body.clone(), new_time, retained_metadata).await,
6152 None,
6153 "test_retained_target_replace",
6154 )
6155 .await
6156 .expect_err("active retention must block stale target replacement");
6157 assert!(err.to_string().contains("complete_multipart_upload"), "unexpected retention error: {err}");
6158
6159 let mut retained_reader = store.pools[1]
6160 .get_object_reader(
6161 &bucket,
6162 object,
6163 None,
6164 HeaderMap::new(),
6165 &ObjectOptions {
6166 versioned: true,
6167 version_id: Some(version_id.to_string()),
6168 ..Default::default()
6169 },
6170 )
6171 .await
6172 .expect("read retained target");
6173 let mut retained_body = Vec::new();
6174 retained_reader
6175 .stream
6176 .read_to_end(&mut retained_body)
6177 .await
6178 .expect("drain retained target");
6179 assert_eq!(retained_body, old_body);
6180 }
6181
6182 for object in [
6183 replaceable,
6184 client_target,
6185 acknowledged_target,
6186 metadata_acknowledged_target,
6187 "compliance-target.bin",
6188 "governance-target.bin",
6189 ] {
6190 let uploads = store.pools[1]
6191 .list_multipart_uploads(&bucket, object, None, None, None, 100)
6192 .await
6193 .expect("list target multipart staging");
6194 assert!(uploads.uploads.is_empty(), "data movement staging must be cleaned for {object}");
6195 }
6196 });
6197 })
6198 .expect("spawn data movement replacement test thread")
6199 .join()
6200 .expect("join data movement replacement test thread");
6201 }
6202
6203 #[tokio::test]
6204 #[serial_test::serial(storage_class_env)]
6205 async fn data_movement_delete_marker_retries_converge_safely() {
6206 let temp_dir = tempfile::tempdir().expect("create delete-marker data movement store dir");
6207 let (_ctx, store, _shutdown) =
6208 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "data-movement-delete-marker", &[4, 4])).await;
6209 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
6210
6211 let bucket = format!("dm-delete-{}", uuid::Uuid::new_v4());
6212 store
6213 .make_bucket(&bucket, &MakeBucketOptions::default())
6214 .await
6215 .expect("create delete-marker data movement bucket");
6216 let object = "delete-marker-retry";
6217 let version = uuid::Uuid::new_v4();
6218 let mod_time = OffsetDateTime::UNIX_EPOCH + time::Duration::SECOND;
6219 let replica_timestamp = mod_time + time::Duration::SECOND;
6220 let replication_timestamp = mod_time;
6221 let replica_timestamp_string = replica_timestamp
6222 .format(&time::format_description::well_known::Rfc3339)
6223 .expect("replica timestamp should format as RFC3339");
6224 let replication_timestamp_string = replication_timestamp
6225 .format(&time::format_description::well_known::Rfc3339)
6226 .expect("replication timestamp should format as RFC3339");
6227 let replication_status = "arn:minio:replication::TenantA:bucket=COMPLETED;";
6228 store.pools[0]
6229 .delete_object(
6230 &bucket,
6231 object,
6232 ObjectOptions {
6233 versioned: true,
6234 version_id: Some(version.to_string()),
6235 delete_marker: true,
6236 mod_time: Some(mod_time),
6237 delete_replication: Some(ReplicationState {
6238 replica_status: ReplicationStatusType::Replica,
6239 replica_timestamp: Some(replica_timestamp),
6240 replication_status_internal: Some(replication_status.to_string()),
6241 replication_timestamp: Some(replication_timestamp),
6242 targets: replication_statuses_map(replication_status),
6243 delete_marker: true,
6244 ..Default::default()
6245 }),
6246 ..Default::default()
6247 },
6248 )
6249 .await
6250 .expect("write source delete marker");
6251 let source = store.pools[0]
6252 .get_object_info(
6253 &bucket,
6254 object,
6255 &ObjectOptions {
6256 versioned: true,
6257 version_id: Some(version.to_string()),
6258 ..Default::default()
6259 },
6260 )
6261 .await
6262 .expect("read source delete marker replication metadata");
6263 assert_eq!(
6264 rustfs_utils::http::get_str(&source.user_defined, rustfs_utils::http::SUFFIX_REPLICA_STATUS).as_deref(),
6265 Some("REPLICA"),
6266 "source metadata: {:?}",
6267 source.user_defined
6268 );
6269
6270 *store.rebalance_meta.write().await = Some(active_rebalance_meta_for_pool(store.pools.len(), 0));
6271 let missing_object = "missing-delete-marker";
6272 let missing_version = uuid::Uuid::new_v4();
6273 let missing_opts = ObjectOptions {
6274 versioned: true,
6275 version_id: Some(missing_version.to_string()),
6276 delete_marker: true,
6277 mod_time: Some(mod_time),
6278 src_pool_idx: 0,
6279 data_movement: true,
6280 skip_decommissioned: true,
6281 ..Default::default()
6282 };
6283 let missing_source_err = store
6284 .delete_object(&bucket, missing_object, missing_opts.clone())
6285 .await
6286 .expect_err("data movement must not recreate a delete marker missing from the source");
6287 assert!(matches!(missing_source_err, StorageError::DataMovementOverwriteErr(_, _, _)));
6288 let missing_target_err = store.pools[1]
6289 .get_object_info(&bucket, missing_object, &missing_opts)
6290 .await
6291 .expect_err("source-missing preflight must not write a target delete marker");
6292 assert!(matches!(
6293 missing_target_err,
6294 StorageError::ObjectNotFound(_, _) | StorageError::VersionNotFound(_, _, _)
6295 ));
6296
6297 let movement_opts = ObjectOptions {
6298 versioned: true,
6299 version_id: Some(version.to_string()),
6300 delete_marker: true,
6301 mod_time: Some(mod_time),
6302 src_pool_idx: 0,
6303 data_movement: true,
6304 skip_decommissioned: true,
6305 ..Default::default()
6306 };
6307 for attempt in 1..=2 {
6308 let moved = store
6309 .delete_object(&bucket, object, movement_opts.clone())
6310 .await
6311 .unwrap_or_else(|err| panic!("delete marker migration attempt {attempt} should converge: {err}"));
6312 assert!(moved.delete_marker);
6313 assert_eq!(moved.version_id, Some(version));
6314 }
6315 let target = store.pools[1]
6316 .get_object_info(&bucket, object, &movement_opts)
6317 .await
6318 .expect("retry must retain the target delete marker");
6319 assert!(target.delete_marker);
6320 assert_eq!(target.mod_time, Some(mod_time));
6321 assert_eq!(
6322 rustfs_utils::http::get_str(&target.user_defined, rustfs_utils::http::SUFFIX_REPLICA_STATUS).as_deref(),
6323 Some("REPLICA")
6324 );
6325 assert_eq!(
6326 rustfs_utils::http::get_str(&target.user_defined, rustfs_utils::http::SUFFIX_REPLICA_TIMESTAMP).as_deref(),
6327 Some(replica_timestamp_string.as_str())
6328 );
6329 assert_eq!(
6330 rustfs_utils::http::get_str(&target.user_defined, rustfs_utils::http::SUFFIX_REPLICATION_STATUS).as_deref(),
6331 Some(replication_status)
6332 );
6333 assert_eq!(
6334 rustfs_utils::http::get_str(&target.user_defined, rustfs_utils::http::SUFFIX_REPLICATION_TIMESTAMP).as_deref(),
6335 Some(replication_timestamp_string.as_str())
6336 );
6337
6338 let null_object = "null-delete-marker-retry";
6339 let null_mod_time = mod_time + time::Duration::seconds(3);
6340 let null_marker = store.pools[0]
6341 .delete_object(
6342 &bucket,
6343 null_object,
6344 ObjectOptions {
6345 version_suspended: true,
6346 version_id: Some(uuid::Uuid::nil().to_string()),
6347 data_movement: true,
6348 delete_marker: true,
6349 mod_time: Some(null_mod_time),
6350 ..Default::default()
6351 },
6352 )
6353 .await
6354 .expect("write source null delete marker");
6355 assert_eq!(null_marker.version_id, Some(uuid::Uuid::nil()));
6356 let null_opts = ObjectOptions {
6357 version_suspended: true,
6358 version_id: Some(uuid::Uuid::nil().to_string()),
6359 delete_marker: true,
6360 mod_time: Some(null_mod_time),
6361 src_pool_idx: 0,
6362 data_movement: true,
6363 skip_decommissioned: true,
6364 ..Default::default()
6365 };
6366 for attempt in 1..=2 {
6367 let moved = store
6368 .delete_object(&bucket, null_object, null_opts.clone())
6369 .await
6370 .unwrap_or_else(|err| panic!("null delete marker migration attempt {attempt} should converge: {err}"));
6371 assert!(moved.delete_marker);
6372 assert_eq!(moved.version_id, Some(uuid::Uuid::nil()));
6373 }
6374 let target_null_marker = store.pools[1]
6375 .get_object_info(&bucket, null_object, &null_opts)
6376 .await
6377 .expect("target must retain the null delete marker identity");
6378 assert!(target_null_marker.delete_marker);
6379 assert_eq!(target_null_marker.version_id, Some(uuid::Uuid::nil()));
6380
6381 let explicit_object = "suspended-explicit-version-delete";
6382 let explicit_version = uuid::Uuid::new_v4();
6383 let mut explicit_reader = PutObjReader::from_vec(b"explicit version".to_vec());
6384 store.pools[0]
6385 .put_object(
6386 &bucket,
6387 explicit_object,
6388 &mut explicit_reader,
6389 &ObjectOptions {
6390 versioned: true,
6391 version_id: Some(explicit_version.to_string()),
6392 ..Default::default()
6393 },
6394 )
6395 .await
6396 .expect("write explicit version before suspended delete");
6397 store.pools[0]
6398 .delete_object(
6399 &bucket,
6400 explicit_object,
6401 ObjectOptions {
6402 version_suspended: true,
6403 version_id: Some(explicit_version.to_string()),
6404 ..Default::default()
6405 },
6406 )
6407 .await
6408 .expect("delete explicit version from suspended bucket");
6409 store.pools[0]
6410 .get_object_info(
6411 &bucket,
6412 explicit_object,
6413 &ObjectOptions {
6414 versioned: true,
6415 version_id: Some(explicit_version.to_string()),
6416 ..Default::default()
6417 },
6418 )
6419 .await
6420 .expect_err("suspended delete must remove the requested UUID version");
6421 }
6422
6423 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6424 #[serial_test::serial(storage_class_env)]
6425 async fn decommission_entry_carries_migration_and_cleanup_mutation_fences() {
6426 let temp_dir = tempfile::tempdir().expect("create decommission delete-fence store dir");
6427 let (_ctx, store, shutdown) = without_storage_class_env(build_isolated_test_store_with_layout(
6428 temp_dir.path(),
6429 "decommission-delete-fence",
6430 &[(2, 4), (1, 4)],
6431 CancellationToken::new(),
6432 None,
6433 ))
6434 .await;
6435 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
6436
6437 let bucket = format!("decom-delete-fence-{}", uuid::Uuid::new_v4());
6438 let object = (0..128)
6439 .map(|index| format!("object-{index}.bin"))
6440 .find(|candidate| store.pools[0].get_disks_by_key(candidate).set_index == 1)
6441 .expect("the deterministic object search should select source set 1");
6442 store
6443 .make_bucket(&bucket, &MakeBucketOptions::default())
6444 .await
6445 .expect("create decommission delete-fence bucket");
6446 let mut source = PutObjReader::from_vec(b"source generation".to_vec());
6447 store.pools[0]
6448 .put_object(&bucket, &object, &mut source, &ObjectOptions::default())
6449 .await
6450 .expect("write source object to the pool being decommissioned");
6451 mark_test_pool_decommissioning(&store, 0).await;
6452 assert!(store.is_suspended(0).await, "pool 0 must be a suspended decommission source");
6453
6454 let barrier = crate::set_disk::PutObjectCommitBarrier::install(
6455 &bucket,
6456 &object,
6457 crate::set_disk::PutObjectCommitPause::AfterNamespace,
6458 );
6459 let cleanup_barrier = crate::data_movement::SourceCleanupDeleteBarrier::install(&bucket, &object);
6460 let source_set = store.pools[0].get_disks_by_key(&object);
6461 assert_eq!(source_set.set_index, 1, "the source entry must exercise the non-fixed set cleanup lock");
6462 let worker_store = Arc::clone(&store);
6463 let worker_bucket = bucket.clone();
6464 let worker_object = object.clone();
6465 let worker = tokio::spawn(async move {
6466 worker_store
6467 .decommission_entry_for_test(
6468 0,
6469 MetaCacheEntry {
6470 name: worker_object,
6471 ..Default::default()
6472 },
6473 worker_bucket,
6474 source_set,
6475 )
6476 .await
6477 });
6478 barrier.wait_until_paused().await;
6479
6480 let delete_barrier = crate::store::object::DeleteAfterObjectLockSnapshotBarrier::install(&bucket);
6481 let delete_store = Arc::clone(&store);
6482 let delete_bucket = bucket.clone();
6483 let delete_object = object.clone();
6484 let delete = tokio::spawn(async move {
6485 delete_store
6486 .delete_object(&delete_bucket, &delete_object, ObjectOptions::default())
6487 .await
6488 });
6489 delete_barrier.wait_until_paused().await;
6490 delete_barrier.release_and_wait_until_namespace_pending().await;
6491 assert!(
6492 !delete_barrier.namespace_acquired() && !delete.is_finished(),
6493 "DELETE must remain before namespace acquisition behind the decommission worker's target-commit mutation fence"
6494 );
6495
6496 barrier.release();
6497 cleanup_barrier.wait_until_paused().await;
6498 drop(barrier);
6499
6500 let fixed_set = Arc::clone(&store.pools[0].disk_set[0]);
6501 let fixed_mutation_barrier = crate::set_disk::PutObjectCommitBarrier::install(
6502 &bucket,
6503 &object,
6504 crate::set_disk::PutObjectCommitPause::BeforeNamespace,
6505 );
6506 let mutation_bucket = bucket.clone();
6507 let mutation_object = object.clone();
6508 let fixed_mutation = tokio::spawn(async move {
6509 let mut reader = PutObjReader::from_vec(b"fixed-domain replacement".to_vec());
6510 fixed_set
6511 .put_object(&mutation_bucket, &mutation_object, &mut reader, &ObjectOptions::default())
6512 .await
6513 });
6514 fixed_mutation_barrier.wait_until_paused().await;
6515 fixed_mutation_barrier.release_and_wait_until_namespace_pending().await;
6516 assert!(
6517 !fixed_mutation_barrier.namespace_acquired() && !fixed_mutation.is_finished(),
6518 "the source cleanup must retain the fixed mutation fence before the set-0 mutation acquires its namespace"
6519 );
6520 fixed_mutation.abort();
6521 assert!(
6522 fixed_mutation
6523 .await
6524 .expect_err("the fixed-domain mutation should be canceled")
6525 .is_cancelled(),
6526 "the competing fixed-domain mutation must remain cancelable while blocked"
6527 );
6528 drop(fixed_mutation_barrier);
6529
6530 cleanup_barrier.release();
6531 worker
6532 .await
6533 .expect("decommission entry worker should join")
6534 .expect("decommission entry should migrate and clean its source");
6535 delete
6536 .await
6537 .expect("DELETE task should join")
6538 .expect("DELETE should remove the source and migrated target generations");
6539
6540 for pool in &store.pools {
6541 let err = pool
6542 .get_object_info(&bucket, &object, &ObjectOptions::default())
6543 .await
6544 .expect_err("DELETE must remove the source and migrated target copies");
6545 assert!(
6546 matches!(err, StorageError::ObjectNotFound(_, _)),
6547 "unexpected post-delete pool result: {err:?}"
6548 );
6549 }
6550 let err = store
6551 .get_object_info(&bucket, &object, &ObjectOptions::default())
6552 .await
6553 .expect_err("the deleted generation must not become visible again");
6554 assert!(
6555 matches!(err, StorageError::ObjectNotFound(_, _)),
6556 "unexpected post-delete store result: {err:?}"
6557 );
6558
6559 shutdown.cancel();
6560 }
6561
6562 #[test]
6563 fn decommission_retry_fault_budget_counts_successes_across_attempt_changes() {
6564 let cases: &[&[(usize, bool, bool)]] = &[
6565 &[(1, true, true), (2, true, true), (3, true, false)],
6566 &[(1, true, true), (1, true, true), (2, true, false)],
6567 &[(1, true, true), (3, true, false), (3, true, false)],
6568 &[(1, true, true), (2, false, false), (1, true, true), (2, true, false)],
6569 &[(1, true, true), (2, false, false), (3, true, false)],
6570 &[(3, true, false), (4, true, false)],
6571 ];
6572 for case in cases {
6573 let faults = Arc::new(AtomicUsize::new(0));
6574 let hook = decommission_retry_fault_hook("bucket", "object", Arc::clone(&faults));
6575
6576 for (stage, bucket, object, succeeded) in [
6577 ("other-stage", "bucket", "object", true),
6578 (DECOMMISSION_TEST_FAULT_STAGE_MIGRATE_OBJECT, "other-bucket", "object", true),
6579 (DECOMMISSION_TEST_FAULT_STAGE_MIGRATE_OBJECT, "bucket", "other-object", true),
6580 (DECOMMISSION_TEST_FAULT_STAGE_MIGRATE_OBJECT, "bucket", "object", false),
6581 ] {
6582 assert!(!hook(stage, bucket, object, 1, succeeded));
6583 }
6584 assert_eq!(faults.load(Ordering::SeqCst), 0, "unrelated or failed copies must not consume faults");
6585
6586 let mut expected_faults = 0;
6587 for &(attempt, succeeded, expected) in *case {
6588 assert_eq!(
6589 hook(DECOMMISSION_TEST_FAULT_STAGE_MIGRATE_OBJECT, "bucket", "object", attempt, succeeded),
6590 expected,
6591 "fault plan {case:?} at attempt {attempt}"
6592 );
6593 expected_faults += usize::from(expected);
6594 assert_eq!(faults.load(Ordering::SeqCst), expected_faults);
6595 }
6596 }
6597 }
6598
6599 #[test]
6600 #[serial_test::serial(storage_class_env)]
6601 fn decommission_entry_retries_source_changed_without_canceling_other_bucket() {
6602 let handle = std::thread::Builder::new()
6603 .name("decommission_entry_retries_source_changed_without_canceling_other_bucket".to_string())
6604 .stack_size(32 * 1024 * 1024)
6605 .spawn(|| {
6606 let runtime = tokio::runtime::Builder::new_multi_thread()
6607 .enable_all()
6608 .worker_threads(2)
6609 .build()
6610 .expect("test runtime should build");
6611 runtime.block_on(async {
6612 let temp_dir = tempfile::tempdir().expect("create decommission retry store dir");
6613 let (_ctx, store, shutdown) = without_storage_class_env(build_isolated_test_store(
6614 temp_dir.path(),
6615 "decommission-entry-retry",
6616 &[4, 4],
6617 ))
6618 .await;
6619 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
6620
6621 let changed_bucket = format!("decom-retry-a-{}", uuid::Uuid::new_v4());
6622 let other_bucket = format!("decom-retry-b-{}", uuid::Uuid::new_v4());
6623 for bucket in [&changed_bucket, &other_bucket] {
6624 store
6625 .make_bucket(bucket, &MakeBucketOptions::default())
6626 .await
6627 .expect("create decommission retry bucket");
6628 }
6629
6630 let changed_object = "changed.bin";
6631 let first_version = uuid::Uuid::new_v4();
6632 let second_version = uuid::Uuid::new_v4();
6633 let base_time = OffsetDateTime::now_utc();
6634 seed_decommission_source(
6635 &store,
6636 &changed_bucket,
6637 changed_object,
6638 b"first generation".to_vec(),
6639 &ObjectOptions {
6640 versioned: true,
6641 version_id: Some(first_version.to_string()),
6642 mod_time: Some(base_time),
6643 ..Default::default()
6644 },
6645 )
6646 .await;
6647 let other_object = "other.bin";
6648 seed_decommission_source(
6649 &store,
6650 &other_bucket,
6651 other_object,
6652 b"other bucket generation".to_vec(),
6653 &ObjectOptions::default(),
6654 )
6655 .await;
6656 mark_test_pool_decommissioning(&store, 0).await;
6657
6658 let mutation_calls = Arc::new(AtomicUsize::new(0));
6659 let mutation_calls_for_hook = Arc::clone(&mutation_calls);
6660 let mutation_store = Arc::clone(&store);
6661 let mutation_bucket = changed_bucket.clone();
6662 let _mutation_guard = crate::core::pools::DecommissionCleanupMutationGuard::install(Arc::new(
6663 move |bucket, object, attempt| {
6664 let is_target = bucket == mutation_bucket.as_str() && object == changed_object;
6665 let calls = Arc::clone(&mutation_calls_for_hook);
6666 let store = Arc::clone(&mutation_store);
6667 let bucket = mutation_bucket.clone();
6668 Box::pin(async move {
6669 if !is_target {
6670 return;
6671 }
6672 calls.fetch_add(1, Ordering::SeqCst);
6673 if attempt == 1 {
6674 seed_decommission_source(
6675 &store,
6676 &bucket,
6677 changed_object,
6678 b"second generation".to_vec(),
6679 &ObjectOptions {
6680 versioned: true,
6681 version_id: Some(second_version.to_string()),
6682 mod_time: Some(base_time + time::Duration::seconds(1)),
6683 ..Default::default()
6684 },
6685 )
6686 .await;
6687 }
6688 })
6689 },
6690 ));
6691
6692 let ordinary_faults = Arc::new(AtomicUsize::new(0));
6693 let fault_hook = decommission_retry_fault_hook(&other_bucket, other_object, Arc::clone(&ordinary_faults));
6694 let _fault_guard = crate::core::pools::DecommissionTestFaultGuard::install(fault_hook);
6695
6696 let rx = CancellationToken::new();
6697 let source_changed_exhaustions = Arc::new(AtomicUsize::new(0));
6698 let changed_incarnation = Some(
6699 store
6700 .bucket_incarnation_id(&changed_bucket)
6701 .await
6702 .expect("changed bucket incarnation"),
6703 );
6704 let other_incarnation = Some(
6705 store
6706 .bucket_incarnation_id(&other_bucket)
6707 .await
6708 .expect("other bucket incarnation"),
6709 );
6710 let (changed_result, other_result) = tokio::join!(
6711 run_decommission_entry_retry_test(
6712 &store,
6713 rx.clone(),
6714 &changed_bucket,
6715 changed_object,
6716 changed_incarnation,
6717 Arc::clone(&source_changed_exhaustions),
6718 ),
6719 run_decommission_entry_retry_test(
6720 &store,
6721 rx.clone(),
6722 &other_bucket,
6723 other_object,
6724 other_incarnation,
6725 Arc::clone(&source_changed_exhaustions),
6726 )
6727 );
6728 changed_result.expect("SourceChanged entry retry must converge");
6729 other_result.expect("other bucket entry must continue through ordinary copy retries");
6730
6731 assert_eq!(
6732 store.pool_meta.read().await.pools[0]
6733 .decommission
6734 .as_ref()
6735 .expect("decommission progress should remain available")
6736 .items_decommission_failed,
6737 0,
6738 "entry completion must not hide an exhausted copy failure"
6739 );
6740 assert!(!rx.is_cancelled(), "entry-level SourceChanged must not cancel the shared worker token");
6741 assert_eq!(mutation_calls.load(Ordering::SeqCst), 2, "entry must be re-listed after SourceChanged");
6742 assert_eq!(ordinary_faults.load(Ordering::SeqCst), 2, "ordinary copy must consume the retry budget");
6743 assert_eq!(source_changed_exhaustions.load(Ordering::SeqCst), 0);
6744
6745 for (version_id, expected_body) in [
6746 (first_version, b"first generation".as_slice()),
6747 (second_version, b"second generation".as_slice()),
6748 ] {
6749 let opts = ObjectOptions {
6750 versioned: true,
6751 version_id: Some(version_id.to_string()),
6752 ..Default::default()
6753 };
6754 assert_decommission_source_absent(&store, &changed_bucket, changed_object, &opts).await;
6755 assert_eq!(
6756 read_decommission_target_body(&store, &changed_bucket, changed_object, &opts).await,
6757 expected_body
6758 );
6759 }
6760 assert_decommission_source_absent(&store, &other_bucket, other_object, &ObjectOptions::default()).await;
6761 assert_eq!(
6762 read_decommission_target_body(&store, &other_bucket, other_object, &ObjectOptions::default()).await,
6763 b"other bucket generation"
6764 );
6765
6766 shutdown.cancel();
6767 });
6768 })
6769 .expect("spawn decommission retry test thread");
6770 if let Err(payload) = handle.join() {
6771 std::panic::resume_unwind(payload);
6772 }
6773 }
6774
6775 #[test]
6776 #[serial_test::serial(storage_class_env)]
6777 fn decommission_entry_exhausted_source_changed_retains_source_and_records_failure() {
6778 let handle = std::thread::Builder::new()
6779 .name("decommission_entry_exhausted_source_changed_retains_source_and_records_failure".to_string())
6780 .stack_size(32 * 1024 * 1024)
6781 .spawn(|| {
6782 let runtime = tokio::runtime::Builder::new_multi_thread()
6783 .enable_all()
6784 .worker_threads(2)
6785 .build()
6786 .expect("test runtime should build");
6787 runtime.block_on(async {
6788 let temp_dir = tempfile::tempdir().expect("create decommission exhaustion store dir");
6789 let (_ctx, store, shutdown) = without_storage_class_env(build_isolated_test_store(
6790 temp_dir.path(),
6791 "decommission-entry-exhaustion",
6792 &[4, 4],
6793 ))
6794 .await;
6795 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
6796
6797 let bucket = format!("decom-exhausted-{}", uuid::Uuid::new_v4());
6798 let object = "exhausted.bin";
6799 store
6800 .make_bucket(&bucket, &MakeBucketOptions::default())
6801 .await
6802 .expect("create decommission exhaustion bucket");
6803 let original_version = uuid::Uuid::new_v4();
6804 let base_time = OffsetDateTime::now_utc();
6805 seed_decommission_source(
6806 &store,
6807 &bucket,
6808 object,
6809 b"original generation".to_vec(),
6810 &ObjectOptions {
6811 versioned: true,
6812 version_id: Some(original_version.to_string()),
6813 mod_time: Some(base_time),
6814 ..Default::default()
6815 },
6816 )
6817 .await;
6818 mark_test_pool_decommissioning(&store, 0).await;
6819
6820 let mutation_calls = Arc::new(AtomicUsize::new(0));
6821 let mutation_calls_for_hook = Arc::clone(&mutation_calls);
6822 let mutation_store = Arc::clone(&store);
6823 let mutation_bucket = bucket.clone();
6824 let _mutation_guard = crate::core::pools::DecommissionCleanupMutationGuard::install(Arc::new(
6825 move |called_bucket, called_object, _attempt| {
6826 let is_target = called_bucket == mutation_bucket.as_str() && called_object == object;
6827 let calls = Arc::clone(&mutation_calls_for_hook);
6828 let store = Arc::clone(&mutation_store);
6829 let bucket = mutation_bucket.clone();
6830 Box::pin(async move {
6831 if !is_target {
6832 return;
6833 }
6834 let call = calls.fetch_add(1, Ordering::SeqCst) + 1;
6835 let offset = i64::try_from(call).expect("entry retry count should fit i64");
6836 seed_decommission_source(
6837 &store,
6838 &bucket,
6839 object,
6840 format!("concurrent generation {call}").into_bytes(),
6841 &ObjectOptions {
6842 versioned: true,
6843 version_id: Some(uuid::Uuid::new_v4().to_string()),
6844 mod_time: Some(base_time + time::Duration::seconds(offset)),
6845 ..Default::default()
6846 },
6847 )
6848 .await;
6849 })
6850 },
6851 ));
6852
6853 let rx = CancellationToken::new();
6854 let source_changed_exhaustions = Arc::new(AtomicUsize::new(0));
6855 let incarnation = Some(store.bucket_incarnation_id(&bucket).await.expect("bucket incarnation"));
6856 run_decommission_entry_retry_test(
6857 &store,
6858 rx.clone(),
6859 &bucket,
6860 object,
6861 incarnation,
6862 Arc::clone(&source_changed_exhaustions),
6863 )
6864 .await
6865 .expect("entry-level exhaustion must stay local below the pool threshold");
6866
6867 assert!(!rx.is_cancelled(), "one exhausted entry must not cancel other bucket workers");
6868 assert_eq!(mutation_calls.load(Ordering::SeqCst), crate::core::pools::DECOMMISSION_ENTRY_MAX_ATTEMPTS);
6869 assert_eq!(source_changed_exhaustions.load(Ordering::SeqCst), 1);
6870 store.pools[0]
6871 .get_object_info(
6872 &bucket,
6873 object,
6874 &ObjectOptions {
6875 versioned: true,
6876 version_id: Some(original_version.to_string()),
6877 ..Default::default()
6878 },
6879 )
6880 .await
6881 .expect("retry exhaustion must retain the original source version");
6882 let pool_meta = store.pool_meta.read().await;
6883 let info = pool_meta.pools[0]
6884 .decommission
6885 .as_ref()
6886 .expect("decommission progress must be initialized");
6887 assert_eq!(info.items_decommission_failed, 1, "exhausted entry must be visible as failed");
6888 drop(pool_meta);
6889
6890 shutdown.cancel();
6891 });
6892 })
6893 .expect("spawn decommission retry test thread");
6894 if let Err(payload) = handle.join() {
6895 std::panic::resume_unwind(payload);
6896 }
6897 }
6898
6899 #[test]
6900 #[serial_test::serial(storage_class_env)]
6901 fn decommission_entry_delete_marker_copy_retries_real_path() {
6902 let handle = std::thread::Builder::new()
6903 .name("decommission_entry_delete_marker_copy_retries_real_path".to_string())
6904 .stack_size(32 * 1024 * 1024)
6905 .spawn(|| {
6906 let runtime = tokio::runtime::Builder::new_multi_thread()
6907 .enable_all()
6908 .worker_threads(2)
6909 .build()
6910 .expect("test runtime should build");
6911 runtime.block_on(async {
6912 let temp_dir = tempfile::tempdir().expect("create delete marker retry store dir");
6913 let (_ctx, store, shutdown) = without_storage_class_env(build_isolated_test_store(
6914 temp_dir.path(),
6915 "decommission-delete-marker-retry",
6916 &[4, 4],
6917 ))
6918 .await;
6919 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
6920
6921 let bucket = format!("decom-marker-{}", uuid::Uuid::new_v4());
6922 let object = "marker.bin";
6923 store
6924 .make_bucket(&bucket, &MakeBucketOptions::default())
6925 .await
6926 .expect("create delete marker retry bucket");
6927 let data_version = uuid::Uuid::new_v4();
6928 let marker_version = uuid::Uuid::new_v4();
6929 let base_time = OffsetDateTime::now_utc();
6930 seed_decommission_source(
6931 &store,
6932 &bucket,
6933 object,
6934 b"delete marker data".to_vec(),
6935 &ObjectOptions {
6936 versioned: true,
6937 version_id: Some(data_version.to_string()),
6938 mod_time: Some(base_time),
6939 ..Default::default()
6940 },
6941 )
6942 .await;
6943 store.pools[0]
6944 .delete_object(
6945 &bucket,
6946 object,
6947 ObjectOptions {
6948 versioned: true,
6949 version_id: Some(marker_version.to_string()),
6950 delete_marker: true,
6951 mod_time: Some(base_time + time::Duration::seconds(1)),
6952 ..Default::default()
6953 },
6954 )
6955 .await
6956 .expect("seed source delete marker");
6957 mark_test_pool_decommissioning(&store, 0).await;
6958
6959 let fault_calls = Arc::new(AtomicUsize::new(0));
6960 let fault_calls_for_hook = Arc::clone(&fault_calls);
6961 let fault_bucket = bucket.clone();
6962 let _fault_guard = crate::core::pools::DecommissionTestFaultGuard::install(Arc::new(
6963 move |stage, called_bucket, called_object, attempt, succeeded| {
6964 let injected = succeeded
6965 && stage == DECOMMISSION_TEST_FAULT_STAGE_DELETE_MARKER
6966 && called_bucket == fault_bucket.as_str()
6967 && called_object == object
6968 && attempt < crate::core::pools::DECOMMISSION_VERSION_COPY_ATTEMPTS;
6969 if injected {
6970 fault_calls_for_hook.fetch_add(1, Ordering::SeqCst);
6971 }
6972 injected
6973 },
6974 ));
6975
6976 let incarnation = Some(store.bucket_incarnation_id(&bucket).await.expect("bucket incarnation"));
6977 run_decommission_entry_retry_test(
6978 &store,
6979 CancellationToken::new(),
6980 &bucket,
6981 object,
6982 incarnation,
6983 Arc::new(AtomicUsize::new(0)),
6984 )
6985 .await
6986 .expect("delete marker copy retries must converge");
6987
6988 assert_eq!(
6989 fault_calls.load(Ordering::SeqCst),
6990 crate::core::pools::DECOMMISSION_VERSION_COPY_ATTEMPTS - 1
6991 );
6992 let marker_opts = ObjectOptions {
6993 versioned: true,
6994 version_id: Some(marker_version.to_string()),
6995 ..Default::default()
6996 };
6997 let target_marker = store.pools[1]
6998 .get_object_info(&bucket, object, &marker_opts)
6999 .await
7000 .expect("target delete marker must exist");
7001 assert!(target_marker.delete_marker);
7002 assert_decommission_source_absent(&store, &bucket, object, &marker_opts).await;
7003
7004 let data_opts = ObjectOptions {
7005 versioned: true,
7006 version_id: Some(data_version.to_string()),
7007 ..Default::default()
7008 };
7009 assert_eq!(
7010 read_decommission_target_body(&store, &bucket, object, &data_opts).await,
7011 b"delete marker data"
7012 );
7013 assert_decommission_source_absent(&store, &bucket, object, &data_opts).await;
7014
7015 shutdown.cancel();
7016 });
7017 })
7018 .expect("spawn decommission retry test thread");
7019 if let Err(payload) = handle.join() {
7020 std::panic::resume_unwind(payload);
7021 }
7022 }
7023
7024 #[cfg(feature = "test-util")]
7025 #[test]
7026 #[serial_test::serial(storage_class_env)]
7027 fn decommission_entry_tiered_copy_retries_real_path() {
7028 let handle = std::thread::Builder::new()
7029 .name("decommission_entry_tiered_copy_retries_real_path".to_string())
7030 .stack_size(32 * 1024 * 1024)
7031 .spawn(|| {
7032 let runtime = tokio::runtime::Builder::new_multi_thread()
7033 .enable_all()
7034 .worker_threads(2)
7035 .build()
7036 .expect("test runtime should build");
7037 runtime.block_on(async {
7038 let temp_dir = tempfile::tempdir().expect("create tiered retry store dir");
7039 let (ctx, store, shutdown) = without_storage_class_env(build_isolated_test_store(
7040 temp_dir.path(),
7041 "decommission-tiered-retry",
7042 &[4, 4],
7043 ))
7044 .await;
7045 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
7046
7047 let bucket = format!("decom-tiered-{}", uuid::Uuid::new_v4());
7048 let object = "tiered.bin";
7049 store
7050 .make_bucket(&bucket, &MakeBucketOptions::default())
7051 .await
7052 .expect("create tiered retry bucket");
7053 let mut reader = PutObjReader::from_vec(b"tiered generation".to_vec());
7054 let original = store.pools[0]
7055 .put_object(&bucket, object, &mut reader, &ObjectOptions::default())
7056 .await
7057 .expect("seed tiered source object");
7058 let tier_name = format!("DECOM{}", &uuid::Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
7059 register_mock_tier(&ctx.tier_config_mgr(), &tier_name).await;
7060 store.pools[0]
7061 .transition_object(
7062 &bucket,
7063 object,
7064 &ObjectOptions {
7065 transition: TransitionOptions {
7066 status: TRANSITION_PENDING.to_string(),
7067 tier: tier_name,
7068 etag: original.etag.clone().expect("tiered source ETag"),
7069 ..Default::default()
7070 },
7071 version_id: original.version_id.map(|version_id| version_id.to_string()),
7072 mod_time: original.mod_time,
7073 ..Default::default()
7074 },
7075 )
7076 .await
7077 .expect("transition source object to mock tier");
7078 mark_test_pool_decommissioning(&store, 0).await;
7079
7080 let fault_calls = Arc::new(AtomicUsize::new(0));
7081 let fault_calls_for_hook = Arc::clone(&fault_calls);
7082 let fault_bucket = bucket.clone();
7083 let _fault_guard = crate::core::pools::DecommissionTestFaultGuard::install(Arc::new(
7084 move |stage, called_bucket, called_object, attempt, succeeded| {
7085 let injected = succeeded
7086 && stage == DECOMMISSION_TEST_FAULT_STAGE_TIERED
7087 && called_bucket == fault_bucket.as_str()
7088 && called_object == object
7089 && attempt < crate::core::pools::DECOMMISSION_VERSION_COPY_ATTEMPTS;
7090 if injected {
7091 fault_calls_for_hook.fetch_add(1, Ordering::SeqCst);
7092 }
7093 injected
7094 },
7095 ));
7096
7097 let incarnation = Some(store.bucket_incarnation_id(&bucket).await.expect("bucket incarnation"));
7098 run_decommission_entry_retry_test(
7099 &store,
7100 CancellationToken::new(),
7101 &bucket,
7102 object,
7103 incarnation,
7104 Arc::new(AtomicUsize::new(0)),
7105 )
7106 .await
7107 .expect("tiered copy retries must converge");
7108
7109 assert_eq!(
7110 fault_calls.load(Ordering::SeqCst),
7111 crate::core::pools::DECOMMISSION_VERSION_COPY_ATTEMPTS - 1
7112 );
7113 let target = store.pools[1]
7114 .get_object_info(&bucket, object, &ObjectOptions::default())
7115 .await
7116 .expect("tiered target metadata must exist");
7117 assert_eq!(target.transitioned_object.status, rustfs_filemeta::TRANSITION_COMPLETE);
7118 assert_decommission_source_absent(&store, &bucket, object, &ObjectOptions::default()).await;
7119
7120 shutdown.cancel();
7121 });
7122 })
7123 .expect("spawn decommission retry test thread");
7124 if let Err(payload) = handle.join() {
7125 std::panic::resume_unwind(payload);
7126 }
7127 }
7128
7129 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7130 #[serial_test::serial(storage_class_env)]
7131 async fn decommission_outer_fence_loss_blocks_target_put_commit() {
7132 let temp_dir = tempfile::tempdir().expect("create decommission PUT fence-loss store dir");
7133 let (_ctx, store, shutdown) =
7134 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "decommission-put-fence-loss", &[4, 4])).await;
7135 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
7136
7137 let bucket = format!("decom-put-fence-loss-{}", uuid::Uuid::new_v4());
7138 let object = "ordinary.bin";
7139 store
7140 .make_bucket(&bucket, &MakeBucketOptions::default())
7141 .await
7142 .expect("create decommission PUT fence-loss bucket");
7143 let mut source = PutObjReader::from_vec(b"source generation".to_vec());
7144 store.pools[0]
7145 .put_object(&bucket, object, &mut source, &ObjectOptions::default())
7146 .await
7147 .expect("write decommission PUT source");
7148 mark_test_pool_decommissioning(&store, 0).await;
7149
7150 let loss_hook = crate::store::object::DecommissionMutationFenceLossHook::install(
7151 &bucket,
7152 object,
7153 crate::store::object::DecommissionMutationFenceTestPhase::Migration,
7154 );
7155 let barrier = crate::set_disk::PutObjectCommitBarrier::install(
7156 &bucket,
7157 object,
7158 crate::set_disk::PutObjectCommitPause::BeforeQuotaRename,
7159 );
7160 let source_set = store.pools[0].get_disks_by_key(object);
7161 let worker_store = Arc::clone(&store);
7162 let worker_bucket = bucket.clone();
7163 let worker = tokio::spawn(async move {
7164 worker_store
7165 .decommission_entry_for_test(
7166 0,
7167 MetaCacheEntry {
7168 name: object.to_string(),
7169 ..Default::default()
7170 },
7171 worker_bucket,
7172 source_set,
7173 )
7174 .await
7175 });
7176
7177 barrier.wait_until_paused().await;
7178 loss_hook.mark_lost();
7179 barrier.release();
7180 drop(barrier);
7181 worker
7182 .await
7183 .expect("decommission PUT fence-loss worker should join")
7184 .expect("a fenced migration failure should remain retryable at entry scope");
7185
7186 assert_pool_object_absent(&store.pools[1], &bucket, object).await;
7187 assert_pool_object_present(&store.pools[0], &bucket, object).await;
7188 shutdown.cancel();
7189 }
7190
7191 #[test]
7192 #[serial_test::serial(storage_class_env)]
7193 fn decommission_outer_fence_loss_blocks_multipart_commits() {
7194 run_large_stack_async_test(
7195 "decommission-multipart-outer-fence-loss",
7196 decommission_outer_fence_loss_blocks_multipart_commits_case,
7197 );
7198 }
7199
7200 async fn decommission_outer_fence_loss_blocks_multipart_commits_case() {
7201 for (object, pause) in [
7202 ("complete.bin", crate::set_disk::MultipartCommitPause::BeforeLockLost),
7203 ("new-upload.bin", crate::set_disk::MultipartCommitPause::NewUploadBeforeLockLost),
7204 ] {
7205 let temp_dir = tempfile::tempdir().expect("create decommission multipart fence-loss store dir");
7209 let (_ctx, store, shutdown) = without_storage_class_env(build_isolated_test_store(
7210 temp_dir.path(),
7211 "decommission-multipart-fence-loss",
7212 &[4, 4],
7213 ))
7214 .await;
7215 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
7216
7217 let bucket = format!("decom-mpu-fence-loss-{}", uuid::Uuid::new_v4());
7218 store
7219 .make_bucket(&bucket, &MakeBucketOptions::default())
7220 .await
7221 .expect("create decommission multipart fence-loss bucket");
7222 write_decommission_test_multipart_source(&store, 0, &bucket, object).await;
7223 mark_test_pool_decommissioning(&store, 0).await;
7224
7225 let loss_hook = crate::store::object::DecommissionMutationFenceLossHook::install(
7226 &bucket,
7227 object,
7228 crate::store::object::DecommissionMutationFenceTestPhase::Migration,
7229 );
7230 let commit_observation = (pause == crate::set_disk::MultipartCommitPause::NewUploadBeforeLockLost)
7231 .then(|| crate::set_disk::NewMultipartUploadCommitObservation::install(&bucket, object));
7232 let barrier = crate::set_disk::MultipartCommitBarrier::install(&bucket, object, pause);
7233 let source_set = store.pools[0].get_disks_by_key(object);
7234 let recovery_source_set = Arc::clone(&source_set);
7235 let worker_store = Arc::clone(&store);
7236 let worker_bucket = bucket.clone();
7237 let mut worker = tokio::spawn(async move {
7238 worker_store
7239 .decommission_entry_for_test(
7240 0,
7241 MetaCacheEntry {
7242 name: object.to_string(),
7243 ..Default::default()
7244 },
7245 worker_bucket,
7246 source_set,
7247 )
7248 .await
7249 });
7250
7251 tokio::select! {
7252 _ = barrier.wait_until_paused() => {}
7253 result = &mut worker => {
7254 panic!("decommission multipart worker finished before the requested {pause:?} commit barrier: {result:?}");
7255 }
7256 }
7257 loss_hook.mark_lost();
7258 barrier.release();
7259 drop(barrier);
7260 worker
7261 .await
7262 .expect("decommission multipart fence-loss worker should join")
7263 .expect("a fenced multipart migration failure should remain retryable at entry scope");
7264
7265 if let Some(commit_observation) = commit_observation {
7266 assert!(
7267 !commit_observation.committed(),
7268 "NewMultipart must reject a lost outer decommission/capacity fence even though it does not acquire the final remote-tuple publication fence"
7269 );
7270 }
7271 assert_pool_object_absent(&store.pools[1], &bucket, object).await;
7272 assert_pool_object_present(&store.pools[0], &bucket, object).await;
7273 let uploads = store.pools[1]
7274 .list_multipart_uploads(&bucket, object, None, None, None, 100)
7275 .await
7276 .expect("list target multipart uploads after fenced migration");
7277 assert!(uploads.uploads.is_empty(), "fenced multipart migration must not retain target staging");
7278 drop(loss_hook);
7279 store
7280 .decommission_entry_for_test(
7281 0,
7282 MetaCacheEntry {
7283 name: object.to_string(),
7284 ..Default::default()
7285 },
7286 bucket.clone(),
7287 recovery_source_set,
7288 )
7289 .await
7290 .expect("same-mutation retry should recover the durable capacity intent");
7291
7292 shutdown.cancel();
7293 }
7294 }
7295
7296 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7297 #[serial_test::serial(storage_class_env)]
7298 async fn decommission_outer_fence_loss_blocks_source_cleanup_delete_commit() {
7299 let temp_dir = tempfile::tempdir().expect("create decommission cleanup fence-loss store dir");
7300 let (_ctx, store, shutdown) =
7301 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "decommission-cleanup-fence-loss", &[4, 4]))
7302 .await;
7303 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
7304
7305 let bucket = format!("decom-cleanup-fence-loss-{}", uuid::Uuid::new_v4());
7306 let object = "cleanup.bin";
7307 store
7308 .make_bucket(&bucket, &MakeBucketOptions::default())
7309 .await
7310 .expect("create decommission cleanup fence-loss bucket");
7311 let mut source = PutObjReader::from_vec(b"source generation".to_vec());
7312 store.pools[0]
7313 .put_object(&bucket, object, &mut source, &ObjectOptions::default())
7314 .await
7315 .expect("write decommission cleanup source");
7316 mark_test_pool_decommissioning(&store, 0).await;
7317
7318 let loss_hook = crate::store::object::DecommissionMutationFenceLossHook::install(
7319 &bucket,
7320 object,
7321 crate::store::object::DecommissionMutationFenceTestPhase::SourceCleanup,
7322 );
7323 let barrier = crate::data_movement::SourceCleanupDeleteBarrier::install(&bucket, object);
7324 let source_set = store.pools[0].get_disks_by_key(object);
7325 let worker_store = Arc::clone(&store);
7326 let worker_bucket = bucket.clone();
7327 let worker = tokio::spawn(async move {
7328 worker_store
7329 .decommission_entry_for_test(
7330 0,
7331 MetaCacheEntry {
7332 name: object.to_string(),
7333 ..Default::default()
7334 },
7335 worker_bucket,
7336 source_set,
7337 )
7338 .await
7339 });
7340
7341 barrier.wait_until_paused().await;
7342 loss_hook.mark_lost();
7343 barrier.release();
7344 drop(barrier);
7345 let err = worker
7346 .await
7347 .expect("decommission cleanup fence-loss worker should join")
7348 .expect_err("source cleanup must fail after its outer fence is lost");
7349 assert!(
7350 err.to_string().contains("delete_object_commit"),
7351 "cleanup failure must come from the delete commit fence: {err:?}"
7352 );
7353
7354 assert_pool_object_present(&store.pools[0], &bucket, object).await;
7355 assert_pool_object_present(&store.pools[1], &bucket, object).await;
7356 shutdown.cancel();
7357 }
7358
7359 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7360 #[serial_test::serial(storage_class_env)]
7361 async fn reverse_decommission_reuses_fixed_target_fence_for_put_and_multipart() {
7362 let temp_dir = tempfile::tempdir().expect("create reverse decommission store dir");
7363 let (_ctx, store, shutdown) = without_storage_class_env(build_isolated_test_store_with_layout(
7364 temp_dir.path(),
7365 "reverse-decommission-fixed-target",
7366 &[(1, 4), (1, 4)],
7367 CancellationToken::new(),
7368 None,
7369 ))
7370 .await;
7371 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
7372
7373 let bucket = format!("reverse-decom-fixed-target-{}", uuid::Uuid::new_v4());
7374 let object = "ordinary.bin";
7375 let object_body = b"reverse ordinary generation".to_vec();
7376 let self_copy_object = "source-only-self-copy.bin";
7377 let self_copy_body = b"source only copy generation".to_vec();
7378 let multipart_object = "multipart.bin";
7379 let first_part = vec![b'm'; 5 * 1024 * 1024];
7380 let second_part = b"reverse multipart tail".to_vec();
7381 let mut multipart_body = first_part.clone();
7382 multipart_body.extend_from_slice(&second_part);
7383
7384 store
7385 .make_bucket(&bucket, &MakeBucketOptions::default())
7386 .await
7387 .expect("create reverse decommission bucket");
7388 let mut source = PutObjReader::from_vec(object_body.clone());
7389 store.pools[1]
7390 .put_object(&bucket, object, &mut source, &ObjectOptions::default())
7391 .await
7392 .expect("write ordinary source object to pool 1");
7393 let mut self_copy_source = PutObjReader::from_vec(self_copy_body.clone());
7394 store.pools[1]
7395 .put_object(&bucket, self_copy_object, &mut self_copy_source, &ObjectOptions::default())
7396 .await
7397 .expect("write self-copy source object to pool 1");
7398
7399 let upload = store.pools[1]
7400 .new_multipart_upload(&bucket, multipart_object, &ObjectOptions::default())
7401 .await
7402 .expect("create source multipart upload in pool 1");
7403 let mut completed_parts = Vec::with_capacity(2);
7404 for (part_number, bytes) in [(1, first_part.as_slice()), (2, second_part.as_slice())] {
7405 let mut reader = PutObjReader::from_vec(bytes.to_vec());
7406 let part = store.pools[1]
7407 .put_object_part(
7408 &bucket,
7409 multipart_object,
7410 &upload.upload_id,
7411 part_number,
7412 &mut reader,
7413 &ObjectOptions::default(),
7414 )
7415 .await
7416 .expect("write source multipart part");
7417 completed_parts.push(crate::storage_api_contracts::multipart::CompletePart {
7418 part_num: part.part_num,
7419 etag: part.etag,
7420 ..Default::default()
7421 });
7422 }
7423 store.pools[1]
7424 .clone()
7425 .complete_multipart_upload(&bucket, multipart_object, &upload.upload_id, completed_parts, &ObjectOptions::default())
7426 .await
7427 .expect("complete source multipart object in pool 1");
7428
7429 {
7430 let mut pool_meta = store.pool_meta.write().await;
7431 pool_meta.pools[1].decommission = Some(PoolDecommissionInfo {
7432 start_time: Some(OffsetDateTime::now_utc()),
7433 ..Default::default()
7434 });
7435 }
7436 assert!(store.is_suspended(1).await, "pool 1 must be the reverse decommission source");
7437
7438 let self_copy_opts = ObjectOptions {
7439 no_lock: true,
7440 ..Default::default()
7441 };
7442 let mut self_copy_reader = store
7443 .get_object_reader(&bucket, self_copy_object, None, HeaderMap::new(), &self_copy_opts)
7444 .await
7445 .expect("read the suspended source before its target is committed");
7446 let mut self_copy_info = self_copy_reader.object_info.clone();
7447 let mut self_copy_source_body = Vec::new();
7448 self_copy_reader
7449 .stream
7450 .read_to_end(&mut self_copy_source_body)
7451 .await
7452 .expect("drain the suspended self-copy source");
7453 assert_eq!(self_copy_source_body, self_copy_body);
7454 self_copy_info.metadata_only = true;
7455 self_copy_info.put_object_reader = Some(PutObjReader::from_vec(self_copy_source_body));
7456 store
7457 .copy_object(
7458 &bucket,
7459 self_copy_object,
7460 &bucket,
7461 self_copy_object,
7462 &mut self_copy_info,
7463 &self_copy_opts,
7464 &self_copy_opts,
7465 )
7466 .await
7467 .expect("self-copy should read the suspended source and commit to an active pool");
7468 assert_pool_object_present(&store.pools[0], &bucket, self_copy_object).await;
7469 assert_pool_object_present(&store.pools[1], &bucket, self_copy_object).await;
7470
7471 let mut active_copy_reader = store
7472 .get_object_reader(&bucket, self_copy_object, None, HeaderMap::new(), &self_copy_opts)
7473 .await
7474 .expect("read the active self-copy target while the source remains");
7475 let active_copy_data_dir = active_copy_reader.object_info.data_dir;
7476 let mut active_copy_info = active_copy_reader.object_info.clone();
7477 let mut active_copy_body = Vec::new();
7478 active_copy_reader
7479 .stream
7480 .read_to_end(&mut active_copy_body)
7481 .await
7482 .expect("drain the active self-copy target");
7483 assert_eq!(active_copy_body, self_copy_body);
7484 active_copy_info.metadata_only = true;
7485 active_copy_info.put_object_reader = Some(PutObjReader::from_vec(active_copy_body));
7486 let active_copy_result = store
7487 .copy_object(
7488 &bucket,
7489 self_copy_object,
7490 &bucket,
7491 self_copy_object,
7492 &mut active_copy_info,
7493 &self_copy_opts,
7494 &self_copy_opts,
7495 )
7496 .await
7497 .expect("self-copy should keep using the committed active target");
7498 assert_eq!(active_copy_result.data_dir, active_copy_data_dir);
7499
7500 {
7501 let mut pool_meta = store.pool_meta.write().await;
7502 pool_meta.pools[1].decommission = None;
7503 }
7504 mark_test_pool_decommissioning(&store, 1).await;
7505
7506 let cleanup_barrier = crate::data_movement::SourceCleanupDeleteBarrier::install(&bucket, object);
7507 let commit_barrier = crate::set_disk::PutObjectCommitBarrier::install(
7508 &bucket,
7509 object,
7510 crate::set_disk::PutObjectCommitPause::AfterNamespace,
7511 );
7512 let source_set = store.pools[1].get_disks_by_key(object);
7513 let worker_store = Arc::clone(&store);
7514 let worker_bucket = bucket.clone();
7515 let worker = tokio::spawn(async move {
7516 worker_store
7517 .decommission_entry_for_test(
7518 1,
7519 MetaCacheEntry {
7520 name: object.to_string(),
7521 ..Default::default()
7522 },
7523 worker_bucket,
7524 source_set,
7525 )
7526 .await
7527 });
7528 commit_barrier.wait_until_paused().await;
7529
7530 let delete_barrier = crate::store::object::DeleteAfterObjectLockSnapshotBarrier::install(&bucket);
7531 let delete_store = Arc::clone(&store);
7532 let delete_bucket = bucket.clone();
7533 let delete = tokio::spawn(async move {
7534 delete_store
7535 .delete_object(&delete_bucket, object, ObjectOptions::default())
7536 .await
7537 });
7538 delete_barrier.wait_until_paused().await;
7539 delete_barrier.release_and_wait_until_namespace_pending().await;
7540 assert!(
7541 !delete_barrier.namespace_acquired() && !delete.is_finished(),
7542 "the reverse target commit must keep DELETE behind the fixed read fence"
7543 );
7544 delete.abort();
7545 assert!(
7546 delete
7547 .await
7548 .expect_err("the blocked DELETE should be canceled")
7549 .is_cancelled(),
7550 "canceling the blocked DELETE must not mutate either pool"
7551 );
7552 drop(delete_barrier);
7553
7554 commit_barrier.release();
7555 cleanup_barrier.wait_until_paused().await;
7556 drop(commit_barrier);
7557
7558 let read_opts = ObjectOptions {
7559 no_lock: true,
7560 ..Default::default()
7561 };
7562 let target_info = store.pools[0]
7563 .get_object_info(&bucket, object, &read_opts)
7564 .await
7565 .expect("read the committed active target before source cleanup");
7566 store.pools[1]
7567 .get_object_info(&bucket, object, &read_opts)
7568 .await
7569 .expect("the source must remain present until cleanup is released");
7570 let routed_info = store
7571 .get_object_info(&bucket, object, &read_opts)
7572 .await
7573 .expect("HEAD routing should prefer the active target before source cleanup");
7574 assert_eq!(routed_info.data_dir, target_info.data_dir);
7575
7576 let mut ranged_reader = store
7577 .get_object_reader(
7578 &bucket,
7579 object,
7580 Some(HTTPRangeSpec {
7581 is_suffix_length: false,
7582 start: 8,
7583 end: 15,
7584 }),
7585 HeaderMap::new(),
7586 &read_opts,
7587 )
7588 .await
7589 .expect("ranged GET routing should prefer the active target before source cleanup");
7590 assert_eq!(ranged_reader.object_info.data_dir, target_info.data_dir);
7591 let mut ranged_body = Vec::new();
7592 ranged_reader
7593 .stream
7594 .read_to_end(&mut ranged_body)
7595 .await
7596 .expect("drain the routed target range");
7597 assert_eq!(ranged_body, object_body[8..=15]);
7598
7599 cleanup_barrier.release();
7600 drop(cleanup_barrier);
7601 tokio::time::timeout(Duration::from_secs(60), worker)
7602 .await
7603 .expect("reverse ordinary decommission must not self-deadlock on the fixed target set")
7604 .expect("reverse ordinary decommission worker should join")
7605 .expect("reverse ordinary decommission should complete");
7606
7607 let mut ordinary_reader = store.pools[0]
7608 .get_object_reader(&bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
7609 .await
7610 .expect("read the ordinary object from the fixed target set");
7611 let mut ordinary_target_body = Vec::new();
7612 ordinary_reader
7613 .stream
7614 .read_to_end(&mut ordinary_target_body)
7615 .await
7616 .expect("drain the ordinary target body");
7617 assert_eq!(ordinary_target_body, object_body, "ordinary migration must preserve the full body");
7618 let ordinary_source_err = store.pools[1]
7619 .get_object_info(&bucket, object, &ObjectOptions::default())
7620 .await
7621 .expect_err("ordinary source generation must be cleaned after migration");
7622 assert!(matches!(ordinary_source_err, StorageError::ObjectNotFound(_, _)));
7623
7624 let multipart_source_set = store.pools[1].get_disks_by_key(multipart_object);
7625 let multipart_store = Arc::clone(&store);
7626 let multipart_bucket = bucket.clone();
7627 let multipart_worker = tokio::spawn(async move {
7628 multipart_store
7629 .decommission_entry_for_test(
7630 1,
7631 MetaCacheEntry {
7632 name: multipart_object.to_string(),
7633 ..Default::default()
7634 },
7635 multipart_bucket,
7636 multipart_source_set,
7637 )
7638 .await
7639 });
7640 tokio::time::timeout(Duration::from_secs(60), multipart_worker)
7641 .await
7642 .expect("reverse multipart decommission must not self-deadlock on new or complete")
7643 .expect("reverse multipart decommission worker should join")
7644 .expect("reverse multipart decommission should complete");
7645
7646 let target_info = store.pools[0]
7647 .get_object_info(&bucket, multipart_object, &ObjectOptions::default())
7648 .await
7649 .expect("read migrated multipart metadata from the fixed target set");
7650 assert!(target_info.is_multipart(), "migration must retain multipart identity");
7651 let mut multipart_reader = store.pools[0]
7652 .get_object_reader(&bucket, multipart_object, None, HeaderMap::new(), &ObjectOptions::default())
7653 .await
7654 .expect("read migrated multipart object from the fixed target set");
7655 let mut multipart_target_body = Vec::new();
7656 multipart_reader
7657 .stream
7658 .read_to_end(&mut multipart_target_body)
7659 .await
7660 .expect("drain the multipart target body");
7661 assert_eq!(multipart_target_body, multipart_body, "multipart migration must preserve the full body");
7662 let multipart_source_err = store.pools[1]
7663 .get_object_info(&bucket, multipart_object, &ObjectOptions::default())
7664 .await
7665 .expect_err("multipart source generation must be cleaned after migration");
7666 assert!(matches!(multipart_source_err, StorageError::ObjectNotFound(_, _)));
7667
7668 shutdown.cancel();
7669 }
7670
7671 #[tokio::test]
7672 #[serial_test::serial(storage_class_env)]
7673 async fn batch_delete_real_path_preserves_source_pool_errors_in_any_pool_order() {
7674 let temp_dir = tempfile::tempdir().expect("create batch delete pool-error store dir");
7675 let (_ctx, store, shutdown) =
7676 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "batch-delete-pool-errors", &[4, 4])).await;
7677 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
7678
7679 for source_pool_idx in [0, 1] {
7680 {
7681 let mut pool_meta = store.pool_meta.write().await;
7682 for pool in &mut pool_meta.pools {
7683 pool.decommission = None;
7684 }
7685 }
7686
7687 let bucket = format!("batch-del-pool-error-{source_pool_idx}-{}", uuid::Uuid::new_v4());
7688 let object_names = vec![
7689 format!("third-{source_pool_idx}.bin"),
7690 format!("first-{source_pool_idx}.bin"),
7691 format!("second-{source_pool_idx}.bin"),
7692 ];
7693 store
7694 .make_bucket(&bucket, &MakeBucketOptions::default())
7695 .await
7696 .expect("create batch delete pool-error bucket");
7697 for pool in &store.pools {
7698 for object_name in &object_names {
7699 let mut reader = PutObjReader::from_vec(format!("pool {} {object_name}", pool.pool_idx).into_bytes());
7700 pool.put_object(&bucket, object_name, &mut reader, &ObjectOptions::default())
7701 .await
7702 .expect("seed each object in both the source and active pools");
7703 }
7704 }
7705 {
7706 let mut pool_meta = store.pool_meta.write().await;
7707 pool_meta.pools[source_pool_idx].decommission = Some(PoolDecommissionInfo {
7708 start_time: Some(OffsetDateTime::now_utc()),
7709 ..Default::default()
7710 });
7711 }
7712 assert!(
7713 store.is_suspended(source_pool_idx).await,
7714 "the injected error pool must be the decommission source"
7715 );
7716
7717 let expected_errors = [
7718 StorageError::ErasureWriteQuorum,
7719 StorageError::NamespaceLockQuorumUnavailable {
7720 mode: "delete_objects_commit",
7721 bucket: bucket.clone(),
7722 object: object_names[1].clone(),
7723 required: 3,
7724 achieved: 2,
7725 },
7726 StorageError::ErasureWriteQuorum,
7727 ];
7728 let injection = crate::store::object::BatchDeletePoolErrorInjection::install(
7729 &bucket,
7730 source_pool_idx,
7731 object_names.iter().cloned().zip(expected_errors.iter().cloned()).collect(),
7732 );
7733 let requests = object_names
7734 .iter()
7735 .map(|object_name| ObjectToDelete {
7736 object_name: object_name.clone(),
7737 ..Default::default()
7738 })
7739 .collect();
7740
7741 let (deleted, errors) = store.delete_objects(&bucket, requests, ObjectOptions::default()).await;
7742
7743 assert_eq!(
7744 injection.observed(),
7745 object_names.len(),
7746 "the source pool must first complete every real delete"
7747 );
7748 assert_eq!(
7749 errors,
7750 expected_errors.iter().cloned().map(Some).collect::<Vec<_>>(),
7751 "a successful pool must not clear a source pool failure at any request index"
7752 );
7753 assert_eq!(
7754 deleted.iter().map(|object| object.object_name.as_str()).collect::<Vec<_>>(),
7755 object_names.iter().map(String::as_str).collect::<Vec<_>>(),
7756 "DeleteObjects must preserve request index mapping while aggregating pool failures"
7757 );
7758 assert!(
7759 deleted.iter().all(|object| object.found),
7760 "the injected source results must retain real delete success data"
7761 );
7762
7763 for pool in &store.pools {
7764 for object_name in &object_names {
7765 let error = pool
7766 .get_object_info(&bucket, object_name, &ObjectOptions::default())
7767 .await
7768 .expect_err("both the active and source pool delete calls must execute");
7769 assert!(
7770 matches!(error, StorageError::ObjectNotFound(_, _)),
7771 "unexpected residual object: {error:?}"
7772 );
7773 }
7774 }
7775 drop(injection);
7776 }
7777
7778 shutdown.cancel();
7779 }
7780
7781 #[tokio::test]
7782 #[serial_test::serial(storage_class_env)]
7783 async fn decommission_source_cleanup_holds_hashed_set_lock_across_preflight() {
7784 let temp_dir = tempfile::tempdir().expect("create multi-set decommission cleanup store dir");
7785 let (_ctx, store, shutdown) = without_storage_class_env(build_isolated_test_store_with_layout(
7786 temp_dir.path(),
7787 "multi-set-decommission-source-cleanup",
7788 &[(2, 4)],
7789 CancellationToken::new(),
7790 None,
7791 ))
7792 .await;
7793 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
7794
7795 let bucket = format!("decom-source-cleanup-lock-{}", uuid::Uuid::new_v4());
7796 let object = (0..128)
7797 .map(|index| format!("object-{index}.bin"))
7798 .find(|candidate| store.pools[0].get_disks_by_key(candidate).set_index == 1)
7799 .expect("the deterministic object search should select source set 1");
7800 store
7801 .make_bucket(&bucket, &MakeBucketOptions::default())
7802 .await
7803 .expect("create multi-set decommission cleanup bucket");
7804 let mut source = PutObjReader::from_vec(b"source generation".to_vec());
7805 store.pools[0]
7806 .put_object(&bucket, &object, &mut source, &ObjectOptions::default())
7807 .await
7808 .expect("write the source generation to set 1");
7809 let source_set = store.pools[0].get_disks_by_key(&object);
7810 assert_eq!(source_set.set_index, 1, "the source must not share the fixed set-0 namespace");
7811 let expected_source_versions = source_set
7812 .load_file_info_versions_exact(&bucket, &object)
7813 .await
7814 .expect("source versions should be readable")
7815 .expect("the source generation should exist");
7816
7817 let cleanup_barrier = crate::data_movement::SourceCleanupDeleteBarrier::install(&bucket, &object);
7818 let cleanup_store = Arc::clone(&store);
7819 let cleanup_bucket = bucket.clone();
7820 let cleanup_object = object.clone();
7821 let cleanup = tokio::spawn(async move {
7822 let mutation_fence = cleanup_store
7823 .acquire_decommission_source_cleanup_fence(&cleanup_bucket, &cleanup_object, source_set.as_ref())
7824 .await?;
7825 crate::data_movement::cleanup_source_entry_if_unchanged(
7826 source_set,
7827 &cleanup_bucket,
7828 &cleanup_object,
7829 &expected_source_versions,
7830 &[],
7831 crate::data_movement::SourceCleanupBucketFence {
7832 object_mutation_fence: Some(&mutation_fence),
7833 ..Default::default()
7834 },
7835 "test_multi_set_decommission_source_cleanup",
7836 )
7837 .await
7838 });
7839 cleanup_barrier.wait_until_paused().await;
7840
7841 let put_barrier = crate::set_disk::PutObjectCommitBarrier::install(
7842 &bucket,
7843 &object,
7844 crate::set_disk::PutObjectCommitPause::BeforeNamespace,
7845 );
7846 let mutation_pool = Arc::clone(&store.pools[0]);
7847 let mutation_bucket = bucket.clone();
7848 let mutation_object = object.clone();
7849 let replacement = b"replacement generation".to_vec();
7850 let expected_replacement = replacement.clone();
7851 let mutation = tokio::spawn(async move {
7852 let mut reader = PutObjReader::from_vec(replacement);
7853 mutation_pool
7854 .put_object(&mutation_bucket, &mutation_object, &mut reader, &ObjectOptions::default())
7855 .await
7856 });
7857 put_barrier.wait_until_paused().await;
7858 put_barrier.release_and_wait_until_namespace_pending().await;
7859 assert!(!mutation.is_finished(), "a source mutation must wait behind cleanup's set-1 write lock");
7860
7861 cleanup_barrier.release();
7862 cleanup
7863 .await
7864 .expect("source cleanup task should join")
7865 .expect("source cleanup should remove only the preflight generation");
7866 mutation
7867 .await
7868 .expect("source mutation task should join")
7869 .expect("source mutation should commit after cleanup releases the set lock");
7870
7871 let mut reader = store.pools[0]
7872 .get_object_reader(&bucket, &object, None, HeaderMap::new(), &ObjectOptions::default())
7873 .await
7874 .expect("the replacement generation must remain readable");
7875 let mut actual = Vec::new();
7876 reader
7877 .stream
7878 .read_to_end(&mut actual)
7879 .await
7880 .expect("read the replacement generation");
7881 assert_eq!(actual, expected_replacement, "cleanup must not delete the replacement generation");
7882
7883 shutdown.cancel();
7884 }
7885
7886 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7887 #[serial_test::serial(storage_class_env)]
7888 async fn versioned_delete_marker_survives_decommission_source_cleanup() {
7889 let temp_dir = tempfile::tempdir().expect("create versioned decommission delete-fence store dir");
7890 let (_ctx, store, shutdown) =
7891 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "versioned-decommission-delete-fence", &[4, 4]))
7892 .await;
7893 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
7894
7895 let bucket = format!("versioned-decom-delete-{}", uuid::Uuid::new_v4());
7896 let object = "object.bin";
7897 store
7898 .make_bucket(&bucket, &MakeBucketOptions::default())
7899 .await
7900 .expect("create versioned decommission delete-fence bucket");
7901 let (source_version, expected_source_versions) = migrate_versioned_decommission_test_object(
7902 &store,
7903 &bucket,
7904 object,
7905 b"source generation",
7906 "test_versioned_decommission_delete_fence",
7907 )
7908 .await;
7909
7910 let delete_barrier = crate::store::object::VersionedDeleteMarkerCommitBarrier::install(&bucket, object);
7911 let delete_store = Arc::clone(&store);
7912 let delete_bucket = bucket.clone();
7913 let delete = tokio::spawn(async move {
7914 delete_store
7915 .delete_object(
7916 &delete_bucket,
7917 object,
7918 ObjectOptions {
7919 versioned: true,
7920 ..Default::default()
7921 },
7922 )
7923 .await
7924 });
7925 delete_barrier.wait_until_paused().await;
7926 let cleanup_set = store.pools[0].get_disks_by_key(object);
7927 crate::data_movement::ensure_source_cleanup_versions_unchanged(
7928 Arc::clone(&cleanup_set),
7929 &bucket,
7930 object,
7931 &expected_source_versions,
7932 &[],
7933 "test_versioned_decommission_delete_fence",
7934 )
7935 .await
7936 .expect("the committed delete marker must not be published to the suspended source pool");
7937
7938 let cleanup_delete_barrier = crate::data_movement::SourceCleanupDeleteBarrier::install(&bucket, object);
7939 let cleanup_store = Arc::clone(&store);
7940 let cleanup_bucket = bucket.clone();
7941 let cleanup = tokio::spawn(async move {
7942 let mutation_fence = cleanup_store
7943 .acquire_decommission_source_cleanup_fence(&cleanup_bucket, object, cleanup_set.as_ref())
7944 .await?;
7945 crate::data_movement::cleanup_source_entry_if_unchanged(
7946 cleanup_set,
7947 &cleanup_bucket,
7948 object,
7949 &expected_source_versions,
7950 &[],
7951 crate::data_movement::SourceCleanupBucketFence {
7952 object_mutation_fence: Some(&mutation_fence),
7953 ..Default::default()
7954 },
7955 "test_versioned_decommission_delete_fence",
7956 )
7957 .await
7958 });
7959 cleanup_delete_barrier.wait_until_fence_pending().await;
7960 assert!(
7961 !cleanup_delete_barrier.is_paused(),
7962 "source cleanup must wait for the versioned DELETE mutation fence"
7963 );
7964
7965 delete_barrier.release();
7966 let marker = delete
7967 .await
7968 .expect("versioned DELETE task should join")
7969 .expect("versioned DELETE should publish a delete marker after migration");
7970 assert!(marker.delete_marker, "versioned DELETE must publish a delete marker");
7971 assert!(
7972 marker.version_id.is_some_and(|version_id| !version_id.is_nil()),
7973 "the delete marker must have a non-nil version ID"
7974 );
7975
7976 cleanup_delete_barrier.wait_until_paused().await;
7977 cleanup_delete_barrier.release();
7978 cleanup
7979 .await
7980 .expect("source cleanup task should join")
7981 .expect("source cleanup should preserve the active-pool delete marker");
7982
7983 let err = store
7984 .get_object_info(
7985 &bucket,
7986 object,
7987 &ObjectOptions {
7988 versioned: true,
7989 ..Default::default()
7990 },
7991 )
7992 .await
7993 .expect_err("the post-migration delete marker must hide the migrated version");
7994 assert!(
7995 matches!(err, StorageError::ObjectNotFound(_, _)),
7996 "unexpected latest-version result: {err:?}"
7997 );
7998 store
7999 .get_object_info(
8000 &bucket,
8001 object,
8002 &ObjectOptions {
8003 versioned: true,
8004 version_id: Some(source_version.to_string()),
8005 ..Default::default()
8006 },
8007 )
8008 .await
8009 .expect("the migrated source version must remain addressable below the delete marker");
8010 store.pools[0]
8011 .get_object_info(
8012 &bucket,
8013 object,
8014 &ObjectOptions {
8015 versioned: true,
8016 version_id: Some(source_version.to_string()),
8017 ..Default::default()
8018 },
8019 )
8020 .await
8021 .expect_err("source cleanup must remove the decommissioned source versions");
8022
8023 shutdown.cancel();
8024 }
8025
8026 #[cfg(feature = "test-util")]
8027 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8028 #[serial_test::serial(storage_class_env)]
8029 async fn decommission_entry_skips_cleanup_only_marker_when_free_version_is_present() {
8030 let temp_dir = tempfile::tempdir().expect("create free-version decommission store dir");
8031 let (ctx, store, shutdown) =
8032 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "decommission-free-marker", &[4, 4])).await;
8033 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
8034 let bucket = format!("decom-free-marker-{}", uuid::Uuid::new_v4());
8035 let object = "free-marker-object";
8036 store
8037 .make_bucket(&bucket, &MakeBucketOptions::default())
8038 .await
8039 .expect("create free-version decommission bucket");
8040 let (_, free_version) = seed_transitioned_free_version(&ctx, &store, &bucket, object).await;
8041 let source_free = store.pools[0]
8042 .get_disks_by_key(object)
8043 .load_file_info_versions_exact(&bucket, object)
8044 .await
8045 .expect("source free-version metadata should decode")
8046 .and_then(|versions| {
8047 versions
8048 .versions
8049 .into_iter()
8050 .find(|version| version.version_id == Some(free_version) && version.tier_free_version())
8051 })
8052 .expect("source free-version identity should be present before decommission");
8053 let mut target_reader = PutObjReader::from_vec(b"target ordinary bytes".to_vec());
8054 let target_w = store.pools[1]
8055 .put_object(
8056 &bucket,
8057 object,
8058 &mut target_reader,
8059 &ObjectOptions {
8060 versioned: true,
8061 ..Default::default()
8062 },
8063 )
8064 .await
8065 .expect("write unrelated target version");
8066 let target_w_version = target_w.version_id.expect("target version should have an id");
8067 let marker = store.pools[0]
8068 .delete_object(
8069 &bucket,
8070 object,
8071 ObjectOptions {
8072 versioned: true,
8073 ..Default::default()
8074 },
8075 )
8076 .await
8077 .expect("write cleanup-only delete marker");
8078 assert!(marker.delete_marker);
8079
8080 mark_test_pool_decommissioning(&store, 0).await;
8081 let source_set = store.pools[0].get_disks_by_key(object);
8082 store
8083 .decommission_entry_for_test_with_bucket_incarnation(
8084 0,
8085 MetaCacheEntry {
8086 name: object.to_string(),
8087 ..Default::default()
8088 },
8089 bucket.clone(),
8090 source_set.clone(),
8091 )
8092 .await
8093 .expect("real decommission entry should migrate the free version");
8094
8095 let target_versions = store.pools[1]
8096 .get_disks_by_key(object)
8097 .load_file_info_versions_exact(&bucket, object)
8098 .await
8099 .expect("target versions should decode")
8100 .expect("target free version should be present");
8101 assert!(
8102 target_versions
8103 .versions
8104 .iter()
8105 .any(|version| { version.version_id == Some(free_version) && version.tier_free_version() })
8106 );
8107 let migrated_free = target_versions
8108 .versions
8109 .iter()
8110 .find(|version| version.version_id == Some(free_version) && version.tier_free_version())
8111 .expect("migrated free-version identity should remain readable from target disks");
8112 assert!(
8113 crate::store::tiered_data_movement_source_matches(&source_free, migrated_free)
8114 .expect("migrated free-version identity should decode")
8115 );
8116 let retained_w = target_versions
8117 .versions
8118 .iter()
8119 .find(|version| version.version_id == Some(target_w_version))
8120 .expect("unrelated target version should remain");
8121 assert!(!retained_w.deleted && !retained_w.tier_free_version());
8122 assert_eq!(retained_w.size, target_w.size);
8123 assert_eq!(retained_w.get_etag(), target_w.etag);
8124 assert!(
8125 target_versions
8126 .versions
8127 .iter()
8128 .all(|version| { version.tier_free_version() || !version.deleted })
8129 );
8130 assert!(
8131 source_set
8132 .load_file_info_versions_exact(&bucket, object)
8133 .await
8134 .expect("source versions should be readable after cleanup")
8135 .is_none(),
8136 "successful free-version migration should permit source cleanup"
8137 );
8138 let (heal_versions, _, _) = store
8139 .heal_walk_versions_page(1, 0, &bucket, "", None, 2, 16, true)
8140 .await
8141 .expect("heal walk should decode the migrated free version");
8142 let free_version_string = free_version.to_string();
8143 let healed_free = heal_versions
8144 .iter()
8145 .find(|version| version.version_id.as_deref() == Some(free_version_string.as_str()))
8146 .expect("heal walk should surface the migrated free version");
8147 let healed_info = healed_free
8148 .lifecycle_object_info
8149 .as_ref()
8150 .expect("heal walk should retain lifecycle identity for the migrated free version");
8151 assert!(healed_info.transitioned_object.free_version);
8152 assert_eq!(healed_info.transitioned_object.tier, source_free.transition_tier);
8153 assert_eq!(healed_info.transitioned_object.name, source_free.transitioned_objname);
8154 shutdown.cancel();
8155 }
8156
8157 #[cfg(feature = "test-util")]
8158 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8159 #[serial_test::serial(storage_class_env)]
8160 async fn decommission_entry_allows_free_version_consumed_before_source_lock() {
8161 let temp_dir = tempfile::tempdir().expect("create consumed free-version store dir");
8162 let (ctx, store, shutdown) =
8163 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "decommission-free-consumed", &[4, 4])).await;
8164 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
8165 let bucket = format!("decom-free-consumed-{}", uuid::Uuid::new_v4());
8166 let object = "free-consumed-object";
8167 store
8168 .make_bucket(&bucket, &MakeBucketOptions::default())
8169 .await
8170 .expect("create consumed free-version bucket");
8171 let (_, free_version) = seed_transitioned_free_version(&ctx, &store, &bucket, object).await;
8172
8173 mark_test_pool_decommissioning(&store, 0).await;
8174 let source_set = store.pools[0].get_disks_by_key(object);
8175 let barrier = crate::store::object::DecommissionFreeVersionSourceRaceBarrier::install(&bucket, object);
8176 let decommission = tokio::spawn({
8177 let store = store.clone();
8178 let bucket = bucket.clone();
8179 let source_set = source_set.clone();
8180 async move {
8181 store
8182 .decommission_entry_for_test_with_bucket_incarnation(
8183 0,
8184 MetaCacheEntry {
8185 name: object.to_string(),
8186 ..Default::default()
8187 },
8188 bucket,
8189 source_set,
8190 )
8191 .await
8192 }
8193 });
8194
8195 barrier.wait_until_paused().await;
8196 store.pools[0]
8197 .delete_object(
8198 &bucket,
8199 object,
8200 ObjectOptions {
8201 versioned: true,
8202 version_id: Some(free_version.to_string()),
8203 incl_free_versions: true,
8204 ..Default::default()
8205 },
8206 )
8207 .await
8208 .expect("lifecycle should consume the source free version before decommission locks it");
8209 assert!(
8210 source_set
8211 .load_file_info_versions_exact(&bucket, object)
8212 .await
8213 .expect("consumed source metadata should remain readable")
8214 .is_none(),
8215 "the lifecycle delete should remove the source free version"
8216 );
8217
8218 barrier.release();
8219 decommission
8220 .await
8221 .expect("decommission task should join")
8222 .expect("a concurrently consumed free version should not fail source cleanup");
8223 assert!(
8224 store.pools[1]
8225 .get_disks_by_key(object)
8226 .load_file_info_versions_exact(&bucket, object)
8227 .await
8228 .expect("target metadata should remain readable")
8229 .is_none(),
8230 "an already consumed free version should not be recreated on the target"
8231 );
8232 shutdown.cancel();
8233 }
8234
8235 #[cfg(feature = "test-util")]
8236 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8237 #[serial_test::serial(storage_class_env)]
8238 async fn decommission_entry_rejects_subquorum_free_version_conflict_and_retains_source() {
8239 let temp_dir = tempfile::tempdir().expect("create sub-quorum free-version store dir");
8240 let (ctx, store, shutdown) =
8241 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "decommission-free-conflict", &[4, 4])).await;
8242 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
8243 let bucket = format!("decom-free-conflict-{}", uuid::Uuid::new_v4());
8244 let object = "free-conflict-object";
8245 store
8246 .make_bucket(&bucket, &MakeBucketOptions::default())
8247 .await
8248 .expect("create sub-quorum conflict bucket");
8249 let (_, free_version) = seed_transitioned_free_version(&ctx, &store, &bucket, object).await;
8250 let source_free = store.pools[0]
8251 .get_disks_by_key(object)
8252 .load_file_info_versions_exact(&bucket, object)
8253 .await
8254 .expect("source free version should decode before crash replay setup")
8255 .and_then(|versions| {
8256 versions
8257 .versions
8258 .into_iter()
8259 .find(|version| version.version_id == Some(free_version))
8260 })
8261 .expect("source free version should be available for crash replay setup");
8262
8263 let mut target_reader = PutObjReader::from_vec(b"target ordinary bytes".to_vec());
8264 let target = store.pools[1]
8265 .put_object(
8266 &bucket,
8267 object,
8268 &mut target_reader,
8269 &ObjectOptions {
8270 versioned: true,
8271 ..Default::default()
8272 },
8273 )
8274 .await
8275 .expect("seed ordinary target version");
8276 let target_version = target.version_id.expect("target version must have an ID");
8277 store.pools[1]
8281 .get_object_info(
8282 &bucket,
8283 object,
8284 &ObjectOptions {
8285 versioned: true,
8286 version_id: Some(target_version.to_string()),
8287 ..Default::default()
8288 },
8289 )
8290 .await
8291 .expect("target PUT tail should drain before per-disk fixture mutation");
8292 let target_disks = store.pools[1].get_disks_by_key(object).disks.read().await.clone();
8293 for disk in target_disks.iter().skip(1) {
8294 disk.as_ref()
8295 .expect("target crash replay quorum disk should be online")
8296 .write_metadata("", &bucket, object, source_free.clone())
8297 .await
8298 .expect("seed an equivalent free version on the target quorum");
8299 }
8300 let conflict_path = temp_dir
8301 .path()
8302 .join(format!("pool1/set0/disk0/{bucket}/{object}/{STORAGE_FORMAT_FILE}"));
8303 let encoded = tokio::fs::read(&conflict_path)
8304 .await
8305 .expect("target metadata should be readable");
8306 let mut metadata = FileMeta::load(&encoded).expect("target metadata should decode");
8307 let target_index = metadata
8308 .versions
8309 .iter()
8310 .position(|version| version.header.version_id == Some(target_version))
8311 .expect("target version should be present on the conflict disk");
8312 let mut target_meta = metadata.versions[target_index]
8313 .parse_version_meta()
8314 .expect("target version metadata should decode");
8315 target_meta
8316 .object
8317 .as_mut()
8318 .expect("target conflict must remain an ordinary object")
8319 .version_id = Some(free_version);
8320 metadata.versions[target_index] = target_meta.try_into().expect("conflict metadata should encode");
8321 let expected_conflict_meta = metadata.versions[target_index].meta.clone();
8322 let expected_conflict = metadata.versions[target_index]
8323 .into_fileinfo(&bucket, object, true)
8324 .expect("conflict metadata should decode as an ordinary object");
8325 let duplicate_free: rustfs_filemeta::FileMetaShallowVersion = rustfs_filemeta::FileMetaVersion::from(source_free.clone())
8326 .try_into()
8327 .expect("duplicate free metadata should encode");
8328 metadata.versions.insert(target_index, duplicate_free);
8329 tokio::fs::write(&conflict_path, metadata.marshal_msg().expect("conflict metadata should encode"))
8330 .await
8331 .expect("write sub-quorum conflict metadata");
8332
8333 let mut expected_target_metadata = Vec::with_capacity(4);
8334 for disk_index in 0..4 {
8335 let target_path = temp_dir
8336 .path()
8337 .join(format!("pool1/set0/disk{disk_index}/{bucket}/{object}/{STORAGE_FORMAT_FILE}"));
8338 let target_encoded = tokio::fs::read(&target_path)
8339 .await
8340 .expect("target fixture metadata should be readable before decommission");
8341 let target_meta = FileMeta::load(&target_encoded).expect("target fixture metadata should decode");
8342 let same_id = target_meta
8343 .versions
8344 .iter()
8345 .filter(|version| version.header.version_id == Some(free_version))
8346 .collect::<Vec<_>>();
8347 if disk_index == 0 {
8348 assert_eq!(same_id.len(), 2, "conflict fixture should contain both same-ID records");
8349 assert!(same_id[0].header.free_version());
8350 assert!(!same_id[1].header.free_version());
8351 } else {
8352 assert_eq!(same_id.len(), 1, "target fixture disk {disk_index} should contain one free version");
8353 assert!(same_id[0].header.free_version());
8354 }
8355 expected_target_metadata.push(target_encoded);
8356 }
8357
8358 mark_test_pool_decommissioning(&store, 0).await;
8359 let source_set = store.pools[0].get_disks_by_key(object);
8360 store
8361 .decommission_entry_for_test_with_bucket_incarnation(
8362 0,
8363 MetaCacheEntry {
8364 name: object.to_string(),
8365 ..Default::default()
8366 },
8367 bucket.clone(),
8368 source_set.clone(),
8369 )
8370 .await
8371 .expect("conflicted decommission entry should retain the source and retry later");
8372
8373 let source_versions = source_set
8374 .load_file_info_versions_exact(&bucket, object)
8375 .await
8376 .expect("retained source versions should decode")
8377 .expect("source free version should be retained after conflict");
8378 assert!(
8379 source_versions
8380 .versions
8381 .iter()
8382 .any(|version| { version.version_id == Some(free_version) && version.tier_free_version() })
8383 );
8384 let post_encoded = tokio::fs::read(&conflict_path)
8385 .await
8386 .expect("conflict metadata should remain readable");
8387 let post_metadata = FileMeta::load(&post_encoded).expect("post-conflict metadata should decode");
8388 let same_id = post_metadata
8389 .versions
8390 .iter()
8391 .filter(|version| version.header.version_id == Some(free_version))
8392 .collect::<Vec<_>>();
8393 assert_eq!(same_id.len(), 2, "conflict metadata should retain both same-ID records");
8394 assert_eq!(same_id.iter().filter(|version| version.header.free_version()).count(), 1);
8395 assert_eq!(same_id.iter().filter(|version| !version.header.free_version()).count(), 1);
8396 let post_conflict = same_id
8397 .into_iter()
8398 .find(|version| !version.header.free_version())
8399 .expect("ordinary conflict version must remain addressable by the source ID");
8400 let post_conflict_info = post_conflict
8401 .into_fileinfo(&bucket, object, true)
8402 .expect("post-conflict ordinary metadata should decode");
8403 assert!(!post_conflict_info.deleted && !post_conflict_info.tier_free_version());
8404 assert_eq!(post_conflict.meta, expected_conflict_meta);
8405 assert_eq!(post_conflict_info.size, expected_conflict.size);
8406 assert_eq!(post_conflict_info.data_dir, expected_conflict.data_dir);
8407 assert_eq!(post_conflict_info.metadata, expected_conflict.metadata);
8408 assert_eq!(post_conflict_info.get_etag(), expected_conflict.get_etag());
8409
8410 for (disk_index, expected_target_encoded) in expected_target_metadata.iter().enumerate() {
8411 let target_path = temp_dir
8412 .path()
8413 .join(format!("pool1/set0/disk{disk_index}/{bucket}/{object}/{STORAGE_FORMAT_FILE}"));
8414 let target_encoded = tokio::fs::read(&target_path)
8415 .await
8416 .expect("target metadata should remain readable");
8417 assert_eq!(
8418 &target_encoded, expected_target_encoded,
8419 "conflicted decommission must not mutate target disk {disk_index}"
8420 );
8421 let target_meta = FileMeta::load(&target_encoded).expect("target metadata should decode");
8422 let same_id = target_meta
8423 .versions
8424 .iter()
8425 .filter(|version| version.header.version_id == Some(free_version))
8426 .collect::<Vec<_>>();
8427 if disk_index == 0 {
8428 assert_eq!(same_id.len(), 2);
8429 assert!(same_id[0].header.free_version());
8430 assert!(!same_id[1].header.free_version());
8431 } else {
8432 assert_eq!(same_id.len(), 1);
8433 assert!(same_id[0].header.free_version());
8434 }
8435 }
8436 let sweep_err = store
8437 .check_after_decommission_for_test(0)
8438 .await
8439 .expect_err("final sweep must report the retained free version");
8440 assert!(
8441 sweep_err.to_string().contains("version(s) were found"),
8442 "unexpected final sweep error: {sweep_err}"
8443 );
8444 shutdown.cancel();
8445 }
8446
8447 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8448 #[serial_test::serial(storage_class_env)]
8449 async fn versioned_batch_delete_marker_skips_decommission_source() {
8450 let temp_dir = tempfile::tempdir().expect("create versioned batch decommission store dir");
8451 let (_ctx, store, shutdown) = without_storage_class_env(build_isolated_test_store(
8452 temp_dir.path(),
8453 "versioned-batch-decommission-delete-fence",
8454 &[4, 4, 4],
8455 ))
8456 .await;
8457 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
8458
8459 let bucket = format!("vbatch-decom-delete-{}", uuid::Uuid::new_v4());
8460 let object = "batch-object.bin";
8461 store
8462 .make_bucket(&bucket, &MakeBucketOptions::default())
8463 .await
8464 .expect("create versioned batch decommission bucket");
8465 let (_source_version, expected_source_versions) = migrate_versioned_decommission_test_object(
8466 &store,
8467 &bucket,
8468 object,
8469 b"batch source generation",
8470 "test_versioned_batch_decommission_delete_fence",
8471 )
8472 .await;
8473
8474 let delete_config_snapshot =
8475 Arc::new(crate::bucket::replication::DeleteReplicationConfigSnapshot::from_configs_for_test(
8476 s3s::dto::VersioningConfiguration {
8477 status: Some(s3s::dto::BucketVersioningStatus::from_static(s3s::dto::BucketVersioningStatus::ENABLED)),
8478 ..Default::default()
8479 },
8480 None,
8481 ));
8482 let delete_barrier = crate::store::object::VersionedDeleteMarkerCommitBarrier::install(&bucket, object);
8483 let delete_store = Arc::clone(&store);
8484 let delete_bucket = bucket.clone();
8485 let delete = tokio::spawn(async move {
8486 delete_store
8487 .delete_objects(
8488 &delete_bucket,
8489 vec![ObjectToDelete {
8490 object_name: object.to_string(),
8491 ..Default::default()
8492 }],
8493 ObjectOptions {
8494 delete_replication_config_snapshot: Some(delete_config_snapshot),
8495 ..Default::default()
8496 },
8497 )
8498 .await
8499 });
8500 delete_barrier.wait_until_paused().await;
8501
8502 let source_set = store.pools[0].get_disks_by_key(object);
8503 crate::data_movement::ensure_source_cleanup_versions_unchanged(
8504 source_set,
8505 &bucket,
8506 object,
8507 &expected_source_versions,
8508 &[],
8509 "test_versioned_batch_decommission_delete_fence",
8510 )
8511 .await
8512 .expect("batch DELETE must not publish a marker to the suspended source");
8513
8514 delete_barrier.release();
8515 let (deleted, errors) = delete.await.expect("versioned batch DELETE task should join");
8516 assert!(errors.iter().all(Option::is_none), "versioned batch DELETE should succeed: {errors:?}");
8517 assert_eq!(deleted.len(), 1);
8518 assert!(deleted[0].delete_marker, "versioned batch DELETE must return a marker");
8519 assert!(
8520 deleted[0]
8521 .delete_marker_version_id
8522 .is_some_and(|version_id| !version_id.is_nil()),
8523 "versioned batch DELETE marker must have a non-nil version ID"
8524 );
8525
8526 let mut active_marker_count = 0;
8527 for pool in store.pools.iter().skip(1) {
8528 let Some(versions) = pool
8529 .get_disks_by_key(object)
8530 .load_file_info_versions_exact(&bucket, object)
8531 .await
8532 .expect("active-pool versions should be readable")
8533 else {
8534 continue;
8535 };
8536 active_marker_count += versions.versions.iter().filter(|version| version.deleted).count();
8537 }
8538 assert_eq!(active_marker_count, 1, "batch DELETE must publish exactly one active-pool marker");
8539
8540 shutdown.cancel();
8541 }
8542
8543 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8544 #[serial_test::serial(storage_class_env)]
8545 async fn suspended_delete_marker_then_decommission_worker_converges_null_source() {
8546 let temp_dir = tempfile::tempdir().expect("create suspended decommission DELETE store dir");
8547 let (_ctx, store, shutdown) = without_storage_class_env(build_isolated_test_store(
8548 temp_dir.path(),
8549 "suspended-decommission-delete-convergence",
8550 &[4, 4],
8551 ))
8552 .await;
8553 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
8554
8555 let bucket = format!("suspended-decom-delete-{}", uuid::Uuid::new_v4());
8556 let object = "single.bin";
8557 store
8558 .make_bucket(&bucket, &MakeBucketOptions::default())
8559 .await
8560 .expect("create suspended decommission DELETE bucket");
8561 write_suspended_decommission_source(&store, &bucket, object).await;
8562 mark_test_pool_decommissioning(&store, 0).await;
8563
8564 let deleted = store
8565 .delete_object(
8566 &bucket,
8567 object,
8568 ObjectOptions {
8569 version_suspended: true,
8570 ..Default::default()
8571 },
8572 )
8573 .await
8574 .expect("a healthy reserved target must accept suspended DELETE");
8575 assert!(deleted.delete_marker);
8576 assert_eq!(deleted.version_id, Some(uuid::Uuid::nil()));
8577 let marker_mod_time = deleted.mod_time.expect("DELETE must return the marker generation");
8578 assert_suspended_null_source_present(&store, &bucket, object).await;
8579 assert_suspended_null_delete_marker_visible(&store, &bucket, object, marker_mod_time).await;
8580
8581 let source_set = store.pools[0].get_disks_by_key(object);
8582 let worker_store = Arc::clone(&store);
8583 let worker_bucket = bucket.clone();
8584 tokio::spawn(async move {
8585 worker_store
8586 .decommission_entry_for_test(
8587 0,
8588 MetaCacheEntry {
8589 name: object.to_string(),
8590 ..Default::default()
8591 },
8592 worker_bucket,
8593 source_set,
8594 )
8595 .await
8596 })
8597 .await
8598 .expect("suspended decommission worker should join")
8599 .expect("worker must converge the old null source behind the newer DELETE marker");
8600
8601 assert_decommission_source_absent(
8602 &store,
8603 &bucket,
8604 object,
8605 &ObjectOptions {
8606 version_suspended: true,
8607 ..Default::default()
8608 },
8609 )
8610 .await;
8611 assert_suspended_null_delete_marker_visible(&store, &bucket, object, marker_mod_time).await;
8612 shutdown.cancel();
8613 }
8614
8615 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8616 #[serial_test::serial(storage_class_env)]
8617 async fn suspended_batch_delete_marker_then_decommission_worker_converges_null_source() {
8618 let temp_dir = tempfile::tempdir().expect("create suspended batch decommission DELETE store dir");
8619 let (_ctx, store, shutdown) = without_storage_class_env(build_isolated_test_store(
8620 temp_dir.path(),
8621 "suspended-batch-decommission-delete-convergence",
8622 &[4, 4],
8623 ))
8624 .await;
8625 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
8626
8627 let bucket = format!("susp-batch-decom-delete-{}", uuid::Uuid::new_v4());
8628 let object = "batch.bin";
8629 store
8630 .make_bucket(&bucket, &MakeBucketOptions::default())
8631 .await
8632 .expect("create suspended batch decommission DELETE bucket");
8633 write_suspended_decommission_source(&store, &bucket, object).await;
8634 mark_test_pool_decommissioning(&store, 0).await;
8635
8636 let delete_config_snapshot =
8637 Arc::new(crate::bucket::replication::DeleteReplicationConfigSnapshot::from_configs_for_test(
8638 s3s::dto::VersioningConfiguration {
8639 status: Some(s3s::dto::BucketVersioningStatus::from_static(s3s::dto::BucketVersioningStatus::SUSPENDED)),
8640 ..Default::default()
8641 },
8642 None,
8643 ));
8644 let (deleted, errors) = store
8645 .delete_objects(
8646 &bucket,
8647 vec![ObjectToDelete {
8648 object_name: object.to_string(),
8649 ..Default::default()
8650 }],
8651 ObjectOptions {
8652 delete_replication_config_snapshot: Some(delete_config_snapshot),
8653 ..Default::default()
8654 },
8655 )
8656 .await;
8657 assert!(
8658 matches!(errors.as_slice(), [None]),
8659 "unexpected suspended batch DELETE result: {errors:?}"
8660 );
8661 assert_eq!(deleted.len(), 1);
8662 assert!(deleted[0].delete_marker);
8663 assert_eq!(deleted[0].object_name, object);
8664 assert!(
8665 deleted[0]
8666 .delete_marker_version_id
8667 .is_none_or(|version_id| version_id.is_nil()),
8668 "batch DELETE must retain the native null version identity"
8669 );
8670 let marker_mod_time = deleted[0]
8671 .delete_marker_mtime
8672 .expect("batch DELETE must return the marker generation");
8673 assert_suspended_null_source_present(&store, &bucket, object).await;
8674 assert_suspended_null_delete_marker_visible(&store, &bucket, object, marker_mod_time).await;
8675
8676 let source_set = store.pools[0].get_disks_by_key(object);
8677 let worker_store = Arc::clone(&store);
8678 let worker_bucket = bucket.clone();
8679 tokio::spawn(async move {
8680 worker_store
8681 .decommission_entry_for_test(
8682 0,
8683 MetaCacheEntry {
8684 name: object.to_string(),
8685 ..Default::default()
8686 },
8687 worker_bucket,
8688 source_set,
8689 )
8690 .await
8691 })
8692 .await
8693 .expect("suspended batch decommission worker should join")
8694 .expect("worker must converge the old null source behind the newer batch DELETE marker");
8695
8696 assert_decommission_source_absent(
8697 &store,
8698 &bucket,
8699 object,
8700 &ObjectOptions {
8701 version_suspended: true,
8702 ..Default::default()
8703 },
8704 )
8705 .await;
8706 assert_suspended_null_delete_marker_visible(&store, &bucket, object, marker_mod_time).await;
8707 shutdown.cancel();
8708 }
8709
8710 #[cfg(feature = "test-util")]
8711 #[tokio::test]
8712 #[serial_test::serial(storage_class_env)]
8713 async fn tiered_data_movement_rejects_a_stale_source_snapshot_before_target_write() {
8714 let temp_dir = tempfile::tempdir().expect("create stale-source data movement store dir");
8715 let (ctx, store, _shutdown) =
8716 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "stale-tier-source", &[4, 4])).await;
8717 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
8718 let bucket = "stale-tier-source-bucket";
8719 let object = "stale-tier-source-object";
8720 store
8721 .make_bucket(bucket, &MakeBucketOptions::default())
8722 .await
8723 .expect("bucket should be created");
8724 register_mock_tier(&ctx.tier_config_mgr(), "STALE-TIER").await;
8725 let incarnation = store
8726 .bucket_incarnation_id(bucket)
8727 .await
8728 .expect("bucket incarnation should exist");
8729 let version_id = uuid::Uuid::new_v4();
8730 let stale = FileInfo {
8731 volume: bucket.to_string(),
8732 name: object.to_string(),
8733 version_id: Some(version_id),
8734 transition_status: rustfs_filemeta::TRANSITION_COMPLETE.to_string(),
8735 transitioned_objname: "remote/stale-source".to_string(),
8736 transition_tier: "STALE-TIER".to_string(),
8737 transition_version_id: Some(uuid::Uuid::new_v4()),
8738 transition_version_state: rustfs_filemeta::TransitionVersionState::Exact,
8739 data_dir: Some(uuid::Uuid::new_v4()),
8740 mod_time: Some(OffsetDateTime::UNIX_EPOCH),
8741 size: 1,
8742 metadata: HashMap::new(),
8743 ..Default::default()
8744 };
8745 let err = store
8746 .decommission_tiered_object(
8747 bucket,
8748 object,
8749 &stale,
8750 &ObjectOptions {
8751 versioned: true,
8752 version_id: Some(version_id.to_string()),
8753 src_pool_idx: 0,
8754 data_movement: true,
8755 expected_bucket_incarnation_id: Some(incarnation),
8756 ..Default::default()
8757 },
8758 )
8759 .await
8760 .expect_err("a source removed after queue capture must fail closed");
8761 assert!(matches!(err, Error::ObjectNotFound(_, _) | Error::FileNotFound));
8762 let target_err = store.pools[1]
8763 .get_object_info(bucket, object, &ObjectOptions::default())
8764 .await
8765 .expect_err("stale source rejection must not write target metadata");
8766 assert!(matches!(
8767 target_err,
8768 StorageError::ObjectNotFound(_, _) | StorageError::VersionNotFound(_, _, _)
8769 ));
8770 }
8771
8772 #[tokio::test]
8773 #[serial_test::serial(storage_class_env)]
8774 async fn data_movement_put_conflict_validates_only_selected_target_pool() {
8775 let temp_dir = tempfile::tempdir().expect("create three-pool data movement store dir");
8776 let (_ctx, store, _shutdown) =
8777 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "data-movement-selected-target", &[4, 4, 4]))
8778 .await;
8779 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
8780
8781 let bucket = format!("dm-selected-target-{}", uuid::Uuid::new_v4());
8782 let object = "object.bin";
8783 let source_body = b"source-and-equivalent".to_vec();
8784 let conflicting_body = b"newer-conflicting-target".to_vec();
8785 let source_mod_time = OffsetDateTime::UNIX_EPOCH + time::Duration::SECOND;
8786 let version = uuid::Uuid::new_v4();
8787 store
8788 .make_bucket(&bucket, &MakeBucketOptions::default())
8789 .await
8790 .expect("create selected-target bucket");
8791 for (pool_idx, body, mod_time) in [
8792 (0, source_body.clone(), source_mod_time),
8793 (1, conflicting_body.clone(), source_mod_time + time::Duration::SECOND),
8794 (2, source_body.clone(), source_mod_time),
8795 ] {
8796 let mut reader = PutObjReader::from_vec(body);
8797 store.pools[pool_idx]
8798 .put_object(
8799 &bucket,
8800 object,
8801 &mut reader,
8802 &ObjectOptions {
8803 versioned: true,
8804 version_id: Some(version.to_string()),
8805 mod_time: Some(mod_time),
8806 ..Default::default()
8807 },
8808 )
8809 .await
8810 .expect("seed data movement pool");
8811 }
8812 *store.rebalance_meta.write().await = Some(active_rebalance_meta_for_pool(store.pools.len(), 0));
8813
8814 let source_reader = store.pools[0]
8815 .get_object_reader(
8816 &bucket,
8817 object,
8818 None,
8819 HeaderMap::new(),
8820 &ObjectOptions {
8821 versioned: true,
8822 version_id: Some(version.to_string()),
8823 raw_data_movement_read: true,
8824 ..Default::default()
8825 },
8826 )
8827 .await
8828 .expect("read source for selected-target migration");
8829 crate::data_movement::migrate_object(store.clone(), 0, bucket.clone(), source_reader, None, "test_selected_target")
8830 .await
8831 .expect_err("an equivalent object in another pool must not mask the selected target conflict");
8832
8833 let mut selected_target = store.pools[1]
8834 .get_object_reader(
8835 &bucket,
8836 object,
8837 None,
8838 HeaderMap::new(),
8839 &ObjectOptions {
8840 versioned: true,
8841 version_id: Some(version.to_string()),
8842 ..Default::default()
8843 },
8844 )
8845 .await
8846 .expect("read selected conflicting target");
8847 let mut selected_body = Vec::new();
8848 selected_target
8849 .stream
8850 .read_to_end(&mut selected_body)
8851 .await
8852 .expect("drain selected conflicting target");
8853 assert_eq!(selected_body, conflicting_body);
8854 }
8855
8856 #[tokio::test]
8857 #[serial_test::serial(storage_class_env)]
8858 async fn data_movement_target_remains_readable_before_source_cleanup() {
8859 let temp_dir = tempfile::tempdir().expect("create data movement read-window store dir");
8860 let (_ctx, store, shutdown) =
8861 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "data-movement-read-window", &[4, 4])).await;
8862 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
8863
8864 let bucket = format!("dm-read-window-{}", uuid::Uuid::new_v4());
8865 let object = "object.bin";
8866 let version = uuid::Uuid::new_v4();
8867 let mod_time = OffsetDateTime::UNIX_EPOCH + time::Duration::SECOND;
8868 store
8869 .make_bucket(&bucket, &MakeBucketOptions::default())
8870 .await
8871 .expect("create data movement read-window bucket");
8872
8873 let mut reader = PutObjReader::from_vec(b"source body".to_vec());
8874 store.pools[0]
8875 .put_object(
8876 &bucket,
8877 object,
8878 &mut reader,
8879 &ObjectOptions {
8880 versioned: true,
8881 version_id: Some(version.to_string()),
8882 mod_time: Some(mod_time),
8883 ..Default::default()
8884 },
8885 )
8886 .await
8887 .expect("seed data movement source");
8888 *store.rebalance_meta.write().await = Some(active_rebalance_meta_for_pool(store.pools.len(), 0));
8889
8890 let source_reader = store.pools[0]
8891 .get_object_reader(
8892 &bucket,
8893 object,
8894 None,
8895 HeaderMap::new(),
8896 &ObjectOptions {
8897 versioned: true,
8898 version_id: Some(version.to_string()),
8899 raw_data_movement_read: true,
8900 ..Default::default()
8901 },
8902 )
8903 .await
8904 .expect("read data movement source");
8905 let source_data_dir = source_reader.object_info.data_dir;
8906 crate::data_movement::migrate_object(store.clone(), 0, bucket.clone(), source_reader, None, "test_read_window")
8907 .await
8908 .expect("commit data movement target");
8909
8910 let target = store.pools[1]
8911 .get_object_info(
8912 &bucket,
8913 object,
8914 &ObjectOptions {
8915 versioned: true,
8916 version_id: Some(version.to_string()),
8917 ..Default::default()
8918 },
8919 )
8920 .await
8921 .expect("read committed data movement target");
8922 assert_ne!(source_data_dir, target.data_dir, "the target must use its own data directory");
8923 assert_eq!(
8924 rustfs_utils::http::get_consistent_str(&target.user_defined, rustfs_utils::http::SUFFIX_DATA_MOVED),
8925 Some("true")
8926 );
8927
8928 let resolved = store
8929 .get_object_info(
8930 &bucket,
8931 object,
8932 &ObjectOptions {
8933 versioned: true,
8934 version_id: Some(version.to_string()),
8935 ..Default::default()
8936 },
8937 )
8938 .await
8939 .expect("the equal-time migration window must remain readable");
8940 assert_eq!(resolved.data_dir, target.data_dir);
8941
8942 shutdown.cancel();
8943 }
8944
8945 #[tokio::test]
8946 #[serial_test::serial(storage_class_env)]
8947 async fn data_movement_multipart_conflict_validates_exact_version_target_pool() {
8948 let temp_dir = tempfile::tempdir().expect("create three-pool multipart data movement store dir");
8949 let (_ctx, store, _shutdown) =
8950 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "data-movement-multipart-target", &[4, 4, 4]))
8951 .await;
8952 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
8953
8954 let bucket = format!("dm-multipart-target-{}", uuid::Uuid::new_v4());
8955 let object = "object.bin";
8956 let source_version = uuid::Uuid::new_v4();
8957 let other_version = uuid::Uuid::new_v4();
8958 let source_mod_time = OffsetDateTime::UNIX_EPOCH + time::Duration::SECOND;
8959 store
8960 .make_bucket(&bucket, &MakeBucketOptions::default())
8961 .await
8962 .expect("create multipart selected-target bucket");
8963 let first_part_size = 5 * 1024 * 1024;
8964 let mut source_body = vec![b'a'; first_part_size];
8965 source_body.push(b'b');
8966 for (pool_idx, version_id, body, mod_time) in [
8967 (
8968 1,
8969 other_version,
8970 b"newer different version".to_vec(),
8971 source_mod_time + time::Duration::seconds(2),
8972 ),
8973 (2, source_version, b"conflicting exact version".to_vec(), source_mod_time),
8974 ] {
8975 let mut reader = PutObjReader::from_vec(body);
8976 store.pools[pool_idx]
8977 .put_object(
8978 &bucket,
8979 object,
8980 &mut reader,
8981 &ObjectOptions {
8982 versioned: true,
8983 version_id: Some(version_id.to_string()),
8984 mod_time: Some(mod_time),
8985 ..Default::default()
8986 },
8987 )
8988 .await
8989 .expect("seed multipart data movement pool");
8990 }
8991 let source_reader = seed_data_movement_source_reader(
8992 &store,
8993 &bucket,
8994 object,
8995 DataMovementSourceSeed {
8996 pool_idx: 0,
8997 version_id: Some(source_version),
8998 body: source_body,
8999 multipart_split: Some(first_part_size),
9000 mod_time: source_mod_time,
9001 user_defined: HashMap::new(),
9002 },
9003 )
9004 .await;
9005 *store.rebalance_meta.write().await = Some(active_rebalance_meta_for_pool(store.pools.len(), 0));
9006
9007 let completion_barrier = crate::store::multipart::DataMovementMultipartCompletionBarrier::install(&bucket);
9008 let migration_store = store.clone();
9009 let migration_bucket = bucket.clone();
9010 let migration = tokio::spawn(async move {
9011 crate::data_movement::migrate_object(
9012 migration_store,
9013 0,
9014 migration_bucket,
9015 source_reader,
9016 None,
9017 "test_multipart_selected_target",
9018 )
9019 .await
9020 });
9021 completion_barrier.wait_until_paused().await;
9022 let unrelated_set = store.pools[1].get_disks_by_key(object);
9023 let original_disks = {
9024 let mut disks = unrelated_set.disks.write().await;
9025 let original = disks.clone();
9026 for disk in disks.iter_mut().take(3) {
9027 *disk = None;
9028 }
9029 original
9030 };
9031 drop(completion_barrier);
9032 let err = migration
9033 .await
9034 .expect("multipart migration task should join")
9035 .expect_err("the exact-version target conflict must not be bypassed by a newer different version");
9036 *unrelated_set.disks.write().await = original_disks;
9037 let rendered = err.to_string();
9038 assert!(
9039 rendered.contains("complete_multipart_upload"),
9040 "the selected target should be reached despite an unrelated degraded pool: {rendered}"
9041 );
9042 assert!(
9043 !rendered.contains("put_object_part failed"),
9044 "part upload must not scan the unrelated degraded pool: {rendered}"
9045 );
9046
9047 let unexpected_version = store.pools[1]
9048 .get_object_info(
9049 &bucket,
9050 object,
9051 &ObjectOptions {
9052 versioned: true,
9053 version_id: Some(source_version.to_string()),
9054 ..Default::default()
9055 },
9056 )
9057 .await
9058 .expect_err("multipart migration must not duplicate the source version into the latest-version pool");
9059 assert!(matches!(
9060 unexpected_version,
9061 StorageError::ObjectNotFound(_, _) | StorageError::VersionNotFound(_, _, _)
9062 ));
9063 }
9064
9065 #[cfg(feature = "test-util")]
9066 #[tokio::test]
9067 #[serial_test::serial(storage_class_env)]
9068 async fn decommission_durable_ilm_target_read_error_is_not_masked_by_peer_success() {
9069 let temp_dir = tempfile::tempdir().expect("create temp store dir");
9070 let (_ctx, store, _shutdown) =
9071 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "durable-ilm-target-read-error", &[4, 4, 4]))
9072 .await;
9073 let job_id = uuid::Uuid::new_v4();
9074 let job =
9075 ManualTransitionJobRecord::new(job_id, "manual-target-read-error", &ManualTransitionRunOptions::default(), "owner");
9076 let path = manual_transition_job_record_object_name(job_id).expect("manual job path should build");
9077 let data = job.encode().expect("manual job should encode");
9078 for pool in &store.pools {
9079 com::save_config(pool.clone(), &path, data.clone())
9080 .await
9081 .expect("manual job fixture should persist in every pool");
9082 }
9083 store.pool_meta.write().await.pools[0].decommission = Some(PoolDecommissionInfo {
9084 start_time: Some(OffsetDateTime::now_utc()),
9085 ..Default::default()
9086 });
9087
9088 let failing_target = store.pools[2].get_disks_by_key(&path);
9089 let original_disks = {
9090 let mut disks = failing_target.disks.write().await;
9091 let original = disks.clone();
9092 for disk in disks.iter_mut().take(3) {
9093 *disk = None;
9094 }
9095 original
9096 };
9097 let error = store
9098 .verify_and_cleanup_decommissioned_durable_ilm_record_for_test(0, store.pools[0].get_disks_by_key(&path), &path)
9099 .await
9100 .expect_err("one target read-quorum error must fail closed despite another target success")
9101 .to_string();
9102 *failing_target.disks.write().await = original_disks;
9103
9104 assert!(error.contains(&path));
9105 assert!(error.contains("pool 2"));
9106 assert_eq!(
9107 com::read_config(store.pools[0].clone(), &path)
9108 .await
9109 .expect("target read error must retain the source"),
9110 data
9111 );
9112 assert_eq!(
9113 store
9114 .decommission_durable_ilm_receipt_count_for_test(0)
9115 .await
9116 .expect("failed target verification should not create a receipt"),
9117 0
9118 );
9119 }
9120
9121 #[cfg(feature = "test-util")]
9122 #[tokio::test]
9123 #[serial_test::serial(storage_class_env)]
9124 async fn decommission_durable_ilm_terminal_receipt_recovers_failed_source_cleanup() {
9125 let temp_dir = tempfile::tempdir().expect("create temp store dir");
9126 let (ctx, store, _shutdown) =
9127 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "durable-ilm-terminal-receipt", &[4, 4])).await;
9128 let tier_name = "DECOMMISSION-RECEIPT";
9129 let backend = register_transition_reconcile_test_tier(&ctx.tier_config_mgr(), tier_name).await;
9130 let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
9131 .await
9132 .expect("tier lease should resolve")
9133 .backend_identity();
9134 let entry = Jentry {
9135 persisted_version: 0,
9136 obj_name: "receipt-recovery-object".to_string(),
9137 version_id: "receipt-recovery-version".to_string(),
9138 tier_name: tier_name.to_string(),
9139 backend_identity: Some(backend_identity),
9140 version_id_exact: true,
9141 version_state: rustfs_filemeta::TransitionVersionState::Exact,
9142 state: TierDeleteJournalState::Committed,
9143 source: None,
9144 dispatch: None,
9145 };
9146 let path = tier_delete_journal_object_name(&entry);
9147 let data = encode_tier_delete_journal_entry(&entry).expect("tier journal should encode");
9148 com::save_config(store.pools[0].clone(), &path, data.clone())
9149 .await
9150 .expect("source tier journal should persist");
9151 com::save_config(store.pools[1].clone(), &path, data.clone())
9152 .await
9153 .expect("target tier journal should persist");
9154 let active_pool_meta = {
9155 let mut pool_meta = store.pool_meta.write().await;
9156 pool_meta.pools[0].decommission = Some(PoolDecommissionInfo {
9157 start_time: Some(OffsetDateTime::now_utc()),
9158 ..Default::default()
9159 });
9160 pool_meta.clone()
9161 };
9162 active_pool_meta
9163 .save(store.pools.clone())
9164 .await
9165 .expect("active decommission run identity should persist");
9166
9167 let source_set = store.pools[0].get_disks_by_key(&path);
9168 let barrier = SourceCleanupDeleteBarrier::install(RUSTFS_META_BUCKET, &path);
9169 let cleanup_store = store.clone();
9170 let cleanup_set = source_set.clone();
9171 let cleanup_path = path.clone();
9172 let cleanup = tokio::spawn(async move {
9173 cleanup_store
9174 .verify_and_cleanup_decommissioned_durable_ilm_record_for_test(0, cleanup_set, &cleanup_path)
9175 .await
9176 });
9177 barrier.wait_until_paused().await;
9178 let original_source_disks = {
9179 let mut disks = source_set.disks.write().await;
9180 let original = disks.clone();
9181 for disk in disks.iter_mut().take(3) {
9182 *disk = None;
9183 }
9184 original
9185 };
9186 barrier.release();
9187 let cleanup_error = cleanup
9188 .await
9189 .expect("source cleanup task should not panic")
9190 .expect_err("injected source delete quorum failure must fail cleanup")
9191 .to_string();
9192 *source_set.disks.write().await = original_source_disks;
9193 drop(barrier);
9194
9195 assert!(cleanup_error.contains("source durable ILM cleanup failed"));
9196 assert_eq!(
9197 store
9198 .decommission_durable_ilm_receipt_count_for_test(0)
9199 .await
9200 .expect("receipt should persist before source cleanup"),
9201 1
9202 );
9203 assert_eq!(
9204 com::read_config(store.pools[0].clone(), &path)
9205 .await
9206 .expect("failed cleanup must retain the source"),
9207 data
9208 );
9209
9210 let mut restarted_pool_meta = PoolMeta::default();
9211 restarted_pool_meta
9212 .load(store.pools[0].clone(), store.pools.clone())
9213 .await
9214 .expect("decommission run identity should reload after restart");
9215 *store.pool_meta.write().await = restarted_pool_meta;
9216 let stats = recover_tier_delete_journal_entries(store.clone(), 100, None)
9217 .await
9218 .expect("target recovery should commit terminal proof and delete the target");
9219 assert_eq!((stats.scanned, stats.deleted, stats.failed), (1, 1, 0));
9220 assert!(matches!(
9221 com::read_config(store.pools[1].clone(), &path).await,
9222 Err(Error::ConfigNotFound)
9223 ));
9224 assert_eq!(
9225 com::read_config(store.pools[0].clone(), &path)
9226 .await
9227 .expect("target recovery must not delete the decommission source"),
9228 data
9229 );
9230
9231 store
9232 .verify_and_cleanup_decommissioned_durable_ilm_record_for_test(0, source_set, &path)
9233 .await
9234 .expect("terminal receipt should authorize cleanup after target deletion");
9235 assert!(matches!(
9236 com::read_config(store.pools[0].clone(), &path).await,
9237 Err(Error::ConfigNotFound)
9238 ));
9239 assert!(backend.remove_versions().await.contains(&(entry.obj_name, entry.version_id)));
9240 }
9241
9242 #[cfg(feature = "test-util")]
9243 #[tokio::test]
9244 #[serial_test::serial(storage_class_env)]
9245 async fn decommission_final_sweep_blocks_cancel_until_source_cleanup_finishes() {
9246 let temp_dir = tempfile::tempdir().expect("create final sweep gate store dir");
9247 let (_ctx, store, _shutdown) =
9248 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "durable-ilm-final-sweep-gate", &[4, 4])).await;
9249 let job_id = uuid::Uuid::new_v4();
9250 let job = ManualTransitionJobRecord::new(job_id, "final-sweep-gate", &ManualTransitionRunOptions::default(), "owner");
9251 let path = manual_transition_job_record_object_name(job_id).expect("manual job path should build");
9252 let data = job.encode().expect("manual job should encode");
9253 for pool in &store.pools {
9254 com::save_config(pool.clone(), &path, data.clone())
9255 .await
9256 .expect("manual job fixture should persist in both pools");
9257 }
9258 let active_pool_meta = {
9259 let mut pool_meta = store.pool_meta.write().await;
9260 pool_meta.pools[0].decommission = Some(PoolDecommissionInfo {
9261 start_time: Some(OffsetDateTime::now_utc()),
9262 ..Default::default()
9263 });
9264 pool_meta.clone()
9265 };
9266 active_pool_meta
9267 .save(store.pools.clone())
9268 .await
9269 .expect("active decommission run identity should persist");
9270
9271 let barrier = SourceCleanupDeleteBarrier::install(RUSTFS_META_BUCKET, &path);
9272 let final_sweep = tokio::spawn({
9273 let store = store.clone();
9274 async move { store.check_after_decommission_for_test(0).await }
9275 });
9276 barrier.wait_until_paused().await;
9277
9278 let mut cancel = tokio::spawn({
9279 let store = store.clone();
9280 async move { store.decommission_cancel(0).await }
9281 });
9282 assert!(
9283 tokio::time::timeout(Duration::from_millis(100), &mut cancel).await.is_err(),
9284 "cancel must wait for the final sweep source cleanup"
9285 );
9286 {
9287 let pool_meta = store.pool_meta.read().await;
9288 let decommission = pool_meta.pools[0]
9289 .decommission
9290 .as_ref()
9291 .expect("decommission state should remain present");
9292 assert!(
9293 decommission.canceled,
9294 "cancel must publish its durable terminal state before waiting for the final sweep"
9295 );
9296 assert!(
9297 decommission.start_time.is_none(),
9298 "the durable cancel state must clear the active run identity before quiescence"
9299 );
9300 }
9301
9302 barrier.release();
9303 final_sweep
9304 .await
9305 .expect("final sweep task should not panic")
9306 .expect("final sweep should finish after the barrier releases");
9307 cancel
9308 .await
9309 .expect("cancel task should not panic")
9310 .expect("cancel should complete after the final sweep releases the operation gate");
9311 let pool_meta = store.pool_meta.read().await;
9312 let decommission = pool_meta.pools[0]
9313 .decommission
9314 .as_ref()
9315 .expect("decommission state should remain present");
9316 assert!(decommission.canceled);
9317 assert!(decommission.start_time.is_none());
9318 }
9319
9320 #[cfg(feature = "test-util")]
9321 #[tokio::test]
9322 #[serial_test::serial(storage_class_env)]
9323 async fn decommission_durable_ilm_recovery_keeps_multiple_active_sources() {
9324 let temp_dir = tempfile::tempdir().expect("create multi-source recovery store dir");
9325 let (ctx, store, _shutdown) = without_storage_class_env(build_isolated_test_store(
9326 temp_dir.path(),
9327 "durable-ilm-multi-source-recovery",
9328 &[4, 4, 4],
9329 ))
9330 .await;
9331 let tier_name = "DECOMMISSION-MULTI-SOURCE";
9332 let backend = register_transition_reconcile_test_tier(&ctx.tier_config_mgr(), tier_name).await;
9333 let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
9334 .await
9335 .expect("tier lease should resolve")
9336 .backend_identity();
9337 let entry = Jentry {
9338 persisted_version: 0,
9339 obj_name: "multi-source-recovery-object".to_string(),
9340 version_id: "multi-source-recovery-version".to_string(),
9341 tier_name: tier_name.to_string(),
9342 backend_identity: Some(backend_identity),
9343 version_id_exact: true,
9344 version_state: rustfs_filemeta::TransitionVersionState::Exact,
9345 state: TierDeleteJournalState::Committed,
9346 source: None,
9347 dispatch: None,
9348 };
9349 let path = tier_delete_journal_object_name(&entry);
9350 let data = encode_tier_delete_journal_entry(&entry).expect("tier journal should encode");
9351 for pool in &store.pools {
9352 com::save_config(pool.clone(), &path, data.clone())
9353 .await
9354 .expect("source and target tier journals should persist");
9355 }
9356
9357 let active_pool_meta = {
9358 let mut pool_meta = store.pool_meta.write().await;
9359 let start_time = OffsetDateTime::now_utc();
9360 for pool_idx in [0, 1] {
9361 pool_meta.pools[pool_idx].decommission = Some(PoolDecommissionInfo {
9362 start_time: Some(start_time),
9363 ..Default::default()
9364 });
9365 }
9366 pool_meta.clone()
9367 };
9368 active_pool_meta
9369 .save(store.pools.clone())
9370 .await
9371 .expect("multiple active decommission runs should persist");
9372 let mut restarted_pool_meta = PoolMeta::default();
9373 restarted_pool_meta
9374 .load(store.pools[0].clone(), store.pools.clone())
9375 .await
9376 .expect("multiple active decommission runs should reload");
9377 *store.pool_meta.write().await = restarted_pool_meta;
9378
9379 let record = validate_durable_ilm_record(&path, &data).expect("tier journal should validate");
9380 let source_zero_receipt = store
9381 .persist_decommission_durable_ilm_receipt_for_test(0, 1, &path, &record, false)
9382 .await
9383 .expect("source pool zero receipt should persist on the other active source");
9384 let source_one_receipt = store
9385 .persist_decommission_durable_ilm_receipt_for_test(1, 0, &path, &record, false)
9386 .await
9387 .expect("source pool one receipt should persist on the other active source");
9388 assert_ne!(
9389 source_zero_receipt, source_one_receipt,
9390 "active source runs must have distinct receipt paths"
9391 );
9392
9393 let stats = recover_tier_delete_journal_entries(store.clone(), 100, None)
9394 .await
9395 .expect("cross-source receipts should not remove active source journals");
9396 assert_eq!((stats.scanned, stats.deleted, stats.failed), (1, 1, 0));
9397 for pool in &store.pools {
9398 assert_eq!(
9399 com::read_config(pool.clone(), &path)
9400 .await
9401 .expect("cross-source receipts alone must retain every journal copy"),
9402 data
9403 );
9404 }
9405
9406 store
9407 .persist_decommission_durable_ilm_receipt_for_test(0, 2, &path, &record, false)
9408 .await
9409 .expect("source pool zero receipt should persist on the target");
9410 store
9411 .persist_decommission_durable_ilm_receipt_for_test(1, 2, &path, &record, false)
9412 .await
9413 .expect("source pool one receipt should persist on the target");
9414
9415 let stats = recover_tier_delete_journal_entries(store.clone(), 100, None)
9416 .await
9417 .expect("multi-source tier journal recovery should complete");
9418 assert_eq!((stats.scanned, stats.deleted, stats.failed), (1, 1, 0));
9419 assert_eq!(
9420 com::read_config(store.pools[0].clone(), &path)
9421 .await
9422 .expect("first active source must remain after target recovery"),
9423 data
9424 );
9425 assert_eq!(
9426 com::read_config(store.pools[1].clone(), &path)
9427 .await
9428 .expect("second active source must remain after target recovery"),
9429 data
9430 );
9431 assert!(matches!(
9432 com::read_config(store.pools[2].clone(), &path).await,
9433 Err(Error::ConfigNotFound)
9434 ));
9435 assert!(backend.remove_versions().await.contains(&(entry.obj_name, entry.version_id)));
9436 }
9437
9438 #[cfg(feature = "test-util")]
9439 #[tokio::test]
9440 #[serial_test::serial(storage_class_env)]
9441 async fn decommission_durable_ilm_receipt_pagination_fails_closed_on_second_page() {
9442 const RECEIPT_COUNT: usize = 1001;
9443
9444 let temp_dir = tempfile::tempdir().expect("create paginated receipt store dir");
9445 let (_ctx, store, _shutdown) =
9446 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "durable-ilm-receipt-pages", &[4, 4])).await;
9447 store.pool_meta.write().await.pools[0].decommission = Some(PoolDecommissionInfo {
9448 start_time: Some(OffsetDateTime::now_utc()),
9449 ..Default::default()
9450 });
9451
9452 futures::stream::iter(0..RECEIPT_COUNT)
9453 .map(|index| {
9454 let store = store.clone();
9455 async move {
9456 let id = format!("{index:064x}");
9457 let source_path = format!("ilm/tier-delete-journal/{id}.json");
9458 let record = ValidatedDurableIlmRecord {
9459 namespace: "tier-delete-journal",
9460 id_kind: "operation_id",
9461 id,
9462 checkpoint: DurableIlmRecordCheckpoint::TierDeleteJournal {
9463 content_sha256: format!("{:064x}", index + RECEIPT_COUNT),
9464 identity_sha256: "f".repeat(64),
9465 committed: false,
9466 dispatch_identity_sha256: None,
9467 state: None,
9468 },
9469 };
9470 store
9471 .persist_decommission_durable_ilm_receipt_for_test(0, 0, &source_path, &record, true)
9472 .await?;
9473 store
9474 .persist_decommission_durable_ilm_receipt_for_test(0, 1, &source_path, &record, true)
9475 .await?;
9476 Ok::<(), Error>(())
9477 }
9478 })
9479 .buffer_unordered(32)
9480 .try_collect::<Vec<_>>()
9481 .await
9482 .expect("more than one receipt page should persist");
9483 store
9484 .persist_decommission_durable_ilm_manifest_for_test(0)
9485 .await
9486 .expect("paginated source receipts should produce a manifest");
9487
9488 let target_receipts = store
9489 .decommission_durable_ilm_receipt_paths_for_test(0)
9490 .await
9491 .expect("paginated target receipts should list");
9492 assert_eq!(target_receipts.len(), RECEIPT_COUNT);
9493 let (target_pool_idx, second_page_path) = target_receipts
9494 .get(1000)
9495 .cloned()
9496 .expect("the real 1000-item page boundary should expose a second page receipt");
9497 let receipt_bytes = com::read_config(store.pools[target_pool_idx].clone(), &second_page_path)
9498 .await
9499 .expect("second page receipt should be readable");
9500
9501 com::delete_config(store.pools[target_pool_idx].clone(), &second_page_path)
9502 .await
9503 .expect("second page receipt should delete");
9504 let missing = store
9505 .complete_decommission(0)
9506 .await
9507 .expect_err("a missing second page receipt must block completion")
9508 .to_string();
9509 assert!(missing.contains(&second_page_path));
9510 assert!(
9511 !store.pool_meta.read().await.pools[0]
9512 .decommission
9513 .as_ref()
9514 .expect("source pool should remain in decommission")
9515 .complete
9516 );
9517 assert!(com::read_config(store.pools[0].clone(), &second_page_path).await.is_ok());
9518
9519 let full_tail = ObjectOptions {
9520 max_parity: true,
9521 write_completion: crate::object_api::WriteCompletion::TailDrained,
9522 ..Default::default()
9523 };
9524 com::save_config_with_opts(store.pools[target_pool_idx].clone(), &second_page_path, receipt_bytes.clone(), &full_tail)
9525 .await
9526 .expect("second page receipt should restore");
9527 com::save_config_with_opts(store.pools[target_pool_idx].clone(), &second_page_path, b"{corrupt".to_vec(), &full_tail)
9528 .await
9529 .expect("second page receipt should corrupt deterministically");
9530 let corrupt = store
9531 .complete_decommission(0)
9532 .await
9533 .expect_err("a corrupt second page receipt must block completion")
9534 .to_string();
9535 assert!(corrupt.contains(&second_page_path));
9536 assert!(corrupt.contains("invalid"));
9537 assert!(
9538 !store.pool_meta.read().await.pools[0]
9539 .decommission
9540 .as_ref()
9541 .expect("source pool should remain in decommission")
9542 .complete
9543 );
9544 assert!(com::read_config(store.pools[0].clone(), &second_page_path).await.is_ok());
9545 }
9546
9547 #[cfg(feature = "test-util")]
9548 #[test]
9549 #[serial_test::serial(storage_class_env)]
9550 fn decommission_migrates_and_verifies_registered_durable_ilm_records() {
9551 std::thread::Builder::new()
9552 .name("durable-ilm-decommission-test".to_string())
9553 .stack_size(16 * 1024 * 1024)
9554 .spawn(|| {
9555 let runtime = tokio::runtime::Builder::new_multi_thread()
9556 .enable_all()
9557 .worker_threads(2)
9558 .build()
9559 .expect("durable ILM decommission runtime should build");
9560 runtime.block_on(decommission_migrates_and_verifies_registered_durable_ilm_records_scenario());
9561 })
9562 .expect("durable ILM decommission scenario thread should spawn")
9563 .join()
9564 .expect("durable ILM decommission scenario should not panic");
9565 }
9566
9567 #[cfg(feature = "test-util")]
9568 async fn decommission_migrates_and_verifies_registered_durable_ilm_records_scenario() {
9569 let temp_dir = tempfile::tempdir().expect("create temp store dir");
9570 let (ctx, store, _shutdown) =
9571 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "durable-ilm-decommission", &[4, 4])).await;
9572
9573 let tier_name = "DECOMMISSION-ILM";
9574 let backend = register_transition_reconcile_test_tier(&ctx.tier_config_mgr(), tier_name).await;
9575 let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
9576 .await
9577 .expect("tier lease should resolve")
9578 .backend_identity();
9579 let tier_entry = Jentry {
9580 persisted_version: 0,
9581 obj_name: "decommissioned-remote-object".to_string(),
9582 version_id: "decommissioned-remote-version".to_string(),
9583 tier_name: tier_name.to_string(),
9584 backend_identity: Some(backend_identity),
9585 version_id_exact: true,
9586 version_state: rustfs_filemeta::TransitionVersionState::Exact,
9587 state: TierDeleteJournalState::Committed,
9588 source: None,
9589 dispatch: None,
9590 };
9591 let tier_path = tier_delete_journal_object_name(&tier_entry);
9592 let tier_bytes = encode_tier_delete_journal_entry(&tier_entry).expect("tier journal should encode");
9593
9594 let mut transaction = TransitionTransaction::new(TransitionTransactionInit {
9595 deployment_id: ctx.deployment_id().expect("test store should initialize deployment id"),
9596 transaction_id: uuid::Uuid::new_v4(),
9597 owner_epoch: uuid::Uuid::new_v4(),
9598 write_id: uuid::Uuid::new_v4(),
9599 source: TransitionSourceIdentity {
9600 bucket: "source-bucket".to_string(),
9601 object: "source-object".to_string(),
9602 version_id: Some(uuid::Uuid::new_v4()),
9603 data_dir: uuid::Uuid::new_v4(),
9604 mod_time_unix_nanos: 1_770_000_000_000_000_000,
9605 size: 42,
9606 etag: "source-etag".to_string(),
9607 version_mode: TransitionSourceVersionMode::Versioned,
9608 },
9609 tier_name: tier_name.to_string(),
9610 backend_fingerprint: backend_identity,
9611 not_after_unix_nanos: 1_780_000_000_000_000_000,
9612 })
9613 .expect("transition transaction should build");
9614 let regressed_transaction = transaction.clone();
9615 transaction
9616 .advance(transaction.fence(), TransitionTransactionState::UploadOutcomeUnknown, None)
9617 .expect("transition transaction should advance before migration");
9618 let transaction_path = transition_transaction_record_object_name(transaction.transaction_id)
9619 .expect("transition transaction path should build");
9620 let transaction_bytes = transaction.encode().expect("transition transaction should encode");
9621
9622 let manual_job_id = uuid::Uuid::new_v4();
9623 let manual_bucket = format!("manual-decommission-{}", manual_job_id.simple());
9624 let manual_options = ManualTransitionRunOptions {
9625 prefix: "logs/".to_string(),
9626 tier: Some(tier_name.to_string()),
9627 ..Default::default()
9628 };
9629 let mut manual_job = ManualTransitionJobRecord::new(manual_job_id, &manual_bucket, &manual_options, "old-owner");
9630 manual_job.scan_completed = true;
9631 manual_job.report.enqueued = 1;
9632 manual_job.lease_expires_at_unix_nanos = 0;
9633 let manual_scope = ManualTransitionScopeAdmission::from_job(&manual_job);
9634 let task_key = manual_transition_worker_result_task_key(&manual_bucket, "logs/a", None);
9635 let manual_task = ManualTransitionTaskRecord::new(manual_job_id, &task_key, &manual_bucket, "logs/a", None, tier_name);
9636 let manual_result =
9637 ManualTransitionWorkerResultRecord::new(manual_job_id, &task_key, ManualTransitionWorkerResult::Completed);
9638
9639 let manual_job_path = manual_transition_job_record_object_name(manual_job_id).expect("manual job path should build");
9640 let manual_scope_path =
9641 manual_transition_scope_record_object_name(&manual_scope.scope_key).expect("manual scope path should build");
9642 let manual_task_path =
9643 manual_transition_task_object_name(manual_job_id, &task_key).expect("manual task path should build");
9644 let manual_result_path = manual_transition_worker_result_object_name(manual_job_id, &task_key)
9645 .expect("manual worker result path should build");
9646 let manual_job_bytes = manual_job.encode().expect("manual job should encode");
9647 let manual_scope_bytes = serde_json::to_vec(&manual_scope).expect("manual scope should encode");
9648 let manual_task_bytes = manual_task.encode().expect("manual task should encode");
9649 let manual_result_bytes = manual_result.encode().expect("manual result should encode");
9650
9651 let records = vec![
9652 (tier_path.clone(), tier_bytes.clone()),
9653 (transaction_path.clone(), transaction_bytes.clone()),
9654 (manual_job_path.clone(), manual_job_bytes.clone()),
9655 (manual_scope_path.clone(), manual_scope_bytes.clone()),
9656 (manual_task_path.clone(), manual_task_bytes.clone()),
9657 (manual_result_path.clone(), manual_result_bytes.clone()),
9658 ];
9659 for (path, data) in &records {
9660 com::save_config(store.pools[0].clone(), path, data.clone())
9661 .await
9662 .expect("durable ILM source record should persist");
9663 }
9664
9665 let legacy_queue = [com::CONFIG_PREFIX, BUCKET_META_PREFIX]
9666 .into_iter()
9667 .map(|prefix| {
9668 DecomBucketInfo {
9669 name: RUSTFS_META_BUCKET.to_string(),
9670 prefix: prefix.to_string(),
9671 }
9672 .to_string()
9673 })
9674 .collect();
9675 let legacy_pool_meta = {
9676 let mut pool_meta = store.pool_meta.write().await;
9677 pool_meta.pools[0].decommission = Some(PoolDecommissionInfo {
9678 queued: true,
9679 queued_buckets: legacy_queue,
9680 ..Default::default()
9681 });
9682 pool_meta.clone()
9683 };
9684 legacy_pool_meta
9685 .save(store.pools.clone())
9686 .await
9687 .expect("legacy decommission queue should persist before restart");
9688 let mut restarted_pool_meta = PoolMeta::default();
9689 restarted_pool_meta
9690 .load(store.pools[0].clone(), store.pools.clone())
9691 .await
9692 .expect("legacy decommission queue should reload after restart");
9693 *store.pool_meta.write().await = restarted_pool_meta;
9694 set_test_decommission_capacity_override(&store, 0);
9695 store
9696 .promote_queued_decommission_for_test(0)
9697 .await
9698 .expect("legacy queued decommission should resume");
9699 let expected_ilm_queue = DecomBucketInfo {
9700 name: RUSTFS_META_BUCKET.to_string(),
9701 prefix: ILM_META_PREFIX.to_string(),
9702 }
9703 .to_string();
9704 {
9705 let pool_meta = store.pool_meta.read().await;
9706 let decommission = pool_meta.pools[0]
9707 .decommission
9708 .as_ref()
9709 .expect("decommission state should remain present");
9710 assert!(!decommission.queued);
9711 assert!(decommission.queued_buckets.contains(&expected_ilm_queue));
9712 }
9713
9714 let ilm_bucket = DecomBucketInfo {
9715 name: RUSTFS_META_BUCKET.to_string(),
9716 prefix: ILM_META_PREFIX.to_string(),
9717 };
9718 for _ in 0..2 {
9719 store
9720 .decommission_pool_for_test(CancellationToken::new(), 0, store.pools[0].clone(), ilm_bucket.clone())
9721 .await
9722 .expect("durable ILM decommission should be idempotent");
9723 }
9724 for (path, expected) in &records {
9725 assert_eq!(
9726 com::read_config(store.pools[0].clone(), path)
9727 .await
9728 .expect("source should remain until the final sweep"),
9729 *expected
9730 );
9731 assert_eq!(
9732 com::read_config(store.pools[1].clone(), path)
9733 .await
9734 .expect("target should contain the migrated record"),
9735 *expected
9736 );
9737 }
9738 assert_eq!(
9739 store
9740 .decommission_durable_ilm_receipt_count_for_test(0)
9741 .await
9742 .expect("no receipt should exist before the final sweep"),
9743 0
9744 );
9745 let isolated_tier_stats = recover_tier_delete_journal_entries(store.clone(), 100, None)
9746 .await
9747 .expect("tier recovery should retain a terminal record until its receipt is committed");
9748 assert!(isolated_tier_stats.scanned >= 1);
9749 assert_eq!(isolated_tier_stats.deleted, 0);
9750 assert!(isolated_tier_stats.failed >= 1);
9751 assert_eq!(
9752 com::read_config(store.pools[1].clone(), &tier_path)
9753 .await
9754 .expect("receipt isolation must retain the target tier journal"),
9755 tier_bytes
9756 );
9757
9758 com::delete_config(store.pools[1].clone(), &manual_job_path)
9759 .await
9760 .expect("target manual job should delete");
9761 let missing = store
9762 .verify_and_cleanup_decommissioned_durable_ilm_record_for_test(
9763 0,
9764 store.pools[0].get_disks_by_key(&manual_job_path),
9765 &manual_job_path,
9766 )
9767 .await
9768 .expect_err("missing target must block source cleanup");
9769 let missing = missing.to_string();
9770 assert!(missing.contains(&manual_job_path) && missing.contains(&manual_job_id.to_string()));
9771 assert_eq!(
9772 com::read_config(store.pools[0].clone(), &manual_job_path)
9773 .await
9774 .expect("missing target must retain source"),
9775 manual_job_bytes
9776 );
9777 com::save_config(store.pools[1].clone(), &manual_job_path, manual_job_bytes.clone())
9778 .await
9779 .expect("target manual job should restore");
9780
9781 com::save_config(store.pools[1].clone(), &transaction_path, b"{corrupt".to_vec())
9782 .await
9783 .expect("target transaction should corrupt deterministically");
9784 let corrupt = store
9785 .verify_and_cleanup_decommissioned_durable_ilm_record_for_test(
9786 0,
9787 store.pools[0].get_disks_by_key(&transaction_path),
9788 &transaction_path,
9789 )
9790 .await
9791 .expect_err("corrupt target must block source cleanup");
9792 let corrupt = corrupt.to_string();
9793 assert!(corrupt.contains(&transaction_path) && corrupt.contains(&transaction.transaction_id.to_string()));
9794 assert_eq!(
9795 com::read_config(store.pools[0].clone(), &transaction_path)
9796 .await
9797 .expect("corrupt target must retain source"),
9798 transaction_bytes
9799 );
9800 com::save_config(store.pools[1].clone(), &transaction_path, transaction_bytes.clone())
9801 .await
9802 .expect("target transaction should restore");
9803
9804 com::save_config(store.pools[1].clone(), &manual_scope_path, manual_scope_bytes.clone())
9805 .await
9806 .expect("target scope rewrite should invalidate cached metadata before the quorum check");
9807 let target_scope_set = store.pools[1].get_disks_by_key(&manual_scope_path);
9808 let original_target_scope_disks = {
9809 let mut disks = target_scope_set.disks.write().await;
9810 let original = disks.clone();
9811 for disk in disks.iter_mut().take(3) {
9812 *disk = None;
9813 }
9814 original
9815 };
9816 let quorum_error = store
9817 .verify_and_cleanup_decommissioned_durable_ilm_record_for_test(
9818 0,
9819 store.pools[0].get_disks_by_key(&manual_scope_path),
9820 &manual_scope_path,
9821 )
9822 .await
9823 .expect_err("target below read quorum must block source cleanup");
9824 *target_scope_set.disks.write().await = original_target_scope_disks;
9825 let quorum_error = quorum_error.to_string();
9826 assert!(quorum_error.contains(&manual_scope_path) && quorum_error.contains(&manual_job_id.to_string()));
9827 assert!(com::read_config(store.pools[0].clone(), &manual_scope_path).await.is_ok());
9828
9829 com::save_config(store.pools[1].clone(), &manual_task_path, manual_task_bytes.clone())
9830 .await
9831 .expect("target task rewrite should invalidate cached metadata before the quorum check");
9832 let manual_task_record =
9833 validate_durable_ilm_record(&manual_task_path, &manual_task_bytes).expect("manual task should validate");
9834 let manual_task_receipt_path = store
9835 .decommission_durable_ilm_receipt_path_for_test(0, &manual_task_path, &manual_task_record)
9836 .await
9837 .expect("manual task receipt path should resolve");
9838 let target_task_set = store.pools[1].get_disks_by_key(&manual_task_receipt_path);
9839 let offline_target_task_disks = force_set_disk_range_offline_for_test(&target_task_set, 0..2).await;
9840 let receipt_quorum_error = store
9841 .verify_and_cleanup_decommissioned_durable_ilm_record_for_test(
9842 0,
9843 store.pools[0].get_disks_by_key(&manual_task_path),
9844 &manual_task_path,
9845 )
9846 .await
9847 .expect_err("target read quorum without receipt write quorum must retain the source");
9848 drop(offline_target_task_disks);
9849 let receipt_quorum_error = receipt_quorum_error.to_string();
9850 assert!(receipt_quorum_error.contains("receipt"));
9851 assert!(receipt_quorum_error.contains(&manual_task_path));
9852 assert!(receipt_quorum_error.contains(&manual_job_id.to_string()));
9853 assert!(com::read_config(store.pools[0].clone(), &manual_task_path).await.is_ok());
9854 store
9855 .verify_and_cleanup_decommissioned_durable_ilm_record_for_test(
9856 0,
9857 store.pools[0].get_disks_by_key(&manual_task_path),
9858 &manual_task_path,
9859 )
9860 .await
9861 .expect("healthy target should persist the receipt before source cleanup");
9862
9863 let unknown_path = "ilm/future-durable/jobs/one.json";
9864 com::save_config(store.pools[0].clone(), unknown_path, b"{}".to_vec())
9865 .await
9866 .expect("unknown durable ILM record should persist for the guard test");
9867 let unknown_migration = store
9868 .decommission_pool_for_test(CancellationToken::new(), 0, store.pools[0].clone(), ilm_bucket)
9869 .await
9870 .expect_err("unregistered durable ILM namespace must block migration");
9871 assert!(unknown_migration.to_string().contains(unknown_path));
9872 let unknown_final_sweep = store
9873 .check_after_decommission_for_test(0)
9874 .await
9875 .expect_err("unregistered durable ILM namespace must block completion");
9876 assert!(unknown_final_sweep.to_string().contains(unknown_path));
9877 com::delete_config(store.pools[0].clone(), unknown_path)
9878 .await
9879 .expect("unknown guard fixture should be removed before the successful final sweep");
9880
9881 store
9882 .check_after_decommission_for_test(0)
9883 .await
9884 .expect("production final sweep should validate every target before cleanup");
9885 assert_eq!(
9886 store
9887 .decommission_durable_ilm_receipt_count_for_test(0)
9888 .await
9889 .expect("durable ILM receipts should be listable"),
9890 records.len(),
9891 "every cleaned source record must have a durable validation receipt"
9892 );
9893 for (path, expected) in &records {
9894 assert!(
9895 matches!(com::read_config(store.pools[0].clone(), path).await, Err(Error::ConfigNotFound)),
9896 "final sweep should remove the validated source `{path}`"
9897 );
9898 assert_eq!(
9899 com::read_config(store.pools[1].clone(), path)
9900 .await
9901 .expect("final sweep must preserve the target"),
9902 *expected
9903 );
9904 }
9905
9906 let mut crash_restarted_pool_meta = PoolMeta::default();
9907 crash_restarted_pool_meta
9908 .load(store.pools[0].clone(), store.pools.clone())
9909 .await
9910 .expect("pool metadata should reload after the simulated pre-complete crash");
9911 *store.pool_meta.write().await = crash_restarted_pool_meta;
9912
9913 let (manual_job_receipt_pool, manual_job_receipt_path) = store
9914 .decommission_durable_ilm_receipt_paths_for_test(0)
9915 .await
9916 .expect("durable ILM receipt paths should be listable")
9917 .into_iter()
9918 .find(|(_, path)| path.contains(&manual_job_path))
9919 .expect("manual job should have one target receipt");
9920 let manual_job_receipt_bytes = com::read_config(store.pools[manual_job_receipt_pool].clone(), &manual_job_receipt_path)
9921 .await
9922 .expect("manual job receipt should be readable before deletion");
9923 com::delete_config(store.pools[manual_job_receipt_pool].clone(), &manual_job_receipt_path)
9924 .await
9925 .expect("manual job receipt should delete after source cleanup");
9926 com::delete_config(store.pools[1].clone(), &manual_job_path)
9927 .await
9928 .expect("post-crash target manual job should delete");
9929 let missing_after_crash = store
9930 .complete_decommission(0)
9931 .await
9932 .expect_err("completion must reject a missing target after source cleanup and restart")
9933 .to_string();
9934 assert!(missing_after_crash.contains("receipt"));
9935 assert!(missing_after_crash.contains(&manual_job_path));
9936 assert!(missing_after_crash.contains(&manual_job_id.to_string()));
9937 assert!(
9938 !store.pool_meta.read().await.pools[0]
9939 .decommission
9940 .as_ref()
9941 .expect("decommission state should survive restart")
9942 .complete
9943 );
9944 com::save_config(store.pools[1].clone(), &manual_job_path, manual_job_bytes.clone())
9945 .await
9946 .expect("post-crash target manual job should restore");
9947 com::save_config(
9948 store.pools[manual_job_receipt_pool].clone(),
9949 &manual_job_receipt_path,
9950 manual_job_receipt_bytes,
9951 )
9952 .await
9953 .expect("manual job receipt should restore after the missing-receipt check");
9954
9955 com::save_config(store.pools[1].clone(), &transaction_path, b"{corrupt".to_vec())
9956 .await
9957 .expect("post-crash target transaction should corrupt deterministically");
9958 let corrupt_after_crash = store
9959 .complete_decommission(0)
9960 .await
9961 .expect_err("completion must reject a corrupt target after source cleanup and restart")
9962 .to_string();
9963 assert!(corrupt_after_crash.contains(&transaction_path));
9964 assert!(corrupt_after_crash.contains(&transaction.transaction_id.to_string()));
9965 com::save_config(store.pools[1].clone(), &transaction_path, transaction_bytes.clone())
9966 .await
9967 .expect("post-crash target transaction should restore");
9968
9969 let mut wrong_manual_job = manual_job.clone();
9970 wrong_manual_job.job_id = uuid::Uuid::new_v4();
9971 com::save_config(
9972 store.pools[1].clone(),
9973 &manual_job_path,
9974 wrong_manual_job.encode().expect("wrong-id job should encode"),
9975 )
9976 .await
9977 .expect("post-crash target manual job should accept the wrong-id fixture");
9978 let wrong_id_after_crash = store
9979 .complete_decommission(0)
9980 .await
9981 .expect_err("completion must reject a target record with the wrong id")
9982 .to_string();
9983 assert!(wrong_id_after_crash.contains(&manual_job_path));
9984 assert!(wrong_id_after_crash.contains(&manual_job_id.to_string()));
9985 com::save_config(store.pools[1].clone(), &manual_job_path, manual_job_bytes.clone())
9986 .await
9987 .expect("post-crash target manual job should restore after the wrong-id check");
9988
9989 com::save_config(
9990 store.pools[1].clone(),
9991 &transaction_path,
9992 regressed_transaction
9993 .encode()
9994 .expect("regressed transition transaction should encode"),
9995 )
9996 .await
9997 .expect("post-crash target transaction should accept the regression fixture");
9998 let regression_after_crash = store
9999 .complete_decommission(0)
10000 .await
10001 .expect_err("completion must reject a lower transition transaction revision")
10002 .to_string();
10003 assert!(regression_after_crash.contains("generation mismatch"));
10004 assert!(regression_after_crash.contains(&transaction_path));
10005 assert!(regression_after_crash.contains(&transaction.transaction_id.to_string()));
10006 com::save_config(store.pools[1].clone(), &transaction_path, transaction_bytes.clone())
10007 .await
10008 .expect("post-crash target transaction should restore after the regression check");
10009
10010 let (manual_task_receipt_pool, manual_task_receipt_path) = store
10011 .decommission_durable_ilm_receipt_paths_for_test(0)
10012 .await
10013 .expect("durable ILM receipt paths should be listable")
10014 .into_iter()
10015 .find(|(_, path)| path.contains(&manual_task_path))
10016 .expect("manual task receipt should retain its reversible source path");
10017 let manual_task_receipt_bytes =
10018 com::read_config(store.pools[manual_task_receipt_pool].clone(), &manual_task_receipt_path)
10019 .await
10020 .expect("manual task receipt should be readable before corruption");
10021 com::save_config(
10022 store.pools[manual_task_receipt_pool].clone(),
10023 &manual_task_receipt_path,
10024 b"{corrupt".to_vec(),
10025 )
10026 .await
10027 .expect("manual task receipt should corrupt deterministically");
10028 let corrupt_receipt = store
10029 .complete_decommission(0)
10030 .await
10031 .expect_err("completion must fail closed on a corrupt receipt")
10032 .to_string();
10033 assert!(corrupt_receipt.contains(&manual_task_path));
10034 assert!(corrupt_receipt.contains(&manual_job_id.to_string()));
10035 com::save_config(
10036 store.pools[manual_task_receipt_pool].clone(),
10037 &manual_task_receipt_path,
10038 manual_task_receipt_bytes,
10039 )
10040 .await
10041 .expect("manual task receipt should restore after the corruption check");
10042
10043 let tier_stats = recover_tier_delete_journal_entries(store.clone(), 100, None)
10044 .await
10045 .expect("tier journal recovery should consume the migrated record before completion");
10046 assert_eq!((tier_stats.scanned, tier_stats.deleted, tier_stats.failed), (1, 1, 0));
10047 assert!(matches!(com::read_config(store.clone(), &tier_path).await, Err(Error::ConfigNotFound)));
10048
10049 let recovered_transition_version = "recovered-transition-version".to_string();
10050 backend
10051 .set_transition_candidate_probe_override(Some(TransitionCandidateProbe::VersionedPresent(
10052 recovered_transition_version.clone(),
10053 )))
10054 .await;
10055 let transaction_stats = {
10056 let _proof = crate::services::notification_sys::install_current_remote_version_state_fleet_proof_for_test();
10057 temp_env::async_with_vars(
10058 [
10059 (rustfs_config::ENV_TIER_REMOTE_VERSION_STATE_WRITE, Some("true")),
10060 (rustfs_config::ENV_TIER_REMOTE_VERSION_STATE_FLEET_CONFIRMED, Some("true")),
10061 ],
10062 recover_transition_transaction_records(store.clone(), 100, None),
10063 )
10064 .await
10065 .expect("transition recovery should advance and consume the migrated transaction before completion")
10066 };
10067 backend.set_transition_candidate_probe_override(None).await;
10068 assert_eq!(
10069 (
10070 transaction_stats.scanned,
10071 transaction_stats.recovered,
10072 transaction_stats.retained,
10073 transaction_stats.failed,
10074 ),
10075 (1, 1, 0, 0)
10076 );
10077 assert!(matches!(
10078 com::read_config(store.clone(), &transaction_path).await,
10079 Err(Error::ConfigNotFound)
10080 ));
10081 com::save_config(
10082 store.pools[1].clone(),
10083 &transaction_path,
10084 regressed_transaction
10085 .encode()
10086 .expect("post-terminal transition rollback should encode"),
10087 )
10088 .await
10089 .expect("target should accept the post-terminal rollback fixture");
10090 let post_terminal_regression = store
10091 .complete_decommission(0)
10092 .await
10093 .expect_err("terminal proof must not mask a lower transition revision")
10094 .to_string();
10095 assert!(post_terminal_regression.contains("generation mismatch"));
10096 assert!(post_terminal_regression.contains(&transaction_path));
10097 assert!(post_terminal_regression.contains(&transaction.transaction_id.to_string()));
10098 com::delete_config(store.pools[1].clone(), &transaction_path)
10099 .await
10100 .expect("post-terminal rollback fixture should be removed");
10101
10102 let manual_stats = recover_manual_transition_jobs_once(store.clone(), 100, None)
10103 .await
10104 .expect("manual recovery should advance the migrated job and consume its scope before completion");
10105 assert_eq!(
10106 (manual_stats.scanned, manual_stats.resumed, manual_stats.skipped, manual_stats.failed,),
10107 (1, 1, 0, 0)
10108 );
10109 assert!(matches!(
10110 com::read_config(store.clone(), &manual_scope_path).await,
10111 Err(Error::ConfigNotFound)
10112 ));
10113 let recovered_manual_job_bytes = com::read_config(store.pools[1].clone(), &manual_job_path)
10114 .await
10115 .expect("manual recovery should retain the advanced job record");
10116 assert_ne!(recovered_manual_job_bytes, manual_job_bytes);
10117
10118 com::save_config(store.pools[1].clone(), &manual_job_path, manual_job_bytes.clone())
10119 .await
10120 .expect("target manual job should accept the rollback fixture");
10121 let manual_regression = store
10122 .complete_decommission(0)
10123 .await
10124 .expect_err("completion must reject a manual job generation rollback")
10125 .to_string();
10126 assert!(manual_regression.contains("generation mismatch"));
10127 assert!(manual_regression.contains(&manual_job_path));
10128 assert!(manual_regression.contains(&manual_job_id.to_string()));
10129 com::save_config(store.pools[1].clone(), &manual_job_path, recovered_manual_job_bytes)
10130 .await
10131 .expect("target manual job should restore its recovered generation");
10132
10133 store
10134 .complete_decommission(0)
10135 .await
10136 .expect("completion should persist before receipt cleanup");
10137 assert!(
10138 store.pool_meta.read().await.pools[0]
10139 .decommission
10140 .as_ref()
10141 .expect("completed decommission state should remain present")
10142 .complete
10143 );
10144 assert_eq!(
10145 store
10146 .decommission_durable_ilm_receipt_count_for_test(0)
10147 .await
10148 .expect("receipt cleanup should be observable"),
10149 0
10150 );
10151 store
10152 .cleanup_decommission_durable_ilm_receipts_for_test(0)
10153 .await
10154 .expect("receipt cleanup should be idempotent");
10155 let removed_versions = backend.remove_versions().await;
10156 assert!(removed_versions.contains(&(tier_entry.obj_name.clone(), tier_entry.version_id.clone())));
10157 assert!(removed_versions.contains(&(transaction.remote_object.clone(), recovered_transition_version)));
10158 }
10159
10160 #[cfg(feature = "test-util")]
10161 async fn tier_delete_journal_count(store: Arc<crate::store::ECStore>) -> usize {
10162 store
10163 .list_objects_v2(RUSTFS_META_BUCKET, TIER_DELETE_JOURNAL_PREFIX, None, None, 100, false, None, false)
10164 .await
10165 .expect("tier delete journal should be listable")
10166 .objects
10167 .len()
10168 }
10169
10170 #[cfg(feature = "test-util")]
10171 async fn tier_delete_dispatch_manifest_count(store: Arc<crate::store::ECStore>) -> usize {
10172 store
10173 .list_objects_v2(
10174 RUSTFS_META_BUCKET,
10175 TIER_DELETE_DISPATCH_MANIFEST_PREFIX,
10176 None,
10177 None,
10178 100,
10179 false,
10180 None,
10181 false,
10182 )
10183 .await
10184 .expect("tier delete dispatch manifests should be listable")
10185 .objects
10186 .len()
10187 }
10188
10189 #[cfg(feature = "test-util")]
10190 async fn install_committed_tier_delete_journals(
10191 store: Arc<crate::store::ECStore>,
10192 tier_name: &str,
10193 backend_identity: [u8; 32],
10194 count: usize,
10195 ) -> Vec<Jentry> {
10196 let mut entries = Vec::with_capacity(count);
10197 for index in 0..count {
10198 let entry = Jentry {
10199 persisted_version: 0,
10200 obj_name: format!("remote/pass-drain-{index:06}.bin"),
10201 version_id: uuid::Uuid::new_v4().to_string(),
10202 tier_name: tier_name.to_string(),
10203 backend_identity: Some(backend_identity),
10204 version_id_exact: true,
10205 version_state: rustfs_filemeta::TransitionVersionState::Exact,
10206 state: TierDeleteJournalState::Committed,
10207 source: None,
10208 dispatch: None,
10209 };
10210 persist_tier_delete_journal_entry(store.clone(), &entry)
10211 .await
10212 .expect("committed tier-delete journal fixture should persist");
10213 entries.push(entry);
10214 }
10215 entries
10216 }
10217
10218 #[cfg(feature = "test-util")]
10219 fn synthetic_v6_dispatch_entry(
10220 bucket: &str,
10221 object: &str,
10222 tier_name: &str,
10223 backend_identity: [u8; 32],
10224 remote_version: &str,
10225 ) -> Jentry {
10226 Jentry {
10227 persisted_version: 0,
10228 obj_name: format!("remote/{object}"),
10229 version_id: remote_version.to_string(),
10230 tier_name: tier_name.to_string(),
10231 backend_identity: Some(backend_identity),
10232 version_id_exact: true,
10233 version_state: rustfs_filemeta::TransitionVersionState::Exact,
10234 state: TierDeleteJournalState::Prepared,
10235 source: Some(TierDeleteSourceIdentity {
10236 bucket: bucket.to_string(),
10237 object: object.to_string(),
10238 version_id: Some(uuid::Uuid::new_v4().to_string()),
10239 versioned: true,
10240 version_suspended: false,
10241 data_dir: Some(uuid::Uuid::new_v4().to_string()),
10242 etag: Some(format!("etag-{object}")),
10243 mod_time: Some(OffsetDateTime::UNIX_EPOCH.to_string()),
10244 }),
10245 dispatch: None,
10246 }
10247 }
10248
10249 #[cfg(feature = "test-util")]
10250 async fn install_aborting_dispatch_fixture(
10251 store: Arc<crate::store::ECStore>,
10252 bucket: &str,
10253 incarnation: uuid::Uuid,
10254 prefix: &str,
10255 tier_name: &str,
10256 backend_identity: [u8; 32],
10257 journal_count: usize,
10258 ) -> (String, Vec<Jentry>) {
10259 let entries = (0..journal_count)
10260 .map(|index| {
10261 (
10262 synthetic_v6_dispatch_entry(
10263 bucket,
10264 &format!("{prefix}{index:06}.bin"),
10265 tier_name,
10266 backend_identity,
10267 &uuid::Uuid::new_v4().to_string(),
10268 ),
10269 Some(TierDeleteJournalState::Prepared),
10270 )
10271 })
10272 .collect();
10273 install_test_tier_delete_dispatch_fixture(
10274 store,
10275 bucket,
10276 incarnation,
10277 prefix,
10278 entries,
10279 TierDeleteDispatchManifestState::Aborting,
10280 )
10281 .await
10282 .expect("Aborting rollback fixture should persist")
10283 }
10284
10285 #[cfg(feature = "test-util")]
10286 async fn drive_tier_delete_dispatch_restart_to_convergence(store: Arc<crate::store::ECStore>) {
10287 tokio::time::timeout(Duration::from_secs(30), async {
10288 let mut stable_empty_observations = 0;
10289 loop {
10290 recover_tier_delete_dispatch_manifests(store.clone(), 100, None)
10291 .await
10292 .expect("restarted manifest recovery pass should finish");
10293 recover_tier_delete_journal_entries(store.clone(), 100, None)
10294 .await
10295 .expect("restarted journal recovery pass should finish");
10296 if tier_delete_journal_count(store.clone()).await == 0
10297 && tier_delete_dispatch_manifest_count(store.clone()).await == 0
10298 && tier_delete_dispatch_manifest_recovery_count_for_test(&store) == 0
10299 {
10300 stable_empty_observations += 1;
10301 if stable_empty_observations == 10 {
10302 break;
10303 }
10304 } else {
10305 stable_empty_observations = 0;
10306 }
10307 tokio::time::sleep(Duration::from_millis(50)).await;
10308 }
10309 })
10310 .await
10311 .expect("same-disk restart recovery should converge without retained dispatch state");
10312 }
10313
10314 #[cfg(feature = "test-util")]
10315 #[tokio::test]
10316 #[serial_test::serial(storage_class_env)]
10317 async fn tier_delete_recovery_pass_drains_multiple_fast_manifest_pages() {
10318 const MANIFEST_COUNT: usize = 10;
10319
10320 let temp_dir = tempfile::tempdir().expect("create fast manifest pass recovery store dir");
10321 let mut instance_ctx = crate::runtime::instance::InstanceContext::new();
10322 instance_ctx.suppress_tier_delete_journal_recovery_for_test();
10323 let (ctx, store, shutdown) = without_storage_class_env(build_isolated_test_store_with_layout(
10324 temp_dir.path(),
10325 "tier-delete-fast-manifest-pass",
10326 &[(1, 4)],
10327 CancellationToken::new(),
10328 Some(Arc::new(instance_ctx)),
10329 ))
10330 .await;
10331 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
10332 let tier_name = "FAST-MANIFEST-PASS";
10333 let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
10334 let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
10335 .await
10336 .expect("fast manifest pass tier lease should resolve")
10337 .backend_identity();
10338 for index in 0..MANIFEST_COUNT {
10339 let bucket = format!("tier-delete-fast-manifest-pass-{index}");
10341 store
10342 .make_bucket(&bucket, &MakeBucketOptions::default())
10343 .await
10344 .expect("fast manifest pass bucket should be created");
10345 let incarnation = store
10346 .bucket_incarnation_id(&bucket)
10347 .await
10348 .expect("fast manifest pass bucket incarnation should resolve");
10349 install_aborting_dispatch_fixture(
10350 store.clone(),
10351 &bucket,
10352 incarnation,
10353 &format!("manifest-page-{index:06}/"),
10354 tier_name,
10355 backend_identity,
10356 1,
10357 )
10358 .await;
10359 }
10360 assert_eq!(tier_delete_dispatch_manifest_count(store.clone()).await, MANIFEST_COUNT);
10361 assert_eq!(tier_delete_journal_count(store.clone()).await, MANIFEST_COUNT);
10362
10363 let cancel = CancellationToken::new();
10364 let mut manifest_marker = None;
10365 let mut journal_marker = None;
10366 let stats = recover_test_tier_delete_journal_pass_with_budget(
10367 store.clone(),
10368 &cancel,
10369 &mut manifest_marker,
10370 &mut journal_marker,
10371 Duration::from_secs(30),
10372 )
10373 .await;
10374
10375 assert!(!stats.canceled);
10376 assert!(!stats.deadline_exhausted);
10377 assert!(
10378 stats.manifest_pages > 1,
10379 "one production pass must cross the default eight-manifest page limit"
10380 );
10381 assert_eq!(stats.manifests.scanned, MANIFEST_COUNT);
10382 assert_eq!(stats.manifests.deleted, MANIFEST_COUNT, "full recovery result: {stats:?}");
10383 assert_eq!(stats.manifests.failed, 0, "full recovery result: {stats:?}");
10384 assert_eq!(manifest_marker, None);
10385 assert_eq!(tier_delete_dispatch_manifest_count(store.clone()).await, 0);
10386 assert_eq!(tier_delete_journal_count(store).await, 0);
10387 assert_eq!(backend.remove_count().await, 0, "rollback recovery must not call the remote tier");
10388 shutdown.cancel();
10389 }
10390
10391 #[cfg(feature = "test-util")]
10392 #[tokio::test]
10393 #[serial_test::serial(storage_class_env)]
10394 async fn tier_delete_manual_pass_retains_manifest_owned_by_startup_recovery() {
10395 let temp_dir = tempfile::tempdir().expect("create automatic recovery ownership store dir");
10396 let (ctx, store, shutdown) =
10397 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "tier-delete-auto-owner", &[4])).await;
10398 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
10399 let bucket = "tier-delete-auto-owner-bucket";
10400 store
10401 .make_bucket(bucket, &MakeBucketOptions::default())
10402 .await
10403 .expect("automatic recovery bucket should be created");
10404 let incarnation = store.bucket_incarnation_id(bucket).await.expect("bucket incarnation");
10405 let tier_name = "AUTO-OWNER";
10406 let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
10407 let identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
10408 .await
10409 .expect("automatic recovery tier lease")
10410 .backend_identity();
10411
10412 let lifecycle_guard = store
10414 .acquire_bucket_lifecycle_write_lock(bucket)
10415 .await
10416 .expect("fixture lifecycle lock");
10417 let (manifest_name, entries) =
10418 install_aborting_dispatch_fixture(store.clone(), bucket, incarnation, "auto-owner/", tier_name, identity, 1).await;
10419 let journal_name = tier_delete_journal_object_name(&entries[0]);
10420 let hook = TierDeleteDispatchRollbackTestHook::install_slow_delete(&journal_name, &journal_name);
10421 drop(lifecycle_guard);
10422 ctx.wake_tier_delete_journal_recovery();
10423 tokio::time::timeout(Duration::from_secs(30), hook.wait_until_delete_paused())
10424 .await
10425 .expect("startup recovery should own the manifest before a manual pass");
10426 assert!(tier_delete_dispatch_manifest_recovery_inflight_for_test(&store, &manifest_name));
10427
10428 let stats = recover_tier_delete_dispatch_manifests(store.clone(), 8, None)
10429 .await
10430 .expect("manual recovery scan");
10431 assert_eq!(stats.scanned, 1, "{stats:?}");
10432 assert_eq!(stats.retained, 1, "{stats:?}");
10433 assert_eq!(stats.deleted, 0, "{stats:?}");
10434 assert_eq!(stats.failed, 0, "{stats:?}");
10435 assert_eq!(tier_delete_dispatch_manifest_count(store.clone()).await, 1);
10436 assert_eq!(tier_delete_journal_count(store.clone()).await, 1);
10437
10438 hook.release_delete();
10439 tokio::time::timeout(Duration::from_secs(30), async {
10440 loop {
10441 let manifest_gone = matches!(com::read_config(store.clone(), &manifest_name).await, Err(Error::ConfigNotFound));
10442 if manifest_gone && !tier_delete_dispatch_manifest_recovery_inflight_for_test(&store, &manifest_name) {
10443 break;
10444 }
10445 tokio::time::sleep(Duration::from_millis(10)).await;
10446 }
10447 })
10448 .await
10449 .expect("automatic recovery should converge without a manual retry");
10450 assert_eq!(tier_delete_dispatch_manifest_count(store.clone()).await, 0);
10451 assert_eq!(tier_delete_journal_count(store).await, 0);
10452 assert_eq!(backend.remove_count().await, 0, "rollback must not delete from the remote tier");
10453 shutdown.cancel();
10454 }
10455
10456 #[cfg(feature = "test-util")]
10457 #[tokio::test]
10458 #[serial_test::serial(storage_class_env)]
10459 async fn tier_delete_recovery_pass_drains_multiple_fast_journal_pages() {
10460 const JOURNAL_COUNT: usize = 25;
10461
10462 let temp_dir = tempfile::tempdir().expect("create fast pass recovery store dir");
10463 let (ctx, store, _shutdown) =
10464 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "tier-delete-fast-pass", &[4])).await;
10465 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
10466 let tier_name = "FAST-PASS-RECOVERY";
10467 let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
10468 let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
10469 .await
10470 .expect("fast pass tier lease should resolve")
10471 .backend_identity();
10472 let entries = install_committed_tier_delete_journals(store.clone(), tier_name, backend_identity, JOURNAL_COUNT).await;
10473 assert_eq!(tier_delete_journal_count(store.clone()).await, JOURNAL_COUNT);
10474
10475 let cancel = CancellationToken::new();
10476 let mut manifest_marker = None;
10477 let mut journal_marker = None;
10478 let stats = recover_test_tier_delete_journal_pass_with_budget(
10479 store.clone(),
10480 &cancel,
10481 &mut manifest_marker,
10482 &mut journal_marker,
10483 Duration::from_secs(30),
10484 )
10485 .await;
10486
10487 assert!(!stats.canceled);
10488 assert!(!stats.deadline_exhausted);
10489 assert!(
10490 stats.journal_pages > 1,
10491 "one production pass must cross the default eight-record page limit"
10492 );
10493 assert_eq!(stats.journals.scanned, JOURNAL_COUNT);
10494 assert_eq!(stats.journals.deleted, JOURNAL_COUNT);
10495 assert_eq!(stats.journals.failed, 0);
10496 assert_eq!(journal_marker, None);
10497 assert_eq!(tier_delete_journal_count(store).await, 0);
10498 assert_eq!(backend.exact_remove_count(), JOURNAL_COUNT);
10499 let mut removed = backend.remove_versions().await;
10500 removed.sort();
10501 let mut expected = entries
10502 .into_iter()
10503 .map(|entry| (entry.obj_name, entry.version_id))
10504 .collect::<Vec<_>>();
10505 expected.sort();
10506 assert_eq!(removed, expected);
10507 }
10508
10509 #[cfg(feature = "test-util")]
10510 #[tokio::test]
10511 #[serial_test::serial(storage_class_env)]
10512 async fn tier_delete_recovery_pass_rotates_failed_journal_pages_without_busy_loop() {
10513 const JOURNAL_COUNT: usize = 9;
10514
10515 let temp_dir = tempfile::tempdir().expect("create failed pass recovery store dir");
10516 let (ctx, store, _shutdown) =
10517 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "tier-delete-failed-pass", &[4])).await;
10518 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
10519 let tier_name = "FAILED-PASS-RECOVERY";
10520 let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
10521 let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
10522 .await
10523 .expect("failed pass tier lease should resolve")
10524 .backend_identity();
10525 install_committed_tier_delete_journals(store.clone(), tier_name, backend_identity, JOURNAL_COUNT).await;
10526 backend.set_remove_failure(true);
10527
10528 let cancel = CancellationToken::new();
10529 let mut manifest_marker = None;
10530 let mut journal_marker = None;
10531 let first = recover_test_tier_delete_journal_pass_with_budget(
10532 store.clone(),
10533 &cancel,
10534 &mut manifest_marker,
10535 &mut journal_marker,
10536 Duration::from_secs(30),
10537 )
10538 .await;
10539 assert_eq!(first.journal_pages, 1, "a no-progress failed page must stop the pass");
10540 assert_eq!(first.journals.scanned, 8);
10541 assert_eq!(first.journals.deleted, 0);
10542 assert_eq!(first.journals.failed, 8);
10543 assert!(journal_marker.is_some(), "a truncated failed page must retain its continuation marker");
10544 assert_eq!(tier_delete_journal_count(store.clone()).await, JOURNAL_COUNT);
10545
10546 let second = recover_test_tier_delete_journal_pass_with_budget(
10547 store.clone(),
10548 &cancel,
10549 &mut manifest_marker,
10550 &mut journal_marker,
10551 Duration::from_secs(30),
10552 )
10553 .await;
10554 assert_eq!(second.journal_pages, 1);
10555 assert_eq!(second.journals.scanned, 1);
10556 assert_eq!(second.journals.deleted, 0);
10557 assert_eq!(second.journals.failed, 1);
10558 assert_eq!(
10559 journal_marker, None,
10560 "end-of-list rotation must revisit the failed prefix on a later pass"
10561 );
10562 assert_eq!(tier_delete_journal_count(store.clone()).await, JOURNAL_COUNT);
10563
10564 backend.set_remove_failure(false);
10565 let third = recover_test_tier_delete_journal_pass_with_budget(
10566 store.clone(),
10567 &cancel,
10568 &mut manifest_marker,
10569 &mut journal_marker,
10570 Duration::from_secs(30),
10571 )
10572 .await;
10573 assert_eq!(third.journals.scanned, JOURNAL_COUNT);
10574 assert_eq!(third.journals.deleted, JOURNAL_COUNT);
10575 assert_eq!(third.journals.failed, 0);
10576 assert_eq!(tier_delete_journal_count(store).await, 0);
10577 }
10578
10579 #[cfg(feature = "test-util")]
10580 #[tokio::test]
10581 #[serial_test::serial(storage_class_env)]
10582 async fn tier_delete_journal_rejects_unbound_new_prepare_without_persisting() {
10583 let temp_dir = tempfile::tempdir().expect("create journal admission store dir");
10584 let (_ctx, store, _shutdown) =
10585 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "tier-journal-unbound-prepare", &[4])).await;
10586 let entry = Jentry {
10587 persisted_version: 0,
10588 obj_name: "remote/object".to_string(),
10589 version_id: uuid::Uuid::new_v4().to_string(),
10590 tier_name: "COLD-A".to_string(),
10591 backend_identity: Some([7; 32]),
10592 version_id_exact: true,
10593 version_state: rustfs_filemeta::TransitionVersionState::Exact,
10594 state: TierDeleteJournalState::Prepared,
10595 source: Some(TierDeleteSourceIdentity {
10596 bucket: "source-bucket".to_string(),
10597 object: "source-object".to_string(),
10598 version_id: Some(uuid::Uuid::new_v4().to_string()),
10599 versioned: true,
10600 version_suspended: false,
10601 data_dir: Some(uuid::Uuid::new_v4().to_string()),
10602 etag: Some("source-etag".to_string()),
10603 mod_time: Some(OffsetDateTime::UNIX_EPOCH.to_string()),
10604 }),
10605 dispatch: None,
10606 };
10607 let error = persist_tier_delete_journal_entry(store.clone(), &entry)
10608 .await
10609 .expect_err("a new source-owning journal without a v6 manifest binding must fail closed");
10610 assert!(error.to_string().contains("dispatch manifest binding"), "unexpected error: {error}");
10611 assert_eq!(tier_delete_journal_count(store).await, 0, "rejected prepare must not write a journal");
10612 }
10613
10614 #[cfg(feature = "test-util")]
10615 #[tokio::test]
10616 #[serial_test::serial(storage_class_env)]
10617 async fn dispatch_manifest_recovery_rolls_back_unapproved_crash_states_without_remote_io() {
10618 let temp_dir = tempfile::tempdir().expect("create dispatch rollback store dir");
10619 let (ctx, store, _shutdown) =
10620 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "dispatch-manifest-rollback", &[4])).await;
10621 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
10622 let bucket = "dispatch-manifest-rollback-bucket";
10623 store
10624 .make_bucket(bucket, &MakeBucketOptions::default())
10625 .await
10626 .expect("dispatch rollback bucket should be created");
10627 let incarnation = store
10628 .bucket_incarnation_id(bucket)
10629 .await
10630 .expect("bucket incarnation should resolve");
10631 let tier_name = "DISPATCH-ROLLBACK";
10632 let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
10633 let identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
10634 .await
10635 .expect("tier lease should resolve")
10636 .backend_identity();
10637
10638 let preparing = [
10639 ("preparing/a.bin", TierDeleteJournalState::Prepared),
10640 ("preparing/b.bin", TierDeleteJournalState::Dispatched),
10641 ]
10642 .into_iter()
10643 .map(|(object, state)| {
10644 (
10645 synthetic_v6_dispatch_entry(bucket, object, tier_name, identity, &uuid::Uuid::new_v4().to_string()),
10646 Some(state),
10647 )
10648 })
10649 .collect();
10650 install_test_tier_delete_dispatch_fixture(
10651 store.clone(),
10652 bucket,
10653 incarnation,
10654 "preparing/",
10655 preparing,
10656 TierDeleteDispatchManifestState::Preparing,
10657 )
10658 .await
10659 .expect("Preparing crash fixture should persist");
10660
10661 let aborting = [
10662 ("aborting/missing.bin", None),
10663 ("aborting/prepared.bin", Some(TierDeleteJournalState::Prepared)),
10664 ("aborting/dispatched.bin", Some(TierDeleteJournalState::Dispatched)),
10665 ]
10666 .into_iter()
10667 .map(|(object, state)| {
10668 (
10669 synthetic_v6_dispatch_entry(bucket, object, tier_name, identity, &uuid::Uuid::new_v4().to_string()),
10670 state,
10671 )
10672 })
10673 .collect();
10674 install_test_tier_delete_dispatch_fixture(
10675 store.clone(),
10676 bucket,
10677 incarnation,
10678 "aborting/",
10679 aborting,
10680 TierDeleteDispatchManifestState::Aborting,
10681 )
10682 .await
10683 .expect("Aborting crash fixture should persist");
10684 install_test_tier_delete_dispatch_fixture(
10685 store.clone(),
10686 bucket,
10687 incarnation,
10688 "empty/",
10689 Vec::new(),
10690 TierDeleteDispatchManifestState::Completed,
10691 )
10692 .await
10693 .expect("empty Completed crash fixture should persist");
10694
10695 let deleted = tokio::time::timeout(Duration::from_secs(30), async {
10696 let mut deleted = 0;
10697 loop {
10698 let stats = recover_tier_delete_dispatch_manifests(store.clone(), 100, None)
10699 .await
10700 .expect("manifest recovery should reconcile negative-seal crash states");
10701 deleted += stats.deleted;
10702 if tier_delete_dispatch_manifest_count(store.clone()).await == 0 {
10703 break deleted;
10704 }
10705 tokio::time::sleep(Duration::from_millis(10)).await;
10709 }
10710 })
10711 .await
10712 .expect("valid negative-seal crash states should eventually converge");
10713 assert_eq!(deleted, 3);
10714 assert_eq!(tier_delete_journal_count(store.clone()).await, 0);
10715 assert_eq!(tier_delete_dispatch_manifest_count(store).await, 0);
10716 assert_eq!(
10717 backend.remove_count().await,
10718 0,
10719 "rollback recovery must perform zero remote delete operations"
10720 );
10721 }
10722
10723 #[cfg(feature = "test-util")]
10724 #[tokio::test]
10725 #[serial_test::serial(storage_class_env)]
10726 async fn dispatch_manifest_recovery_quarantines_impossible_committed_rollback_set() {
10727 let temp_dir = tempfile::tempdir().expect("create dispatch quarantine store dir");
10728 let (ctx, store, _shutdown) =
10729 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "dispatch-manifest-quarantine", &[4])).await;
10730 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
10731 let bucket = "dispatch-manifest-quarantine-bucket";
10732 store
10733 .make_bucket(bucket, &MakeBucketOptions::default())
10734 .await
10735 .expect("dispatch quarantine bucket should be created");
10736 let incarnation = store
10737 .bucket_incarnation_id(bucket)
10738 .await
10739 .expect("bucket incarnation should resolve");
10740 let tier_name = "DISPATCH-QUARANTINE";
10741 let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
10742 let identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
10743 .await
10744 .expect("tier lease should resolve")
10745 .backend_identity();
10746 let entries = [TierDeleteJournalState::Prepared, TierDeleteJournalState::Committed]
10747 .into_iter()
10748 .enumerate()
10749 .map(|(index, state)| {
10750 (
10751 synthetic_v6_dispatch_entry(
10752 bucket,
10753 &format!("quarantine/{index}.bin"),
10754 tier_name,
10755 identity,
10756 &uuid::Uuid::new_v4().to_string(),
10757 ),
10758 Some(state),
10759 )
10760 })
10761 .collect();
10762 let (manifest_name, _) = install_test_tier_delete_dispatch_fixture(
10763 store.clone(),
10764 bucket,
10765 incarnation,
10766 "quarantine/",
10767 entries,
10768 TierDeleteDispatchManifestState::Aborting,
10769 )
10770 .await
10771 .expect("impossible rollback fixture should persist");
10772
10773 let stats = recover_tier_delete_dispatch_manifests(store.clone(), 100, None)
10774 .await
10775 .expect("quarantined manifest scan should finish");
10776 assert_eq!((stats.scanned, stats.deleted, stats.failed), (1, 0, 1));
10777 assert_eq!(
10778 tier_delete_journal_count(store.clone()).await,
10779 2,
10780 "validation must precede every deletion"
10781 );
10782 assert_eq!(tier_delete_dispatch_manifest_count(store.clone()).await, 1);
10783 assert_eq!(
10784 test_tier_delete_dispatch_manifest_state(store, &manifest_name)
10785 .await
10786 .expect("quarantined manifest should remain readable"),
10787 Some(TierDeleteDispatchManifestState::Aborting)
10788 );
10789 assert_eq!(backend.remove_count().await, 0);
10790 }
10791
10792 #[cfg(feature = "test-util")]
10793 #[tokio::test]
10794 #[serial_test::serial(storage_class_env)]
10795 async fn dispatch_manifest_rollback_bounded_concurrency_reaches_tail_behind_slow_member() {
10796 const JOURNAL_COUNT: usize = 65;
10797
10798 let temp_dir = tempfile::tempdir().expect("create bounded rollback store dir");
10799 let (ctx, store, _shutdown) =
10800 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "bounded-dispatch-rollback", &[4])).await;
10801 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
10802 let bucket = "bounded-dispatch-rollback-bucket";
10803 store
10804 .make_bucket(bucket, &MakeBucketOptions::default())
10805 .await
10806 .expect("bounded rollback bucket should be created");
10807 let incarnation = store
10808 .bucket_incarnation_id(bucket)
10809 .await
10810 .expect("bounded rollback bucket incarnation should resolve");
10811 let tier_name = "BOUNDED-DISPATCH-ROLLBACK";
10812 let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
10813 let identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
10814 .await
10815 .expect("bounded rollback tier lease should resolve")
10816 .backend_identity();
10817 let (manifest_name, entries) =
10818 install_aborting_dispatch_fixture(store.clone(), bucket, incarnation, "slow/", tier_name, identity, JOURNAL_COUNT)
10819 .await;
10820 let first_name = tier_delete_journal_object_name(entries.first().expect("slow rollback fixture should not be empty"));
10821 let last_name = tier_delete_journal_object_name(entries.last().expect("slow rollback fixture should not be empty"));
10822 let hook = TierDeleteDispatchRollbackTestHook::install_slow_delete(&first_name, &last_name);
10823 let worker_store = store.clone();
10824 let worker_manifest = manifest_name.clone();
10825 let worker =
10826 tokio::spawn(async move { recover_test_tier_delete_dispatch_manifest(worker_store, &worker_manifest).await });
10827 tokio::time::timeout(Duration::from_secs(30), hook.wait_until_delete_paused())
10828 .await
10829 .expect("the first rollback member should reach the slow hook");
10830 tokio::time::timeout(Duration::from_secs(30), hook.wait_until_delete_observed())
10831 .await
10832 .expect("a member beyond the first two concurrency windows should remain reachable");
10833 assert!(
10834 (2..=32).contains(&hook.max_in_flight()),
10835 "rollback delete concurrency must make progress without exceeding its bound; observed {}",
10836 hook.max_in_flight()
10837 );
10838 hook.release_delete();
10839 worker
10840 .await
10841 .expect("bounded rollback recovery task should join")
10842 .expect("bounded rollback recovery should converge after the slow member releases");
10843 drop(hook);
10844
10845 assert_eq!(tier_delete_journal_count(store.clone()).await, 0);
10846 assert_eq!(tier_delete_dispatch_manifest_count(store.clone()).await, 0);
10847 assert_eq!(backend.remove_count().await, 0, "rollback must not call the remote tier");
10848 }
10849
10850 #[cfg(feature = "test-util")]
10851 #[tokio::test]
10852 #[serial_test::serial(storage_class_env)]
10853 async fn aborted_dispatch_retry_defers_residual_scan_to_recovery() {
10854 const JOURNAL_COUNT: usize = 65;
10855
10856 let temp_dir = tempfile::tempdir().expect("create aborted retry store dir");
10857 let (ctx, store, shutdown) =
10858 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "aborted-dispatch-retry", &[4])).await;
10859 shutdown.cancel();
10860 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
10861 let bucket = "aborted-dispatch-retry-bucket";
10862 let prefix = "archive/";
10863 store
10864 .make_bucket(bucket, &MakeBucketOptions::default())
10865 .await
10866 .expect("aborted retry bucket should be created");
10867 let incarnation = store
10868 .bucket_incarnation_id(bucket)
10869 .await
10870 .expect("aborted retry bucket incarnation should resolve");
10871 let tier_name = "ABORTED-DISPATCH-RETRY";
10872 let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
10873 let identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
10874 .await
10875 .expect("aborted retry tier lease should resolve")
10876 .backend_identity();
10877 let aborted_entries = (0..JOURNAL_COUNT)
10878 .map(|index| {
10879 (
10880 synthetic_v6_dispatch_entry(
10881 bucket,
10882 &format!("{prefix}{index:06}.bin"),
10883 tier_name,
10884 identity,
10885 &uuid::Uuid::new_v4().to_string(),
10886 ),
10887 Some(TierDeleteJournalState::Prepared),
10888 )
10889 })
10890 .collect::<Vec<_>>();
10891 let entries = aborted_entries.iter().map(|(entry, _)| entry.clone()).collect::<Vec<_>>();
10892 let (manifest_name, _) = install_test_tier_delete_dispatch_fixture(
10893 store.clone(),
10894 bucket,
10895 incarnation,
10896 prefix,
10897 aborted_entries,
10898 TierDeleteDispatchManifestState::Aborted,
10899 )
10900 .await
10901 .expect("Aborted retry fixture should persist");
10902
10903 let lifecycle_guard = store
10904 .acquire_bucket_lifecycle_write_lock(bucket)
10905 .await
10906 .expect("aborted retry should acquire the bucket lifecycle fence");
10907 let mut fence_opts = ObjectOptions::default();
10908 fence_opts.add_bucket_lifecycle_lock_guard(&lifecycle_guard);
10909 let bucket_fence = fence_opts
10910 .bucket_lifecycle_lock_fence
10911 .clone()
10912 .expect("aborted retry should capture the bucket lifecycle fence");
10913 let fleet_proof = acquire_tier_delete_journal_fleet_proof().expect("aborted retry fixture should have a fleet proof");
10914 let hook = TierDeleteDispatchMemberReadTestHook::install_pause(TierDeleteDispatchMemberReadTestStage::Validation);
10915
10916 let error =
10917 match prepare_tier_delete_dispatch(store.clone(), bucket, incarnation, prefix, entries, fleet_proof, &bucket_fence)
10918 .await
10919 {
10920 Ok(_) => panic!("an Aborted manifest must be retained for bounded recovery cleanup"),
10921 Err(err) => err,
10922 };
10923
10924 assert!(
10925 error.to_string().contains("bounded recovery cleanup"),
10926 "unexpected aborted retry error: {error}"
10927 );
10928 assert_eq!(
10929 hook.entry_count(),
10930 0,
10931 "request retry must not scan the retained Aborted journal set under bucket write lock"
10932 );
10933 assert_eq!(
10934 test_tier_delete_dispatch_manifest_state(store.clone(), &manifest_name)
10935 .await
10936 .expect("retained Aborted manifest should remain readable"),
10937 Some(TierDeleteDispatchManifestState::Aborted)
10938 );
10939 assert_eq!(tier_delete_journal_count(store.clone()).await, JOURNAL_COUNT);
10940 assert_eq!(backend.remove_count().await, 0, "retry must not call the remote tier");
10941 drop(hook);
10942 drop(lifecycle_guard);
10943
10944 recover_test_tier_delete_dispatch_manifest(store.clone(), &manifest_name)
10945 .await
10946 .expect("bounded recovery should clean the retained Aborted manifest");
10947 assert_eq!(tier_delete_journal_count(store.clone()).await, 0);
10948 assert_eq!(tier_delete_dispatch_manifest_count(store).await, 0);
10949 }
10950
10951 #[cfg(feature = "test-util")]
10952 #[tokio::test]
10953 #[serial_test::serial(storage_class_env)]
10954 async fn dispatch_manifest_page_timeout_detaches_one_deduplicated_worker_until_convergence() {
10955 let temp_dir = tempfile::tempdir().expect("create detached rollback store dir");
10956 let (ctx, store, _shutdown) =
10957 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "detached-dispatch-rollback", &[4])).await;
10958 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
10959 let bucket = "detached-dispatch-rollback-bucket";
10960 store
10961 .make_bucket(bucket, &MakeBucketOptions::default())
10962 .await
10963 .expect("detached rollback bucket should be created");
10964 let incarnation = store
10965 .bucket_incarnation_id(bucket)
10966 .await
10967 .expect("detached rollback bucket incarnation should resolve");
10968 let tier_name = "DETACHED-DISPATCH-ROLLBACK";
10969 let _backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
10970 let identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
10971 .await
10972 .expect("detached rollback tier lease should resolve")
10973 .backend_identity();
10974 let (manifest_name, entries) =
10975 install_aborting_dispatch_fixture(store.clone(), bucket, incarnation, "detached/", tier_name, identity, 1).await;
10976 let journal_name = tier_delete_journal_object_name(&entries[0]);
10977 let hook = TierDeleteDispatchRollbackTestHook::install_slow_delete(&journal_name, &journal_name);
10978
10979 assert!(
10980 recover_test_tier_delete_dispatch_manifest_with_page_budget(
10981 store.clone(),
10982 &manifest_name,
10983 Duration::from_millis(10),
10984 )
10985 .await,
10986 "the synthetic page budget must elapse while the manifest worker continues"
10987 );
10988 tokio::time::timeout(Duration::from_secs(30), hook.wait_until_delete_paused())
10989 .await
10990 .expect("the detached manifest worker should continue into its delete phase");
10991 assert!(
10992 tier_delete_dispatch_manifest_recovery_inflight_for_test(&store, &manifest_name),
10993 "the detached worker must retain its per-store/object deduplication guard"
10994 );
10995 assert!(
10996 !recover_test_tier_delete_dispatch_manifest_with_page_budget(store.clone(), &manifest_name, Duration::from_secs(30),)
10997 .await,
10998 "a duplicate page observation should be retained without queuing another worker"
10999 );
11000
11001 hook.release_delete();
11002 tokio::time::timeout(Duration::from_secs(30), async {
11003 loop {
11004 let manifest_gone = matches!(com::read_config(store.clone(), &manifest_name).await, Err(Error::ConfigNotFound));
11005 if manifest_gone && !tier_delete_dispatch_manifest_recovery_inflight_for_test(&store, &manifest_name) {
11006 break;
11007 }
11008 tokio::time::sleep(Duration::from_millis(10)).await;
11009 }
11010 })
11011 .await
11012 .expect("the detached manifest worker should release its guard after convergence");
11013 drop(hook);
11014 assert_eq!(tier_delete_journal_count(store.clone()).await, 0);
11015 }
11016
11017 #[cfg(feature = "test-util")]
11018 #[tokio::test]
11019 #[serial_test::serial(storage_class_env)]
11020 async fn dispatch_manifest_detached_workers_are_store_bounded_and_shutdown_cancellable() {
11021 const MANIFEST_COUNT: usize = 6;
11022 const WORKER_LIMIT: usize = 4;
11023
11024 let temp_dir = tempfile::tempdir().expect("create bounded detached rollback store dir");
11025 let (ctx, store, shutdown) =
11026 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "bounded-detached-dispatch-rollback", &[4]))
11027 .await;
11028 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
11029 let tier_name = "BOUNDED-DETACHED-DISPATCH-ROLLBACK";
11030 let _backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
11031 let identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
11032 .await
11033 .expect("bounded detached rollback tier lease should resolve")
11034 .backend_identity();
11035 let mut manifest_names = Vec::with_capacity(MANIFEST_COUNT);
11036 for index in 0..MANIFEST_COUNT {
11037 let bucket = format!("bounded-detached-dispatch-rollback-{index}");
11038 store
11039 .make_bucket(&bucket, &MakeBucketOptions::default())
11040 .await
11041 .expect("bounded detached rollback bucket should be created");
11042 let incarnation = store
11043 .bucket_incarnation_id(&bucket)
11044 .await
11045 .expect("bounded detached rollback bucket incarnation should resolve");
11046 let (manifest_name, _) = install_aborting_dispatch_fixture(
11047 store.clone(),
11048 &bucket,
11049 incarnation,
11050 &format!("detached-{index}/"),
11051 tier_name,
11052 identity,
11053 1,
11054 )
11055 .await;
11056 manifest_names.push(manifest_name);
11057 }
11058 let hook = TierDeleteDispatchRollbackTestHook::install_pause_all_deletes();
11059
11060 let first_pass = futures::stream::iter(manifest_names.iter().cloned().map(|manifest_name| {
11061 let store = store.clone();
11062 async move {
11063 recover_test_tier_delete_dispatch_manifest_with_page_budget(store, &manifest_name, Duration::from_millis(10))
11064 .await
11065 }
11066 }))
11067 .buffer_unordered(MANIFEST_COUNT)
11068 .collect::<Vec<_>>()
11069 .await;
11070 tokio::time::timeout(Duration::from_secs(30), hook.wait_until_delete_pause_count(WORKER_LIMIT))
11071 .await
11072 .expect("every admitted detached worker should pause before its first mutation");
11073 assert_eq!(
11074 first_pass.into_iter().filter(|timed_out| *timed_out).count(),
11075 WORKER_LIMIT,
11076 "only the per-store worker budget may detach"
11077 );
11078 assert_eq!(
11079 tier_delete_dispatch_manifest_recovery_count_for_test(&store),
11080 WORKER_LIMIT,
11081 "detached workers must remain bounded per store"
11082 );
11083
11084 let repeated_pass = futures::stream::iter(manifest_names.iter().cloned().map(|manifest_name| {
11085 let store = store.clone();
11086 async move {
11087 recover_test_tier_delete_dispatch_manifest_with_page_budget(store, &manifest_name, Duration::from_secs(30)).await
11088 }
11089 }))
11090 .buffer_unordered(MANIFEST_COUNT)
11091 .collect::<Vec<_>>()
11092 .await;
11093 assert!(
11094 repeated_pass.into_iter().all(|timed_out| !timed_out),
11095 "duplicates and over-budget manifests must be retained without a waiter queue"
11096 );
11097 assert_eq!(tier_delete_dispatch_manifest_recovery_count_for_test(&store), WORKER_LIMIT);
11098
11099 shutdown.cancel();
11100 tokio::time::sleep(Duration::from_millis(50)).await;
11101 assert_eq!(
11102 tier_delete_dispatch_manifest_recovery_count_for_test(&store),
11103 WORKER_LIMIT,
11104 "shutdown must not drop a bounded batch before its admitted work drains"
11105 );
11106 assert_eq!(tier_delete_journal_count(store.clone()).await, MANIFEST_COUNT);
11107 assert_eq!(tier_delete_dispatch_manifest_count(store.clone()).await, MANIFEST_COUNT);
11108 hook.release_all_deletes();
11109 tokio::time::timeout(Duration::from_secs(30), async {
11110 while tier_delete_dispatch_manifest_recovery_count_for_test(&store) != 0 {
11111 tokio::time::sleep(Duration::from_millis(10)).await;
11112 }
11113 })
11114 .await
11115 .expect("shutdown should cancel every detached worker and release its registry permit");
11116 assert_eq!(tier_delete_journal_count(store.clone()).await, MANIFEST_COUNT);
11117 assert_eq!(tier_delete_dispatch_manifest_count(store.clone()).await, MANIFEST_COUNT);
11118 drop(hook);
11119 tokio::time::sleep(Duration::from_millis(50)).await;
11120 assert_eq!(
11121 tier_delete_journal_count(store.clone()).await,
11122 MANIFEST_COUNT,
11123 "releasing the old hook after shutdown must not resume a cancelled mutation"
11124 );
11125 assert_eq!(
11126 tier_delete_dispatch_manifest_recovery_count_for_test(&store),
11127 0,
11128 "the shutdown store must leave no registry permits behind"
11129 );
11130 drop(store);
11131 drop(ctx);
11132
11133 let (_restarted_ctx, restarted_store, restarted_shutdown) = without_storage_class_env(build_isolated_test_store(
11134 temp_dir.path(),
11135 "bounded-detached-dispatch-rollback-restart",
11136 &[4],
11137 ))
11138 .await;
11139 crate::bucket::metadata_sys::init_bucket_metadata_sys(restarted_store.clone(), Vec::new()).await;
11140 assert!(!restarted_shutdown.is_cancelled(), "the restarted store needs a fresh shutdown token");
11141 tokio::time::timeout(Duration::from_secs(30), async {
11142 loop {
11143 let active = tier_delete_dispatch_manifest_recovery_count_for_test(&restarted_store);
11144 let manifests = tier_delete_dispatch_manifest_count(restarted_store.clone()).await;
11145 if active > 0 || manifests < MANIFEST_COUNT {
11146 break;
11147 }
11148 tokio::task::yield_now().await;
11149 }
11150 })
11151 .await
11152 .expect("the restarted store should start a fresh manifest recovery");
11153 tokio::time::timeout(Duration::from_secs(30), async {
11154 while tier_delete_dispatch_manifest_recovery_count_for_test(&restarted_store) != 0 {
11155 tokio::time::sleep(Duration::from_millis(10)).await;
11156 }
11157 })
11158 .await
11159 .expect("the restarted store recovery should release every worker permit");
11160 if tier_delete_dispatch_manifest_count(restarted_store.clone()).await != 0 {
11161 let stats = recover_tier_delete_dispatch_manifests(restarted_store.clone(), 100, None)
11162 .await
11163 .expect("the restarted store should retry any conservatively retained manifest");
11164 assert_eq!(stats.failed, 0, "the restarted recovery must not quarantine a valid manifest");
11165 }
11166 assert_eq!(tier_delete_journal_count(restarted_store.clone()).await, 0);
11167 assert_eq!(tier_delete_dispatch_manifest_count(restarted_store).await, 0);
11168 restarted_shutdown.cancel();
11169 }
11170
11171 #[cfg(feature = "test-util")]
11172 #[tokio::test]
11173 #[serial_test::serial(storage_class_env)]
11174 async fn dispatch_manifest_shutdown_stops_aborting_validation_tail_reads() {
11175 const JOURNAL_COUNT: usize = 65;
11176 const ADMITTED_READS: usize = 32;
11177
11178 let temp_dir = tempfile::tempdir().expect("create validation-read shutdown store dir");
11179 let mut instance_ctx = crate::runtime::instance::InstanceContext::new();
11181 instance_ctx.suppress_tier_delete_journal_recovery_for_test();
11182 let (ctx, store, shutdown) = without_storage_class_env(build_isolated_test_store_with_layout(
11183 temp_dir.path(),
11184 "dispatch-validation-read-shutdown",
11185 &[(1, 4)],
11186 CancellationToken::new(),
11187 Some(Arc::new(instance_ctx)),
11188 ))
11189 .await;
11190 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
11191 let bucket = "dispatch-validation-read-shutdown-bucket";
11192 store
11193 .make_bucket(bucket, &MakeBucketOptions::default())
11194 .await
11195 .expect("validation-read shutdown bucket should be created");
11196 let incarnation = store
11197 .bucket_incarnation_id(bucket)
11198 .await
11199 .expect("validation-read shutdown bucket incarnation should resolve");
11200 let tier_name = "VALIDATION-READ-SHUTDOWN";
11201 let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
11202 let identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
11203 .await
11204 .expect("validation-read shutdown tier lease should resolve")
11205 .backend_identity();
11206 let (manifest_name, _) =
11207 install_aborting_dispatch_fixture(store.clone(), bucket, incarnation, "archive/", tier_name, identity, JOURNAL_COUNT)
11208 .await;
11209 let hook = TierDeleteDispatchMemberReadTestHook::install_pause(TierDeleteDispatchMemberReadTestStage::Validation);
11210 let worker_store = store.clone();
11211 let worker_manifest = manifest_name.clone();
11212 let worker = tokio::spawn(async move {
11213 recover_test_tier_delete_dispatch_manifest_with_page_budget(worker_store, &worker_manifest, Duration::from_secs(30))
11214 .await
11215 });
11216 tokio::time::timeout(Duration::from_secs(30), hook.wait_until_read_pause_count(ADMITTED_READS))
11217 .await
11218 .expect("the first validation read window should be admitted");
11219 assert_eq!(hook.entry_count(), ADMITTED_READS);
11220 assert_eq!(hook.in_flight(), ADMITTED_READS);
11221
11222 shutdown.cancel();
11223 tokio::time::sleep(Duration::from_millis(50)).await;
11224 assert_eq!(hook.entry_count(), ADMITTED_READS, "shutdown must not admit validation tail reads");
11225 assert_eq!(tier_delete_dispatch_manifest_recovery_count_for_test(&store), 1);
11226 assert!(tier_delete_dispatch_manifest_operation_lock_held_for_test(store.clone(), &manifest_name).await);
11227 assert!(crate::services::notification_sys::tier_delete_journal_fleet_proof_has_inflight_for_test());
11228 assert_eq!(tier_delete_journal_count(store.clone()).await, JOURNAL_COUNT);
11229 assert_eq!(backend.remove_count().await, 0);
11230
11231 hook.release_all_reads();
11232 assert!(
11233 !worker.await.expect("validation-read shutdown worker should join"),
11234 "cooperative shutdown should retain an unvalidated rollback"
11235 );
11236 assert_eq!(hook.entry_count(), ADMITTED_READS);
11237 assert_eq!(hook.in_flight(), 0);
11238 assert_eq!(tier_delete_dispatch_manifest_recovery_count_for_test(&store), 0);
11239 assert_eq!(
11240 tier_delete_journal_count(store.clone()).await,
11241 JOURNAL_COUNT,
11242 "no rollback DELETE may start"
11243 );
11244 assert_eq!(
11245 test_tier_delete_dispatch_manifest_state(store.clone(), &manifest_name)
11246 .await
11247 .expect("retained Aborting manifest should remain readable"),
11248 Some(TierDeleteDispatchManifestState::Aborting)
11249 );
11250 drop(hook);
11251 drop(store);
11252 drop(ctx);
11253
11254 let (_restarted_ctx, restarted_store, restarted_shutdown) = without_storage_class_env(build_isolated_test_store(
11255 temp_dir.path(),
11256 "dispatch-validation-read-shutdown-restart",
11257 &[4],
11258 ))
11259 .await;
11260 crate::bucket::metadata_sys::init_bucket_metadata_sys(restarted_store.clone(), Vec::new()).await;
11261 drive_tier_delete_dispatch_restart_to_convergence(restarted_store.clone()).await;
11262 restarted_shutdown.cancel();
11263 }
11264
11265 #[cfg(feature = "test-util")]
11266 #[tokio::test]
11267 #[serial_test::serial(storage_class_env)]
11268 async fn dispatch_manifest_shutdown_stops_authorized_tail_reads_before_receipt_or_cas() {
11269 const JOURNAL_COUNT: usize = 65;
11270 const ADMITTED_READS: usize = 32;
11271
11272 let temp_dir = tempfile::tempdir().expect("create authorized-read shutdown store dir");
11273 let (ctx, store, shutdown) =
11274 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "dispatch-authorized-read-shutdown", &[4]))
11275 .await;
11276 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
11277 let bucket = "dispatch-authorized-read-shutdown-bucket";
11278 store
11279 .make_bucket(bucket, &MakeBucketOptions::default())
11280 .await
11281 .expect("authorized-read shutdown bucket should be created");
11282 let incarnation = store
11283 .bucket_incarnation_id(bucket)
11284 .await
11285 .expect("authorized-read shutdown bucket incarnation should resolve");
11286 let tier_name = "AUTHORIZED-READ-SHUTDOWN";
11287 let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
11288 let identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
11289 .await
11290 .expect("authorized-read shutdown tier lease should resolve")
11291 .backend_identity();
11292 let entries = (0..JOURNAL_COUNT)
11293 .map(|index| {
11294 (
11295 synthetic_v6_dispatch_entry(
11296 bucket,
11297 &format!("archive/{index:06}.bin"),
11298 tier_name,
11299 identity,
11300 &uuid::Uuid::new_v4().to_string(),
11301 ),
11302 Some(TierDeleteJournalState::Committed),
11303 )
11304 })
11305 .collect();
11306 let (manifest_name, _) = install_test_tier_delete_dispatch_fixture(
11307 store.clone(),
11308 bucket,
11309 incarnation,
11310 "archive/",
11311 entries,
11312 TierDeleteDispatchManifestState::DispatchAuthorized,
11313 )
11314 .await
11315 .expect("authorized-read shutdown fixture should persist");
11316 let tier_config = ctx
11317 .tier_config_mgr()
11318 .read()
11319 .await
11320 .tiers
11321 .get(tier_name)
11322 .cloned()
11323 .expect("authorized-read tier config should remain available for restart");
11324 let hook = TierDeleteDispatchMemberReadTestHook::install_pause(TierDeleteDispatchMemberReadTestStage::Authorized);
11325 let worker_store = store.clone();
11326 let worker_manifest = manifest_name.clone();
11327 let worker = tokio::spawn(async move {
11328 recover_test_tier_delete_dispatch_manifest_with_page_budget(worker_store, &worker_manifest, Duration::from_secs(30))
11329 .await
11330 });
11331 tokio::time::timeout(Duration::from_secs(30), hook.wait_until_read_pause_count(ADMITTED_READS))
11332 .await
11333 .expect("the first authorized read window should be admitted");
11334
11335 shutdown.cancel();
11336 tokio::time::sleep(Duration::from_millis(50)).await;
11337 assert_eq!(hook.entry_count(), ADMITTED_READS, "shutdown must not admit authorized tail reads");
11338 assert_eq!(hook.in_flight(), ADMITTED_READS);
11339 assert_eq!(tier_delete_dispatch_manifest_recovery_count_for_test(&store), 1);
11340 assert!(tier_delete_dispatch_manifest_operation_lock_held_for_test(store.clone(), &manifest_name).await);
11341 assert!(crate::services::notification_sys::tier_delete_journal_fleet_proof_has_inflight_for_test());
11342
11343 hook.release_all_reads();
11344 assert!(
11345 !worker.await.expect("authorized-read shutdown worker should join"),
11346 "cooperative shutdown should retain an unscanned authorized manifest"
11347 );
11348 assert_eq!(hook.entry_count(), ADMITTED_READS);
11349 assert_eq!(hook.in_flight(), 0);
11350 assert_eq!(tier_delete_dispatch_manifest_recovery_count_for_test(&store), 0);
11351 assert_eq!(
11352 test_tier_delete_dispatch_manifest_state(store.clone(), &manifest_name)
11353 .await
11354 .expect("retained Authorized manifest should remain readable"),
11355 Some(TierDeleteDispatchManifestState::DispatchAuthorized),
11356 "cancellation before member reads must prevent the Completed CAS"
11357 );
11358 assert_eq!(
11359 hook.authorized_progress_count(),
11360 0,
11361 "cancellation before member reads must not attempt a decommission progress receipt"
11362 );
11363 assert_eq!(backend.remove_count().await, 0);
11364 drop(hook);
11365 drop(store);
11366 drop(ctx);
11367
11368 let (restarted_ctx, restarted_store, restarted_shutdown) = without_storage_class_env(build_isolated_test_store(
11369 temp_dir.path(),
11370 "dispatch-authorized-read-shutdown-restart",
11371 &[4],
11372 ))
11373 .await;
11374 {
11375 let tier_config_mgr = restarted_ctx.tier_config_mgr();
11376 let mut manager = tier_config_mgr.write().await;
11377 manager.tiers.insert(tier_name.to_string(), tier_config);
11378 manager
11379 .install_test_driver(tier_name, Box::new(backend.clone()))
11380 .expect("the exact authorized-read tier driver should reinstall after restart");
11381 }
11382 crate::bucket::metadata_sys::init_bucket_metadata_sys(restarted_store.clone(), Vec::new()).await;
11383 drive_tier_delete_dispatch_restart_to_convergence(restarted_store.clone()).await;
11384 assert_eq!(backend.remove_count().await, JOURNAL_COUNT);
11385 restarted_shutdown.cancel();
11386 }
11387
11388 #[cfg(feature = "test-util")]
11389 #[tokio::test]
11390 #[serial_test::serial(storage_class_env)]
11391 async fn dispatch_completion_cas_is_bounded_and_reaches_the_tail() {
11392 const JOURNAL_COUNT: usize = 65;
11393 const ADMITTED_COMMITS: usize = 32;
11394
11395 let temp_dir = tempfile::tempdir().expect("create dispatch completion concurrency store dir");
11396 let (ctx, store, shutdown) =
11397 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "dispatch-completion-concurrency", &[4])).await;
11398 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
11399 let bucket = "dispatch-completion-concurrency-bucket";
11400 let prefix = "archive/";
11401 store
11402 .make_bucket(bucket, &MakeBucketOptions::default())
11403 .await
11404 .expect("dispatch completion concurrency bucket should be created");
11405 let incarnation = store
11406 .bucket_incarnation_id(bucket)
11407 .await
11408 .expect("dispatch completion concurrency bucket incarnation should resolve");
11409 let tier_name = "DISPATCH-COMPLETION-CONCURRENCY";
11410 let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
11411 backend.set_remove_failure(true);
11412 let identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
11413 .await
11414 .expect("dispatch completion concurrency tier lease should resolve")
11415 .backend_identity();
11416 let entries = (0..JOURNAL_COUNT)
11417 .map(|index| {
11418 synthetic_v6_dispatch_entry(
11419 bucket,
11420 &format!("{prefix}{index:06}.bin"),
11421 tier_name,
11422 identity,
11423 &uuid::Uuid::new_v4().to_string(),
11424 )
11425 })
11426 .collect::<Vec<_>>();
11427 let (manifest_name, _) = install_test_tier_delete_dispatch_fixture(
11428 store.clone(),
11429 bucket,
11430 incarnation,
11431 prefix,
11432 entries
11433 .iter()
11434 .cloned()
11435 .map(|entry| (entry, Some(TierDeleteJournalState::Dispatched)))
11436 .collect(),
11437 TierDeleteDispatchManifestState::DispatchAuthorized,
11438 )
11439 .await
11440 .expect("dispatch completion concurrency fixture should persist");
11441
11442 let lifecycle_guard = store
11443 .acquire_bucket_lifecycle_write_lock(bucket)
11444 .await
11445 .expect("dispatch completion should acquire the bucket lifecycle fence");
11446 let mut fence_opts = ObjectOptions::default();
11447 fence_opts.add_bucket_lifecycle_lock_guard(&lifecycle_guard);
11448 let bucket_fence = fence_opts
11449 .bucket_lifecycle_lock_fence
11450 .clone()
11451 .expect("dispatch completion should capture the bucket lifecycle fence");
11452 let fleet_proof =
11453 acquire_tier_delete_journal_fleet_proof().expect("dispatch completion concurrency fixture should have a fleet proof");
11454 let prepared =
11455 prepare_tier_delete_dispatch(store.clone(), bucket, incarnation, prefix, entries, fleet_proof, &bucket_fence)
11456 .await
11457 .expect("the existing Authorized dispatch should reload");
11458 let active = prepared
11459 .consume(bucket, incarnation, prefix)
11460 .expect("the existing Authorized dispatch permit should be consumable");
11461 active
11462 .authorization()
11463 .mark_mutation_started(bucket, incarnation, prefix)
11464 .expect("the completion test should mark its synthetic local mutation");
11465
11466 let hook = TierDeleteDispatchMemberReadTestHook::install_pause(TierDeleteDispatchMemberReadTestStage::Completion);
11467 let worker_store = store.clone();
11468 let worker = tokio::spawn(async move { complete_tier_delete_dispatch(worker_store, &active, &bucket_fence).await });
11469 tokio::time::timeout(Duration::from_secs(30), hook.wait_until_read_pause_count(ADMITTED_COMMITS))
11470 .await
11471 .expect("the first completion CAS window should be admitted");
11472 tokio::time::sleep(Duration::from_millis(50)).await;
11473 assert_eq!(hook.entry_count(), ADMITTED_COMMITS, "completion must not admit an unbounded CAS tail");
11474 assert_eq!(hook.in_flight(), ADMITTED_COMMITS);
11475
11476 hook.release_all_reads();
11477 worker
11478 .await
11479 .expect("dispatch completion concurrency worker should join")
11480 .expect("every bounded completion CAS should succeed");
11481 assert_eq!(
11482 hook.entry_count(),
11483 JOURNAL_COUNT,
11484 "completion must eventually admit the entire journal set"
11485 );
11486 assert_eq!(hook.in_flight(), 0);
11487 assert_eq!(
11488 test_tier_delete_dispatch_manifest_state(store.clone(), &manifest_name)
11489 .await
11490 .expect("completed concurrency manifest should remain readable"),
11491 Some(TierDeleteDispatchManifestState::Completed)
11492 );
11493 drop(hook);
11494 shutdown.cancel();
11495 }
11496
11497 #[cfg(feature = "test-util")]
11498 #[tokio::test]
11499 #[serial_test::serial(storage_class_env)]
11500 async fn dispatch_manifest_shutdown_stops_completed_missing_journal_tail_reads() {
11501 const JOURNAL_COUNT: usize = 65;
11502 const ADMITTED_READS: usize = 32;
11503
11504 let temp_dir = tempfile::tempdir().expect("create completed-read shutdown store dir");
11505 let (ctx, store, shutdown) =
11506 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "dispatch-completed-read-shutdown", &[4])).await;
11507 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
11508 let bucket = "dispatch-completed-read-shutdown-bucket";
11509 store
11510 .make_bucket(bucket, &MakeBucketOptions::default())
11511 .await
11512 .expect("completed-read shutdown bucket should be created");
11513 let incarnation = store
11514 .bucket_incarnation_id(bucket)
11515 .await
11516 .expect("completed-read shutdown bucket incarnation should resolve");
11517 let tier_name = "COMPLETED-READ-SHUTDOWN";
11518 let _backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
11519 let identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
11520 .await
11521 .expect("completed-read shutdown tier lease should resolve")
11522 .backend_identity();
11523 let entries = (0..JOURNAL_COUNT)
11524 .map(|index| {
11525 (
11526 synthetic_v6_dispatch_entry(
11527 bucket,
11528 &format!("archive/{index:06}.bin"),
11529 tier_name,
11530 identity,
11531 &uuid::Uuid::new_v4().to_string(),
11532 ),
11533 None,
11534 )
11535 })
11536 .collect();
11537 let (manifest_name, _) = install_test_tier_delete_dispatch_fixture(
11538 store.clone(),
11539 bucket,
11540 incarnation,
11541 "archive/",
11542 entries,
11543 TierDeleteDispatchManifestState::Completed,
11544 )
11545 .await
11546 .expect("completed-read shutdown fixture should persist");
11547 let hook = TierDeleteDispatchMemberReadTestHook::install_pause(TierDeleteDispatchMemberReadTestStage::Completed);
11548 let worker_store = store.clone();
11549 let worker_manifest = manifest_name.clone();
11550 let worker = tokio::spawn(async move {
11551 recover_test_tier_delete_dispatch_manifest_with_page_budget(worker_store, &worker_manifest, Duration::from_secs(30))
11552 .await
11553 });
11554 tokio::time::timeout(Duration::from_secs(30), hook.wait_until_read_pause_count(ADMITTED_READS))
11555 .await
11556 .expect("the first Completed read window should be admitted");
11557
11558 shutdown.cancel();
11559 tokio::time::sleep(Duration::from_millis(50)).await;
11560 assert_eq!(hook.entry_count(), ADMITTED_READS, "shutdown must not admit later Completed chunks");
11561 assert_eq!(hook.in_flight(), ADMITTED_READS);
11562 assert_eq!(tier_delete_dispatch_manifest_recovery_count_for_test(&store), 1);
11563 assert!(tier_delete_dispatch_manifest_operation_lock_held_for_test(store.clone(), &manifest_name).await);
11564 assert!(crate::services::notification_sys::tier_delete_journal_fleet_proof_has_inflight_for_test());
11565
11566 hook.release_all_reads();
11567 assert!(
11568 !worker.await.expect("completed-read shutdown worker should join"),
11569 "cooperative shutdown should retain a partially scanned Completed manifest"
11570 );
11571 assert_eq!(hook.entry_count(), ADMITTED_READS);
11572 assert_eq!(hook.in_flight(), 0);
11573 assert_eq!(tier_delete_dispatch_manifest_recovery_count_for_test(&store), 0);
11574 assert_eq!(tier_delete_journal_count(store.clone()).await, 0);
11575 assert_eq!(
11576 tier_delete_dispatch_manifest_count(store.clone()).await,
11577 1,
11578 "manifest DELETE must not run"
11579 );
11580 assert_eq!(
11581 test_tier_delete_dispatch_manifest_state(store.clone(), &manifest_name)
11582 .await
11583 .expect("retained Completed manifest should remain readable"),
11584 Some(TierDeleteDispatchManifestState::Completed)
11585 );
11586 drop(hook);
11587 drop(store);
11588 drop(ctx);
11589
11590 let (_restarted_ctx, restarted_store, restarted_shutdown) = without_storage_class_env(build_isolated_test_store(
11591 temp_dir.path(),
11592 "dispatch-completed-read-shutdown-restart",
11593 &[4],
11594 ))
11595 .await;
11596 crate::bucket::metadata_sys::init_bucket_metadata_sys(restarted_store.clone(), Vec::new()).await;
11597 drive_tier_delete_dispatch_restart_to_convergence(restarted_store.clone()).await;
11598 restarted_shutdown.cancel();
11599 }
11600
11601 #[cfg(feature = "test-util")]
11602 #[tokio::test]
11603 #[serial_test::serial(storage_class_env)]
11604 async fn dispatch_manifest_shutdown_drains_preparing_cas_before_releasing_fences() {
11605 let temp_dir = tempfile::tempdir().expect("create preparing CAS shutdown store dir");
11606 let (ctx, store, shutdown) =
11607 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "dispatch-preparing-cas-shutdown", &[4])).await;
11608 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
11609 let bucket = "dispatch-preparing-cas-shutdown-bucket";
11610 store
11611 .make_bucket(bucket, &MakeBucketOptions::default())
11612 .await
11613 .expect("preparing CAS shutdown bucket should be created");
11614 let incarnation = store
11615 .bucket_incarnation_id(bucket)
11616 .await
11617 .expect("preparing CAS shutdown bucket incarnation should resolve");
11618 let entry = synthetic_v6_dispatch_entry(
11619 bucket,
11620 "archive/preparing-cas.bin",
11621 "PREPARING-CAS-SHUTDOWN",
11622 [31; 32],
11623 &uuid::Uuid::new_v4().to_string(),
11624 );
11625 let (manifest_name, _) = install_test_tier_delete_dispatch_fixture(
11626 store.clone(),
11627 bucket,
11628 incarnation,
11629 "archive/",
11630 vec![(entry, Some(TierDeleteJournalState::Prepared))],
11631 TierDeleteDispatchManifestState::Preparing,
11632 )
11633 .await
11634 .expect("preparing CAS shutdown fixture should persist");
11635 let barrier = crate::set_disk::PutObjectCommitBarrier::install(
11636 RUSTFS_META_BUCKET,
11637 &manifest_name,
11638 crate::set_disk::PutObjectCommitPause::BeforeQuotaRename,
11639 );
11640 let worker_store = store.clone();
11641 let worker_manifest = manifest_name.clone();
11642 let worker = tokio::spawn(async move {
11643 recover_test_tier_delete_dispatch_manifest_with_page_budget(worker_store, &worker_manifest, Duration::from_secs(30))
11644 .await
11645 });
11646 barrier.wait_until_paused().await;
11647
11648 shutdown.cancel();
11649 tokio::time::sleep(Duration::from_millis(50)).await;
11650 assert_eq!(
11651 tier_delete_dispatch_manifest_recovery_count_for_test(&store),
11652 1,
11653 "a paused CAS must retain its registry permit after shutdown"
11654 );
11655 assert!(
11656 tier_delete_dispatch_manifest_operation_lock_held_for_test(store.clone(), &manifest_name).await,
11657 "a paused CAS must retain its manifest operation lock after shutdown"
11658 );
11659 assert!(
11660 crate::services::notification_sys::tier_delete_journal_fleet_proof_has_inflight_for_test(),
11661 "a paused CAS must retain its fleet-proof generation permit"
11662 );
11663
11664 barrier.release();
11665 assert!(
11666 !worker.await.expect("preparing CAS shutdown worker should join"),
11667 "cooperative shutdown should retain the partially advanced manifest"
11668 );
11669 assert_eq!(tier_delete_dispatch_manifest_recovery_count_for_test(&store), 0);
11670 assert!(
11671 !tier_delete_dispatch_manifest_operation_lock_held_for_test(store.clone(), &manifest_name).await,
11672 "the operation lock may release only after the admitted CAS drains"
11673 );
11674 assert!(
11675 com::read_config(store.clone(), &manifest_name).await.is_ok(),
11676 "shutdown must retain the durable manifest for restart replay"
11677 );
11678 drop(barrier);
11679 drop(store);
11680 drop(ctx);
11681
11682 let (_restarted_ctx, restarted_store, restarted_shutdown) = without_storage_class_env(build_isolated_test_store(
11683 temp_dir.path(),
11684 "dispatch-preparing-cas-shutdown-restart",
11685 &[4],
11686 ))
11687 .await;
11688 crate::bucket::metadata_sys::init_bucket_metadata_sys(restarted_store.clone(), Vec::new()).await;
11689 tokio::time::timeout(Duration::from_secs(30), async {
11690 let mut stable_empty_observations = 0;
11691 loop {
11692 if tier_delete_journal_count(restarted_store.clone()).await == 0
11693 && tier_delete_dispatch_manifest_count(restarted_store.clone()).await == 0
11694 && tier_delete_dispatch_manifest_recovery_count_for_test(&restarted_store) == 0
11695 {
11696 stable_empty_observations += 1;
11697 if stable_empty_observations == 10 {
11698 break;
11699 }
11700 } else {
11701 stable_empty_observations = 0;
11702 }
11703 tokio::time::sleep(Duration::from_millis(50)).await;
11704 }
11705 })
11706 .await
11707 .expect("the restarted background recovery should replay the drained Preparing CAS");
11708 assert_eq!(tier_delete_journal_count(restarted_store.clone()).await, 0);
11709 assert_eq!(tier_delete_dispatch_manifest_count(restarted_store).await, 0);
11710 restarted_shutdown.cancel();
11711 }
11712
11713 #[cfg(feature = "test-util")]
11714 #[tokio::test]
11715 #[serial_test::serial(storage_class_env)]
11716 async fn dispatch_manifest_shutdown_drains_admitted_delete_batch_without_tail_admission() {
11717 const JOURNAL_COUNT: usize = 65;
11718 const ADMITTED_BATCH: usize = 32;
11719
11720 let temp_dir = tempfile::tempdir().expect("create delete-batch shutdown store dir");
11721 let mut instance_ctx = crate::runtime::instance::InstanceContext::new();
11723 instance_ctx.suppress_tier_delete_journal_recovery_for_test();
11724 let (ctx, store, shutdown) = without_storage_class_env(build_isolated_test_store_with_layout(
11725 temp_dir.path(),
11726 "dispatch-delete-batch-shutdown",
11727 &[(1, 4)],
11728 CancellationToken::new(),
11729 Some(Arc::new(instance_ctx)),
11730 ))
11731 .await;
11732 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
11733 let bucket = "dispatch-delete-batch-shutdown-bucket";
11734 store
11735 .make_bucket(bucket, &MakeBucketOptions::default())
11736 .await
11737 .expect("delete-batch shutdown bucket should be created");
11738 let incarnation = store
11739 .bucket_incarnation_id(bucket)
11740 .await
11741 .expect("delete-batch shutdown bucket incarnation should resolve");
11742 let (manifest_name, entries) = install_aborting_dispatch_fixture(
11743 store.clone(),
11744 bucket,
11745 incarnation,
11746 "archive/",
11747 "DELETE-BATCH-SHUTDOWN",
11748 [32; 32],
11749 JOURNAL_COUNT,
11750 )
11751 .await;
11752 let mut journal_names = entries.iter().map(tier_delete_journal_object_name).collect::<Vec<_>>();
11753 journal_names.sort();
11754 let published_delete_name = journal_names[0].clone();
11755 let hook = TierDeleteDispatchRollbackTestHook::install_pause_all_except_delete(&published_delete_name);
11756 let barrier =
11757 crate::set_disk::DeleteObjectCommitBarrier::install_after_publish(RUSTFS_META_BUCKET, &published_delete_name);
11758 let worker_store = store.clone();
11759 let worker_manifest = manifest_name.clone();
11760 let worker = tokio::spawn(async move {
11761 recover_test_tier_delete_dispatch_manifest_with_page_budget(worker_store, &worker_manifest, Duration::from_secs(30))
11762 .await
11763 });
11764 barrier.wait_until_paused().await;
11765 tokio::time::timeout(Duration::from_secs(30), hook.wait_until_delete_pause_count(ADMITTED_BATCH - 1))
11766 .await
11767 .expect("the rest of the bounded delete batch should pause before mutation");
11768 assert_eq!(hook.delete_entry_count(), ADMITTED_BATCH);
11769
11770 shutdown.cancel();
11771 tokio::time::sleep(Duration::from_millis(50)).await;
11772 assert_eq!(tier_delete_dispatch_manifest_recovery_count_for_test(&store), 1);
11773 assert!(
11774 tier_delete_dispatch_manifest_operation_lock_held_for_test(store.clone(), &manifest_name).await,
11775 "the partially published DELETE must retain its operation lock"
11776 );
11777 assert!(
11778 crate::services::notification_sys::tier_delete_journal_fleet_proof_has_inflight_for_test(),
11779 "the partially published DELETE must retain its fleet-proof permit"
11780 );
11781 assert_eq!(
11782 hook.delete_entry_count(),
11783 ADMITTED_BATCH,
11784 "shutdown must stop admitting the tail behind the bounded batch"
11785 );
11786
11787 barrier.release();
11788 tokio::time::sleep(Duration::from_millis(50)).await;
11789 assert_eq!(
11790 tier_delete_dispatch_manifest_recovery_count_for_test(&store),
11791 1,
11792 "the worker must retain its guards until every admitted sibling drains"
11793 );
11794 hook.release_all_deletes();
11795 assert!(
11796 !worker.await.expect("delete-batch shutdown worker should join"),
11797 "cooperative shutdown should retain the partial rollback for restart"
11798 );
11799 assert_eq!(hook.delete_entry_count(), ADMITTED_BATCH);
11800 assert_eq!(tier_delete_dispatch_manifest_recovery_count_for_test(&store), 0);
11801 assert!(
11802 tier_delete_journal_count(store.clone()).await >= JOURNAL_COUNT - 1,
11803 "only the already-published DELETE may commit before restart"
11804 );
11805 drop(barrier);
11806 drop(hook);
11807 drop(store);
11808 drop(ctx);
11809
11810 let (_restarted_ctx, restarted_store, restarted_shutdown) =
11811 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "dispatch-delete-batch-shutdown-restart", &[4]))
11812 .await;
11813 crate::bucket::metadata_sys::init_bucket_metadata_sys(restarted_store.clone(), Vec::new()).await;
11814 tokio::time::timeout(Duration::from_secs(30), async {
11815 let mut stable_empty_observations = 0;
11816 loop {
11817 if tier_delete_journal_count(restarted_store.clone()).await == 0
11818 && tier_delete_dispatch_manifest_count(restarted_store.clone()).await == 0
11819 && tier_delete_dispatch_manifest_recovery_count_for_test(&restarted_store) == 0
11820 {
11821 stable_empty_observations += 1;
11822 if stable_empty_observations == 10 {
11823 break;
11824 }
11825 } else {
11826 stable_empty_observations = 0;
11827 }
11828 tokio::time::sleep(Duration::from_millis(50)).await;
11829 }
11830 })
11831 .await
11832 .expect("the restarted background recovery should replay the partially drained delete batch");
11833 assert_eq!(tier_delete_journal_count(restarted_store.clone()).await, 0);
11834 assert_eq!(tier_delete_dispatch_manifest_count(restarted_store).await, 0);
11835 restarted_shutdown.cancel();
11836 }
11837
11838 #[cfg(feature = "test-util")]
11839 #[tokio::test]
11840 #[serial_test::serial(storage_class_env)]
11841 async fn dispatch_manifest_rollback_retries_delete_and_confirmation_failures() {
11842 const JOURNAL_COUNT: usize = 40;
11843
11844 let temp_dir = tempfile::tempdir().expect("create rollback retry store dir");
11845 let mut instance_ctx = crate::runtime::instance::InstanceContext::new();
11847 instance_ctx.suppress_tier_delete_journal_recovery_for_test();
11848 let (ctx, store, shutdown) = without_storage_class_env(build_isolated_test_store_with_layout(
11849 temp_dir.path(),
11850 "dispatch-rollback-retry",
11851 &[(1, 4)],
11852 CancellationToken::new(),
11853 Some(Arc::new(instance_ctx)),
11854 ))
11855 .await;
11856 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
11857 let bucket = "dispatch-rollback-retry-bucket";
11858 store
11859 .make_bucket(bucket, &MakeBucketOptions::default())
11860 .await
11861 .expect("rollback retry bucket should be created");
11862 let incarnation = store
11863 .bucket_incarnation_id(bucket)
11864 .await
11865 .expect("rollback retry bucket incarnation should resolve");
11866 let tier_name = "DISPATCH-ROLLBACK-RETRY";
11867 let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
11868 let identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
11869 .await
11870 .expect("rollback retry tier lease should resolve")
11871 .backend_identity();
11872
11873 let (delete_manifest, delete_entries) = install_aborting_dispatch_fixture(
11874 store.clone(),
11875 bucket,
11876 incarnation,
11877 "delete-failure/",
11878 tier_name,
11879 identity,
11880 JOURNAL_COUNT,
11881 )
11882 .await;
11883 let delete_failure_name = tier_delete_journal_object_name(&delete_entries[10]);
11884 let delete_hook = TierDeleteDispatchRollbackTestHook::install_delete_failure(&delete_failure_name);
11885 let delete_error = recover_test_tier_delete_dispatch_manifest(store.clone(), &delete_manifest)
11886 .await
11887 .expect_err("an injected member delete failure must retain the manifest")
11888 .to_string();
11889 assert!(delete_error.contains("injected tier delete dispatch rollback delete failure"));
11890 assert!(
11891 tier_delete_journal_count(store.clone()).await > 0,
11892 "the failed member and all work not admitted after the first error must remain retryable"
11893 );
11894 drop(delete_hook);
11895 recover_test_tier_delete_dispatch_manifest(store.clone(), &delete_manifest)
11896 .await
11897 .expect("the delete-stage retry should converge");
11898
11899 let (confirmation_manifest, confirmation_entries) = install_aborting_dispatch_fixture(
11900 store.clone(),
11901 bucket,
11902 incarnation,
11903 "confirmation-failure/",
11904 tier_name,
11905 identity,
11906 JOURNAL_COUNT,
11907 )
11908 .await;
11909 let confirmation_failure_name = tier_delete_journal_object_name(&confirmation_entries[10]);
11910 let confirmation_hook = TierDeleteDispatchRollbackTestHook::install_confirmation_failure(&confirmation_failure_name);
11911 let confirmation_error = recover_test_tier_delete_dispatch_manifest(store.clone(), &confirmation_manifest)
11912 .await
11913 .expect_err("an injected confirmation failure must retain the manifest")
11914 .to_string();
11915 assert!(confirmation_error.contains("injected tier delete dispatch rollback confirmation failure"));
11916 assert_eq!(
11917 tier_delete_journal_count(store.clone()).await,
11918 0,
11919 "confirmation failure occurs only after the bounded delete phase drains"
11920 );
11921 assert_eq!(tier_delete_dispatch_manifest_count(store.clone()).await, 1);
11922 drop(confirmation_hook);
11923 recover_test_tier_delete_dispatch_manifest(store.clone(), &confirmation_manifest)
11924 .await
11925 .expect("the confirmation-stage retry should converge over missing members");
11926
11927 assert_eq!(tier_delete_dispatch_manifest_count(store.clone()).await, 0);
11928 assert_eq!(backend.remove_count().await, 0, "rollback retries must never call the remote tier");
11929 shutdown.cancel();
11930 }
11931
11932 #[cfg(feature = "test-util")]
11933 #[test]
11934 #[serial_test::serial(storage_class_env)]
11935 fn v6_decommission_checkpoint_no_lock_put_rejects_lost_publication_fence() {
11936 run_large_stack_async_test(
11937 "v6-checkpoint-fence-loss",
11938 v6_decommission_checkpoint_no_lock_put_rejects_lost_publication_fence_case,
11939 );
11940 }
11941
11942 #[cfg(feature = "test-util")]
11943 async fn v6_decommission_checkpoint_no_lock_put_rejects_lost_publication_fence_case() {
11944 let temp_dir = tempfile::tempdir().expect("create v6 checkpoint fence-loss store dir");
11945 let (_ctx, store, _shutdown) =
11946 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "v6-checkpoint-fence-loss", &[4, 4])).await;
11947 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
11948 let bucket = "v6-checkpoint-fence-loss-bucket";
11949 store
11950 .make_bucket(bucket, &MakeBucketOptions::default())
11951 .await
11952 .expect("checkpoint fence-loss bucket should be created");
11953 let incarnation = store
11954 .bucket_incarnation_id(bucket)
11955 .await
11956 .expect("checkpoint fence-loss bucket incarnation should resolve");
11957 let entry = synthetic_v6_dispatch_entry(
11958 bucket,
11959 "archive/fence-loss.bin",
11960 "FENCE-LOSS-TIER",
11961 [18; 32],
11962 &uuid::Uuid::new_v4().to_string(),
11963 );
11964 let (manifest_name, entries) = install_test_tier_delete_dispatch_fixture(
11965 store.clone(),
11966 bucket,
11967 incarnation,
11968 "archive/",
11969 vec![(entry, Some(TierDeleteJournalState::Prepared))],
11970 TierDeleteDispatchManifestState::Preparing,
11971 )
11972 .await
11973 .expect("checkpoint fence-loss fixture should persist");
11974 let journal_name = tier_delete_journal_object_name(&entries[0]);
11975 let manifest_data = com::read_config(store.clone(), &manifest_name)
11976 .await
11977 .expect("checkpoint fence-loss manifest should be readable");
11978 let journal_data = com::read_config(store.clone(), &journal_name)
11979 .await
11980 .expect("checkpoint fence-loss journal should be readable");
11981 for pool in &store.pools {
11982 com::save_config(pool.clone(), &manifest_name, manifest_data.clone())
11983 .await
11984 .expect("manifest copy should persist in each checkpoint fence-loss pool");
11985 com::save_config(pool.clone(), &journal_name, journal_data.clone())
11986 .await
11987 .expect("journal copy should persist in each checkpoint fence-loss pool");
11988 }
11989 mark_test_pool_decommissioning(&store, 0).await;
11990 for (path, data) in [
11991 (&manifest_name, manifest_data.as_slice()),
11992 (&journal_name, journal_data.as_slice()),
11993 ] {
11994 let record = validate_durable_ilm_record(path, data).expect("checkpoint fence-loss v6 record should validate");
11995 store
11996 .persist_decommission_durable_ilm_receipt_for_test(0, 1, path, &record, false)
11997 .await
11998 .expect("checkpoint fence-loss receipt should persist");
11999 }
12000
12001 let loss_hook = crate::store::object::DecommissionMutationFenceLossHook::install(
12002 RUSTFS_META_BUCKET,
12003 &manifest_name,
12004 crate::store::object::DecommissionMutationFenceTestPhase::Migration,
12005 );
12006 let barrier = crate::set_disk::PutObjectCommitBarrier::install(
12007 RUSTFS_META_BUCKET,
12008 &manifest_name,
12009 crate::set_disk::PutObjectCommitPause::BeforeQuotaRename,
12010 );
12011 let worker_store = store.clone();
12012 let worker_manifest = manifest_name.clone();
12013 let worker =
12014 tokio::spawn(async move { recover_test_tier_delete_dispatch_manifest(worker_store, &worker_manifest).await });
12015 barrier.wait_until_paused().await;
12016 loss_hook.mark_lost();
12017 barrier.release();
12018 drop(barrier);
12019 let error = worker
12020 .await
12021 .expect("checkpoint fence-loss recovery task should join")
12022 .expect_err("lost publication fence must reject the no-lock checkpoint PUT")
12023 .to_string();
12024 assert!(
12025 error.contains("lock") || error.contains("fence"),
12026 "unexpected checkpoint fence-loss error: {error}"
12027 );
12028 assert_eq!(
12029 com::read_config(store.pools[1].clone(), &manifest_name)
12030 .await
12031 .expect("fenced target manifest should remain readable"),
12032 manifest_data,
12033 "the target checkpoint must not commit after its publication fence is lost"
12034 );
12035
12036 drop(loss_hook);
12037 recover_test_tier_delete_dispatch_manifest(store.clone(), &manifest_name)
12038 .await
12039 .expect("checkpoint recovery should converge after a fresh fence is acquired");
12040 }
12041
12042 #[cfg(feature = "test-util")]
12043 #[tokio::test]
12044 #[serial_test::serial(storage_class_env)]
12045 async fn v6_decommission_checkpoint_partial_target_failure_retries_without_advancing_receipts() {
12046 let temp_dir = tempfile::tempdir().expect("create v6 partial checkpoint store dir");
12047 let (_ctx, store, _shutdown) =
12048 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "v6-partial-checkpoint", &[4, 4, 4])).await;
12049 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
12050 let bucket = "v6-partial-checkpoint-bucket";
12051 store
12052 .make_bucket(bucket, &MakeBucketOptions::default())
12053 .await
12054 .expect("partial checkpoint bucket should be created");
12055 let incarnation = store
12056 .bucket_incarnation_id(bucket)
12057 .await
12058 .expect("partial checkpoint bucket incarnation should resolve");
12059 let entry = synthetic_v6_dispatch_entry(
12060 bucket,
12061 "archive/partial.bin",
12062 "PARTIAL-CHECKPOINT-TIER",
12063 [19; 32],
12064 &uuid::Uuid::new_v4().to_string(),
12065 );
12066 let (manifest_name, entries) = install_test_tier_delete_dispatch_fixture(
12067 store.clone(),
12068 bucket,
12069 incarnation,
12070 "archive/",
12071 vec![(entry, Some(TierDeleteJournalState::Prepared))],
12072 TierDeleteDispatchManifestState::Preparing,
12073 )
12074 .await
12075 .expect("partial checkpoint fixture should persist");
12076 let journal_name = tier_delete_journal_object_name(&entries[0]);
12077 let manifest_data = com::read_config(store.clone(), &manifest_name)
12078 .await
12079 .expect("partial checkpoint manifest should be readable");
12080 let journal_data = com::read_config(store.clone(), &journal_name)
12081 .await
12082 .expect("partial checkpoint journal should be readable");
12083 for pool in &store.pools {
12084 com::save_config(pool.clone(), &manifest_name, manifest_data.clone())
12085 .await
12086 .expect("manifest copy should persist in each partial checkpoint pool");
12087 com::save_config(pool.clone(), &journal_name, journal_data.clone())
12088 .await
12089 .expect("journal copy should persist in each partial checkpoint pool");
12090 }
12091 mark_test_pool_decommissioning_with_split_targets(&store, 0).await;
12092 for target_pool_index in [1, 2] {
12093 for (path, data) in [
12094 (&manifest_name, manifest_data.as_slice()),
12095 (&journal_name, journal_data.as_slice()),
12096 ] {
12097 let record = validate_durable_ilm_record(path, data).expect("partial checkpoint v6 record should validate");
12098 store
12099 .persist_decommission_durable_ilm_receipt_for_test(0, target_pool_index, path, &record, false)
12100 .await
12101 .expect("partial checkpoint receipt should persist on each target");
12102 }
12103 }
12104 let (aborting_data, preparing_etag) = test_tier_delete_dispatch_manifest_checkpoint(
12105 store.clone(),
12106 &manifest_name,
12107 TierDeleteDispatchManifestState::Aborting,
12108 )
12109 .await
12110 .expect("partial checkpoint Aborting generation should encode");
12111 let targets = store
12112 .decommission_durable_ilm_checkpoint_targets(&manifest_name, &aborting_data, &preparing_etag)
12113 .await
12114 .expect("partial checkpoint targets should resolve")
12115 .expect("the active decommission should own the partial checkpoint");
12116 assert_eq!(
12117 targets.iter().map(|target| target.target_pool_index).collect::<Vec<_>>(),
12118 vec![1, 2],
12119 "the capacity reservation must exercise two independently confirmed targets"
12120 );
12121
12122 let failure_hook = DecommissionCheckpointTargetFailureHook::install(2);
12123 let error = recover_test_tier_delete_dispatch_manifest(store.clone(), &manifest_name)
12124 .await
12125 .expect_err("the injected second-target failure must reject the checkpoint");
12126 assert!(
12127 error.to_string().contains("injected decommission checkpoint target failure"),
12128 "unexpected partial checkpoint error: {error}"
12129 );
12130 let mut diagnostic = &error as &(dyn std::error::Error + 'static);
12131 let mut target_context_found = false;
12132 while let Some(source) = diagnostic.source() {
12133 if source.to_string().contains("target pool 2") {
12134 target_context_found = true;
12135 break;
12136 }
12137 diagnostic = source;
12138 }
12139 assert!(
12140 target_context_found,
12141 "the stable checkpoint error must retain the failed target in its diagnostic source chain: {error:?}"
12142 );
12143 assert_eq!(
12144 com::read_config(store.pools[1].clone(), &manifest_name)
12145 .await
12146 .expect("first target checkpoint should be readable"),
12147 aborting_data,
12148 "the first target should model a committed checkpoint before the later failure"
12149 );
12150 assert_eq!(
12151 com::read_config(store.pools[2].clone(), &manifest_name)
12152 .await
12153 .expect("second target checkpoint should be readable"),
12154 manifest_data,
12155 "the failed second target must retain the prior generation"
12156 );
12157 let retry_targets = store
12158 .decommission_durable_ilm_checkpoint_targets(&manifest_name, &aborting_data, &preparing_etag)
12159 .await
12160 .expect("unadvanced receipts must still authorize the exact partial checkpoint retry")
12161 .expect("the partial checkpoint retry should remain decommission-owned");
12162 assert!(retry_targets[0].already_committed);
12163 assert!(!retry_targets[1].already_committed);
12164
12165 drop(failure_hook);
12166 recover_test_tier_delete_dispatch_manifest(store.clone(), &manifest_name)
12167 .await
12168 .expect("the partial checkpoint should converge after the target recovers");
12169 }
12170
12171 #[cfg(feature = "test-util")]
12172 #[tokio::test]
12173 #[serial_test::serial(storage_class_env)]
12174 async fn v6_decommission_checkpoint_shutdown_drains_target_put_and_replays_capacity() {
12175 let temp_dir = tempfile::tempdir().expect("create checkpoint shutdown store dir");
12176 let (ctx, store, shutdown) =
12177 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "v6-checkpoint-shutdown", &[4, 4])).await;
12178 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
12179 let bucket = "v6-checkpoint-shutdown-bucket";
12180 store
12181 .make_bucket(bucket, &MakeBucketOptions::default())
12182 .await
12183 .expect("checkpoint shutdown bucket should be created");
12184 let incarnation = store
12185 .bucket_incarnation_id(bucket)
12186 .await
12187 .expect("checkpoint shutdown bucket incarnation should resolve");
12188 let entry = synthetic_v6_dispatch_entry(
12189 bucket,
12190 "archive/checkpoint-shutdown.bin",
12191 "CHECKPOINT-SHUTDOWN",
12192 [33; 32],
12193 &uuid::Uuid::new_v4().to_string(),
12194 );
12195 let (manifest_name, entries) = install_test_tier_delete_dispatch_fixture(
12196 store.clone(),
12197 bucket,
12198 incarnation,
12199 "archive/",
12200 vec![(entry, Some(TierDeleteJournalState::Prepared))],
12201 TierDeleteDispatchManifestState::Preparing,
12202 )
12203 .await
12204 .expect("checkpoint shutdown fixture should persist");
12205 let journal_name = tier_delete_journal_object_name(&entries[0]);
12206 let manifest_data = com::read_config(store.clone(), &manifest_name)
12207 .await
12208 .expect("checkpoint shutdown manifest should be readable");
12209 let journal_data = com::read_config(store.clone(), &journal_name)
12210 .await
12211 .expect("checkpoint shutdown journal should be readable");
12212 for pool in &store.pools {
12213 com::save_config(pool.clone(), &manifest_name, manifest_data.clone())
12214 .await
12215 .expect("manifest copy should persist in each checkpoint shutdown pool");
12216 com::save_config(pool.clone(), &journal_name, journal_data.clone())
12217 .await
12218 .expect("journal copy should persist in each checkpoint shutdown pool");
12219 }
12220 mark_test_pool_decommissioning(&store, 0).await;
12221 for (path, data) in [
12222 (&manifest_name, manifest_data.as_slice()),
12223 (&journal_name, journal_data.as_slice()),
12224 ] {
12225 let record = validate_durable_ilm_record(path, data).expect("checkpoint shutdown v6 record should validate");
12226 store
12227 .persist_decommission_durable_ilm_receipt_for_test(0, 1, path, &record, false)
12228 .await
12229 .expect("checkpoint shutdown receipt should persist");
12230 }
12231 let (aborting_data, preparing_etag) = test_tier_delete_dispatch_manifest_checkpoint(
12232 store.clone(),
12233 &manifest_name,
12234 TierDeleteDispatchManifestState::Aborting,
12235 )
12236 .await
12237 .expect("checkpoint shutdown Aborting generation should encode");
12238 let targets = store
12239 .decommission_durable_ilm_checkpoint_targets(&manifest_name, &aborting_data, &preparing_etag)
12240 .await
12241 .expect("checkpoint shutdown target should resolve")
12242 .expect("the active decommission should own the checkpoint target");
12243 assert_eq!(targets.len(), 1);
12244 let target = targets[0].clone();
12245 let consumed_before_checkpoint = store.pool_meta.read().await.pools[0]
12246 .decommission
12247 .as_ref()
12248 .and_then(|info| info.capacity_reservation.as_ref())
12249 .expect("checkpoint shutdown capacity reservation should exist")
12250 .consumed_target_physical_bytes;
12251 let precommit_error = store
12252 .run_decommission_capacity_non_growing_replacement_with_capacity_lease(
12253 target.target_pool_index,
12254 Some(target.capacity_owner),
12255 Some(aborting_data.len()),
12256 |_| async { Err::<(), Error>(Error::other("injected checkpoint failure before commit")) },
12257 )
12258 .await
12259 .expect_err("the first checkpoint attempt should fail before writing its target")
12260 .to_string();
12261 assert!(precommit_error.contains("injected checkpoint failure before commit"));
12262 assert!(
12263 store
12264 .has_decommission_capacity_temporary_mutation_state(target.target_pool_index, target.capacity_owner)
12265 .await,
12266 "a failed checkpoint attempt must retain exact retry state"
12267 );
12268 let barrier = crate::set_disk::PutObjectCommitBarrier::install(
12269 RUSTFS_META_BUCKET,
12270 &manifest_name,
12271 crate::set_disk::PutObjectCommitPause::BeforeQuotaRename,
12272 );
12273 let worker_store = store.clone();
12274 let worker_manifest = manifest_name.clone();
12275 let worker = tokio::spawn(async move {
12276 recover_test_tier_delete_dispatch_manifest_with_page_budget(worker_store, &worker_manifest, Duration::from_secs(30))
12277 .await
12278 });
12279 barrier.wait_until_paused().await;
12280 assert!(
12281 store
12282 .has_decommission_capacity_temporary_mutation_state(target.target_pool_index, target.capacity_owner)
12283 .await,
12284 "the target PUT must persist its capacity intent before commit"
12285 );
12286
12287 shutdown.cancel();
12288 tokio::time::sleep(Duration::from_millis(50)).await;
12289 assert_eq!(tier_delete_dispatch_manifest_recovery_count_for_test(&store), 1);
12290 assert!(
12291 tier_delete_dispatch_manifest_operation_lock_held_for_test(store.clone(), &manifest_name).await,
12292 "the target checkpoint PUT must retain its operation lock after shutdown"
12293 );
12294 assert!(
12295 crate::services::notification_sys::tier_delete_journal_fleet_proof_has_inflight_for_test(),
12296 "the target checkpoint PUT must retain its fleet-proof permit after shutdown"
12297 );
12298
12299 barrier.release();
12300 assert!(
12301 !worker.await.expect("checkpoint shutdown worker should join"),
12302 "cooperative shutdown should retain the partially committed checkpoint"
12303 );
12304 assert_eq!(tier_delete_dispatch_manifest_recovery_count_for_test(&store), 0);
12305 assert_eq!(
12306 com::read_config(store.pools[target.target_pool_index].clone(), &manifest_name)
12307 .await
12308 .expect("the committed target checkpoint should be readable"),
12309 aborting_data,
12310 "the admitted target PUT must finish before its fences release"
12311 );
12312 assert!(
12313 !store
12314 .has_decommission_capacity_temporary_mutation_state(target.target_pool_index, target.capacity_owner)
12315 .await,
12316 "the admitted target PUT must drain its capacity transaction before releasing the recovery fences"
12317 );
12318 {
12319 let pool_meta = store.pool_meta.read().await;
12320 let reservation = pool_meta.pools[0]
12321 .decommission
12322 .as_ref()
12323 .and_then(|info| info.capacity_reservation.as_ref())
12324 .expect("checkpoint shutdown capacity reservation should remain active");
12325 let capacity_target = reservation
12326 .targets
12327 .iter()
12328 .find(|candidate| candidate.pool_index == target.target_pool_index)
12329 .expect("checkpoint shutdown capacity target should remain allocated");
12330 assert_eq!(
12331 reservation.consumed_target_physical_bytes, consumed_before_checkpoint,
12332 "a non-growing checkpoint replacement must not consume durable migration capacity"
12333 );
12334 assert_eq!(reservation.pending_target_physical_bytes, 0);
12335 assert_eq!(reservation.inflight_target_physical_bytes, 0);
12336 assert_eq!(capacity_target.pending_physical_bytes, 0);
12337 assert!(capacity_target.temporary_mutations.is_empty());
12338 }
12339 drop(barrier);
12340 let mut reloaded_pool_meta = PoolMeta::default();
12341 reloaded_pool_meta
12342 .load(store.pools[0].clone(), store.pools.clone())
12343 .await
12344 .expect("checkpoint shutdown pool metadata should reload from disk");
12345 *store.pool_meta.write().await = reloaded_pool_meta;
12346 recover_test_tier_delete_dispatch_manifest(store.clone(), &manifest_name)
12347 .await
12348 .expect("a fresh recovery attempt should replay the committed checkpoint and receipt");
12349 assert!(
12350 !store
12351 .has_decommission_capacity_temporary_mutation_state(target.target_pool_index, target.capacity_owner)
12352 .await,
12353 "durable replay must reconcile the exact capacity transaction"
12354 );
12355 assert_eq!(
12356 store
12357 .decommission_durable_ilm_receipt_count_for_test(0)
12358 .await
12359 .expect("checkpoint shutdown receipts should be listable"),
12360 2,
12361 "durable replay must advance both the manifest and journal receipts"
12362 );
12363 drop(ctx);
12364 }
12365
12366 #[cfg(feature = "test-util")]
12367 #[tokio::test]
12368 #[serial_test::serial(storage_class_env)]
12369 async fn v6_dispatch_recovery_advances_decommission_receipts_and_cleans_target_copies() {
12370 let temp_dir = tempfile::tempdir().expect("create v6 decommission receipt store dir");
12371 let (_ctx, store, _shutdown) =
12372 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "v6-dispatch-decommission-receipts", &[4, 4]))
12373 .await;
12374 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
12375 let bucket = "v6-dispatch-decommission-receipts-bucket";
12376 store
12377 .make_bucket(bucket, &MakeBucketOptions::default())
12378 .await
12379 .expect("receipt bucket should be created");
12380 let incarnation = store
12381 .bucket_incarnation_id(bucket)
12382 .await
12383 .expect("receipt bucket incarnation should resolve");
12384 let entry = synthetic_v6_dispatch_entry(
12385 bucket,
12386 "archive/receipt.bin",
12387 "RECEIPT-TIER",
12388 [17; 32],
12389 &uuid::Uuid::new_v4().to_string(),
12390 );
12391 let (manifest_name, entries) = install_test_tier_delete_dispatch_fixture(
12392 store.clone(),
12393 bucket,
12394 incarnation,
12395 "archive/",
12396 vec![(entry, Some(TierDeleteJournalState::Prepared))],
12397 TierDeleteDispatchManifestState::Preparing,
12398 )
12399 .await
12400 .expect("Preparing v6 fixture should persist");
12401 let journal_name = tier_delete_journal_object_name(&entries[0]);
12402 let manifest_data = com::read_config(store.clone(), &manifest_name)
12403 .await
12404 .expect("manifest fixture should be readable");
12405 let journal_data = com::read_config(store.clone(), &journal_name)
12406 .await
12407 .expect("journal fixture should be readable");
12408
12409 for pool in &store.pools {
12412 com::save_config(pool.clone(), &manifest_name, manifest_data.clone())
12413 .await
12414 .expect("manifest copy should persist in each pool");
12415 com::save_config(pool.clone(), &journal_name, journal_data.clone())
12416 .await
12417 .expect("journal copy should persist in each pool");
12418 }
12419 mark_test_pool_decommissioning(&store, 0).await;
12424
12425 let missing_receipt_error = recover_test_tier_delete_dispatch_manifest(store.clone(), &manifest_name)
12426 .await
12427 .expect_err("an active decommission without receipt coverage must block v6 checkpoint mutation")
12428 .to_string();
12429 assert!(
12430 missing_receipt_error.contains("missing receipt coverage"),
12431 "unexpected missing-receipt error: {missing_receipt_error}"
12432 );
12433 assert_eq!(
12434 com::read_config(store.pools[1].clone(), &manifest_name)
12435 .await
12436 .expect("blocked checkpoint must retain the target manifest generation"),
12437 manifest_data,
12438 "missing receipt coverage must fail before mutating a target copy"
12439 );
12440
12441 for (path, data) in [
12442 (&manifest_name, manifest_data.as_slice()),
12443 (&journal_name, journal_data.as_slice()),
12444 ] {
12445 let record = validate_durable_ilm_record(path, data).expect("v6 durable record should validate");
12446 store
12447 .persist_decommission_durable_ilm_receipt_for_test(0, 1, path, &record, false)
12448 .await
12449 .expect("initial decommission receipt should persist");
12450 }
12451
12452 let (aborting_data, preparing_etag) = test_tier_delete_dispatch_manifest_checkpoint(
12457 store.clone(),
12458 &manifest_name,
12459 TierDeleteDispatchManifestState::Aborting,
12460 )
12461 .await
12462 .expect("the Aborting checkpoint should encode");
12463 let checkpoint_targets = store
12464 .decommission_durable_ilm_checkpoint_targets(&manifest_name, &aborting_data, &preparing_etag)
12465 .await
12466 .expect("receipt-bearing checkpoint targets should resolve")
12467 .expect("an active decommission should own the checkpoint");
12468 assert_eq!(checkpoint_targets.len(), 1);
12469 let checkpoint_target = checkpoint_targets
12470 .into_iter()
12471 .next()
12472 .expect("one checkpoint target should exist");
12473 let checkpoint_owner = checkpoint_target.capacity_owner;
12474 let target_pool_index = checkpoint_target.target_pool_index;
12475 let injected_write_error = store
12476 .run_decommission_capacity_temporary_mutation_with_capacity_lease(
12477 target_pool_index,
12478 Some(checkpoint_owner),
12479 Some(aborting_data.len()),
12480 |_| {
12481 let target_pool = store.pools[target_pool_index].clone();
12482 let manifest_name = manifest_name.clone();
12483 let aborting_data = aborting_data.clone();
12484 async move {
12485 com::save_config(target_pool, &manifest_name, aborting_data)
12486 .await
12487 .expect("the injected target checkpoint should commit");
12488 Err::<(), Error>(Error::other("injected crash after checkpoint target commit"))
12489 }
12490 },
12491 )
12492 .await
12493 .expect_err("the injected post-commit crash should leave a pending capacity intent")
12494 .to_string();
12495 assert!(injected_write_error.contains("injected crash after checkpoint target commit"));
12496 assert!(
12497 store
12498 .has_decommission_capacity_temporary_mutation_state(target_pool_index, checkpoint_owner)
12499 .await,
12500 "the committed target must retain its exact unresolved capacity intent"
12501 );
12502
12503 recover_test_tier_delete_dispatch_manifest(store.clone(), &manifest_name)
12504 .await
12505 .expect("Preparing rollback should advance receipts and finish terminal cleanup");
12506 assert!(
12507 !store
12508 .has_decommission_capacity_temporary_mutation_state(target_pool_index, checkpoint_owner)
12509 .await,
12510 "already-committed checkpoint recovery must resolve the pending capacity intent"
12511 );
12512 let stats = recover_tier_delete_dispatch_manifests(store.clone(), 100, None)
12513 .await
12514 .expect("the terminal source checkpoint should remain non-actionable");
12515 assert_eq!((stats.scanned, stats.deleted, stats.retained, stats.failed), (1, 0, 1, 0));
12516 assert_eq!(
12517 store
12518 .decommission_durable_ilm_receipt_count_for_test(0)
12519 .await
12520 .expect("v6 terminal receipts should be listable"),
12521 2
12522 );
12523
12524 {
12525 let _proof_guard = crate::services::notification_sys::without_cross_pool_fence_fleet_proof_for_test();
12526 let proof_error = store
12527 .verify_and_cleanup_decommissioned_durable_ilm_record_for_test(
12528 0,
12529 store.pools[0].get_disks_by_key(&manifest_name),
12530 &manifest_name,
12531 )
12532 .await
12533 .expect_err("revoked v6 fleet proof must block final source cleanup")
12534 .to_string();
12535 assert!(
12536 proof_error.contains("fleet capability is unavailable"),
12537 "unexpected revoked-proof cleanup error: {proof_error}"
12538 );
12539 assert!(
12540 com::read_config(store.pools[0].clone(), &manifest_name).await.is_ok(),
12541 "revoked fleet proof must retain the last source manifest copy"
12542 );
12543 }
12544
12545 for path in [&manifest_name, &journal_name] {
12546 store
12547 .verify_and_cleanup_decommissioned_durable_ilm_record_for_test(0, store.pools[0].get_disks_by_key(path), path)
12548 .await
12549 .expect("terminal receipt should authorize verified source cleanup");
12550 }
12551 let stats = recover_tier_delete_dispatch_manifests(store.clone(), 100, None)
12552 .await
12553 .expect("verified source cleanup should remove the last manifest copy");
12554 assert_eq!((stats.scanned, stats.deleted, stats.failed), (0, 0, 0));
12555 for path in [&manifest_name, &journal_name] {
12556 for pool in &store.pools {
12557 assert!(
12558 matches!(com::read_config(pool.clone(), path).await, Err(Error::ConfigNotFound)),
12559 "terminal receipt cleanup must remove `{path}` from every staged pool"
12560 );
12561 }
12562 }
12563 }
12564
12565 #[cfg(feature = "test-util")]
12566 #[tokio::test]
12567 #[serial_test::serial(storage_class_env)]
12568 async fn authorized_all_committed_manifest_recovers_after_last_commit_crash() {
12569 let temp_dir = tempfile::tempdir().expect("create authorized manifest recovery store dir");
12570 let (ctx, store, _shutdown) =
12571 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "authorized-manifest-last-commit", &[4])).await;
12572 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
12573 let bucket = "authorized-manifest-last-commit-bucket";
12574 store
12575 .make_bucket(bucket, &MakeBucketOptions::default())
12576 .await
12577 .expect("authorized recovery bucket should be created");
12578 let incarnation = store
12579 .bucket_incarnation_id(bucket)
12580 .await
12581 .expect("bucket incarnation should resolve");
12582 let tier_name = "AUTHORIZED-LAST-COMMIT";
12583 let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
12584 let lease = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
12585 .await
12586 .expect("tier lease should resolve");
12587 let mut entries = Vec::new();
12588 for index in 0..2 {
12589 let remote_version = uuid::Uuid::new_v4().to_string();
12590 let entry = synthetic_v6_dispatch_entry(
12591 bucket,
12592 &format!("archive/{index}.bin"),
12593 tier_name,
12594 lease.backend_identity(),
12595 &remote_version,
12596 );
12597 backend.set_put_remote_version(Some(remote_version)).await;
12598 lease
12599 .put(&entry.obj_name, ReaderImpl::Body(bytes::Bytes::from_static(b"remote candidate")), 16)
12600 .await
12601 .expect("remote candidate should be seeded");
12602 entries.push((entry, Some(TierDeleteJournalState::Committed)));
12603 }
12604 let (manifest_name, _) = install_test_tier_delete_dispatch_fixture(
12605 store.clone(),
12606 bucket,
12607 incarnation,
12608 "archive/",
12609 entries,
12610 TierDeleteDispatchManifestState::DispatchAuthorized,
12611 )
12612 .await
12613 .expect("last-commit crash fixture should persist");
12614
12615 let manifest_stats = recover_tier_delete_dispatch_manifests(store.clone(), 100, None)
12616 .await
12617 .expect("all-Committed Authorized manifest should advance");
12618 assert_eq!((manifest_stats.scanned, manifest_stats.advanced, manifest_stats.failed), (1, 1, 0));
12619 assert_eq!(
12620 test_tier_delete_dispatch_manifest_state(store.clone(), &manifest_name)
12621 .await
12622 .expect("advanced manifest should remain readable"),
12623 Some(TierDeleteDispatchManifestState::Completed)
12624 );
12625 let journal_stats = recover_tier_delete_journal_entries(store.clone(), 100, None)
12626 .await
12627 .expect("Completed journal set should recover remote candidates");
12628 assert_eq!((journal_stats.scanned, journal_stats.deleted, journal_stats.failed), (2, 2, 0));
12629 assert_eq!(tier_delete_journal_count(store.clone()).await, 0);
12630 assert_eq!(tier_delete_dispatch_manifest_count(store.clone()).await, 1);
12631 let gc_stats = recover_tier_delete_dispatch_manifests(store.clone(), 100, None)
12632 .await
12633 .expect("manifest coordinator should remove the empty Completed manifest");
12634 assert_eq!((gc_stats.scanned, gc_stats.deleted, gc_stats.failed), (1, 1, 0));
12635 assert_eq!(tier_delete_dispatch_manifest_count(store).await, 0);
12636 assert_eq!(backend.object_count().await, 0);
12637 assert_eq!(backend.remove_versions().await.len(), 2);
12638 }
12639
12640 #[cfg(feature = "test-util")]
12641 async fn transition_transaction_record_count(store: Arc<crate::store::ECStore>) -> usize {
12642 store
12643 .list_objects_v2(
12644 RUSTFS_META_BUCKET,
12645 TRANSITION_TRANSACTION_RECORD_PREFIX,
12646 None,
12647 None,
12648 100,
12649 false,
12650 None,
12651 false,
12652 )
12653 .await
12654 .expect("transition transaction records should be listable")
12655 .objects
12656 .len()
12657 }
12658
12659 #[cfg(feature = "test-util")]
12660 async fn only_transition_transaction(store: Arc<crate::store::ECStore>) -> TransitionTransaction {
12661 let records = store
12662 .clone()
12663 .list_objects_v2(
12664 RUSTFS_META_BUCKET,
12665 TRANSITION_TRANSACTION_RECORD_PREFIX,
12666 None,
12667 None,
12668 100,
12669 false,
12670 None,
12671 false,
12672 )
12673 .await
12674 .expect("transition transaction records should be listable")
12675 .objects;
12676 assert_eq!(records.len(), 1, "test fixture should have exactly one transition transaction");
12677 let transaction_id = records[0]
12678 .name
12679 .rsplit('/')
12680 .next()
12681 .and_then(|name| name.strip_suffix(".json"))
12682 .and_then(|name| uuid::Uuid::parse_str(name).ok())
12683 .expect("transition transaction path should end in its UUID");
12684 load_transition_transaction_record(store, transaction_id)
12685 .await
12686 .expect("transition transaction should load")
12687 }
12688
12689 #[cfg(feature = "test-util")]
12690 async fn register_transition_reconcile_test_tier(
12691 handle: &Arc<tokio::sync::RwLock<TierConfigMgr>>,
12692 tier_name: &str,
12693 ) -> MockWarmBackend {
12694 let backend = MockWarmBackend::new();
12695 let mut manager = handle.write().await;
12696 manager.tiers.insert(
12697 tier_name.to_string(),
12698 TierConfig {
12699 version: "v1".to_string(),
12700 tier_type: TierType::Wasabi,
12701 name: tier_name.to_string(),
12702 wasabi: Some(TierWasabi {
12703 name: tier_name.to_string(),
12704 endpoint: "https://s3.wasabisys.com".to_string(),
12705 access_key: "test-access-key".to_string(),
12706 secret_key: "test-secret-key".to_string(),
12707 bucket: "mock-tier".to_string(),
12708 prefix: format!("mock/{}/", uuid::Uuid::new_v4()),
12709 region: "us-east-1".to_string(),
12710 }),
12711 ..Default::default()
12712 },
12713 );
12714 manager
12715 .install_test_driver(tier_name, Box::new(backend.clone()))
12716 .expect("mock fallback tier driver should install");
12717 backend
12718 }
12719
12720 #[cfg(feature = "test-util")]
12721 async fn wait_for_tier_delete_journal_recovery(
12722 store: Arc<crate::store::ECStore>,
12723 backend: &MockWarmBackend,
12724 expected_removes: usize,
12725 ) {
12726 tokio::time::timeout(Duration::from_secs(30), async {
12727 loop {
12728 if backend.remove_versions().await.len() >= expected_removes
12729 && tier_delete_journal_count(store.clone()).await == 0
12730 {
12731 return;
12732 }
12733 tokio::time::sleep(Duration::from_millis(10)).await;
12734 }
12735 })
12736 .await
12737 .expect("tier delete journal recovery should complete");
12738 }
12739
12740 #[cfg(feature = "test-util")]
12741 async fn wait_for_tier_free_version_recovery(
12742 store: Arc<crate::store::ECStore>,
12743 backend: &MockWarmBackend,
12744 expected_removes: usize,
12745 ) {
12746 ExpiryState::resize_workers(1, store.clone()).await;
12747 let mut last_progress = None;
12748 let result = tokio::time::timeout(Duration::from_secs(30), async {
12749 loop {
12750 let stats = recover_tier_free_versions(store.clone(), 100, None, None)
12751 .await
12752 .expect("tier free-version recovery scan should succeed");
12753 let removes = backend.remove_versions().await.len();
12754 let recovered = removes >= expected_removes && stats.enqueued == 0 && stats.failed == 0;
12755 last_progress = Some((removes, stats));
12756 if recovered {
12757 return;
12758 }
12759 tokio::time::sleep(Duration::from_millis(50)).await;
12760 }
12761 })
12762 .await;
12763 if result.is_err() {
12764 let expiry_state = store.ctx.expiry_state();
12765 let workers = expiry_state
12766 .try_read()
12767 .ok()
12768 .map(|state| (state.pending_tasks(), state.active_tasks()));
12769 panic!(
12770 "tier free-version recovery should complete: expected_removes={expected_removes}, \
12771 last_progress={last_progress:?}, workers(pending, active)={workers:?}"
12772 );
12773 }
12774 wait_for_expiry_workers_idle(&store).await;
12775 }
12776
12777 #[cfg(feature = "test-util")]
12782 async fn exact_metadata_absent_during_cleanup(store: &crate::store::ECStore, bucket: &str, object: &str) -> bool {
12783 match store.pools[0]
12784 .get_disks_by_key(object)
12785 .load_file_info_versions_exact(bucket, object)
12786 .await
12787 {
12788 Ok(metadata) => metadata.is_none(),
12789 Err(StorageError::InsufficientReadQuorum(_, _)) => false,
12790 Err(error) => panic!("{object} cleanup metadata should remain readable: {error:?}"),
12791 }
12792 }
12793
12794 #[cfg(feature = "test-util")]
12800 async fn assert_every_disk_holds_only_the_cleanup_owner(
12801 set: &crate::set_disk::SetDisks,
12802 bucket: &str,
12803 object: &str,
12804 remote: &str,
12805 ) {
12806 use crate::disk::DiskAPI as _;
12807
12808 let disk_object = rustfs_utils::path::encode_dir_object(object);
12809 for (index, disk) in set.disk_inventory().await.into_iter().enumerate() {
12810 let disk = disk.unwrap_or_else(|| panic!("disk{index} should be online"));
12811 let raw = disk
12812 .read_xl(bucket, &disk_object, false)
12813 .await
12814 .unwrap_or_else(|err| panic!("disk{index} xl.meta should be readable after the overwrite: {err:?}"));
12815 let versions = rustfs_filemeta::FileMeta::load(&raw.buf)
12816 .and_then(|meta| meta.get_all_file_info_versions(bucket, object, true))
12817 .unwrap_or_else(|err| panic!("disk{index} xl.meta should decode: {err:?}"));
12818 let all: Vec<_> = versions.versions.iter().chain(versions.free_versions.iter()).collect();
12819 let owners = all.iter().filter(|fi| fi.tier_free_version()).count();
12820 let live_sources = all
12821 .iter()
12822 .filter(|fi| !fi.tier_free_version() && fi.transitioned_objname == remote)
12823 .count();
12824 assert_eq!(owners, 1, "disk{index} must hold exactly one cleanup owner after the drained overwrite");
12825 assert_eq!(live_sources, 0, "disk{index} must not retain the pre-overwrite live transitioned source");
12826 }
12827 }
12828
12829 #[cfg(feature = "test-util")]
12830 async fn wait_for_expiry_workers_idle(store: &crate::store::ECStore) {
12831 let expiry_state = store.ctx.expiry_state();
12832 let idle = tokio::time::timeout(Duration::from_secs(30), async {
12833 loop {
12834 let idle = {
12835 let state = expiry_state.read().await;
12836 state.pending_tasks() == 0 && state.active_tasks() == 0
12837 };
12838 if idle {
12839 tokio::time::sleep(Duration::from_millis(10)).await;
12843 let state = expiry_state.read().await;
12844 if state.pending_tasks() == 0 && state.active_tasks() == 0 {
12845 return;
12846 }
12847 } else {
12848 tokio::time::sleep(Duration::from_millis(10)).await;
12849 }
12850 }
12851 })
12852 .await;
12853 if idle.is_err() {
12854 let state = expiry_state.read().await;
12855 panic!(
12856 "lifecycle expiry workers should become idle: pending={} active={}",
12857 state.pending_tasks(),
12858 state.active_tasks()
12859 );
12860 }
12861 }
12862
12863 #[cfg(feature = "test-util")]
12864 async fn run_transitioned_delete_free_version_owner_case(
12865 store_name: &str,
12866 tier_name: &str,
12867 bucket: &str,
12868 object: &str,
12869 restore_before_delete: bool,
12870 causal_enqueue: bool,
12871 delete_with_journal: bool,
12872 ) {
12873 let temp_dir = tempfile::tempdir().expect("create transitioned delete store dir");
12874 let (ctx, store, _shutdown) =
12875 without_storage_class_env(build_isolated_test_store(temp_dir.path(), store_name, &[4])).await;
12876 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
12877
12878 let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
12879 store
12880 .make_bucket(bucket, &MakeBucketOptions::default())
12881 .await
12882 .expect("source bucket should be created");
12883 let mut reader = PutObjReader::from_vec(vec![b'd'; 1024 * 1024]);
12884 let original = store
12885 .put_object(bucket, object, &mut reader, &ObjectOptions::default())
12886 .await
12887 .expect("source object should be written");
12888 store
12889 .transition_object(
12890 bucket,
12891 object,
12892 &ObjectOptions {
12893 transition: TransitionOptions {
12894 status: TRANSITION_PENDING.to_string(),
12895 tier: tier_name.to_string(),
12896 etag: original.etag.clone().expect("source should have an etag"),
12897 ..Default::default()
12898 },
12899 mod_time: original.mod_time,
12900 ..Default::default()
12901 },
12902 )
12903 .await
12904 .expect("source transition should commit");
12905 assert_eq!(backend.object_count().await, 1, "transition should create one remote object");
12906 if restore_before_delete {
12907 store
12908 .clone()
12909 .restore_transitioned_object(
12910 bucket,
12911 object,
12912 &ObjectOptions {
12913 transition: TransitionOptions {
12914 restore_request: s3s::dto::RestoreRequest {
12915 days: Some(1),
12916 ..Default::default()
12917 },
12918 ..Default::default()
12919 },
12920 ..Default::default()
12921 },
12922 )
12923 .await
12924 .expect("restore should complete before transitioned delete");
12925 let restored = store
12926 .get_object_info(bucket, object, &ObjectOptions::default())
12927 .await
12928 .expect("restored transitioned source should remain readable");
12929 assert!(
12930 restored.user_defined.contains_key(rustfs_filemeta::metadata_keys::RESTORE),
12931 "restore completion metadata must be present before the delete regression"
12932 );
12933 }
12934
12935 if causal_enqueue {
12936 ExpiryState::resize_workers(1, store.clone()).await;
12937 }
12938 backend.set_remove_failure(!causal_enqueue);
12939 if delete_with_journal {
12940 store
12941 .delete_object_with_tier_delete_journal(bucket, object, ObjectOptions::default())
12942 .await
12943 .expect("transitioned source journal-wrapper delete should commit");
12944 } else {
12945 store
12946 .delete_object(bucket, object, ObjectOptions::default())
12947 .await
12948 .expect("transitioned source plain object-layer delete should commit");
12949 }
12950
12951 if causal_enqueue {
12952 tokio::time::timeout(Duration::from_secs(30), async {
12953 loop {
12954 let metadata_absent = {
12955 let mut read_opts = ObjectOptions::default();
12957 let _guards = store
12958 .acquire_all_physical_object_read_locks(
12959 "transitioned_delete_cleanup_test",
12960 bucket,
12961 object,
12962 &mut read_opts,
12963 )
12964 .await
12965 .expect("causal free-version cleanup observation should acquire object read locks");
12966 store.pools[0]
12967 .get_disks_by_key(object)
12968 .load_file_info_versions_exact(bucket, object)
12969 .await
12970 .expect("causal free-version cleanup metadata should remain readable")
12971 .is_none()
12972 };
12973 if metadata_absent && backend.remove_versions().await.len() == 1 {
12974 return;
12975 }
12976 tokio::time::sleep(Duration::from_millis(10)).await;
12977 }
12978 })
12979 .await
12980 .expect("committed free-version should be cleaned without a recovery scan");
12981 assert_eq!(backend.object_count().await, 0, "causal cleanup should remove the remote object");
12982 assert_eq!(
12983 tier_delete_journal_count(store.clone()).await,
12984 0,
12985 "ordinary causal cleanup must not create a journal"
12986 );
12987 store
12988 .delete_bucket(bucket, &DeleteBucketOptions::default())
12989 .await
12990 .expect("bucket delete should succeed after causal free-version cleanup");
12991 return;
12992 }
12993
12994 let local_versions = store.pools[0]
12995 .get_disks_by_key(object)
12996 .load_file_info_versions_exact(bucket, object)
12997 .await
12998 .expect("post-delete metadata should remain decodable");
12999 let local_versions = local_versions.expect("ordinary delete must leave a durable free-version cleanup owner");
13000 assert_eq!(
13001 local_versions
13002 .versions
13003 .iter()
13004 .chain(local_versions.free_versions.iter())
13005 .filter(|version| version.tier_free_version())
13006 .count(),
13007 1,
13008 "ordinary delete must leave exactly one hidden free-version owner"
13009 );
13010 assert_eq!(
13011 local_versions
13012 .versions
13013 .iter()
13014 .filter(|version| !version.tier_free_version())
13015 .count(),
13016 0,
13017 "ordinary delete must remove the visible transitioned source"
13018 );
13019 let listed = store
13020 .clone()
13021 .list_object_versions(bucket, "", None, None, None, 100)
13022 .await
13023 .expect("source versions should be listable");
13024 assert!(listed.objects.is_empty(), "source must have no visible versions after delete");
13025
13026 assert_eq!(
13027 tier_delete_journal_count(store.clone()).await,
13028 0,
13029 "ordinary delete must not create a journal"
13030 );
13031
13032 backend.set_remove_failure(false);
13033 ExpiryState::resize_workers(1, store.clone()).await;
13034 let recovered = recover_tier_free_versions(store.clone(), 100, None, None)
13035 .await
13036 .expect("free-version recovery should scan the hidden owner");
13037 assert_eq!((recovered.enqueued, recovered.failed), (1, 0));
13038 tokio::time::timeout(Duration::from_secs(30), async {
13039 loop {
13040 let metadata_absent = {
13041 let mut read_opts = ObjectOptions::default();
13043 let _guards = store
13044 .acquire_all_physical_object_read_locks(
13045 "transitioned_delete_cleanup_test",
13046 bucket,
13047 object,
13048 &mut read_opts,
13049 )
13050 .await
13051 .expect("free-version cleanup observation should acquire object read locks");
13052 store.pools[0]
13053 .get_disks_by_key(object)
13054 .load_file_info_versions_exact(bucket, object)
13055 .await
13056 .expect("free-version cleanup metadata should remain readable")
13057 .is_none()
13058 };
13059 if metadata_absent && backend.remove_versions().await.len() == 1 {
13060 return;
13061 }
13062 tokio::time::sleep(Duration::from_millis(10)).await;
13063 }
13064 })
13065 .await
13066 .expect("free-version recovery should delete the remote and exact local owner");
13067 assert_eq!(backend.object_count().await, 0, "free-version recovery should remove the remote object");
13068 store
13069 .delete_bucket(bucket, &DeleteBucketOptions::default())
13070 .await
13071 .expect("ordinary bucket delete should succeed after the last transitioned object");
13072 }
13073
13074 #[cfg(feature = "test-util")]
13075 #[tokio::test]
13076 #[serial_test::serial(storage_class_env)]
13077 async fn transitioned_delete_uses_free_version_as_cleanup_owner() {
13078 run_transitioned_delete_free_version_owner_case(
13079 "transitioned-delete-journal-owner",
13080 "DELETEOWNER",
13081 "transitioned-delete-journal-owner-bucket",
13082 "transition/archive.bin",
13083 false,
13084 false,
13085 true,
13086 )
13087 .await;
13088 }
13089
13090 #[cfg(feature = "test-util")]
13091 #[tokio::test]
13092 #[serial_test::serial(storage_class_env)]
13093 async fn restored_transitioned_delete_uses_free_version_as_cleanup_owner() {
13094 run_transitioned_delete_free_version_owner_case(
13095 "restored-transitioned-delete-journal-owner",
13096 "RESTOREDELETEOWNER",
13097 "restored-transitioned-delete-journal-owner-bucket",
13098 "transition/archive.bin",
13099 true,
13100 false,
13101 true,
13102 )
13103 .await;
13104 }
13105
13106 #[cfg(feature = "test-util")]
13107 #[tokio::test]
13108 #[serial_test::serial(storage_class_env)]
13109 async fn transitioned_delete_causally_enqueues_free_version() {
13110 run_transitioned_delete_free_version_owner_case(
13111 "transitioned-delete-causal-enqueue",
13112 "DELETE-CAUSAL",
13113 "transitioned-delete-causal-enqueue-bucket",
13114 "transition/archive.bin",
13115 false,
13116 true,
13117 false,
13118 )
13119 .await;
13120 }
13121
13122 #[cfg(feature = "test-util")]
13123 #[tokio::test]
13124 #[serial_test::serial(storage_class_env)]
13125 async fn restored_transitioned_delete_causally_enqueues_free_version() {
13126 run_transitioned_delete_free_version_owner_case(
13127 "restored-transitioned-delete-causal-enqueue",
13128 "RESTORE-DELETE-CAUSAL",
13129 "restored-transitioned-delete-causal-enqueue-bucket",
13130 "transition/archive.bin",
13131 true,
13132 true,
13133 true,
13134 )
13135 .await;
13136 }
13137
13138 #[cfg(feature = "test-util")]
13139 #[tokio::test]
13140 #[serial_test::serial(storage_class_env)]
13141 async fn force_tier_remove_blocks_on_physical_free_version_hidden_by_other_pool() {
13142 let temp_dir = tempfile::tempdir().expect("create tier reference store dir");
13143 let (ctx, store, _shutdown) =
13144 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "tier-free-reference-proof", &[4, 4])).await;
13145 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
13146
13147 let tier_name = "FREE-PROOF";
13148 let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
13149 let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
13150 .await
13151 .expect("test tier identity should resolve")
13152 .backend_identity();
13153 let bucket = "tier-free-reference-proof-bucket";
13154 let object = "archive.bin";
13155 store
13156 .make_bucket(bucket, &MakeBucketOptions::default())
13157 .await
13158 .expect("source bucket should be created");
13159 let mut reader = PutObjReader::from_vec(vec![b'f'; 1024 * 1024]);
13160 let original = store.pools[0]
13161 .put_object(bucket, object, &mut reader, &ObjectOptions::default())
13162 .await
13163 .expect("source object should be written");
13164 store.pools[0]
13165 .transition_object(
13166 bucket,
13167 object,
13168 &ObjectOptions {
13169 transition: TransitionOptions {
13170 status: TRANSITION_PENDING.to_string(),
13171 tier: tier_name.to_string(),
13172 etag: original.etag.clone().expect("source should have an etag"),
13173 ..Default::default()
13174 },
13175 mod_time: original.mod_time,
13176 ..Default::default()
13177 },
13178 )
13179 .await
13180 .expect("source transition should commit");
13181 backend.set_remove_failure(true);
13182 store.pools[0]
13183 .delete_object(
13184 bucket,
13185 object,
13186 ObjectOptions {
13187 tier_delete_journal_api: Some(store.clone()),
13188 ..Default::default()
13189 },
13190 )
13191 .await
13192 .expect("journal-less compatibility delete should retain a free-version");
13193
13194 let exact = store.pools[0]
13195 .get_disks_by_key(object)
13196 .load_file_info_versions_exact(bucket, object)
13197 .await
13198 .expect("free-version metadata should remain decodable")
13199 .expect("free-version metadata should remain on disk");
13200 assert_eq!(
13201 exact
13202 .versions
13203 .iter()
13204 .chain(exact.free_versions.iter())
13205 .filter(|version| version.tier_free_version())
13206 .count(),
13207 1,
13208 "fixture must contain one real serialized hidden free-version"
13209 );
13210
13211 let mut hot_reader = PutObjReader::from_vec(vec![b'h'; 1024 * 1024]);
13212 store.pools[1]
13213 .put_object(
13214 bucket,
13215 object,
13216 &mut hot_reader,
13217 &ObjectOptions {
13218 mod_time: Some(OffsetDateTime::now_utc() + time::Duration::hours(1)),
13219 ..Default::default()
13220 },
13221 )
13222 .await
13223 .expect("second pool hot source should be written");
13224 let merged = store
13225 .get_object_info(bucket, object, &ObjectOptions::default())
13226 .await
13227 .expect("merged lookup should return the second-pool hot source");
13228 assert!(
13229 merged.transitioned_object.status.is_empty() && !merged.transitioned_object.free_version,
13230 "the ordinary second-pool source must hide the first-pool free-version from merged lookup"
13231 );
13232
13233 let err = ensure_no_authoritative_tier_references_for_test(store, tier_name, backend_identity, true)
13234 .await
13235 .expect_err("force tier removal must not bypass a physical free-version cleanup obligation");
13236 assert_eq!(err.code, ERR_TIER_BACKEND_IN_USE.code);
13237 }
13238
13239 #[cfg(feature = "test-util")]
13240 #[tokio::test]
13241 #[serial_test::serial(storage_class_env)]
13242 async fn tier_reference_proof_fails_closed_when_physical_bucket_topology_is_incomplete() {
13243 let temp_dir = tempfile::tempdir().expect("create incomplete tier reference store dir");
13244 let (ctx, store, _shutdown) =
13245 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "tier-reference-incomplete-topology", &[4]))
13246 .await;
13247 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
13248
13249 let tier_name = "INCOMPLETE-PROOF";
13250 register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
13251 let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
13252 .await
13253 .expect("test tier identity should resolve")
13254 .backend_identity();
13255 store.pools[0].disk_set[0].disks.write().await[0] = None;
13256
13257 let err = ensure_no_authoritative_tier_references_for_test(store, tier_name, backend_identity, true)
13258 .await
13259 .expect_err("an incomplete physical bucket enumeration must block tier mutation");
13260 assert_eq!(err.code, ERR_TIER_INVALID_CONFIG.code);
13261 assert!(
13262 err.message.contains("complete set quorum"),
13263 "unexpected reference proof error: {}",
13264 err.message
13265 );
13266 }
13267
13268 #[cfg(feature = "test-util")]
13269 async fn copy_test_xlmeta_between_pools(
13270 root: &std::path::Path,
13271 source_pool: usize,
13272 target_pool: usize,
13273 bucket: &str,
13274 object: &str,
13275 ) {
13276 for disk_index in 0..4 {
13277 let source = root.join(format!("pool{source_pool}/set0/disk{disk_index}/{bucket}/{object}/{STORAGE_FORMAT_FILE}"));
13278 let target = root.join(format!("pool{target_pool}/set0/disk{disk_index}/{bucket}/{object}/{STORAGE_FORMAT_FILE}"));
13279 tokio::fs::create_dir_all(target.parent().expect("target xl.meta should have a parent"))
13280 .await
13281 .expect("target object directory should be created");
13282 tokio::fs::copy(&source, &target)
13283 .await
13284 .expect("xl.meta should copy exactly between pools");
13285 }
13286 }
13287
13288 #[cfg(feature = "test-util")]
13289 async fn rewrite_transitioned_xlmeta_as_legacy_unknown(
13290 root: &std::path::Path,
13291 pool_index: usize,
13292 bucket: &str,
13293 object: &str,
13294 minio_unversioned: bool,
13295 ) {
13296 for disk_index in 0..4 {
13297 let metadata_path =
13298 root.join(format!("pool{pool_index}/set0/disk{disk_index}/{bucket}/{object}/{STORAGE_FORMAT_FILE}"));
13299 let encoded = tokio::fs::read(&metadata_path)
13300 .await
13301 .expect("transitioned xl.meta should be readable");
13302 let mut metadata = FileMeta::load(&encoded).expect("transitioned xl.meta should decode");
13303 let mut found = false;
13304 for shallow in &mut metadata.versions {
13305 let mut version = shallow
13306 .parse_version_meta()
13307 .expect("transitioned version record should decode");
13308 let Some(object_meta) = version.object.as_mut() else {
13309 continue;
13310 };
13311 if rustfs_utils::http::metadata_compat::get_bytes(
13312 &object_meta.meta_sys,
13313 rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_STATUS,
13314 )
13315 .as_deref()
13316 != Some(rustfs_filemeta::TRANSITION_COMPLETE.as_bytes())
13317 {
13318 continue;
13319 }
13320 found = true;
13321 for suffix in [
13322 rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
13323 rustfs_utils::http::metadata_compat::SUFFIX_TRANSITIONED_VERSION_STATE,
13324 rustfs_utils::http::metadata_compat::SUFFIX_TRANSITIONED_VERSION_ID,
13325 ] {
13326 rustfs_utils::http::metadata_compat::remove_bytes(&mut object_meta.meta_sys, suffix);
13327 }
13328 if minio_unversioned {
13329 object_meta
13330 .meta_sys
13331 .insert("x-minio-internal-transitioned-versionID".to_string(), Vec::new());
13332 }
13333 *shallow = rustfs_filemeta::FileMetaShallowVersion::try_from(version)
13334 .expect("legacy transitioned version should re-encode");
13335 }
13336 assert!(found, "transitioned source should exist in serialized xl.meta");
13337 tokio::fs::write(&metadata_path, metadata.marshal_msg().expect("legacy transitioned xl.meta should encode"))
13338 .await
13339 .expect("legacy transitioned xl.meta should persist");
13340 }
13341 }
13342
13343 #[cfg(feature = "test-util")]
13344 async fn read_store_body(
13345 store: &Arc<crate::store::ECStore>,
13346 bucket: &str,
13347 object: &str,
13348 range: Option<HTTPRangeSpec>,
13349 opts: &ObjectOptions,
13350 ) -> Vec<u8> {
13351 let mut reader = store
13352 .get_object_reader(bucket, object, range, HeaderMap::new(), opts)
13353 .await
13354 .expect("object reader should open");
13355 let mut body = Vec::new();
13356 reader.stream.read_to_end(&mut body).await.expect("object body should drain");
13357 body
13358 }
13359
13360 #[cfg(feature = "test-util")]
13361 #[test]
13362 #[serial_test::serial(storage_class_env)]
13363 fn legacy_transition_state_inspection_and_apply_keep_all_disk_copies_unchanged() {
13364 run_large_stack_async_test("legacy-state-reconcile-inspection", || {
13365 legacy_transition_state_inspection_and_apply_case(false)
13366 });
13367 }
13368
13369 #[cfg(feature = "test-util")]
13370 #[cfg(not(windows))]
13371 #[test]
13372 #[serial_test::serial(storage_class_env)]
13373 fn legacy_transition_state_backfill_retries_partial_commits_and_preserves_other_bytes() {
13374 run_large_stack_async_test("legacy-state-reconcile-backfill", || {
13375 legacy_transition_state_inspection_and_apply_case(true)
13376 });
13377 }
13378
13379 #[cfg(feature = "test-util")]
13380 async fn legacy_transition_state_inspection_and_apply_case(write_enabled: bool) {
13381 #[cfg(windows)]
13382 assert!(!write_enabled, "Windows supports inspection but cannot prove repair directory durability");
13383 use crate::bucket::lifecycle::legacy_transition_state_reconcile::{
13384 LegacyTransitionStateReconcileOutcome as Outcome, LegacyTransitionStateReconcileRequest,
13385 LegacyTransitionStateReconcileSelector,
13386 };
13387 for (remote_version, expected_state) in [
13388 ("", rustfs_filemeta::TransitionVersionState::KnownDisabled),
13389 ("null", rustfs_filemeta::TransitionVersionState::SuspendedNull),
13390 ("opaque-version", rustfs_filemeta::TransitionVersionState::Exact),
13391 ] {
13392 let temp_dir = tempfile::tempdir().expect("legacy reconcile store directory");
13393 let (ctx, store, _shutdown) =
13394 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "legacy-state-reconcile-inspect", &[4]))
13395 .await;
13396 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
13397 let tier_name = "LEGACY-RECONCILE";
13398 let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
13399 backend.set_put_remote_version(Some(remote_version.to_string())).await;
13400 let bucket = "legacy-state-reconcile-bucket";
13401 let object = "archive.bin";
13402 store
13403 .make_bucket(bucket, &MakeBucketOptions::default())
13404 .await
13405 .expect("create legacy fixture bucket");
13406 let mut reader = PutObjReader::from_vec(b"legacy reconcile body".repeat(1024));
13407 let source = store
13408 .put_object(bucket, object, &mut reader, &ObjectOptions::default())
13409 .await
13410 .expect("write source");
13411 {
13412 let _proof = crate::services::notification_sys::install_current_remote_version_state_fleet_proof_for_test();
13415 temp_env::async_with_vars(
13416 [
13417 (rustfs_config::ENV_TIER_REMOTE_VERSION_STATE_WRITE, Some("true")),
13418 (rustfs_config::ENV_TIER_REMOTE_VERSION_STATE_FLEET_CONFIRMED, Some("true")),
13419 ],
13420 store.transition_object(
13421 bucket,
13422 object,
13423 &ObjectOptions {
13424 transition: TransitionOptions {
13425 status: TRANSITION_PENDING.to_string(),
13426 tier: tier_name.to_string(),
13427 etag: source.etag.clone().expect("source ETag"),
13428 ..Default::default()
13429 },
13430 mod_time: source.mod_time,
13431 ..Default::default()
13432 },
13433 ),
13434 )
13435 .await
13436 .expect("transition source");
13437 }
13438 assert!(
13439 crate::services::notification_sys::acquire_legacy_transition_state_reconcile_fleet_proof()
13440 .await
13441 .is_none(),
13442 "fixture setup must not grant the missing reconciliation write capability"
13443 );
13444 let selector = LegacyTransitionStateReconcileSelector {
13445 bucket: bucket.to_string(),
13446 object: object.to_string(),
13447 version_id: "null".to_string(),
13448 };
13449 if expected_state == rustfs_filemeta::TransitionVersionState::Exact {
13450 backend
13451 .set_transition_candidate_probe_override(Some(
13452 crate::services::tier::warm_backend::TransitionCandidateProbe::Ambiguous,
13453 ))
13454 .await;
13455 }
13456 let converged = store
13457 .inspect_legacy_transition_state(selector.clone())
13458 .await
13459 .expect("inspect an already explicit transition");
13460 assert_eq!(converged.outcome, Outcome::Migrated, "{converged:?}");
13461 assert!(!converged.changed);
13462 backend.set_transition_candidate_probe_override(None).await;
13463 rewrite_transitioned_xlmeta_as_legacy_unknown(temp_dir.path(), 0, bucket, object, remote_version.is_empty()).await;
13464 let paths = (0..4)
13465 .map(|disk| {
13466 temp_dir
13467 .path()
13468 .join(format!("pool0/set0/disk{disk}/{bucket}/{object}/{STORAGE_FORMAT_FILE}"))
13469 })
13470 .collect::<Vec<_>>();
13471 let mut original = Vec::new();
13472 for path in &paths {
13473 original.push(tokio::fs::read(path).await.expect("original xl.meta"));
13474 }
13475 backend.clear_op_log().await;
13476 let inspection = store.inspect_legacy_transition_state(selector.clone());
13477 assert!(
13478 std::mem::size_of_val(&inspection) <= 4 * 1024,
13479 "admin inspection future must remain stack-bounded"
13480 );
13481 let inspected = inspection.await.expect("inspect legacy state");
13482 assert_eq!(inspected.outcome, Outcome::ReadyToMigrate, "{inspected:?}");
13483 assert!(!inspected.readiness.post_ready, "current fleet cannot authorize conditional writes");
13484 let target = inspected.target.expect("live probe should establish one model");
13485 assert_eq!(target.state, expected_state);
13486 let request = LegacyTransitionStateReconcileRequest {
13487 confirm: true,
13488 selector,
13489 source: inspected.source.expect("immutable source"),
13490 original_sets: inspected.original_sets,
13491 target,
13492 reconciliation_digest: inspected.reconciliation_digest.expect("expected tuple digest"),
13493 };
13494 #[cfg(not(windows))]
13495 if write_enabled {
13496 crate::services::notification_sys::with_legacy_transition_state_fleet_proof_for_test(async {
13497 crate::disk::local::bucket_durability::set(bucket, Some(crate::disk::local::DurabilityMode::None));
13498 let unsynced = store.reconcile_legacy_transition_state(request.clone()).await;
13499 crate::disk::local::bucket_durability::set(bucket, None);
13500 let unsynced = unsynced.expect("repair without metadata durability");
13501 assert_eq!(unsynced.outcome, Outcome::BackendUnavailable);
13502 assert!(!unsynced.changed);
13503 let rollback = paths[0].parent().expect("object directory").join(Uuid::new_v4().to_string());
13504 tokio::fs::create_dir(&rollback).await.expect("pending rollback directory");
13505 tokio::fs::write(rollback.join(crate::disk::STORAGE_FORMAT_FILE_BACKUP), &original[0])
13506 .await
13507 .expect("pending old metadata backup");
13508 let unsettled = store.reconcile_legacy_transition_state(request.clone()).await;
13509 tokio::fs::remove_dir_all(&rollback).await.expect("settle fixture rollback");
13510 let unsettled = unsettled.expect("repair must wait for rollback");
13511 assert_eq!(unsettled.outcome, Outcome::BackendUnavailable);
13512 assert!(!unsettled.changed);
13513 for (path, bytes) in paths.iter().zip(&original) {
13514 assert_eq!(tokio::fs::read(path).await.expect("blocked repair leaves original bytes"), *bytes);
13515 }
13516 let disks = store.all_set_disks()[0].disk_inventory().await;
13519 let first_disk = disks[0].as_ref().expect("first physical disk");
13520 let publication_path = first_disk
13521 .get_object_path_for_io_if_local(bucket, &format!("{object}/{STORAGE_FORMAT_FILE}"))
13522 .expect("local disk")
13523 .expect("publication path");
13524 let crash_key = format!("{object}/{STORAGE_FORMAT_FILE}");
13525 let _hook = crate::disk::os::prepared_publication_test_hooks::install_at(
13526 crate::disk::os::prepared_publication_test_hooks::Stage::Rename,
13527 &publication_path,
13528 move || {
13529 crate::crash_inject::arm(crate::crash_inject::CrashPoint::MetaWriteAfterTmpBeforeRename, &crash_key);
13530 },
13531 );
13532 let partial = store
13533 .reconcile_legacy_transition_state(request.clone())
13534 .await
13535 .expect("partial repair response");
13536 assert_eq!(partial.outcome, Outcome::BackendUnavailable, "{partial:?}");
13537 assert!(partial.changed, "first copy was committed: {partial:?}");
13538 assert!(partial.changes_indeterminate);
13539 assert_ne!(tokio::fs::read(&paths[0]).await.expect("first committed copy"), original[0]);
13540 for (path, bytes) in paths[1..].iter().zip(&original[1..]) {
13541 assert_eq!(tokio::fs::read(path).await.expect("uncommitted copy"), *bytes);
13542 }
13543 assert!(
13544 store.all_set_disks()[0]
13545 .load_file_info_versions_exact(bucket, object)
13546 .await
13547 .is_err(),
13548 "cleanup cannot select a partial repair subset"
13549 );
13550 let repaired = store
13551 .reconcile_legacy_transition_state(request.clone())
13552 .await
13553 .expect("retry original snapshot");
13554 assert_eq!(repaired.outcome, Outcome::Migrated, "{repaired:?}");
13555 assert!(repaired.changed);
13556 assert!(!repaired.changes_indeterminate);
13557 let mut committed = Vec::new();
13558 for (path, original) in paths.iter().zip(&original) {
13559 let raw = tokio::fs::read(path).await.expect("repaired copy");
13560 let metadata = FileMeta::load(&raw).expect("decode repaired copy");
13561 let previous = FileMeta::load(original).expect("decode original copy");
13562 assert_eq!(
13563 metadata.transition_reconcile_generation(None).unwrap(),
13564 previous.transition_reconcile_generation(None).unwrap()
13565 );
13566 let (_, version) = metadata.find_version(None).expect("selected version");
13567 let info = version.into_fileinfo(bucket, object, true).expect("repaired FileInfo");
13568 assert_eq!(info.transition_version_state, expected_state);
13569 assert_eq!(info.transition_version, request.target.remote_version);
13570 committed.push(raw);
13571 }
13572 assert!(
13573 store.all_set_disks()[0]
13574 .load_file_info_versions_exact(bucket, object)
13575 .await
13576 .expect("converged cleanup snapshot")
13577 .is_some()
13578 );
13579 let replay = store
13580 .reconcile_legacy_transition_state(request.clone())
13581 .await
13582 .expect("idempotent original request replay");
13583 assert_eq!(replay.outcome, Outcome::Migrated, "{replay:?}");
13584 assert!(!replay.changed);
13585 for (path, expected) in paths.iter().zip(&committed) {
13586 assert_eq!(
13587 tokio::fs::read(path).await.expect("replayed copy"),
13588 *expected,
13589 "idempotence preserves raw encoding"
13590 );
13591 }
13592 for (path, bytes) in paths.iter().zip(&original) {
13593 tokio::fs::write(path, bytes)
13594 .await
13595 .expect("reset independent cancellation fixture");
13596 }
13597 let (entered_tx, entered) = tokio::sync::oneshot::channel();
13598 let (release, released) = std::sync::mpsc::channel::<()>();
13599 let _pause = crate::disk::os::prepared_publication_test_hooks::install_at(
13600 crate::disk::os::prepared_publication_test_hooks::Stage::Rename,
13601 &publication_path,
13602 move || {
13603 let _ = entered_tx.send(());
13604 let _ = released.recv();
13605 },
13606 );
13607 let mut repair = Box::pin(store.reconcile_legacy_transition_state(request.clone()));
13608 tokio::select! {
13609 result = &mut repair => panic!("repair completed before publication pause: {result:?}"),
13610 result = entered => result.expect("publication executor entered"),
13611 }
13612 let update_options = crate::disk::UpdateMetadataOpts::default();
13613 let mut update = Box::pin(first_disk.update_metadata(
13614 bucket,
13615 object,
13616 FileInfo {
13617 metadata: HashMap::from([("x-amz-meta-concurrent".to_string(), "kept".to_string())]),
13618 ..Default::default()
13619 },
13620 &update_options,
13621 ));
13622 assert!(
13623 tokio::time::timeout(std::time::Duration::from_millis(25), update.as_mut())
13624 .await
13625 .is_err(),
13626 "another metadata RMW must wait for publication"
13627 );
13628 drop(repair);
13629 assert!(
13630 tokio::time::timeout(std::time::Duration::from_millis(25), update.as_mut())
13631 .await
13632 .is_err(),
13633 "cancelling the coordinator must not release an in-flight disk mutation"
13634 );
13635 release.send(()).expect("resume owned publication");
13636 update.await.expect("serialized metadata update");
13637 let raw = tokio::fs::read(&paths[0])
13638 .await
13639 .expect("cancelled repair and later metadata update");
13640 let (_, version) = FileMeta::load(&raw)
13641 .expect("metadata after cancellation")
13642 .find_version(None)
13643 .expect("selected version");
13644 let info = version
13645 .into_fileinfo(bucket, object, true)
13646 .expect("metadata after serialized update");
13647 assert_eq!(info.transition_version_state, expected_state);
13648 assert_eq!(info.metadata.get("x-amz-meta-concurrent").map(String::as_str), Some("kept"));
13649 let stale = store
13650 .reconcile_legacy_transition_state(request.clone())
13651 .await
13652 .expect("stale original request");
13653 assert_eq!(
13654 stale.outcome,
13655 Outcome::Corrupt,
13656 "unrelated metadata change invalidates the original generation: {stale:?}"
13657 );
13658 assert!(!stale.changed);
13659 assert_eq!(tokio::fs::read(&paths[0]).await.expect("stale write leaves bytes unchanged"), raw);
13660 assert_eq!(backend.remove_count().await, 0);
13661 let operations = backend.op_log().await;
13664 assert!(
13665 operations.iter().all(|operation| match operation {
13666 MockWarmOp::Probe { object } | MockWarmOp::Get { object } => object == &request.source.remote_object,
13667 _ => false,
13668 }),
13669 "unexpected backend effects: {operations:?}"
13670 );
13671 })
13672 .await;
13673 continue;
13674 }
13675 let mut tampered = request.clone();
13676 tampered.source.remote_object.push_str("-other");
13677 let probes_before = backend.op_log().await.len();
13678 let rejected = store
13679 .reconcile_legacy_transition_state(tampered)
13680 .await
13681 .expect("reject tampered tuple");
13682 assert_eq!(rejected.outcome, Outcome::Corrupt);
13683 assert_eq!(backend.op_log().await.len(), probes_before, "invalid digest must not probe the backend");
13684 let applied = store
13685 .reconcile_legacy_transition_state(request)
13686 .await
13687 .expect("apply must report unavailable write authority");
13688 assert_eq!(applied.outcome, Outcome::BackendUnavailable, "{applied:?}");
13689 assert_eq!(applied.reason_code, "write_fence_unavailable");
13690 assert!(!applied.changed);
13691 for (path, expected) in paths.iter().zip(&original) {
13692 assert_eq!(tokio::fs::read(path).await.expect("xl.meta after inspection"), *expected);
13693 }
13694 assert_eq!(backend.remove_count().await, 0);
13695 assert!(
13696 backend
13697 .op_log()
13698 .await
13699 .iter()
13700 .all(|operation| matches!(operation, MockWarmOp::Probe { .. }))
13701 );
13702
13703 backend.set_unreachable(true).await;
13704 let unavailable = store
13705 .inspect_legacy_transition_state(LegacyTransitionStateReconcileSelector {
13706 bucket: bucket.to_string(),
13707 object: object.to_string(),
13708 version_id: "null".to_string(),
13709 })
13710 .await
13711 .expect("unreachable tier is a diagnostic outcome");
13712 assert_eq!(unavailable.outcome, Outcome::BackendUnavailable);
13713 assert!(!unavailable.changed);
13714 backend.set_unreachable(false).await;
13715 for candidate in ["", "00000000-0000-0000-0000-000000000000", "bad\nversion"] {
13716 backend
13717 .set_transition_candidate_probe_override(Some(
13718 crate::services::tier::warm_backend::TransitionCandidateProbe::VersionedPresent(candidate.to_string()),
13719 ))
13720 .await;
13721 let invalid_proof = store
13722 .inspect_legacy_transition_state(LegacyTransitionStateReconcileSelector {
13723 bucket: bucket.to_string(),
13724 object: object.to_string(),
13725 version_id: "null".to_string(),
13726 })
13727 .await
13728 .expect("invalid backend proof is a diagnostic outcome");
13729 assert_eq!(invalid_proof.outcome, Outcome::BackendUnavailable, "{invalid_proof:?}");
13730 assert!(invalid_proof.target.is_none());
13731 }
13732 backend.set_transition_candidate_probe_override(None).await;
13733
13734 backend.clear_op_log().await;
13735 for path in &paths[1..] {
13736 tokio::fs::remove_file(path)
13737 .await
13738 .expect("hide majority metadata copies in fixture");
13739 }
13740 let minority = store
13741 .inspect_legacy_transition_state(LegacyTransitionStateReconcileSelector {
13742 bucket: bucket.to_string(),
13743 object: object.to_string(),
13744 version_id: "null".to_string(),
13745 })
13746 .await
13747 .expect("inspect minority legacy record");
13748 assert_eq!(
13749 minority.outcome,
13750 Outcome::BackendUnavailable,
13751 "a minority owner must remain visible: {minority:?}"
13752 );
13753 assert!(
13754 backend.op_log().await.is_empty(),
13755 "unproven metadata quorum cannot initiate a remote probe"
13756 );
13757 for (path, bytes) in paths.iter().zip(&original) {
13758 tokio::fs::write(path, bytes).await.expect("restore fixture copies");
13759 }
13760 tokio::fs::write(&paths[0], b"corrupt-xl-meta")
13761 .await
13762 .expect("inject corrupt copy");
13763 let corrupt = store
13764 .inspect_legacy_transition_state(LegacyTransitionStateReconcileSelector {
13765 bucket: bucket.to_string(),
13766 object: object.to_string(),
13767 version_id: "null".to_string(),
13768 })
13769 .await
13770 .expect("inspect corrupt legacy record");
13771 assert_eq!(corrupt.outcome, Outcome::Corrupt, "{corrupt:?}");
13772 assert!(backend.op_log().await.is_empty(), "corruption must fail before backend I/O");
13773 }
13774 }
13775
13776 #[cfg(feature = "test-util")]
13777 #[tokio::test]
13778 #[serial_test::serial(storage_class_env)]
13779 async fn legacy_unknown_unversioned_transition_supports_head_get_and_range_without_backfill() {
13780 let temp_dir = tempfile::tempdir().expect("create legacy unknown unversioned store dir");
13781 let (ctx, store, _shutdown) =
13782 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "legacy-unknown-unversioned-read", &[4])).await;
13783 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
13784 let tier_name = "LEGACY-UNKNOWN-UNVERSIONED-READ";
13785 let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
13786 backend.set_put_remote_version(Some(String::new())).await;
13787 let bucket = "legacy-unknown-unversioned-read-bucket";
13788 let object = "object.bin";
13789 let payload = b"legacy unversioned remote tier object remains readable".repeat(1024);
13790 store
13791 .make_bucket(bucket, &MakeBucketOptions::default())
13792 .await
13793 .expect("legacy source bucket should be created");
13794 let mut reader = PutObjReader::from_vec(payload.clone());
13795 let source = store
13796 .put_object(bucket, object, &mut reader, &ObjectOptions::default())
13797 .await
13798 .expect("legacy source should be written");
13799 store
13800 .transition_object(
13801 bucket,
13802 object,
13803 &ObjectOptions {
13804 transition: TransitionOptions {
13805 status: TRANSITION_PENDING.to_string(),
13806 tier: tier_name.to_string(),
13807 etag: source.etag.clone().expect("legacy source should have an etag"),
13808 ..Default::default()
13809 },
13810 mod_time: source.mod_time,
13811 ..Default::default()
13812 },
13813 )
13814 .await
13815 .expect("legacy source should transition");
13816 rewrite_transitioned_xlmeta_as_legacy_unknown(temp_dir.path(), 0, bucket, object, true).await;
13817 backend.clear_op_log().await;
13818
13819 let opts = ObjectOptions {
13820 metadata_cache_safe: false,
13821 ..Default::default()
13822 };
13823 let head = store
13824 .get_object_info(bucket, object, &opts)
13825 .await
13826 .expect("legacy transitioned HEAD should use local metadata");
13827 assert_eq!(head.transition_version_state, rustfs_filemeta::TransitionVersionState::Unknown);
13828 assert!(head.transitioned_object.version_id.is_empty());
13829 assert_eq!(
13830 head.user_defined
13831 .get("x-minio-internal-transitioned-versionID")
13832 .map(String::as_str),
13833 Some(""),
13834 "the MinIO empty version-key provenance must survive xl.meta decoding"
13835 );
13836 assert!(
13837 !rustfs_utils::http::metadata_compat::contains_key_str(
13838 &head.user_defined,
13839 rustfs_utils::http::metadata_compat::SUFFIX_TRANSITIONED_VERSION_STATE,
13840 ),
13841 "the compatibility read must not synthesize version-state metadata"
13842 );
13843
13844 let full_body = read_store_body(&store, bucket, object, None, &opts).await;
13845 assert_eq!(full_body, payload);
13846
13847 let range = HTTPRangeSpec {
13848 is_suffix_length: false,
13849 start: 7,
13850 end: 38,
13851 };
13852 let ranged_body = read_store_body(&store, bucket, object, Some(range), &opts).await;
13853 assert_eq!(ranged_body, &payload[7..=38]);
13854
13855 let after_read = store.pools[0]
13856 .get_disks_by_key(object)
13857 .load_file_info_versions_exact(bucket, object)
13858 .await
13859 .expect("legacy metadata should remain readable after GET")
13860 .expect("legacy object metadata should remain on disk")
13861 .versions
13862 .into_iter()
13863 .find(|version| version.transition_status == rustfs_filemeta::TRANSITION_COMPLETE)
13864 .expect("legacy transitioned source should remain visible after GET");
13865 assert_eq!(after_read.transition_version_state, rustfs_filemeta::TransitionVersionState::Unknown);
13866 assert!(after_read.transition_version.is_none());
13867 assert!(after_read.transition_version_id.is_none());
13868 assert_eq!(
13869 after_read
13870 .metadata
13871 .get("x-minio-internal-transitioned-versionID")
13872 .map(String::as_str),
13873 Some(""),
13874 "the MinIO empty version-key provenance must remain after GET and Range GET"
13875 );
13876 assert!(
13877 !rustfs_utils::http::metadata_compat::contains_key_str(
13878 &after_read.metadata,
13879 rustfs_utils::http::metadata_compat::SUFFIX_TRANSITIONED_VERSION_STATE,
13880 ),
13881 "the compatibility read must remain side-effect free"
13882 );
13883
13884 assert_eq!(
13885 backend.op_log().await,
13886 vec![
13887 MockWarmOp::Probe {
13888 object: after_read.transitioned_objname.clone(),
13889 },
13890 MockWarmOp::Get {
13891 object: after_read.transitioned_objname.clone(),
13892 },
13893 MockWarmOp::Probe {
13894 object: after_read.transitioned_objname.clone(),
13895 },
13896 MockWarmOp::Get {
13897 object: after_read.transitioned_objname,
13898 },
13899 ],
13900 "legacy reads should probe before each unversioned GET and never mutate local metadata"
13901 );
13902 assert_eq!(backend.remove_count().await, 0);
13903 }
13904
13905 #[cfg(feature = "test-util")]
13906 #[tokio::test]
13907 #[serial_test::serial(storage_class_env)]
13908 async fn legacy_unknown_transition_delete_falls_back_for_single_batch_and_blocks_prefix() {
13909 let temp_dir = tempfile::tempdir().expect("create legacy transitioned delete store dir");
13910 let (ctx, store, _shutdown) =
13911 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "legacy-unknown-transitioned-delete", &[4]))
13912 .await;
13913 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
13914 let tier_name = "LEGACY-UNKNOWN-DELETE";
13915 let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
13916 let bucket = "legacy-unknown-transitioned-delete-bucket";
13917 store
13918 .make_bucket(bucket, &MakeBucketOptions::default())
13919 .await
13920 .expect("legacy source bucket should be created");
13921
13922 for (index, object) in ["single.bin", "batch.bin", "prefix/legacy.bin"].into_iter().enumerate() {
13923 let mut reader = PutObjReader::from_vec(vec![b'l' + index as u8; 1024 * 1024]);
13924 let source = store
13925 .put_object(bucket, object, &mut reader, &ObjectOptions::default())
13926 .await
13927 .expect("legacy source should be written");
13928 store
13929 .transition_object(
13930 bucket,
13931 object,
13932 &ObjectOptions {
13933 transition: TransitionOptions {
13934 status: TRANSITION_PENDING.to_string(),
13935 tier: tier_name.to_string(),
13936 etag: source.etag.clone().expect("legacy source should have an etag"),
13937 ..Default::default()
13938 },
13939 mod_time: source.mod_time,
13940 ..Default::default()
13941 },
13942 )
13943 .await
13944 .expect("legacy source should transition");
13945 rewrite_transitioned_xlmeta_as_legacy_unknown(temp_dir.path(), 0, bucket, object, false).await;
13946 let legacy = store.pools[0]
13947 .get_disks_by_key(object)
13948 .load_file_info_versions_exact(bucket, object)
13949 .await
13950 .expect("legacy source metadata should remain decodable")
13951 .expect("legacy source should remain on disk")
13952 .versions
13953 .into_iter()
13954 .find(|version| version.transition_status == rustfs_filemeta::TRANSITION_COMPLETE)
13955 .expect("legacy transitioned source should remain visible before delete");
13956 assert_eq!(legacy.transition_version_state, rustfs_filemeta::TransitionVersionState::Unknown);
13957 assert_eq!(
13958 crate::services::tier::tier::tier_destination_id_from_metadata(&legacy.metadata)
13959 .expect("legacy metadata identity lookup should not fail"),
13960 None
13961 );
13962 }
13963
13964 backend.set_remove_failure(true);
13965 store
13966 .delete_object_with_tier_delete_journal(bucket, "single.bin", ObjectOptions::default())
13967 .await
13968 .expect("legacy single delete should retain its free-version fallback");
13969 let free_version_prefix_error = store
13970 .delete_object_with_tier_delete_journal(
13971 bucket,
13972 "single.bin",
13973 ObjectOptions {
13974 delete_prefix: true,
13975 ..Default::default()
13976 },
13977 )
13978 .await
13979 .expect_err("recursive prefix delete must not discard an existing hidden cleanup owner");
13980 assert!(
13981 free_version_prefix_error
13982 .to_string()
13983 .contains("tier free-version cleanup obligation"),
13984 "unexpected hidden-owner prefix failure: {free_version_prefix_error}"
13985 );
13986 let (_deleted, errors) = store
13987 .delete_objects_with_tier_delete_journal(
13988 bucket,
13989 vec![ObjectToDelete {
13990 object_name: "batch.bin".to_string(),
13991 ..Default::default()
13992 }],
13993 ObjectOptions::default(),
13994 )
13995 .await;
13996 assert!(errors.iter().all(Option::is_none), "legacy batch delete should succeed: {errors:?}");
13997 let prefix_error = store
13998 .delete_object_with_tier_delete_journal(
13999 bucket,
14000 "prefix/",
14001 ObjectOptions {
14002 delete_prefix: true,
14003 ..Default::default()
14004 },
14005 )
14006 .await
14007 .expect_err("legacy recursive prefix delete must fail closed before physical removal");
14008 assert!(
14009 prefix_error.to_string().contains("legacy Unknown"),
14010 "unexpected prefix failure: {prefix_error}"
14011 );
14012
14013 assert_eq!(
14014 tier_delete_journal_count(store.clone()).await,
14015 0,
14016 "legacy cleanup must not create an unsafe journal"
14017 );
14018 assert_eq!(
14019 backend.object_count().await,
14020 3,
14021 "failed free-version cleanup must retain all remote objects"
14022 );
14023 for object in ["single.bin", "batch.bin"] {
14024 let exact = store.pools[0]
14025 .get_disks_by_key(object)
14026 .load_file_info_versions_exact(bucket, object)
14027 .await
14028 .expect("legacy post-delete metadata should decode")
14029 .expect("legacy cleanup owner should remain on disk");
14030 assert_eq!(
14031 exact
14032 .versions
14033 .iter()
14034 .chain(exact.free_versions.iter())
14035 .filter(|version| version.tier_free_version())
14036 .count(),
14037 1,
14038 "legacy {object} must retain exactly one hidden cleanup owner"
14039 );
14040 assert_eq!(
14041 exact.versions.iter().filter(|version| !version.tier_free_version()).count(),
14042 0,
14043 "legacy {object} must have no visible source after delete"
14044 );
14045 }
14046 let retained_prefix = store.pools[0]
14047 .get_disks_by_key("prefix/legacy.bin")
14048 .load_file_info_versions_exact(bucket, "prefix/legacy.bin")
14049 .await
14050 .expect("rejected prefix metadata should decode")
14051 .expect("rejected prefix delete must retain its source");
14052 assert_eq!(
14053 retained_prefix
14054 .versions
14055 .iter()
14056 .filter(|version| !version.tier_free_version())
14057 .count(),
14058 1
14059 );
14060 }
14061
14062 #[cfg(feature = "test-util")]
14063 #[tokio::test]
14064 #[serial_test::serial(storage_class_env)]
14065 async fn stable_transitioned_recursive_prefix_delete_uses_journal_owners() {
14066 let temp_dir = tempfile::tempdir().expect("create stable transitioned prefix-delete store dir");
14067 let (ctx, store, _shutdown) =
14068 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "stable-transitioned-prefix-delete", &[4]))
14069 .await;
14070 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
14071 let tier_name = "STABLE-PREFIX-DELETE";
14072 let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
14073 let bucket = "stable-transitioned-prefix-delete-bucket";
14074 store
14075 .make_bucket(bucket, &MakeBucketOptions::default())
14076 .await
14077 .expect("stable prefix source bucket should be created");
14078
14079 let objects = ["prefix/first.bin", "prefix/second.bin"];
14080 for (index, object) in objects.into_iter().enumerate() {
14081 let mut reader = PutObjReader::from_vec(vec![b'a' + index as u8; 1024 * 1024]);
14082 let source = store
14083 .put_object(bucket, object, &mut reader, &ObjectOptions::default())
14084 .await
14085 .expect("stable prefix source should be written");
14086 store
14087 .transition_object(
14088 bucket,
14089 object,
14090 &ObjectOptions {
14091 transition: TransitionOptions {
14092 status: TRANSITION_PENDING.to_string(),
14093 tier: tier_name.to_string(),
14094 etag: source.etag.clone().expect("stable prefix source should have an etag"),
14095 ..Default::default()
14096 },
14097 mod_time: source.mod_time,
14098 ..Default::default()
14099 },
14100 )
14101 .await
14102 .expect("stable prefix source should transition");
14103 }
14104 assert_eq!(backend.object_count().await, objects.len());
14105 backend.set_remove_failure(true);
14106
14107 store
14108 .delete_object_with_tier_delete_journal(
14109 bucket,
14110 "prefix/",
14111 ObjectOptions {
14112 delete_prefix: true,
14113 ..Default::default()
14114 },
14115 )
14116 .await
14117 .expect("stable recursive prefix delete should use journal ownership");
14118
14119 assert_eq!(tier_delete_journal_count(store.clone()).await, objects.len());
14120 assert_eq!(backend.object_count().await, objects.len());
14121 for object in objects {
14122 assert!(
14123 store.pools[0]
14124 .get_disks_by_key(object)
14125 .load_file_info_versions_exact(bucket, object)
14126 .await
14127 .expect("stable prefix metadata lookup should succeed")
14128 .is_none(),
14129 "stable recursive delete must leave neither a visible source nor a free-version for {object}"
14130 );
14131 }
14132
14133 backend.set_remove_failure(false);
14134 let stats = recover_tier_delete_journal_entries(store.clone(), 100, None)
14135 .await
14136 .expect("stable prefix journals should recover");
14137 assert_eq!(stats.failed, 0);
14138 assert!(stats.scanned <= objects.len() && stats.deleted <= objects.len());
14139 assert_eq!(tier_delete_journal_count(store).await, 0);
14140 assert_eq!(backend.object_count().await, 0);
14141 assert_eq!(backend.remove_versions().await.len(), objects.len());
14142 }
14143
14144 #[cfg(feature = "test-util")]
14145 #[test]
14146 #[serial_test::serial(storage_class_env)]
14147 fn tier_delete_prefix_limit_and_multi_chunk_batches_converge() {
14148 run_large_stack_async_test(
14149 "tier-delete-prefix-limit-and-multi-chunk",
14150 tier_delete_prefix_limit_and_multi_chunk_batches_converge_case,
14151 );
14152 }
14153
14154 #[cfg(feature = "test-util")]
14155 async fn tier_delete_prefix_limit_and_multi_chunk_batches_converge_case() {
14156 let _batch_limit = TierDeleteDispatchBatchLimitGuard::install(2);
14157 let temp_dir = tempfile::tempdir().expect("create chunked prefix-delete store dir");
14158 let (ctx, store, shutdown) =
14159 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "chunked-prefix-delete", &[4])).await;
14160 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
14161 let tier_name = "CHUNKED-PREFIX-DELETE";
14162 let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
14163 let bucket = "chunked-prefix-delete-bucket";
14164 store
14165 .make_bucket(bucket, &MakeBucketOptions::default())
14166 .await
14167 .expect("chunked prefix source bucket should be created");
14168
14169 for (prefix, count) in [("at-limit", 2), ("limit-plus-one", 3), ("multi-chunk", 5)] {
14170 for index in 0..count {
14171 let object = format!("{prefix}/object-{index}.bin");
14172 let mut reader = PutObjReader::from_vec(vec![b'a' + index as u8; 1024 * 1024]);
14173 let source = store
14174 .put_object(bucket, &object, &mut reader, &ObjectOptions::default())
14175 .await
14176 .expect("chunked prefix source should be written");
14177 store
14178 .transition_object(
14179 bucket,
14180 &object,
14181 &ObjectOptions {
14182 transition: TransitionOptions {
14183 status: TRANSITION_PENDING.to_string(),
14184 tier: tier_name.to_string(),
14185 etag: source.etag.clone().expect("chunked prefix source should have an etag"),
14186 ..Default::default()
14187 },
14188 mod_time: source.mod_time,
14189 ..Default::default()
14190 },
14191 )
14192 .await
14193 .expect("chunked prefix source should transition");
14194 }
14195 }
14196 backend.set_remove_failure(true);
14197
14198 store
14199 .delete_object_with_tier_delete_journal(
14200 bucket,
14201 "at-limit/",
14202 ObjectOptions {
14203 delete_prefix: true,
14204 ..Default::default()
14205 },
14206 )
14207 .await
14208 .expect("exactly one batch must retain the v1 one-shot path");
14209
14210 let limit_plus_one_first = store
14211 .delete_object_with_tier_delete_journal(
14212 bucket,
14213 "limit-plus-one/",
14214 ObjectOptions {
14215 delete_prefix: true,
14216 ..Default::default()
14217 },
14218 )
14219 .await
14220 .expect_err("limit plus one must start a bounded parent transaction");
14221 assert!(
14222 limit_plus_one_first.to_string().contains("retry the next durable batch"),
14223 "unexpected first limit-plus-one result: {limit_plus_one_first}"
14224 );
14225 let active_limit_plus_one_records = store
14226 .clone()
14227 .list_objects_v2(
14228 RUSTFS_META_BUCKET,
14229 TIER_DELETE_DISPATCH_MANIFEST_PREFIX,
14230 None,
14231 None,
14232 10,
14233 false,
14234 None,
14235 false,
14236 )
14237 .await
14238 .expect("active limit-plus-one records should be listable");
14239 let mut active_limit_plus_one_parent_seen = false;
14240 for record in active_limit_plus_one_records
14241 .objects
14242 .iter()
14243 .filter(|record| !record.name.contains("/chunks/"))
14244 {
14245 let data = com::read_config(store.clone(), &record.name)
14246 .await
14247 .expect("an active dispatch root should be readable");
14248 let value: serde_json::Value = serde_json::from_slice(&data).expect("an active dispatch root should contain JSON");
14249 if value["prefix"] == "limit-plus-one/" {
14250 assert_eq!(value["record_type"], "chunked_parent");
14251 active_limit_plus_one_parent_seen = true;
14252 }
14253 }
14254 assert!(
14255 active_limit_plus_one_parent_seen,
14256 "limit plus one must establish the fail-closed parent at the legacy root"
14257 );
14258
14259 let mut limit_plus_one_completed = false;
14260 let mut limit_plus_one_retries = 1;
14261 for _ in 0..3 {
14262 match store
14263 .delete_object_with_tier_delete_journal(
14264 bucket,
14265 "limit-plus-one/",
14266 ObjectOptions {
14267 delete_prefix: true,
14268 ..Default::default()
14269 },
14270 )
14271 .await
14272 {
14273 Ok(_) => {
14274 limit_plus_one_completed = true;
14275 break;
14276 }
14277 Err(err) if err.to_string().contains("retry the next durable batch") => {
14278 limit_plus_one_retries += 1;
14279 }
14280 Err(err) => panic!("limit-plus-one delete returned an unexpected error: {err}"),
14281 }
14282 }
14283 assert!(limit_plus_one_completed, "limit plus one must converge through two bounded children");
14284 assert_eq!(limit_plus_one_retries, 2, "limit plus one must require exactly two child batches");
14285
14286 let first_batch = store
14287 .delete_object_with_tier_delete_journal(
14288 bucket,
14289 "multi-chunk/",
14290 ObjectOptions {
14291 delete_prefix: true,
14292 ..Default::default()
14293 },
14294 )
14295 .await
14296 .expect_err("the first bounded child must request a successor batch");
14297 assert!(
14298 first_batch.to_string().contains("retry the next durable batch"),
14299 "unexpected first child result: {first_batch}"
14300 );
14301
14302 let newcomer = "multi-chunk/zzz-newcomer.bin";
14303 let mut newcomer_reader = PutObjReader::from_vec(vec![b'n'; 1024 * 1024]);
14304 let newcomer_source = store
14305 .put_object(bucket, newcomer, &mut newcomer_reader, &ObjectOptions::default())
14306 .await
14307 .expect("a source created between chunks should be written");
14308 store
14309 .transition_object(
14310 bucket,
14311 newcomer,
14312 &ObjectOptions {
14313 transition: TransitionOptions {
14314 status: TRANSITION_PENDING.to_string(),
14315 tier: tier_name.to_string(),
14316 etag: newcomer_source
14317 .etag
14318 .clone()
14319 .expect("the between-chunks source should have an etag"),
14320 ..Default::default()
14321 },
14322 mod_time: newcomer_source.mod_time,
14323 ..Default::default()
14324 },
14325 )
14326 .await
14327 .expect("a source created between chunks should transition");
14328
14329 let mut completed = false;
14330 let mut durable_batch_retries = 1;
14331 for _ in 0..7 {
14332 match store
14333 .delete_object_with_tier_delete_journal(
14334 bucket,
14335 "multi-chunk/",
14336 ObjectOptions {
14337 delete_prefix: true,
14338 ..Default::default()
14339 },
14340 )
14341 .await
14342 {
14343 Ok(_) => {
14344 completed = true;
14345 break;
14346 }
14347 Err(err)
14348 if err.to_string().contains("retry the next durable batch")
14349 || err.to_string().contains("retry the next batch") =>
14350 {
14351 durable_batch_retries += 1;
14352 }
14353 Err(err) => panic!("chunked prefix delete returned an unexpected error: {err}"),
14354 }
14355 }
14356 assert!(completed, "six entries must converge through three bounded child batches");
14357 assert_eq!(durable_batch_retries, 3, "limit two should require exactly three child batches");
14358 assert_eq!(tier_delete_journal_count(store.clone()).await, 11);
14359 assert_eq!(
14360 backend.object_count().await,
14361 11,
14362 "remote cleanup must remain durable while the tier is unavailable"
14363 );
14364 let dispatch_records = store
14365 .clone()
14366 .list_objects_v2(
14367 RUSTFS_META_BUCKET,
14368 TIER_DELETE_DISPATCH_MANIFEST_PREFIX,
14369 None,
14370 None,
14371 20,
14372 false,
14373 None,
14374 false,
14375 )
14376 .await
14377 .expect("bounded dispatch records should be listable");
14378 let mut child_batches = 0;
14379 let mut legacy_at_limit_seen = false;
14380 for record in &dispatch_records.objects {
14381 let data = com::read_config(store.clone(), &record.name)
14382 .await
14383 .expect("a retained dispatch record should be readable");
14384 let value: serde_json::Value = serde_json::from_slice(&data).expect("a retained dispatch record should contain JSON");
14385 if record.name.contains("/chunks/") {
14386 let journal_count = value["journal_count"]
14387 .as_u64()
14388 .expect("a retained child should declare its journal count");
14389 assert!(journal_count <= 2, "a child batch exceeded the configured resource bound");
14390 child_batches += 1;
14391 } else if value["prefix"] == "at-limit/" {
14392 assert!(
14393 value.get("record_type").is_none(),
14394 "the exact-limit root must remain a legacy v1 manifest"
14395 );
14396 assert_eq!(value["journal_count"], 2);
14397 legacy_at_limit_seen = true;
14398 }
14399 }
14400 assert!(
14401 legacy_at_limit_seen,
14402 "the exact-limit dispatch must retain its byte-compatible root shape"
14403 );
14404 assert_eq!(child_batches, 5, "nine chunked sources should persist exactly five bounded children");
14405
14406 backend.set_remove_failure(false);
14407 drive_tier_delete_dispatch_restart_to_convergence(store.clone()).await;
14408 assert_eq!(tier_delete_journal_count(store.clone()).await, 0);
14409 assert_eq!(tier_delete_dispatch_manifest_count(store.clone()).await, 0);
14410 assert_eq!(backend.object_count().await, 0);
14411 assert_eq!(
14412 backend.remove_versions().await.len(),
14413 11,
14414 "each remote version must be removed exactly once"
14415 );
14416 shutdown.cancel();
14417 }
14418
14419 #[cfg(feature = "test-util")]
14420 #[test]
14421 #[serial_test::serial(storage_class_env)]
14422 fn tier_delete_chunk_crash_boundaries_resume_without_skipping_sources() {
14423 run_large_stack_async_test(
14424 "tier-delete-chunk-crash-boundaries",
14425 tier_delete_chunk_crash_boundaries_resume_without_skipping_sources_case,
14426 );
14427 }
14428
14429 #[cfg(feature = "test-util")]
14430 async fn tier_delete_chunk_crash_boundaries_resume_without_skipping_sources_case() {
14431 let _batch_limit = TierDeleteDispatchBatchLimitGuard::install(1);
14432 let stages = [
14433 TierDeleteChunkTestStage::ParentPersisted,
14434 TierDeleteChunkTestStage::ChildManifestPersisted,
14435 TierDeleteChunkTestStage::ParentBound,
14436 TierDeleteChunkTestStage::DispatchAuthorized,
14437 TierDeleteChunkTestStage::LocalReplayCompleted,
14438 TierDeleteChunkTestStage::ChildCompleted,
14439 TierDeleteChunkTestStage::ParentProgressed,
14440 TierDeleteChunkTestStage::FinalLocalDeletionCompleted,
14441 TierDeleteChunkTestStage::ParentCompleted,
14442 ];
14443 for (case, stage) in stages.into_iter().enumerate() {
14444 let temp_dir = tempfile::tempdir().expect("create chunk crash-boundary store dir");
14445 let initial_name = format!("chunk-crash-boundary-{case}");
14446 let (ctx, store, shutdown) =
14447 without_storage_class_env(build_isolated_test_store(temp_dir.path(), &initial_name, &[4])).await;
14448 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
14449 let tier_name = format!("CHUNK-CRASH-BOUNDARY-{case}");
14450 let backend = register_mock_tier(&ctx.tier_config_mgr(), &tier_name).await;
14451 backend.set_remove_failure(true);
14452 let bucket = format!("chunk-crash-boundary-{case}-bucket");
14453 let prefix = "prefix/";
14454 store
14455 .make_bucket(&bucket, &MakeBucketOptions::default())
14456 .await
14457 .expect("chunk crash-boundary bucket should be created");
14458 for index in 0..2 {
14459 let object = format!("{prefix}object-{index}.bin");
14460 let mut reader = PutObjReader::from_vec(vec![b'a' + index as u8; 1024 * 1024]);
14461 let source = store
14462 .put_object(&bucket, &object, &mut reader, &ObjectOptions::default())
14463 .await
14464 .expect("chunk crash-boundary source should be written");
14465 store
14466 .transition_object(
14467 &bucket,
14468 &object,
14469 &ObjectOptions {
14470 transition: TransitionOptions {
14471 status: TRANSITION_PENDING.to_string(),
14472 tier: tier_name.clone(),
14473 etag: source.etag.clone().expect("chunk crash-boundary source should have an etag"),
14474 ..Default::default()
14475 },
14476 mod_time: source.mod_time,
14477 ..Default::default()
14478 },
14479 )
14480 .await
14481 .expect("chunk crash-boundary source should transition");
14482 }
14483 let local_only = format!("{prefix}local-only.bin");
14484 let mut local_reader = PutObjReader::from_vec(vec![b'l'; 1024 * 1024]);
14485 store
14486 .put_object(&bucket, &local_only, &mut local_reader, &ObjectOptions::default())
14487 .await
14488 .expect("chunk crash-boundary local-only object should be written");
14489 let tier_config = ctx
14490 .tier_config_mgr()
14491 .read()
14492 .await
14493 .tiers
14494 .get(&tier_name)
14495 .expect("chunk crash-boundary tier config should remain available for restart")
14496 .clone_with_credentials();
14497
14498 if matches!(
14499 stage,
14500 TierDeleteChunkTestStage::FinalLocalDeletionCompleted | TierDeleteChunkTestStage::ParentCompleted
14501 ) {
14502 for _ in 0..2 {
14503 let retry = store
14504 .delete_object_with_tier_delete_journal(
14505 &bucket,
14506 prefix,
14507 ObjectOptions {
14508 delete_prefix: true,
14509 ..Default::default()
14510 },
14511 )
14512 .await
14513 .expect_err("each bounded child must complete before the final crash boundary");
14514 assert!(
14515 retry.to_string().contains("retry the next durable batch"),
14516 "unexpected bounded-child result before {stage:?}: {retry}"
14517 );
14518 }
14519 }
14520
14521 let barrier = TierDeleteChunkTestBarrier::install(stage);
14522 let worker_store = store.clone();
14523 let worker_bucket = bucket.clone();
14524 let worker = tokio::spawn(async move {
14525 worker_store
14526 .delete_object_with_tier_delete_journal(
14527 &worker_bucket,
14528 prefix,
14529 ObjectOptions {
14530 delete_prefix: true,
14531 ..Default::default()
14532 },
14533 )
14534 .await
14535 });
14536 tokio::time::timeout(Duration::from_secs(30), barrier.wait_until_paused())
14537 .await
14538 .unwrap_or_else(|_| panic!("chunk delete did not reach crash boundary {stage:?}"));
14539 worker.abort();
14540 let _ = worker.await;
14541 drop(barrier);
14542 let released_bucket_guard =
14543 tokio::time::timeout(Duration::from_secs(5), store.acquire_bucket_lifecycle_write_lock(&bucket))
14544 .await
14545 .unwrap_or_else(|_| panic!("canceling at {stage:?} did not release the bucket lifecycle lock"))
14546 .unwrap_or_else(|err| {
14547 panic!("bucket lifecycle lock reacquire failed after cancellation at {stage:?}: {err}")
14548 });
14549 drop(released_bucket_guard);
14550 shutdown.cancel();
14551 drop(store);
14552 drop(ctx);
14553
14554 let restarted_name = format!("chunk-crash-boundary-{case}-restart");
14555 let (restarted_ctx, restarted_store, restarted_shutdown) =
14556 without_storage_class_env(build_isolated_test_store(temp_dir.path(), &restarted_name, &[4])).await;
14557 {
14558 let tier_config_mgr = restarted_ctx.tier_config_mgr();
14559 let mut manager = tier_config_mgr.write().await;
14560 manager.tiers.insert(tier_name.clone(), tier_config);
14561 manager
14562 .install_test_driver(&tier_name, Box::new(backend.clone()))
14563 .expect("the exact chunk crash-boundary tier driver should reinstall after restart");
14564 }
14565 crate::bucket::metadata_sys::init_bucket_metadata_sys(restarted_store.clone(), Vec::new()).await;
14566
14567 let mut completed = false;
14568 for _ in 0..20 {
14569 let recovery = recover_tier_delete_dispatch_manifests(restarted_store.clone(), 100, None)
14570 .await
14571 .expect("chunk crash-boundary manifest recovery should remain readable");
14572 assert_eq!(recovery.failed, 0, "recovery must not quarantine a valid chunk boundary");
14573 match restarted_store
14574 .delete_object_with_tier_delete_journal(
14575 &bucket,
14576 prefix,
14577 ObjectOptions {
14578 delete_prefix: true,
14579 ..Default::default()
14580 },
14581 )
14582 .await
14583 {
14584 Ok(_) => {
14585 completed = true;
14586 break;
14587 }
14588 Err(err)
14589 if err.to_string().contains("retry")
14590 || err.to_string().contains("rollback")
14591 || err.to_string().contains("durable cleanup") => {}
14592 Err(err) => panic!("chunk crash-boundary retry returned an unexpected error at {stage:?}: {err}"),
14593 }
14594 }
14595 assert!(completed, "chunk state must converge after cancellation at {stage:?}");
14596 for index in 0..2 {
14597 let object = format!("{prefix}object-{index}.bin");
14598 assert!(
14599 restarted_store.pools[0]
14600 .get_disks_by_key(&object)
14601 .load_file_info_versions_exact(&bucket, &object)
14602 .await
14603 .expect("chunk crash-boundary source lookup should succeed")
14604 .is_none(),
14605 "source {object} must not be skipped after cancellation at {stage:?}"
14606 );
14607 }
14608 assert!(
14609 restarted_store.pools[0]
14610 .get_disks_by_key(&local_only)
14611 .load_file_info_versions_exact(&bucket, &local_only)
14612 .await
14613 .expect("chunk crash-boundary local-only source lookup should succeed")
14614 .is_none(),
14615 "the final raw delete must remove the local-only source after cancellation at {stage:?}"
14616 );
14617 assert_eq!(backend.object_count().await, 2);
14618 backend.set_remove_failure(false);
14619 drive_tier_delete_dispatch_restart_to_convergence(restarted_store.clone()).await;
14620 assert_eq!(tier_delete_journal_count(restarted_store.clone()).await, 0);
14621 assert_eq!(tier_delete_dispatch_manifest_count(restarted_store.clone()).await, 0);
14622 assert_eq!(backend.object_count().await, 0);
14623 assert_eq!(
14624 backend.remove_versions().await.len(),
14625 2,
14626 "each crash case must retain exactly one cleanup owner per remote version"
14627 );
14628 restarted_shutdown.cancel();
14629 }
14630 }
14631
14632 #[cfg(feature = "test-util")]
14633 #[test]
14634 #[serial_test::serial(storage_class_env)]
14635 fn tier_delete_chunk_missing_child_with_journals_fails_closed() {
14636 run_large_stack_async_test(
14637 "tier-delete-chunk-missing-child",
14638 tier_delete_chunk_missing_child_with_journals_fails_closed_case,
14639 );
14640 }
14641
14642 #[cfg(feature = "test-util")]
14643 async fn tier_delete_chunk_missing_child_with_journals_fails_closed_case() {
14644 let _batch_limit = TierDeleteDispatchBatchLimitGuard::install(1);
14645 let temp_dir = tempfile::tempdir().expect("create missing-child store dir");
14646 let (ctx, store, shutdown) =
14647 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "chunk-missing-child", &[4])).await;
14648 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
14649 let tier_name = "CHUNK-MISSING-CHILD";
14650 let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
14651 backend.set_remove_failure(true);
14652 let bucket = "chunk-missing-child-bucket";
14653 let prefix = "prefix/";
14654 store
14655 .make_bucket(bucket, &MakeBucketOptions::default())
14656 .await
14657 .expect("missing-child bucket should be created");
14658 for index in 0..2 {
14659 let object = format!("{prefix}object-{index}.bin");
14660 let mut reader = PutObjReader::from_vec(vec![b'm' + index as u8; 1024 * 1024]);
14661 let source = store
14662 .put_object(bucket, &object, &mut reader, &ObjectOptions::default())
14663 .await
14664 .expect("missing-child source should be written");
14665 store
14666 .transition_object(
14667 bucket,
14668 &object,
14669 &ObjectOptions {
14670 transition: TransitionOptions {
14671 status: TRANSITION_PENDING.to_string(),
14672 tier: tier_name.to_string(),
14673 etag: source.etag.clone().expect("missing-child source should have an etag"),
14674 ..Default::default()
14675 },
14676 mod_time: source.mod_time,
14677 ..Default::default()
14678 },
14679 )
14680 .await
14681 .expect("missing-child source should transition");
14682 }
14683
14684 let barrier = TierDeleteChunkTestBarrier::install(TierDeleteChunkTestStage::DispatchAuthorized);
14685 let worker_store = store.clone();
14686 let worker = tokio::spawn(async move {
14687 worker_store
14688 .delete_object_with_tier_delete_journal(
14689 bucket,
14690 prefix,
14691 ObjectOptions {
14692 delete_prefix: true,
14693 ..Default::default()
14694 },
14695 )
14696 .await
14697 });
14698 tokio::time::timeout(Duration::from_secs(30), barrier.wait_until_paused())
14699 .await
14700 .expect("chunk delete should persist its authorization before corruption injection");
14701 worker.abort();
14702 let _ = worker.await;
14703 drop(barrier);
14704
14705 let records = store
14706 .clone()
14707 .list_objects_v2(
14708 RUSTFS_META_BUCKET,
14709 TIER_DELETE_DISPATCH_MANIFEST_PREFIX,
14710 None,
14711 None,
14712 10,
14713 false,
14714 None,
14715 false,
14716 )
14717 .await
14718 .expect("parent and child records should be listable");
14719 let child = records
14720 .objects
14721 .iter()
14722 .find(|object| object.name.contains("/chunks/"))
14723 .expect("the bound child manifest should exist")
14724 .name
14725 .clone();
14726 com::delete_config(store.clone(), &child)
14727 .await
14728 .expect("the test should remove only the child manifest");
14729 assert_eq!(tier_delete_journal_count(store.clone()).await, 1);
14730
14731 let error = store
14732 .delete_object_with_tier_delete_journal(
14733 bucket,
14734 prefix,
14735 ObjectOptions {
14736 delete_prefix: true,
14737 ..Default::default()
14738 },
14739 )
14740 .await
14741 .expect_err("a missing child with retained journals must quarantine the parent");
14742 assert!(
14743 error.to_string().contains("missing child with retained journals"),
14744 "unexpected missing-child result: {error}"
14745 );
14746 assert_eq!(backend.object_count().await, 2, "fail-closed inspection must not remove a remote version");
14747 shutdown.cancel();
14748 }
14749
14750 #[cfg(feature = "test-util")]
14751 #[test]
14752 #[serial_test::serial(storage_class_env)]
14753 fn authorized_prefix_retry_replays_predecessor_before_newcomer() {
14754 let handle = std::thread::Builder::new()
14755 .name("authorized-prefix-predecessor-replay-test".to_string())
14756 .stack_size(32 * 1024 * 1024)
14757 .spawn(|| {
14758 let runtime = tokio::runtime::Builder::new_multi_thread()
14759 .enable_all()
14760 .worker_threads(2)
14761 .build()
14762 .expect("authorized predecessor replay test runtime should build");
14763 runtime.block_on(authorized_prefix_retry_replays_predecessor_before_newcomer_case());
14764 })
14765 .expect("authorized predecessor replay test thread should spawn");
14766 if let Err(payload) = handle.join() {
14767 std::panic::resume_unwind(payload);
14768 }
14769 }
14770
14771 #[cfg(feature = "test-util")]
14772 async fn authorized_prefix_retry_replays_predecessor_before_newcomer_case() {
14773 let _batch_limit = TierDeleteDispatchBatchLimitGuard::install(1);
14774 let temp_dir = tempfile::tempdir().expect("create authorized predecessor replay store dir");
14775 let (ctx, store, _shutdown) =
14776 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "authorized-prefix-predecessor-replay", &[4]))
14777 .await;
14778 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
14779 let tier_name = "AUTHORIZED-PREFIX-REPLAY";
14780 let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
14781 let lease = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
14782 .await
14783 .expect("authorized replay tier lease should resolve");
14784 let bucket = "authorized-prefix-predecessor-replay-bucket";
14785 let prefix = "prefix/";
14786 let predecessor = "prefix/000-predecessor.bin";
14787 let newcomer = "prefix/zzz-newcomer.bin";
14788 store
14789 .make_bucket(bucket, &MakeBucketOptions::default())
14790 .await
14791 .expect("authorized replay bucket should be created");
14792
14793 let mut predecessor_reader = PutObjReader::from_vec(vec![b'p'; 1024 * 1024]);
14794 let predecessor_source = store
14795 .put_object(bucket, predecessor, &mut predecessor_reader, &ObjectOptions::default())
14796 .await
14797 .expect("predecessor source should be written");
14798 store
14799 .transition_object(
14800 bucket,
14801 predecessor,
14802 &ObjectOptions {
14803 transition: TransitionOptions {
14804 status: TRANSITION_PENDING.to_string(),
14805 tier: tier_name.to_string(),
14806 etag: predecessor_source.etag.clone().expect("predecessor should have an etag"),
14807 ..Default::default()
14808 },
14809 mod_time: predecessor_source.mod_time,
14810 ..Default::default()
14811 },
14812 )
14813 .await
14814 .expect("predecessor should transition");
14815 let predecessor_info = store
14816 .get_object_info(bucket, predecessor, &ObjectOptions::default())
14817 .await
14818 .expect("transitioned predecessor should be readable");
14819 let mut predecessor_entry =
14820 transitioned_delete_journal_entry_for_source(None, false, false, bucket, predecessor, &predecessor_info)
14821 .expect("transitioned predecessor should produce a dispatch journal");
14822 predecessor_entry.backend_identity = Some(lease.backend_identity());
14823 let incarnation = store
14824 .bucket_incarnation_id(bucket)
14825 .await
14826 .expect("authorized replay bucket incarnation should resolve");
14827 let (manifest_name, _) = install_test_tier_delete_dispatch_fixture(
14828 store.clone(),
14829 bucket,
14830 incarnation,
14831 prefix,
14832 vec![(predecessor_entry, Some(TierDeleteJournalState::Dispatched))],
14833 TierDeleteDispatchManifestState::DispatchAuthorized,
14834 )
14835 .await
14836 .expect("authorized predecessor fixture should persist");
14837
14838 let mut newcomer_reader = PutObjReader::from_vec(vec![b'n'; 1024 * 1024]);
14839 let newcomer_source = store
14840 .put_object(bucket, newcomer, &mut newcomer_reader, &ObjectOptions::default())
14841 .await
14842 .expect("newcomer source should be written after authorization");
14843 store
14844 .transition_object(
14845 bucket,
14846 newcomer,
14847 &ObjectOptions {
14848 transition: TransitionOptions {
14849 status: TRANSITION_PENDING.to_string(),
14850 tier: tier_name.to_string(),
14851 etag: newcomer_source.etag.clone().expect("newcomer should have an etag"),
14852 ..Default::default()
14853 },
14854 mod_time: newcomer_source.mod_time,
14855 ..Default::default()
14856 },
14857 )
14858 .await
14859 .expect("newcomer should transition");
14860 backend.set_remove_failure(true);
14861
14862 let publication_epoch = store
14863 .scanner_data_usage_publication_epoch()
14864 .await
14865 .expect("authorized replay should observe the current scanner publication epoch");
14866 let publication_scope = store
14867 .scanner_data_usage_publication_commit_scope(
14868 publication_epoch,
14869 tokio::time::Instant::now() + Duration::from_secs(30),
14870 Vec::new(),
14871 )
14872 .await
14873 .expect("authorized replay should acquire a scanner publication scope");
14874
14875 let retry = store
14876 .delete_object_with_tier_delete_journal(
14877 bucket,
14878 prefix,
14879 ObjectOptions {
14880 delete_prefix: true,
14881 scanner_publication_commit_scope: Some(publication_scope.clone()),
14882 ..Default::default()
14883 },
14884 )
14885 .await
14886 .expect_err("the predecessor-only replay must request a successor retry");
14887 assert!(
14888 retry.to_string().contains("retry the successor dispatch"),
14889 "unexpected predecessor replay result: {retry}"
14890 );
14891 assert_eq!(
14892 publication_scope.state(),
14893 crate::object_api::ScannerPublicationCommitState::Committed,
14894 "an exact predecessor replay must report its local mutation as committed"
14895 );
14896 assert!(publication_scope.release_movement_permit().await);
14897 assert!(
14898 store.pools[0]
14899 .get_disks_by_key(predecessor)
14900 .load_file_info_versions_exact(bucket, predecessor)
14901 .await
14902 .expect("predecessor metadata should remain readable")
14903 .is_none(),
14904 "the authorized predecessor must be removed exactly"
14905 );
14906 assert!(
14907 store.pools[0]
14908 .get_disks_by_key(newcomer)
14909 .load_file_info_versions_exact(bucket, newcomer)
14910 .await
14911 .expect("newcomer metadata should remain readable")
14912 .is_some(),
14913 "the predecessor authorization must not delete the newcomer"
14914 );
14915 assert_eq!(
14916 test_tier_delete_dispatch_manifest_state(store.clone(), &manifest_name)
14917 .await
14918 .expect("completed predecessor manifest should remain readable"),
14919 Some(TierDeleteDispatchManifestState::Completed)
14920 );
14921
14922 backend.set_remove_failure(false);
14923 let predecessor_journals = recover_tier_delete_journal_entries(store.clone(), 100, None)
14924 .await
14925 .expect("completed predecessor journal should recover");
14926 assert_eq!(predecessor_journals.failed, 0);
14927 assert!(predecessor_journals.scanned <= 1 && predecessor_journals.deleted <= 1);
14928 assert_eq!(tier_delete_journal_count(store.clone()).await, 0);
14929 assert_eq!(backend.object_count().await, 1, "only the newcomer remote object should remain");
14930 assert_eq!(backend.remove_versions().await.len(), 1);
14931 let predecessor_manifest = recover_tier_delete_dispatch_manifests(store.clone(), 100, None)
14932 .await
14933 .expect("completed predecessor manifest should be collected");
14934 assert_eq!(predecessor_manifest.failed, 0);
14935 assert!(predecessor_manifest.deleted <= 1);
14936 assert_eq!(tier_delete_dispatch_manifest_count(store.clone()).await, 0);
14937
14938 store
14939 .delete_object_with_tier_delete_journal(
14940 bucket,
14941 prefix,
14942 ObjectOptions {
14943 delete_prefix: true,
14944 ..Default::default()
14945 },
14946 )
14947 .await
14948 .expect("a successor dispatch should delete the remaining newcomer");
14949 assert!(
14950 store.pools[0]
14951 .get_disks_by_key(newcomer)
14952 .load_file_info_versions_exact(bucket, newcomer)
14953 .await
14954 .expect("newcomer metadata lookup should succeed after successor")
14955 .is_none()
14956 );
14957 drive_tier_delete_dispatch_restart_to_convergence(store.clone()).await;
14958 assert_eq!(tier_delete_journal_count(store.clone()).await, 0);
14959 assert_eq!(tier_delete_dispatch_manifest_count(store.clone()).await, 0);
14960 assert_eq!(backend.object_count().await, 0);
14961 assert_eq!(backend.remove_versions().await.len(), 2);
14962 }
14963
14964 #[cfg(feature = "test-util")]
14965 #[tokio::test]
14966 #[serial_test::serial(storage_class_env)]
14967 async fn multi_pool_same_tuple_recursive_prefix_uses_one_journal_owner() {
14968 let temp_dir = tempfile::tempdir().expect("create shared-tuple prefix-delete store dir");
14969 let (ctx, store, _shutdown) = without_storage_class_env(build_isolated_test_store(
14970 temp_dir.path(),
14971 "multi-pool-shared-tuple-prefix-delete",
14972 &[4, 4],
14973 ))
14974 .await;
14975 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
14976 let tier_name = "SHARED-PREFIX-DELETE";
14977 let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
14978 let bucket = "multi-pool-shared-tuple-prefix-delete-bucket";
14979 let object = "prefix/shared.bin";
14980 store
14981 .make_bucket(bucket, &MakeBucketOptions::default())
14982 .await
14983 .expect("shared-tuple prefix bucket should be created");
14984 let mut reader = PutObjReader::from_vec(vec![b's'; 1024 * 1024]);
14985 let source = store.pools[0]
14986 .put_object(bucket, object, &mut reader, &ObjectOptions::default())
14987 .await
14988 .expect("shared-tuple prefix source should be written");
14989 store.pools[0]
14990 .transition_object(
14991 bucket,
14992 object,
14993 &ObjectOptions {
14994 transition: TransitionOptions {
14995 status: TRANSITION_PENDING.to_string(),
14996 tier: tier_name.to_string(),
14997 etag: source.etag.clone().expect("shared-tuple prefix source should have an etag"),
14998 ..Default::default()
14999 },
15000 mod_time: source.mod_time,
15001 ..Default::default()
15002 },
15003 )
15004 .await
15005 .expect("shared-tuple prefix source should transition");
15006 copy_test_xlmeta_between_pools(temp_dir.path(), 0, 1, bucket, object).await;
15007 assert_eq!(backend.object_count().await, 1);
15008 backend.set_remove_failure(true);
15009
15010 store
15011 .delete_object_with_tier_delete_journal(
15012 bucket,
15013 "prefix/",
15014 ObjectOptions {
15015 delete_prefix: true,
15016 ..Default::default()
15017 },
15018 )
15019 .await
15020 .expect("shared-tuple recursive prefix delete should commit");
15021
15022 assert_eq!(tier_delete_journal_count(store.clone()).await, 1);
15023 for pool_idx in 0..2 {
15024 assert!(
15025 store.pools[pool_idx]
15026 .get_disks_by_key(object)
15027 .load_file_info_versions_exact(bucket, object)
15028 .await
15029 .expect("shared-tuple post-delete metadata should decode")
15030 .is_none(),
15031 "pool {pool_idx} must be physically empty"
15032 );
15033 }
15034 backend.set_remove_failure(false);
15035 ctx.wake_tier_delete_journal_recovery();
15036 wait_for_tier_delete_journal_recovery(store.clone(), &backend, 1).await;
15037 assert_eq!(backend.object_count().await, 0);
15038 assert_eq!(backend.remove_versions().await.len(), 1);
15039 }
15040
15041 #[cfg(feature = "test-util")]
15042 #[tokio::test]
15043 #[serial_test::serial(storage_class_env)]
15044 async fn multi_pool_recursive_prefix_rejects_legacy_or_hidden_merge_loser_before_delete() {
15045 let temp_dir = tempfile::tempdir().expect("create merge-loser prefix-delete store dir");
15046 let (ctx, store, _shutdown) = without_storage_class_env(build_isolated_test_store(
15047 temp_dir.path(),
15048 "multi-pool-merge-loser-prefix-delete",
15049 &[4, 4],
15050 ))
15051 .await;
15052 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
15053 let tier_name = "MERGE-LOSER-PREFIX-DELETE";
15054 let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
15055 let bucket = "multi-pool-merge-loser-prefix-delete-bucket";
15056 store
15057 .make_bucket(bucket, &MakeBucketOptions::default())
15058 .await
15059 .expect("merge-loser prefix bucket should be created");
15060
15061 for (object, byte) in [("legacy/item.bin", b'l'), ("hidden/item.bin", b'h')] {
15062 let mut reader = PutObjReader::from_vec(vec![byte; 1024 * 1024]);
15063 let source = store.pools[0]
15064 .put_object(bucket, object, &mut reader, &ObjectOptions::default())
15065 .await
15066 .expect("merge-loser source should be written");
15067 store.pools[0]
15068 .transition_object(
15069 bucket,
15070 object,
15071 &ObjectOptions {
15072 transition: TransitionOptions {
15073 status: TRANSITION_PENDING.to_string(),
15074 tier: tier_name.to_string(),
15075 etag: source.etag.clone().expect("merge-loser source should have an etag"),
15076 ..Default::default()
15077 },
15078 mod_time: source.mod_time,
15079 ..Default::default()
15080 },
15081 )
15082 .await
15083 .expect("merge-loser source should transition");
15084 copy_test_xlmeta_between_pools(temp_dir.path(), 0, 1, bucket, object).await;
15085 }
15086 rewrite_transitioned_xlmeta_as_legacy_unknown(temp_dir.path(), 1, bucket, "legacy/item.bin", false).await;
15087 backend.set_remove_failure(true);
15088 store.pools[1]
15089 .delete_object(bucket, "hidden/item.bin", ObjectOptions::default())
15090 .await
15091 .expect("second-pool source delete should retain a hidden free-version");
15092
15093 let legacy_error = store
15094 .delete_object_with_tier_delete_journal(
15095 bucket,
15096 "legacy/",
15097 ObjectOptions {
15098 delete_prefix: true,
15099 ..Default::default()
15100 },
15101 )
15102 .await
15103 .expect_err("a legacy merge loser must reject recursive deletion");
15104 assert!(legacy_error.to_string().contains("legacy Unknown"), "unexpected error: {legacy_error}");
15105 let hidden_error = store
15106 .delete_object_with_tier_delete_journal(
15107 bucket,
15108 "hidden/",
15109 ObjectOptions {
15110 delete_prefix: true,
15111 ..Default::default()
15112 },
15113 )
15114 .await
15115 .expect_err("a hidden free-version merge loser must reject recursive deletion");
15116 assert!(
15117 hidden_error.to_string().contains("free-version cleanup obligation"),
15118 "unexpected error: {hidden_error}"
15119 );
15120
15121 assert_eq!(tier_delete_journal_count(store.clone()).await, 0);
15122 assert_eq!(backend.object_count().await, 2);
15123 assert_eq!(backend.remove_count().await, 0);
15124 for object in ["legacy/item.bin", "hidden/item.bin"] {
15125 let pool0 = store.pools[0]
15126 .get_disks_by_key(object)
15127 .load_file_info_versions_exact(bucket, object)
15128 .await
15129 .expect("winner metadata should decode")
15130 .expect("winner source must remain before destructive deletion");
15131 assert_eq!(pool0.versions.iter().filter(|version| !version.tier_free_version()).count(), 1);
15132 }
15133 let hidden_pool1 = store.pools[1]
15134 .get_disks_by_key("hidden/item.bin")
15135 .load_file_info_versions_exact(bucket, "hidden/item.bin")
15136 .await
15137 .expect("hidden loser metadata should decode")
15138 .expect("hidden loser cleanup owner must remain");
15139 assert_eq!(
15140 hidden_pool1
15141 .versions
15142 .iter()
15143 .chain(hidden_pool1.free_versions.iter())
15144 .filter(|version| version.tier_free_version())
15145 .count(),
15146 1
15147 );
15148 }
15149
15150 #[cfg(feature = "test-util")]
15151 #[tokio::test]
15152 #[serial_test::serial(storage_class_env)]
15153 async fn recursive_prefix_partial_pool_failure_keeps_prepared_cleanup_owners() {
15154 let temp_dir = tempfile::tempdir().expect("create partial prefix-delete store dir");
15155 let (ctx, store, _shutdown) =
15156 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "partial-pool-prefix-delete", &[4, 4])).await;
15157 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
15158 let tier_name = "PARTIAL-PREFIX-DELETE";
15159 let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
15160 let bucket = "partial-pool-prefix-delete-bucket";
15161 let object = "prefix/item.bin";
15162 store
15163 .make_bucket(bucket, &MakeBucketOptions::default())
15164 .await
15165 .expect("partial prefix bucket should be created");
15166 for (pool_idx, byte) in [(0, b'a'), (1, b'b')] {
15167 let mut reader = PutObjReader::from_vec(vec![byte; 1024 * 1024]);
15168 let source = store.pools[pool_idx]
15169 .put_object(bucket, object, &mut reader, &ObjectOptions::default())
15170 .await
15171 .expect("pool-local prefix source should be written");
15172 store.pools[pool_idx]
15173 .transition_object(
15174 bucket,
15175 object,
15176 &ObjectOptions {
15177 transition: TransitionOptions {
15178 status: TRANSITION_PENDING.to_string(),
15179 tier: tier_name.to_string(),
15180 etag: source.etag.clone().expect("pool-local source should have an etag"),
15181 ..Default::default()
15182 },
15183 mod_time: source.mod_time,
15184 ..Default::default()
15185 },
15186 )
15187 .await
15188 .expect("pool-local prefix source should transition");
15189 }
15190 assert_eq!(backend.object_count().await, 2);
15191 backend.set_remove_failure(true);
15192
15193 let barrier = crate::store::object::PrefixDeleteAfterJournalPrepareBarrier::install(bucket, "prefix/");
15194 let delete_store = store.clone();
15195 let delete = tokio::spawn(async move {
15196 delete_store
15197 .delete_object_with_tier_delete_journal(
15198 bucket,
15199 "prefix/",
15200 ObjectOptions {
15201 delete_prefix: true,
15202 ..Default::default()
15203 },
15204 )
15205 .await
15206 });
15207 tokio::time::timeout(Duration::from_secs(10), barrier.wait_until_paused())
15208 .await
15209 .expect("prefix delete should pause after every journal is Prepared");
15210 assert_eq!(tier_delete_journal_count(store.clone()).await, 2);
15211
15212 let failed_set = store.pools[1].get_disks_by_key(object);
15213 let offline_disks = force_set_disks_offline_for_test(&failed_set).await;
15214 barrier.release();
15215 let error = delete
15216 .await
15217 .expect("partial prefix delete task should join")
15218 .expect_err("an offline second pool must fail the recursive prefix delete");
15219 assert!(
15220 matches!(
15221 error,
15222 StorageError::InsufficientWriteQuorum(_, _) | StorageError::InsufficientReadQuorum(_, _)
15223 ),
15224 "unexpected error: {error:?}"
15225 );
15226 drop(offline_disks);
15227 drop(barrier);
15228
15229 assert_eq!(
15230 tier_delete_journal_count(store.clone()).await,
15231 2,
15232 "an ambiguous partial mutation must retain every Dispatched owner"
15233 );
15234 let first_pool_state = store.pools[0]
15235 .get_disks_by_key(object)
15236 .load_file_info_versions_exact(bucket, object)
15237 .await
15238 .expect("first-pool metadata should remain decodable after an ambiguous failure");
15239 if let Some(first_pool_state) = first_pool_state {
15240 assert_eq!(
15241 first_pool_state
15242 .versions
15243 .iter()
15244 .filter(|version| !version.tier_free_version())
15245 .count(),
15246 1,
15247 "a retained first-pool source must remain a visible source, not an unjournaled fallback owner"
15248 );
15249 }
15250 assert!(
15251 store.pools[1]
15252 .get_disks_by_key(object)
15253 .load_file_info_versions_exact(bucket, object)
15254 .await
15255 .expect("failed-pool metadata should decode")
15256 .is_some(),
15257 "the failed pool should retain its live source"
15258 );
15259
15260 let _ = recover_tier_delete_journal_entries(store.clone(), 100, None)
15261 .await
15262 .expect("Dispatched owners should reconcile after the failed pool returns");
15263 assert_eq!(
15264 tier_delete_journal_count(store.clone()).await,
15265 2,
15266 "recovery must never abort or delete a Dispatched journal while any source is still live"
15267 );
15268 store
15269 .delete_object_with_tier_delete_journal(
15270 bucket,
15271 "prefix/",
15272 ObjectOptions {
15273 delete_prefix: true,
15274 ..Default::default()
15275 },
15276 )
15277 .await
15278 .expect("retry should delete the remaining pool source");
15279 backend.set_remove_failure(false);
15280 for _ in 0..3 {
15281 let _ = recover_tier_delete_journal_entries(store.clone(), 100, None)
15282 .await
15283 .expect("remaining pool journals should settle after backend recovery");
15284 if tier_delete_journal_count(store.clone()).await == 0 {
15285 break;
15286 }
15287 }
15288 assert_eq!(tier_delete_journal_count(store.clone()).await, 0);
15289 assert_eq!(backend.object_count().await, 0);
15290 assert_eq!(backend.remove_versions().await.len(), 2);
15291 }
15292
15293 #[cfg(feature = "test-util")]
15294 #[tokio::test]
15295 #[serial_test::serial(storage_class_env)]
15296 async fn recursive_prefix_partial_set_failure_keeps_prepared_cleanup_owners() {
15297 let temp_dir = tempfile::tempdir().expect("create partial-set prefix-delete store dir");
15298 let (ctx, store, _shutdown) = without_storage_class_env(build_isolated_test_store_with_layout(
15299 temp_dir.path(),
15300 "partial-set-prefix-delete",
15301 &[(2, 4)],
15302 CancellationToken::new(),
15303 None,
15304 ))
15305 .await;
15306 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
15307 let tier_name = "PARTIAL-SET-PREFIX-DELETE";
15308 let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
15309 let bucket = "partial-set-prefix-delete-bucket";
15310 store
15311 .make_bucket(bucket, &MakeBucketOptions::default())
15312 .await
15313 .expect("partial-set prefix bucket should be created");
15314
15315 let objects = (0..2)
15316 .map(|target_set| {
15317 (0..512)
15318 .map(|index| format!("prefix/set-{target_set}-{index}.bin"))
15319 .find(|candidate| store.pools[0].get_disks_by_key(candidate).set_index == target_set)
15320 .expect("a deterministic key should map to each test set")
15321 })
15322 .collect::<Vec<_>>();
15323 for (index, object) in objects.iter().enumerate() {
15324 let mut reader = PutObjReader::from_vec(vec![b'0' + index as u8; 1024 * 1024]);
15325 let source = store.pools[0]
15326 .put_object(bucket, object, &mut reader, &ObjectOptions::default())
15327 .await
15328 .expect("set-local prefix source should be written");
15329 store.pools[0]
15330 .transition_object(
15331 bucket,
15332 object,
15333 &ObjectOptions {
15334 transition: TransitionOptions {
15335 status: TRANSITION_PENDING.to_string(),
15336 tier: tier_name.to_string(),
15337 etag: source.etag.clone().expect("set-local source should have an etag"),
15338 ..Default::default()
15339 },
15340 mod_time: source.mod_time,
15341 ..Default::default()
15342 },
15343 )
15344 .await
15345 .expect("set-local prefix source should transition");
15346 }
15347 assert_eq!(backend.object_count().await, 2);
15348 backend.set_remove_failure(true);
15349
15350 let barrier = crate::store::object::PrefixDeleteAfterJournalPrepareBarrier::install(bucket, "prefix/");
15351 let delete_store = store.clone();
15352 let delete = tokio::spawn(async move {
15353 delete_store
15354 .delete_object_with_tier_delete_journal(
15355 bucket,
15356 "prefix/",
15357 ObjectOptions {
15358 delete_prefix: true,
15359 ..Default::default()
15360 },
15361 )
15362 .await
15363 });
15364 tokio::time::timeout(Duration::from_secs(10), barrier.wait_until_paused())
15365 .await
15366 .expect("set-scoped prefix delete should pause after journal Prepare");
15367 assert_eq!(tier_delete_journal_count(store.clone()).await, 2);
15368
15369 let failed_set = store.pools[0].disk_set[1].clone();
15370 let offline_disks = force_set_disks_offline_for_test(&failed_set).await;
15371 barrier.release();
15372 let error = delete
15373 .await
15374 .expect("partial-set prefix delete task should join")
15375 .expect_err("an offline second set must fail recursive prefix deletion");
15376 assert!(
15377 matches!(
15378 error,
15379 StorageError::InsufficientWriteQuorum(_, _) | StorageError::InsufficientReadQuorum(_, _)
15380 ),
15381 "unexpected error: {error:?}"
15382 );
15383 drop(offline_disks);
15384 drop(barrier);
15385
15386 assert_eq!(tier_delete_journal_count(store.clone()).await, 2);
15387 let first_set_state = store.pools[0]
15388 .get_disks_by_key(&objects[0])
15389 .load_file_info_versions_exact(bucket, &objects[0])
15390 .await
15391 .expect("first-set metadata should remain decodable after an ambiguous failure");
15392 if let Some(first_set_state) = first_set_state {
15393 assert_eq!(
15394 first_set_state
15395 .versions
15396 .iter()
15397 .filter(|version| !version.tier_free_version())
15398 .count(),
15399 1,
15400 "a retained first-set source must remain visible under its Dispatched journal"
15401 );
15402 }
15403 assert!(
15404 store.pools[0]
15405 .get_disks_by_key(&objects[1])
15406 .load_file_info_versions_exact(bucket, &objects[1])
15407 .await
15408 .expect("failed-set metadata should decode")
15409 .is_some()
15410 );
15411
15412 let _ = recover_tier_delete_journal_entries(store.clone(), 100, None)
15413 .await
15414 .expect("set-scoped Dispatched owners should reconcile");
15415 assert_eq!(
15416 tier_delete_journal_count(store.clone()).await,
15417 2,
15418 "recovery must retain the complete Authorized dispatch until every source is absent"
15419 );
15420 store
15421 .delete_object_with_tier_delete_journal(
15422 bucket,
15423 "prefix/",
15424 ObjectOptions {
15425 delete_prefix: true,
15426 ..Default::default()
15427 },
15428 )
15429 .await
15430 .expect("set-scoped retry should delete the remaining source");
15431 backend.set_remove_failure(false);
15432 for _ in 0..3 {
15433 let _ = recover_tier_delete_journal_entries(store.clone(), 100, None)
15434 .await
15435 .expect("remaining set journals should settle after backend recovery");
15436 if tier_delete_journal_count(store.clone()).await == 0 {
15437 break;
15438 }
15439 }
15440 assert_eq!(tier_delete_journal_count(store.clone()).await, 0);
15441 assert_eq!(backend.object_count().await, 0);
15442 assert_eq!(backend.remove_versions().await.len(), 2);
15443 }
15444
15445 #[cfg(feature = "test-util")]
15446 #[tokio::test]
15447 #[serial_test::serial(storage_class_env)]
15448 async fn versioned_explicit_transition_delete_preserves_other_version_then_allows_bucket_delete() {
15449 let temp_dir = tempfile::tempdir().expect("create versioned transitioned delete store dir");
15450 let (ctx, store, _shutdown) =
15451 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "versioned-transitioned-delete-owner", &[4]))
15452 .await;
15453 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
15454 let tier_name = "VERSIONED-DELETE";
15455 let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
15456 let bucket = "versioned-transitioned-delete-owner-bucket";
15457 let object = "archive.bin";
15458 store
15459 .make_bucket(bucket, &MakeBucketOptions::default())
15460 .await
15461 .expect("versioned source bucket should be created");
15462 crate::bucket::metadata_sys::update_in(
15463 &ctx,
15464 bucket,
15465 BUCKET_VERSIONING_CONFIG,
15466 b"<VersioningConfiguration><Status>Enabled</Status></VersioningConfiguration>".to_vec(),
15467 )
15468 .await
15469 .expect("bucket versioning should be enabled");
15470
15471 let mut first_reader = PutObjReader::from_vec(vec![b'1'; 1024 * 1024]);
15472 let first = store
15473 .put_object(
15474 bucket,
15475 object,
15476 &mut first_reader,
15477 &ObjectOptions {
15478 versioned: true,
15479 ..Default::default()
15480 },
15481 )
15482 .await
15483 .expect("first version should be written");
15484 let first_version = first.version_id.expect("versioned PUT should return an identity");
15485 store
15486 .transition_object(
15487 bucket,
15488 object,
15489 &ObjectOptions {
15490 version_id: Some(first_version.to_string()),
15491 versioned: true,
15492 transition: TransitionOptions {
15493 status: TRANSITION_PENDING.to_string(),
15494 tier: tier_name.to_string(),
15495 etag: first.etag.clone().expect("first version should have an etag"),
15496 ..Default::default()
15497 },
15498 mod_time: first.mod_time,
15499 ..Default::default()
15500 },
15501 )
15502 .await
15503 .expect("first version should transition");
15504 let mut second_reader = PutObjReader::from_vec(vec![b'2'; 1024 * 1024]);
15505 let second = store
15506 .put_object(
15507 bucket,
15508 object,
15509 &mut second_reader,
15510 &ObjectOptions {
15511 versioned: true,
15512 ..Default::default()
15513 },
15514 )
15515 .await
15516 .expect("second version should be written");
15517 let second_version = second.version_id.expect("second PUT should return an identity");
15518
15519 backend.set_remove_failure(true);
15520 store
15521 .delete_object_with_tier_delete_journal(
15522 bucket,
15523 object,
15524 ObjectOptions {
15525 version_id: Some(first_version.to_string()),
15526 versioned: true,
15527 ..Default::default()
15528 },
15529 )
15530 .await
15531 .expect("explicit transitioned version delete should commit");
15532 store
15533 .get_object_info(
15534 bucket,
15535 object,
15536 &ObjectOptions {
15537 version_id: Some(second_version.to_string()),
15538 versioned: true,
15539 ..Default::default()
15540 },
15541 )
15542 .await
15543 .expect("the unrelated second version must remain readable");
15544 let exact = store.pools[0]
15545 .get_disks_by_key(object)
15546 .load_file_info_versions_exact(bucket, object)
15547 .await
15548 .expect("remaining version metadata should decode")
15549 .expect("the second version should remain");
15550 assert_eq!(
15551 exact
15552 .versions
15553 .iter()
15554 .chain(exact.free_versions.iter())
15555 .filter(|version| version.tier_free_version())
15556 .count(),
15557 1,
15558 "ordinary exact-version delete must leave one hidden free-version owner"
15559 );
15560 assert!(
15561 exact
15562 .versions
15563 .iter()
15564 .any(|version| version.version_id == Some(second_version)),
15565 "the explicit delete must not remove the other version"
15566 );
15567 assert_eq!(
15568 tier_delete_journal_count(store.clone()).await,
15569 0,
15570 "ordinary exact-version delete must not create a journal"
15571 );
15572 backend.set_remove_failure(false);
15573 wait_for_tier_free_version_recovery(store.clone(), &backend, 1).await;
15574 let exact = store.pools[0]
15575 .get_disks_by_key(object)
15576 .load_file_info_versions_exact(bucket, object)
15577 .await
15578 .expect("post-cleanup metadata should decode")
15579 .expect("the unrelated second version should remain after cleanup");
15580 assert_eq!(
15581 exact
15582 .versions
15583 .iter()
15584 .chain(exact.free_versions.iter())
15585 .filter(|version| version.tier_free_version())
15586 .count(),
15587 0,
15588 "free-version recovery must remove only its exact owner"
15589 );
15590 assert!(
15591 exact
15592 .versions
15593 .iter()
15594 .any(|version| version.version_id == Some(second_version)),
15595 "free-version recovery must preserve the unrelated version"
15596 );
15597
15598 store
15599 .delete_object_with_tier_delete_journal(
15600 bucket,
15601 object,
15602 ObjectOptions {
15603 version_id: Some(second_version.to_string()),
15604 versioned: true,
15605 ..Default::default()
15606 },
15607 )
15608 .await
15609 .expect("last ordinary version should be removed");
15610 store
15611 .delete_bucket(bucket, &DeleteBucketOptions::default())
15612 .await
15613 .expect("bucket delete should succeed after the last version is removed");
15614 }
15615
15616 #[cfg(feature = "test-util")]
15617 #[tokio::test]
15618 #[serial_test::serial(storage_class_env)]
15619 async fn suspended_null_transition_delete_uses_free_version_as_sole_owner() {
15620 let temp_dir = tempfile::tempdir().expect("create suspended transitioned delete store dir");
15621 let (ctx, store, _shutdown) =
15622 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "suspended-transitioned-delete-owner", &[4]))
15623 .await;
15624 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
15625 let tier_name = "SUSPENDED-DELETE";
15626 let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
15627 let bucket = "suspended-transitioned-delete-owner-bucket";
15628 let object = "archive.bin";
15629 store
15630 .make_bucket(bucket, &MakeBucketOptions::default())
15631 .await
15632 .expect("suspended source bucket should be created");
15633 crate::bucket::metadata_sys::update_in(
15634 &ctx,
15635 bucket,
15636 BUCKET_VERSIONING_CONFIG,
15637 b"<VersioningConfiguration><Status>Suspended</Status></VersioningConfiguration>".to_vec(),
15638 )
15639 .await
15640 .expect("bucket versioning should be suspended");
15641
15642 let mut reader = PutObjReader::from_vec(vec![b's'; 1024 * 1024]);
15643 let source = store
15644 .put_object(
15645 bucket,
15646 object,
15647 &mut reader,
15648 &ObjectOptions {
15649 version_suspended: true,
15650 ..Default::default()
15651 },
15652 )
15653 .await
15654 .expect("suspended null source should be written");
15655 let null_version = source.version_id.expect("suspended source should expose the null identity");
15656 assert!(null_version.is_nil(), "suspended source identity must be null");
15657 store
15658 .transition_object(
15659 bucket,
15660 object,
15661 &ObjectOptions {
15662 version_id: Some(null_version.to_string()),
15663 version_suspended: true,
15664 transition: TransitionOptions {
15665 status: TRANSITION_PENDING.to_string(),
15666 tier: tier_name.to_string(),
15667 etag: source.etag.clone().expect("suspended source should have an etag"),
15668 ..Default::default()
15669 },
15670 mod_time: source.mod_time,
15671 ..Default::default()
15672 },
15673 )
15674 .await
15675 .expect("suspended null source should transition");
15676
15677 backend.set_remove_failure(true);
15678 store
15679 .delete_object_with_tier_delete_journal(
15680 bucket,
15681 object,
15682 ObjectOptions {
15683 version_id: Some(null_version.to_string()),
15684 version_suspended: true,
15685 ..Default::default()
15686 },
15687 )
15688 .await
15689 .expect("explicit null transitioned version delete should commit");
15690 let exact = store.pools[0]
15691 .get_disks_by_key(object)
15692 .load_file_info_versions_exact(bucket, object)
15693 .await
15694 .expect("post-delete null metadata should decode")
15695 .expect("ordinary null-version delete must retain a cleanup owner");
15696 assert_eq!(
15697 exact
15698 .versions
15699 .iter()
15700 .chain(exact.free_versions.iter())
15701 .filter(|version| version.tier_free_version())
15702 .count(),
15703 1,
15704 "ordinary null-version delete must retain exactly one hidden free-version owner"
15705 );
15706 assert_eq!(
15707 exact.versions.iter().filter(|version| !version.tier_free_version()).count(),
15708 0,
15709 "ordinary null-version delete must remove the visible source"
15710 );
15711 assert_eq!(tier_delete_journal_count(store.clone()).await, 0);
15712 backend.set_remove_failure(false);
15713 wait_for_tier_free_version_recovery(store.clone(), &backend, 1).await;
15714 store
15715 .delete_bucket(bucket, &DeleteBucketOptions::default())
15716 .await
15717 .expect("suspended bucket should be empty after exact null-version cleanup");
15718 }
15719
15720 #[cfg(feature = "test-util")]
15721 #[tokio::test]
15722 #[serial_test::serial(storage_class_env)]
15723 async fn versioned_delete_marker_keeps_transitioned_source_and_remote_object() {
15724 let temp_dir = tempfile::tempdir().expect("create versioned delete-marker store dir");
15725 let (ctx, store, _shutdown) =
15726 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "transitioned-delete-marker", &[4])).await;
15727 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
15728 let tier_name = "DELETE-MARKER";
15729 let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
15730 let bucket = "transitioned-delete-marker-bucket";
15731 let object = "archive.bin";
15732 store
15733 .make_bucket(bucket, &MakeBucketOptions::default())
15734 .await
15735 .expect("versioned source bucket should be created");
15736 crate::bucket::metadata_sys::update_in(
15737 &ctx,
15738 bucket,
15739 BUCKET_VERSIONING_CONFIG,
15740 b"<VersioningConfiguration><Status>Enabled</Status></VersioningConfiguration>".to_vec(),
15741 )
15742 .await
15743 .expect("bucket versioning should be enabled");
15744
15745 let mut reader = PutObjReader::from_vec(vec![b'm'; 1024 * 1024]);
15746 let source = store
15747 .put_object(
15748 bucket,
15749 object,
15750 &mut reader,
15751 &ObjectOptions {
15752 versioned: true,
15753 ..Default::default()
15754 },
15755 )
15756 .await
15757 .expect("versioned source should be written");
15758 let source_version = source.version_id.expect("versioned source should have an identity");
15759 store
15760 .transition_object(
15761 bucket,
15762 object,
15763 &ObjectOptions {
15764 version_id: Some(source_version.to_string()),
15765 versioned: true,
15766 transition: TransitionOptions {
15767 status: TRANSITION_PENDING.to_string(),
15768 tier: tier_name.to_string(),
15769 etag: source.etag.clone().expect("versioned source should have an etag"),
15770 ..Default::default()
15771 },
15772 mod_time: source.mod_time,
15773 ..Default::default()
15774 },
15775 )
15776 .await
15777 .expect("versioned source should transition");
15778
15779 let marker = store
15780 .delete_object_with_tier_delete_journal(
15781 bucket,
15782 object,
15783 ObjectOptions {
15784 versioned: true,
15785 ..Default::default()
15786 },
15787 )
15788 .await
15789 .expect("versioned DELETE without versionId should create a marker");
15790 assert!(marker.delete_marker);
15791 let marker_version = marker.version_id.expect("delete marker should have an identity");
15792 assert_eq!(backend.object_count().await, 1, "delete-marker creation must retain the remote object");
15793 assert_eq!(
15794 tier_delete_journal_count(store.clone()).await,
15795 0,
15796 "delete-marker creation must not schedule cleanup"
15797 );
15798 let exact = store.pools[0]
15799 .get_disks_by_key(object)
15800 .load_file_info_versions_exact(bucket, object)
15801 .await
15802 .expect("versioned delete-marker metadata should decode")
15803 .expect("source and delete marker should remain on disk");
15804 assert!(
15805 exact.versions.iter().any(|version| {
15806 version.version_id == Some(source_version) && version.transition_status == rustfs_filemeta::TRANSITION_COMPLETE
15807 }),
15808 "the transitioned source must remain behind the delete marker"
15809 );
15810 assert_eq!(
15811 exact
15812 .versions
15813 .iter()
15814 .chain(exact.free_versions.iter())
15815 .filter(|version| version.tier_free_version())
15816 .count(),
15817 0,
15818 "delete-marker creation must not create a cleanup free-version"
15819 );
15820
15821 store
15822 .delete_object_with_tier_delete_journal(
15823 bucket,
15824 object,
15825 ObjectOptions {
15826 version_id: Some(marker_version.to_string()),
15827 versioned: true,
15828 ..Default::default()
15829 },
15830 )
15831 .await
15832 .expect("delete marker should be removable by exact identity");
15833 store
15834 .delete_object_with_tier_delete_journal(
15835 bucket,
15836 object,
15837 ObjectOptions {
15838 version_id: Some(source_version.to_string()),
15839 versioned: true,
15840 ..Default::default()
15841 },
15842 )
15843 .await
15844 .expect("transitioned source should be removable by exact identity");
15845 wait_for_tier_free_version_recovery(store.clone(), &backend, 1).await;
15846 if let Some(exact) = store.pools[0]
15847 .get_disks_by_key(object)
15848 .load_file_info_versions_exact(bucket, object)
15849 .await
15850 .expect("post-recovery version metadata should decode")
15851 {
15852 assert!(
15853 exact
15854 .versions
15855 .iter()
15856 .chain(exact.free_versions.iter())
15857 .all(|version| !version.tier_free_version()),
15858 "free-version recovery must remove the all-physical cleanup owner"
15859 );
15860 }
15861 store
15862 .delete_bucket(bucket, &DeleteBucketOptions::default())
15863 .await
15864 .expect("versioned source bucket should be empty after explicit cleanup");
15865 }
15866
15867 #[cfg(feature = "test-util")]
15868 #[tokio::test]
15869 #[serial_test::serial(storage_class_env)]
15870 async fn batch_transitioned_delete_uses_free_version_per_item() {
15871 let temp_dir = tempfile::tempdir().expect("create batch transitioned delete store dir");
15872 let (ctx, store, _shutdown) =
15873 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "batch-transitioned-delete-owner", &[4])).await;
15874 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
15875 let tier_name = "BATCH-DELETE";
15876 let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
15877 let bucket = "batch-transitioned-delete-owner-bucket";
15878 let transitioned = "transitioned.bin";
15879 let ordinary = "ordinary.bin";
15880 store
15881 .make_bucket(bucket, &MakeBucketOptions::default())
15882 .await
15883 .expect("batch source bucket should be created");
15884 let mut transitioned_reader = PutObjReader::from_vec(vec![b't'; 1024 * 1024]);
15885 let transitioned_source = store
15886 .put_object(bucket, transitioned, &mut transitioned_reader, &ObjectOptions::default())
15887 .await
15888 .expect("transitioned batch source should be written");
15889 store
15890 .transition_object(
15891 bucket,
15892 transitioned,
15893 &ObjectOptions {
15894 transition: TransitionOptions {
15895 status: TRANSITION_PENDING.to_string(),
15896 tier: tier_name.to_string(),
15897 etag: transitioned_source
15898 .etag
15899 .clone()
15900 .expect("transitioned batch source should have an etag"),
15901 ..Default::default()
15902 },
15903 mod_time: transitioned_source.mod_time,
15904 ..Default::default()
15905 },
15906 )
15907 .await
15908 .expect("batch source transition should commit");
15909 let mut ordinary_reader = PutObjReader::from_vec(vec![b'o'; 1024 * 1024]);
15910 store
15911 .put_object(bucket, ordinary, &mut ordinary_reader, &ObjectOptions::default())
15912 .await
15913 .expect("ordinary batch source should be written");
15914
15915 backend.set_remove_failure(true);
15916 let (_deleted, errors) = store
15917 .delete_objects_with_tier_delete_journal(
15918 bucket,
15919 vec![
15920 ObjectToDelete {
15921 object_name: transitioned.to_string(),
15922 ..Default::default()
15923 },
15924 ObjectToDelete {
15925 object_name: ordinary.to_string(),
15926 ..Default::default()
15927 },
15928 ObjectToDelete {
15929 object_name: "missing.bin".to_string(),
15930 ..Default::default()
15931 },
15932 ],
15933 ObjectOptions::default(),
15934 )
15935 .await;
15936 assert!(errors.iter().all(Option::is_none), "S3 batch delete should be idempotent: {errors:?}");
15937 let transitioned_versions = store.pools[0]
15938 .get_disks_by_key(transitioned)
15939 .load_file_info_versions_exact(bucket, transitioned)
15940 .await
15941 .expect("transitioned batch metadata should remain decodable")
15942 .expect("transitioned batch item must retain a cleanup owner");
15943 assert_eq!(
15944 transitioned_versions
15945 .versions
15946 .iter()
15947 .chain(transitioned_versions.free_versions.iter())
15948 .filter(|version| version.tier_free_version())
15949 .count(),
15950 1,
15951 "transitioned batch item must retain exactly one hidden free-version owner"
15952 );
15953 assert_eq!(
15954 transitioned_versions
15955 .versions
15956 .iter()
15957 .filter(|version| !version.tier_free_version())
15958 .count(),
15959 0,
15960 "transitioned batch item must have no visible source"
15961 );
15962 assert!(
15963 store.pools[0]
15964 .get_disks_by_key(ordinary)
15965 .load_file_info_versions_exact(bucket, ordinary)
15966 .await
15967 .expect("ordinary batch metadata should remain decodable")
15968 .is_none(),
15969 "ordinary batch item should have no local metadata"
15970 );
15971 assert_eq!(
15972 tier_delete_journal_count(store.clone()).await,
15973 0,
15974 "ordinary batch delete must not create a tier-delete journal"
15975 );
15976 assert_eq!(backend.object_count().await, 1, "failed remote cleanup must retain its object");
15977
15978 backend.set_remove_failure(false);
15979 wait_for_tier_free_version_recovery(store.clone(), &backend, 1).await;
15980 assert!(
15981 store.pools[0]
15982 .get_disks_by_key(transitioned)
15983 .load_file_info_versions_exact(bucket, transitioned)
15984 .await
15985 .expect("cleaned batch metadata should remain decodable")
15986 .is_none(),
15987 "free-version recovery must remove the exact cleanup owner"
15988 );
15989
15990 let causal = "causal.bin";
15991 let mut causal_reader = PutObjReader::from_vec(vec![b'c'; 1024 * 1024]);
15992 let causal_source = store
15993 .put_object(bucket, causal, &mut causal_reader, &ObjectOptions::default())
15994 .await
15995 .expect("causal batch source should be written");
15996 store
15997 .transition_object(
15998 bucket,
15999 causal,
16000 &ObjectOptions {
16001 transition: TransitionOptions {
16002 status: TRANSITION_PENDING.to_string(),
16003 tier: tier_name.to_string(),
16004 etag: causal_source.etag.clone().expect("causal batch source should have an etag"),
16005 ..Default::default()
16006 },
16007 mod_time: causal_source.mod_time,
16008 ..Default::default()
16009 },
16010 )
16011 .await
16012 .expect("causal batch source transition should commit");
16013 let (_deleted, errors) = store
16014 .delete_objects(
16015 bucket,
16016 vec![
16017 ObjectToDelete {
16018 object_name: causal.to_string(),
16019 ..Default::default()
16020 },
16021 ObjectToDelete {
16022 object_name: causal.to_string(),
16023 ..Default::default()
16024 },
16025 ],
16026 ObjectOptions::default(),
16027 )
16028 .await;
16029 assert!(
16030 errors.iter().all(Option::is_none),
16031 "duplicate causal batch deletes should remain idempotent: {errors:?}"
16032 );
16033 tokio::time::timeout(Duration::from_secs(30), async {
16034 loop {
16035 let metadata_absent = exact_metadata_absent_during_cleanup(&store, bucket, causal).await;
16036 if metadata_absent && backend.remove_versions().await.len() >= 2 {
16037 return;
16038 }
16039 tokio::time::sleep(Duration::from_millis(10)).await;
16040 }
16041 })
16042 .await
16043 .expect("batch free-version receipt should converge without another recovery scan");
16044 assert_eq!(
16045 backend.remove_versions().await.len(),
16046 2,
16047 "duplicate batch requests must cause only one remote delete for the causal object"
16048 );
16049
16050 crate::bucket::metadata_sys::update_in(
16051 &ctx,
16052 bucket,
16053 BUCKET_VERSIONING_CONFIG,
16054 b"<VersioningConfiguration><Status>Enabled</Status></VersioningConfiguration>".to_vec(),
16055 )
16056 .await
16057 .expect("causal batch bucket versioning should be enabled");
16058 let versioned_causal = "versioned-causal.bin";
16059 let mut versioned_reader = PutObjReader::from_vec(vec![b'v'; 1024 * 1024]);
16060 let versioned_source = store
16061 .put_object(
16062 bucket,
16063 versioned_causal,
16064 &mut versioned_reader,
16065 &ObjectOptions {
16066 versioned: true,
16067 ..Default::default()
16068 },
16069 )
16070 .await
16071 .expect("versioned causal batch source should be written");
16072 let versioned_source_id = versioned_source
16073 .version_id
16074 .expect("versioned causal batch source should have an identity");
16075 store
16076 .transition_object(
16077 bucket,
16078 versioned_causal,
16079 &ObjectOptions {
16080 version_id: Some(versioned_source_id.to_string()),
16081 versioned: true,
16082 transition: TransitionOptions {
16083 status: TRANSITION_PENDING.to_string(),
16084 tier: tier_name.to_string(),
16085 etag: versioned_source
16086 .etag
16087 .clone()
16088 .expect("versioned causal batch source should have an etag"),
16089 ..Default::default()
16090 },
16091 mod_time: versioned_source.mod_time,
16092 ..Default::default()
16093 },
16094 )
16095 .await
16096 .expect("versioned causal batch source transition should commit");
16097 let (_deleted, errors) = store
16098 .delete_objects(
16099 bucket,
16100 vec![ObjectToDelete {
16101 object_name: versioned_causal.to_string(),
16102 version_id: Some(versioned_source_id),
16103 ..Default::default()
16104 }],
16105 ObjectOptions::default(),
16106 )
16107 .await;
16108 assert!(
16109 errors.iter().all(Option::is_none),
16110 "explicit-version causal batch delete should commit: {errors:?}"
16111 );
16112 tokio::time::timeout(Duration::from_secs(30), async {
16113 loop {
16114 let metadata_absent = exact_metadata_absent_during_cleanup(&store, bucket, versioned_causal).await;
16115 if metadata_absent && backend.remove_versions().await.len() == 3 {
16116 return;
16117 }
16118 tokio::time::sleep(Duration::from_millis(10)).await;
16119 }
16120 })
16121 .await
16122 .expect("explicit-version batch receipt should converge without a recovery scan");
16123 store
16124 .delete_bucket(bucket, &DeleteBucketOptions::default())
16125 .await
16126 .expect("batch source bucket should be physically empty");
16127 }
16128
16129 #[cfg(feature = "test-util")]
16130 #[tokio::test]
16131 #[serial_test::serial(storage_class_env)]
16132 async fn batch_transitioned_delete_aggregate_error_still_enqueues_committed_free_version() {
16133 let temp_dir = tempfile::tempdir().expect("create aggregate-error batch delete store dir");
16134 let (ctx, store, shutdown) =
16135 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "batch-transitioned-aggregate-error", &[4, 4]))
16136 .await;
16137 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
16138 let tier_name = "BATCH-AGGREGATE-ERROR";
16139 let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
16140 let bucket = "batch-transitioned-aggregate-error-bucket";
16141 let object = "archive.bin";
16142 store
16143 .make_bucket(bucket, &MakeBucketOptions::default())
16144 .await
16145 .expect("aggregate-error source bucket should be created");
16146 let mut reader = PutObjReader::from_vec(vec![b'a'; 1024 * 1024]);
16147 let source = store.pools[0]
16148 .put_object(bucket, object, &mut reader, &ObjectOptions::default())
16149 .await
16150 .expect("aggregate-error source should be written");
16151 store.pools[0]
16152 .transition_object(
16153 bucket,
16154 object,
16155 &ObjectOptions {
16156 transition: TransitionOptions {
16157 status: TRANSITION_PENDING.to_string(),
16158 tier: tier_name.to_string(),
16159 etag: source.etag.clone().expect("aggregate-error source should have an etag"),
16160 ..Default::default()
16161 },
16162 mod_time: source.mod_time,
16163 ..Default::default()
16164 },
16165 )
16166 .await
16167 .expect("aggregate-error source should transition");
16168
16169 for disk_index in 0..4 {
16173 let source_meta = temp_dir
16174 .path()
16175 .join(format!("pool0/set0/disk{disk_index}/{bucket}/{object}/{STORAGE_FORMAT_FILE}"));
16176 let target_meta = temp_dir
16177 .path()
16178 .join(format!("pool1/set0/disk{disk_index}/{bucket}/{object}/{STORAGE_FORMAT_FILE}"));
16179 tokio::fs::create_dir_all(target_meta.parent().expect("target xl.meta should have a parent"))
16180 .await
16181 .expect("second-pool object directory should be created");
16182 tokio::fs::copy(&source_meta, &target_meta)
16183 .await
16184 .expect("transitioned xl.meta should copy exactly to the second pool");
16185 }
16186
16187 ExpiryState::resize_workers(1, store.clone()).await;
16188 let injection = crate::store::object::BatchDeletePoolErrorInjection::install(
16189 bucket,
16190 1,
16191 vec![(object.to_string(), StorageError::ErasureWriteQuorum)],
16192 );
16193 let (deleted, errors) = store
16194 .delete_objects(
16195 bucket,
16196 vec![ObjectToDelete {
16197 object_name: object.to_string(),
16198 ..Default::default()
16199 }],
16200 ObjectOptions::default(),
16201 )
16202 .await;
16203 assert_eq!(injection.observed(), 1, "the second pool should inject one post-commit aggregate error");
16204 assert_eq!(errors, vec![Some(StorageError::ErasureWriteQuorum)]);
16205 assert!(deleted[0].found, "the aggregate error must retain the committed pool result");
16206 drop(injection);
16207
16208 wait_for_expiry_workers_idle(&store).await;
16211 for pool in &store.pools {
16212 assert!(
16213 pool.get_disks_by_key(object)
16214 .load_file_info_versions_exact(bucket, object)
16215 .await
16216 .expect("aggregate-error cleanup metadata should remain readable")
16217 .is_none(),
16218 "aggregate failure must not suppress committed receipt cleanup"
16219 );
16220 }
16221 assert_eq!(
16222 backend.remove_count().await,
16223 1,
16224 "committed receipts must remove the shared remote object once"
16225 );
16226 assert_eq!(backend.object_count().await, 0, "the shared remote object should be removed exactly once");
16227 store
16228 .delete_bucket(bucket, &DeleteBucketOptions::default())
16229 .await
16230 .expect("aggregate-error bucket should be physically empty");
16231 shutdown.cancel();
16232 }
16233
16234 #[cfg(feature = "test-util")]
16235 #[tokio::test]
16236 #[serial_test::serial(storage_class_env)]
16237 async fn transitioned_delete_free_version_replays_after_store_restart() {
16238 let temp_dir = tempfile::tempdir().expect("create restarted free-version store dir");
16239 let (ctx, store, shutdown) =
16240 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "transitioned-delete-restart-before", &[4]))
16241 .await;
16242 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
16243 let tier_name = "DELETE-RESTART";
16244 let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
16245 let tier_config = ctx
16246 .tier_config_mgr()
16247 .read()
16248 .await
16249 .tiers
16250 .get(tier_name)
16251 .expect("mock tier config should exist")
16252 .clone_with_credentials();
16253 let bucket = "transitioned-delete-restart-bucket";
16254 let object = "archive.bin";
16255 store
16256 .make_bucket(bucket, &MakeBucketOptions::default())
16257 .await
16258 .expect("restart source bucket should be created");
16259 let mut reader = PutObjReader::from_vec(vec![b'r'; 1024 * 1024]);
16260 let original = store
16261 .put_object(bucket, object, &mut reader, &ObjectOptions::default())
16262 .await
16263 .expect("restart source object should be written");
16264 store
16265 .transition_object(
16266 bucket,
16267 object,
16268 &ObjectOptions {
16269 transition: TransitionOptions {
16270 status: TRANSITION_PENDING.to_string(),
16271 tier: tier_name.to_string(),
16272 etag: original.etag.clone().expect("restart source should have an etag"),
16273 ..Default::default()
16274 },
16275 mod_time: original.mod_time,
16276 ..Default::default()
16277 },
16278 )
16279 .await
16280 .expect("restart source transition should commit");
16281 backend.set_remove_failure(true);
16282 store
16283 .delete_object_with_tier_delete_journal(bucket, object, ObjectOptions::default())
16284 .await
16285 .expect("local source delete should commit while remote cleanup is retryable");
16286 let retained = store.pools[0]
16287 .get_disks_by_key(object)
16288 .load_file_info_versions_exact(bucket, object)
16289 .await
16290 .expect("post-delete local metadata should decode")
16291 .expect("ordinary delete must retain a free-version across restart");
16292 assert_eq!(
16293 retained
16294 .versions
16295 .iter()
16296 .chain(retained.free_versions.iter())
16297 .filter(|version| version.tier_free_version())
16298 .count(),
16299 1,
16300 "ordinary delete must retain exactly one free-version across restart"
16301 );
16302 assert_eq!(
16303 retained
16304 .versions
16305 .iter()
16306 .filter(|version| !version.tier_free_version())
16307 .count(),
16308 0,
16309 "ordinary delete must remove the visible source before restart"
16310 );
16311 assert_eq!(tier_delete_journal_count(store.clone()).await, 0);
16312 assert_eq!(backend.object_count().await, 1);
16313
16314 shutdown.cancel();
16315 drop(store);
16316 drop(ctx);
16317 let (restarted_ctx, restarted_store, restarted_shutdown) =
16318 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "transitioned-delete-restart-after", &[4]))
16319 .await;
16320 crate::bucket::metadata_sys::init_bucket_metadata_sys(restarted_store.clone(), Vec::new()).await;
16321 {
16322 let restarted_tier_config_mgr = restarted_ctx.tier_config_mgr();
16323 let mut manager = restarted_tier_config_mgr.write().await;
16324 manager.tiers.insert(tier_name.to_string(), tier_config);
16325 manager
16326 .install_test_driver(tier_name, Box::new(backend.clone()))
16327 .expect("restarted store should bind the original tier destination");
16328 }
16329 backend.set_remove_failure(false);
16330 wait_for_tier_free_version_recovery(restarted_store.clone(), &backend, 1).await;
16331 assert_eq!(tier_delete_journal_count(restarted_store.clone()).await, 0);
16332 assert_eq!(backend.object_count().await, 0);
16333 assert!(
16334 restarted_store.pools[0]
16335 .get_disks_by_key(object)
16336 .load_file_info_versions_exact(bucket, object)
16337 .await
16338 .expect("restarted cleanup metadata should decode")
16339 .is_none(),
16340 "restarted recovery must remove the exact free-version owner"
16341 );
16342 restarted_store
16343 .delete_bucket(bucket, &DeleteBucketOptions::default())
16344 .await
16345 .expect("source bucket should be empty after restarted free-version recovery");
16346 restarted_shutdown.cancel();
16347 }
16348
16349 #[cfg(feature = "test-util")]
16350 #[tokio::test]
16351 #[serial_test::serial(storage_class_env)]
16352 async fn transitioned_delete_local_quorum_failure_rolls_back_without_cleanup_owner() {
16353 let temp_dir = tempfile::tempdir().expect("create failed local delete store dir");
16354 let (ctx, store, _shutdown) =
16355 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "transitioned-delete-local-failure", &[4]))
16356 .await;
16357 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
16358 let tier_name = "DELETE-LOCAL-FAIL";
16359 let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
16360 let bucket = "transitioned-delete-local-failure-bucket";
16361 let object = "archive.bin";
16362 store
16363 .make_bucket(bucket, &MakeBucketOptions::default())
16364 .await
16365 .expect("failed-delete source bucket should be created");
16366 let mut reader = PutObjReader::from_vec(vec![b'q'; 1024 * 1024]);
16367 let source = store
16368 .put_object(bucket, object, &mut reader, &ObjectOptions::default())
16369 .await
16370 .expect("failed-delete source should be written");
16371 store
16372 .transition_object(
16373 bucket,
16374 object,
16375 &ObjectOptions {
16376 transition: TransitionOptions {
16377 status: TRANSITION_PENDING.to_string(),
16378 tier: tier_name.to_string(),
16379 etag: source.etag.clone().expect("failed-delete source should have an etag"),
16380 ..Default::default()
16381 },
16382 mod_time: source.mod_time,
16383 ..Default::default()
16384 },
16385 )
16386 .await
16387 .expect("failed-delete source should transition");
16388
16389 let set = store.pools[0].get_disks_by_key(object);
16390 let saved_disk = {
16391 let mut disks = set.disks.write().await;
16392 let first = disks[0].as_ref().expect("first disk should be online");
16393 crate::disk::local::set_delete_version_fail_after_commit(first.path().as_path(), object);
16394 disks[1].take().expect("second disk should be online before fault injection")
16395 };
16396 let err = store
16397 .delete_object_with_tier_delete_journal(bucket, object, ObjectOptions::default())
16398 .await
16399 .expect_err("one post-commit error plus one offline disk must miss delete quorum");
16400 assert!(matches!(err, StorageError::Io(_)), "unexpected local delete failure: {err:?}");
16401 assert!(
16402 err.to_string()
16403 .contains("Storage resources are insufficient for the write operation"),
16404 "the caller must observe the wrapped quorum failure: {err}"
16405 );
16406 set.disks.write().await[1] = Some(saved_disk);
16407
16408 let retained = store
16409 .get_object_info(bucket, object, &ObjectOptions::default())
16410 .await
16411 .expect("failed local delete must roll back the transitioned source");
16412 assert_eq!(retained.transitioned_object.status, rustfs_filemeta::TRANSITION_COMPLETE);
16413 let retained_versions = set
16414 .load_file_info_versions_exact(bucket, object)
16415 .await
16416 .expect("rolled-back metadata should decode")
16417 .expect("rolled-back source should remain on disk");
16418 assert_eq!(
16419 retained_versions
16420 .versions
16421 .iter()
16422 .chain(retained_versions.free_versions.iter())
16423 .filter(|version| version.tier_free_version())
16424 .count(),
16425 0,
16426 "failed local quorum must not create a cleanup owner"
16427 );
16428 assert_eq!(
16429 tier_delete_journal_count(store.clone()).await,
16430 0,
16431 "ordinary delete must not create a journal, including on rollback"
16432 );
16433 assert_eq!(backend.object_count().await, 1, "failed local commit must retain the remote object");
16434 assert_eq!(backend.remove_count().await, 0, "failed local commit must not dispatch remote cleanup");
16435
16436 backend.set_remove_failure(true);
16437 store
16438 .delete_object_with_tier_delete_journal(bucket, object, ObjectOptions::default())
16439 .await
16440 .expect("retry after disk recovery should commit");
16441 let cleanup_owner = set
16442 .load_file_info_versions_exact(bucket, object)
16443 .await
16444 .expect("retry metadata should decode")
16445 .expect("successful retry must leave a free-version owner");
16446 assert_eq!(
16447 cleanup_owner
16448 .versions
16449 .iter()
16450 .chain(cleanup_owner.free_versions.iter())
16451 .filter(|version| version.tier_free_version())
16452 .count(),
16453 1,
16454 "successful retry must leave exactly one free-version owner"
16455 );
16456 assert_eq!(tier_delete_journal_count(store.clone()).await, 0);
16457 backend.set_remove_failure(false);
16458 wait_for_tier_free_version_recovery(store.clone(), &backend, 1).await;
16459 store
16460 .delete_bucket(bucket, &DeleteBucketOptions::default())
16461 .await
16462 .expect("retry should leave the source bucket empty");
16463 }
16464
16465 #[cfg(feature = "test-util")]
16466 #[tokio::test]
16467 #[serial_test::serial(storage_class_env)]
16468 async fn batch_transitioned_delete_post_commit_failures_roll_back_without_free_version_receipt() {
16469 let temp_dir = tempfile::tempdir().expect("create failed batch delete store dir");
16470 let (ctx, store, shutdown) =
16471 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "batch-delete-local-failure", &[4])).await;
16472 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
16473 let tier_name = "BATCH-DELETE-LOCAL-FAIL";
16474 let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
16475 let bucket = "batch-delete-local-failure-bucket";
16476 let object = "archive.bin";
16477 store
16478 .make_bucket(bucket, &MakeBucketOptions::default())
16479 .await
16480 .expect("failed batch source bucket should be created");
16481 crate::bucket::metadata_sys::update_in(
16482 &ctx,
16483 bucket,
16484 BUCKET_VERSIONING_CONFIG,
16485 b"<VersioningConfiguration><Status>Enabled</Status></VersioningConfiguration>".to_vec(),
16486 )
16487 .await
16488 .expect("failed batch bucket versioning should be enabled");
16489 let mut transitioned_reader = PutObjReader::from_vec(vec![b't'; 1024 * 1024]);
16490 let transitioned_source = store
16491 .put_object(
16492 bucket,
16493 object,
16494 &mut transitioned_reader,
16495 &ObjectOptions {
16496 versioned: true,
16497 ..Default::default()
16498 },
16499 )
16500 .await
16501 .expect("failed batch transitioned version should be written");
16502 let transitioned_version_id = transitioned_source
16503 .version_id
16504 .expect("failed batch transitioned source should have a version identity");
16505 store
16506 .transition_object(
16507 bucket,
16508 object,
16509 &ObjectOptions {
16510 version_id: Some(transitioned_version_id.to_string()),
16511 versioned: true,
16512 transition: TransitionOptions {
16513 status: TRANSITION_PENDING.to_string(),
16514 tier: tier_name.to_string(),
16515 etag: transitioned_source
16516 .etag
16517 .clone()
16518 .expect("failed batch transitioned source should have an etag"),
16519 ..Default::default()
16520 },
16521 mod_time: transitioned_source.mod_time,
16522 ..Default::default()
16523 },
16524 )
16525 .await
16526 .expect("failed batch source version should transition");
16527 let mut ordinary_reader = PutObjReader::from_vec(vec![b'o'; 1024 * 1024]);
16528 let ordinary_source = store
16529 .put_object(
16530 bucket,
16531 object,
16532 &mut ordinary_reader,
16533 &ObjectOptions {
16534 versioned: true,
16535 ..Default::default()
16536 },
16537 )
16538 .await
16539 .expect("failed batch ordinary sibling should be written");
16540 let ordinary_version_id = ordinary_source
16541 .version_id
16542 .expect("failed batch ordinary sibling should have a version identity");
16543 let delete_requests = || {
16544 vec![
16545 ObjectToDelete {
16546 object_name: object.to_string(),
16547 version_id: Some(transitioned_version_id),
16548 ..Default::default()
16549 },
16550 ObjectToDelete {
16551 object_name: object.to_string(),
16552 version_id: Some(ordinary_version_id),
16553 ..Default::default()
16554 },
16555 ]
16556 };
16557
16558 let set = store.pools[0].get_disks_by_key(object);
16559 let disks = set.disks.read().await;
16560 assert_eq!(disks.len(), 4, "the rollback fixture must use four disks");
16561 for disk in disks.iter().take(3) {
16564 let disk = disk.as_ref().expect("injected rollback disks should be online");
16565 crate::disk::local::set_delete_version_fail_after_commit(disk.path().as_path(), object);
16566 }
16567 drop(disks);
16568 let receipt_sink = crate::object_api::TierFreeVersionReceiptSink::new();
16569 let (_deleted, errors) = store
16570 .delete_objects(
16571 bucket,
16572 delete_requests(),
16573 ObjectOptions {
16574 tier_free_version_receipt_sink: Some(receipt_sink.clone()),
16575 ..Default::default()
16576 },
16577 )
16578 .await;
16579 assert_eq!(
16580 errors,
16581 vec![Some(StorageError::Unexpected), Some(StorageError::Unexpected),],
16582 "three post-commit disk errors must fail batch delete before receipts publish"
16583 );
16584
16585 assert!(
16586 receipt_sink
16587 .drain()
16588 .expect("the test-owned failed-batch sink should drain exactly once")
16589 .is_empty(),
16590 "a rolled-back physical group must publish no cleanup receipt"
16591 );
16592 let retained_transitioned = store
16593 .get_object_info(
16594 bucket,
16595 object,
16596 &ObjectOptions {
16597 version_id: Some(transitioned_version_id.to_string()),
16598 versioned: true,
16599 ..Default::default()
16600 },
16601 )
16602 .await
16603 .expect("failed batch delete must restore the transitioned sibling");
16604 assert_eq!(retained_transitioned.transitioned_object.status, rustfs_filemeta::TRANSITION_COMPLETE);
16605 let retained_ordinary = store
16606 .get_object_info(
16607 bucket,
16608 object,
16609 &ObjectOptions {
16610 version_id: Some(ordinary_version_id.to_string()),
16611 versioned: true,
16612 ..Default::default()
16613 },
16614 )
16615 .await
16616 .expect("failed batch delete must restore the ordinary sibling");
16617 assert_ne!(retained_ordinary.transitioned_object.status, rustfs_filemeta::TRANSITION_COMPLETE);
16618 let retained_versions = set
16619 .load_file_info_versions_exact(bucket, object)
16620 .await
16621 .expect("rolled-back batch metadata should decode")
16622 .expect("rolled-back batch source should remain on disk");
16623 assert_eq!(
16624 retained_versions
16625 .versions
16626 .iter()
16627 .chain(retained_versions.free_versions.iter())
16628 .filter(|version| version.tier_free_version())
16629 .count(),
16630 0,
16631 "failed batch quorum must not retain a free-version owner"
16632 );
16633 let retained_version_ids = retained_versions
16634 .versions
16635 .iter()
16636 .filter_map(|version| version.version_id)
16637 .collect::<std::collections::HashSet<_>>();
16638 assert_eq!(
16639 retained_version_ids,
16640 std::collections::HashSet::from([transitioned_version_id, ordinary_version_id]),
16641 "the physical-group rollback must restore both explicit siblings"
16642 );
16643 assert_eq!(backend.object_count().await, 1, "failed batch commit must retain the remote object");
16644 assert_eq!(backend.remove_count().await, 0, "failed batch commit must not dispatch remote cleanup");
16645
16646 ExpiryState::resize_workers(1, store.clone()).await;
16647 let (_deleted, retry_errors) = store
16648 .delete_objects(bucket, delete_requests(), ObjectOptions::default())
16649 .await;
16650 assert!(
16651 retry_errors.iter().all(Option::is_none),
16652 "retry after disk recovery should commit: {retry_errors:?}"
16653 );
16654 tokio::time::timeout(Duration::from_secs(30), async {
16655 loop {
16656 let metadata_absent = match set.load_file_info_versions_exact(bucket, object).await {
16659 Ok(versions) => versions.is_none(),
16660 Err(StorageError::InsufficientReadQuorum(..)) => false,
16661 Err(err) => panic!("retry cleanup metadata should remain readable: {err:?}"),
16662 };
16663 if metadata_absent && backend.remove_count().await == 1 {
16664 return;
16665 }
16666 tokio::time::sleep(Duration::from_millis(10)).await;
16667 }
16668 })
16669 .await
16670 .expect("successful batch retry should converge without a recovery scan");
16671 store
16672 .delete_bucket(bucket, &DeleteBucketOptions::default())
16673 .await
16674 .expect("successful batch retry should leave the bucket empty");
16675 shutdown.cancel();
16676 }
16677
16678 #[cfg(feature = "test-util")]
16679 async fn run_multi_pool_same_remote_tuple_delete_case(batch: bool) {
16680 let temp_dir = tempfile::tempdir().expect("create shared-tuple multi-pool store dir");
16681 let case = if batch { "batch" } else { "single" };
16682 let (ctx, store, _shutdown) = without_storage_class_env(build_isolated_test_store(
16683 temp_dir.path(),
16684 &format!("multi-pool-same-tuple-{case}"),
16685 &[4, 4],
16686 ))
16687 .await;
16688 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
16689 let tier_name = if batch { "SAME-TUPLE-BATCH" } else { "SAME-TUPLE-SINGLE" };
16690 let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
16691 let bucket = format!("multi-pool-same-tuple-{case}-bucket");
16692 let object = "archive.bin";
16693 store
16694 .make_bucket(&bucket, &MakeBucketOptions::default())
16695 .await
16696 .expect("shared-tuple bucket should be created");
16697 let mut reader = PutObjReader::from_vec(vec![b's'; 1024 * 1024]);
16698 let source = store.pools[0]
16699 .put_object(&bucket, object, &mut reader, &ObjectOptions::default())
16700 .await
16701 .expect("first pool source should be written");
16702 store.pools[0]
16703 .transition_object(
16704 &bucket,
16705 object,
16706 &ObjectOptions {
16707 transition: TransitionOptions {
16708 status: TRANSITION_PENDING.to_string(),
16709 tier: tier_name.to_string(),
16710 etag: source.etag.clone().expect("shared source should have an etag"),
16711 ..Default::default()
16712 },
16713 mod_time: source.mod_time,
16714 ..Default::default()
16715 },
16716 )
16717 .await
16718 .expect("first pool source should transition");
16719 let first = store.pools[0]
16720 .get_object_info(&bucket, object, &ObjectOptions::default())
16721 .await
16722 .expect("first pool transitioned source should be readable");
16723
16724 for disk_index in 0..4 {
16728 let source_meta = temp_dir
16729 .path()
16730 .join(format!("pool0/set0/disk{disk_index}/{bucket}/{object}/{STORAGE_FORMAT_FILE}"));
16731 let target_meta = temp_dir
16732 .path()
16733 .join(format!("pool1/set0/disk{disk_index}/{bucket}/{object}/{STORAGE_FORMAT_FILE}"));
16734 tokio::fs::create_dir_all(target_meta.parent().expect("target xl.meta should have a parent"))
16735 .await
16736 .expect("second pool object directory should be created");
16737 tokio::fs::copy(&source_meta, &target_meta)
16738 .await
16739 .expect("transitioned xl.meta should copy exactly to the second pool");
16740 }
16741 let second = store.pools[1]
16742 .get_object_info(&bucket, object, &ObjectOptions::default())
16743 .await
16744 .expect("copied second-pool source should be readable");
16745 assert_eq!(
16746 (first.transitioned_object.name.clone(), first.transitioned_object.version_id.clone()),
16747 (second.transitioned_object.name.clone(), second.transitioned_object.version_id.clone())
16748 );
16749 assert_eq!(backend.object_count().await, 1);
16750
16751 let receipt_sink = crate::object_api::TierFreeVersionReceiptSink::new();
16752 let mut delete_opts = ObjectOptions {
16753 tier_delete_journal_api: Some(store.clone()),
16754 tier_free_version_receipt_sink: Some(receipt_sink.clone()),
16755 ..Default::default()
16756 };
16757 if batch {
16758 delete_opts.delete_replication_config_snapshot = Some(Arc::new(
16759 ReplicationObjectBridge::delete_request_config_in(&store.ctx, &bucket)
16760 .await
16761 .expect("batch delete replication snapshot should load"),
16762 ));
16763 let (_deleted, errors, _accounting) = store.pools[0]
16764 .delete_objects_with_accounting(
16765 &bucket,
16766 vec![ObjectToDelete {
16767 object_name: object.to_string(),
16768 ..Default::default()
16769 }],
16770 delete_opts.clone(),
16771 )
16772 .await;
16773 assert!(errors.iter().all(Option::is_none), "first pool batch delete should commit: {errors:?}");
16774 } else {
16775 store.pools[0]
16776 .delete_object(&bucket, object, delete_opts.clone())
16777 .await
16778 .expect("first pool single delete should commit");
16779 }
16780
16781 let first_pool_versions = store.pools[0]
16782 .get_disks_by_key(object)
16783 .load_file_info_versions_exact(&bucket, object)
16784 .await
16785 .expect("first-pool metadata should decode")
16786 .expect("first pool must retain a free-version owner");
16787 assert_eq!(
16788 first_pool_versions
16789 .versions
16790 .iter()
16791 .chain(first_pool_versions.free_versions.iter())
16792 .filter(|version| version.tier_free_version())
16793 .count(),
16794 1,
16795 "first pool must retain exactly one free-version owner"
16796 );
16797 assert_eq!(
16798 first_pool_versions
16799 .versions
16800 .iter()
16801 .filter(|version| !version.tier_free_version())
16802 .count(),
16803 0,
16804 "first pool must remove its visible source"
16805 );
16806 assert_eq!(
16807 tier_delete_journal_count(store.clone()).await,
16808 0,
16809 "ordinary pool-local delete must not create a journal"
16810 );
16811
16812 ExpiryState::resize_workers(1, store.clone()).await;
16813 let first_recovery = recover_tier_free_versions(store.clone(), 100, None, None)
16814 .await
16815 .expect("first-pool free-version should be recoverable");
16816 assert_eq!(
16817 (first_recovery.enqueued, first_recovery.failed),
16818 (0, 0),
16819 "the live second-pool source is the merged walk winner, so its hidden loser must not dispatch cleanup"
16820 );
16821 wait_for_expiry_workers_idle(&store).await;
16822 assert_eq!(backend.remove_count().await, 0, "live second-pool source must gate remote DELETE");
16823 assert_eq!(backend.object_count().await, 1, "live source must retain its remote body");
16824 assert!(
16825 store.pools[0]
16826 .get_disks_by_key(object)
16827 .load_file_info_versions_exact(&bucket, object)
16828 .await
16829 .expect("blocked cleanup metadata should decode")
16830 .is_some(),
16831 "blocked cleanup must retain the first-pool free-version"
16832 );
16833 let surviving = store.pools[1]
16834 .get_object_info(&bucket, object, &ObjectOptions::default())
16835 .await
16836 .expect("second-pool source must remain GET-addressable");
16837 assert_eq!(surviving.transitioned_object.name, first.transitioned_object.name);
16838
16839 backend.set_remove_failure(true);
16840 if batch {
16841 let (_deleted, errors, _accounting) = store.pools[1]
16842 .delete_objects_with_accounting(
16843 &bucket,
16844 vec![ObjectToDelete {
16845 object_name: object.to_string(),
16846 ..Default::default()
16847 }],
16848 delete_opts,
16849 )
16850 .await;
16851 assert!(errors.iter().all(Option::is_none), "second pool batch retry should commit: {errors:?}");
16852 } else {
16853 store.pools[1]
16854 .delete_object(&bucket, object, delete_opts)
16855 .await
16856 .expect("second pool single retry should commit");
16857 }
16858 wait_for_expiry_workers_idle(&store).await;
16859 let mut free_version_ids = Vec::new();
16860 for pool_idx in 0..2 {
16861 let retained = store.pools[pool_idx]
16862 .get_disks_by_key(object)
16863 .load_file_info_versions_exact(&bucket, object)
16864 .await
16865 .expect("shared-tuple free-version metadata should decode")
16866 .expect("remote failure must retain every shared-tuple cleanup owner");
16867 let owners = retained
16868 .versions
16869 .iter()
16870 .chain(retained.free_versions.iter())
16871 .filter(|version| version.tier_free_version())
16872 .collect::<Vec<_>>();
16873 assert_eq!(owners.len(), 1, "pool {pool_idx} must retain one free-version owner");
16874 free_version_ids.push(owners[0].version_id);
16875 }
16876 assert_ne!(
16877 free_version_ids[0], free_version_ids[1],
16878 "ordinary pool-local deletes must model distinct local free-version identities"
16879 );
16880
16881 backend.set_remove_failure(false);
16882 let receipts = receipt_sink
16883 .drain()
16884 .expect("the simulated outer multi-pool wrapper should drain exactly once");
16885 assert_eq!(
16886 receipts.len(),
16887 1,
16888 "the same physical key and remote tuple must collapse to one causal task"
16889 );
16890 assert_eq!(
16891 crate::bucket::lifecycle::bucket_lifecycle_ops::enqueue_committed_free_versions(&store, receipts).await,
16892 1,
16893 "the committed shared-tuple task should enter the running worker"
16894 );
16895 wait_for_expiry_workers_idle(&store).await;
16896 assert_eq!(backend.remove_count().await, 1, "shared remote tuple should be deleted exactly once");
16897 for pool_idx in 0..2 {
16898 assert!(
16899 store.pools[pool_idx]
16900 .get_disks_by_key(object)
16901 .load_file_info_versions_exact(&bucket, object)
16902 .await
16903 .expect("post-delete metadata should decode")
16904 .is_none(),
16905 "pool {pool_idx} must not retain source or free-version metadata"
16906 );
16907 }
16908 store
16909 .delete_bucket(&bucket, &DeleteBucketOptions::default())
16910 .await
16911 .expect("shared-tuple bucket should be physically empty");
16912 }
16913
16914 #[cfg(feature = "test-util")]
16915 #[tokio::test]
16916 #[serial_test::serial(storage_class_env)]
16917 async fn multi_pool_same_remote_tuple_single_delete_waits_for_all_sources() {
16918 run_multi_pool_same_remote_tuple_delete_case(false).await;
16919 }
16920
16921 #[cfg(feature = "test-util")]
16922 #[tokio::test]
16923 #[serial_test::serial(storage_class_env)]
16924 async fn multi_pool_same_remote_tuple_batch_delete_waits_for_all_sources() {
16925 run_multi_pool_same_remote_tuple_delete_case(true).await;
16926 }
16927
16928 #[cfg(feature = "test-util")]
16929 #[tokio::test]
16930 #[serial_test::serial(storage_class_env)]
16931 async fn multi_pool_transitioned_delete_persists_one_free_version_per_remote_tuple() {
16932 let temp_dir = tempfile::tempdir().expect("create multi-pool transitioned delete store dir");
16933 let (ctx, store, _shutdown) = without_storage_class_env(build_isolated_test_store(
16934 temp_dir.path(),
16935 "multi-pool-transitioned-delete-owner",
16936 &[4, 4],
16937 ))
16938 .await;
16939 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
16940 let tier_name = "MULTIPOOL-DELETE";
16941 let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
16942 let bucket = "multi-pool-transitioned-delete-owner-bucket";
16943 let object = "archive.bin";
16944 store
16945 .make_bucket(bucket, &MakeBucketOptions::default())
16946 .await
16947 .expect("multi-pool source bucket should be created");
16948
16949 let mut remote_tuples = Vec::new();
16950 for pool_idx in 0..2 {
16951 let mut reader = PutObjReader::from_vec(vec![b'a' + pool_idx as u8; 1024 * 1024]);
16952 let source = store.pools[pool_idx]
16953 .put_object(
16954 bucket,
16955 object,
16956 &mut reader,
16957 &ObjectOptions {
16958 mod_time: Some(OffsetDateTime::now_utc() + time::Duration::seconds(pool_idx as i64)),
16959 ..Default::default()
16960 },
16961 )
16962 .await
16963 .expect("pool-local source should be written");
16964 store.pools[pool_idx]
16965 .transition_object(
16966 bucket,
16967 object,
16968 &ObjectOptions {
16969 transition: TransitionOptions {
16970 status: TRANSITION_PENDING.to_string(),
16971 tier: tier_name.to_string(),
16972 etag: source.etag.clone().expect("pool-local source should have an etag"),
16973 ..Default::default()
16974 },
16975 mod_time: source.mod_time,
16976 ..Default::default()
16977 },
16978 )
16979 .await
16980 .expect("pool-local source should transition");
16981 let transitioned = store.pools[pool_idx]
16982 .get_object_info(bucket, object, &ObjectOptions::default())
16983 .await
16984 .expect("pool-local transitioned source should be readable");
16985 remote_tuples.push((
16986 transitioned.transitioned_object.name.clone(),
16987 transitioned.transitioned_object.version_id.clone(),
16988 ));
16989 }
16990 assert_ne!(remote_tuples[0], remote_tuples[1], "fixture must use distinct remote cleanup tuples");
16991 assert_eq!(backend.object_count().await, 2);
16992
16993 backend.set_remove_failure(true);
16994 store
16995 .delete_object_with_tier_delete_journal(bucket, object, ObjectOptions::default())
16996 .await
16997 .expect("store-level delete should remove both pool copies");
16998 for pool_idx in 0..2 {
16999 let retained = store.pools[pool_idx]
17000 .get_disks_by_key(object)
17001 .load_file_info_versions_exact(bucket, object)
17002 .await
17003 .expect("pool-local post-delete metadata should decode")
17004 .expect("each pool-specific remote tuple must retain a cleanup owner");
17005 assert_eq!(
17006 retained
17007 .versions
17008 .iter()
17009 .chain(retained.free_versions.iter())
17010 .filter(|version| version.tier_free_version())
17011 .count(),
17012 1,
17013 "pool {pool_idx} must retain exactly one free-version owner"
17014 );
17015 assert_eq!(
17016 retained
17017 .versions
17018 .iter()
17019 .filter(|version| !version.tier_free_version())
17020 .count(),
17021 0,
17022 "pool {pool_idx} must not retain a visible source"
17023 );
17024 }
17025 assert_eq!(
17026 tier_delete_journal_count(store.clone()).await,
17027 0,
17028 "ordinary multi-pool delete must not create tier-delete journals"
17029 );
17030 assert_eq!(backend.object_count().await, 2, "failed remote cleanup must retain both remote objects");
17031
17032 backend.set_remove_failure(false);
17033 wait_for_tier_free_version_recovery(store.clone(), &backend, 2).await;
17034 let mut removed = backend.remove_versions().await;
17035 removed.sort();
17036 remote_tuples.sort();
17037 assert_eq!(removed, remote_tuples, "recovery must use each pool's exact remote tuple");
17038 for pool_idx in 0..2 {
17039 assert!(
17040 store.pools[pool_idx]
17041 .get_disks_by_key(object)
17042 .load_file_info_versions_exact(bucket, object)
17043 .await
17044 .expect("post-recovery pool metadata should decode")
17045 .is_none(),
17046 "pool {pool_idx} must not retain source or free-version metadata"
17047 );
17048 }
17049 store
17050 .delete_bucket(bucket, &DeleteBucketOptions::default())
17051 .await
17052 .expect("multi-pool source bucket should be physically empty");
17053 }
17054
17055 #[tokio::test]
17061 #[serial_test::serial(storage_class_env)]
17062 async fn new_with_instance_ctx_threads_context_through_store_graph() {
17063 let temp_dir = tempfile::tempdir().expect("create temp store dir");
17064 let (instance_ctx, store, _shutdown) =
17065 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "instance-ctx-store-graph-test", &[4])).await;
17066
17067 assert!(
17068 Arc::ptr_eq(&store.ctx, &instance_ctx),
17069 "the store must adopt the explicitly passed instance context"
17070 );
17071 for sets in &store.pools {
17072 assert!(
17073 Arc::ptr_eq(sets.instance_ctx(), &instance_ctx),
17074 "every pool's Sets must carry the passed instance context"
17075 );
17076 }
17077 assert_eq!(
17078 instance_ctx.deployment_id(),
17079 Some(store.id),
17080 "the deployment id must land on the passed context and mirror the store id"
17081 );
17082
17083 let registered: Vec<String> = instance_ctx.local_disk_map().read().await.keys().cloned().collect();
17084 assert_eq!(registered.len(), 4, "the passed context must register all four local disks");
17085 let registered_disk_ids = instance_ctx.local_disk_id_map();
17086 let registered_disk_ids = registered_disk_ids.read().await;
17087 assert_eq!(registered_disk_ids.len(), 4, "the passed context must publish all four disk IDs");
17088 for endpoint in registered_disk_ids.values() {
17089 assert!(
17090 registered.contains(endpoint),
17091 "every disk ID in the passed context must resolve to one of its registered endpoints"
17092 );
17093 }
17094 drop(registered_disk_ids);
17095 let bootstrap = crate::runtime::instance::bootstrap_ctx();
17096 assert_ne!(
17097 bootstrap.deployment_id(),
17098 Some(store.id),
17099 "the bootstrap context must not absorb the fresh store's deployment id"
17100 );
17101 let bootstrap_map = bootstrap.local_disk_map();
17102 let bootstrap_map = bootstrap_map.read().await;
17103 for key in ®istered {
17104 assert!(
17105 !bootstrap_map.contains_key(key),
17106 "the bootstrap context must not absorb the fresh store's disks"
17107 );
17108 }
17109 drop(bootstrap_map);
17110 let bootstrap_disk_ids = bootstrap.local_disk_id_map();
17111 let bootstrap_disk_ids = bootstrap_disk_ids.read().await;
17112 for endpoint in bootstrap_disk_ids.values() {
17113 assert!(
17114 !registered.contains(endpoint),
17115 "the bootstrap context must not absorb the fresh store's disk IDs"
17116 );
17117 }
17118 }
17119
17120 #[tokio::test]
17121 #[serial_test::serial(storage_class_env)]
17122 async fn new_with_instance_ctx_applies_default_parity_to_each_real_pool() {
17123 let temp_dir = tempfile::tempdir().expect("create multi-pool store dir");
17124 let (_, store, _shutdown) =
17125 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "pool-parity-regression", &[4, 2])).await;
17126
17127 assert_eq!(store.pools.len(), 2);
17128 assert_eq!(store.pools[0].default_parity_count, 2);
17129 assert_eq!(store.pools[0].disk_set[0].default_parity_count, 2);
17130 assert_eq!(store.pools[1].default_parity_count, 1);
17131 assert_eq!(store.pools[1].disk_set[0].default_parity_count, 1);
17132 }
17133
17134 #[tokio::test]
17139 #[serial_test::serial(storage_class_env)]
17140 async fn two_stores_initialize_their_own_bucket_metadata_sys() {
17141 let temp_a = tempfile::tempdir().expect("create temp store dir a");
17142 let temp_b = tempfile::tempdir().expect("create temp store dir b");
17143 let (ctx_a, store_a, _shutdown_a) =
17144 without_storage_class_env(build_isolated_test_store(temp_a.path(), "bucket-metadata-isolation-a", &[4])).await;
17145 let (ctx_b, store_b, _shutdown_b) =
17146 without_storage_class_env(build_isolated_test_store(temp_b.path(), "bucket-metadata-isolation-b", &[4])).await;
17147
17148 crate::bucket::metadata_sys::init_bucket_metadata_sys(store_a.clone(), Vec::new()).await;
17149 crate::bucket::metadata_sys::init_bucket_metadata_sys(store_b.clone(), Vec::new()).await;
17151
17152 let sys_a = ctx_a
17153 .bucket_metadata_sys()
17154 .expect("store A's context must hold its metadata system");
17155 let sys_b = ctx_b
17156 .bucket_metadata_sys()
17157 .expect("store B's context must hold its metadata system");
17158 assert!(!Arc::ptr_eq(&sys_a, &sys_b), "each store must own a distinct bucket metadata system");
17159 }
17160
17161 #[tokio::test]
17162 #[serial_test::serial(storage_class_env)]
17163 async fn same_name_bucket_recursive_delete_uses_owning_instance_object_lock_state() {
17164 let temp_a = tempfile::tempdir().expect("create temp store dir a");
17165 let temp_b = tempfile::tempdir().expect("create temp store dir b");
17166 let (_ctx_a, store_a, _shutdown_a) =
17167 without_storage_class_env(build_isolated_test_store(temp_a.path(), "delete-scope-a", &[4])).await;
17168 let (_ctx_b, store_b, _shutdown_b) =
17169 without_storage_class_env(build_isolated_test_store(temp_b.path(), "delete-scope-b", &[4])).await;
17170 crate::bucket::metadata_sys::init_bucket_metadata_sys(store_a.clone(), Vec::new()).await;
17171 crate::bucket::metadata_sys::init_bucket_metadata_sys(store_b.clone(), Vec::new()).await;
17172
17173 let bucket = format!("same-name-delete-scope-{}", uuid::Uuid::new_v4());
17174 store_a
17175 .make_bucket(&bucket, &MakeBucketOptions::default())
17176 .await
17177 .expect("create plain bucket in store A");
17178 store_b
17179 .make_bucket(
17180 &bucket,
17181 &MakeBucketOptions {
17182 lock_enabled: true,
17183 ..Default::default()
17184 },
17185 )
17186 .await
17187 .expect("create Object Lock bucket in store B");
17188
17189 for store in [&store_a, &store_b] {
17190 let mut reader = PutObjReader::from_vec(b"payload".to_vec());
17191 store
17192 .put_object(
17193 &bucket,
17194 "prefix/object.bin",
17195 &mut reader,
17196 &ObjectOptions {
17197 versioned: Arc::ptr_eq(store, &store_b),
17198 ..Default::default()
17199 },
17200 )
17201 .await
17202 .expect("put same-name object");
17203 }
17204
17205 store_a
17206 .delete_object(
17207 &bucket,
17208 "prefix",
17209 ObjectOptions {
17210 delete_prefix: true,
17211 ..Default::default()
17212 },
17213 )
17214 .await
17215 .expect("plain store prefix delete should remain allowed");
17216
17217 let err = store_b
17218 .delete_object(
17219 &bucket,
17220 "prefix",
17221 ObjectOptions {
17222 delete_prefix: true,
17223 ..Default::default()
17224 },
17225 )
17226 .await
17227 .expect_err("store B must not consult store A's plain bucket metadata");
17228 assert!(matches!(err, StorageError::InvalidArgument(_, _, _)));
17229 store_b
17230 .get_object_info(&bucket, "prefix/object.bin", &ObjectOptions::default())
17231 .await
17232 .expect("locked store object must survive the rejected prefix delete");
17233 }
17234
17235 #[tokio::test]
17236 #[serial_test::serial(storage_class_env)]
17237 async fn recursive_delete_checks_object_lock_on_every_version_page() {
17238 let temp = tempfile::tempdir().expect("create temp store dir");
17239 let (_ctx, store, _shutdown) =
17240 without_storage_class_env(build_isolated_test_store(temp.path(), "recursive-delete-pagination", &[4])).await;
17241 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
17242
17243 let bucket = format!("recursive-page-{}", uuid::Uuid::new_v4());
17244 store
17245 .make_bucket(&bucket, &MakeBucketOptions::default())
17246 .await
17247 .expect("create plain bucket");
17248 for object in ["prefix/a.bin", "prefix/b.bin"] {
17249 let mut reader = PutObjReader::from_vec(b"plain".to_vec());
17250 store
17251 .put_object(&bucket, object, &mut reader, &ObjectOptions::default())
17252 .await
17253 .expect("write first-page object");
17254 }
17255 let locked_object = "prefix/z-locked.bin";
17256 let mut reader = PutObjReader::from_vec(b"locked".to_vec());
17257 store
17258 .put_object(
17259 &bucket,
17260 locked_object,
17261 &mut reader,
17262 &ObjectOptions {
17263 user_defined: HashMap::from([
17264 (
17265 rustfs_filemeta::metadata_keys::OBJECT_LOCK_MODE.to_string(),
17266 s3s::dto::ObjectLockRetentionMode::COMPLIANCE.to_string(),
17267 ),
17268 (
17269 rustfs_filemeta::metadata_keys::OBJECT_LOCK_RETAIN_UNTIL_DATE.to_string(),
17270 "2099-01-01T00:00:00Z".to_string(),
17271 ),
17272 ]),
17273 ..Default::default()
17274 },
17275 )
17276 .await
17277 .expect("write second-page locked object");
17278
17279 let first_page = store.pools[0].disk_set[0]
17280 .clone()
17281 .inner_list_object_versions_for_recursive_delete(&bucket, "prefix/", None, None, 2)
17282 .await
17283 .expect("list the first recursive-delete validation page");
17284 assert!(first_page.is_truncated, "the fixture must require a continuation page");
17285 assert!(
17286 first_page.objects.iter().all(|object| object.name != locked_object),
17287 "the locked object must not be visible on the first page"
17288 );
17289 let second_page = store.pools[0].disk_set[0]
17290 .clone()
17291 .inner_list_object_versions_for_recursive_delete(
17292 &bucket,
17293 "prefix/",
17294 first_page.next_marker,
17295 first_page.next_version_idmarker,
17296 2,
17297 )
17298 .await
17299 .expect("list the recursive-delete continuation page");
17300 assert!(
17301 second_page.objects.iter().any(|object| object.name == locked_object),
17302 "the continuation page must contain the locked object"
17303 );
17304
17305 let err = store
17306 .delete_object(
17307 &bucket,
17308 "prefix/",
17309 ObjectOptions {
17310 delete_prefix: true,
17311 ..Default::default()
17312 },
17313 )
17314 .await
17315 .expect_err("recursive delete must scan the continuation page and find the locked object");
17316 assert!(
17317 matches!(err, StorageError::PrefixAccessDenied(ref name, ref object) if name == &bucket && object == locked_object),
17318 "unexpected recursive delete error: {err:?}"
17319 );
17320 for object in ["prefix/a.bin", "prefix/b.bin", locked_object] {
17321 store
17322 .get_object_info(&bucket, object, &ObjectOptions::default())
17323 .await
17324 .expect("a rejected recursive delete must leave every page intact");
17325 }
17326 }
17327
17328 #[tokio::test]
17329 #[serial_test::serial(storage_class_env)]
17330 async fn delete_holds_bucket_incarnation_sentinel_through_object_commit() {
17331 let temp = tempfile::tempdir().expect("create temp store dir");
17332 let (_ctx, store, _shutdown) =
17333 without_storage_class_env(build_isolated_test_store(temp.path(), "delete-bucket-incarnation", &[4])).await;
17334 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
17335
17336 let bucket = format!("delete-bucket-incarnation-{}", uuid::Uuid::new_v4());
17337 let object = "same-key.bin";
17338 store
17339 .make_bucket(&bucket, &MakeBucketOptions::default())
17340 .await
17341 .expect("create initial plain bucket");
17342 let old_incarnation = store
17343 .bucket_incarnation_id(&bucket)
17344 .await
17345 .expect("read initial bucket incarnation");
17346 let mut reader = PutObjReader::from_vec(b"old generation".to_vec());
17347 store
17348 .put_object(&bucket, object, &mut reader, &ObjectOptions::default())
17349 .await
17350 .expect("put old-generation object");
17351
17352 let barrier = crate::store::object::DeleteAfterObjectLockSnapshotBarrier::install(&bucket);
17353 let delete_store = Arc::clone(&store);
17354 let delete_bucket = bucket.clone();
17355 let delete = tokio::spawn(async move {
17356 delete_store
17357 .delete_object(&delete_bucket, object, ObjectOptions::default())
17358 .await
17359 });
17360 barrier.wait_until_paused().await;
17361
17362 let recreate_store = Arc::clone(&store);
17363 let recreate_bucket = bucket.clone();
17364 let mut recreate = tokio::spawn(async move {
17365 recreate_store
17366 .delete_bucket(
17367 &recreate_bucket,
17368 &crate::storage_api_contracts::bucket::DeleteBucketOptions {
17369 force: true,
17370 ..Default::default()
17371 },
17372 )
17373 .await?;
17374 recreate_store
17375 .make_bucket(
17376 &recreate_bucket,
17377 &MakeBucketOptions {
17378 lock_enabled: true,
17379 ..Default::default()
17380 },
17381 )
17382 .await?;
17383 let mut reader = PutObjReader::from_vec(b"new protected generation".to_vec());
17384 recreate_store
17385 .put_object(
17386 &recreate_bucket,
17387 object,
17388 &mut reader,
17389 &ObjectOptions {
17390 versioned: true,
17391 ..Default::default()
17392 },
17393 )
17394 .await?;
17395 Ok::<_, Error>(())
17396 });
17397
17398 assert!(
17399 tokio::time::timeout(Duration::from_millis(100), &mut recreate).await.is_err(),
17400 "bucket delete and same-name recreation must wait for the in-flight object delete"
17401 );
17402 barrier.release();
17403 delete
17404 .await
17405 .expect("delete task should join")
17406 .expect("old-generation delete should complete");
17407 tokio::time::timeout(Duration::from_secs(10), recreate)
17408 .await
17409 .expect("bucket recreation should finish after delete releases the sentinel")
17410 .expect("bucket recreation task should join")
17411 .expect("bucket recreation should succeed");
17412
17413 let stale_delete_err = store
17414 .delete_object(
17415 &bucket,
17416 object,
17417 ObjectOptions {
17418 expected_bucket_incarnation_id: Some(old_incarnation),
17419 versioned: true,
17420 ..Default::default()
17421 },
17422 )
17423 .await
17424 .expect_err("an authorization snapshot from the old bucket incarnation must be rejected");
17425 assert!(matches!(stale_delete_err, StorageError::BucketNotFound(name) if name == bucket));
17426
17427 let (_deleted, stale_batch_errors) = store
17428 .delete_objects(
17429 &bucket,
17430 vec![crate::storage_api_contracts::object::ObjectToDelete {
17431 object_name: object.to_string(),
17432 ..Default::default()
17433 }],
17434 ObjectOptions {
17435 expected_bucket_incarnation_id: Some(old_incarnation),
17436 versioned: true,
17437 ..Default::default()
17438 },
17439 )
17440 .await;
17441 assert!(
17442 matches!(stale_batch_errors.as_slice(), [Some(StorageError::BucketNotFound(name))] if name.as_str() == bucket),
17443 "batch delete must reject an authorization snapshot from the old bucket incarnation"
17444 );
17445
17446 store
17447 .get_object_info(
17448 &bucket,
17449 object,
17450 &ObjectOptions {
17451 versioned: true,
17452 ..Default::default()
17453 },
17454 )
17455 .await
17456 .expect("the new locked-bucket object must survive the old-generation delete");
17457 }
17458
17459 #[tokio::test]
17460 #[serial_test::serial(storage_class_env)]
17461 async fn delete_objects_holds_bucket_incarnation_sentinel_through_commit() {
17462 let temp = tempfile::tempdir().expect("create temp store dir");
17463 let (_ctx, store, _shutdown) =
17464 without_storage_class_env(build_isolated_test_store(temp.path(), "batch-delete-bucket-incarnation", &[4])).await;
17465 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
17466
17467 let bucket = format!("batch-incarnation-{}", uuid::Uuid::new_v4());
17468 let object = "same-key.bin";
17469 store
17470 .make_bucket(&bucket, &MakeBucketOptions::default())
17471 .await
17472 .expect("create initial bucket");
17473 let mut reader = PutObjReader::from_vec(b"old generation".to_vec());
17474 store
17475 .put_object(&bucket, object, &mut reader, &ObjectOptions::default())
17476 .await
17477 .expect("put old-generation object");
17478
17479 let barrier = crate::store::object::DeleteAfterObjectLockSnapshotBarrier::install(&bucket);
17480 let delete_store = Arc::clone(&store);
17481 let delete_bucket = bucket.clone();
17482 let delete = tokio::spawn(async move {
17483 delete_store
17484 .delete_objects(
17485 &delete_bucket,
17486 vec![crate::storage_api_contracts::object::ObjectToDelete {
17487 object_name: object.to_string(),
17488 ..Default::default()
17489 }],
17490 ObjectOptions::default(),
17491 )
17492 .await
17493 });
17494 barrier.wait_until_paused().await;
17495
17496 let recreate_store = Arc::clone(&store);
17497 let recreate_bucket = bucket.clone();
17498 let mut recreate = tokio::spawn(async move {
17499 recreate_store
17500 .delete_bucket(
17501 &recreate_bucket,
17502 &crate::storage_api_contracts::bucket::DeleteBucketOptions {
17503 force: true,
17504 ..Default::default()
17505 },
17506 )
17507 .await?;
17508 recreate_store
17509 .make_bucket(&recreate_bucket, &MakeBucketOptions::default())
17510 .await?;
17511 let mut reader = PutObjReader::from_vec(b"new generation".to_vec());
17512 recreate_store
17513 .put_object(&recreate_bucket, object, &mut reader, &ObjectOptions::default())
17514 .await?;
17515 Ok::<_, Error>(())
17516 });
17517
17518 assert!(
17519 tokio::time::timeout(Duration::from_millis(100), &mut recreate).await.is_err(),
17520 "bucket recreation must wait for the in-flight batch delete"
17521 );
17522 barrier.release();
17523 let (_deleted, errors) = delete.await.expect("batch delete task should join");
17524 assert!(
17525 errors.iter().all(Option::is_none),
17526 "old-generation batch delete should complete: {errors:?}"
17527 );
17528 tokio::time::timeout(Duration::from_secs(10), recreate)
17529 .await
17530 .expect("bucket recreation should finish after batch delete releases the sentinel")
17531 .expect("bucket recreation task should join")
17532 .expect("bucket recreation should succeed");
17533 store
17534 .get_object_info(&bucket, object, &ObjectOptions::default())
17535 .await
17536 .expect("new-generation object must survive the old batch delete");
17537 }
17538
17539 #[tokio::test]
17540 #[serial_test::serial(storage_class_env)]
17541 async fn prefix_delete_serializes_with_concurrent_object_lock_metadata_updates() {
17542 let temp = tempfile::tempdir().expect("create temp store dir");
17543 let (_ctx, store, _shutdown) =
17544 without_storage_class_env(build_isolated_test_store(temp.path(), "prefix-delete-lock-metadata", &[4])).await;
17545 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
17546
17547 let cases = [
17548 (
17549 "compliance",
17550 HashMap::from([
17551 ("x-amz-object-lock-mode".to_string(), "COMPLIANCE".to_string()),
17552 ("x-amz-object-lock-retain-until-date".to_string(), "2099-01-01T00:00:00Z".to_string()),
17553 ]),
17554 ),
17555 (
17556 "legal-hold",
17557 HashMap::from([("x-amz-object-lock-legal-hold".to_string(), "ON".to_string())]),
17558 ),
17559 ];
17560
17561 for (case, eval_metadata) in cases {
17562 let bucket = format!("prefix-lock-{case}-{}", uuid::Uuid::new_v4());
17563 let object = "protected-after-scan.bin";
17564 store
17565 .make_bucket(
17566 &bucket,
17567 &MakeBucketOptions {
17568 lock_enabled: true,
17569 ..Default::default()
17570 },
17571 )
17572 .await
17573 .expect("create Object Lock bucket");
17574 let mut reader = PutObjReader::from_vec(b"body".to_vec());
17575 let object_info = store
17576 .put_object(
17577 &bucket,
17578 object,
17579 &mut reader,
17580 &ObjectOptions {
17581 versioned: true,
17582 ..Default::default()
17583 },
17584 )
17585 .await
17586 .expect("write unprotected version");
17587 let version_id = object_info.version_id.expect("versioned PUT should return an ID").to_string();
17588
17589 let barrier = crate::store::object::DeleteAfterObjectLockSnapshotBarrier::install(&bucket);
17590 let delete_store = Arc::clone(&store);
17591 let delete_bucket = bucket.clone();
17592 let delete = tokio::spawn(async move {
17593 delete_store
17594 .delete_object(
17595 &delete_bucket,
17596 object,
17597 ObjectOptions {
17598 delete_prefix: true,
17599 delete_prefix_object: true,
17600 versioned: true,
17601 ..Default::default()
17602 },
17603 )
17604 .await
17605 });
17606 barrier.wait_until_paused().await;
17607
17608 let update_store = Arc::clone(&store);
17609 let update_bucket = bucket.clone();
17610 let mut update = tokio::spawn(async move {
17611 update_store
17612 .put_object_metadata(
17613 &update_bucket,
17614 object,
17615 &ObjectOptions {
17616 version_id: Some(version_id),
17617 versioned: true,
17618 eval_metadata: Some(eval_metadata),
17619 ..Default::default()
17620 },
17621 )
17622 .await
17623 });
17624 assert!(
17625 tokio::time::timeout(Duration::from_millis(100), &mut update).await.is_err(),
17626 "{case} metadata update must wait behind the prefix-delete lifecycle write fence"
17627 );
17628
17629 barrier.release();
17630 delete
17631 .await
17632 .expect("prefix delete task should join")
17633 .expect("prefix delete should remove the unprotected version");
17634 update
17635 .await
17636 .expect("metadata update task should join")
17637 .expect_err("metadata update must not succeed after the version was deleted");
17638 assert!(
17639 store
17640 .get_object_info(
17641 &bucket,
17642 object,
17643 &ObjectOptions {
17644 versioned: true,
17645 ..Default::default()
17646 },
17647 )
17648 .await
17649 .is_err(),
17650 "{case} case must not report a successful protected write followed by deletion"
17651 );
17652 }
17653 }
17654
17655 #[tokio::test]
17656 #[serial_test::serial(storage_class_env)]
17657 async fn put_rejects_same_name_bucket_recreated_after_request_snapshot() {
17658 let temp = tempfile::tempdir().expect("create temp store dir");
17659 let (_ctx, store, _shutdown) =
17660 without_storage_class_env(build_isolated_test_store(temp.path(), "put-bucket-incarnation", &[4])).await;
17661 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
17662
17663 let bucket = format!("put-bucket-incarnation-{}", uuid::Uuid::new_v4());
17664 let object = "object.bin";
17665 store
17666 .make_bucket(&bucket, &MakeBucketOptions::default())
17667 .await
17668 .expect("create initial bucket");
17669 let old_incarnation = store
17670 .bucket_incarnation_id(&bucket)
17671 .await
17672 .expect("read initial bucket incarnation");
17673
17674 let barrier = crate::set_disk::PutObjectCommitBarrier::install(
17675 &bucket,
17676 object,
17677 crate::set_disk::PutObjectCommitPause::BeforeNamespace,
17678 );
17679 let put_store = Arc::clone(&store);
17680 let put_bucket = bucket.clone();
17681 let put = tokio::spawn(async move {
17682 let mut reader = PutObjReader::from_vec(b"old generation".to_vec());
17683 put_store
17684 .put_object(
17685 &put_bucket,
17686 object,
17687 &mut reader,
17688 &ObjectOptions {
17689 versioned: true,
17690 ..Default::default()
17691 },
17692 )
17693 .await
17694 });
17695 barrier.wait_until_paused().await;
17696
17697 store
17698 .delete_bucket(&bucket, &crate::storage_api_contracts::bucket::DeleteBucketOptions::default())
17699 .await
17700 .expect("delete initial empty bucket while PUT is staged");
17701 store
17702 .make_bucket(&bucket, &MakeBucketOptions::default())
17703 .await
17704 .expect("recreate same-name bucket");
17705 let new_incarnation = store
17706 .bucket_incarnation_id(&bucket)
17707 .await
17708 .expect("read recreated bucket incarnation");
17709 assert_ne!(old_incarnation, new_incarnation);
17710
17711 barrier.release();
17712 let err = put
17713 .await
17714 .expect("PUT task should join")
17715 .expect_err("old-generation PUT must not commit into the recreated bucket");
17716 assert!(
17717 matches!(err, StorageError::BucketNotFound(ref name) if name == &bucket),
17718 "unexpected stale PUT error: {err:?}"
17719 );
17720 assert!(
17721 store
17722 .get_object_info(&bucket, object, &ObjectOptions::default())
17723 .await
17724 .is_err(),
17725 "the recreated bucket must not contain the staged old-generation object"
17726 );
17727 }
17728
17729 #[tokio::test]
17730 #[serial_test::serial(storage_class_env)]
17731 async fn exact_version_put_uses_only_its_own_bucket_object_lock_snapshot() {
17732 let target_temp = tempfile::tempdir().expect("create target store dir");
17733 let foreign_temp = tempfile::tempdir().expect("create foreign store dir");
17734 let (target_ctx, target_store, _target_shutdown) =
17735 without_storage_class_env(build_isolated_test_store(target_temp.path(), "put-object-lock-target", &[4])).await;
17736 let (_foreign_ctx, foreign_store, _foreign_shutdown) =
17737 without_storage_class_env(build_isolated_test_store(foreign_temp.path(), "put-object-lock-foreign", &[4])).await;
17738 crate::bucket::metadata_sys::init_bucket_metadata_sys(target_store.clone(), Vec::new()).await;
17739 crate::bucket::metadata_sys::init_bucket_metadata_sys(foreign_store.clone(), Vec::new()).await;
17740
17741 let bucket = format!("put-object-lock-snapshot-{}", uuid::Uuid::new_v4());
17742 let object = "object.bin";
17743 target_store
17744 .make_bucket(
17745 &bucket,
17746 &MakeBucketOptions {
17747 lock_enabled: true,
17748 ..Default::default()
17749 },
17750 )
17751 .await
17752 .expect("create target Object Lock bucket");
17753 foreign_store
17754 .make_bucket(&bucket, &MakeBucketOptions::default())
17755 .await
17756 .expect("create same-name plain bucket in foreign store");
17757 let stale_snapshot = target_store
17758 .object_lock_config_snapshot(&bucket)
17759 .await
17760 .expect("load target snapshot before changing its default retention");
17761
17762 let mut metadata = crate::bucket::metadata_sys::get_in(&target_ctx, &bucket)
17763 .await
17764 .expect("load target bucket metadata")
17765 .as_ref()
17766 .clone();
17767 metadata
17768 .update_config(
17769 crate::bucket::metadata::OBJECT_LOCK_CONFIG,
17770 b"<ObjectLockConfiguration><ObjectLockEnabled>Enabled</ObjectLockEnabled><Rule><DefaultRetention><Mode>COMPLIANCE</Mode><Days>1</Days></DefaultRetention></Rule></ObjectLockConfiguration>".to_vec(),
17771 )
17772 .expect("set default COMPLIANCE metadata");
17773 let update_started = Arc::new(tokio::sync::Barrier::new(2));
17774 let update_ctx = Arc::clone(&target_ctx);
17775 let update_started_task = Arc::clone(&update_started);
17776 let mut update = tokio::spawn(async move {
17777 update_started_task.wait().await;
17778 let _transaction_guard =
17779 crate::bucket::metadata_sys::acquire_bucket_metadata_transaction_lock_in(&update_ctx, &metadata.name).await?;
17780 crate::bucket::metadata_sys::set_bucket_metadata_in(&update_ctx, metadata).await
17781 });
17782 update_started.wait().await;
17783 assert!(
17784 tokio::time::timeout(Duration::from_millis(100), &mut update).await.is_err(),
17785 "an active Object Lock snapshot must hold the metadata revision read fence"
17786 );
17787 drop(stale_snapshot);
17788 update
17789 .await
17790 .expect("Object Lock metadata update task should join")
17791 .expect("persist target Object Lock metadata after snapshot release");
17792
17793 let original_body = b"protected body".to_vec();
17794 let mut original_reader = PutObjReader::from_vec(original_body.clone());
17795 let original = target_store
17796 .put_object(
17797 &bucket,
17798 object,
17799 &mut original_reader,
17800 &ObjectOptions {
17801 versioned: true,
17802 ..Default::default()
17803 },
17804 )
17805 .await
17806 .expect("write protected target version");
17807 let version_id = original
17808 .version_id
17809 .expect("versioned PUT should return a version ID")
17810 .to_string();
17811
17812 let mut replacement = PutObjReader::from_vec(b"replacement".to_vec());
17813 let err = target_store
17814 .put_object(
17815 &bucket,
17816 object,
17817 &mut replacement,
17818 &ObjectOptions {
17819 version_id: Some(version_id.clone()),
17820 versioned: true,
17821 ..Default::default()
17822 },
17823 )
17824 .await
17825 .expect_err("ECStore must load default COMPLIANCE before an exact-version overwrite");
17826 assert!(matches!(err, StorageError::PrefixAccessDenied(_, _)));
17827
17828 let foreign_snapshot = foreign_store
17829 .object_lock_config_snapshot(&bucket)
17830 .await
17831 .expect("load same-name foreign bucket snapshot");
17832 let mut foreign_replacement = PutObjReader::from_vec(b"foreign replacement".to_vec());
17833 target_store
17834 .put_object(
17835 &bucket,
17836 object,
17837 &mut foreign_replacement,
17838 &ObjectOptions {
17839 version_id: Some(version_id.clone()),
17840 versioned: true,
17841 object_lock_config_snapshot: Some(foreign_snapshot),
17842 ..Default::default()
17843 },
17844 )
17845 .await
17846 .expect_err("a snapshot from another store must fail closed");
17847
17848 let mut reader = target_store
17849 .get_object_reader(
17850 &bucket,
17851 object,
17852 None,
17853 HeaderMap::new(),
17854 &ObjectOptions {
17855 version_id: Some(version_id),
17856 versioned: true,
17857 ..Default::default()
17858 },
17859 )
17860 .await
17861 .expect("protected target version should remain readable");
17862 let mut body = Vec::new();
17863 reader
17864 .stream
17865 .read_to_end(&mut body)
17866 .await
17867 .expect("protected body should drain");
17868 assert_eq!(body, original_body);
17869 }
17870
17871 #[cfg(feature = "test-util")]
17872 #[tokio::test]
17873 #[serial_test::serial(storage_class_env)]
17874 async fn object_lock_snapshot_uses_read_quorum_bucket_existence_probe() {
17875 let temp = tempfile::tempdir().expect("create degraded snapshot store dir");
17876 let (ctx, store, _shutdown) = without_storage_class_env(build_isolated_test_store_with_layout(
17877 temp.path(),
17878 "degraded-object-lock-snapshot",
17879 &[(2, 12)],
17880 CancellationToken::new(),
17881 None,
17882 ))
17883 .await;
17884 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
17885
17886 let bucket = format!("degraded-ol-{}", uuid::Uuid::new_v4());
17887 store
17888 .make_bucket(&bucket, &MakeBucketOptions::default())
17889 .await
17890 .expect("create snapshot bucket");
17891 let expected_incarnation = store
17892 .bucket_incarnation_id(&bucket)
17893 .await
17894 .expect("read bucket incarnation before degrading sets");
17895
17896 let mut offline_disks = Vec::new();
17897 for set in store.all_set_disks() {
17898 offline_disks.push(force_set_disk_range_offline_for_test(&set, 6..12).await);
17899 }
17900
17901 let snapshot = store
17902 .object_lock_config_snapshot(&bucket)
17903 .await
17904 .expect("read-quorum bucket existence should admit guarded Object Lock snapshot");
17905 assert!(matches!(
17906 snapshot.state(),
17907 crate::bucket::metadata_sys::ObjectLockConfigState::ConfirmedAbsent
17908 ));
17909 assert!(snapshot.is_valid_for_destructive_put(store.id, &bucket, expected_incarnation));
17910
17911 let current_incarnation = crate::bucket::metadata_sys::get_object_lock_config_and_incarnation_from_disk_in(&ctx, &bucket)
17912 .await
17913 .expect("authoritative metadata read should also survive at read quorum")
17914 .1;
17915 assert_eq!(current_incarnation, expected_incarnation);
17916
17917 drop(offline_disks);
17918 }
17919
17920 #[tokio::test]
17921 #[serial_test::serial(storage_class_env)]
17922 async fn force_create_existing_bucket_preserves_incarnation_and_inflight_request() {
17923 let temp = tempfile::tempdir().expect("create temp store dir");
17924 let (_ctx, store, _shutdown) =
17925 without_storage_class_env(build_isolated_test_store(temp.path(), "force-create-incarnation", &[4])).await;
17926 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
17927
17928 let bucket = format!("force-create-incarnation-{}", uuid::Uuid::new_v4());
17929 store
17930 .make_bucket(&bucket, &MakeBucketOptions::default())
17931 .await
17932 .expect("create initial bucket");
17933 let incarnation = store.bucket_incarnation_id(&bucket).await.expect("read initial incarnation");
17934
17935 store
17936 .make_bucket(
17937 &bucket,
17938 &MakeBucketOptions {
17939 force_create: true,
17940 ..Default::default()
17941 },
17942 )
17943 .await
17944 .expect("idempotent force-create should succeed");
17945 assert_eq!(
17946 store.bucket_incarnation_id(&bucket).await.unwrap(),
17947 incarnation,
17948 "idempotent force-create must not rotate the bucket generation"
17949 );
17950
17951 let mut reader = PutObjReader::from_vec(b"authorized before force-create".to_vec());
17952 store
17953 .put_object(
17954 &bucket,
17955 "inflight.bin",
17956 &mut reader,
17957 &ObjectOptions {
17958 expected_bucket_incarnation_id: Some(incarnation),
17959 ..Default::default()
17960 },
17961 )
17962 .await
17963 .expect("a request authorized before idempotent force-create must still commit");
17964 }
17965
17966 #[tokio::test]
17967 #[serial_test::serial(storage_class_env)]
17968 async fn lock_enabled_create_retries_metadata_intent_without_unlocking() {
17969 let temp = tempfile::tempdir().expect("create temp store dir");
17970 let (ctx, store, _shutdown) =
17971 without_storage_class_env(build_isolated_test_store(temp.path(), "lock-create-intent-retry", &[4])).await;
17972 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
17973
17974 let bucket = format!("lock-create-intent-retry-{}", uuid::Uuid::new_v4());
17975 let mut intent = crate::bucket::metadata::BucketMetadata::new(&bucket);
17976 intent.lock_enabled = true;
17977 crate::bucket::metadata_sys::set_new_bucket_metadata_in(&ctx, intent)
17978 .await
17979 .expect("persist pre-visibility lock intent");
17980 assert!(
17981 store
17982 .peer_sys
17983 .get_bucket_info(&bucket, &crate::storage_api_contracts::bucket::BucketOptions::default())
17984 .await
17985 .is_err(),
17986 "the intent must not make the physical bucket visible"
17987 );
17988
17989 store
17990 .make_bucket(
17991 &bucket,
17992 &MakeBucketOptions {
17993 lock_enabled: true,
17994 ..Default::default()
17995 },
17996 )
17997 .await
17998 .expect("retry lock-enabled creation");
17999 assert!(matches!(
18000 crate::bucket::metadata_sys::get_object_lock_config_state_in(&ctx, &bucket)
18001 .await
18002 .expect("read retried lock state"),
18003 crate::bucket::metadata_sys::ObjectLockConfigState::Configured { .. }
18004 ));
18005 }
18006
18007 #[tokio::test]
18008 #[serial_test::serial(storage_class_env)]
18009 async fn physical_lock_bucket_with_incomplete_metadata_fails_closed() {
18010 let temp = tempfile::tempdir().expect("create temp store dir");
18011 let (ctx, store, _shutdown) =
18012 without_storage_class_env(build_isolated_test_store(temp.path(), "lock-create-incomplete-metadata", &[4])).await;
18013 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
18014
18015 let bucket = format!("lock-create-incomplete-metadata-{}", uuid::Uuid::new_v4());
18016 let mut metadata = crate::bucket::metadata::BucketMetadata::new(&bucket);
18017 metadata.lock_enabled = true;
18018 metadata
18019 .save_with_store(store.clone())
18020 .await
18021 .expect("persist lock metadata before sidecar");
18022 store
18023 .peer_sys
18024 .make_bucket(&bucket, &MakeBucketOptions::default())
18025 .await
18026 .expect("expose physical bucket after metadata phase");
18027
18028 let err = crate::bucket::metadata_sys::get_object_lock_config_state_in(&ctx, &bucket)
18029 .await
18030 .expect_err("missing sidecar must not fabricate an unlocked bucket");
18031 assert!(err.to_string().contains("sidecar is missing"), "unexpected error: {err}");
18032 }
18033
18034 #[tokio::test]
18035 #[serial_test::serial(storage_class_env)]
18036 async fn successful_lock_enabled_create_has_complete_metadata_intent() {
18037 let temp = tempfile::tempdir().expect("create temp store dir");
18038 let (ctx, store, _shutdown) =
18039 without_storage_class_env(build_isolated_test_store(temp.path(), "lock-create-complete-metadata", &[4])).await;
18040 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
18041
18042 let bucket = format!("lock-create-{}", uuid::Uuid::new_v4());
18043 store
18044 .make_bucket(
18045 &bucket,
18046 &MakeBucketOptions {
18047 lock_enabled: true,
18048 ..Default::default()
18049 },
18050 )
18051 .await
18052 .expect("create lock-enabled bucket");
18053 let (metadata, persisted) = crate::bucket::metadata_sys::get_config_from_disk_with_presence_in(&ctx, &bucket)
18054 .await
18055 .expect("read completed metadata intent");
18056 assert!(persisted && metadata.bucket_incarnation_sidecar && metadata.lock_enabled);
18057 assert!(matches!(
18058 crate::bucket::metadata_sys::get_object_lock_config_state_in(&ctx, &bucket)
18059 .await
18060 .expect("read completed lock state"),
18061 crate::bucket::metadata_sys::ObjectLockConfigState::Configured { .. }
18062 ));
18063 }
18064
18065 #[tokio::test]
18066 #[serial_test::serial(storage_class_env)]
18067 async fn lifecycle_expiry_rejects_task_from_recreated_bucket_incarnation() {
18068 let temp = tempfile::tempdir().expect("create temp store dir");
18069 let (_ctx, store, _shutdown) =
18070 without_storage_class_env(build_isolated_test_store(temp.path(), "lifecycle-expiry-incarnation", &[4])).await;
18071 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
18072
18073 let bucket = format!("lifecycle-incarnation-{}", uuid::Uuid::new_v4());
18074 let object = "same-key.bin";
18075 store
18076 .make_bucket(&bucket, &MakeBucketOptions::default())
18077 .await
18078 .expect("create initial bucket");
18079 let old_incarnation = store.bucket_incarnation_id(&bucket).await.expect("read old incarnation");
18080 let mut old_reader = PutObjReader::from_vec(b"old-generation".to_vec());
18081 store
18082 .put_object(&bucket, object, &mut old_reader, &ObjectOptions::default())
18083 .await
18084 .expect("put old-generation object");
18085 let old_object = store
18086 .get_object_info(&bucket, object, &ObjectOptions::default())
18087 .await
18088 .expect("snapshot old-generation object");
18089
18090 store
18091 .delete_object(&bucket, object, ObjectOptions::default())
18092 .await
18093 .expect("delete old-generation object");
18094 store
18095 .delete_bucket(&bucket, &crate::storage_api_contracts::bucket::DeleteBucketOptions::default())
18096 .await
18097 .expect("delete old bucket");
18098 store
18099 .make_bucket(&bucket, &MakeBucketOptions::default())
18100 .await
18101 .expect("recreate bucket");
18102 let mut new_reader = PutObjReader::from_vec(b"new-generation".to_vec());
18103 store
18104 .put_object(&bucket, object, &mut new_reader, &ObjectOptions::default())
18105 .await
18106 .expect("put replacement object");
18107
18108 let applied = crate::bucket::lifecycle::bucket_lifecycle_ops::apply_expiry_on_non_transitioned_objects(
18109 store.clone(),
18110 &old_object,
18111 &crate::bucket::lifecycle::lifecycle::Event {
18112 action: crate::bucket::lifecycle::lifecycle::IlmAction::DeleteAction,
18113 ..Default::default()
18114 },
18115 &crate::bucket::lifecycle::bucket_lifecycle_audit::LcEventSrc::Scanner,
18116 old_incarnation,
18117 )
18118 .await;
18119 assert!(!applied, "an expiry task from the old bucket incarnation must fail closed");
18120
18121 let mut reader = store
18122 .get_object_reader(&bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
18123 .await
18124 .expect("new-generation object must survive stale expiry");
18125 let mut body = Vec::new();
18126 reader.stream.read_to_end(&mut body).await.expect("read replacement object");
18127 assert_eq!(body, b"new-generation");
18128 }
18129
18130 #[tokio::test]
18131 #[serial_test::serial(storage_class_env)]
18132 async fn pre_upgrade_multipart_upload_can_complete_and_abort_in_the_same_bucket_lifetime() {
18133 let temp = tempfile::tempdir().expect("create temp store dir");
18134 let (_ctx, store, _shutdown) =
18135 without_storage_class_env(build_isolated_test_store(temp.path(), "multipart-upgrade-compat", &[4])).await;
18136 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
18137
18138 let bucket = format!("multipart-upgrade-compat-{}", uuid::Uuid::new_v4());
18139 let created = OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("fixed timestamp should be valid");
18140 store
18141 .make_bucket(
18142 &bucket,
18143 &MakeBucketOptions {
18144 created_at: Some(created),
18145 ..Default::default()
18146 },
18147 )
18148 .await
18149 .expect("create bucket");
18150
18151 let complete_object = "complete.bin";
18152 let complete_upload = store.pools[0]
18153 .new_multipart_upload(
18154 &bucket,
18155 complete_object,
18156 &ObjectOptions {
18157 mod_time: Some(created + time::Duration::seconds(1)),
18158 ..Default::default()
18159 },
18160 )
18161 .await
18162 .expect("create a pre-upgrade multipart upload without an incarnation stamp");
18163 let mut part_reader = PutObjReader::from_vec(b"pre-upgrade multipart body".to_vec());
18164 let part = store.pools[0]
18165 .put_object_part(
18166 &bucket,
18167 complete_object,
18168 &complete_upload.upload_id,
18169 1,
18170 &mut part_reader,
18171 &ObjectOptions::default(),
18172 )
18173 .await
18174 .expect("stage a pre-upgrade multipart part");
18175 store
18176 .clone()
18177 .complete_multipart_upload(
18178 &bucket,
18179 complete_object,
18180 &complete_upload.upload_id,
18181 vec![crate::storage_api_contracts::multipart::CompletePart {
18182 part_num: part.part_num,
18183 etag: part.etag,
18184 ..Default::default()
18185 }],
18186 &ObjectOptions::default(),
18187 )
18188 .await
18189 .expect("a pre-upgrade upload from the current bucket lifetime should complete");
18190 store
18191 .get_object_info(&bucket, complete_object, &ObjectOptions::default())
18192 .await
18193 .expect("completed pre-upgrade upload should be visible");
18194
18195 let abort_object = "abort.bin";
18196 let abort_upload = store.pools[0]
18197 .new_multipart_upload(
18198 &bucket,
18199 abort_object,
18200 &ObjectOptions {
18201 mod_time: Some(created + time::Duration::seconds(2)),
18202 ..Default::default()
18203 },
18204 )
18205 .await
18206 .expect("create another pre-upgrade multipart upload");
18207 store
18208 .abort_multipart_upload(&bucket, abort_object, &abort_upload.upload_id, &ObjectOptions::default())
18209 .await
18210 .expect("a pre-upgrade upload from the current bucket lifetime should abort");
18211 let err = store
18212 .get_multipart_info(&bucket, abort_object, &abort_upload.upload_id, &ObjectOptions::default())
18213 .await
18214 .expect_err("aborted pre-upgrade upload should be removed");
18215 assert!(matches!(err, StorageError::InvalidUploadID(..)));
18216 }
18217
18218 #[tokio::test]
18219 #[serial_test::serial(storage_class_env)]
18220 async fn complete_multipart_rejects_legacy_upload_from_recreated_bucket_incarnation() {
18221 let temp = tempfile::tempdir().expect("create temp store dir");
18222 let (_ctx, store, _shutdown) =
18223 without_storage_class_env(build_isolated_test_store(temp.path(), "multipart-bucket-incarnation", &[4])).await;
18224 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
18225
18226 let bucket = format!("multipart-incarnation-{}", uuid::Uuid::new_v4());
18227 let object = "object.bin";
18228 let created = OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("fixed timestamp should be valid");
18229 store
18230 .make_bucket(
18231 &bucket,
18232 &MakeBucketOptions {
18233 created_at: Some(created),
18234 ..Default::default()
18235 },
18236 )
18237 .await
18238 .expect("create initial bucket");
18239 let old_incarnation = store
18240 .bucket_incarnation_id(&bucket)
18241 .await
18242 .expect("read initial bucket incarnation");
18243 let upload = store.pools[0]
18244 .new_multipart_upload(
18245 &bucket,
18246 object,
18247 &ObjectOptions {
18248 mod_time: Some(created + time::Duration::seconds(1)),
18249 ..Default::default()
18250 },
18251 )
18252 .await
18253 .expect("create old-generation legacy multipart upload");
18254
18255 store
18256 .delete_bucket(&bucket, &crate::storage_api_contracts::bucket::DeleteBucketOptions::default())
18257 .await
18258 .expect("delete bucket with an incomplete multipart upload");
18259 store
18260 .make_bucket(
18261 &bucket,
18262 &MakeBucketOptions {
18263 created_at: Some(created + time::Duration::seconds(2)),
18264 ..Default::default()
18265 },
18266 )
18267 .await
18268 .expect("recreate same-name bucket");
18269 let new_incarnation = store
18270 .bucket_incarnation_id(&bucket)
18271 .await
18272 .expect("read recreated bucket incarnation");
18273 assert_ne!(old_incarnation, new_incarnation);
18274
18275 let err = store
18276 .clone()
18277 .complete_multipart_upload(&bucket, object, &upload.upload_id, Vec::new(), &ObjectOptions::default())
18278 .await
18279 .expect_err("an upload initiated in the old bucket incarnation must not commit into the replacement");
18280 assert!(
18281 match &err {
18282 StorageError::BucketNotFound(name) => name == &bucket,
18283 StorageError::InvalidUploadID(name, key, _) => name == &bucket && key == object,
18284 _ => false,
18285 },
18286 "unexpected stale multipart completion error: {err:?}"
18287 );
18288 assert!(
18289 store
18290 .get_object_info(&bucket, object, &ObjectOptions::default())
18291 .await
18292 .is_err(),
18293 "the recreated bucket must not contain the old-generation multipart upload"
18294 );
18295 }
18296
18297 #[tokio::test]
18298 #[serial_test::serial(storage_class_env)]
18299 async fn delete_after_bucket_validation_preserves_bucket_not_found() {
18300 let temp = tempfile::tempdir().expect("create temp store dir");
18301 let (_ctx, store, _shutdown) =
18302 without_storage_class_env(build_isolated_test_store(temp.path(), "delete-missing-bucket", &[4])).await;
18303 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
18304
18305 let bucket = format!("delete-missing-bucket-{}", uuid::Uuid::new_v4());
18306 store
18307 .make_bucket(&bucket, &MakeBucketOptions::default())
18308 .await
18309 .expect("create bucket before validation");
18310 store
18311 .get_bucket_info(&bucket, &crate::storage_api_contracts::bucket::BucketOptions::default())
18312 .await
18313 .expect("request-level bucket validation should succeed");
18314 store
18315 .delete_bucket(&bucket, &crate::storage_api_contracts::bucket::DeleteBucketOptions::default())
18316 .await
18317 .expect("delete bucket after validation");
18318
18319 let err = store
18320 .delete_object(&bucket, "missing.bin", ObjectOptions::default())
18321 .await
18322 .expect_err("post-validation bucket deletion must not become an object-not-found error");
18323
18324 assert!(matches!(err, StorageError::BucketNotFound(name) if name == bucket));
18325 }
18326
18327 #[tokio::test]
18328 #[serial_test::serial(storage_class_env)]
18329 async fn object_lock_metadata_read_error_blocks_every_destructive_delete_shape() {
18330 let temp = tempfile::tempdir().expect("create temp store dir");
18331 let (ctx, store, _shutdown) =
18332 without_storage_class_env(build_isolated_test_store(temp.path(), "delete-config-read-error", &[4])).await;
18333 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
18334
18335 let bucket = format!("delete-config-read-error-{}", uuid::Uuid::new_v4());
18336 store
18337 .make_bucket(&bucket, &MakeBucketOptions::default())
18338 .await
18339 .expect("create bucket");
18340 for object in ["single.bin", "batch.bin", "prefix/child.bin", "lifecycle.bin"] {
18341 let mut reader = PutObjReader::from_vec(b"must survive".to_vec());
18342 store
18343 .put_object(&bucket, object, &mut reader, &ObjectOptions::default())
18344 .await
18345 .expect("put object before metadata corruption");
18346 }
18347
18348 crate::bucket::metadata_sys::inject_object_lock_disk_read_error_in(&ctx, &bucket)
18349 .await
18350 .expect("inject single-delete metadata read failure");
18351
18352 let single_err = store
18353 .delete_object(&bucket, "single.bin", ObjectOptions::default())
18354 .await
18355 .expect_err("single delete must propagate the metadata read failure");
18356 assert!(
18357 single_err
18358 .to_string()
18359 .contains("injected Object Lock metadata disk read failure")
18360 );
18361
18362 crate::bucket::metadata_sys::inject_object_lock_disk_read_error_in(&ctx, &bucket)
18363 .await
18364 .expect("inject batch-delete metadata read failure");
18365 let (_deleted, batch_errs) = store
18366 .delete_objects(
18367 &bucket,
18368 vec![crate::storage_api_contracts::object::ObjectToDelete {
18369 object_name: "batch.bin".to_string(),
18370 ..Default::default()
18371 }],
18372 ObjectOptions::default(),
18373 )
18374 .await;
18375 assert!(
18376 batch_errs[0]
18377 .as_ref()
18378 .is_some_and(|err| err.to_string().contains("injected Object Lock metadata disk read failure"))
18379 );
18380
18381 crate::bucket::metadata_sys::inject_object_lock_disk_read_error_in(&ctx, &bucket)
18382 .await
18383 .expect("inject prefix-delete metadata read failure");
18384 let prefix_err = store
18385 .delete_object(
18386 &bucket,
18387 "prefix",
18388 ObjectOptions {
18389 delete_prefix: true,
18390 ..Default::default()
18391 },
18392 )
18393 .await
18394 .expect_err("prefix delete must propagate the metadata read failure");
18395 assert!(
18396 prefix_err
18397 .to_string()
18398 .contains("injected Object Lock metadata disk read failure")
18399 );
18400
18401 crate::bucket::metadata_sys::inject_object_lock_disk_read_error_in(&ctx, &bucket)
18402 .await
18403 .expect("inject lifecycle-style delete-all metadata read failure");
18404 let lifecycle_err = store
18405 .delete_object(
18406 &bucket,
18407 "lifecycle.bin",
18408 ObjectOptions {
18409 delete_prefix: true,
18410 delete_prefix_object: true,
18411 ..Default::default()
18412 },
18413 )
18414 .await
18415 .expect_err("lifecycle-style delete-all must propagate the metadata read failure");
18416 assert!(
18417 lifecycle_err
18418 .to_string()
18419 .contains("injected Object Lock metadata disk read failure")
18420 );
18421
18422 for object in ["single.bin", "batch.bin", "prefix/child.bin", "lifecycle.bin"] {
18423 store
18424 .get_object_info(&bucket, object, &ObjectOptions::default())
18425 .await
18426 .expect("metadata failure must preserve every object");
18427 }
18428 }
18429
18430 #[cfg(feature = "test-util")]
18431 #[tokio::test]
18432 #[serial_test::serial(storage_class_env)]
18433 async fn legacy_tier_delete_journals_create_redacted_recovery_controls_without_remote_calls() {
18434 let temp_dir = tempfile::tempdir().expect("create legacy journal recovery store dir");
18435 let (ctx, store, _shutdown) =
18436 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "legacy-tier-journal-recovery", &[4])).await;
18437 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
18438
18439 let tier_name = "LEGACY-RECOVERY";
18440 let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
18441 let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
18442 .await
18443 .expect("legacy recovery tier lease should resolve")
18444 .backend_identity();
18445 let fixtures = [
18446 serde_json::json!({
18447 "version": 1,
18448 "obj_name": "legacy/remote-v1",
18449 "version_id": "opaque-v1",
18450 "tier_name": tier_name,
18451 }),
18452 serde_json::json!({
18453 "version": 2,
18454 "obj_name": "legacy/remote-v2",
18455 "version_id": "opaque-v2",
18456 "tier_name": tier_name,
18457 "backend_identity": backend_identity,
18458 }),
18459 ];
18460 let mut journal_paths = Vec::new();
18461 for fixture in &fixtures {
18462 let data = serde_json::to_vec(&fixture).expect("legacy journal fixture should encode");
18463 let entry = crate::bucket::lifecycle::tier_delete_journal::decode_tier_delete_journal_entry(&data)
18464 .expect("legacy journal fixture should decode");
18465 let path = tier_delete_journal_object_name(&entry);
18466 com::save_config(store.clone(), &path, data)
18467 .await
18468 .expect("legacy journal fixture should persist");
18469 journal_paths.push(path);
18470 }
18471 let corrupt_path = format!(
18472 "{TIER_DELETE_JOURNAL_PREFIX}/{}.json",
18473 rustfs_utils::crypto::hex_sha256(b"corrupt legacy tier journal", ToOwned::to_owned)
18474 );
18475 com::save_config(store.clone(), &corrupt_path, b"{corrupt".to_vec())
18476 .await
18477 .expect("corrupt legacy journal fixture should persist");
18478
18479 let (first, concurrent) = tokio::join!(
18480 recover_tier_delete_journal_entries(store.clone(), 100, None),
18481 recover_tier_delete_journal_entries(store.clone(), 100, None),
18482 );
18483 for stats in [first, concurrent] {
18484 let stats = stats.expect("concurrent legacy journal recovery scan should finish");
18485 assert_eq!((stats.scanned, stats.deleted, stats.failed), (3, 0, 0));
18486 }
18487 assert_eq!(tier_delete_journal_count(store.clone()).await, 3);
18488 assert_eq!(backend.remove_count().await, 0, "legacy recovery must not call the remote tier");
18489 assert_eq!(backend.exact_remove_count(), 0, "legacy recovery must not issue exact remote DELETE");
18490 assert!(backend.op_log().await.is_empty(), "legacy recovery must not invoke any backend operation");
18491
18492 let mut first_controls = list_recovery_controls(store.clone(), IlmRecoveryProtocol::TierDeleteJournal, None, 100, None)
18493 .await
18494 .expect("legacy recovery controls should be listable")
18495 .records;
18496 first_controls.sort_by(|left, right| left.control_id.cmp(&right.control_id));
18497 assert_eq!(first_controls.len(), 3);
18498 assert_eq!(
18499 first_controls
18500 .iter()
18501 .filter(|control| control.classification == IlmRecoveryClassification::RetainedAmbiguous)
18502 .count(),
18503 2
18504 );
18505 assert_eq!(
18506 first_controls
18507 .iter()
18508 .filter(|control| control.classification == IlmRecoveryClassification::Corrupt)
18509 .count(),
18510 1
18511 );
18512 for view in &first_controls {
18513 assert_eq!(view.protocol, IlmRecoveryProtocol::TierDeleteJournal);
18514 assert_eq!(view.revision, 1);
18515 assert_eq!(view.attempt_count, 0);
18516 let encoded = serde_json::to_string(view).expect("recovery control view should encode");
18517 for secret in ["legacy/remote-v1", "legacy/remote-v2", "opaque-v1", "opaque-v2", tier_name] {
18518 assert!(!encoded.contains(secret), "recovery control view must redact `{secret}`");
18519 }
18520 let persisted = load_recovery_control(store.clone(), IlmRecoveryProtocol::TierDeleteJournal, &view.control_id)
18521 .await
18522 .expect("legacy recovery control should load");
18523 match view.source_schema.as_str() {
18524 "rustfs-tier-delete-journal-v1" => {
18525 assert_eq!(persisted.control.identity.record_class, "tier_delete_journal_v1");
18526 assert_eq!(view.last_error_code, IlmRecoveryErrorCode::RemoteVersionUnknown);
18527 }
18528 "rustfs-tier-delete-journal-v2" => {
18529 assert_eq!(persisted.control.identity.record_class, "tier_delete_journal_v2");
18530 assert_eq!(view.last_error_code, IlmRecoveryErrorCode::RemoteVersionUnknown);
18531 }
18532 "rustfs-tier-delete-journal-unknown" => {
18533 assert_eq!(persisted.control.identity.record_class, "tier_delete_journal_corrupt");
18534 assert_eq!(view.last_error_code, IlmRecoveryErrorCode::SourceCorrupt);
18535 }
18536 schema => panic!("unexpected legacy recovery source schema: {schema}"),
18537 }
18538 }
18539
18540 let creator_sha256 = rustfs_utils::crypto::hex_sha256(b"legacy-export-actor", ToOwned::to_owned);
18541 let mut created_exports = Vec::new();
18542 for exportable in first_controls
18543 .iter()
18544 .filter(|control| control.classification == IlmRecoveryClassification::RetainedAmbiguous)
18545 {
18546 let observation = inspect_recovery_export_observation(store.clone(), &exportable.control_id)
18547 .await
18548 .expect("fresh legacy recovery observation should be exportable");
18549 let created = create_recovery_export(store.clone(), &observation, &creator_sha256)
18550 .await
18551 .expect("legacy recovery export should be created exactly once");
18552 assert!(!created.replayed);
18553 let loaded = load_recovery_export(store.clone(), &created.export_id)
18554 .await
18555 .expect("created legacy recovery export should load");
18556 assert_eq!(loaded.encoded, created.encoded, "export readback must preserve the exact committed bytes");
18557 let replayed = create_recovery_export(store.clone(), &observation, &creator_sha256)
18558 .await
18559 .expect("the same observed generation should replay its immutable export");
18560 assert!(replayed.replayed);
18561 assert_eq!(replayed.encoded, created.encoded);
18562 created_exports.push(created);
18563 }
18564 assert_eq!(created_exports.len(), 2, "both v1 and v2 legacy journals must have an export path");
18565
18566 let corrupt_export_id = &created_exports[0].export_id;
18567 let export_path = recovery_export_record_object_name(IlmRecoveryProtocol::TierDeleteJournal, corrupt_export_id)
18568 .expect("export path should build");
18569 com::save_config(store.clone(), &export_path, Vec::new())
18570 .await
18571 .expect("zero-byte corruption fixture should persist");
18572 let corrupt_export = load_recovery_export(store.clone(), corrupt_export_id)
18573 .await
18574 .expect_err("an existing zero-byte export must fail closed");
18575 assert!(!matches!(corrupt_export, Error::ConfigNotFound));
18576
18577 com::save_config(
18578 store.clone(),
18579 &journal_paths[0],
18580 serde_json::to_vec_pretty(&fixtures[0]).expect("rewritten legacy journal fixture should encode"),
18581 )
18582 .await
18583 .expect("equivalent legacy journal rewrite should persist");
18584 let second = recover_tier_delete_journal_entries(store.clone(), 100, None)
18585 .await
18586 .expect("repeated legacy journal recovery scan should finish");
18587 assert_eq!((second.scanned, second.deleted, second.failed), (3, 0, 0));
18588 let mut second_controls = list_recovery_controls(store, IlmRecoveryProtocol::TierDeleteJournal, None, 100, None)
18589 .await
18590 .expect("repeated legacy recovery controls should remain listable")
18591 .records;
18592 second_controls.sort_by(|left, right| left.control_id.cmp(&right.control_id));
18593 assert_eq!(second_controls, first_controls, "repeated scans must not reset durable controls");
18594 assert_eq!(backend.remove_count().await, 0, "repeated recovery must remain remote-call free");
18595 assert_eq!(backend.exact_remove_count(), 0);
18596 assert!(
18597 backend.op_log().await.is_empty(),
18598 "repeated recovery must not invoke any backend operation"
18599 );
18600 }
18601
18602 #[cfg(feature = "test-util")]
18603 #[tokio::test]
18604 #[serial_test::serial(storage_class_env)]
18605 async fn legacy_recovery_disposition_removes_only_local_journals_and_replays() {
18606 Box::pin(legacy_recovery_disposition_removes_only_local_journals_and_replays_case()).await;
18607 }
18608
18609 #[cfg(feature = "test-util")]
18610 async fn legacy_recovery_disposition_removes_only_local_journals_and_replays_case() {
18611 let temp_dir = tempfile::tempdir().expect("create legacy disposition store dir");
18612 let (ctx, store, _shutdown) =
18613 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "legacy-recovery-disposition", &[4])).await;
18614 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
18615
18616 let tier_name = "LEGACY-DISPOSITION";
18617 let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
18618 let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
18619 .await
18620 .expect("legacy disposition tier lease should resolve")
18621 .backend_identity();
18622 let fixtures = [
18623 serde_json::json!({
18624 "version": 1,
18625 "obj_name": "legacy/disposition-v1",
18626 "version_id": "opaque-disposition-v1",
18627 "tier_name": tier_name,
18628 }),
18629 serde_json::json!({
18630 "version": 2,
18631 "obj_name": "legacy/disposition-v2",
18632 "version_id": "opaque-disposition-v2",
18633 "tier_name": tier_name,
18634 "backend_identity": backend_identity,
18635 }),
18636 ];
18637 let mut journal_paths = Vec::new();
18638 for fixture in &fixtures {
18639 let data = serde_json::to_vec(fixture).expect("legacy disposition fixture should encode");
18640 let entry = crate::bucket::lifecycle::tier_delete_journal::decode_tier_delete_journal_entry(&data)
18641 .expect("legacy disposition fixture should decode");
18642 let path = tier_delete_journal_object_name(&entry);
18643 com::save_config(store.clone(), &path, data)
18644 .await
18645 .expect("legacy disposition fixture should persist");
18646 journal_paths.push(path);
18647 }
18648
18649 let recovered = recover_tier_delete_journal_entries(store.clone(), 100, None)
18650 .await
18651 .expect("legacy disposition recovery scan should finish");
18652 assert_eq!((recovered.scanned, recovered.deleted, recovered.failed), (2, 0, 0));
18653 assert_eq!(tier_delete_journal_count(store.clone()).await, 2);
18654
18655 let mut controls = list_recovery_controls(
18656 store.clone(),
18657 IlmRecoveryProtocol::TierDeleteJournal,
18658 Some(IlmRecoveryClassification::RetainedAmbiguous),
18659 100,
18660 None,
18661 )
18662 .await
18663 .expect("legacy disposition controls should be listable")
18664 .records;
18665 controls.sort_by(|left, right| left.control_id.cmp(&right.control_id));
18666 assert_eq!(controls.len(), 2, "both legacy schemas must support disposition");
18667
18668 let actor_sha256 = rustfs_utils::crypto::hex_sha256(b"legacy-disposition-actor", ToOwned::to_owned);
18669 let wrong_actor_sha256 = rustfs_utils::crypto::hex_sha256(b"different-disposition-actor", ToOwned::to_owned);
18670 let wrong_export_sha256 = "ff".repeat(32);
18671 for (index, control) in controls.iter().enumerate() {
18672 let observation = inspect_recovery_export_observation(store.clone(), &control.control_id)
18673 .await
18674 .expect("legacy disposition source should be observable");
18675 let export = create_recovery_export(store.clone(), &observation, &actor_sha256)
18676 .await
18677 .expect("legacy disposition export should persist");
18678 let confirmed_at_unix_nanos = i64::try_from(OffsetDateTime::now_utc().unix_timestamp_nanos())
18679 .expect("legacy disposition timestamp should fit i64");
18680
18681 if index == 0 {
18682 let wrong_hash = Box::pin(execute_recovery_disposition(
18683 store.clone(),
18684 &observation,
18685 &export.export_id,
18686 &wrong_export_sha256,
18687 &actor_sha256,
18688 confirmed_at_unix_nanos,
18689 ))
18690 .await
18691 .expect_err("a mismatched export checksum must fail before local deletion");
18692 assert_eq!(wrong_hash, Error::PreconditionFailed);
18693 assert_eq!(tier_delete_journal_count(store.clone()).await, 2);
18694 }
18695
18696 let dry_run = dry_run_recovery_disposition(
18697 store.clone(),
18698 &observation,
18699 &export.export_id,
18700 &export.content_sha256,
18701 &actor_sha256,
18702 confirmed_at_unix_nanos,
18703 )
18704 .await
18705 .expect("legacy disposition dry-run should validate exact local state");
18706 assert_eq!(dry_run.source_copy_count, observation.source_generation.copies.len());
18707 assert_eq!(
18708 tier_delete_journal_count(store.clone()).await,
18709 fixtures.len() - index,
18710 "dry-run must not delete a legacy journal"
18711 );
18712 assert!(
18713 matches!(
18714 load_recovery_disposition(store.clone(), IlmRecoveryProtocol::TierDeleteJournal, &dry_run.disposition_id,)
18715 .await,
18716 Err(Error::ConfigNotFound)
18717 ),
18718 "dry-run must not persist a disposition record"
18719 );
18720
18721 if index == 0 {
18722 inject_recovery_disposition_crash_once(RecoveryDispositionCrashStage::AfterLocalDelete);
18723 Box::pin(execute_recovery_disposition(
18724 store.clone(),
18725 &observation,
18726 &export.export_id,
18727 &export.content_sha256,
18728 &actor_sha256,
18729 confirmed_at_unix_nanos,
18730 ))
18731 .await
18732 .expect_err("the injected crash must stop after local delete commits");
18733 assert_eq!(tier_delete_journal_count(store.clone()).await, 1);
18734 let interrupted =
18735 load_recovery_disposition(store.clone(), IlmRecoveryProtocol::TierDeleteJournal, &dry_run.disposition_id)
18736 .await
18737 .expect("the applying disposition must survive the post-delete crash");
18738 assert_eq!(interrupted.disposition.state, IlmRecoveryDispositionState::Applying);
18739 assert!(
18740 interrupted.disposition.confirmed_absent.is_empty(),
18741 "the crash must occur before absence progress is persisted"
18742 );
18743 } else {
18744 inject_recovery_disposition_crash_once(RecoveryDispositionCrashStage::AfterControlAbandon);
18745 Box::pin(execute_recovery_disposition(
18746 store.clone(),
18747 &observation,
18748 &export.export_id,
18749 &export.content_sha256,
18750 &actor_sha256,
18751 confirmed_at_unix_nanos,
18752 ))
18753 .await
18754 .expect_err("the injected crash must stop after control abandonment commits");
18755 let interrupted =
18756 load_recovery_disposition(store.clone(), IlmRecoveryProtocol::TierDeleteJournal, &dry_run.disposition_id)
18757 .await
18758 .expect("the applying disposition must survive the post-control crash");
18759 assert_eq!(interrupted.disposition.state, IlmRecoveryDispositionState::Applying);
18760 assert_eq!(
18761 interrupted.disposition.confirmed_absent.len(),
18762 interrupted.disposition.identity.source_generation.copies.len()
18763 );
18764
18765 let abandoned =
18766 load_recovery_control(store.clone(), IlmRecoveryProtocol::TierDeleteJournal, &observation.control_id)
18767 .await
18768 .expect("the abandoned control must survive the injected crash");
18769 let exact_abandoned = abandoned.control.encode().expect("the exact abandoned control should encode");
18770 let mut wrong_history = abandoned.control;
18771 wrong_history.last_error_code = IlmRecoveryErrorCode::CleanupFailed;
18772 let control_path = recovery_control_record_object_name(observation.protocol, &observation.control_id)
18773 .expect("control path should remain canonical");
18774 com::save_config(
18775 store.clone(),
18776 &control_path,
18777 wrong_history
18778 .encode()
18779 .expect("the alternate valid control history should encode"),
18780 )
18781 .await
18782 .expect("the alternate control history fixture should persist");
18783 let wrong_history_err = Box::pin(execute_recovery_disposition(
18784 store.clone(),
18785 &observation,
18786 &export.export_id,
18787 &export.content_sha256,
18788 &actor_sha256,
18789 confirmed_at_unix_nanos + 1,
18790 ))
18791 .await
18792 .expect_err("a different abandoned control history must not bridge to completion");
18793 assert_eq!(wrong_history_err, Error::PreconditionFailed);
18794 com::save_config(store.clone(), &control_path, exact_abandoned)
18795 .await
18796 .expect("the exact abandoned control fixture should be restored");
18797 }
18798
18799 let replay_confirmed_at_unix_nanos = confirmed_at_unix_nanos + 2;
18800 let executed = Box::pin(execute_recovery_disposition(
18801 store.clone(),
18802 &observation,
18803 &export.export_id,
18804 &export.content_sha256,
18805 &actor_sha256,
18806 replay_confirmed_at_unix_nanos,
18807 ))
18808 .await
18809 .expect("a later request must resume and complete the interrupted disposition");
18810 assert_eq!(executed.state, IlmRecoveryDispositionState::Completed);
18811 assert_eq!(executed.outcome, IlmRecoveryDispositionExecutionOutcome::Completed);
18812 assert_eq!(executed.confirmed_absent_copy_count, executed.source_copy_count);
18813 assert_eq!(tier_delete_journal_count(store.clone()).await, fixtures.len() - index - 1);
18814 assert!(matches!(
18815 com::read_config(store.clone(), &observation.canonical_source_path).await,
18816 Err(Error::ConfigNotFound)
18817 ));
18818 if index == 0 {
18819 let untouched = journal_paths
18820 .iter()
18821 .find(|path| *path != &observation.canonical_source_path)
18822 .expect("the other legacy journal should remain");
18823 com::read_config(store.clone(), untouched)
18824 .await
18825 .expect("disposition must not remove a different legacy journal");
18826 }
18827
18828 let abandoned = load_recovery_control(store.clone(), IlmRecoveryProtocol::TierDeleteJournal, &observation.control_id)
18829 .await
18830 .expect("abandoned recovery control should remain inspectable");
18831 assert_eq!(abandoned.control.classification, IlmRecoveryClassification::Abandoned);
18832 assert_eq!(abandoned.control.revision, observation.control_revision + 1);
18833 assert_eq!(abandoned.control.observed_source_generation, observation.source_generation);
18834
18835 let persisted =
18836 load_recovery_disposition(store.clone(), IlmRecoveryProtocol::TierDeleteJournal, &executed.disposition_id)
18837 .await
18838 .expect("completed disposition should remain durable");
18839 assert_eq!(persisted.disposition.state, IlmRecoveryDispositionState::Completed);
18840
18841 let replayed = Box::pin(execute_recovery_disposition(
18842 store.clone(),
18843 &observation,
18844 &export.export_id,
18845 &export.content_sha256,
18846 &actor_sha256,
18847 replay_confirmed_at_unix_nanos + 1,
18848 ))
18849 .await
18850 .expect("same actor should replay the completed disposition");
18851 assert_eq!(replayed.state, IlmRecoveryDispositionState::Completed);
18852 assert_eq!(replayed.outcome, IlmRecoveryDispositionExecutionOutcome::Replayed);
18853
18854 let wrong_actor = Box::pin(execute_recovery_disposition(
18855 store.clone(),
18856 &observation,
18857 &export.export_id,
18858 &export.content_sha256,
18859 &wrong_actor_sha256,
18860 replay_confirmed_at_unix_nanos + 2,
18861 ))
18862 .await
18863 .expect_err("a different actor must not replay a completed disposition");
18864 assert_eq!(wrong_actor, Error::PreconditionFailed);
18865
18866 assert!(
18867 !Box::pin(garbage_collect_completed_recovery_disposition(
18868 store.clone(),
18869 &persisted,
18870 persisted.disposition.retain_until_unix_nanos - 1,
18871 ))
18872 .await
18873 .expect("completed disposition should remain before retention expires")
18874 );
18875 assert!(
18876 Box::pin(garbage_collect_completed_recovery_disposition(
18877 store.clone(),
18878 &persisted,
18879 persisted.disposition.retain_until_unix_nanos,
18880 ))
18881 .await
18882 .expect("expired completed disposition should be garbage collected")
18883 );
18884 assert!(matches!(
18885 load_recovery_disposition(store.clone(), IlmRecoveryProtocol::TierDeleteJournal, &executed.disposition_id).await,
18886 Err(Error::ConfigNotFound)
18887 ));
18888 assert_eq!(backend.remove_count().await, 0, "legacy disposition must not call the remote tier");
18889 assert_eq!(backend.exact_remove_count(), 0, "legacy disposition must not issue exact remote DELETE");
18890 assert!(
18891 backend.op_log().await.is_empty(),
18892 "legacy disposition must not invoke any backend operation"
18893 );
18894 }
18895 }
18896
18897 #[cfg(feature = "test-util")]
18898 #[tokio::test]
18899 #[serial_test::serial(storage_class_env)]
18900 async fn v4_tier_delete_journal_recovery_is_exact_and_store_scoped() {
18901 let temp_a = tempfile::tempdir().expect("create temp store dir a");
18902 let temp_b = tempfile::tempdir().expect("create temp store dir b");
18903 let (ctx_a, store_a, _shutdown_a) =
18904 without_storage_class_env(build_isolated_test_store(temp_a.path(), "tier-journal-recovery-a", &[4])).await;
18905 let (ctx_b, store_b, _shutdown_b) =
18906 without_storage_class_env(build_isolated_test_store(temp_b.path(), "tier-journal-recovery-b", &[4])).await;
18907 crate::bucket::metadata_sys::init_bucket_metadata_sys(store_a.clone(), Vec::new()).await;
18908 crate::bucket::metadata_sys::init_bucket_metadata_sys(store_b.clone(), Vec::new()).await;
18909
18910 let tier_a = "JOURNAL-A";
18911 let tier_b = "JOURNAL-B";
18912 let backend_a = register_mock_tier(&ctx_a.tier_config_mgr(), tier_a).await;
18913 let backend_b = register_mock_tier(&ctx_b.tier_config_mgr(), tier_b).await;
18914 let identity_a = TierConfigMgr::acquire_operation_lease(&ctx_a.tier_config_mgr(), tier_a)
18915 .await
18916 .expect("store A tier lease should resolve")
18917 .backend_identity();
18918 let identity_b = TierConfigMgr::acquire_operation_lease(&ctx_b.tier_config_mgr(), tier_b)
18919 .await
18920 .expect("store B tier lease should resolve")
18921 .backend_identity();
18922 let entry_a = Jentry {
18923 persisted_version: 0,
18924 obj_name: "remote-a".to_string(),
18925 version_id: "version-a".to_string(),
18926 tier_name: tier_a.to_string(),
18927 backend_identity: Some(identity_a),
18928 version_id_exact: true,
18929 version_state: rustfs_filemeta::TransitionVersionState::Exact,
18930 state: crate::bucket::lifecycle::tier_sweeper::TierDeleteJournalState::Committed,
18931 source: None,
18932 dispatch: None,
18933 };
18934 let entry_b = Jentry {
18935 persisted_version: 0,
18936 obj_name: "remote-b".to_string(),
18937 version_id: "version-b".to_string(),
18938 tier_name: tier_b.to_string(),
18939 backend_identity: Some(identity_b),
18940 version_id_exact: true,
18941 version_state: rustfs_filemeta::TransitionVersionState::Exact,
18942 state: crate::bucket::lifecycle::tier_sweeper::TierDeleteJournalState::Committed,
18943 source: None,
18944 dispatch: None,
18945 };
18946 persist_tier_delete_journal_entry(store_a.clone(), &entry_a)
18947 .await
18948 .expect("store A v4 compatibility fixture should persist");
18949 persist_tier_delete_journal_entry(store_b.clone(), &entry_b)
18950 .await
18951 .expect("store B v4 compatibility fixture should persist");
18952 let entry_a_v3 = Jentry {
18953 obj_name: "remote-a-v3".to_string(),
18954 version_id: "version-a-v3".to_string(),
18955 ..entry_a.clone()
18956 };
18957 let mut raw_v3: serde_json::Value = serde_json::from_slice(
18958 &encode_tier_delete_journal_entry(&entry_a_v3).expect("store A v3 fixture should encode from the v4 shape"),
18959 )
18960 .expect("store A v3 fixture should be JSON");
18961 raw_v3["version"] = serde_json::json!(3);
18962 raw_v3
18963 .as_object_mut()
18964 .expect("store A v3 fixture should be an object")
18965 .remove("version_state");
18966 let path_a_v3 = tier_delete_journal_object_name(&entry_a_v3);
18967 com::save_config(
18968 store_a.clone(),
18969 &path_a_v3,
18970 serde_json::to_vec(&raw_v3).expect("store A raw v3 fixture should encode"),
18971 )
18972 .await
18973 .expect("store A raw v3 compatibility fixture should persist");
18974 let path_a = tier_delete_journal_object_name(&entry_a);
18975 let path_b = tier_delete_journal_object_name(&entry_b);
18976 let recovered_a = recover_tier_delete_journal_entries(store_a.clone(), 100, None)
18977 .await
18978 .expect("store A recovery scan should finish");
18979 let recovered_b = recover_tier_delete_journal_entries(store_b.clone(), 100, None)
18980 .await
18981 .expect("store B recovery scan should finish");
18982 assert_eq!((recovered_a.scanned, recovered_a.deleted, recovered_a.failed), (2, 2, 0));
18983 assert_eq!((recovered_b.scanned, recovered_b.deleted, recovered_b.failed), (1, 1, 0));
18984 let mut removed_a = backend_a.remove_versions().await;
18985 removed_a.sort();
18986 assert_eq!(
18987 removed_a,
18988 vec![
18989 ("remote-a".to_string(), "version-a".to_string()),
18990 ("remote-a-v3".to_string(), "version-a-v3".to_string()),
18991 ]
18992 );
18993 assert_eq!(backend_b.remove_versions().await, vec![("remote-b".to_string(), "version-b".to_string())]);
18994 assert!(matches!(com::read_config(store_a.clone(), &path_a).await, Err(Error::ConfigNotFound)));
18995 assert!(matches!(com::read_config(store_a.clone(), &path_a_v3).await, Err(Error::ConfigNotFound)));
18996 assert!(matches!(com::read_config(store_b.clone(), &path_b).await, Err(Error::ConfigNotFound)));
18997 assert_eq!(tier_delete_journal_count(store_a).await, 0);
18998 assert_eq!(tier_delete_journal_count(store_b).await, 0);
18999 }
19000
19001 #[cfg(feature = "test-util")]
19002 #[tokio::test]
19003 #[serial_test::serial(storage_class_env)]
19004 async fn v5_tier_delete_recovery_retains_live_sources_and_aborts_only_prepared() {
19005 let temp_dir = tempfile::tempdir().expect("create v5 live-source store dir");
19006 let (ctx, store, _shutdown) =
19007 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "v5-live-source-recovery", &[4])).await;
19008 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
19009
19010 let tier_name = "V5-LIVE-SOURCE";
19011 let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
19012 let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
19013 .await
19014 .expect("v5 live-source tier lease should resolve")
19015 .backend_identity();
19016 let bucket = "v5-live-source-recovery-bucket";
19017 store
19018 .make_bucket(bucket, &MakeBucketOptions::default())
19019 .await
19020 .expect("v5 live-source bucket should be created");
19021
19022 let mut paths = Vec::new();
19023 for (index, state) in [
19024 TierDeleteJournalState::Prepared,
19025 TierDeleteJournalState::Dispatched,
19026 TierDeleteJournalState::Committed,
19027 ]
19028 .into_iter()
19029 .enumerate()
19030 {
19031 let object = format!("live-{index}.bin");
19032 let remote_version = uuid::Uuid::new_v4().to_string();
19033 backend.set_put_remote_version(Some(remote_version)).await;
19034 let mut reader = PutObjReader::from_vec(format!("v5 live source {index}").into_bytes());
19035 let source = store
19036 .put_object(bucket, &object, &mut reader, &ObjectOptions::default())
19037 .await
19038 .expect("v5 live source should be written");
19039 store
19040 .transition_object(
19041 bucket,
19042 &object,
19043 &ObjectOptions {
19044 transition: TransitionOptions {
19045 status: TRANSITION_PENDING.to_string(),
19046 tier: tier_name.to_string(),
19047 etag: source.etag.clone().expect("v5 live source should have an ETag"),
19048 ..Default::default()
19049 },
19050 version_id: source.version_id.map(|version| version.to_string()),
19051 mod_time: source.mod_time,
19052 ..Default::default()
19053 },
19054 )
19055 .await
19056 .expect("v5 live source should transition");
19057 let transitioned = store
19058 .get_object_info(
19059 bucket,
19060 &object,
19061 &ObjectOptions {
19062 no_lock: true,
19063 metadata_cache_safe: false,
19064 ..Default::default()
19065 },
19066 )
19067 .await
19068 .expect("v5 transitioned source should be readable");
19069 let mut entry = transitioned_delete_journal_entry_for_source(None, false, false, bucket, &object, &transitioned)
19070 .expect("v5 transitioned source should produce a stable journal identity");
19071 entry.persisted_version = 5;
19072 entry.backend_identity = Some(backend_identity);
19073 entry.state = state;
19074 entry.dispatch = None;
19075 let path = tier_delete_journal_object_name(&entry);
19076 com::save_config(
19077 store.clone(),
19078 &path,
19079 encode_tier_delete_journal_entry(&entry).expect("raw v5 live-source journal should encode"),
19080 )
19081 .await
19082 .expect("raw v5 live-source journal should persist");
19083 paths.push((state, path));
19084 }
19085
19086 let stats = recover_tier_delete_journal_entries(store.clone(), 100, None)
19087 .await
19088 .expect("v5 live-source recovery should finish");
19089 assert_eq!((stats.scanned, stats.deleted, stats.failed), (3, 1, 2));
19090 assert_eq!(backend.remove_count().await, 0, "a live source must block every remote DELETE");
19091 for (state, path) in paths {
19092 let observed = com::read_config(store.clone(), &path).await;
19093 if state == TierDeleteJournalState::Prepared {
19094 assert!(matches!(observed, Err(Error::ConfigNotFound)), "Prepared v5 evidence should abort");
19095 } else {
19096 assert!(observed.is_ok(), "{state:?} v5 evidence must remain while its exact source exists");
19097 }
19098 }
19099 }
19100
19101 #[cfg(feature = "test-util")]
19102 #[tokio::test]
19103 #[serial_test::serial(storage_class_env)]
19104 async fn v5_tier_delete_recovery_replays_all_crash_states_without_sources() {
19105 let temp_dir = tempfile::tempdir().expect("create v5 crash-replay store dir");
19106 let (ctx, store, _shutdown) =
19107 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "v5-crash-replay", &[4])).await;
19108 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
19109
19110 let tier_name = "V5-CRASH-REPLAY";
19111 let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
19112 let lease = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
19113 .await
19114 .expect("v5 crash-replay tier lease should resolve");
19115 let backend_identity = lease.backend_identity();
19116 let bucket = "v5-crash-replay-bucket";
19117 store
19118 .make_bucket(bucket, &MakeBucketOptions::default())
19119 .await
19120 .expect("v5 crash-replay bucket should be created");
19121
19122 for (index, state, seed_remote) in [
19123 (0, TierDeleteJournalState::Prepared, true),
19124 (1, TierDeleteJournalState::Dispatched, true),
19125 (2, TierDeleteJournalState::Committed, true),
19126 (3, TierDeleteJournalState::Committed, false),
19129 ] {
19130 let remote_version = uuid::Uuid::new_v4().to_string();
19131 let mut entry = synthetic_v6_dispatch_entry(
19132 bucket,
19133 &format!("missing-{index}.bin"),
19134 tier_name,
19135 backend_identity,
19136 &remote_version,
19137 );
19138 entry.persisted_version = 5;
19139 entry.state = state;
19140 entry.dispatch = None;
19141 if seed_remote {
19142 backend.set_put_remote_version(Some(remote_version)).await;
19143 let body = bytes::Bytes::from(format!("v5 remote candidate {index}"));
19144 lease
19145 .put(
19146 &entry.obj_name,
19147 ReaderImpl::Body(body.clone()),
19148 i64::try_from(body.len()).expect("v5 candidate length should fit i64"),
19149 )
19150 .await
19151 .expect("v5 remote candidate should be seeded");
19152 }
19153 let path = tier_delete_journal_object_name(&entry);
19154 com::save_config(
19155 store.clone(),
19156 &path,
19157 encode_tier_delete_journal_entry(&entry).expect("raw v5 crash journal should encode"),
19158 )
19159 .await
19160 .expect("raw v5 crash journal should persist");
19161 }
19162
19163 let stats = recover_tier_delete_journal_entries(store.clone(), 100, None)
19164 .await
19165 .expect("v5 crash-state recovery should finish");
19166 assert_eq!((stats.scanned, stats.deleted, stats.failed), (4, 4, 0));
19167 assert_eq!(tier_delete_journal_count(store).await, 0);
19168 assert_eq!(backend.object_count().await, 0);
19169 assert_eq!(backend.exact_remove_count(), 4, "every v5 state must use exact-version cleanup");
19170 }
19171
19172 #[cfg(feature = "test-util")]
19173 #[tokio::test]
19174 #[serial_test::serial(storage_class_env)]
19175 async fn dispatched_tier_delete_recovery_finds_directory_source_on_encoded_set() {
19176 let temp_dir = tempfile::tempdir().expect("create temp store dir");
19177 let shutdown = CancellationToken::new();
19178 let (ctx, store, _shutdown) = without_storage_class_env(build_isolated_test_store_with_layout(
19179 temp_dir.path(),
19180 "prepared-directory-recovery",
19181 &[(2, 4)],
19182 shutdown,
19183 None,
19184 ))
19185 .await;
19186 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
19187
19188 let pool = &store.pools[0];
19189 let object = (0..10_000)
19190 .map(|index| format!("directory-{index}/"))
19191 .find(|candidate| {
19192 let encoded = rustfs_utils::path::encode_dir_object(candidate);
19193 !Arc::ptr_eq(&pool.get_disks_by_key(candidate), &pool.get_disks_by_key(&encoded))
19194 })
19195 .expect("test topology should have a directory key whose encoded form hashes to another set");
19196 let encoded = rustfs_utils::path::encode_dir_object(&object);
19197 assert!(!Arc::ptr_eq(&pool.get_disks_by_key(&object), &pool.get_disks_by_key(&encoded)));
19198
19199 let tier_name = "PREPAREDDIRECTORY";
19200 let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
19201 let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
19202 .await
19203 .expect("tier lease should resolve")
19204 .backend_identity();
19205 let bucket = "prepared-directory-recovery-bucket";
19206 store
19207 .make_bucket(bucket, &MakeBucketOptions::default())
19208 .await
19209 .expect("bucket should be created");
19210 let mut reader = PutObjReader::from_vec(b"directory source".to_vec());
19211 let original = store
19212 .put_object(bucket, &object, &mut reader, &ObjectOptions::default())
19213 .await
19214 .expect("directory source should be written");
19215 store
19216 .transition_object(
19217 bucket,
19218 &object,
19219 &ObjectOptions {
19220 transition: TransitionOptions {
19221 status: TRANSITION_PENDING.to_string(),
19222 tier: tier_name.to_string(),
19223 etag: original.etag.clone().expect("source should have an ETag"),
19224 ..Default::default()
19225 },
19226 mod_time: original.mod_time,
19227 ..Default::default()
19228 },
19229 )
19230 .await
19231 .expect("directory transition should commit");
19232 let committed = store
19233 .get_object_info(
19234 bucket,
19235 &object,
19236 &ObjectOptions {
19237 no_lock: true,
19238 metadata_cache_safe: false,
19239 ..Default::default()
19240 },
19241 )
19242 .await
19243 .expect("transitioned directory source should be readable");
19244 let mut entry = transitioned_delete_journal_entry_for_source(None, false, false, bucket, &object, &committed)
19245 .expect("transitioned source should produce a prepared journal entry");
19246 entry.backend_identity = Some(backend_identity);
19247 let incarnation = store
19248 .bucket_incarnation_id(bucket)
19249 .await
19250 .expect("bucket incarnation should resolve");
19251 install_test_tier_delete_dispatch_fixture(
19252 store.clone(),
19253 bucket,
19254 incarnation,
19255 &object,
19256 vec![(entry, Some(TierDeleteJournalState::Dispatched))],
19257 TierDeleteDispatchManifestState::DispatchAuthorized,
19258 )
19259 .await
19260 .expect("authorized dispatched journal should persist");
19261
19262 let stats = recover_tier_delete_journal_entries(store.clone(), 100, None)
19263 .await
19264 .expect("prepared recovery should complete");
19265
19266 assert_eq!((stats.scanned, stats.deleted, stats.failed), (1, 0, 1));
19267 assert_eq!(tier_delete_journal_count(store).await, 1);
19268 assert_eq!(backend.remove_count().await, 0);
19269 assert_eq!(backend.object_count().await, 1, "live directory source must retain its remote object");
19270 }
19271
19272 #[cfg(feature = "test-util")]
19273 #[tokio::test]
19274 #[serial_test::serial(storage_class_env)]
19275 async fn dispatched_tier_delete_recovery_checks_later_pool_then_commits_after_source_removal() {
19276 let temp_dir = tempfile::tempdir().expect("create cross-pool recovery store dir");
19277 let (ctx, store, _shutdown) =
19278 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "prepared-cross-pool-recovery", &[4, 4])).await;
19279 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
19280 let tier_name = "PREPAREDCROSSPOOL";
19281 let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
19282 let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
19283 .await
19284 .expect("tier lease should resolve")
19285 .backend_identity();
19286 let bucket = "prepared-cross-pool-recovery-bucket";
19287 let object = "object";
19288 store
19289 .make_bucket(bucket, &MakeBucketOptions::default())
19290 .await
19291 .expect("bucket should be created");
19292 let mut reader = PutObjReader::from_vec(b"later pool source".to_vec());
19293 let original = store.pools[1]
19294 .put_object(bucket, object, &mut reader, &ObjectOptions::default())
19295 .await
19296 .expect("source should be written only to the later pool");
19297 store.pools[1]
19298 .transition_object(
19299 bucket,
19300 object,
19301 &ObjectOptions {
19302 transition: TransitionOptions {
19303 status: TRANSITION_PENDING.to_string(),
19304 tier: tier_name.to_string(),
19305 etag: original.etag.clone().expect("source should have an ETag"),
19306 ..Default::default()
19307 },
19308 mod_time: original.mod_time,
19309 ..Default::default()
19310 },
19311 )
19312 .await
19313 .expect("later-pool transition should commit");
19314 let committed = store.pools[1]
19315 .get_object_info(bucket, object, &ObjectOptions::default())
19316 .await
19317 .expect("transitioned source should be readable");
19318 let mut entry = transitioned_delete_journal_entry_for_source(None, false, false, bucket, object, &committed)
19319 .expect("transitioned source should produce a prepared journal");
19320 entry.backend_identity = Some(backend_identity);
19321 let incarnation = store
19322 .bucket_incarnation_id(bucket)
19323 .await
19324 .expect("bucket incarnation should resolve");
19325 install_test_tier_delete_dispatch_fixture(
19326 store.clone(),
19327 bucket,
19328 incarnation,
19329 object,
19330 vec![(entry, Some(TierDeleteJournalState::Dispatched))],
19331 TierDeleteDispatchManifestState::DispatchAuthorized,
19332 )
19333 .await
19334 .expect("authorized dispatched journal should persist");
19335
19336 let retained = recover_tier_delete_journal_entries(store.clone(), 100, None)
19337 .await
19338 .expect("recovery should scan the later pool");
19339 assert_eq!((retained.scanned, retained.deleted, retained.failed), (1, 0, 1));
19340 assert_eq!(backend.remove_count().await, 0);
19341 assert_eq!(backend.object_count().await, 1);
19342 assert_eq!(
19343 tier_delete_journal_count(store.clone()).await,
19344 1,
19345 "an authorized journal must remain while its live source is durable"
19346 );
19347
19348 store.pools[1]
19349 .delete_object(
19350 bucket,
19351 object,
19352 ObjectOptions {
19353 version_id: committed.version_id.map(|version| version.to_string()),
19354 expiration: crate::storage_api_contracts::lifecycle::ExpirationOptions { expire: true },
19355 skip_free_version: true,
19356 ..Default::default()
19357 },
19358 )
19359 .await
19360 .expect("source version should be removed before recovery retry");
19361 let committed_stats = recover_tier_delete_journal_entries(store.clone(), 100, None)
19362 .await
19363 .expect("journal recovery should commit an absent stable source");
19364 assert_eq!((committed_stats.scanned, committed_stats.deleted, committed_stats.failed), (1, 0, 1));
19365 assert_eq!(backend.remove_count().await, 0);
19366
19367 let manifest_stats = recover_tier_delete_dispatch_manifests(store.clone(), 100, None)
19368 .await
19369 .expect("manifest recovery should authorize committed remote cleanup");
19370 assert_eq!((manifest_stats.advanced, manifest_stats.failed), (1, 0));
19371
19372 let deleted = recover_tier_delete_journal_entries(store.clone(), 100, None)
19373 .await
19374 .expect("completed journal should delete the exact remote version");
19375 assert_eq!((deleted.scanned, deleted.deleted, deleted.failed), (1, 1, 0));
19376 assert_eq!(tier_delete_journal_count(store).await, 0);
19377 assert_eq!(backend.remove_count().await, 1);
19378 assert_eq!(backend.object_count().await, 0);
19379 }
19380
19381 #[cfg(feature = "test-util")]
19382 #[tokio::test]
19383 #[serial_test::serial(storage_class_env)]
19384 async fn dispatched_tier_delete_recovery_retains_journal_on_source_metadata_error() {
19385 let temp_dir = tempfile::tempdir().expect("create metadata-error recovery store dir");
19386 let (ctx, store, _shutdown) =
19387 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "prepared-metadata-error-recovery", &[4])).await;
19388 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
19389 let tier_name = "PREPAREDMETADATAERROR";
19390 let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
19391 let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
19392 .await
19393 .expect("tier lease should resolve")
19394 .backend_identity();
19395 let bucket = "prepared-metadata-error-recovery-bucket";
19396 let object = "object";
19397 store
19398 .make_bucket(bucket, &MakeBucketOptions::default())
19399 .await
19400 .expect("bucket should be created");
19401 let mut reader = PutObjReader::from_vec(b"source with unreadable metadata".to_vec());
19402 let original = store
19403 .put_object(bucket, object, &mut reader, &ObjectOptions::default())
19404 .await
19405 .expect("source should be written");
19406 store
19407 .transition_object(
19408 bucket,
19409 object,
19410 &ObjectOptions {
19411 transition: TransitionOptions {
19412 status: TRANSITION_PENDING.to_string(),
19413 tier: tier_name.to_string(),
19414 etag: original.etag.clone().expect("source should have an ETag"),
19415 ..Default::default()
19416 },
19417 mod_time: original.mod_time,
19418 ..Default::default()
19419 },
19420 )
19421 .await
19422 .expect("source transition should commit");
19423 let committed = store
19424 .get_object_info(bucket, object, &ObjectOptions::default())
19425 .await
19426 .expect("transitioned source should be readable");
19427 let mut entry = transitioned_delete_journal_entry_for_source(None, false, false, bucket, object, &committed)
19428 .expect("transitioned source should produce a prepared journal");
19429 entry.backend_identity = Some(backend_identity);
19430 let incarnation = store
19431 .bucket_incarnation_id(bucket)
19432 .await
19433 .expect("bucket incarnation should resolve");
19434 install_test_tier_delete_dispatch_fixture(
19435 store.clone(),
19436 bucket,
19437 incarnation,
19438 object,
19439 vec![(entry, Some(TierDeleteJournalState::Dispatched))],
19440 TierDeleteDispatchManifestState::DispatchAuthorized,
19441 )
19442 .await
19443 .expect("authorized dispatched journal should persist");
19444
19445 for disk_index in 0..4 {
19446 let metadata_path = temp_dir
19447 .path()
19448 .join(format!("pool0/set0/disk{disk_index}/{bucket}/{object}/{STORAGE_FORMAT_FILE}"));
19449 tokio::fs::write(metadata_path, b"not-xl-meta")
19450 .await
19451 .expect("source metadata should be corrupted");
19452 }
19453
19454 let stats = recover_tier_delete_journal_entries(store.clone(), 100, None)
19455 .await
19456 .expect("recovery scan should complete despite the entry failure");
19457
19458 assert_eq!((stats.scanned, stats.deleted, stats.failed), (1, 0, 1));
19459 assert_eq!(tier_delete_journal_count(store).await, 1, "journal must remain dispatched for retry");
19460 assert_eq!(backend.remove_count().await, 0, "unreadable source metadata must block remote deletion");
19461 assert_eq!(backend.object_count().await, 1, "remote source must remain intact");
19462 }
19463
19464 #[cfg(feature = "test-util")]
19465 #[tokio::test]
19466 #[serial_test::serial(storage_class_env)]
19467 async fn tier_delete_recovery_retains_content_that_does_not_match_its_object_name() {
19468 let temp_dir = tempfile::tempdir().expect("create mismatched journal store dir");
19469 let (ctx, store, _shutdown) =
19470 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "mismatched-tier-journal", &[4])).await;
19471 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
19472 let tier_name = "MISMATCHEDJOURNAL";
19473 let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
19474 let lease = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
19475 .await
19476 .expect("tier lease should resolve");
19477 let remote_version = uuid::Uuid::new_v4().to_string();
19478 backend.set_put_remote_version(Some(remote_version.clone())).await;
19479 lease
19480 .put("remote/original", ReaderImpl::Body(bytes::Bytes::from_static(b"remote body")), 11)
19481 .await
19482 .expect("remote body should be seeded");
19483 let entry = Jentry {
19484 persisted_version: 0,
19485 obj_name: "remote/original".to_string(),
19486 version_id: remote_version,
19487 tier_name: tier_name.to_string(),
19488 backend_identity: Some(lease.backend_identity()),
19489 version_id_exact: true,
19490 version_state: rustfs_filemeta::TransitionVersionState::Exact,
19491 state: TierDeleteJournalState::Prepared,
19492 source: Some(TierDeleteSourceIdentity {
19493 bucket: "absent-source-bucket".to_string(),
19494 object: "absent-source-object".to_string(),
19495 version_id: Some(uuid::Uuid::new_v4().to_string()),
19496 versioned: true,
19497 version_suspended: false,
19498 data_dir: Some(uuid::Uuid::new_v4().to_string()),
19499 etag: Some("etag".to_string()),
19500 mod_time: Some(OffsetDateTime::UNIX_EPOCH.to_string()),
19501 }),
19502 dispatch: None,
19503 };
19504 let (_manifest_name, bound) = install_test_tier_delete_dispatch_fixture(
19505 store.clone(),
19506 "absent-source-bucket",
19507 uuid::Uuid::new_v4(),
19508 "absent-source-object",
19509 vec![(entry, Some(TierDeleteJournalState::Dispatched))],
19510 TierDeleteDispatchManifestState::DispatchAuthorized,
19511 )
19512 .await
19513 .expect("authorized dispatched journal should persist");
19514 let journal_name = tier_delete_journal_object_name(&bound[0]);
19515 let data = com::read_config(store.clone(), &journal_name)
19516 .await
19517 .expect("prepared journal should be readable");
19518 let mut value: serde_json::Value = serde_json::from_slice(&data).expect("journal should contain JSON");
19519 value["obj_name"] = serde_json::json!("remote/replaced");
19520 com::save_config(
19521 store.clone(),
19522 &journal_name,
19523 serde_json::to_vec(&value).expect("mismatched journal should encode"),
19524 )
19525 .await
19526 .expect("mismatched journal content should be written under the original name");
19527
19528 let stats = recover_tier_delete_journal_entries(store.clone(), 100, None)
19529 .await
19530 .expect("recovery scan should complete");
19531
19532 assert_eq!((stats.scanned, stats.deleted, stats.failed), (1, 0, 1));
19533 assert_eq!(tier_delete_journal_count(store).await, 1, "mismatched journal must be retained");
19534 assert_eq!(backend.remove_count().await, 0);
19535 assert_eq!(backend.object_count().await, 1);
19536 }
19537
19538 #[cfg(feature = "test-util")]
19539 #[test]
19540 #[serial_test::serial(storage_class_env)]
19541 fn transitioned_history_expiry_uses_journal_or_legacy_free_version() {
19542 std::thread::Builder::new()
19543 .name("transitioned-delete-all-test".to_string())
19544 .stack_size(32 * 1024 * 1024)
19545 .spawn(|| {
19546 let runtime = tokio::runtime::Builder::new_multi_thread()
19547 .enable_all()
19548 .worker_threads(2)
19549 .build()
19550 .expect("test runtime should build");
19551 runtime.block_on(async {
19552 let temp_dir = tempfile::tempdir().expect("create transitioned delete-all store dir");
19553 let (ctx, store, _shutdown) =
19554 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "transitioned-delete-all", &[4]))
19555 .await;
19556 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
19557 let tier_name = "DELETEALLTRANSITIONED";
19558 let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
19559 let bucket = "transitioned-delete-all-bucket";
19560 let object = "object";
19561 store
19562 .make_bucket(bucket, &MakeBucketOptions::default())
19563 .await
19564 .expect("bucket should be created");
19565 crate::bucket::metadata_sys::update_in(
19566 &ctx,
19567 bucket,
19568 BUCKET_VERSIONING_CONFIG,
19569 b"<VersioningConfiguration><Status>Enabled</Status></VersioningConfiguration>".to_vec(),
19570 )
19571 .await
19572 .expect("bucket versioning should be enabled");
19573 crate::bucket::metadata_sys::update_in(
19574 &ctx,
19575 bucket,
19576 BUCKET_LIFECYCLE_CONFIG,
19577 br#"<LifecycleConfiguration>
19578 <Rule>
19579 <ID>delete-all-versions</ID>
19580 <Status>Enabled</Status>
19581 <Filter><Prefix></Prefix></Filter>
19582 <Expiration><Days>1</Days><ExpiredObjectAllVersions>true</ExpiredObjectAllVersions></Expiration>
19583 </Rule>
19584</LifecycleConfiguration>"#
19585 .to_vec(),
19586 )
19587 .await
19588 .expect("delete-all lifecycle should be configured");
19589
19590 let old_time = OffsetDateTime::now_utc() - time::Duration::days(3);
19591 let mut history_reader = PutObjReader::from_vec(b"transitioned history".to_vec());
19592 let history = store
19593 .put_object(
19594 bucket,
19595 object,
19596 &mut history_reader,
19597 &ObjectOptions {
19598 versioned: true,
19599 mod_time: Some(old_time - time::Duration::hours(1)),
19600 ..Default::default()
19601 },
19602 )
19603 .await
19604 .expect("historical version should be written");
19605 let mut current_reader = PutObjReader::from_vec(b"current version".to_vec());
19606 let current = store
19607 .put_object(
19608 bucket,
19609 object,
19610 &mut current_reader,
19611 &ObjectOptions {
19612 versioned: true,
19613 mod_time: Some(old_time),
19614 ..Default::default()
19615 },
19616 )
19617 .await
19618 .expect("current version should be written");
19619 store
19620 .transition_object(
19621 bucket,
19622 object,
19623 &ObjectOptions {
19624 versioned: true,
19625 version_id: history.version_id.map(|version_id| version_id.to_string()),
19626 transition: TransitionOptions {
19627 status: TRANSITION_PENDING.to_string(),
19628 tier: tier_name.to_string(),
19629 etag: history.etag.clone().expect("history should have an ETag"),
19630 ..Default::default()
19631 },
19632 mod_time: history.mod_time,
19633 ..Default::default()
19634 },
19635 )
19636 .await
19637 .expect("historical version should transition");
19638 assert_eq!(backend.object_count().await, 1);
19639 assert_eq!(backend.put_versions().await.len(), 1);
19640 backend.set_remove_failure(true);
19641
19642 let incarnation = store
19643 .bucket_incarnation_id_from_disk(bucket)
19644 .await
19645 .expect("bucket incarnation should be available");
19646 for disk_index in 0..4 {
19647 let metadata_path = temp_dir
19648 .path()
19649 .join(format!("pool0/set0/disk{disk_index}/{bucket}/{object}/{STORAGE_FORMAT_FILE}"));
19650 let encoded = tokio::fs::read(&metadata_path)
19651 .await
19652 .expect("transition metadata should be readable");
19653 let mut metadata = FileMeta::load(&encoded).expect("transition metadata should decode");
19654 let (version_index, mut transitioned) = metadata
19655 .find_version(history.version_id)
19656 .expect("transitioned history should exist");
19657 rustfs_utils::http::metadata_compat::remove_bytes(
19660 &mut transitioned.object.as_mut().expect("history should be an object").meta_sys,
19661 rustfs_utils::http::metadata_compat::SUFFIX_TRANSITIONED_VERSION_STATE,
19662 );
19663 metadata.versions[version_index] = rustfs_filemeta::FileMetaShallowVersion::try_from(transitioned)
19664 .expect("legacy history should re-encode");
19665 tokio::fs::write(
19666 &metadata_path,
19667 metadata.marshal_msg().expect("unknown transition metadata should encode"),
19668 )
19669 .await
19670 .expect("unknown transition metadata should be written");
19671 let encoded = tokio::fs::read(&metadata_path)
19672 .await
19673 .expect("legacy transition metadata should be readable");
19674 let legacy = FileMeta::load(&encoded)
19675 .expect("legacy transition metadata should decode")
19676 .find_version(history.version_id)
19677 .expect("legacy history should exist")
19678 .1
19679 .into_fileinfo(bucket, object, true)
19680 .expect("legacy history should decode");
19681 assert_eq!(legacy.transition_version_state, rustfs_filemeta::TransitionVersionState::Unknown);
19682 }
19683 let lifecycle_event = crate::bucket::lifecycle::lifecycle::Event {
19684 action: rustfs_scanner_metrics::metrics::IlmAction::DeleteAllVersionsAction,
19685 rule_id: "delete-all-versions".to_string(),
19686 ..Default::default()
19687 };
19688 let legacy_applied =
19689 crate::bucket::lifecycle::bucket_lifecycle_ops::apply_expiry_on_non_transitioned_objects(
19690 store.clone(),
19691 ¤t,
19692 &lifecycle_event,
19693 &crate::bucket::lifecycle::bucket_lifecycle_audit::LcEventSrc::Scanner,
19694 incarnation,
19695 )
19696 .await;
19697 assert!(legacy_applied, "legacy Unknown transition metadata should use the free-version fallback");
19698 assert_eq!(tier_delete_journal_count(store.clone()).await, 0);
19699 assert_eq!(backend.remove_count().await, 0);
19700 let retained = store.pools[0].disk_set[0]
19701 .load_file_info_versions_exact(bucket, object)
19702 .await
19703 .expect("fallback delete-all metadata should remain readable")
19704 .expect("fallback delete-all should retain a hidden cleanup owner");
19705 assert_eq!(
19706 retained
19707 .versions
19708 .iter()
19709 .filter(|version| !version.tier_free_version())
19710 .count(),
19711 0
19712 );
19713 assert_eq!(
19714 retained.versions.iter().filter(|version| version.tier_free_version()).count(),
19715 1,
19716 "legacy cleanup must retain exactly one hidden free-version"
19717 );
19718 assert_eq!(backend.object_count().await, 1, "legacy remote cleanup must remain pending");
19719
19720 let exact_tier_name = "DELETEALLTRANSITIONEDEXACT";
19724 let exact_backend = register_mock_tier(&ctx.tier_config_mgr(), exact_tier_name).await;
19725 let exact_object = "exact-object";
19726 let mut exact_history_reader = PutObjReader::from_vec(b"exact transitioned history".to_vec());
19727 let exact_history = store
19728 .put_object(
19729 bucket,
19730 exact_object,
19731 &mut exact_history_reader,
19732 &ObjectOptions {
19733 versioned: true,
19734 mod_time: Some(old_time - time::Duration::hours(1)),
19735 ..Default::default()
19736 },
19737 )
19738 .await
19739 .expect("second historical version should be written");
19740 let mut exact_current_reader = PutObjReader::from_vec(b"exact current version".to_vec());
19741 let exact_current = store
19742 .put_object(
19743 bucket,
19744 exact_object,
19745 &mut exact_current_reader,
19746 &ObjectOptions {
19747 versioned: true,
19748 mod_time: Some(old_time),
19749 ..Default::default()
19750 },
19751 )
19752 .await
19753 .expect("second current version should be written");
19754 store
19755 .transition_object(
19756 bucket,
19757 exact_object,
19758 &ObjectOptions {
19759 versioned: true,
19760 version_id: exact_history.version_id.map(|version_id| version_id.to_string()),
19761 transition: TransitionOptions {
19762 status: TRANSITION_PENDING.to_string(),
19763 tier: exact_tier_name.to_string(),
19764 etag: exact_history.etag.clone().expect("second history should have an ETag"),
19765 ..Default::default()
19766 },
19767 mod_time: exact_history.mod_time,
19768 ..Default::default()
19769 },
19770 )
19771 .await
19772 .expect("second historical version should transition exactly");
19773 let exact_remote_version = exact_backend
19774 .put_versions()
19775 .await
19776 .last()
19777 .cloned()
19778 .expect("second transition should publish a remote version");
19779 exact_backend.set_remove_failure(true);
19780 let applied = crate::bucket::lifecycle::bucket_lifecycle_ops::apply_expiry_on_non_transitioned_objects(
19781 store.clone(),
19782 &exact_current,
19783 &lifecycle_event,
19784 &crate::bucket::lifecycle::bucket_lifecycle_audit::LcEventSrc::Scanner,
19785 incarnation,
19786 )
19787 .await;
19788
19789 assert!(applied, "delete-all should remove current and transitioned history");
19790 assert!(
19791 store.pools[0].disk_set[0]
19792 .load_file_info_versions_exact(bucket, exact_object)
19793 .await
19794 .expect("exact journal-owned metadata lookup should succeed")
19795 .is_none(),
19796 "exact journal ownership must remove the source key without leaving a free-version"
19797 );
19798 let versions = store.pools[0].disk_set[0]
19799 .load_file_info_versions_exact(bucket, object)
19800 .await
19801 .expect("remaining fallback metadata should be readable")
19802 .expect("the prior legacy cleanup owner should remain");
19803 assert_eq!(
19804 versions
19805 .versions
19806 .iter()
19807 .filter(|version| !version.tier_free_version())
19808 .count(),
19809 0,
19810 "exact delete-all must remove every visible version"
19811 );
19812 assert_eq!(
19813 versions.versions.iter().filter(|version| version.tier_free_version()).count(),
19814 1,
19815 "the exact v6 journal must not add another free-version"
19816 );
19817 assert_eq!(tier_delete_journal_count(store.clone()).await, 1);
19818 assert_eq!(backend.object_count().await, 1, "legacy remote cleanup must remain pending");
19819 assert_eq!(exact_backend.object_count().await, 1, "exact remote cleanup must remain journal-owned");
19820
19821 exact_backend.set_remove_failure(false);
19822 let stats = recover_tier_delete_journal_entries(store.clone(), 100, None)
19823 .await
19824 .expect("committed journal should recover");
19825 assert_eq!(stats.failed, 0);
19826 assert!(stats.scanned <= 1 && stats.deleted <= 1);
19827 assert_eq!(tier_delete_journal_count(store).await, 0);
19828 assert_eq!(
19829 backend.object_count().await,
19830 1,
19831 "journal recovery must leave the legacy free-version remote"
19832 );
19833 assert_eq!(exact_backend.object_count().await, 0);
19834 assert!(exact_backend.exact_remove_count() >= 1);
19835 assert_eq!(exact_backend.remove_versions().await, vec![exact_remote_version]);
19836 });
19837 })
19838 .expect("test thread should spawn")
19839 .join()
19840 .expect("test thread should complete");
19841 }
19842
19843 #[cfg(feature = "test-util")]
19844 #[tokio::test]
19845 #[serial_test::serial(storage_class_env)]
19846 async fn odm_write_back_requires_one_set_and_enabled_namespace_locking() {
19847 for (layout, locking, supported) in [
19848 (&[(1, 4)][..], true, true),
19849 (&[(1, 4), (1, 4)][..], true, false),
19850 (&[(2, 4)][..], true, false),
19851 (&[(1, 4)][..], false, false),
19852 ] {
19853 temp_env::async_with_vars([("RUSTFS_LOCK_ENABLED", Some(if locking { "true" } else { "false" }))], async {
19854 let dir = tempfile::tempdir().expect("isolated topology");
19855 let shutdown = CancellationToken::new();
19856 let (_ctx, store, _) = without_storage_class_env(build_isolated_test_store_with_layout(
19857 dir.path(),
19858 "odm-topology",
19859 layout,
19860 shutdown.clone(),
19861 None,
19862 ))
19863 .await;
19864 assert_eq!(
19865 store.supports_atomic_create_only_write_back(),
19866 supported,
19867 "layout={layout:?}, locking={locking}"
19868 );
19869 shutdown.cancel();
19870 })
19871 .await;
19872 }
19873 }
19874
19875 #[cfg(feature = "test-util")]
19876 #[tokio::test]
19877 #[serial_test::serial(storage_class_env)]
19878 async fn dispatched_tier_delete_recovery_requires_namespace_locking() {
19879 temp_env::async_with_vars([("RUSTFS_LOCK_ENABLED", Some("false"))], async {
19880 let temp_dir = tempfile::tempdir().expect("create lock-disabled store dir");
19881 let (ctx, store, _shutdown) =
19882 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "prepared-recovery-lock-disabled", &[4]))
19883 .await;
19884 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
19885 assert!(ctx.lock_manager().is_disabled());
19886
19887 let tier_name = "PREPAREDLOCKDISABLED";
19888 let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
19889 let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
19890 .await
19891 .expect("tier lease should resolve")
19892 .backend_identity();
19893 let entry = Jentry {
19894 persisted_version: 0,
19895 obj_name: "remote/lock-disabled".to_string(),
19896 version_id: uuid::Uuid::new_v4().to_string(),
19897 tier_name: tier_name.to_string(),
19898 backend_identity: Some(backend_identity),
19899 version_id_exact: true,
19900 version_state: rustfs_filemeta::TransitionVersionState::Exact,
19901 state: TierDeleteJournalState::Prepared,
19902 source: Some(TierDeleteSourceIdentity {
19903 bucket: "absent-source-bucket".to_string(),
19904 object: "absent-source-object".to_string(),
19905 version_id: Some(uuid::Uuid::new_v4().to_string()),
19906 versioned: true,
19907 version_suspended: false,
19908 data_dir: Some(uuid::Uuid::new_v4().to_string()),
19909 etag: Some("etag".to_string()),
19910 mod_time: Some(OffsetDateTime::UNIX_EPOCH.to_string()),
19911 }),
19912 dispatch: None,
19913 };
19914 install_test_tier_delete_dispatch_fixture(
19915 store.clone(),
19916 "absent-source-bucket",
19917 uuid::Uuid::new_v4(),
19918 "absent-source-object",
19919 vec![(entry, Some(TierDeleteJournalState::Dispatched))],
19920 TierDeleteDispatchManifestState::DispatchAuthorized,
19921 )
19922 .await
19923 .expect("authorized dispatched journal should persist");
19924
19925 let stats = recover_tier_delete_journal_entries(store.clone(), 100, None)
19926 .await
19927 .expect("recovery scan should complete");
19928
19929 assert_eq!((stats.scanned, stats.deleted, stats.failed), (1, 0, 1));
19930 assert_eq!(tier_delete_journal_count(store).await, 1, "journal must remain dispatched for retry");
19931 assert_eq!(backend.remove_count().await, 0, "lock-disabled recovery must not delete remotely");
19932 })
19933 .await;
19934 }
19935
19936 #[cfg(feature = "test-util")]
19937 #[tokio::test]
19938 #[serial_test::serial(storage_class_env)]
19939 async fn lifecycle_delete_all_requires_namespace_locking_before_mutation() {
19940 temp_env::async_with_vars([("RUSTFS_LOCK_ENABLED", Some("false"))], async {
19941 let temp_dir = tempfile::tempdir().expect("create lock-disabled delete-all store dir");
19942 let (ctx, store, _shutdown) =
19943 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "delete-all-lock-disabled", &[4])).await;
19944 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
19945 assert!(ctx.lock_manager().is_disabled());
19946 let bucket = "delete-all-lock-disabled-bucket";
19947 let object = "object";
19948 store
19949 .make_bucket(bucket, &MakeBucketOptions::default())
19950 .await
19951 .expect("bucket should be created");
19952 let mut reader = PutObjReader::from_vec(b"must survive".to_vec());
19953 let no_lock_opts = ObjectOptions {
19954 no_lock: true,
19955 ..Default::default()
19956 };
19957 let original = store.pools[0]
19958 .put_object(bucket, object, &mut reader, &no_lock_opts)
19959 .await
19960 .expect("source should be written");
19961
19962 let mut delete_opts = ObjectOptions {
19963 delete_prefix: true,
19964 delete_prefix_object: true,
19965 lifecycle_delete_all: Some(crate::object_api::LifecycleDeleteAllRequest {
19966 version_id: original.version_id,
19967 delete_marker: false,
19968 action: rustfs_scanner_metrics::metrics::IlmAction::DeleteAllVersionsAction,
19969 rule_id: "rule".to_string(),
19970 phase: crate::object_api::LifecycleDeleteAllPhase::Preflight,
19971 }),
19972 delete_replication_config_snapshot: Some(Arc::new(
19973 crate::bucket::replication::DeleteReplicationConfigSnapshot::default(),
19974 )),
19975 ..Default::default()
19976 };
19977 delete_opts.ensure_lifecycle_delete_all_journal();
19978
19979 let err = store
19980 .delete_object_with_tier_delete_journal(bucket, object, delete_opts)
19981 .await
19982 .expect_err("delete-all must reject disabled namespace locking");
19983
19984 assert!(err.to_string().contains("requires namespace locking"));
19985 let retained = store.pools[0]
19986 .get_object_info(bucket, object, &no_lock_opts)
19987 .await
19988 .expect("rejected delete-all must retain the source");
19989 assert_eq!(retained.etag, original.etag);
19990 assert_eq!(tier_delete_journal_count(store).await, 0, "rejected delete-all must not prepare journals");
19991 })
19992 .await;
19993 }
19994
19995 #[cfg(feature = "test-util")]
19996 #[tokio::test]
19997 #[serial_test::serial(storage_class_env)]
19998 async fn tier_mutation_intent_record_round_trips_through_config_store() {
19999 let temp_dir = tempfile::tempdir().expect("create temp store dir");
20000 let (_ctx, store, _shutdown) =
20001 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "tier-mutation-intent-record", &[4])).await;
20002 let mutation_id = uuid::Uuid::new_v4();
20003 let intent = TierMutationIntent {
20004 mutation_id,
20005 revision: 1,
20006 kind: TierMutationIntentKind::Edit,
20007 state: TierMutationIntentState::Prepared,
20008 old_config_etag: Some("old-etag".to_string()),
20009 committed_config_etag: None,
20010 candidate_digest: [3; 32],
20011 affected_targets: vec![TierMutationIntentTarget {
20012 tier_name: "COLD-A".to_string(),
20013 old_backend_identity: Some([1; 32]),
20014 new_backend_identity: Some([2; 32]),
20015 }],
20016 expires_at_unix_nanos: 1_780_000_000_000_000_000,
20017 };
20018
20019 save_tier_mutation_intent_record(store.clone(), &intent)
20020 .await
20021 .expect("tier mutation intent record should persist");
20022 let loaded = load_tier_mutation_intent_record(store.clone(), mutation_id)
20023 .await
20024 .expect("tier mutation intent record should load");
20025
20026 assert_eq!(loaded, intent);
20027
20028 delete_tier_mutation_intent_record(store.clone(), mutation_id)
20029 .await
20030 .expect("tier mutation intent record delete should be idempotent");
20031 delete_tier_mutation_intent_record(store.clone(), mutation_id)
20032 .await
20033 .expect("tier mutation intent record delete should tolerate missing records");
20034 let err = load_tier_mutation_intent_record(store, mutation_id)
20035 .await
20036 .expect_err("deleted tier mutation intent record should not load");
20037 assert!(matches!(err, Error::ConfigNotFound));
20038 }
20039
20040 #[cfg(feature = "test-util")]
20041 #[tokio::test]
20042 #[serial_test::serial(storage_class_env)]
20043 async fn tier_probe_intent_store_enforces_create_cas_and_terminal_delete_preconditions() {
20044 let temp_dir = tempfile::tempdir().expect("create temp store dir");
20045 let (_ctx, store, _shutdown) =
20046 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "tier-probe-intent-cas", &[4])).await;
20047 let probe_id = uuid::Uuid::new_v4();
20048 let creator_epoch = uuid::Uuid::new_v4();
20049 let initial = TierProbeIntent {
20050 probe_id,
20051 revision: 1,
20052 state: TierProbeIntentState::UploadOutcomeUnknown,
20053 operation: TierProbeOperationIdentity::Verify {
20054 config_etag: "config-etag".to_string(),
20055 backend_identity: [1; 32],
20056 },
20057 tier_name: "COLD-A".to_string(),
20058 destination_id: [1; 32],
20059 probe_object: format!("rustfs-tier-probe-{probe_id}"),
20060 creator_id: "node-a".to_string(),
20061 creator_epoch,
20062 created_at_unix_nanos: 1_780_000_000_000_000_000,
20063 owner: TierProbeOwnerFence {
20064 owner_id: "node-a".to_string(),
20065 owner_epoch: creator_epoch,
20066 not_after_unix_nanos: 1_780_000_900_000_000_000,
20067 },
20068 remote_version: TierProbeRemoteVersion::default(),
20069 };
20070
20071 save_tier_probe_intent_record_if_absent(store.clone(), &initial)
20072 .await
20073 .expect("initial probe intent should persist with create-only semantics");
20074 let duplicate = save_tier_probe_intent_record_if_absent(store.clone(), &initial)
20075 .await
20076 .expect_err("duplicate create must fail closed");
20077 assert!(matches!(duplicate, Error::PreconditionFailed));
20078
20079 let observed_initial = load_tier_probe_intent_record(store.clone(), probe_id)
20080 .await
20081 .expect("initial probe intent should load with an ETag");
20082 assert_eq!(observed_initial.intent(), &initial);
20083
20084 let nonterminal_delete = delete_tier_probe_intent_record_if_current(store.clone(), &observed_initial)
20085 .await
20086 .expect_err("nonterminal evidence must not be deleted");
20087 assert!(nonterminal_delete.to_string().contains("must be terminal"));
20088
20089 let mut fabricated_current_intent = initial.clone();
20090 fabricated_current_intent.tier_name = "COLD-B".to_string();
20091 let mut fabricated_successor = fabricated_current_intent.clone();
20092 fabricated_successor
20093 .advance(
20094 TierProbeIntentState::Uploaded,
20095 TierProbeRemoteVersion::versioned(uuid::Uuid::new_v4().to_string()),
20096 )
20097 .expect("fabricated successor should be internally valid");
20098 let fabricated_current = observed_initial.with_intent_for_test(fabricated_current_intent.clone());
20099 let crossed_cas = save_tier_probe_intent_record_if_current(store.clone(), &fabricated_current, &fabricated_successor)
20100 .await
20101 .expect_err("a live ETag must not authorize a different caller record");
20102 assert!(matches!(crossed_cas, Error::PreconditionFailed));
20103 assert_eq!(
20104 load_tier_probe_intent_record(store.clone(), probe_id)
20105 .await
20106 .expect("crossed CAS must retain the authoritative record")
20107 .intent(),
20108 &initial
20109 );
20110
20111 let mut fabricated_terminal_intent = fabricated_current_intent;
20112 fabricated_terminal_intent
20113 .advance(TierProbeIntentState::AbortedNoRemote, TierProbeRemoteVersion::default())
20114 .expect("fabricated terminal should be internally valid");
20115 let fabricated_terminal = observed_initial.with_intent_for_test(fabricated_terminal_intent);
20116 let crossed_delete = delete_tier_probe_intent_record_if_current(store.clone(), &fabricated_terminal)
20117 .await
20118 .expect_err("a live ETag must not delete for a different caller record");
20119 assert!(matches!(crossed_delete, Error::PreconditionFailed));
20120 assert_eq!(
20121 load_tier_probe_intent_record(store.clone(), probe_id)
20122 .await
20123 .expect("crossed delete must retain the authoritative record")
20124 .intent(),
20125 &initial
20126 );
20127
20128 let remote_version = TierProbeRemoteVersion::versioned(uuid::Uuid::new_v4().to_string());
20129 let mut uploaded = observed_initial.intent().clone();
20130 uploaded
20131 .advance(TierProbeIntentState::Uploaded, remote_version.clone())
20132 .expect("known PUT result should advance");
20133 save_tier_probe_intent_record_if_current(store.clone(), &observed_initial, &uploaded)
20134 .await
20135 .expect("the matching initial ETag should admit one successor");
20136
20137 let stale_cas = save_tier_probe_intent_record_if_current(store.clone(), &observed_initial, &uploaded)
20138 .await
20139 .expect_err("a consumed ETag must not overwrite the current generation");
20140 assert!(matches!(stale_cas, Error::PreconditionFailed));
20141
20142 let observed_uploaded = load_tier_probe_intent_record(store.clone(), probe_id)
20143 .await
20144 .expect("uploaded generation should load");
20145 assert_eq!(observed_uploaded.intent(), &uploaded);
20146 let mut cleanup = observed_uploaded.intent().clone();
20147 cleanup
20148 .advance(TierProbeIntentState::CleanupPending, remote_version.clone())
20149 .expect("known candidate should become cleanup-pending");
20150 save_tier_probe_intent_record_if_current(store.clone(), &observed_uploaded, &cleanup)
20151 .await
20152 .expect("cleanup generation should persist by exact ETag");
20153
20154 let observed_cleanup = load_tier_probe_intent_record(store.clone(), probe_id)
20155 .await
20156 .expect("cleanup generation should load");
20157 let mut completed = observed_cleanup.intent().clone();
20158 completed
20159 .advance(TierProbeIntentState::Completed, remote_version)
20160 .expect("exact cleanup should become terminal");
20161 save_tier_probe_intent_record_if_current(store.clone(), &observed_cleanup, &completed)
20162 .await
20163 .expect("terminal generation should persist by exact ETag");
20164
20165 let stale_terminal = observed_cleanup.with_intent_for_test(completed.clone());
20166 let stale_delete = delete_tier_probe_intent_record_if_current(store.clone(), &stale_terminal)
20167 .await
20168 .expect_err("a stale ETag must not delete terminal evidence");
20169 assert!(matches!(stale_delete, Error::PreconditionFailed));
20170
20171 let observed_completed = load_tier_probe_intent_record(store.clone(), probe_id)
20172 .await
20173 .expect("terminal generation should remain after stale delete");
20174 assert_eq!(observed_completed.intent(), &completed);
20175 delete_tier_probe_intent_record_if_current(store.clone(), &observed_completed)
20176 .await
20177 .expect("the exact terminal ETag should delete the record");
20178 assert!(matches!(load_tier_probe_intent_record(store, probe_id).await, Err(Error::ConfigNotFound)));
20179 }
20180
20181 #[cfg(feature = "test-util")]
20182 #[tokio::test]
20183 #[serial_test::serial(storage_class_env)]
20184 async fn tier_mutation_intent_record_scan_retains_good_records_and_counts_bad_records() {
20185 let temp_dir = tempfile::tempdir().expect("create temp store dir");
20186 let (_ctx, store, _shutdown) =
20187 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "tier-mutation-intent-scan", &[4])).await;
20188 let build_intent = |mutation_id: uuid::Uuid, tier_name: &str| TierMutationIntent {
20189 mutation_id,
20190 revision: 1,
20191 kind: TierMutationIntentKind::Edit,
20192 state: TierMutationIntentState::Prepared,
20193 old_config_etag: Some("old-etag".to_string()),
20194 committed_config_etag: None,
20195 candidate_digest: [3; 32],
20196 affected_targets: vec![TierMutationIntentTarget {
20197 tier_name: tier_name.to_string(),
20198 old_backend_identity: Some([1; 32]),
20199 new_backend_identity: Some([2; 32]),
20200 }],
20201 expires_at_unix_nanos: 1_780_000_000_000_000_000,
20202 };
20203 let first_id = uuid::Uuid::parse_str("12345678-1234-5678-9abc-def012345678").expect("first uuid should parse");
20204 let second_id = uuid::Uuid::parse_str("22345678-1234-5678-9abc-def012345678").expect("second uuid should parse");
20205 let first = build_intent(first_id, "COLD-A");
20206 let second = build_intent(second_id, "COLD-B");
20207 save_tier_mutation_intent_record(store.clone(), &first)
20208 .await
20209 .expect("first tier mutation intent record should persist");
20210 save_tier_mutation_intent_record(store.clone(), &second)
20211 .await
20212 .expect("second tier mutation intent record should persist");
20213 com::save_config(
20214 store.clone(),
20215 &format!("{TIER_MUTATION_INTENT_RECORD_PREFIX}/00/00/33345678123456789abcdef012345678.json"),
20216 b"{}".to_vec(),
20217 )
20218 .await
20219 .expect("malformed-shard intent record should persist");
20220 com::save_config(
20221 store.clone(),
20222 &format!("{TIER_MUTATION_INTENT_RECORD_PREFIX}/44/44/44445678123456789abcdef012345678.json"),
20223 b"{".to_vec(),
20224 )
20225 .await
20226 .expect("corrupt-json intent record should persist");
20227
20228 let scan = list_tier_mutation_intent_records(store, 100, None)
20229 .await
20230 .expect("tier mutation intent records should scan");
20231 let mut loaded_ids: Vec<_> = scan.intents.into_iter().map(|intent| intent.mutation_id).collect();
20232 loaded_ids.sort();
20233
20234 assert_eq!(scan.scanned, 4);
20235 assert_eq!(scan.failed, 2);
20236 assert_eq!(loaded_ids, vec![first_id, second_id]);
20237 assert!(!scan.truncated);
20238 assert_eq!(scan.next_marker, None);
20239 }
20240
20241 #[cfg(feature = "test-util")]
20242 #[tokio::test]
20243 #[serial_test::serial(storage_class_env)]
20244 async fn tier_mutation_intent_record_advance_is_idempotent_in_config_store() {
20245 let temp_dir = tempfile::tempdir().expect("create temp store dir");
20246 let (_ctx, store, _shutdown) =
20247 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "tier-mutation-intent-advance", &[4])).await;
20248 let mutation_id = uuid::Uuid::new_v4();
20249 let intent = TierMutationIntent {
20250 mutation_id,
20251 revision: 1,
20252 kind: TierMutationIntentKind::Edit,
20253 state: TierMutationIntentState::Prepared,
20254 old_config_etag: Some("old-etag".to_string()),
20255 committed_config_etag: None,
20256 candidate_digest: [3; 32],
20257 affected_targets: vec![TierMutationIntentTarget {
20258 tier_name: "COLD-A".to_string(),
20259 old_backend_identity: Some([1; 32]),
20260 new_backend_identity: Some([2; 32]),
20261 }],
20262 expires_at_unix_nanos: 1_780_000_000_000_000_000,
20263 };
20264 save_tier_mutation_intent_record(store.clone(), &intent)
20265 .await
20266 .expect("prepared tier mutation intent record should persist");
20267 let (loaded_before_advance, stale_etag) = load_tier_mutation_intent_record_with_etag(store.clone(), mutation_id)
20268 .await
20269 .expect("prepared tier mutation intent record should load with etag");
20270 assert_eq!(loaded_before_advance, intent);
20271 assert!(!stale_etag.is_empty());
20272
20273 let (committed, first_advanced) = advance_tier_mutation_intent_record_idempotent(
20274 store.clone(),
20275 mutation_id,
20276 TierMutationIntentState::Committed,
20277 Some("new-etag".to_string()),
20278 )
20279 .await
20280 .expect("first commit should advance the record");
20281 assert!(first_advanced);
20282 assert_eq!(committed.state, TierMutationIntentState::Committed);
20283 assert_eq!(committed.revision, 2);
20284 assert_eq!(committed.committed_config_etag.as_deref(), Some("new-etag"));
20285
20286 let (retried, retry_advanced) = advance_tier_mutation_intent_record_idempotent(
20287 store.clone(),
20288 mutation_id,
20289 TierMutationIntentState::Committed,
20290 Some("new-etag".to_string()),
20291 )
20292 .await
20293 .expect("same commit retry should be idempotent");
20294 assert!(!retry_advanced);
20295 assert_eq!(retried, committed);
20296
20297 let conflict = advance_tier_mutation_intent_record_idempotent(
20298 store.clone(),
20299 mutation_id,
20300 TierMutationIntentState::Committed,
20301 Some("other-etag".to_string()),
20302 )
20303 .await
20304 .expect_err("conflicting commit retry should fail closed");
20305 assert!(matches!(conflict, Error::Io(_)));
20306 assert!(conflict.to_string().contains("committed config etag does not match"));
20307
20308 let mut stale_conflict = intent;
20309 stale_conflict
20310 .advance_idempotent(TierMutationIntentState::Committed, Some("other-etag".to_string()))
20311 .expect("stale conflicting intent should advance locally");
20312 let stale_save = save_tier_mutation_intent_record_if_current(store.clone(), &stale_conflict, &stale_etag)
20313 .await
20314 .expect_err("stale etag must fail closed instead of overwriting the committed record");
20315 assert!(matches!(stale_save, Error::PreconditionFailed));
20316
20317 let loaded = load_tier_mutation_intent_record(store.clone(), mutation_id)
20318 .await
20319 .expect("conflicting retry must not overwrite the durable record");
20320 assert_eq!(loaded, committed);
20321
20322 let abort_id = uuid::Uuid::new_v4();
20323 let abort_intent = TierMutationIntent {
20324 mutation_id: abort_id,
20325 revision: 1,
20326 kind: TierMutationIntentKind::Edit,
20327 state: TierMutationIntentState::Prepared,
20328 old_config_etag: Some("old-etag".to_string()),
20329 committed_config_etag: None,
20330 candidate_digest: [4; 32],
20331 affected_targets: vec![TierMutationIntentTarget {
20332 tier_name: "COLD-B".to_string(),
20333 old_backend_identity: Some([1; 32]),
20334 new_backend_identity: Some([2; 32]),
20335 }],
20336 expires_at_unix_nanos: 1_780_000_000_000_000_000,
20337 };
20338 save_tier_mutation_intent_record(store.clone(), &abort_intent)
20339 .await
20340 .expect("prepared abort intent record should persist");
20341
20342 let (aborted, first_abort_advanced) =
20343 advance_tier_mutation_intent_record_idempotent(store.clone(), abort_id, TierMutationIntentState::Aborted, None)
20344 .await
20345 .expect("first abort should advance the record");
20346 assert!(first_abort_advanced);
20347 assert_eq!(aborted.state, TierMutationIntentState::Aborted);
20348 assert_eq!(aborted.revision, 2);
20349 assert_eq!(aborted.committed_config_etag, None);
20350
20351 let (aborted_retry, retry_abort_advanced) =
20352 advance_tier_mutation_intent_record_idempotent(store, abort_id, TierMutationIntentState::Aborted, None)
20353 .await
20354 .expect("same abort retry should be idempotent");
20355 assert!(!retry_abort_advanced);
20356 assert_eq!(aborted_retry, aborted);
20357 }
20358
20359 #[cfg(feature = "test-util")]
20360 fn tier_mutation_peer_test_intent(
20361 mutation_id: uuid::Uuid,
20362 tier_name: &str,
20363 candidate_digest: [u8; 32],
20364 ) -> TierMutationIntent {
20365 TierMutationIntent {
20366 mutation_id,
20367 revision: 1,
20368 kind: TierMutationIntentKind::Edit,
20369 state: TierMutationIntentState::Prepared,
20370 old_config_etag: Some("old-etag".to_string()),
20371 committed_config_etag: None,
20372 candidate_digest,
20373 affected_targets: vec![TierMutationIntentTarget {
20374 tier_name: tier_name.to_string(),
20375 old_backend_identity: Some([1; 32]),
20376 new_backend_identity: Some([2; 32]),
20377 }],
20378 expires_at_unix_nanos: i64::MAX,
20379 }
20380 }
20381
20382 #[cfg(feature = "test-util")]
20383 #[tokio::test]
20384 #[serial_test::serial(storage_class_env)]
20385 async fn tier_config_initial_reload_lock_failure_keeps_recovery_worker() {
20386 use crate::storage_api_contracts::namespace::NamespaceLocking as _;
20387
20388 let temp_dir = tempfile::tempdir().expect("create contended tier startup store dir");
20389 let (ctx, store, shutdown) =
20390 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "tier-startup-contention", &[4])).await;
20391 let manager = ctx.tier_config_mgr();
20392 register_mock_tier(&manager, "COLD-A").await;
20393 let candidate = TierConfigMgr::new();
20394 let candidate = candidate.read().await;
20395 let candidate_digest = tier_config_candidate_digest(&candidate).expect("empty candidate digest should resolve");
20396 candidate
20397 .save_tiering_config(store.clone())
20398 .await
20399 .expect("committed removal config should persist");
20400 let config_info = store
20401 .get_object_info(
20402 RUSTFS_META_BUCKET,
20403 &format!("{}/{TIER_CONFIG_FILE}", com::CONFIG_PREFIX),
20404 &ObjectOptions::default(),
20405 )
20406 .await
20407 .expect("committed removal config metadata should load");
20408 let mutation_id = uuid::Uuid::new_v4();
20409 let mut intent = tier_mutation_peer_test_intent(mutation_id, "COLD-A", candidate_digest);
20410 intent.kind = TierMutationIntentKind::Remove;
20411 intent.affected_targets[0].new_backend_identity = None;
20412 intent
20413 .advance(
20414 TierMutationIntentState::Committed,
20415 Some(config_info.etag.expect("committed config should have an ETag")),
20416 )
20417 .expect("removal intent should commit");
20418 save_tier_mutation_intent_record(store.clone(), &intent)
20419 .await
20420 .expect("committed removal intent should persist");
20421
20422 let config_lock = format!("{}/{TIER_CONFIG_FILE}.lock", com::CONFIG_PREFIX);
20423 let namespace = store
20424 .new_ns_lock(RUSTFS_META_BUCKET, &config_lock)
20425 .await
20426 .expect("tier config namespace should resolve");
20427 let owner = namespace
20428 .get_write_lock(crate::set_disk::get_lock_acquire_timeout())
20429 .await
20430 .expect("competing recovery should hold the config lock");
20431 let initial_error = runtime_sources::init_tier_config_mgr_handle(manager.clone(), store.clone())
20432 .await
20433 .expect_err("startup must still report the initial recovery lock failure");
20434 assert!(initial_error.to_string().contains(&config_lock));
20435 match TierConfigMgr::acquire_operation_lease(&manager, "COLD-A").await {
20436 Err(err) => assert_eq!(err.message, "Remote tier configuration is being replaced"),
20437 Ok(_) => panic!("failed initial recovery must retain its mutation fence"),
20438 }
20439 drop(owner);
20440
20441 tokio::time::timeout(Duration::from_secs(30), async {
20442 loop {
20443 if manager.read().await.tiers.is_empty()
20444 && matches!(
20445 TierConfigMgr::acquire_operation_lease(&manager, "COLD-A").await,
20446 Err(err) if err.code == crate::services::tier::tier_handlers::ERR_TIER_NOT_FOUND.code
20447 )
20448 {
20449 break;
20450 }
20451 tokio::time::sleep(Duration::from_millis(10)).await;
20452 }
20453 })
20454 .await
20455 .expect("recovery worker must publish and clear its fence without another admin mutation");
20456 assert_eq!(
20457 load_tier_mutation_intent_record(store.clone(), mutation_id)
20458 .await
20459 .expect("unexpired committed tombstone must remain durable"),
20460 intent
20461 );
20462 shutdown.cancel();
20463 }
20464
20465 #[cfg(feature = "test-util")]
20466 #[tokio::test]
20467 #[serial_test::serial(storage_class_env)]
20468 async fn tier_mutation_peer_prepare_reloads_after_save_and_load_lock_timeouts() {
20469 use crate::storage_api_contracts::namespace::NamespaceLocking as _;
20470 use std::sync::atomic::Ordering;
20471
20472 let temp_dir = tempfile::tempdir().expect("create contended tier prepare store dir");
20473 let (ctx, store, shutdown) =
20474 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "tier-prepare-contention", &[4])).await;
20475 let mutation_id = uuid::Uuid::new_v4();
20476 let intent = tier_mutation_peer_test_intent(mutation_id, "COLD-A", [9; 32]);
20477 let object = crate::services::tier::tier_mutation_intent::tier_mutation_intent_record_object_name(mutation_id)
20478 .expect("intent record object should resolve");
20479 let barrier = crate::set_disk::PutObjectCommitBarrier::install(
20480 RUSTFS_META_BUCKET,
20481 &object,
20482 crate::set_disk::PutObjectCommitPause::BeforeNamespace,
20483 );
20484 let prepare_store = store.clone();
20485 let payload = intent.encode().expect("prepare intent should encode");
20486 let mut prepare = tokio::spawn(async move {
20487 handle_tier_mutation_peer_request(
20488 prepare_store,
20489 TIER_MUTATION_RPC_PROTOCOL_VERSION,
20490 TierMutationRpcPhase::Prepare,
20491 mutation_id,
20492 &payload,
20493 )
20494 .await
20495 });
20496 barrier.wait_until_paused().await;
20497 let namespace = store.pools[0].disk_set[0]
20498 .new_ns_lock(RUSTFS_META_BUCKET, &object)
20499 .await
20500 .expect("intent record namespace should resolve");
20501 let owner = namespace
20502 .get_write_lock(crate::set_disk::get_lock_acquire_timeout())
20503 .await
20504 .expect("competing publisher should hold the intent namespace");
20505 com::save_config_with_opts(
20508 store.clone(),
20509 &object,
20510 intent.encode().expect("winning intent should encode"),
20511 &ObjectOptions {
20512 no_lock: true,
20513 max_parity: true,
20514 write_completion: crate::object_api::WriteCompletion::TailDrained,
20515 ..Default::default()
20516 },
20517 )
20518 .await
20519 .expect("competing publisher should persist the matching intent under its lock");
20520 let manager = ctx
20521 .lock_manager()
20522 .as_fast_lock_manager()
20523 .expect("fixture should use real locks");
20524 let key = rustfs_lock::fast_lock::types::ObjectKey {
20525 bucket: Arc::from(RUSTFS_META_BUCKET),
20526 object: Arc::from(object.as_str()),
20527 version: None,
20528 };
20529 let timeouts = &manager.get_shard(&key).metrics().timeouts;
20530 let before = timeouts.load(Ordering::Relaxed);
20531 barrier.release_and_wait_until_namespace_pending().await;
20532 drop(barrier);
20533 tokio::select! {
20534 result = &mut prepare => panic!("Prepare must retry the contended save and load before completing: {result:?}"),
20535 result = tokio::time::timeout(Duration::from_secs(30), async {
20536 while timeouts.load(Ordering::Relaxed) - before < 2 {
20537 tokio::time::sleep(Duration::from_millis(10)).await;
20538 }
20539 }) => result.expect("the save and subsequent load should each exhaust their lock wait"),
20540 }
20541 assert_eq!(timeouts.load(Ordering::Relaxed) - before, 2);
20542 drop(owner);
20543 let outcome = tokio::time::timeout(Duration::from_secs(30), prepare)
20544 .await
20545 .expect("third prepare round should complete after the competing publisher releases its lock")
20546 .expect("prepare task should join")
20547 .expect("third prepare round should reload the matching durable intent");
20548 assert_eq!(outcome.state, TierMutationPeerState::Prepared);
20549 assert!(!outcome.applied, "the retry must reuse the winning record rather than recreate it");
20550 assert_eq!(
20551 load_tier_mutation_intent_record(store.clone(), mutation_id)
20552 .await
20553 .expect("winning record should remain readable"),
20554 intent
20555 );
20556 assert_eq!(timeouts.load(Ordering::Relaxed) - before, 2);
20557 shutdown.cancel();
20558 }
20559
20560 #[cfg(feature = "test-util")]
20561 #[tokio::test]
20562 #[serial_test::serial(storage_class_env)]
20563 async fn tier_mutation_peer_prepare_preserves_lock_timeout_after_three_rounds() {
20564 use crate::storage_api_contracts::namespace::NamespaceLocking as _;
20565 use std::sync::atomic::Ordering;
20566
20567 let temp_dir = tempfile::tempdir().expect("create exhausted tier prepare store dir");
20568 let (ctx, store, shutdown) =
20569 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "tier-prepare-exhaustion", &[4])).await;
20570 let mutation_id = uuid::Uuid::new_v4();
20571 let intent = tier_mutation_peer_test_intent(mutation_id, "COLD-A", [9; 32]);
20572 let object = crate::services::tier::tier_mutation_intent::tier_mutation_intent_record_object_name(mutation_id)
20573 .expect("intent record object should resolve");
20574 let barrier = crate::set_disk::PutObjectCommitBarrier::install(
20575 RUSTFS_META_BUCKET,
20576 &object,
20577 crate::set_disk::PutObjectCommitPause::BeforeNamespace,
20578 );
20579 let prepare_store = store.clone();
20580 let payload = intent.encode().expect("prepare intent should encode");
20581 let prepare = tokio::spawn(async move {
20582 handle_tier_mutation_peer_request(
20583 prepare_store,
20584 TIER_MUTATION_RPC_PROTOCOL_VERSION,
20585 TierMutationRpcPhase::Prepare,
20586 mutation_id,
20587 &payload,
20588 )
20589 .await
20590 });
20591 barrier.wait_until_paused().await;
20592 let namespace = store.pools[0].disk_set[0]
20593 .new_ns_lock(RUSTFS_META_BUCKET, &object)
20594 .await
20595 .expect("intent record namespace should resolve");
20596 let lock_timeout = crate::set_disk::get_lock_acquire_timeout();
20597 let owner = namespace
20598 .get_write_lock(lock_timeout)
20599 .await
20600 .expect("competing publisher should hold the intent namespace");
20601 let manager = ctx
20602 .lock_manager()
20603 .as_fast_lock_manager()
20604 .expect("fixture should use real locks");
20605 let key = rustfs_lock::fast_lock::types::ObjectKey {
20606 bucket: Arc::from(RUSTFS_META_BUCKET),
20607 object: Arc::from(object.as_str()),
20608 version: None,
20609 };
20610 let timeouts = &manager.get_shard(&key).metrics().timeouts;
20611 let before = timeouts.load(Ordering::Relaxed);
20612 barrier.release_and_wait_until_namespace_pending().await;
20613 drop(barrier);
20614 let error = tokio::time::timeout(Duration::from_secs(30), prepare)
20615 .await
20616 .expect("three contended prepare rounds should exhaust the bounded budget")
20617 .expect("prepare task should join")
20618 .expect_err("Prepare must fail while the competing publisher keeps the namespace");
20619 match error {
20620 TierMutationPeerError::Store(Error::Lock(rustfs_lock::LockError::Timeout { resource, timeout })) => {
20621 assert_eq!(resource, key.to_string());
20622 assert_eq!(timeout, lock_timeout);
20623 }
20624 other => panic!("Prepare must preserve the final typed lock timeout: {other:?}"),
20625 }
20626 assert_eq!(timeouts.load(Ordering::Relaxed) - before, 3, "save and load must share three rounds");
20627 drop(owner);
20628 assert!(matches!(
20629 load_tier_mutation_intent_record(store, mutation_id).await,
20630 Err(Error::ConfigNotFound)
20631 ));
20632 shutdown.cancel();
20633 }
20634
20635 #[cfg(feature = "test-util")]
20636 #[tokio::test]
20637 #[serial_test::serial(storage_class_env)]
20638 async fn tier_mutation_abort_tombstone_rejects_delayed_prepare() {
20639 let temp_dir = tempfile::tempdir().expect("create temp store dir");
20640 let (_ctx, store, _shutdown) =
20641 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "tier-mutation-abort-tombstone", &[4])).await;
20642 register_mock_tier(&store.tier_config_mgr(), "COLD-A").await;
20643 store
20644 .tier_config_mgr()
20645 .read()
20646 .await
20647 .save_tiering_config(store.clone())
20648 .await
20649 .expect("base tier config should persist");
20650 let config_info = store
20651 .get_object_info(
20652 RUSTFS_META_BUCKET,
20653 &format!("{}/{}", com::CONFIG_PREFIX, TIER_CONFIG_FILE),
20654 &ObjectOptions::default(),
20655 )
20656 .await
20657 .expect("base tier config metadata should load");
20658 let mutation_id = uuid::Uuid::new_v4();
20659 let mut intent = tier_mutation_peer_test_intent(mutation_id, "COLD-A", [7; 32]);
20660 intent.old_config_etag = config_info.etag;
20661 intent.expires_at_unix_nanos =
20662 i64::try_from((OffsetDateTime::now_utc() + time::Duration::hours(1)).unix_timestamp_nanos())
20663 .expect("future tombstone expiry should fit i64");
20664 let payload = intent.encode().expect("prepare identity should encode");
20665
20666 let mismatch_id = uuid::Uuid::new_v4();
20667 let mut mismatch = intent.clone();
20668 mismatch.mutation_id = mismatch_id;
20669 mismatch.old_config_etag = Some("not-the-current-config-etag".to_string());
20670 let mismatch_error = handle_tier_mutation_peer_request(
20671 store.clone(),
20672 TIER_MUTATION_RPC_PROTOCOL_VERSION,
20673 TierMutationRpcPhase::Abort,
20674 mismatch_id,
20675 &mismatch.encode().expect("proof-mismatch Abort should encode"),
20676 )
20677 .await
20678 .expect_err("a missing-record Abort with the wrong config proof must fail closed");
20679 assert!(matches!(mismatch_error, TierMutationPeerError::AbortProofMismatch));
20680 assert!(matches!(
20681 load_tier_mutation_intent_record(store.clone(), mismatch_id).await,
20682 Err(Error::ConfigNotFound)
20683 ));
20684
20685 let aborted = handle_tier_mutation_peer_request(
20686 store.clone(),
20687 TIER_MUTATION_RPC_PROTOCOL_VERSION,
20688 TierMutationRpcPhase::Abort,
20689 mutation_id,
20690 &payload,
20691 )
20692 .await
20693 .expect("missing-record abort should persist a tombstone");
20694 assert!(aborted.applied);
20695 assert_eq!(aborted.state, TierMutationPeerState::Aborted);
20696 let tombstone = load_tier_mutation_intent_record(store.clone(), mutation_id)
20697 .await
20698 .expect("abort tombstone should be durable");
20699 assert_eq!(tombstone.state, TierMutationIntentState::Aborted);
20700 assert!(tombstone.same_identity_as(&intent));
20701
20702 let manager = store.tier_config_mgr();
20703 TierConfigMgr::reload_handle(&manager, store.clone())
20704 .await
20705 .expect("reload should retain an unexpired abort tombstone");
20706 let tombstone_after_reload = load_tier_mutation_intent_record(store.clone(), mutation_id)
20707 .await
20708 .expect("reload must not clean an unexpired abort tombstone");
20709 assert_eq!(tombstone_after_reload.state, TierMutationIntentState::Aborted);
20710 assert!(tombstone_after_reload.same_identity_as(&intent));
20711
20712 register_mock_tier(&store.tier_config_mgr(), "COLD-B").await;
20713 store
20714 .tier_config_mgr()
20715 .read()
20716 .await
20717 .save_tiering_config(store.clone())
20718 .await
20719 .expect("later config change should persist before idempotent Abort replay");
20720 let retried_abort = handle_tier_mutation_peer_request(
20721 store.clone(),
20722 TIER_MUTATION_RPC_PROTOCOL_VERSION,
20723 TierMutationRpcPhase::Abort,
20724 mutation_id,
20725 &payload,
20726 )
20727 .await
20728 .expect("terminal Abort retry must not depend on the mutable current config proof");
20729 assert!(!retried_abort.applied);
20730 assert_eq!(retried_abort.state, TierMutationPeerState::Aborted);
20731
20732 let delayed = handle_tier_mutation_peer_request(
20733 store.clone(),
20734 TIER_MUTATION_RPC_PROTOCOL_VERSION,
20735 TierMutationRpcPhase::Prepare,
20736 mutation_id,
20737 &payload,
20738 )
20739 .await
20740 .expect("delayed matching prepare should converge on the tombstone");
20741 assert!(!delayed.applied);
20742 assert_eq!(delayed.state, TierMutationPeerState::Aborted);
20743 TierConfigMgr::acquire_operation_lease(&store.tier_config_mgr(), "COLD-A")
20744 .await
20745 .expect("delayed prepare must not reinstall the runtime block");
20746
20747 let mut conflicting = intent.clone();
20748 conflicting.candidate_digest = [8; 32];
20749 let conflict = handle_tier_mutation_peer_request(
20750 store,
20751 TIER_MUTATION_RPC_PROTOCOL_VERSION,
20752 TierMutationRpcPhase::Prepare,
20753 mutation_id,
20754 &conflicting.encode().expect("conflicting prepare should encode"),
20755 )
20756 .await
20757 .expect_err("different identity must not reuse an abort tombstone");
20758 assert!(matches!(conflict, TierMutationPeerError::ConflictingIntent));
20759 }
20760
20761 #[cfg(feature = "test-util")]
20762 #[tokio::test]
20763 #[serial_test::serial(storage_class_env)]
20764 async fn tier_mutation_expired_prepare_cannot_recreate_a_cleaned_tombstone() {
20765 let temp_dir = tempfile::tempdir().expect("create temp store dir");
20766 let (_ctx, store, _shutdown) =
20767 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "tier-mutation-expired-replay", &[4])).await;
20768 register_mock_tier(&store.tier_config_mgr(), "COLD-A").await;
20769 let mutation_id = uuid::Uuid::new_v4();
20770 let mut intent = tier_mutation_peer_test_intent(mutation_id, "COLD-A", [11; 32]);
20771 intent.expires_at_unix_nanos =
20772 i64::try_from((OffsetDateTime::now_utc() - time::Duration::seconds(1)).unix_timestamp_nanos())
20773 .expect("expired timestamp should fit i64");
20774 let payload = intent.encode().expect("expired signed Prepare should still encode");
20775
20776 let error = handle_tier_mutation_peer_request(
20777 store.clone(),
20778 TIER_MUTATION_RPC_PROTOCOL_VERSION,
20779 TierMutationRpcPhase::Prepare,
20780 mutation_id,
20781 &payload,
20782 )
20783 .await
20784 .expect_err("a missing-record expired Prepare must fail closed");
20785 assert!(matches!(error, TierMutationPeerError::ExpiredIntent));
20786 assert!(matches!(
20787 load_tier_mutation_intent_record(store.clone(), mutation_id).await,
20788 Err(Error::ConfigNotFound)
20789 ));
20790 TierConfigMgr::acquire_operation_lease(&store.tier_config_mgr(), "COLD-A")
20791 .await
20792 .expect("expired replay must not install a peer-only runtime fence");
20793 }
20794
20795 #[cfg(feature = "test-util")]
20796 #[tokio::test]
20797 #[serial_test::serial(storage_class_env)]
20798 async fn tier_mutation_peer_prepare_waits_for_inflight_reference_lease() {
20799 let temp_dir = tempfile::tempdir().expect("create temp store dir");
20800 let (_ctx, store, _shutdown) =
20801 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "tier-mutation-peer-drain", &[4])).await;
20802 register_mock_tier(&store.tier_config_mgr(), "COLD-A").await;
20803 let old_lease = TierConfigMgr::acquire_operation_lease(&store.tier_config_mgr(), "COLD-A")
20804 .await
20805 .expect("old tier operation lease should be available");
20806 let mutation_id = uuid::Uuid::new_v4();
20807 let mut intent = tier_mutation_peer_test_intent(mutation_id, "COLD-A", [9; 32]);
20808 intent.affected_targets[0].old_backend_identity = Some(old_lease.backend_identity());
20809 let payload = intent.encode().expect("prepare intent should encode");
20810 let prepare_store = store.clone();
20811 let prepare = tokio::spawn(async move {
20812 handle_tier_mutation_peer_request(
20813 prepare_store,
20814 TIER_MUTATION_RPC_PROTOCOL_VERSION,
20815 TierMutationRpcPhase::Prepare,
20816 mutation_id,
20817 &payload,
20818 )
20819 .await
20820 });
20821
20822 tokio::time::timeout(Duration::from_secs(5), async {
20823 loop {
20824 if TierConfigMgr::acquire_operation_lease(&store.tier_config_mgr(), "COLD-A")
20825 .await
20826 .is_err()
20827 {
20828 return;
20829 }
20830 tokio::task::yield_now().await;
20831 }
20832 })
20833 .await
20834 .expect("peer Prepare should install its runtime block");
20835 assert!(
20836 !prepare.is_finished(),
20837 "peer Prepare must not acknowledge while a pre-existing reference creator still holds a lease"
20838 );
20839
20840 drop(old_lease);
20841 let outcome = prepare
20842 .await
20843 .expect("peer Prepare task should join")
20844 .expect("peer Prepare should finish after the lease drains");
20845 assert_eq!(outcome.state, TierMutationPeerState::Prepared);
20846 TierConfigMgr::clear_prepared_mutation_intent_block(&store.tier_config_mgr(), mutation_id)
20847 .await
20848 .expect("test should clear the prepared runtime block");
20849 }
20850
20851 #[cfg(feature = "test-util")]
20852 #[test]
20853 #[serial_test::serial(storage_class_env)]
20854 fn tier_mutation_peer_handler_applies_prepare_commit_and_abort_idempotently() {
20855 std::thread::Builder::new()
20860 .name("tier-mutation-peer-handler-test".to_string())
20861 .stack_size(32 * 1024 * 1024)
20862 .spawn(|| {
20863 let runtime = tokio::runtime::Builder::new_multi_thread()
20864 .enable_all()
20865 .worker_threads(2)
20866 .build()
20867 .expect("tier mutation peer handler test runtime should build");
20868 runtime.block_on(tier_mutation_peer_handler_case());
20869 })
20870 .expect("tier mutation peer handler test thread should spawn")
20871 .join()
20872 .expect("tier mutation peer handler test thread should complete");
20873 }
20874
20875 #[cfg(feature = "test-util")]
20876 async fn tier_mutation_peer_handler_case() {
20877 let temp_dir = tempfile::tempdir().expect("create temp store dir");
20878 let (_ctx, store, _shutdown) =
20879 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "tier-mutation-peer-handler", &[4])).await;
20880 register_mock_tier(&store.tier_config_mgr(), "COLD-A").await;
20881 let (candidate_digest, config_etag) = {
20882 let tier_config_mgr = store.tier_config_mgr();
20883 let manager = tier_config_mgr.read().await;
20884 let candidate_digest = tier_config_candidate_digest(&manager).expect("peer commit candidate digest should build");
20885 manager
20886 .save_tiering_config(store.clone())
20887 .await
20888 .expect("peer commit config fixture should persist");
20889 let config_info = store
20890 .get_object_info(
20891 RUSTFS_META_BUCKET,
20892 &format!("{}/{}", com::CONFIG_PREFIX, TIER_CONFIG_FILE),
20893 &ObjectOptions::default(),
20894 )
20895 .await
20896 .expect("peer commit config fixture should load");
20897 (candidate_digest, config_info.etag.expect("peer commit config should carry an ETag"))
20898 };
20899 let mutation_id = uuid::Uuid::new_v4();
20900 let mut intent = tier_mutation_peer_test_intent(mutation_id, "COLD-A", candidate_digest);
20901 intent.old_config_etag = Some(config_etag.clone());
20902 let prepare_payload = intent.encode().expect("prepare intent should encode");
20903
20904 let prepared = handle_tier_mutation_peer_request(
20905 store.clone(),
20906 TIER_MUTATION_RPC_PROTOCOL_VERSION,
20907 TierMutationRpcPhase::Prepare,
20908 mutation_id,
20909 &prepare_payload,
20910 )
20911 .await
20912 .expect("first prepare should create the peer intent");
20913 assert!(prepared.applied);
20914 assert_eq!(prepared.state, TierMutationPeerState::Prepared);
20915 let blocked = match TierConfigMgr::acquire_operation_lease(&store.tier_config_mgr(), "COLD-A").await {
20916 Ok(_) => panic!("prepared peer mutation should block new tier operation leases"),
20917 Err(err) => err,
20918 };
20919 assert!(
20920 blocked.message.contains("being replaced"),
20921 "prepared peer mutation should reuse the existing blocked-tier error: {blocked}"
20922 );
20923
20924 let retried_prepare = handle_tier_mutation_peer_request(
20925 store.clone(),
20926 TIER_MUTATION_RPC_PROTOCOL_VERSION,
20927 TierMutationRpcPhase::Prepare,
20928 mutation_id,
20929 &prepare_payload,
20930 )
20931 .await
20932 .expect("same prepare retry should be idempotent");
20933 assert!(!retried_prepare.applied);
20934 assert_eq!(retried_prepare.state, TierMutationPeerState::Prepared);
20935 let retried_blocked = match TierConfigMgr::acquire_operation_lease(&store.tier_config_mgr(), "COLD-A").await {
20936 Ok(_) => panic!("prepared retry should keep blocking new tier operation leases"),
20937 Err(err) => err,
20938 };
20939 assert!(
20940 retried_blocked.message.contains("being replaced"),
20941 "prepared retry should keep the existing blocked-tier error: {retried_blocked}"
20942 );
20943
20944 let mismatched_commit = handle_tier_mutation_peer_request(
20945 store.clone(),
20946 TIER_MUTATION_RPC_PROTOCOL_VERSION,
20947 TierMutationRpcPhase::Commit,
20948 mutation_id,
20949 b"not-the-current-etag",
20950 )
20951 .await
20952 .expect_err("commit with a mismatched config proof must fail closed");
20953 assert!(matches!(mismatched_commit, TierMutationPeerError::CommitProofMismatch));
20954
20955 register_mock_tier(&store.tier_config_mgr(), "COLD-C").await;
20956 let bad_digest_id = uuid::Uuid::new_v4();
20957 let mut bad_digest_intent = tier_mutation_peer_test_intent(bad_digest_id, "COLD-C", [9; 32]);
20958 bad_digest_intent.old_config_etag = Some(config_etag.clone());
20959 handle_tier_mutation_peer_request(
20960 store.clone(),
20961 TIER_MUTATION_RPC_PROTOCOL_VERSION,
20962 TierMutationRpcPhase::Prepare,
20963 bad_digest_id,
20964 &bad_digest_intent.encode().expect("bad digest prepare intent should encode"),
20965 )
20966 .await
20967 .expect("bad digest prepare should install a prepared intent");
20968 let mismatched_digest_commit = handle_tier_mutation_peer_request(
20969 store.clone(),
20970 TIER_MUTATION_RPC_PROTOCOL_VERSION,
20971 TierMutationRpcPhase::Commit,
20972 bad_digest_id,
20973 config_etag.as_bytes(),
20974 )
20975 .await
20976 .expect_err("a correct ETag with a mismatched candidate digest must fail closed");
20977 assert!(matches!(mismatched_digest_commit, TierMutationPeerError::CommitProofMismatch));
20978 let bad_digest_loaded = load_tier_mutation_intent_record(store.clone(), bad_digest_id)
20979 .await
20980 .expect("mismatched digest must leave the prepared intent durable");
20981 assert_eq!(bad_digest_loaded.state, TierMutationIntentState::Prepared);
20982 let bad_digest_blocked = match TierConfigMgr::acquire_operation_lease(&store.tier_config_mgr(), "COLD-C").await {
20983 Ok(_) => panic!("mismatched digest must retain the prepared runtime fence"),
20984 Err(err) => err,
20985 };
20986 assert!(bad_digest_blocked.message.contains("being replaced"), "{bad_digest_blocked}");
20987 handle_tier_mutation_peer_request(
20988 store.clone(),
20989 TIER_MUTATION_RPC_PROTOCOL_VERSION,
20990 TierMutationRpcPhase::Abort,
20991 bad_digest_id,
20992 &bad_digest_intent.encode().expect("bad digest abort identity should encode"),
20993 )
20994 .await
20995 .expect("the negative digest proof fixture should clean up through abort");
20996
20997 let committed = handle_tier_mutation_peer_request(
20998 store.clone(),
20999 TIER_MUTATION_RPC_PROTOCOL_VERSION,
21000 TierMutationRpcPhase::Commit,
21001 mutation_id,
21002 config_etag.as_bytes(),
21003 )
21004 .await
21005 .expect("commit should advance the prepared peer intent");
21006 assert!(committed.applied);
21007 assert_eq!(committed.state, TierMutationPeerState::Committed);
21008 let committed_blocked = match TierConfigMgr::acquire_operation_lease(&store.tier_config_mgr(), "COLD-A").await {
21009 Ok(_) => panic!("committed peer mutation must remain blocked until local reload publishes the config"),
21010 Err(err) => err,
21011 };
21012 assert!(
21013 committed_blocked.message.contains("being replaced"),
21014 "committed peer mutation should keep the existing blocked-tier error: {committed_blocked}"
21015 );
21016
21017 let retried_commit = handle_tier_mutation_peer_request(
21018 store.clone(),
21019 TIER_MUTATION_RPC_PROTOCOL_VERSION,
21020 TierMutationRpcPhase::Commit,
21021 mutation_id,
21022 config_etag.as_bytes(),
21023 )
21024 .await
21025 .expect("same commit retry should be idempotent");
21026 assert!(!retried_commit.applied);
21027 assert_eq!(retried_commit.state, TierMutationPeerState::Committed);
21028 let retried_commit_blocked = match TierConfigMgr::acquire_operation_lease(&store.tier_config_mgr(), "COLD-A").await {
21029 Ok(_) => panic!("committed retry must keep the tier blocked until local reload"),
21030 Err(err) => err,
21031 };
21032 assert!(
21033 retried_commit_blocked.message.contains("being replaced"),
21034 "committed retry should keep the existing blocked-tier error: {retried_commit_blocked}"
21035 );
21036
21037 let delayed_prepare_retry = handle_tier_mutation_peer_request(
21038 store.clone(),
21039 TIER_MUTATION_RPC_PROTOCOL_VERSION,
21040 TierMutationRpcPhase::Prepare,
21041 mutation_id,
21042 &prepare_payload,
21043 )
21044 .await
21045 .expect("delayed duplicate prepare should report the durable committed state");
21046 assert!(!delayed_prepare_retry.applied);
21047 assert_eq!(delayed_prepare_retry.state, TierMutationPeerState::Committed);
21048 let delayed_prepare_blocked = match TierConfigMgr::acquire_operation_lease(&store.tier_config_mgr(), "COLD-A").await {
21049 Ok(_) => panic!("delayed committed prepare retry must preserve the committed runtime block"),
21050 Err(err) => err,
21051 };
21052 assert!(
21053 delayed_prepare_blocked.message.contains("being replaced"),
21054 "delayed committed prepare retry should keep the existing blocked-tier error: {delayed_prepare_blocked}"
21055 );
21056
21057 let loaded = load_tier_mutation_intent_record(store.clone(), mutation_id)
21058 .await
21059 .expect("committed peer intent should remain durable");
21060 assert_eq!(loaded.state, TierMutationIntentState::Committed);
21061 assert_eq!(loaded.committed_config_etag.as_deref(), Some(config_etag.as_str()));
21062
21063 store
21064 .tier_config_mgr()
21065 .read()
21066 .await
21067 .save_tiering_config(store.clone())
21068 .await
21069 .expect("tier config should persist for cleaned intent commit proof");
21070 let tier_config_info = store
21071 .get_object_info(
21072 RUSTFS_META_BUCKET,
21073 &format!("{}/{}", com::CONFIG_PREFIX, TIER_CONFIG_FILE),
21074 &ObjectOptions::default(),
21075 )
21076 .await
21077 .expect("tier config object info should load");
21078 let tier_config_etag = tier_config_info.etag.expect("tier config should carry an ETag");
21079 delete_tier_mutation_intent_record(store.clone(), mutation_id)
21080 .await
21081 .expect("simulate another node cleaning the shared committed peer intent");
21082 let cleaned_commit_retry = handle_tier_mutation_peer_request(
21083 store.clone(),
21084 TIER_MUTATION_RPC_PROTOCOL_VERSION,
21085 TierMutationRpcPhase::Commit,
21086 mutation_id,
21087 tier_config_etag.as_bytes(),
21088 )
21089 .await
21090 .expect("commit retry after durable cleanup should be idempotently terminal");
21091 assert!(!cleaned_commit_retry.applied);
21092 assert_eq!(cleaned_commit_retry.state, TierMutationPeerState::Committed);
21093 let cleaned_commit_blocked = match TierConfigMgr::acquire_operation_lease(&store.tier_config_mgr(), "COLD-A").await {
21094 Ok(_) => panic!("shared intent cleanup must not clear this node's committed runtime block"),
21095 Err(err) => err,
21096 };
21097 assert!(
21098 cleaned_commit_blocked.message.contains("being replaced"),
21099 "commit retry after shared cleanup should keep the existing blocked-tier error: {cleaned_commit_blocked}"
21100 );
21101 let mismatched_cleaned_commit = handle_tier_mutation_peer_request(
21102 store.clone(),
21103 TIER_MUTATION_RPC_PROTOCOL_VERSION,
21104 TierMutationRpcPhase::Commit,
21105 mutation_id,
21106 b"not-the-current-etag",
21107 )
21108 .await
21109 .expect_err("missing intent without a matching committed config ETag must fail closed");
21110 assert!(matches!(mismatched_cleaned_commit, TierMutationPeerError::Store(Error::ConfigNotFound)));
21111 let refresh_store = store.clone();
21112 let refresh_manager = store.tier_config_mgr();
21113 let refresh_worker = tokio::spawn(async move {
21114 TierConfigMgr::refresh_tier_config_handle_with(refresh_manager, refresh_store).await;
21115 });
21116 tokio::time::timeout(Duration::from_secs(5), async {
21117 loop {
21118 if let Ok(lease) = TierConfigMgr::acquire_operation_lease(&store.tier_config_mgr(), "COLD-A").await {
21119 drop(lease);
21120 break;
21121 }
21122 tokio::task::yield_now().await;
21123 }
21124 })
21125 .await
21126 .expect("matching cleaned commit should wake the refresh worker and clear the committed fence");
21127 refresh_worker.abort();
21128 let _ = refresh_worker.await;
21129
21130 let abort_id = uuid::Uuid::new_v4();
21131 register_mock_tier(&store.tier_config_mgr(), "COLD-B").await;
21132 store
21133 .tier_config_mgr()
21134 .read()
21135 .await
21136 .save_tiering_config(store.clone())
21137 .await
21138 .expect("abort target tier config should persist");
21139 let abort_config_info = store
21140 .get_object_info(
21141 RUSTFS_META_BUCKET,
21142 &format!("{}/{}", com::CONFIG_PREFIX, TIER_CONFIG_FILE),
21143 &ObjectOptions::default(),
21144 )
21145 .await
21146 .expect("abort target config metadata should load");
21147 let abort_config_etag = abort_config_info.etag.expect("abort target config should carry an ETag");
21148 let mut abort_intent = tier_mutation_peer_test_intent(abort_id, "COLD-B", [4; 32]);
21149 abort_intent.old_config_etag = Some(abort_config_etag);
21150 let abort_prepare_payload = abort_intent.encode().expect("abort prepare intent should encode");
21151 handle_tier_mutation_peer_request(
21152 store.clone(),
21153 TIER_MUTATION_RPC_PROTOCOL_VERSION,
21154 TierMutationRpcPhase::Prepare,
21155 abort_id,
21156 &abort_prepare_payload,
21157 )
21158 .await
21159 .expect("abort target prepare should create the peer intent");
21160 let abort_blocked = match TierConfigMgr::acquire_operation_lease(&store.tier_config_mgr(), "COLD-B").await {
21161 Ok(_) => panic!("abort target prepare should block new tier operation leases"),
21162 Err(err) => err,
21163 };
21164 assert!(
21165 abort_blocked.message.contains("being replaced"),
21166 "abort target prepare should reuse the existing blocked-tier error: {abort_blocked}"
21167 );
21168
21169 let aborted = handle_tier_mutation_peer_request(
21170 store.clone(),
21171 TIER_MUTATION_RPC_PROTOCOL_VERSION,
21172 TierMutationRpcPhase::Abort,
21173 abort_id,
21174 &abort_prepare_payload,
21175 )
21176 .await
21177 .expect("abort should advance the prepared peer intent");
21178 assert!(aborted.applied);
21179 assert_eq!(aborted.state, TierMutationPeerState::Aborted);
21180 TierConfigMgr::acquire_operation_lease(&store.tier_config_mgr(), "COLD-B")
21181 .await
21182 .expect("durable abort must clear the peer runtime fence immediately");
21183
21184 let retried_abort = handle_tier_mutation_peer_request(
21185 store.clone(),
21186 TIER_MUTATION_RPC_PROTOCOL_VERSION,
21187 TierMutationRpcPhase::Abort,
21188 abort_id,
21189 &abort_prepare_payload,
21190 )
21191 .await
21192 .expect("same abort retry should be idempotent before recovery cleanup");
21193 assert!(!retried_abort.applied);
21194 assert_eq!(retried_abort.state, TierMutationPeerState::Aborted);
21195 }
21196
21197 #[cfg(feature = "test-util")]
21198 #[tokio::test]
21199 #[serial_test::serial(storage_class_env)]
21200 async fn late_abort_after_config_commit_keeps_fence_until_commit_recovery() {
21201 let temp_dir = tempfile::tempdir().expect("create temp store dir");
21202 let (_ctx, store, _shutdown) =
21203 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "tier-mutation-late-abort", &[4])).await;
21204 register_mock_tier(&store.tier_config_mgr(), "COLD-A").await;
21205 store
21206 .tier_config_mgr()
21207 .read()
21208 .await
21209 .save_tiering_config(store.clone())
21210 .await
21211 .expect("base tier config should persist");
21212 let base_config_info = store
21213 .get_object_info(
21214 RUSTFS_META_BUCKET,
21215 &format!("{}/{}", com::CONFIG_PREFIX, TIER_CONFIG_FILE),
21216 &ObjectOptions::default(),
21217 )
21218 .await
21219 .expect("base tier config metadata should load");
21220 let base_etag = base_config_info.etag.expect("base tier config should carry an ETag");
21221
21222 register_mock_tier(&store.tier_config_mgr(), "COLD-B").await;
21223 let candidate_digest = {
21224 let manager = store.tier_config_mgr();
21225 let manager = manager.read().await;
21226 tier_config_candidate_digest(&manager).expect("candidate digest should build")
21227 };
21228 let mutation_id = uuid::Uuid::new_v4();
21229 let mut intent = tier_mutation_peer_test_intent(mutation_id, "COLD-B", candidate_digest);
21230 intent.old_config_etag = Some(base_etag.clone());
21231 let prepare_payload = intent.encode().expect("late abort prepare intent should encode");
21232 handle_tier_mutation_peer_request(
21233 store.clone(),
21234 TIER_MUTATION_RPC_PROTOCOL_VERSION,
21235 TierMutationRpcPhase::Prepare,
21236 mutation_id,
21237 &prepare_payload,
21238 )
21239 .await
21240 .expect("prepare should install the runtime fence");
21241
21242 store
21243 .tier_config_mgr()
21244 .read()
21245 .await
21246 .save_tiering_config(store.clone())
21247 .await
21248 .expect("candidate tier config should persist before the late abort");
21249 let committed_config_info = store
21250 .get_object_info(
21251 RUSTFS_META_BUCKET,
21252 &format!("{}/{}", com::CONFIG_PREFIX, TIER_CONFIG_FILE),
21253 &ObjectOptions::default(),
21254 )
21255 .await
21256 .expect("committed tier config metadata should load");
21257 let committed_etag = committed_config_info
21258 .etag
21259 .expect("committed tier config should carry an ETag");
21260 assert_ne!(committed_etag, base_etag);
21261
21262 let late_abort = handle_tier_mutation_peer_request(
21263 store.clone(),
21264 TIER_MUTATION_RPC_PROTOCOL_VERSION,
21265 TierMutationRpcPhase::Abort,
21266 mutation_id,
21267 &prepare_payload,
21268 )
21269 .await
21270 .expect_err("an abort after the candidate config commit must fail closed");
21271 assert!(matches!(late_abort, TierMutationPeerError::AbortProofMismatch));
21272 let prepared = load_tier_mutation_intent_record(store.clone(), mutation_id)
21273 .await
21274 .expect("rejected late abort must retain the prepared intent");
21275 assert_eq!(prepared.state, TierMutationIntentState::Prepared);
21276 let blocked = match TierConfigMgr::acquire_operation_lease(&store.tier_config_mgr(), "COLD-B").await {
21277 Ok(_) => panic!("rejected late abort must retain the runtime fence"),
21278 Err(err) => err,
21279 };
21280 assert!(blocked.message.contains("being replaced"), "{blocked}");
21281
21282 let committed = handle_tier_mutation_peer_request(
21283 store.clone(),
21284 TIER_MUTATION_RPC_PROTOCOL_VERSION,
21285 TierMutationRpcPhase::Commit,
21286 mutation_id,
21287 committed_etag.as_bytes(),
21288 )
21289 .await
21290 .expect("matching commit should converge the rejected late abort fixture");
21291 assert!(committed.applied);
21292 assert_eq!(committed.state, TierMutationPeerState::Committed);
21293
21294 let refresh_store = store.clone();
21295 let refresh_manager = store.tier_config_mgr();
21296 let refresh_worker = tokio::spawn(async move {
21297 TierConfigMgr::refresh_tier_config_handle_with(refresh_manager, refresh_store).await;
21298 });
21299 tokio::time::timeout(Duration::from_secs(5), async {
21300 loop {
21301 if let Ok(lease) = TierConfigMgr::acquire_operation_lease(&store.tier_config_mgr(), "COLD-B").await {
21302 drop(lease);
21303 break;
21304 }
21305 tokio::task::yield_now().await;
21306 }
21307 })
21308 .await
21309 .expect("commit recovery should publish before clearing the late-abort fence");
21310 refresh_worker.abort();
21311 let _ = refresh_worker.await;
21312 }
21313
21314 #[cfg(feature = "test-util")]
21315 #[tokio::test]
21316 #[serial_test::serial(storage_class_env)]
21317 async fn missing_record_commit_promotes_prepared_fence_until_worker_publish() {
21318 let temp_dir = tempfile::tempdir().expect("create temp store dir");
21319 let (_ctx, store, _shutdown) =
21320 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "tier-mutation-missing-record-commit", &[4]))
21321 .await;
21322 register_mock_tier(&store.tier_config_mgr(), "COLD-A").await;
21323 let tier_config_mgr = store.tier_config_mgr();
21324 let candidate_digest = {
21325 let manager = tier_config_mgr.read().await;
21326 let digest = tier_config_candidate_digest(&manager).expect("candidate digest should build");
21327 manager
21328 .save_tiering_config(store.clone())
21329 .await
21330 .expect("candidate config should persist");
21331 digest
21332 };
21333 let config_info = store
21334 .get_object_info(
21335 RUSTFS_META_BUCKET,
21336 &format!("{}/{}", com::CONFIG_PREFIX, TIER_CONFIG_FILE),
21337 &ObjectOptions::default(),
21338 )
21339 .await
21340 .expect("candidate config metadata should load");
21341 let config_etag = config_info.etag.expect("candidate config should carry an ETag");
21342 let mutation_id = uuid::Uuid::new_v4();
21343 let intent = tier_mutation_peer_test_intent(mutation_id, "COLD-A", candidate_digest);
21344 handle_tier_mutation_peer_request(
21345 store.clone(),
21346 TIER_MUTATION_RPC_PROTOCOL_VERSION,
21347 TierMutationRpcPhase::Prepare,
21348 mutation_id,
21349 &intent.encode().expect("prepare intent should encode"),
21350 )
21351 .await
21352 .expect("prepare should install the runtime fence");
21353 delete_tier_mutation_intent_record(store.clone(), mutation_id)
21354 .await
21355 .expect("simulate shared intent cleanup before the local commit arrives");
21356
21357 let committed = handle_tier_mutation_peer_request(
21358 store.clone(),
21359 TIER_MUTATION_RPC_PROTOCOL_VERSION,
21360 TierMutationRpcPhase::Commit,
21361 mutation_id,
21362 config_etag.as_bytes(),
21363 )
21364 .await
21365 .expect("matching commit after shared cleanup should be terminal");
21366 assert!(!committed.applied);
21367 assert_eq!(committed.state, TierMutationPeerState::Committed);
21368 let blocked = match TierConfigMgr::acquire_operation_lease(&tier_config_mgr, "COLD-A").await {
21369 Ok(_) => panic!("the promoted committed fence must block old-generation leases"),
21370 Err(err) => err,
21371 };
21372 assert!(blocked.message.contains("being replaced"), "{blocked}");
21373
21374 let refresh_store = store.clone();
21375 let refresh_manager = tier_config_mgr.clone();
21376 let refresh_worker = tokio::spawn(async move {
21377 TierConfigMgr::refresh_tier_config_handle_with(refresh_manager, refresh_store).await;
21378 });
21379 tokio::time::timeout(Duration::from_secs(5), async {
21380 loop {
21381 if let Ok(lease) = TierConfigMgr::acquire_operation_lease(&tier_config_mgr, "COLD-A").await {
21382 drop(lease);
21383 break;
21384 }
21385 tokio::task::yield_now().await;
21386 }
21387 })
21388 .await
21389 .expect("the commit notification should drive publish and clear the promoted fence");
21390 refresh_worker.abort();
21391 let _ = refresh_worker.await;
21392 }
21393
21394 #[cfg(feature = "test-util")]
21395 #[tokio::test]
21396 #[serial_test::serial(storage_class_env)]
21397 async fn tier_mutation_peer_handler_rejects_conflicting_prepare_without_overwrite() {
21398 let temp_dir = tempfile::tempdir().expect("create temp store dir");
21399 let (_ctx, store, _shutdown) =
21400 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "tier-mutation-peer-conflict", &[4])).await;
21401 let mutation_id = uuid::Uuid::new_v4();
21402 let intent = tier_mutation_peer_test_intent(mutation_id, "COLD-A", [3; 32]);
21403 let prepare_payload = intent.encode().expect("prepare intent should encode");
21404 handle_tier_mutation_peer_request(
21405 store.clone(),
21406 TIER_MUTATION_RPC_PROTOCOL_VERSION,
21407 TierMutationRpcPhase::Prepare,
21408 mutation_id,
21409 &prepare_payload,
21410 )
21411 .await
21412 .expect("first prepare should create the peer intent");
21413
21414 let conflicting = tier_mutation_peer_test_intent(mutation_id, "COLD-A", [4; 32]);
21415 let conflicting_payload = conflicting.encode().expect("conflicting intent should encode");
21416 let conflict = handle_tier_mutation_peer_request(
21417 store.clone(),
21418 TIER_MUTATION_RPC_PROTOCOL_VERSION,
21419 TierMutationRpcPhase::Prepare,
21420 mutation_id,
21421 &conflicting_payload,
21422 )
21423 .await
21424 .expect_err("conflicting prepare must fail closed");
21425 assert!(matches!(conflict, TierMutationPeerError::ConflictingIntent));
21426
21427 let loaded = load_tier_mutation_intent_record(store, mutation_id)
21428 .await
21429 .expect("conflicting prepare must not overwrite the first record");
21430 assert_eq!(loaded.candidate_digest, [3; 32]);
21431 assert_eq!(loaded.state, TierMutationIntentState::Prepared);
21432 }
21433
21434 #[cfg(feature = "test-util")]
21435 #[tokio::test]
21436 #[serial_test::serial(storage_class_env)]
21437 async fn tier_mutation_intent_record_scan_paginates_exact_limit() {
21438 let temp_dir = tempfile::tempdir().expect("create temp store dir");
21439 let (_ctx, store, _shutdown) =
21440 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "tier-mutation-intent-scan-page", &[4])).await;
21441 let build_intent = |mutation_id: uuid::Uuid, tier_name: &str| TierMutationIntent {
21442 mutation_id,
21443 revision: 1,
21444 kind: TierMutationIntentKind::Edit,
21445 state: TierMutationIntentState::Prepared,
21446 old_config_etag: Some("old-etag".to_string()),
21447 committed_config_etag: None,
21448 candidate_digest: [3; 32],
21449 affected_targets: vec![TierMutationIntentTarget {
21450 tier_name: tier_name.to_string(),
21451 old_backend_identity: Some([1; 32]),
21452 new_backend_identity: Some([2; 32]),
21453 }],
21454 expires_at_unix_nanos: 1_780_000_000_000_000_000,
21455 };
21456 let intent_ids = vec![
21457 uuid::Uuid::parse_str("11345678-1234-5678-9abc-def012345678").expect("first uuid should parse"),
21458 uuid::Uuid::parse_str("22345678-1234-5678-9abc-def012345678").expect("second uuid should parse"),
21459 uuid::Uuid::parse_str("33345678-1234-5678-9abc-def012345678").expect("third uuid should parse"),
21460 ];
21461 for (index, mutation_id) in intent_ids.iter().copied().enumerate() {
21462 let intent = build_intent(mutation_id, &format!("COLD-{index}"));
21463 save_tier_mutation_intent_record(store.clone(), &intent)
21464 .await
21465 .expect("tier mutation intent record should persist");
21466 }
21467
21468 let exact_page = list_tier_mutation_intent_records(store.clone(), 3, None)
21469 .await
21470 .expect("exact full page should scan");
21471 assert_eq!(exact_page.scanned, 3);
21472 assert_eq!(exact_page.intents.len(), 3);
21473 assert_eq!(exact_page.failed, 0);
21474 assert!(!exact_page.truncated);
21475 assert_eq!(exact_page.next_marker, None);
21476
21477 let first_page = list_tier_mutation_intent_records(store.clone(), 2, None)
21478 .await
21479 .expect("first page should scan");
21480 assert_eq!(first_page.scanned, 2);
21481 assert_eq!(first_page.intents.len(), 2);
21482 assert_eq!(first_page.failed, 0);
21483 assert!(first_page.truncated);
21484 assert!(first_page.next_marker.is_some());
21485
21486 let second_page = list_tier_mutation_intent_records(store, 2, first_page.next_marker)
21487 .await
21488 .expect("second page should scan");
21489 let mut loaded_ids: Vec<_> = first_page
21490 .intents
21491 .into_iter()
21492 .chain(second_page.intents.into_iter())
21493 .map(|intent| intent.mutation_id)
21494 .collect();
21495 loaded_ids.sort();
21496
21497 assert_eq!(second_page.scanned, 1);
21498 assert_eq!(second_page.failed, 0);
21499 assert!(!second_page.truncated);
21500 assert_eq!(second_page.next_marker, None);
21501 assert_eq!(loaded_ids, intent_ids);
21502 }
21503
21504 #[cfg(feature = "test-util")]
21505 fn transition_mutation_measurement(observations: &[TransitionTransactionMutationObservation]) -> (usize, usize, u128) {
21506 (
21507 observations.len(),
21508 observations.iter().map(|observation| observation.encoded_bytes).sum(),
21509 observations.iter().map(|observation| observation.elapsed.as_micros()).sum(),
21510 )
21511 }
21512
21513 #[cfg(feature = "test-util")]
21514 #[tokio::test]
21515 #[serial_test::serial(storage_class_env)]
21516 async fn compact_transition_transactions_halve_success_path_quorum_mutations() {
21517 let temp_dir = tempfile::tempdir().expect("create transition mutation measurement dir");
21518 let (ctx, store, shutdown) =
21519 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "transition-mutation-measurement", &[4])).await;
21520 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
21521 let tier_name = "MUTATION-MEASURE";
21522 register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
21523 let bucket = "transition-mutation-measurement";
21524 store
21525 .make_bucket(bucket, &MakeBucketOptions::default())
21526 .await
21527 .expect("measurement bucket should be created");
21528
21529 let run_case = |profile: &'static str, size: usize| {
21530 let store = store.clone();
21531 async move {
21532 let object = format!("{profile}-{size}.bin");
21533 let mut reader = PutObjReader::from_vec(vec![b'm'; size]);
21534 let original = store
21535 .put_object(bucket, &object, &mut reader, &ObjectOptions::default())
21536 .await
21537 .expect("measurement source should be written");
21538 let probe = TransitionTransactionMutationProbe::install(bucket, &object);
21539 store
21540 .transition_object(
21541 bucket,
21542 &object,
21543 &ObjectOptions {
21544 transition: TransitionOptions {
21545 status: TRANSITION_PENDING.to_string(),
21546 tier: tier_name.to_string(),
21547 etag: original.etag.clone().expect("measurement source should have an ETag"),
21548 ..Default::default()
21549 },
21550 version_id: original.version_id.map(|version| version.to_string()),
21551 mod_time: original.mod_time,
21552 ..Default::default()
21553 },
21554 )
21555 .await
21556 .expect("measurement transition should commit");
21557 probe.observations()
21558 }
21559 };
21560
21561 let sizes = [4 * 1024, 1024 * 1024];
21562 let mut legacy = Vec::with_capacity(sizes.len());
21563 for size in sizes {
21564 legacy.push((size, run_case("legacy", size).await));
21565 }
21566
21567 let compaction_proof = install_transition_transaction_compaction_fleet_proof_for_test("object-transaction-fencing-test");
21568 for ((size, legacy_observations), compact_size) in legacy.into_iter().zip(sizes) {
21569 assert_eq!(size, compact_size);
21570 let compact_observations = run_case("compact", size).await;
21571 let legacy_measurement = transition_mutation_measurement(&legacy_observations);
21572 let compact_measurement = transition_mutation_measurement(&compact_observations);
21573 assert_eq!(legacy_measurement.0, 6, "legacy success should use five saves and one delete");
21574 assert_eq!(compact_measurement.0, 3, "compact success should use two saves and one delete");
21575 assert!(
21576 compact_measurement.1 < legacy_measurement.1,
21577 "compact transaction bodies should write fewer aggregate bytes"
21578 );
21579 assert_eq!(
21580 legacy_observations
21581 .iter()
21582 .map(|observation| (observation.kind, observation.previous_state, observation.state))
21583 .collect::<Vec<_>>(),
21584 vec![
21585 (TransitionTransactionMutationKind::Create, None, TransitionTransactionState::UploadStarted),
21586 (
21587 TransitionTransactionMutationKind::CompareAndSave,
21588 Some(TransitionTransactionState::UploadStarted),
21589 TransitionTransactionState::UploadOutcomeUnknown,
21590 ),
21591 (
21592 TransitionTransactionMutationKind::CompareAndSave,
21593 Some(TransitionTransactionState::UploadOutcomeUnknown),
21594 TransitionTransactionState::Uploaded,
21595 ),
21596 (
21597 TransitionTransactionMutationKind::CompareAndSave,
21598 Some(TransitionTransactionState::Uploaded),
21599 TransitionTransactionState::LocalCommitStarted,
21600 ),
21601 (
21602 TransitionTransactionMutationKind::CompareAndSave,
21603 Some(TransitionTransactionState::LocalCommitStarted),
21604 TransitionTransactionState::Committed,
21605 ),
21606 (
21607 TransitionTransactionMutationKind::Delete,
21608 Some(TransitionTransactionState::Committed),
21609 TransitionTransactionState::Committed,
21610 ),
21611 ]
21612 );
21613 assert_eq!(
21614 compact_observations
21615 .iter()
21616 .map(|observation| (observation.kind, observation.previous_state, observation.state))
21617 .collect::<Vec<_>>(),
21618 vec![
21619 (
21620 TransitionTransactionMutationKind::Create,
21621 None,
21622 TransitionTransactionState::UploadOutcomeUnknown,
21623 ),
21624 (
21625 TransitionTransactionMutationKind::CompareAndSave,
21626 Some(TransitionTransactionState::UploadOutcomeUnknown),
21627 TransitionTransactionState::LocalCommitStarted,
21628 ),
21629 (
21630 TransitionTransactionMutationKind::Delete,
21631 Some(TransitionTransactionState::LocalCommitStarted),
21632 TransitionTransactionState::LocalCommitStarted,
21633 ),
21634 ]
21635 );
21636 assert!(legacy_observations.iter().all(|observation| observation.succeeded));
21637 assert!(compact_observations.iter().all(|observation| observation.succeeded));
21638 println!(
21639 "transition_mutation_measurement,profile=legacy,size={size},mutations={},encoded_bytes={},latency_us={},quorum_operations={}",
21640 legacy_measurement.0, legacy_measurement.1, legacy_measurement.2, legacy_measurement.0
21641 );
21642 println!(
21643 "transition_mutation_measurement,profile=compact,size={size},mutations={},encoded_bytes={},latency_us={},quorum_operations={}",
21644 compact_measurement.0, compact_measurement.1, compact_measurement.2, compact_measurement.0
21645 );
21646 }
21647 assert_eq!(transition_transaction_record_count(store.clone()).await, 0);
21648 let admitted = acquire_transition_transaction_compaction_fleet_proof()
21649 .expect("published homogeneous proof should admit one compact writer");
21650 assert!(transition_transaction_compaction_fleet_proof_matches(&admitted));
21651 drop(compaction_proof);
21652 assert!(
21653 !transition_transaction_compaction_fleet_proof_matches(&admitted),
21654 "revocation must fence a writer admitted by the previous process-epoch snapshot"
21655 );
21656 drop(admitted);
21657 assert!(
21658 acquire_transition_transaction_compaction_fleet_proof().is_none(),
21659 "revoking the homogeneous proof must restore the legacy writer profile"
21660 );
21661 shutdown.cancel();
21662 }
21663
21664 #[cfg(feature = "test-util")]
21665 #[tokio::test]
21666 #[serial_test::serial(storage_class_env)]
21667 async fn compact_transition_kill_points_preserve_the_only_remote_owner() {
21668 let temp_dir = tempfile::tempdir().expect("create compact transition kill-point dir");
21669 let (ctx, store, shutdown) =
21670 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "compact-transition-kill-points", &[4])).await;
21671 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
21672 let tier_name = "COMPACT-KILL";
21673 let backend = register_transition_reconcile_test_tier(&ctx.tier_config_mgr(), tier_name).await;
21674 let _tier_lease = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
21675 .await
21676 .expect("mock tier lease should remain available during recovery");
21677 let bucket = "compact-transition-kill-points";
21678 store
21679 .make_bucket(bucket, &MakeBucketOptions::default())
21680 .await
21681 .expect("kill-point bucket should be created");
21682 let _compaction_proof = install_transition_transaction_compaction_fleet_proof_for_test("object-transaction-fencing-test");
21683
21684 for (index, point, expected_state, expected_revision, expected_committed, expected_recovered) in [
21685 (
21686 0,
21687 TransitionTransactionKillPoint::PrePutFence,
21688 TransitionTransactionState::UploadOutcomeUnknown,
21689 1,
21690 false,
21691 true,
21692 ),
21693 (
21694 1,
21695 TransitionTransactionKillPoint::UploadBeforeCommitFence,
21696 TransitionTransactionState::UploadOutcomeUnknown,
21697 1,
21698 false,
21699 true,
21700 ),
21701 (
21702 2,
21703 TransitionTransactionKillPoint::LocalCommitBeforeDelete,
21704 TransitionTransactionState::LocalCommitStarted,
21705 2,
21706 true,
21707 true,
21708 ),
21709 (
21710 3,
21711 TransitionTransactionKillPoint::CommitFenceBeforeLocalCommit,
21712 TransitionTransactionState::LocalCommitStarted,
21713 2,
21714 false,
21715 false,
21716 ),
21717 ] {
21718 let object = format!("kill-point-{index}.bin");
21719 let payload = vec![b'k' + u8::try_from(index).expect("small case index should fit u8"); 64 * 1024];
21720 let mut reader = PutObjReader::from_vec(payload.clone());
21721 let source = store
21722 .put_object(bucket, &object, &mut reader, &ObjectOptions::default())
21723 .await
21724 .expect("kill-point source should be written");
21725 let remote_before = backend.object_count().await;
21726 let barrier = TransitionTransactionKillPointBarrier::install(bucket, &object, point);
21727 let transition_store = store.clone();
21728 let transition_object = object.clone();
21729 let transition = tokio::spawn(async move {
21730 transition_store
21731 .transition_object(
21732 bucket,
21733 &transition_object,
21734 &ObjectOptions {
21735 transition: TransitionOptions {
21736 status: TRANSITION_PENDING.to_string(),
21737 tier: tier_name.to_string(),
21738 etag: source.etag.clone().expect("kill-point source should have an ETag"),
21739 ..Default::default()
21740 },
21741 version_id: source.version_id.map(|version| version.to_string()),
21742 mod_time: source.mod_time,
21743 ..Default::default()
21744 },
21745 )
21746 .await
21747 });
21748 barrier.wait_until_paused().await;
21749 transition.abort();
21750 assert!(
21751 transition
21752 .await
21753 .expect_err("kill-point transition should be cancelled")
21754 .is_cancelled(),
21755 "kill-point transition should stop without unwinding"
21756 );
21757 drop(barrier);
21758
21759 let transaction = only_transition_transaction(store.clone()).await;
21760 assert_eq!((transaction.state, transaction.revision), (expected_state, expected_revision));
21761 let paused_source = store
21762 .get_object_info(
21763 bucket,
21764 &object,
21765 &ObjectOptions {
21766 metadata_cache_safe: false,
21767 ..Default::default()
21768 },
21769 )
21770 .await
21771 .expect("kill-point source metadata should remain readable");
21772 assert_eq!(
21773 paused_source.transitioned_object.status == rustfs_filemeta::TRANSITION_COMPLETE,
21774 expected_committed
21775 );
21776 let expected_remote_at_pause = remote_before + usize::from(point != TransitionTransactionKillPoint::PrePutFence);
21777 assert_eq!(backend.object_count().await, expected_remote_at_pause);
21778
21779 let stats = recover_transition_transaction_records_at(
21780 store.clone(),
21781 100,
21782 None,
21783 i128::from(transaction.not_after_unix_nanos) + 1,
21784 )
21785 .await
21786 .expect("kill-point transaction recovery should complete");
21787 if expected_recovered {
21788 assert_eq!((stats.scanned, stats.recovered, stats.retained, stats.failed), (1, 1, 0, 0));
21789 assert_eq!(transition_transaction_record_count(store.clone()).await, 0);
21790 } else {
21791 assert_eq!((stats.scanned, stats.recovered, stats.retained, stats.failed), (1, 0, 1, 0));
21792 assert_eq!(
21793 transition_transaction_record_count(store.clone()).await,
21794 1,
21795 "an uncommitted local-commit fence must retain its exact remote owner"
21796 );
21797 }
21798 let expected_remote_after_recovery = if matches!(
21799 point,
21800 TransitionTransactionKillPoint::LocalCommitBeforeDelete
21801 | TransitionTransactionKillPoint::CommitFenceBeforeLocalCommit
21802 ) {
21803 remote_before + 1
21804 } else {
21805 remote_before
21806 };
21807 assert_eq!(backend.object_count().await, expected_remote_after_recovery);
21808 assert_eq!(backend.remove_count().await, usize::from(index >= 1));
21809
21810 let mut restored = Vec::new();
21811 store
21812 .get_object_reader(bucket, &object, None, HeaderMap::new(), &ObjectOptions::default())
21813 .await
21814 .expect("kill-point source should remain readable through its authoritative location")
21815 .stream
21816 .read_to_end(&mut restored)
21817 .await
21818 .expect("kill-point source body should drain");
21819 assert_eq!(restored, payload);
21820
21821 if !expected_recovered {
21822 break;
21823 }
21824 }
21825 shutdown.cancel();
21826 }
21827
21828 #[cfg(feature = "test-util")]
21829 #[tokio::test]
21830 #[serial_test::serial(storage_class_env)]
21831 async fn cancelled_transition_cleanup_recovers_from_its_own_instance_transaction() {
21832 struct ResolverReset(Arc<std::sync::Mutex<Option<std::sync::Weak<crate::store::ECStore>>>>);
21833
21834 impl Drop for ResolverReset {
21835 fn drop(&mut self) {
21836 *self.0.lock().unwrap_or_else(std::sync::PoisonError::into_inner) = None;
21837 }
21838 }
21839
21840 let temp_a = tempfile::tempdir().expect("create transition store dir a");
21841 let temp_b = tempfile::tempdir().expect("create transition store dir b");
21842 let shutdown_a = CancellationToken::new();
21843 let shutdown_b = CancellationToken::new();
21844 shutdown_a.cancel();
21845 shutdown_b.cancel();
21846 let (ctx_a, store_a, shutdown_a) = without_storage_class_env(build_isolated_test_store_with_shutdown(
21847 temp_a.path(),
21848 "transition-cleanup-context-a",
21849 &[4],
21850 shutdown_a,
21851 ))
21852 .await;
21853 let (ctx_b, store_b, shutdown_b) = without_storage_class_env(build_isolated_test_store_with_shutdown(
21854 temp_b.path(),
21855 "transition-cleanup-context-b",
21856 &[4],
21857 shutdown_b,
21858 ))
21859 .await;
21860 assert!(shutdown_a.is_cancelled());
21861 assert!(shutdown_b.is_cancelled());
21862 crate::bucket::metadata_sys::init_bucket_metadata_sys(store_a.clone(), Vec::new()).await;
21863
21864 let resolver_target = Arc::new(std::sync::Mutex::new(Some(Arc::downgrade(&store_b))));
21865 let resolver_store = resolver_target.clone();
21866 assert!(
21867 set_object_store_resolver(Arc::new(move || {
21868 resolver_store
21869 .lock()
21870 .unwrap_or_else(std::sync::PoisonError::into_inner)
21871 .as_ref()
21872 .and_then(std::sync::Weak::upgrade)
21873 })),
21874 "the cross-context regression test must install the only process object-store resolver"
21875 );
21876 let _resolver_reset = ResolverReset(resolver_target);
21877 assert!(
21878 runtime_sources::object_store_handle().is_some_and(|store| Arc::ptr_eq(&store, &store_b)),
21879 "the process resolver must deliberately point at store B"
21880 );
21881
21882 let tier_name = "CROSSCTXA";
21883 let backend = register_transition_reconcile_test_tier(&ctx_a.tier_config_mgr(), tier_name).await;
21884 backend.set_put_remote_version(Some(uuid::Uuid::new_v4().to_string())).await;
21885 backend.reject_next_non_empty_remote_version_validation();
21886 let remove_barrier = backend.arm_failing_remove_barrier().await;
21887
21888 let bucket = "transition-cleanup-context-a";
21889 let object = "rejected-candidate.bin";
21890 store_a
21891 .make_bucket(bucket, &MakeBucketOptions::default())
21892 .await
21893 .expect("store A bucket should be created");
21894 let mut reader = PutObjReader::from_vec(b"cross-context rejected transition cleanup".repeat(1024));
21895 let original = store_a
21896 .put_object(bucket, object, &mut reader, &ObjectOptions::default())
21897 .await
21898 .expect("store A source object should be written");
21899 let opts = ObjectOptions {
21900 no_lock: true,
21901 transition: TransitionOptions {
21902 status: TRANSITION_PENDING.to_string(),
21903 tier: tier_name.to_string(),
21904 etag: original.etag.clone().expect("the source object should have an ETag"),
21905 ..Default::default()
21906 },
21907 version_id: original.version_id.map(|version| version.to_string()),
21908 mod_time: original.mod_time,
21909 ..Default::default()
21910 };
21911
21912 let cleanup_store_barrier = TransitionCleanupStoreBarrier::install();
21913 let transition_store = store_a.clone();
21914 let transition = tokio::spawn(async move { transition_store.transition_object(bucket, object, &opts).await });
21915 cleanup_store_barrier.wait_until_paused().await;
21916 transition.abort();
21917 assert!(
21918 transition
21919 .await
21920 .expect_err("the transition task should observe cancellation")
21921 .is_cancelled()
21922 );
21923
21924 remove_barrier.wait_until_paused().await;
21925 let transaction_counts = (
21926 transition_transaction_record_count(store_a.clone()).await,
21927 transition_transaction_record_count(store_b.clone()).await,
21928 );
21929 assert_eq!(
21930 transaction_counts,
21931 (1, 0),
21932 "the transition transaction must remain only on store A even while the process resolver points at store B"
21933 );
21934 let journal_counts = (
21935 tier_delete_journal_count(store_a.clone()).await,
21936 tier_delete_journal_count(store_b.clone()).await,
21937 );
21938 assert_eq!(
21939 journal_counts,
21940 (0, 0),
21941 "rejected transition uploads must not create a tier-delete journal"
21942 );
21943 assert_eq!(backend.object_count().await, 1, "failed cleanup should retain the remote candidate");
21944 remove_barrier.release();
21945 remove_barrier.wait_until_operation_dropped().await;
21946
21947 let recovered = recover_transition_transaction_records(store_a.clone(), 100, None)
21948 .await
21949 .expect("store A should recover its own cancelled-transition transaction");
21950 assert_eq!(
21951 (recovered.scanned, recovered.recovered, recovered.retained, recovered.failed),
21952 (1, 1, 0, 0)
21953 );
21954 assert_eq!(transition_transaction_record_count(store_a.clone()).await, 0);
21955 assert_eq!(transition_transaction_record_count(store_b.clone()).await, 0);
21956 assert_eq!(tier_delete_journal_count(store_a.clone()).await, 0);
21957 assert_eq!(tier_delete_journal_count(store_b.clone()).await, 0);
21958 assert_eq!(
21959 backend.object_count().await,
21960 0,
21961 "store A recovery should delete the exact remote candidate"
21962 );
21963 assert!(!Arc::ptr_eq(&ctx_a, &ctx_b), "the regression requires two distinct instance contexts");
21964 }
21965
21966 #[cfg(feature = "test-util")]
21967 #[tokio::test]
21968 #[serial_test::serial(storage_class_env)]
21969 async fn transition_transaction_recovery_expires_abandoned_attempt_at_budget_bound() {
21970 let temp_dir = tempfile::tempdir().expect("create temp store dir");
21971 let (ctx, store, _shutdown) = without_storage_class_env(build_isolated_test_store(
21972 temp_dir.path(),
21973 "transition-transaction-expired-attempt-budget",
21974 &[4],
21975 ))
21976 .await;
21977 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
21978
21979 let transaction = TransitionTransaction::new(TransitionTransactionInit {
21980 deployment_id: ctx.deployment_id().expect("test store should initialize deployment id"),
21981 transaction_id: uuid::Uuid::new_v4(),
21982 owner_epoch: uuid::Uuid::new_v4(),
21983 write_id: uuid::Uuid::new_v4(),
21984 source: TransitionSourceIdentity {
21985 bucket: "source-bucket".to_string(),
21986 object: "source-object".to_string(),
21987 version_id: Some(uuid::Uuid::new_v4()),
21988 data_dir: uuid::Uuid::new_v4(),
21989 mod_time_unix_nanos: 1_770_000_000_000_000_000,
21990 size: 42,
21991 etag: "source-etag".to_string(),
21992 version_mode: TransitionSourceVersionMode::Versioned,
21993 },
21994 tier_name: "UNUSEDABANDONEDTIER".to_string(),
21995 backend_fingerprint: [7; 32],
21996 not_after_unix_nanos: 1,
21997 })
21998 .expect("transaction should build");
21999 save_transition_transaction_record(store.clone(), &transaction)
22000 .await
22001 .expect("transaction record should persist");
22002
22003 let record_name =
22004 transition_transaction_record_object_name(transaction.transaction_id).expect("transaction record name should derive");
22005 let source = observe_recovery_source(
22006 store.clone(),
22007 &record_name,
22008 crate::bucket::lifecycle::transition_transaction::TRANSITION_TRANSACTION_SCHEMA,
22009 )
22010 .await
22011 .expect("transaction source generation should be observable");
22012 let mut control = IlmRecoveryControl::new(
22013 IlmRecoveryControlIdentity {
22014 protocol: IlmRecoveryProtocol::TransitionTransaction,
22015 canonical_source_path: record_name,
22016 stable_operation_identity: transaction.transaction_id.to_string(),
22017 record_class: "transition_transaction_v1".to_string(),
22018 },
22019 source.generation,
22020 IlmRecoveryClassification::Retrying,
22021 2_000_000_000,
22022 IlmRecoveryErrorCode::None,
22023 )
22024 .expect("recovery control should build");
22025 let mut now = 3_000_000_000;
22026 for _ in 1..MAX_RECOVERY_ATTEMPTS {
22027 now = now.max(control.next_attempt_at_unix_nanos.unwrap_or(now));
22028 control
22029 .claim("cancelled-or-timed-out-owner", uuid::Uuid::new_v4(), now, 1)
22030 .expect("abandoned attempt should claim");
22031 control
22032 .record_expired_attempt(now + 1)
22033 .expect("expired attempt should consume retry budget");
22034 now += 2;
22035 }
22036 now = now.max(control.next_attempt_at_unix_nanos.expect("last retry should have a backoff"));
22037 control
22038 .claim("cancelled-or-timed-out-owner", uuid::Uuid::new_v4(), now, 1)
22039 .expect("final abandoned attempt should claim");
22040 save_recovery_control_if_absent(store.clone(), &control)
22041 .await
22042 .expect("claimed recovery control should persist");
22043
22044 let stats = recover_transition_transaction_records_at(store.clone(), 100, None, i128::from(now + 1))
22045 .await
22046 .expect("recovery should account for the expired attempt");
22047 assert_eq!((stats.scanned, stats.recovered, stats.retained, stats.failed), (1, 0, 1, 0));
22048
22049 let control_id = transition_recovery_control_id(&transaction).expect("control id should derive");
22050 let persisted = load_recovery_control(store.clone(), IlmRecoveryProtocol::TransitionTransaction, &control_id)
22051 .await
22052 .expect("expired recovery control should remain inspectable");
22053 assert_eq!(persisted.control.classification, IlmRecoveryClassification::OperatorRequired);
22054 assert_eq!(persisted.control.attempt_count, u64::from(MAX_RECOVERY_ATTEMPTS));
22055 assert_eq!(persisted.control.consecutive_failure_count, MAX_RECOVERY_ATTEMPTS);
22056 assert_eq!(persisted.control.last_error_code, IlmRecoveryErrorCode::AttemptLeaseExpired);
22057 assert!(persisted.control.owner.is_none());
22058 assert_eq!(
22059 transition_transaction_record_count(store).await,
22060 1,
22061 "budget exhaustion must retain the source record"
22062 );
22063 }
22064
22065 #[cfg(feature = "test-util")]
22066 #[tokio::test]
22067 #[serial_test::serial(storage_class_env)]
22068 async fn transition_transaction_recovery_retains_active_uploaded_candidate_until_commit() {
22069 let temp_dir = tempfile::tempdir().expect("create temp store dir");
22070 let (ctx, store, _shutdown) =
22071 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "transition-transaction-active-uploaded", &[4]))
22072 .await;
22073 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
22074
22075 let tier_name = "TXACTIVEUPLOADED";
22076 let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
22077 let bucket = "transition-transaction-active-uploaded-bucket";
22078 let object = "object.bin";
22079 let payload = b"active Uploaded ownership must survive recovery until local commit".repeat(1024);
22080 store
22081 .make_bucket(bucket, &MakeBucketOptions::default())
22082 .await
22083 .expect("source bucket should be created");
22084 let mut reader = PutObjReader::from_vec(payload.clone());
22085 let source = store
22086 .put_object(bucket, object, &mut reader, &ObjectOptions::default())
22087 .await
22088 .expect("source object should be written");
22089 let barrier = TransitionUploadedCommitBarrier::install(bucket, object);
22090 let transition_store = store.clone();
22091 let transition = tokio::spawn(async move {
22092 transition_store
22093 .transition_object(
22094 bucket,
22095 object,
22096 &ObjectOptions {
22097 transition: TransitionOptions {
22098 status: TRANSITION_PENDING.to_string(),
22099 tier: tier_name.to_string(),
22100 etag: source.etag.clone().expect("source object should have an ETag"),
22101 ..Default::default()
22102 },
22103 version_id: source.version_id.map(|version_id| version_id.to_string()),
22104 mod_time: source.mod_time,
22105 ..Default::default()
22106 },
22107 )
22108 .await
22109 });
22110 barrier.wait_until_paused().await;
22111
22112 let stats = recover_transition_transaction_records(store.clone(), 100, None)
22113 .await
22114 .expect("recovery should inspect the active Uploaded transaction");
22115 assert_eq!((stats.scanned, stats.recovered, stats.retained, stats.failed), (1, 0, 1, 0));
22116 assert_eq!(backend.object_count().await, 1, "active ownership must retain the remote candidate");
22117 assert_eq!(backend.remove_count().await, 0, "active ownership must fence remote DELETE");
22118 assert_eq!(transition_transaction_record_count(store.clone()).await, 1);
22119
22120 barrier.release();
22121 transition
22122 .await
22123 .expect("transition task should join")
22124 .expect("transition should commit after the barrier is released");
22125 assert_eq!(transition_transaction_record_count(store.clone()).await, 0);
22126 assert_eq!(backend.remove_count().await, 0);
22127
22128 let mut body = Vec::new();
22129 store
22130 .get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
22131 .await
22132 .expect("committed tier object should remain readable")
22133 .stream
22134 .read_to_end(&mut body)
22135 .await
22136 .expect("tier object body should drain");
22137 assert_eq!(body, payload);
22138 }
22139
22140 #[cfg(feature = "test-util")]
22141 #[tokio::test]
22142 #[serial_test::serial(storage_class_env)]
22143 async fn transition_transaction_recovery_deletes_expired_uploaded_remote_candidate() {
22144 let temp_dir = tempfile::tempdir().expect("create temp store dir");
22145 let (ctx, store, _shutdown) =
22146 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "transition-transaction-recovery", &[4])).await;
22147 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
22148
22149 let tier_name = "TXRECOVERY";
22150 let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
22151 let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
22152 .await
22153 .expect("tier lease should resolve")
22154 .backend_identity();
22155 let exact_version = uuid::Uuid::new_v4().to_string();
22156 let cases = [
22157 (
22158 "exact",
22159 exact_version.clone(),
22160 TransitionRemoteVersion::versioned(exact_version),
22161 TransitionSourceVersionMode::Versioned,
22162 ),
22163 (
22164 "suspended-null",
22165 "null".to_string(),
22166 TransitionRemoteVersion::versioned("null"),
22167 TransitionSourceVersionMode::VersionSuspended,
22168 ),
22169 (
22170 "known-unversioned",
22171 String::new(),
22172 TransitionRemoteVersion::unversioned(),
22173 TransitionSourceVersionMode::Unversioned,
22174 ),
22175 ];
22176 let mut expected_removes = Vec::new();
22177 let mut recovery_control_ids = Vec::new();
22178 for (case, put_version, remote_version, source_mode) in cases {
22179 let mut transaction = TransitionTransaction::new(TransitionTransactionInit {
22180 deployment_id: ctx.deployment_id().expect("test store should initialize deployment id"),
22181 transaction_id: uuid::Uuid::new_v4(),
22182 owner_epoch: uuid::Uuid::new_v4(),
22183 write_id: uuid::Uuid::new_v4(),
22184 source: TransitionSourceIdentity {
22185 bucket: "source-bucket".to_string(),
22186 object: format!("source-{case}"),
22187 version_id: (source_mode == TransitionSourceVersionMode::Versioned).then(uuid::Uuid::new_v4),
22188 data_dir: uuid::Uuid::new_v4(),
22189 mod_time_unix_nanos: 1_770_000_000_000_000_000,
22190 size: 42,
22191 etag: "source-etag".to_string(),
22192 version_mode: source_mode,
22193 },
22194 tier_name: tier_name.to_string(),
22195 backend_fingerprint: backend_identity,
22196 not_after_unix_nanos: 1,
22197 })
22198 .expect("transaction should build");
22199 transaction
22200 .advance(transaction.fence(), TransitionTransactionState::Uploaded, Some(remote_version))
22201 .expect("transaction should enter uploaded state");
22202 backend.set_put_remote_version(Some(put_version.clone())).await;
22203 let candidate = bytes::Bytes::from_static(b"orphan candidate");
22204 backend
22205 .put(
22206 &transaction.remote_object,
22207 ReaderImpl::Body(candidate.clone()),
22208 i64::try_from(candidate.len()).expect("test candidate length should fit i64"),
22209 )
22210 .await
22211 .expect("mock backend should accept candidate");
22212 save_transition_transaction_record(store.clone(), &transaction)
22213 .await
22214 .expect("transaction record should persist");
22215 recovery_control_ids
22216 .push(transition_recovery_control_id(&transaction).expect("transition recovery control id should derive"));
22217 expected_removes.push((transaction.remote_object, put_version));
22218 }
22219
22220 let stats = {
22221 let _proof = crate::services::notification_sys::install_current_remote_version_state_fleet_proof_for_test();
22222 temp_env::async_with_vars(
22223 [
22224 (rustfs_config::ENV_TIER_REMOTE_VERSION_STATE_WRITE, Some("true")),
22225 (rustfs_config::ENV_TIER_REMOTE_VERSION_STATE_FLEET_CONFIRMED, Some("true")),
22226 ],
22227 recover_transition_transaction_records(store.clone(), 100, None),
22228 )
22229 .await
22230 .expect("transition transaction recovery should run")
22231 };
22232
22233 assert_eq!((stats.scanned, stats.recovered, stats.retained, stats.failed), (3, 3, 0, 0));
22234 assert_eq!(transition_transaction_record_count(store.clone()).await, 0);
22235 let mut actual_removes = backend.remove_versions().await;
22236 actual_removes.sort();
22237 expected_removes.sort();
22238 assert_eq!(actual_removes, expected_removes, "recovery must preserve each remote version shape");
22239 assert_eq!(backend.exact_remove_count(), 2);
22240 assert_eq!(backend.object_count().await, 0);
22241 for control_id in recovery_control_ids {
22242 let control = load_recovery_control(store.clone(), IlmRecoveryProtocol::TransitionTransaction, &control_id)
22243 .await
22244 .expect("completed recovery control should remain inspectable");
22245 assert_eq!(control.control.classification, IlmRecoveryClassification::Terminal);
22246 assert_eq!(control.control.attempt_count, 1);
22247 assert!(control.control.owner.is_none());
22248 }
22249
22250 let replay = recover_transition_transaction_records(store, 100, None)
22251 .await
22252 .expect("replayed recovery should remain idempotent");
22253 assert_eq!((replay.scanned, replay.recovered, replay.retained, replay.failed), (0, 0, 0, 0));
22254 assert_eq!(
22255 backend.remove_versions().await.len(),
22256 3,
22257 "each expired candidate must be deleted only once"
22258 );
22259 }
22260
22261 #[cfg(feature = "test-util")]
22262 #[tokio::test]
22263 #[serial_test::serial(storage_class_env)]
22264 async fn transition_transaction_recovery_resumes_source_cleanup_after_terminal_crash() {
22265 let temp_dir = tempfile::tempdir().expect("create temp store dir");
22266 let (ctx, store, _shutdown) =
22267 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "transition-transaction-terminal-crash", &[4]))
22268 .await;
22269 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
22270
22271 let tier_name = "TXTERMINALCRASH";
22272 let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
22273 let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
22274 .await
22275 .expect("tier lease should resolve")
22276 .backend_identity();
22277 let remote_version = uuid::Uuid::new_v4().to_string();
22278 let mut transaction = TransitionTransaction::new(TransitionTransactionInit {
22279 deployment_id: ctx.deployment_id().expect("test store should initialize deployment id"),
22280 transaction_id: uuid::Uuid::new_v4(),
22281 owner_epoch: uuid::Uuid::new_v4(),
22282 write_id: uuid::Uuid::new_v4(),
22283 source: TransitionSourceIdentity {
22284 bucket: "source-bucket".to_string(),
22285 object: "source-object".to_string(),
22286 version_id: Some(uuid::Uuid::new_v4()),
22287 data_dir: uuid::Uuid::new_v4(),
22288 mod_time_unix_nanos: 1_770_000_000_000_000_000,
22289 size: 42,
22290 etag: "source-etag".to_string(),
22291 version_mode: TransitionSourceVersionMode::Versioned,
22292 },
22293 tier_name: tier_name.to_string(),
22294 backend_fingerprint: backend_identity,
22295 not_after_unix_nanos: 1,
22296 })
22297 .expect("transaction should build");
22298 transaction
22299 .advance(
22300 transaction.fence(),
22301 TransitionTransactionState::Uploaded,
22302 Some(TransitionRemoteVersion::versioned(remote_version.clone())),
22303 )
22304 .expect("transaction should enter uploaded state");
22305 backend.set_put_remote_version(Some(remote_version)).await;
22306 let candidate = bytes::Bytes::from_static(b"terminal crash candidate");
22307 backend
22308 .put(
22309 &transaction.remote_object,
22310 ReaderImpl::Body(candidate.clone()),
22311 i64::try_from(candidate.len()).expect("test candidate length should fit i64"),
22312 )
22313 .await
22314 .expect("mock backend should accept candidate");
22315 save_transition_transaction_record(store.clone(), &transaction)
22316 .await
22317 .expect("transaction record should persist");
22318 let control_id = transition_recovery_control_id(&transaction).expect("control id should derive");
22319
22320 let barrier = TransitionRecoveryTerminalBarrier::install(transaction.transaction_id);
22321 let recovery_store = store.clone();
22322 let recovery = tokio::spawn(async move { recover_transition_transaction_records(recovery_store, 100, None).await });
22323 barrier.wait_until_paused().await;
22324 let terminal = load_recovery_control(store.clone(), IlmRecoveryProtocol::TransitionTransaction, &control_id)
22325 .await
22326 .expect("terminal control should persist before source cleanup");
22327 assert_eq!(terminal.control.classification, IlmRecoveryClassification::Terminal);
22328 assert_eq!(transition_transaction_record_count(store.clone()).await, 1);
22329 assert_eq!(backend.object_count().await, 0);
22330 assert_eq!(backend.exact_remove_count(), 1);
22331
22332 recovery.abort();
22333 assert!(
22334 recovery
22335 .await
22336 .expect_err("recovery should be cancelled at the crash boundary")
22337 .is_cancelled()
22338 );
22339 drop(barrier);
22340
22341 let replay = recover_transition_transaction_records(store.clone(), 100, None)
22342 .await
22343 .expect("terminal control should resume source cleanup without another remote delete");
22344 assert_eq!((replay.scanned, replay.recovered, replay.retained, replay.failed), (1, 1, 0, 0));
22345 assert_eq!(transition_transaction_record_count(store).await, 0);
22346 assert_eq!(backend.exact_remove_count(), 1, "terminal replay must not repeat the remote delete");
22347 }
22348
22349 #[cfg(feature = "test-util")]
22350 #[tokio::test]
22351 #[serial_test::serial(storage_class_env)]
22352 async fn transition_transaction_recovery_cas_loses_to_active_state_advance() {
22353 let temp_dir = tempfile::tempdir().expect("create temp store dir");
22354 let (ctx, store, _shutdown) =
22355 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "transition-transaction-recovery-cas", &[4]))
22356 .await;
22357 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
22358
22359 let tier_name = "TXRECOVERYCAS";
22360 let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
22361 let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
22362 .await
22363 .expect("tier lease should resolve")
22364 .backend_identity();
22365 let remote_version = uuid::Uuid::new_v4().to_string();
22366 let mut uploaded = TransitionTransaction::new(TransitionTransactionInit {
22367 deployment_id: ctx.deployment_id().expect("test store should initialize deployment id"),
22368 transaction_id: uuid::Uuid::new_v4(),
22369 owner_epoch: uuid::Uuid::new_v4(),
22370 write_id: uuid::Uuid::new_v4(),
22371 source: TransitionSourceIdentity {
22372 bucket: "source-bucket".to_string(),
22373 object: "source-object".to_string(),
22374 version_id: Some(uuid::Uuid::new_v4()),
22375 data_dir: uuid::Uuid::new_v4(),
22376 mod_time_unix_nanos: 1_770_000_000_000_000_000,
22377 size: 42,
22378 etag: "source-etag".to_string(),
22379 version_mode: TransitionSourceVersionMode::Versioned,
22380 },
22381 tier_name: tier_name.to_string(),
22382 backend_fingerprint: backend_identity,
22383 not_after_unix_nanos: 1,
22384 })
22385 .expect("transaction should build");
22386 uploaded
22387 .advance(
22388 uploaded.fence(),
22389 TransitionTransactionState::Uploaded,
22390 Some(TransitionRemoteVersion::versioned(remote_version.clone())),
22391 )
22392 .expect("transaction should enter uploaded state");
22393 backend.set_put_remote_version(Some(remote_version)).await;
22394 let candidate = bytes::Bytes::from_static(b"CAS-owned transition remote candidate");
22395 backend
22396 .put(
22397 &uploaded.remote_object,
22398 ReaderImpl::Body(candidate.clone()),
22399 i64::try_from(candidate.len()).expect("test candidate length should fit i64"),
22400 )
22401 .await
22402 .expect("mock backend should accept candidate");
22403 save_transition_transaction_record(store.clone(), &uploaded)
22404 .await
22405 .expect("transaction record should persist");
22406 let recovery_control_id =
22407 transition_recovery_control_id(&uploaded).expect("transition recovery control id should derive");
22408
22409 let barrier = TransitionRecoveryClaimBarrier::install(uploaded.transaction_id);
22410 let recovery_store = store.clone();
22411 let recovery = tokio::spawn(async move { recover_transition_transaction_records(recovery_store, 100, None).await });
22412 barrier.wait_until_paused().await;
22413
22414 let mut active = uploaded.clone();
22415 active
22416 .advance(active.fence(), TransitionTransactionState::LocalCommitStarted, None)
22417 .expect("active owner should advance to local commit");
22418 save_transition_transaction_record_if_current(store.clone(), &uploaded, &active)
22419 .await
22420 .expect("active owner should win the persisted CAS");
22421 barrier.release();
22422
22423 let stats = recovery
22424 .await
22425 .expect("recovery task should join")
22426 .expect("recovery should treat the lost CAS as a retained transaction");
22427 assert_eq!((stats.scanned, stats.recovered, stats.retained, stats.failed), (1, 0, 1, 0));
22428 assert_eq!(
22429 load_transition_transaction_record(store.clone(), uploaded.transaction_id)
22430 .await
22431 .expect("newer transaction revision must remain"),
22432 active
22433 );
22434 let control = load_recovery_control(store, IlmRecoveryProtocol::TransitionTransaction, &recovery_control_id)
22435 .await
22436 .expect("lost source CAS should retain a retryable recovery control");
22437 assert_eq!(control.control.classification, IlmRecoveryClassification::Retrying);
22438 assert_eq!(control.control.consecutive_failure_count, 1);
22439 assert_eq!(control.control.last_error_code, IlmRecoveryErrorCode::SourceGenerationChanged);
22440 assert_eq!(backend.object_count().await, 1, "a stale recovery must not delete the candidate");
22441 assert_eq!(backend.remove_count().await, 0);
22442 }
22443
22444 #[cfg(feature = "test-util")]
22445 #[tokio::test]
22446 #[serial_test::serial(storage_class_env)]
22447 async fn transition_transaction_recovery_quarantines_missing_required_fields() {
22448 let temp_dir = tempfile::tempdir().expect("create temp store dir");
22449 let (ctx, store, _shutdown) = without_storage_class_env(build_isolated_test_store(
22450 temp_dir.path(),
22451 "transition-transaction-recovery-corrupt",
22452 &[4],
22453 ))
22454 .await;
22455 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
22456
22457 let tier_name = "TXRECOVERYCORRUPT";
22458 let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
22459 let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
22460 .await
22461 .expect("tier lease should resolve")
22462 .backend_identity();
22463 let remote_version = uuid::Uuid::new_v4().to_string();
22464 let mut transaction = TransitionTransaction::new(TransitionTransactionInit {
22465 deployment_id: ctx.deployment_id().expect("test store should initialize deployment id"),
22466 transaction_id: uuid::Uuid::new_v4(),
22467 owner_epoch: uuid::Uuid::new_v4(),
22468 write_id: uuid::Uuid::new_v4(),
22469 source: TransitionSourceIdentity {
22470 bucket: "source-bucket".to_string(),
22471 object: "source-object".to_string(),
22472 version_id: None,
22473 data_dir: uuid::Uuid::new_v4(),
22474 mod_time_unix_nanos: 1_770_000_000_000_000_000,
22475 size: 42,
22476 etag: "source-etag".to_string(),
22477 version_mode: TransitionSourceVersionMode::Unversioned,
22478 },
22479 tier_name: tier_name.to_string(),
22480 backend_fingerprint: backend_identity,
22481 not_after_unix_nanos: 1,
22482 })
22483 .expect("transaction should build");
22484 transaction
22485 .advance(
22486 transaction.fence(),
22487 TransitionTransactionState::Uploaded,
22488 Some(TransitionRemoteVersion::versioned(remote_version.clone())),
22489 )
22490 .expect("transaction should enter uploaded state");
22491 backend.set_put_remote_version(Some(remote_version)).await;
22492 let candidate = bytes::Bytes::from_static(b"corrupt journal candidate");
22493 backend
22494 .put(
22495 &transaction.remote_object,
22496 ReaderImpl::Body(candidate.clone()),
22497 i64::try_from(candidate.len()).expect("test candidate length should fit i64"),
22498 )
22499 .await
22500 .expect("mock backend should accept candidate");
22501
22502 let mut persisted: serde_json::Value = serde_json::from_slice(&transaction.encode().expect("transaction should encode"))
22503 .expect("transaction should be JSON");
22504 persisted["transaction"]
22505 .as_object_mut()
22506 .expect("transaction payload should be an object")
22507 .remove("not_after_unix_nanos");
22508 let path =
22509 transition_transaction_record_object_name(transaction.transaction_id).expect("transaction path should be canonical");
22510 com::save_config(
22511 store.clone(),
22512 &path,
22513 serde_json::to_vec(&persisted).expect("corrupt fixture should encode"),
22514 )
22515 .await
22516 .expect("corrupt transaction fixture should persist");
22517
22518 let stats = recover_transition_transaction_records(store.clone(), 100, None)
22519 .await
22520 .expect("recovery scan should isolate a corrupt record");
22521 assert_eq!((stats.scanned, stats.recovered, stats.retained, stats.failed), (1, 0, 0, 1));
22522 assert_eq!(
22523 transition_transaction_record_count(store).await,
22524 1,
22525 "corrupt evidence must remain quarantined"
22526 );
22527 assert_eq!(backend.object_count().await, 1, "corrupt evidence must never authorize remote DELETE");
22528 assert_eq!(backend.remove_count().await, 0);
22529 }
22530
22531 #[cfg(feature = "test-util")]
22532 #[tokio::test]
22533 #[serial_test::serial(storage_class_env)]
22534 async fn transition_transaction_recovery_retries_cleanup_pending_candidate() {
22535 let temp_dir = tempfile::tempdir().expect("create temp store dir");
22536 let (ctx, store, _shutdown) =
22537 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "transition-transaction-cleanup", &[4])).await;
22538 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
22539
22540 let tier_name = "TXCLEANUP";
22541 let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
22542 let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
22543 .await
22544 .expect("tier lease should resolve")
22545 .backend_identity();
22546 let remote_version = uuid::Uuid::new_v4().to_string();
22547 let mut transaction = TransitionTransaction::new(TransitionTransactionInit {
22548 deployment_id: ctx.deployment_id().expect("test store should initialize deployment id"),
22549 transaction_id: uuid::Uuid::new_v4(),
22550 owner_epoch: uuid::Uuid::new_v4(),
22551 write_id: uuid::Uuid::new_v4(),
22552 source: TransitionSourceIdentity {
22553 bucket: "source-bucket".to_string(),
22554 object: "source-object".to_string(),
22555 version_id: Some(uuid::Uuid::new_v4()),
22556 data_dir: uuid::Uuid::new_v4(),
22557 mod_time_unix_nanos: 1_770_000_000_000_000_000,
22558 size: 42,
22559 etag: "source-etag".to_string(),
22560 version_mode: TransitionSourceVersionMode::Versioned,
22561 },
22562 tier_name: tier_name.to_string(),
22563 backend_fingerprint: backend_identity,
22564 not_after_unix_nanos: 1_780_000_000_000_000_000,
22565 })
22566 .expect("transaction should build");
22567 let uploaded_fence = transaction
22568 .advance(
22569 transaction.fence(),
22570 TransitionTransactionState::Uploaded,
22571 Some(TransitionRemoteVersion::versioned(remote_version.clone())),
22572 )
22573 .expect("transaction should enter uploaded state");
22574 transaction
22575 .mark_cleanup_pending(
22576 uploaded_fence,
22577 TransitionCleanupProof {
22578 transaction_id: transaction.transaction_id,
22579 write_id: transaction.write_id,
22580 remote_object: transaction.remote_object.clone(),
22581 remote_version: transaction.remote_version.clone(),
22582 backend_fingerprint: transaction.backend_fingerprint,
22583 decision: TransitionCleanupDecision::UploadAbortedBeforeLocalCommit,
22584 },
22585 )
22586 .expect("transaction should enter cleanup pending state");
22587 backend.set_put_remote_version(Some(remote_version.clone())).await;
22588 let candidate = bytes::Bytes::from_static(b"cleanup pending candidate");
22589 backend
22590 .put(
22591 &transaction.remote_object,
22592 ReaderImpl::Body(candidate.clone()),
22593 i64::try_from(candidate.len()).expect("test candidate length should fit i64"),
22594 )
22595 .await
22596 .expect("mock backend should accept candidate");
22597 save_transition_transaction_record(store.clone(), &transaction)
22598 .await
22599 .expect("transaction record should persist");
22600
22601 let stats = recover_transition_transaction_records(store.clone(), 100, None)
22602 .await
22603 .expect("transition transaction recovery should run");
22604
22605 assert_eq!((stats.scanned, stats.recovered, stats.retained, stats.failed), (1, 1, 0, 0));
22606 assert_eq!(transition_transaction_record_count(store.clone()).await, 0);
22607 assert_eq!(
22608 backend.remove_versions().await,
22609 vec![(transaction.remote_object.clone(), remote_version)],
22610 "cleanup pending recovery must retry the exact remote candidate delete"
22611 );
22612 assert_eq!(backend.exact_remove_count(), 1);
22613 assert_eq!(backend.object_count().await, 0);
22614 }
22615
22616 #[cfg(feature = "test-util")]
22617 #[tokio::test]
22618 #[serial_test::serial(storage_class_env)]
22619 async fn transition_transaction_recovery_retries_cleanup_pending_unversioned_candidate() {
22620 let temp_dir = tempfile::tempdir().expect("create temp store dir");
22621 let (ctx, store, _shutdown) = without_storage_class_env(build_isolated_test_store(
22622 temp_dir.path(),
22623 "transition-transaction-cleanup-unversioned",
22624 &[4],
22625 ))
22626 .await;
22627 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
22628
22629 let tier_name = "TXCLEANUPUNVERSIONED";
22630 let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
22631 let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
22632 .await
22633 .expect("tier lease should resolve")
22634 .backend_identity();
22635 let mut transaction = TransitionTransaction::new(TransitionTransactionInit {
22636 deployment_id: ctx.deployment_id().expect("test store should initialize deployment id"),
22637 transaction_id: uuid::Uuid::new_v4(),
22638 owner_epoch: uuid::Uuid::new_v4(),
22639 write_id: uuid::Uuid::new_v4(),
22640 source: TransitionSourceIdentity {
22641 bucket: "source-bucket".to_string(),
22642 object: "source-object".to_string(),
22643 version_id: None,
22644 data_dir: uuid::Uuid::new_v4(),
22645 mod_time_unix_nanos: 1_770_000_000_000_000_000,
22646 size: 42,
22647 etag: "source-etag".to_string(),
22648 version_mode: TransitionSourceVersionMode::Unversioned,
22649 },
22650 tier_name: tier_name.to_string(),
22651 backend_fingerprint: backend_identity,
22652 not_after_unix_nanos: 1_780_000_000_000_000_000,
22653 })
22654 .expect("transaction should build");
22655 let uploaded_fence = transaction
22656 .advance(
22657 transaction.fence(),
22658 TransitionTransactionState::Uploaded,
22659 Some(TransitionRemoteVersion::unversioned()),
22660 )
22661 .expect("transaction should enter uploaded state");
22662 transaction
22663 .mark_cleanup_pending(
22664 uploaded_fence,
22665 TransitionCleanupProof {
22666 transaction_id: transaction.transaction_id,
22667 write_id: transaction.write_id,
22668 remote_object: transaction.remote_object.clone(),
22669 remote_version: transaction.remote_version.clone(),
22670 backend_fingerprint: transaction.backend_fingerprint,
22671 decision: TransitionCleanupDecision::UploadAbortedBeforeLocalCommit,
22672 },
22673 )
22674 .expect("transaction should enter cleanup pending state");
22675 backend.set_put_remote_version(Some(String::new())).await;
22676 let candidate = bytes::Bytes::from_static(b"cleanup pending unversioned candidate");
22677 backend
22678 .put(
22679 &transaction.remote_object,
22680 ReaderImpl::Body(candidate.clone()),
22681 i64::try_from(candidate.len()).expect("test candidate length should fit i64"),
22682 )
22683 .await
22684 .expect("mock backend should accept candidate");
22685 save_transition_transaction_record(store.clone(), &transaction)
22686 .await
22687 .expect("transaction record should persist");
22688
22689 let stats = recover_transition_transaction_records(store.clone(), 100, None)
22690 .await
22691 .expect("transition transaction recovery should run");
22692
22693 assert_eq!((stats.scanned, stats.recovered, stats.retained, stats.failed), (1, 1, 0, 0));
22694 assert_eq!(transition_transaction_record_count(store.clone()).await, 0);
22695 assert_eq!(
22696 backend.remove_versions().await,
22697 vec![(transaction.remote_object.clone(), String::new())],
22698 "unversioned cleanup must not send a synthetic remote version"
22699 );
22700 assert_eq!(backend.exact_remove_count(), 0);
22701 assert_eq!(backend.object_count().await, 0);
22702 }
22703
22704 #[cfg(feature = "test-util")]
22705 #[tokio::test]
22706 #[serial_test::serial(storage_class_env)]
22707 async fn transition_transaction_recovery_keeps_cleanup_pending_record_when_delete_fails() {
22708 let temp_dir = tempfile::tempdir().expect("create temp store dir");
22709 let (ctx, store, _shutdown) =
22710 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "transition-transaction-cleanup-fail", &[4]))
22711 .await;
22712 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
22713
22714 let tier_name = "TXCLEANUPFAIL";
22715 let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
22716 let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
22717 .await
22718 .expect("tier lease should resolve")
22719 .backend_identity();
22720 let remote_version = uuid::Uuid::new_v4().to_string();
22721 let mut transaction = TransitionTransaction::new(TransitionTransactionInit {
22722 deployment_id: ctx.deployment_id().expect("test store should initialize deployment id"),
22723 transaction_id: uuid::Uuid::new_v4(),
22724 owner_epoch: uuid::Uuid::new_v4(),
22725 write_id: uuid::Uuid::new_v4(),
22726 source: TransitionSourceIdentity {
22727 bucket: "source-bucket".to_string(),
22728 object: "source-object".to_string(),
22729 version_id: Some(uuid::Uuid::new_v4()),
22730 data_dir: uuid::Uuid::new_v4(),
22731 mod_time_unix_nanos: 1_770_000_000_000_000_000,
22732 size: 42,
22733 etag: "source-etag".to_string(),
22734 version_mode: TransitionSourceVersionMode::Versioned,
22735 },
22736 tier_name: tier_name.to_string(),
22737 backend_fingerprint: backend_identity,
22738 not_after_unix_nanos: 1_780_000_000_000_000_000,
22739 })
22740 .expect("transaction should build");
22741 transaction
22742 .advance(
22743 transaction.fence(),
22744 TransitionTransactionState::Uploaded,
22745 Some(TransitionRemoteVersion::versioned(remote_version.clone())),
22746 )
22747 .expect("transaction should enter uploaded state");
22748 let candidate = bytes::Bytes::from_static(b"cleanup pending candidate retained after failure");
22749 backend.set_put_remote_version(Some(remote_version)).await;
22750 backend
22751 .put(
22752 &transaction.remote_object,
22753 ReaderImpl::Body(candidate.clone()),
22754 i64::try_from(candidate.len()).expect("test candidate length should fit i64"),
22755 )
22756 .await
22757 .expect("mock backend should accept candidate");
22758 save_transition_transaction_record(store.clone(), &transaction)
22759 .await
22760 .expect("transaction record should persist");
22761 let recovery_control_id =
22762 transition_recovery_control_id(&transaction).expect("transition recovery control id should derive");
22763
22764 backend.set_remove_failure(true);
22765 let stats = recover_transition_transaction_records(store.clone(), 100, None)
22766 .await
22767 .expect("transition transaction recovery should keep scanning after cleanup failure");
22768
22769 assert_eq!((stats.scanned, stats.recovered, stats.retained, stats.failed), (1, 0, 0, 1));
22770 assert_eq!(
22771 transition_transaction_record_count(store.clone()).await,
22772 1,
22773 "failed cleanup must keep the cleanup-pending transaction for retry"
22774 );
22775 assert_eq!(backend.remove_versions().await, Vec::<(String, String)>::new());
22776 assert_eq!(backend.exact_remove_count(), 1);
22777 assert_eq!(backend.object_count().await, 1);
22778 let control = load_recovery_control(store.clone(), IlmRecoveryProtocol::TransitionTransaction, &recovery_control_id)
22779 .await
22780 .expect("failed recovery control should persist");
22781 assert_eq!(control.control.classification, IlmRecoveryClassification::Retrying);
22782 assert_eq!(control.control.attempt_count, 1);
22783 assert_eq!(control.control.consecutive_failure_count, 1);
22784 assert!(
22785 control
22786 .control
22787 .next_attempt_at_unix_nanos
22788 .is_some_and(|next| next > OffsetDateTime::now_utc().unix_timestamp_nanos() as i64)
22789 );
22790
22791 backend.set_remove_failure(false);
22792 let replay = recover_transition_transaction_records(store.clone(), 100, None)
22793 .await
22794 .expect("recovery before the persisted deadline should be skipped");
22795 assert_eq!((replay.scanned, replay.recovered, replay.retained, replay.failed), (1, 0, 1, 0));
22796 assert_eq!(backend.exact_remove_count(), 1, "persisted backoff must prevent an immediate retry");
22797
22798 let retry_at = control
22799 .control
22800 .next_attempt_at_unix_nanos
22801 .expect("retry deadline should persist");
22802 let retried = recover_transition_transaction_records_at(store.clone(), 100, None, i128::from(retry_at) + 1)
22803 .await
22804 .expect("recovery at the persisted deadline should retry the advanced source generation");
22805 assert_eq!((retried.scanned, retried.recovered, retried.retained, retried.failed), (1, 1, 0, 0));
22806 assert_eq!(backend.exact_remove_count(), 2);
22807 assert_eq!(backend.object_count().await, 0);
22808 assert_eq!(transition_transaction_record_count(store.clone()).await, 0);
22809 let terminal = load_recovery_control(store, IlmRecoveryProtocol::TransitionTransaction, &recovery_control_id)
22810 .await
22811 .expect("completed retry control should remain inspectable");
22812 assert_eq!(terminal.control.classification, IlmRecoveryClassification::Terminal);
22813 assert_eq!(terminal.control.attempt_count, 2);
22814 }
22815
22816 #[cfg(feature = "test-util")]
22817 #[tokio::test]
22818 #[serial_test::serial(storage_class_env)]
22819 async fn transition_transaction_recovery_keeps_cleanup_pending_local_commit() {
22820 let temp_dir = tempfile::tempdir().expect("create temp store dir");
22821 let (ctx, store, _shutdown) = without_storage_class_env(build_isolated_test_store(
22822 temp_dir.path(),
22823 "transition-transaction-cleanup-committed",
22824 &[4],
22825 ))
22826 .await;
22827 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
22828
22829 let tier_name = "TXCLEANUPCOMMITTED";
22830 let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
22831 let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
22832 .await
22833 .expect("tier lease should resolve")
22834 .backend_identity();
22835 let bucket = "transition-transaction-cleanup-committed-bucket";
22836 let object = "object.bin";
22837 store
22838 .make_bucket(bucket, &MakeBucketOptions::default())
22839 .await
22840 .expect("bucket should be created");
22841 let mut reader = PutObjReader::from_vec(b"cleanup pending local commit record cleanup".repeat(1024));
22842 let original = store
22843 .put_object(bucket, object, &mut reader, &ObjectOptions::default())
22844 .await
22845 .expect("source object should be written");
22846 let opts = ObjectOptions {
22847 no_lock: true,
22848 transition: TransitionOptions {
22849 status: TRANSITION_PENDING.to_string(),
22850 tier: tier_name.to_string(),
22851 etag: original.etag.clone().expect("source object should have etag"),
22852 ..Default::default()
22853 },
22854 mod_time: original.mod_time,
22855 ..Default::default()
22856 };
22857 store
22858 .transition_object(bucket, object, &opts)
22859 .await
22860 .expect("transition should commit");
22861 let committed = store
22862 .get_object_info(
22863 bucket,
22864 object,
22865 &ObjectOptions {
22866 no_lock: true,
22867 metadata_cache_safe: false,
22868 ..Default::default()
22869 },
22870 )
22871 .await
22872 .expect("committed object info should be readable");
22873 let mut remote_parts = committed.transitioned_object.name.rsplit('/');
22874 let write_id = uuid::Uuid::parse_str(remote_parts.next().expect("remote object should contain write id"))
22875 .expect("write id should parse");
22876 let transaction_id = uuid::Uuid::parse_str(remote_parts.next().expect("remote object should contain transaction id"))
22877 .expect("transaction id should parse");
22878 let source = TransitionSourceIdentity {
22879 bucket: bucket.to_string(),
22880 object: object.to_string(),
22881 version_id: None,
22882 data_dir: original.data_dir.expect("source object should have data_dir"),
22883 mod_time_unix_nanos: original
22884 .mod_time
22885 .expect("source object should have mod_time")
22886 .unix_timestamp_nanos()
22887 .try_into()
22888 .expect("test timestamp should fit i64"),
22889 size: original.size,
22890 etag: original.etag.expect("source object should have etag"),
22891 version_mode: TransitionSourceVersionMode::Unversioned,
22892 };
22893 let mut transaction = TransitionTransaction::new(TransitionTransactionInit {
22894 deployment_id: ctx.deployment_id().expect("test store should initialize deployment id"),
22895 transaction_id,
22896 owner_epoch: uuid::Uuid::new_v4(),
22897 write_id,
22898 source,
22899 tier_name: tier_name.to_string(),
22900 backend_fingerprint: backend_identity,
22901 not_after_unix_nanos: 1_780_000_000_000_000_000,
22902 })
22903 .expect("transaction should build");
22904 transaction
22905 .advance(
22906 transaction.fence(),
22907 TransitionTransactionState::Uploaded,
22908 Some(TransitionRemoteVersion::known_from_put_response(
22909 committed.transitioned_object.version_id.clone(),
22910 )),
22911 )
22912 .expect("transaction should enter uploaded state");
22913 let local_commit_fence = transaction
22914 .advance(transaction.fence(), TransitionTransactionState::LocalCommitStarted, None)
22915 .expect("transaction should enter local commit state");
22916 transaction
22917 .mark_cleanup_pending(
22918 local_commit_fence,
22919 TransitionCleanupProof {
22920 transaction_id: transaction.transaction_id,
22921 write_id: transaction.write_id,
22922 remote_object: transaction.remote_object.clone(),
22923 remote_version: transaction.remote_version.clone(),
22924 backend_fingerprint: transaction.backend_fingerprint,
22925 decision: TransitionCleanupDecision::SourceReconciledUnchanged {
22926 observed_source: transaction.source.clone(),
22927 },
22928 },
22929 )
22930 .expect("transaction should enter cleanup pending state");
22931 save_transition_transaction_record(store.clone(), &transaction)
22932 .await
22933 .expect("transaction record should persist");
22934
22935 let stats = recover_transition_transaction_records(store.clone(), 100, None)
22936 .await
22937 .expect("transition transaction recovery should run");
22938
22939 assert_eq!((stats.scanned, stats.recovered, stats.retained, stats.failed), (1, 1, 0, 0));
22940 assert_eq!(transition_transaction_record_count(store.clone()).await, 0);
22941 assert_eq!(
22942 backend.object_count().await,
22943 1,
22944 "cleanup pending recovery must keep the remote body once local metadata references it"
22945 );
22946 assert_eq!(backend.remove_count().await, 0);
22947 }
22948
22949 #[cfg(feature = "test-util")]
22950 #[tokio::test]
22951 #[serial_test::serial(storage_class_env)]
22952 async fn transition_transaction_recovery_drops_record_after_confirmed_local_commit() {
22953 let temp_dir = tempfile::tempdir().expect("create temp store dir");
22954 let (ctx, store, _shutdown) =
22955 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "transition-transaction-committed", &[4])).await;
22956 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
22957
22958 let tier_name = "TXCOMMITTED";
22959 let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
22960 let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
22961 .await
22962 .expect("tier lease should resolve")
22963 .backend_identity();
22964 let bucket = "transition-transaction-committed-bucket";
22965 let object = "object.bin";
22966 store
22967 .make_bucket(bucket, &MakeBucketOptions::default())
22968 .await
22969 .expect("bucket should be created");
22970 let mut reader = PutObjReader::from_vec(b"confirmed local commit record cleanup".repeat(1024));
22971 let original = store
22972 .put_object(bucket, object, &mut reader, &ObjectOptions::default())
22973 .await
22974 .expect("source object should be written");
22975 let opts = ObjectOptions {
22976 no_lock: true,
22977 transition: TransitionOptions {
22978 status: TRANSITION_PENDING.to_string(),
22979 tier: tier_name.to_string(),
22980 etag: original.etag.clone().expect("source object should have etag"),
22981 ..Default::default()
22982 },
22983 mod_time: original.mod_time,
22984 ..Default::default()
22985 };
22986 store
22987 .transition_object(bucket, object, &opts)
22988 .await
22989 .expect("transition should commit");
22990 let committed = store
22991 .get_object_info(
22992 bucket,
22993 object,
22994 &ObjectOptions {
22995 no_lock: true,
22996 metadata_cache_safe: false,
22997 ..Default::default()
22998 },
22999 )
23000 .await
23001 .expect("committed object info should be readable");
23002 let mut remote_parts = committed.transitioned_object.name.rsplit('/');
23003 let write_id = uuid::Uuid::parse_str(remote_parts.next().expect("remote object should contain write id"))
23004 .expect("write id should parse");
23005 let transaction_id = uuid::Uuid::parse_str(remote_parts.next().expect("remote object should contain transaction id"))
23006 .expect("transaction id should parse");
23007 let mut transaction = TransitionTransaction::new(TransitionTransactionInit {
23008 deployment_id: ctx.deployment_id().expect("test store should initialize deployment id"),
23009 transaction_id,
23010 owner_epoch: uuid::Uuid::new_v4(),
23011 write_id,
23012 source: TransitionSourceIdentity {
23013 bucket: bucket.to_string(),
23014 object: object.to_string(),
23015 version_id: None,
23016 data_dir: original.data_dir.expect("source object should have data_dir"),
23017 mod_time_unix_nanos: original
23018 .mod_time
23019 .expect("source object should have mod_time")
23020 .unix_timestamp_nanos()
23021 .try_into()
23022 .expect("test timestamp should fit i64"),
23023 size: original.size,
23024 etag: original.etag.expect("source object should have etag"),
23025 version_mode: TransitionSourceVersionMode::Unversioned,
23026 },
23027 tier_name: tier_name.to_string(),
23028 backend_fingerprint: backend_identity,
23029 not_after_unix_nanos: 1_780_000_000_000_000_000,
23030 })
23031 .expect("transaction should build");
23032 transaction
23033 .advance(
23034 transaction.fence(),
23035 TransitionTransactionState::Uploaded,
23036 Some(TransitionRemoteVersion::known_from_put_response(
23037 committed.transitioned_object.version_id.clone(),
23038 )),
23039 )
23040 .expect("transaction should enter uploaded state");
23041 transaction
23042 .advance(transaction.fence(), TransitionTransactionState::LocalCommitStarted, None)
23043 .expect("transaction should enter local commit state");
23044 save_transition_transaction_record(store.clone(), &transaction)
23045 .await
23046 .expect("transaction record should persist");
23047 assert_eq!(transition_transaction_record_count(store.clone()).await, 1);
23048
23049 let stats = recover_transition_transaction_records(store.clone(), 100, None)
23050 .await
23051 .expect("transition transaction recovery should run");
23052
23053 assert_eq!((stats.scanned, stats.recovered, stats.retained, stats.failed), (1, 1, 0, 0));
23054 assert_eq!(transition_transaction_record_count(store.clone()).await, 0);
23055 assert_eq!(
23056 backend.object_count().await,
23057 1,
23058 "confirmed local commit recovery must not delete the committed remote body"
23059 );
23060 assert_eq!(backend.remove_count().await, 0);
23061 }
23062
23063 #[cfg(feature = "test-util")]
23064 #[tokio::test]
23065 #[serial_test::serial(storage_class_env)]
23066 async fn transition_transaction_recovery_retains_unproven_remote_candidates() {
23067 let temp_dir = tempfile::tempdir().expect("create temp store dir");
23068 let (ctx, store, _shutdown) =
23069 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "transition-transaction-unproven", &[4])).await;
23070 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
23071
23072 let tier_name = "TXUNPROVEN";
23073 let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
23074 let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
23075 .await
23076 .expect("tier lease should resolve")
23077 .backend_identity();
23078 let remote_version = uuid::Uuid::new_v4().to_string();
23079 let new_transaction = || {
23080 TransitionTransaction::new(TransitionTransactionInit {
23081 deployment_id: ctx.deployment_id().expect("test store should initialize deployment id"),
23082 transaction_id: uuid::Uuid::new_v4(),
23083 owner_epoch: uuid::Uuid::new_v4(),
23084 write_id: uuid::Uuid::new_v4(),
23085 source: TransitionSourceIdentity {
23086 bucket: "absent-source-bucket".to_string(),
23087 object: "source-object".to_string(),
23088 version_id: None,
23089 data_dir: uuid::Uuid::new_v4(),
23090 mod_time_unix_nanos: 1_770_000_000_000_000_000,
23091 size: 42,
23092 etag: "source-etag".to_string(),
23093 version_mode: TransitionSourceVersionMode::Unversioned,
23094 },
23095 tier_name: tier_name.to_string(),
23096 backend_fingerprint: backend_identity,
23097 not_after_unix_nanos: 1_780_000_000_000_000_000,
23098 })
23099 .expect("transaction should build")
23100 };
23101
23102 let upload_started = new_transaction();
23103 let mut local_commit_started = new_transaction();
23104 local_commit_started
23105 .advance(
23106 local_commit_started.fence(),
23107 TransitionTransactionState::Uploaded,
23108 Some(TransitionRemoteVersion::versioned(remote_version.clone())),
23109 )
23110 .expect("transaction should enter uploaded state");
23111 local_commit_started
23112 .advance(local_commit_started.fence(), TransitionTransactionState::LocalCommitStarted, None)
23113 .expect("transaction should enter local commit state");
23114 let upload_started_control_id =
23115 transition_recovery_control_id(&upload_started).expect("upload-started control id should derive");
23116 let local_commit_control_id =
23117 transition_recovery_control_id(&local_commit_started).expect("local-commit control id should derive");
23118
23119 backend.set_put_remote_version(Some(remote_version)).await;
23120 for transaction in [&upload_started, &local_commit_started] {
23121 let candidate = bytes::Bytes::from_static(b"unproven transition remote candidate");
23122 backend
23123 .put(
23124 &transaction.remote_object,
23125 ReaderImpl::Body(candidate.clone()),
23126 i64::try_from(candidate.len()).expect("test candidate length should fit i64"),
23127 )
23128 .await
23129 .expect("mock backend should accept candidate");
23130 save_transition_transaction_record(store.clone(), transaction)
23131 .await
23132 .expect("transaction record should persist");
23133 }
23134
23135 let stats = recover_transition_transaction_records(store.clone(), 100, None)
23136 .await
23137 .expect("transition transaction recovery should run");
23138
23139 assert_eq!((stats.scanned, stats.recovered, stats.retained, stats.failed), (2, 0, 2, 0));
23140 assert_eq!(
23141 transition_transaction_record_count(store.clone()).await,
23142 2,
23143 "an upload without completion proof or unproven local commit must remain for authoritative reconcile"
23144 );
23145 assert_eq!(backend.object_count().await, 2, "recovery must not delete an unproven remote candidate");
23146 assert_eq!(backend.remove_count().await, 0);
23147 assert_eq!(backend.exact_remove_count(), 0);
23148 let upload_started_control =
23149 load_recovery_control(store.clone(), IlmRecoveryProtocol::TransitionTransaction, &upload_started_control_id)
23150 .await
23151 .expect("upload-started control should persist");
23152 assert_eq!(
23153 upload_started_control.control.classification,
23154 IlmRecoveryClassification::RetainedAmbiguous
23155 );
23156 let local_commit_control =
23157 load_recovery_control(store.clone(), IlmRecoveryProtocol::TransitionTransaction, &local_commit_control_id)
23158 .await
23159 .expect("local-commit control should persist");
23160 assert_eq!(local_commit_control.control.classification, IlmRecoveryClassification::OperatorRequired);
23161
23162 let upload_status = inspect_transition_recovery_retry_for_operator(store.clone(), &upload_started_control_id)
23163 .await
23164 .expect("retained upload should be inspectable for a bounded retry");
23165 let local_status = inspect_transition_recovery_retry_for_operator(store.clone(), &local_commit_control_id)
23166 .await
23167 .expect("operator-required local commit should be inspectable for a bounded retry");
23168 assert!(upload_status.retry_ready);
23169 assert!(local_status.retry_ready);
23170 assert!(matches!(
23171 retry_transition_recovery_for_operator(
23172 store.clone(),
23173 &upload_started_control_id,
23174 upload_status.control_revision + 1,
23175 &upload_status.source_generation_sha256,
23176 )
23177 .await,
23178 Err(TransitionOperatorError::StaleRecoveryControl)
23179 ));
23180
23181 let put_count_before_retry = backend.put_count().await;
23182 let get_count_before_retry = backend.get_count().await;
23183 let remove_count_before_retry = backend.remove_count().await;
23184 let upload_retry = retry_transition_recovery_for_operator(
23185 store.clone(),
23186 &upload_started_control_id,
23187 upload_status.control_revision,
23188 &upload_status.source_generation_sha256,
23189 )
23190 .await
23191 .expect("exact retained upload generation should be rearmed");
23192 let local_retry = retry_transition_recovery_for_operator(
23193 store.clone(),
23194 &local_commit_control_id,
23195 local_status.control_revision,
23196 &local_status.source_generation_sha256,
23197 )
23198 .await
23199 .expect("exact operator-required local commit generation should be rearmed");
23200 assert_eq!(upload_retry.classification, IlmRecoveryClassification::Retrying);
23201 assert_eq!(local_retry.classification, IlmRecoveryClassification::Retrying);
23202 assert_eq!(upload_retry.attempt_count, upload_status.attempt_count);
23203 assert_eq!(local_retry.attempt_count, local_status.attempt_count);
23204 assert_eq!(backend.put_count().await, put_count_before_retry);
23205 assert_eq!(backend.get_count().await, get_count_before_retry);
23206 assert_eq!(backend.remove_count().await, remove_count_before_retry);
23207 assert_eq!(backend.exact_remove_count(), 0, "operator retry must not directly issue remote DELETE");
23208
23209 let retried = recover_transition_transaction_records(store.clone(), 100, None)
23210 .await
23211 .expect("rearmed records should be re-evaluated through normal recovery");
23212 assert_eq!((retried.scanned, retried.recovered, retried.retained, retried.failed), (2, 0, 2, 0));
23213 let upload_retained =
23214 load_recovery_control(store.clone(), IlmRecoveryProtocol::TransitionTransaction, &upload_started_control_id)
23215 .await
23216 .expect("upload retry result should persist");
23217 let local_retained = load_recovery_control(store, IlmRecoveryProtocol::TransitionTransaction, &local_commit_control_id)
23218 .await
23219 .expect("local commit retry result should persist");
23220 assert_eq!(upload_retained.control.classification, IlmRecoveryClassification::RetainedAmbiguous);
23221 assert_eq!(local_retained.control.classification, IlmRecoveryClassification::OperatorRequired);
23222 assert_eq!(upload_retained.control.attempt_count, upload_status.attempt_count + 1);
23223 assert_eq!(local_retained.control.attempt_count, local_status.attempt_count + 1);
23224 assert_eq!(backend.put_count().await, put_count_before_retry);
23225 assert_eq!(backend.get_count().await, get_count_before_retry);
23226 assert_eq!(backend.remove_count().await, remove_count_before_retry);
23227 assert_eq!(backend.exact_remove_count(), 0);
23228 }
23229
23230 #[cfg(feature = "test-util")]
23231 #[tokio::test]
23232 #[serial_test::serial(storage_class_env)]
23233 async fn transition_transaction_recovery_handles_provider_recovered_unknown_upload() {
23234 let versioned_remote = uuid::Uuid::new_v4().to_string();
23235 let nil_remote = uuid::Uuid::nil().to_string();
23236 for (case, tier_name, remote_version, should_recover) in [
23237 ("missing", "TXPROBEMISSING", None, true),
23238 ("unversioned", "TXPROBEUNVERSIONED", Some(String::new()), true),
23239 ("versioned", "TXPROBEVERSIONED", Some(versioned_remote), true),
23240 ("nil-version", "TXPROBENILVERSION", Some(nil_remote), false),
23241 ] {
23242 let temp_dir = tempfile::tempdir().expect("create temp store dir");
23243 let (ctx, store, _shutdown) = without_storage_class_env(build_isolated_test_store(
23244 temp_dir.path(),
23245 &format!("transition-transaction-probe-{case}"),
23246 &[4],
23247 ))
23248 .await;
23249 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
23250
23251 let backend = register_transition_reconcile_test_tier(&ctx.tier_config_mgr(), tier_name).await;
23252 let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
23253 .await
23254 .expect("tier lease should resolve")
23255 .backend_identity();
23256 let mut transaction = TransitionTransaction::new(TransitionTransactionInit {
23257 deployment_id: ctx.deployment_id().expect("test store should initialize deployment id"),
23258 transaction_id: uuid::Uuid::new_v4(),
23259 owner_epoch: uuid::Uuid::new_v4(),
23260 write_id: uuid::Uuid::new_v4(),
23261 source: TransitionSourceIdentity {
23262 bucket: "source-bucket".to_string(),
23263 object: "source-object".to_string(),
23264 version_id: None,
23265 data_dir: uuid::Uuid::new_v4(),
23266 mod_time_unix_nanos: 1_770_000_000_000_000_000,
23267 size: 42,
23268 etag: "source-etag".to_string(),
23269 version_mode: TransitionSourceVersionMode::Unversioned,
23270 },
23271 tier_name: tier_name.to_string(),
23272 backend_fingerprint: backend_identity,
23273 not_after_unix_nanos: 1,
23274 })
23275 .expect("transaction should build");
23276 transaction
23277 .advance(transaction.fence(), TransitionTransactionState::UploadOutcomeUnknown, None)
23278 .expect("transaction should enter unknown upload outcome state");
23279
23280 if let Some(version) = &remote_version {
23281 backend.set_put_remote_version(Some(version.clone())).await;
23282 let candidate = bytes::Bytes::from_static(b"provider-recovered transition remote candidate");
23283 backend
23284 .put(
23285 &transaction.remote_object,
23286 ReaderImpl::Body(candidate.clone()),
23287 i64::try_from(candidate.len()).expect("test candidate length should fit i64"),
23288 )
23289 .await
23290 .expect("mock backend should accept candidate");
23291 }
23292 save_transition_transaction_record(store.clone(), &transaction)
23293 .await
23294 .expect("transaction record should persist");
23295
23296 let stats = recover_transition_transaction_records(store.clone(), 100, None)
23297 .await
23298 .expect("transition transaction recovery should run");
23299
23300 assert_eq!(
23301 (stats.scanned, stats.recovered, stats.retained, stats.failed),
23302 if should_recover { (1, 1, 0, 0) } else { (1, 0, 1, 0) }
23303 );
23304 assert_eq!(transition_transaction_record_count(store.clone()).await, usize::from(!should_recover));
23305 assert_eq!(
23306 backend.object_count().await,
23307 usize::from(!should_recover),
23308 "case {case}: only a valid provider version state may be recovered destructively"
23309 );
23310 let removed = remote_version
23311 .filter(|_| should_recover)
23312 .map(|version| vec![(transaction.remote_object.clone(), version)])
23313 .unwrap_or_default();
23314 assert_eq!(
23315 backend.remove_versions().await,
23316 removed,
23317 "case {case}: recovery must delete only provider-recovered candidates"
23318 );
23319 assert_eq!(
23320 backend.exact_remove_count(),
23321 usize::from(removed.first().is_some_and(|(_, version)| !version.is_empty()))
23322 );
23323 }
23324 }
23325
23326 #[cfg(feature = "test-util")]
23327 #[tokio::test]
23328 #[serial_test::serial(storage_class_env)]
23329 async fn operator_reconcile_deletes_exact_candidate_before_finalizing_record() {
23330 let temp_dir = tempfile::tempdir().expect("create temp store dir");
23331 let (ctx, store, _shutdown) =
23332 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "transition-operator-reconcile", &[4])).await;
23333 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
23334
23335 let tier_name = "TXOPERATOR";
23336 let backend = register_transition_reconcile_test_tier(&ctx.tier_config_mgr(), tier_name).await;
23337 let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
23338 .await
23339 .expect("tier lease should resolve")
23340 .backend_identity();
23341 let mut transaction = TransitionTransaction::new(TransitionTransactionInit {
23342 deployment_id: ctx.deployment_id().expect("test store should initialize deployment id"),
23343 transaction_id: uuid::Uuid::new_v4(),
23344 owner_epoch: uuid::Uuid::new_v4(),
23345 write_id: uuid::Uuid::new_v4(),
23346 source: TransitionSourceIdentity {
23347 bucket: "source-bucket".to_string(),
23348 object: "source-object".to_string(),
23349 version_id: None,
23350 data_dir: uuid::Uuid::new_v4(),
23351 mod_time_unix_nanos: 1_770_000_000_000_000_000,
23352 size: 42,
23353 etag: "source-etag".to_string(),
23354 version_mode: TransitionSourceVersionMode::Unversioned,
23355 },
23356 tier_name: tier_name.to_string(),
23357 backend_fingerprint: backend_identity,
23358 not_after_unix_nanos: 1,
23359 })
23360 .expect("transaction should build");
23361 transaction
23362 .advance(transaction.fence(), TransitionTransactionState::UploadOutcomeUnknown, None)
23363 .expect("transaction should enter unknown upload outcome state");
23364
23365 let remote_version = uuid::Uuid::new_v4().to_string();
23366 backend.set_put_remote_version(Some(remote_version.clone())).await;
23367 let candidate = bytes::Bytes::from_static(b"operator-confirmed transition candidate");
23368 backend
23369 .put(
23370 &transaction.remote_object,
23371 ReaderImpl::Body(candidate.clone()),
23372 i64::try_from(candidate.len()).expect("test candidate length should fit i64"),
23373 )
23374 .await
23375 .expect("mock backend should accept candidate");
23376 save_transition_transaction_record(store.clone(), &transaction)
23377 .await
23378 .expect("transaction record should persist");
23379
23380 let status = inspect_transition_transaction_for_operator(store.clone(), transaction.transaction_id)
23381 .await
23382 .expect("operator inspection should probe the candidate");
23383 assert_eq!(status.probe, TransitionOperatorProbe::VersionedPresent(remote_version.clone()));
23384
23385 let wrong_version = uuid::Uuid::new_v4().to_string();
23386 let err = delete_transition_candidate_for_operator(store.clone(), transaction.transaction_id, &wrong_version)
23387 .await
23388 .expect_err("a mismatched exact version must fail before deleting a candidate");
23389 assert!(matches!(
23390 err,
23391 TransitionOperatorError::CandidateVersionMismatch {
23392 expected,
23393 actual: TransitionOperatorProbe::VersionedPresent(ref observed),
23394 } if expected == wrong_version && observed == &remote_version
23395 ));
23396 assert!(backend.contains(&transaction.remote_object).await);
23397 assert_eq!(backend.exact_remove_count(), 0);
23398 load_transition_transaction_record(store.clone(), transaction.transaction_id)
23399 .await
23400 .expect("an incorrect exact version must retain the transaction journal");
23401
23402 let result = delete_transition_candidate_for_operator(store.clone(), transaction.transaction_id, &remote_version)
23403 .await
23404 .expect("operator-confirmed exact candidate should be deleted");
23405 assert_eq!(result.status.probe, TransitionOperatorProbe::Missing);
23406 assert!(result.journal_observed_after_delete);
23407 assert_eq!(backend.exact_remove_count(), 1);
23408 assert_eq!(backend.remove_versions().await, vec![(transaction.remote_object.clone(), remote_version)]);
23409 load_transition_transaction_record(store.clone(), transaction.transaction_id)
23410 .await
23411 .expect("candidate deletion must retain the transaction journal");
23412
23413 finalize_missing_transition_transaction_for_operator(store.clone(), transaction.transaction_id)
23414 .await
23415 .expect("a separately confirmed missing candidate should permit finalization");
23416 assert!(matches!(
23417 load_transition_transaction_record(store, transaction.transaction_id).await,
23418 Err(Error::ConfigNotFound)
23419 ));
23420 }
23421
23422 #[cfg(feature = "test-util")]
23423 #[tokio::test]
23424 #[serial_test::serial(storage_class_env)]
23425 async fn operator_finalize_retains_record_without_missing_proof() {
23426 let temp_dir = tempfile::tempdir().expect("create temp store dir");
23427 let (ctx, store, _shutdown) =
23428 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "transition-operator-fail-closed", &[4])).await;
23429 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
23430
23431 let tier_name = "TXOPERATORFAIL";
23432 let backend = register_transition_reconcile_test_tier(&ctx.tier_config_mgr(), tier_name).await;
23433 let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
23434 .await
23435 .expect("tier lease should resolve")
23436 .backend_identity();
23437 let mut transaction = TransitionTransaction::new(TransitionTransactionInit {
23438 deployment_id: ctx.deployment_id().expect("test store should initialize deployment id"),
23439 transaction_id: uuid::Uuid::new_v4(),
23440 owner_epoch: uuid::Uuid::new_v4(),
23441 write_id: uuid::Uuid::new_v4(),
23442 source: TransitionSourceIdentity {
23443 bucket: "source-bucket".to_string(),
23444 object: "source-object".to_string(),
23445 version_id: None,
23446 data_dir: uuid::Uuid::new_v4(),
23447 mod_time_unix_nanos: 1_770_000_000_000_000_000,
23448 size: 42,
23449 etag: "source-etag".to_string(),
23450 version_mode: TransitionSourceVersionMode::Unversioned,
23451 },
23452 tier_name: tier_name.to_string(),
23453 backend_fingerprint: backend_identity,
23454 not_after_unix_nanos: 1,
23455 })
23456 .expect("transaction should build");
23457 transaction
23458 .advance(transaction.fence(), TransitionTransactionState::UploadOutcomeUnknown, None)
23459 .expect("transaction should enter unknown upload outcome state");
23460 save_transition_transaction_record(store.clone(), &transaction)
23461 .await
23462 .expect("transaction record should persist");
23463 backend
23464 .set_transition_candidate_probe_override(Some(TransitionCandidateProbe::Unsupported))
23465 .await;
23466
23467 assert!(matches!(
23468 finalize_missing_transition_transaction_for_operator(store.clone(), transaction.transaction_id).await,
23469 Err(TransitionOperatorError::CandidateNotMissing(TransitionOperatorProbe::Unsupported))
23470 ));
23471 load_transition_transaction_record(store, transaction.transaction_id)
23472 .await
23473 .expect("an unsupported probe must retain the transaction journal");
23474 assert_eq!(backend.remove_count().await, 0);
23475 }
23476
23477 #[cfg(feature = "test-util")]
23478 #[tokio::test]
23479 #[serial_test::serial(storage_class_env)]
23480 async fn transition_response_loss_persists_unknown_outcome_for_provider_recovery() {
23481 let temp_dir = tempfile::tempdir().expect("create temp store dir");
23482 let (ctx, store, _shutdown) =
23483 without_storage_class_env(build_isolated_test_store(temp_dir.path(), "transition-response-loss", &[4])).await;
23484 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
23485
23486 let tier_name = "TXRESPONSELOSS";
23487 let backend = register_transition_reconcile_test_tier(&ctx.tier_config_mgr(), tier_name).await;
23488 let _compaction_proof = install_transition_transaction_compaction_fleet_proof_for_test("object-transaction-fencing-test");
23489 let bucket = "transition-response-loss-bucket";
23490 let object = "source.bin";
23491 store
23492 .make_bucket(bucket, &MakeBucketOptions::default())
23493 .await
23494 .expect("source bucket should be created");
23495 let payload = b"a response-lost tier PUT must remain recoverable".repeat(1024);
23496 let mut reader = PutObjReader::from_vec(payload.clone());
23497 let source = store
23498 .put_object(bucket, object, &mut reader, &ObjectOptions::default())
23499 .await
23500 .expect("source object should be written");
23501 backend.lose_next_put_response();
23502
23503 let error = store
23504 .transition_object(
23505 bucket,
23506 object,
23507 &ObjectOptions {
23508 no_lock: true,
23509 transition: TransitionOptions {
23510 status: TRANSITION_PENDING.to_string(),
23511 tier: tier_name.to_string(),
23512 etag: source.etag.clone().expect("source object should have an ETag"),
23513 ..Default::default()
23514 },
23515 version_id: source.version_id.map(|version| version.to_string()),
23516 mod_time: source.mod_time,
23517 ..Default::default()
23518 },
23519 )
23520 .await
23521 .expect_err("a lost tier PUT response must fail the transition request");
23522 assert!(
23523 matches!(error, StorageError::Io(ref err) if err.kind() == std::io::ErrorKind::ConnectionReset),
23524 "the response-loss error must remain visible to the caller: {error:?}"
23525 );
23526
23527 let records = store
23528 .clone()
23529 .list_objects_v2(
23530 RUSTFS_META_BUCKET,
23531 TRANSITION_TRANSACTION_RECORD_PREFIX,
23532 None,
23533 None,
23534 10,
23535 false,
23536 None,
23537 false,
23538 )
23539 .await
23540 .expect("transition transaction records should be listable");
23541 assert_eq!(records.objects.len(), 1, "response loss must leave one durable transaction record");
23542 let transaction_id = records.objects[0]
23543 .name
23544 .rsplit('/')
23545 .next()
23546 .and_then(|name| name.strip_suffix(".json"))
23547 .and_then(|name| uuid::Uuid::parse_str(name).ok())
23548 .expect("transaction record name should contain a UUID");
23549 let transaction = load_transition_transaction_record(store.clone(), transaction_id)
23550 .await
23551 .expect("response loss transaction record should load");
23552 assert_eq!(
23553 transaction.state,
23554 TransitionTransactionState::UploadOutcomeUnknown,
23555 "a response-lost PUT must not remain in UploadStarted"
23556 );
23557 assert_eq!(transaction.revision, 1, "compact response loss must retain the pre-PUT fence generation");
23558 assert!(
23559 backend.contains(&transaction.remote_object).await,
23560 "the test backend must retain the remote candidate"
23561 );
23562
23563 backend
23564 .set_transition_candidate_probe_override(Some(TransitionCandidateProbe::Unsupported))
23565 .await;
23566 let unsupported_stats = recover_transition_transaction_records(store.clone(), 100, None)
23567 .await
23568 .expect("active unknown ownership should remain fenced before provider recovery");
23569 assert_eq!(
23570 (
23571 unsupported_stats.scanned,
23572 unsupported_stats.recovered,
23573 unsupported_stats.retained,
23574 unsupported_stats.failed
23575 ),
23576 (1, 0, 1, 0),
23577 "active unknown ownership must retain the upload before the recovery deadline"
23578 );
23579 assert_eq!(transition_transaction_record_count(store.clone()).await, 1);
23580 let recovery_control_id =
23581 transition_recovery_control_id(&transaction).expect("transition recovery control id should derive");
23582 assert!(matches!(
23583 load_recovery_control(store.clone(), IlmRecoveryProtocol::TransitionTransaction, &recovery_control_id).await,
23584 Err(Error::ConfigNotFound)
23585 ));
23586 assert!(
23587 backend.contains(&transaction.remote_object).await,
23588 "active ownership must not delete the candidate"
23589 );
23590 assert_eq!(backend.remove_count().await, 0, "active ownership must not attempt cleanup");
23591 assert!(
23592 !backend
23593 .op_log()
23594 .await
23595 .iter()
23596 .any(|operation| matches!(operation, MockWarmOp::Probe { .. })),
23597 "active ownership must not probe the provider"
23598 );
23599
23600 backend.set_transition_candidate_probe_override(None).await;
23601 let stats =
23602 recover_transition_transaction_records_at(store.clone(), 100, None, i128::from(transaction.not_after_unix_nanos) + 1)
23603 .await
23604 .expect("provider-authoritative recovery should run");
23605 assert_eq!((stats.scanned, stats.recovered, stats.retained, stats.failed), (1, 1, 0, 0));
23606 assert_eq!(transition_transaction_record_count(store.clone()).await, 0);
23607 assert_eq!(backend.object_count().await, 0, "recovery must delete the provider-confirmed candidate");
23608 let op_log = backend.op_log().await;
23609 assert!(
23610 op_log.iter().any(|operation| matches!(operation, MockWarmOp::Probe { .. })),
23611 "response-loss recovery must enter the provider probe branch"
23612 );
23613 assert!(
23614 op_log.iter().any(|operation| matches!(operation, MockWarmOp::Put { .. })),
23615 "response-loss fixture must record that the remote PUT reached the backend"
23616 );
23617 let source_after = store
23618 .get_object_info(bucket, object, &ObjectOptions::default())
23619 .await
23620 .expect("recovery must preserve the local source object");
23621 assert_eq!(source_after.size, i64::try_from(payload.len()).expect("payload length should fit i64"));
23622 }
23623}