Skip to main content

rustfs_ecstore/store/
init.rs

1// Copyright 2024 RustFS Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use super::*;
16use crate::core::pools::{
17    PoolMetaReplicaState, PoolMetaWriteState, local_decommission_queue_prefix, persist_pool_meta_identity_for_attested_pools,
18    persist_pool_meta_identity_for_startup, pool_meta_has_active_decommission,
19};
20use crate::runtime::instance::InstanceContext;
21use crate::runtime::sources as runtime_sources;
22use crate::storage_api_contracts::object::EcstoreObjectIO;
23use rustfs_config::server_config::KVS;
24use rustfs_credentials::{RPC_SECRET_REQUIRED_OPERATOR_MESSAGE, try_get_rpc_token};
25use std::future::Future;
26use tracing::{debug, error, info, warn};
27
28const LOG_COMPONENT_ECSTORE: &str = "ecstore";
29const LOG_SUBSYSTEM_STORE_INIT: &str = "store_init";
30const EVENT_DECOMMISSION_RESUME_RETRY: &str = "decommission_resume_retry";
31const EVENT_STORE_FORMAT_RETRY: &str = "store_format_retry";
32const EVENT_ECSTORE_INIT_STATUS: &str = "ecstore_init_status";
33const EVENT_STORE_RPC_SECRET_PREFLIGHT_FAILED: &str = "store_rpc_secret_preflight_failed";
34
35fn pool_first_endpoint_is_local(pool: &crate::layout::endpoints::PoolEndpoints) -> bool {
36    pool.endpoints.as_ref().first().is_some_and(|endpoint| endpoint.is_local)
37}
38
39fn startup_pool_drive_counts(endpoint_pools: &EndpointServerPools) -> Vec<usize> {
40    endpoint_pools.as_ref().iter().map(|pool| pool.drives_per_set).collect()
41}
42
43fn resolve_startup_pool_defaults(endpoint_pools: &EndpointServerPools) -> Result<Vec<usize>> {
44    resolve_startup_pool_defaults_with(endpoint_pools, ECStore::validate_startup_storage_class)
45}
46
47fn resolve_startup_pool_defaults_with(
48    endpoint_pools: &EndpointServerPools,
49    validate: impl FnOnce(&EndpointServerPools) -> Result<()>,
50) -> Result<Vec<usize>> {
51    validate(endpoint_pools)?;
52    let drive_counts = startup_pool_drive_counts(endpoint_pools);
53    drive_counts.into_iter().map(ec_drives_no_config).collect()
54}
55
56/// Fail fast when the topology spans remote nodes but no internode RPC secret
57/// resolves. Every remote format read would otherwise fail client-side with
58/// "No valid auth token" and startup would retry for minutes before dying with
59/// a misleading "erasure read quorum" error (issues #4939, #5153).
60fn preflight_startup_rpc_secret(endpoint_pools: &EndpointServerPools) -> Result<()> {
61    preflight_startup_rpc_secret_with(endpoint_pools, try_get_rpc_token)
62}
63
64fn preflight_startup_rpc_secret_with(
65    endpoint_pools: &EndpointServerPools,
66    resolve_rpc_token: impl FnOnce() -> std::io::Result<String>,
67) -> Result<()> {
68    let has_remote_endpoint = endpoint_pools
69        .as_ref()
70        .iter()
71        .flat_map(|pool| pool.endpoints.as_ref().iter())
72        .any(|endpoint| !endpoint.is_local);
73
74    if !has_remote_endpoint {
75        return Ok(());
76    }
77
78    match resolve_rpc_token() {
79        Ok(_) => Ok(()),
80        Err(err) => {
81            // The message prefix must stay aligned with the runtime log in
82            // cluster/rpc/http_auth.rs: the log-analyzer `rpc-secret-resolution`
83            // rule anchors on it, and this preflight aborts before that log
84            // would ever be emitted.
85            error!(
86                event = EVENT_STORE_RPC_SECRET_PREFLIGHT_FAILED,
87                component = LOG_COMPONENT_ECSTORE,
88                subsystem = LOG_SUBSYSTEM_STORE_INIT,
89                "RPC auth secret resolution failed: {err}; {RPC_SECRET_REQUIRED_OPERATOR_MESSAGE}"
90            );
91            Err(Error::other(format!(
92                "store init aborted: endpoints include remote nodes but {err}; {RPC_SECRET_REQUIRED_OPERATOR_MESSAGE}"
93            )))
94        }
95    }
96}
97
98const LOCAL_DECOMMISSION_RESUME_RETRY_DELAY: Duration = Duration::from_secs(30);
99const LOCAL_DECOMMISSION_WATCHDOG_INTERVAL: Duration = Duration::from_secs(30);
100const LOCAL_DECOMMISSION_WATCHDOG_MAX_RETRY_DELAY: Duration = Duration::from_secs(60 * 5);
101const REBALANCE_INITIAL_RESUME_DELAY: Duration = Duration::from_secs(10);
102const REBALANCE_RESUME_RETRY_DELAY: Duration = Duration::from_secs(10);
103
104fn should_retry_format_load(err: &Error) -> bool {
105    !matches!(
106        err,
107        Error::CorruptedFormat | Error::UnsupportedSnsdExpansion { .. } | Error::PoolTopologyMismatch { .. }
108    )
109}
110
111fn should_auto_start_rebalance_after_init(decommission_running: bool, rebalance_resume_required: bool) -> bool {
112    rebalance_resume_required && !decommission_running
113}
114
115fn should_defer_rebalance_auto_start(distributed: bool, fleet_proof_available: bool) -> bool {
116    distributed && !fleet_proof_available
117}
118
119async fn wait_for_local_decommission_resume_delay(rx: &CancellationToken, delay: Duration) -> bool {
120    tokio::select! {
121        _ = rx.cancelled() => false,
122        _ = tokio::time::sleep(delay) => true,
123    }
124}
125
126fn local_decommission_watchdog_retry_delay(consecutive_failures: u32) -> Duration {
127    let exponent = consecutive_failures.saturating_sub(1).min(4);
128    LOCAL_DECOMMISSION_RESUME_RETRY_DELAY
129        .saturating_mul(1_u32 << exponent)
130        .min(LOCAL_DECOMMISSION_WATCHDOG_MAX_RETRY_DELAY)
131}
132
133async fn wait_for_rebalance_resume_delay(rx: &CancellationToken, delay: Duration) -> bool {
134    tokio::select! {
135        _ = rx.cancelled() => false,
136        _ = tokio::time::sleep(delay) => true,
137    }
138}
139
140async fn wait_for_rebalance_resume_retry(rx: &CancellationToken) -> bool {
141    wait_for_rebalance_resume_delay(rx, REBALANCE_RESUME_RETRY_DELAY).await
142}
143
144fn resolve_store_init_stage_result(result: Result<()>, stage: &str) -> Result<()> {
145    result.map_err(|err| Error::other(format!("store init failed during {stage}: {err}")))
146}
147
148async fn load_pool_meta_for_startup<S>(
149    pools: Vec<Arc<S>>,
150    write_state: &mut PoolMetaWriteState,
151) -> Result<(PoolMeta, PoolMetaReplicaState)>
152where
153    S: EcstoreObjectIO,
154{
155    let mut meta = PoolMeta::default();
156    let replica_state = meta
157        .load_for_startup_observing(pools, write_state)
158        .await
159        .map_err(|err| Error::other_with_context("store init failed during load_pool_meta", err))?;
160    write_state.observe_replicas(replica_state);
161    write_state
162        .ensure_missing_metadata_can_initialize()
163        .map_err(|err| Error::other(format!("store init failed during classify_pool_meta_absence: {err}")))?;
164    Ok((meta, replica_state))
165}
166
167async fn establish_pool_meta_bootstrap_identity_if_proven<S>(
168    pools: Vec<Arc<S>>,
169    write_state: &mut PoolMetaWriteState,
170    elected_writer: bool,
171) -> Result<()>
172where
173    S: EcstoreObjectIO,
174{
175    if elected_writer && write_state.bootstrap_identity_proven() {
176        return persist_pool_meta_identity_for_startup(pools, write_state, false).await;
177    }
178    if write_state.bootstrap_identity_proven() {
179        return Ok(());
180    }
181    // Multi-pool bootstrap whose pools were formatted by different nodes: no
182    // single process can prove the whole deployment fresh in memory, so each
183    // creator attests the pools it formatted first-hand with the shared nonce
184    // and the elected writer waits for a complete, agreeing pending set.
185    let attested = write_state.attested_pool_indices();
186    if attested.is_empty() {
187        return Ok(());
188    }
189    persist_pool_meta_identity_for_attested_pools(pools, write_state, &attested).await
190}
191
192async fn save_validated_pool_meta_for_startup<S>(
193    meta: &PoolMeta,
194    pools: Vec<Arc<S>>,
195    write_state: &mut PoolMetaWriteState,
196) -> Result<PoolMeta>
197where
198    S: EcstoreObjectIO,
199{
200    meta.save_for_startup_observing(pools, write_state)
201        .await
202        .map_err(|err| Error::other(format!("store init failed during save_validated_pool_meta: {err}")))
203}
204
205async fn persist_pool_meta_for_startup_if_safe<S>(
206    meta: &PoolMeta,
207    pools: Vec<Arc<S>>,
208    replica_state: PoolMetaReplicaState,
209    write_state: &mut PoolMetaWriteState,
210    topology_update: bool,
211    elected_writer: bool,
212) -> Result<PoolMeta>
213where
214    S: EcstoreObjectIO,
215{
216    if !elected_writer {
217        return Ok(meta.clone());
218    }
219    let should_write = topology_update || (replica_state.needs_repair && replica_state.repair_write_safe);
220    if topology_update {
221        replica_state.ensure_write_safe("store init failed during save_validated_pool_meta")?;
222    }
223    if should_write || write_state.identity_requires_repair() {
224        write_state.ensure_write_safe("store init failed during save_validated_pool_meta")?;
225    }
226    let mut committed = meta.clone();
227    if should_write {
228        if write_state.bootstrap_identity_proven() || write_state.identity_is_pending() {
229            persist_pool_meta_identity_for_startup(pools.clone(), write_state, false)
230                .await
231                .map_err(|err| Error::other(format!("store init failed during prepare_pool_meta_identity: {err}")))?;
232        }
233        committed = save_validated_pool_meta_for_startup(meta, pools.clone(), write_state).await?;
234    }
235    if should_write || write_state.identity_requires_repair() {
236        persist_pool_meta_identity_for_startup(pools, write_state, true)
237            .await
238            .map_err(|err| Error::other(format!("store init failed during commit_pool_meta_identity: {err}")))?;
239    }
240    Ok(committed)
241}
242
243async fn run_local_decommission_watchdog<F, Fut>(rx: CancellationToken, mut reconcile: F)
244where
245    F: FnMut() -> Fut,
246    Fut: Future<Output = Result<()>>,
247{
248    let mut consecutive_failures = 0_u32;
249
250    loop {
251        if rx.is_cancelled() {
252            return;
253        }
254
255        let delay = match reconcile().await {
256            Ok(()) => {
257                consecutive_failures = 0;
258                LOCAL_DECOMMISSION_WATCHDOG_INTERVAL
259            }
260            Err(err) => {
261                consecutive_failures = consecutive_failures.saturating_add(1);
262                let retry_delay = local_decommission_watchdog_retry_delay(consecutive_failures);
263                warn!(
264                    event = EVENT_DECOMMISSION_RESUME_RETRY,
265                    component = LOG_COMPONENT_ECSTORE,
266                    subsystem = LOG_SUBSYSTEM_STORE_INIT,
267                    consecutive_failures,
268                    retry_delay_secs = retry_delay.as_secs(),
269                    error = %err,
270                    "Retrying decommission worker recovery"
271                );
272                retry_delay
273            }
274        };
275
276        if !wait_for_local_decommission_resume_delay(&rx, delay).await {
277            return;
278        }
279    }
280}
281
282async fn reconcile_local_decommission_after_init(store: &Arc<ECStore>, rx: CancellationToken) -> Result<()> {
283    store
284        .ensure_pool_meta_side_effects_safe("decommission worker recovery blocked while pool metadata requires recovery")
285        .await?;
286    if store.has_active_local_decommission_worker().await {
287        return Ok(());
288    }
289    store.refresh_pool_status_meta().await?;
290    let resume_required = pool_meta_has_active_decommission(&*store.pool_meta.read().await);
291    if resume_required {
292        crate::core::pools::acquire_pool_activation_fleet_proof(&store.ctx).await?;
293    }
294    store.spawn_missing_local_decommission_routines_with_token(rx).await
295}
296
297async fn supervise_local_decommission_after_init(store: Arc<ECStore>, rx: CancellationToken) {
298    run_local_decommission_watchdog(rx.clone(), || {
299        let store = store.clone();
300        let worker_rx = rx.clone();
301        async move { reconcile_local_decommission_after_init(&store, worker_rx).await }
302    })
303    .await;
304}
305
306async fn resume_rebalance_after_init(store: Arc<ECStore>, rx: CancellationToken) {
307    if !wait_for_rebalance_resume_delay(&rx, REBALANCE_INITIAL_RESUME_DELAY).await {
308        return;
309    }
310
311    loop {
312        if rx.is_cancelled() {
313            return;
314        }
315
316        let resume_required = store
317            .rebalance_meta
318            .read()
319            .await
320            .as_ref()
321            .is_some_and(crate::services::rebalance::rebalance_requires_worker_activation);
322        if !resume_required {
323            return;
324        }
325
326        if should_defer_rebalance_auto_start(
327            store.ctx.is_dist_erasure().await,
328            crate::services::notification_sys::acquire_cross_pool_fence_fleet_proof().is_some(),
329        ) {
330            if !wait_for_rebalance_resume_retry(&rx).await {
331                return;
332            }
333            continue;
334        }
335
336        match store.start_rebalance().await {
337            Ok(()) => return,
338            Err(err) if crate::core::pools::is_pool_activation_fleet_proof_error(&err) => {
339                warn!(
340                    event = EVENT_ECSTORE_INIT_STATUS,
341                    component = LOG_COMPONENT_ECSTORE,
342                    subsystem = LOG_SUBSYSTEM_STORE_INIT,
343                    stage = "start_rebalance",
344                    state = "retrying",
345                    reason = "fleet_capability_proof_unavailable",
346                    error = %err,
347                    retry_delay_secs = REBALANCE_RESUME_RETRY_DELAY.as_secs(),
348                    "Retrying deferred rebalance auto-start"
349                );
350                if !wait_for_rebalance_resume_retry(&rx).await {
351                    return;
352                }
353            }
354            Err(err) => {
355                error!(
356                    event = EVENT_ECSTORE_INIT_STATUS,
357                    component = LOG_COMPONENT_ECSTORE,
358                    subsystem = LOG_SUBSYSTEM_STORE_INIT,
359                    stage = "start_rebalance",
360                    state = "failed",
361                    reason = "deferred_resume_failed",
362                    error = %err,
363                    "Failed to resume rebalance after store initialization"
364                );
365                return;
366            }
367        }
368    }
369}
370
371impl ECStore {
372    /// Shutdown token owned by this store instance.
373    pub fn background_cancel_token(&self) -> Option<CancellationToken> {
374        self.ctx.background_cancel_token()
375    }
376
377    /// Validate topology and process storage-class overrides before any disk is opened.
378    pub fn validate_startup_storage_class(endpoint_pools: &EndpointServerPools) -> Result<()> {
379        let drive_counts = startup_pool_drive_counts(endpoint_pools);
380        storageclass::lookup_config_for_pools(&KVS::new(), &drive_counts).map(|_| ())
381    }
382
383    #[allow(clippy::new_ret_no_self)]
384    #[instrument(level = "debug", skip(endpoint_pools))]
385    pub async fn new(address: SocketAddr, endpoint_pools: EndpointServerPools, ctx: CancellationToken) -> Result<Arc<Self>> {
386        Self::new_with_instance_ctx(address, endpoint_pools, ctx, crate::runtime::instance::bootstrap_ctx()).await
387    }
388
389    /// Build a store around an explicit instance context (Phase 5 follow-up,
390    /// backlog#1052). The legacy [`ECStore::new`] entry adopts the process
391    /// bootstrap context, keeping single-instance startup byte-for-byte
392    /// unchanged; a caller that owns its own context (a future second embedded
393    /// server) passes it here so every construction-time write — pool sets,
394    /// local-disk registry, deployment id — lands on that context instead of
395    /// the shared bootstrap one.
396    #[allow(clippy::new_ret_no_self)]
397    #[instrument(level = "debug", skip(endpoint_pools, instance_ctx))]
398    pub async fn new_with_instance_ctx(
399        address: SocketAddr,
400        endpoint_pools: EndpointServerPools,
401        ctx: CancellationToken,
402        instance_ctx: Arc<InstanceContext>,
403    ) -> Result<Arc<Self>> {
404        Box::pin(Self::new_with_instance_ctx_inner(address, endpoint_pools, ctx, instance_ctx)).await
405    }
406
407    async fn new_with_instance_ctx_inner(
408        address: SocketAddr,
409        endpoint_pools: EndpointServerPools,
410        ctx: CancellationToken,
411        instance_ctx: Arc<InstanceContext>,
412    ) -> Result<Arc<Self>> {
413        instance_ctx.bind_background_cancel_token(ctx.clone());
414
415        // let layouts = DisksLayout::from_volumes(endpoints.as_slice())?;
416
417        // Validate topology and environment overrides before opening any disk.
418        // The values stored on SetDisks remain pure per-pool topology defaults;
419        // payload writes use the runtime storage-class snapshot published later
420        // from config before the store is marked ready.
421        let default_pool_parities = resolve_startup_pool_defaults(&endpoint_pools)?;
422
423        preflight_startup_rpc_secret(&endpoint_pools)?;
424
425        let mut deployment_id = None;
426        let mut pool_meta_bootstrap_authorities = Vec::new();
427
428        // let (endpoint_pools, _) = EndpointServerPools::create_server_endpoints(address.as_str(), &layouts)?;
429
430        let mut pools = Vec::with_capacity(endpoint_pools.as_ref().len());
431        let mut disk_map = HashMap::with_capacity(endpoint_pools.as_ref().len());
432
433        let mut local_disks = Vec::new();
434
435        debug!(
436            event = EVENT_ECSTORE_INIT_STATUS,
437            component = LOG_COMPONENT_ECSTORE,
438            subsystem = LOG_SUBSYSTEM_STORE_INIT,
439            address = %address,
440            "Initializing ECStore address"
441        );
442        let mut host = address.ip().to_string();
443        if host.is_empty() {
444            host = runtime_sources::rustfs_host().await
445        }
446        let mut port = address.port().to_string();
447        if port.is_empty() {
448            port = runtime_sources::rustfs_port().to_string()
449        }
450        debug!(
451            event = EVENT_ECSTORE_INIT_STATUS,
452            component = LOG_COMPONENT_ECSTORE,
453            subsystem = LOG_SUBSYSTEM_STORE_INIT,
454            host = %host,
455            port = %port,
456            "Initializing ECStore host"
457        );
458        init_local_peer(&endpoint_pools, &host, &port).await;
459
460        // debug!("endpoint_pools: {:?}", endpoint_pools);
461
462        for (i, pool_eps) in endpoint_pools.as_ref().iter().enumerate() {
463            let pool_first_is_local = pool_first_endpoint_is_local(pool_eps);
464            let parity_drives = default_pool_parities
465                .get(i)
466                .copied()
467                .ok_or_else(|| Error::other(format!("store init failed to resolve default parity for pool {i}")))?;
468
469            // validate_parity(parity_count, pool_eps.drives_per_set)?;
470
471            // Build disks with health monitoring available, but do not start
472            // periodic monitoring until format loading succeeds. Startup RPC
473            // failures can still spawn recovery probes for peers that come up
474            // after this node.
475            let (mut disks, errs) = init_format::init_disks(
476                &pool_eps.endpoints,
477                &DiskOption {
478                    cleanup: true,
479                    health_check: true,
480                },
481            )
482            .await;
483
484            check_disk_fatal_errs(&errs)?;
485
486            let loaded_format = {
487                let mut times = 0;
488                let mut interval = 1;
489                loop {
490                    match init_format::connect_load_init_formats_with_instance_ctx(
491                        &instance_ctx,
492                        pool_first_is_local,
493                        &mut disks,
494                        pool_eps.set_count,
495                        pool_eps.drives_per_set,
496                        deployment_id,
497                    )
498                    .await
499                    {
500                        Ok(fm) => break Ok(fm),
501                        Err(e) if !should_retry_format_load(&e) => break Err(e),
502                        // Wrap the final error if we are giving up
503                        Err(e) if times >= 10 => {
504                            break Err(Error::other(format!("store init failed to load formats after {times} retries: {e}")));
505                        }
506                        // Retrying so just drop the error
507                        Err(_) => {}
508                    }
509                    times += 1;
510                    if interval < 16 {
511                        interval *= 2;
512                    }
513                    debug!(
514                        event = EVENT_STORE_FORMAT_RETRY,
515                        component = LOG_COMPONENT_ECSTORE,
516                        subsystem = LOG_SUBSYSTEM_STORE_INIT,
517                        retry_count = times,
518                        retry_delay_secs = interval,
519                        "Retrying storage format load"
520                    );
521                    select! {
522                        _ = tokio::signal::ctrl_c() => {
523                            info!(
524                                event = EVENT_STORE_FORMAT_RETRY,
525                                component = LOG_COMPONENT_ECSTORE,
526                                subsystem = LOG_SUBSYSTEM_STORE_INIT,
527                                reason = "ctrl_c",
528                                "Interrupted storage format retry loop"
529                            );
530                            exit(0);
531                        }
532                        _ = sleep(Duration::from_secs(interval)) => {
533                        }
534                    }
535                    // After waiting for peers, clear transient faulty marks so the next attempt can open RPCs again
536                    // (these `DiskStore` handles are reused; `is_faulty()` would otherwise short-circuit).
537                    for disk in disks.iter().flatten() {
538                        disk.reset_health_for_store_init_retry();
539                    }
540                }
541            }?;
542            // First-hand authority for this pool only: `Fresh` when this process
543            // formatted it, `LegacyAdoption` when it verified the migration, and
544            // `None` when it merely read a format another node created.
545            pool_meta_bootstrap_authorities.push(loaded_format.pool_meta_bootstrap_authority);
546            let fm = loaded_format.format;
547
548            // Format loading succeeded, enable health monitoring on all disks
549            for disk in disks.iter().flatten() {
550                disk.enable_health_check();
551            }
552
553            if deployment_id.is_none() {
554                deployment_id = Some(fm.id);
555            }
556
557            if deployment_id != Some(fm.id) {
558                return Err(Error::other("store init failed: deployment IDs do not match across pools"));
559            }
560
561            if deployment_id.is_some_and(|id| id.is_nil()) {
562                deployment_id = Some(Uuid::new_v4());
563            }
564
565            for disk in disks.iter() {
566                if disk.is_some() && disk.as_ref().expect("operation should succeed").is_local() {
567                    local_disks.push(disk.as_ref().expect("operation should succeed").clone());
568                }
569            }
570
571            let sets = Sets::new_with_instance_ctx(disks.clone(), pool_eps, &fm, i, parity_drives, instance_ctx.clone()).await?;
572            pools.push(sets);
573
574            disk_map.insert(i, disks);
575        }
576
577        // Replace the local disk
578        if !instance_ctx.is_dist_erasure().await {
579            runtime_sources::record_local_disks(&instance_ctx, local_disks).await;
580        }
581
582        let deployment_id = deployment_id.ok_or_else(|| Error::other("store init failed: deployment id is not initialized"))?;
583        let peer_sys = S3PeerSys::new_with_instance_ctx(&endpoint_pools, instance_ctx.clone());
584        let mut pool_meta = PoolMeta::new(&pools, &PoolMeta::default());
585        pool_meta.dont_save = true;
586        let elected_bootstrap_writer = pools
587            .first()
588            .is_some_and(|pool| pool_first_endpoint_is_local(&pool.endpoints));
589        let pool_meta_write_state = PoolMetaWriteState::for_startup_with_pool_bootstrap_authorities(
590            deployment_id,
591            pool_meta_bootstrap_authorities,
592            elected_bootstrap_writer,
593        );
594
595        let decommission_cancelers = RwLock::new(vec![None; pools.len()]);
596        let ec = Arc::new(ECStore {
597            id: deployment_id,
598            disk_map,
599            pools,
600            peer_sys,
601            pool_meta: RwLock::new(pool_meta),
602            rebalance_meta: RwLock::new(None),
603            decommission_cancelers,
604            start_gate: Mutex::new(()),
605            pool_meta_save_gate: Mutex::new(pool_meta_write_state),
606            // Adopt the caller's context (the process bootstrap one on the
607            // legacy path) so startup writes (erasure type recorded before
608            // this point) and later reads share one cell.
609            ctx: instance_ctx.clone(),
610            bucket_fence_registry: std::sync::Arc::default(),
611        });
612
613        // Only set it when this instance's deployment ID is not yet configured
614        if instance_ctx.deployment_id().is_none() {
615            instance_ctx.set_deployment_id(deployment_id);
616        }
617
618        let wait_sec = 5;
619        let mut exit_count = 0;
620        loop {
621            if let Err(err) = ec.init(ctx.clone()).await {
622                error!("init err: {}", err);
623                error!("retry after  {} second", wait_sec);
624                sleep(Duration::from_secs(wait_sec)).await;
625
626                if exit_count > 10 {
627                    return Err(Error::other("store init failed: init retry budget exhausted"));
628                }
629
630                exit_count += 1;
631
632                continue;
633            }
634
635            break;
636        }
637
638        runtime_sources::publish_object_store(ec.clone()).await;
639
640        Ok(ec)
641    }
642
643    #[instrument(level = "debug", skip(self, rx))]
644    pub async fn init(self: &Arc<Self>, rx: CancellationToken) -> Result<()> {
645        runtime_sources::ensure_boot_time().await;
646
647        let should_persist_pool_meta = self
648            .pools
649            .first()
650            .is_some_and(|pool| pool_first_endpoint_is_local(&pool.endpoints));
651        #[cfg(feature = "e2e-test-hooks")]
652        let startup_attempt = uuid::Uuid::new_v4();
653        let (meta, pool_meta_replica_state) = {
654            let mut write_state = self.pool_meta_save_gate.lock().await;
655            establish_pool_meta_bootstrap_identity_if_proven(self.pools.clone(), &mut write_state, should_persist_pool_meta)
656                .await
657                .map_err(|err| Error::other(format!("store init failed during establish_pool_meta_bootstrap_identity: {err}")))?;
658            let load = load_pool_meta_for_startup(self.pools.clone(), &mut write_state);
659            #[cfg(feature = "e2e-test-hooks")]
660            let load = crate::core::pools::startup_cas_test_scope(startup_attempt, "load", &self.pools, load);
661            load.await?
662        };
663        let update = meta.validate(self.pools.clone())?;
664        #[cfg(feature = "e2e-test-hooks")]
665        crate::core::pools::startup_cas_test_observe(serde_json::json!({
666            "kind": "startup-classifier", "attempt": startup_attempt,
667            "elected_writer": should_persist_pool_meta,
668            "needs_repair": pool_meta_replica_state.needs_repair,
669            "repair_write_safe": pool_meta_replica_state.repair_write_safe,
670            "topology_update": update,
671        }));
672        let endpoints = runtime_sources::endpoint_pools_or_default();
673
674        let mut installed_pool_meta = if update {
675            PoolMeta::new(&self.pools, &meta)
676        } else {
677            meta.clone()
678        };
679        // Only one local node should persist validated pool metadata here; otherwise
680        // distributed startup can race on the same lock and replay the prior init bug.
681        {
682            let mut write_state = self.pool_meta_save_gate.lock().await;
683            let persist = persist_pool_meta_for_startup_if_safe(
684                &installed_pool_meta,
685                self.pools.clone(),
686                pool_meta_replica_state,
687                &mut write_state,
688                update,
689                should_persist_pool_meta,
690            );
691            #[cfg(feature = "e2e-test-hooks")]
692            let persist = crate::core::pools::startup_cas_test_scope(startup_attempt, "persist", &self.pools, persist);
693            installed_pool_meta = persist.await?;
694        }
695
696        {
697            let mut pool_meta = self.pool_meta.write().await;
698            *pool_meta = installed_pool_meta.clone();
699        }
700
701        resolve_store_init_stage_result(self.load_rebalance_meta().await, "load_rebalance_meta")?;
702        let rebalance_resume_required = {
703            let rebalance_meta = self.rebalance_meta.read().await;
704            rebalance_meta
705                .as_ref()
706                .is_some_and(crate::services::rebalance::rebalance_requires_worker_activation)
707        };
708        let decommission_running =
709            pool_meta_has_active_decommission(&installed_pool_meta) || self.is_decommission_running().await;
710        let distributed = self.ctx.is_dist_erasure().await;
711        let fleet_proof_available = crate::services::notification_sys::acquire_cross_pool_fence_fleet_proof().is_some();
712        let mut rebalance_auto_start_deferred = false;
713        if should_auto_start_rebalance_after_init(decommission_running, rebalance_resume_required) {
714            if should_defer_rebalance_auto_start(distributed, fleet_proof_available) {
715                rebalance_auto_start_deferred = true;
716                warn!(
717                    event = EVENT_ECSTORE_INIT_STATUS,
718                    component = LOG_COMPONENT_ECSTORE,
719                    subsystem = LOG_SUBSYSTEM_STORE_INIT,
720                    stage = "start_rebalance",
721                    state = "deferred",
722                    reason = "fleet_capability_proof_unavailable",
723                    retry_delay_secs = REBALANCE_RESUME_RETRY_DELAY.as_secs(),
724                    "Deferred rebalance auto-start until a live fleet capability proof is available"
725                );
726            } else if let Err(err) = self.start_rebalance().await {
727                if crate::core::pools::is_pool_activation_fleet_proof_error(&err) {
728                    rebalance_auto_start_deferred = true;
729                    warn!(
730                        event = EVENT_ECSTORE_INIT_STATUS,
731                        component = LOG_COMPONENT_ECSTORE,
732                        subsystem = LOG_SUBSYSTEM_STORE_INIT,
733                        stage = "start_rebalance",
734                        state = "deferred",
735                        reason = "fleet_capability_proof_changed",
736                        error = %err,
737                        retry_delay_secs = REBALANCE_RESUME_RETRY_DELAY.as_secs(),
738                        "Deferred rebalance auto-start after the fleet capability proof changed"
739                    );
740                } else {
741                    return resolve_store_init_stage_result(Err(err), "start_rebalance");
742                }
743            }
744        } else if decommission_running && rebalance_resume_required {
745            warn!(
746                event = EVENT_ECSTORE_INIT_STATUS,
747                component = LOG_COMPONENT_ECSTORE,
748                subsystem = LOG_SUBSYSTEM_STORE_INIT,
749                stage = "start_rebalance",
750                reason = "active_decommission",
751                "Deferred rebalance auto-start during store init because decommission is active"
752            );
753        }
754
755        // Initialize the storage-owned scanner publication state only after
756        // both movement metadata sources have been loaded. SetDisks cache
757        // writers remain fail-closed until this snapshot is available.
758        let _ = self.scanner_data_usage_publication_blocked().await;
759
760        let pools = installed_pool_meta.return_resumable_pools();
761        let mut pool_indices = Vec::with_capacity(pools.len());
762
763        for p in pools.iter() {
764            if let Some(idx) = endpoints.get_pool_idx(&p.cmd_line) {
765                pool_indices.push(idx);
766            } else {
767                return Err(Error::other(format!(
768                    "store init failed to resolve resumable decommission pool `{}` from current endpoints",
769                    p.cmd_line
770                )));
771            }
772        }
773
774        let local_pool_indices = local_decommission_queue_prefix(&endpoints, &pool_indices)?;
775        let has_local_decommission_leadership = endpoints.as_ref().iter().any(pool_first_endpoint_is_local);
776        let pool_meta_write_safe = self
777            .ensure_pool_meta_side_effects_safe("decommission resume blocked while pool metadata requires recovery")
778            .await
779            .is_ok();
780        if !pool_meta_replica_state.repair_write_safe || !pool_meta_write_safe {
781            warn!(
782                event = EVENT_DECOMMISSION_RESUME_RETRY,
783                component = LOG_COMPONENT_ECSTORE,
784                subsystem = LOG_SUBSYSTEM_STORE_INIT,
785                state = "blocked",
786                pool_indices = ?local_pool_indices,
787                reason = "pool_meta_write_blocked",
788                "Decommission watchdog waiting for pool metadata recovery"
789            );
790        }
791        if has_local_decommission_leadership {
792            // The watchdog checks recovery safety and retries transient failures.
793            // Resume persisted work without an unconditional cold-start delay.
794            tokio::spawn(supervise_local_decommission_after_init(self.clone(), rx.clone()));
795        }
796
797        let recovery_store = self.clone();
798        let recovery_rx = rx.clone();
799        tokio::spawn(async move {
800            let mut delay = std::time::Duration::from_secs(5);
801            loop {
802                tokio::select! {
803                    _ = recovery_rx.cancelled() => return,
804                    _ = tokio::time::sleep(delay) => {}
805                }
806                let result = tokio::select! {
807                    _ = recovery_rx.cancelled() => return,
808                    result = tokio::time::timeout(std::time::Duration::from_secs(30), recovery_store.recover_pool_meta_transaction()) => result,
809                };
810                delay = match result {
811                    Ok(Ok(_)) => std::time::Duration::from_secs(5),
812                    failure => {
813                        let error = match failure {
814                            Ok(Err(error)) => error,
815                            _ => Error::Timeout,
816                        };
817                        recovery_store.record_pool_meta_recovery_failure(error);
818                        (delay * 2).min(std::time::Duration::from_secs(60))
819                    }
820                };
821            }
822        });
823
824        runtime_sources::init_bucket_monitor_for_current_endpoints();
825        crate::bucket::bucket_target_sys::BucketTargetSys::get().start_heartbeat();
826
827        init_background_expiry(self.clone()).await;
828        crate::bucket::lifecycle::bucket_lifecycle_ops::init_background_stale_multipart_upload_cleanup(self.clone());
829
830        TransitionState::init(self.clone()).await;
831        crate::services::tier::tier::try_migrate_tiering_config(self.clone()).await;
832
833        if let Err(err) = runtime_sources::init_tier_config_mgr(self.clone()).await {
834            info!("TierConfigMgr init error: {}", err);
835        }
836
837        if rebalance_auto_start_deferred {
838            let store = self.clone();
839            tokio::spawn(resume_rebalance_after_init(store, rx));
840        }
841
842        Ok(())
843    }
844
845    pub fn init_local_disks() {}
846
847    pub fn single_pool(&self) -> bool {
848        self.pools.len() == 1
849    }
850
851    /// The set-local create-only check is atomic only when every object
852    /// mutation uses that same, enabled namespace lock domain.
853    pub fn supports_atomic_create_only_write_back(&self) -> bool {
854        !self.ctx.lock_manager().is_disabled() && self.pools.len() == 1 && self.pools[0].disk_set.len() == 1
855    }
856}
857
858#[cfg(test)]
859mod tests {
860    use super::{
861        LOCAL_DECOMMISSION_RESUME_RETRY_DELAY, LOCAL_DECOMMISSION_WATCHDOG_MAX_RETRY_DELAY, PoolMetaWriteState,
862        establish_pool_meta_bootstrap_identity_if_proven, load_pool_meta_for_startup, local_decommission_watchdog_retry_delay,
863        persist_pool_meta_for_startup_if_safe, pool_first_endpoint_is_local, pool_meta_has_active_decommission,
864        preflight_startup_rpc_secret_with, resolve_startup_pool_defaults_with, resolve_store_init_stage_result,
865        run_local_decommission_watchdog, save_validated_pool_meta_for_startup, should_auto_start_rebalance_after_init,
866        should_defer_rebalance_auto_start, should_retry_format_load, wait_for_local_decommission_resume_delay,
867    };
868    use crate::core::pools::PoolMetaBootstrapAuthority;
869    #[cfg(feature = "test-util")]
870    use crate::disk::DiskAPI;
871    #[cfg(feature = "test-util")]
872    use crate::{
873        bucket::lifecycle::{
874            DurableIlmRecordCheckpoint, ILM_META_PREFIX, ValidatedDurableIlmRecord,
875            bucket_lifecycle_ops::{ExpiryState, ManualTransitionRunOptions, recover_manual_transition_jobs_once},
876            lifecycle::{TRANSITION_PENDING, TransitionOptions},
877            manual_transition_job::{
878                ManualTransitionJobRecord, ManualTransitionScopeAdmission, ManualTransitionTaskRecord,
879                ManualTransitionWorkerResult, ManualTransitionWorkerResultRecord, manual_transition_job_record_object_name,
880                manual_transition_scope_record_object_name, manual_transition_task_object_name,
881                manual_transition_worker_result_object_name, manual_transition_worker_result_task_key,
882            },
883            recovery_control::{
884                IlmRecoveryClassification, IlmRecoveryControl, IlmRecoveryControlIdentity, IlmRecoveryErrorCode,
885                IlmRecoveryProtocol, MAX_RECOVERY_ATTEMPTS, list_recovery_controls, load_recovery_control,
886                observe_recovery_source, recovery_control_record_object_name, save_recovery_control_if_absent,
887            },
888            recovery_disposition::{
889                IlmRecoveryDispositionExecutionOutcome, IlmRecoveryDispositionState, RecoveryDispositionCrashStage,
890                dry_run_recovery_disposition, execute_recovery_disposition, inject_recovery_disposition_crash_once,
891                load_recovery_disposition,
892            },
893            recovery_disposition_runtime::garbage_collect_completed_recovery_disposition,
894            recovery_export::{
895                create_recovery_export, inspect_recovery_export_observation, load_recovery_export,
896                recovery_export_record_object_name,
897            },
898            tier_delete_journal::{
899                DecommissionCheckpointTargetFailureHook, TIER_DELETE_DISPATCH_MANIFEST_PREFIX, TIER_DELETE_JOURNAL_PREFIX,
900                TierDeleteChunkTestBarrier, TierDeleteChunkTestStage, TierDeleteDispatchBatchLimitGuard,
901                TierDeleteDispatchManifestState, TierDeleteDispatchMemberReadTestHook, TierDeleteDispatchMemberReadTestStage,
902                TierDeleteDispatchRollbackTestHook, complete_tier_delete_dispatch, encode_tier_delete_journal_entry,
903                install_test_tier_delete_dispatch_fixture, persist_tier_delete_journal_entry, prepare_tier_delete_dispatch,
904                recover_test_tier_delete_dispatch_manifest, recover_test_tier_delete_dispatch_manifest_with_page_budget,
905                recover_test_tier_delete_journal_pass_with_budget, recover_tier_delete_dispatch_manifests,
906                recover_tier_delete_journal_entries, test_tier_delete_dispatch_manifest_checkpoint,
907                test_tier_delete_dispatch_manifest_state, tier_delete_dispatch_manifest_operation_lock_held_for_test,
908                tier_delete_dispatch_manifest_recovery_count_for_test, tier_delete_dispatch_manifest_recovery_inflight_for_test,
909                tier_delete_journal_object_name,
910            },
911            tier_free_version_recovery::recover_tier_free_versions,
912            tier_sweeper::{
913                Jentry, TierDeleteJournalState, TierDeleteSourceIdentity, transitioned_delete_journal_entry_for_source,
914            },
915            transition_transaction::{
916                TRANSITION_TRANSACTION_RECORD_PREFIX, TransitionCleanupDecision, TransitionCleanupProof, TransitionOperatorError,
917                TransitionOperatorProbe, TransitionRecoveryClaimBarrier, TransitionRecoveryTerminalBarrier,
918                TransitionRemoteVersion, TransitionSourceIdentity, TransitionSourceVersionMode, TransitionTransaction,
919                TransitionTransactionInit, TransitionTransactionState, delete_transition_candidate_for_operator,
920                finalize_missing_transition_transaction_for_operator, inspect_transition_recovery_retry_for_operator,
921                inspect_transition_transaction_for_operator, load_transition_transaction_record,
922                recover_transition_transaction_records, recover_transition_transaction_records_at,
923                retry_transition_recovery_for_operator, save_transition_transaction_record,
924                save_transition_transaction_record_if_current, transition_recovery_control_id,
925                transition_transaction_record_object_name,
926            },
927            validate_durable_ilm_record,
928        },
929        bucket::metadata::{BUCKET_LIFECYCLE_CONFIG, BUCKET_VERSIONING_CONFIG},
930        config::com,
931        core::pools::DecomBucketInfo,
932        data_movement::SourceCleanupDeleteBarrier,
933        disk::{BUCKET_META_PREFIX, RUSTFS_META_BUCKET, STORAGE_FORMAT_FILE},
934        runtime::{global::set_object_store_resolver, sources as runtime_sources},
935        services::notification_sys::{
936            acquire_tier_delete_journal_fleet_proof, acquire_transition_transaction_compaction_fleet_proof,
937            install_transition_transaction_compaction_fleet_proof_for_test,
938            transition_transaction_compaction_fleet_proof_matches,
939        },
940        services::tier::{
941            test_util::{MockWarmBackend, MockWarmOp, TransitionCleanupStoreBarrier, register_mock_tier},
942            tier::{
943                ERR_TIER_BACKEND_IN_USE, ERR_TIER_INVALID_CONFIG, TIER_CONFIG_FILE, TierConfigMgr,
944                ensure_no_authoritative_tier_references_for_test, tier_config_candidate_digest,
945            },
946            tier_config::{TierConfig, TierType, TierWasabi},
947            tier_mutation_intent::{
948                TIER_MUTATION_INTENT_RECORD_PREFIX, TierMutationIntent, TierMutationIntentKind, TierMutationIntentState,
949                TierMutationIntentTarget, advance_tier_mutation_intent_record_idempotent, delete_tier_mutation_intent_record,
950                list_tier_mutation_intent_records, load_tier_mutation_intent_record, load_tier_mutation_intent_record_with_etag,
951                save_tier_mutation_intent_record, save_tier_mutation_intent_record_if_current,
952            },
953            tier_mutation_peer::{TierMutationPeerError, TierMutationPeerState, handle_tier_mutation_peer_request},
954            tier_probe_intent::{
955                TierProbeIntent, TierProbeIntentState, TierProbeOperationIdentity, TierProbeOwnerFence, TierProbeRemoteVersion,
956                delete_tier_probe_intent_record_if_current, load_tier_probe_intent_record,
957                save_tier_probe_intent_record_if_absent, save_tier_probe_intent_record_if_current,
958            },
959            warm_backend::{TransitionCandidateProbe, WarmBackend},
960        },
961        set_disk::{
962            SetDiskTransitionTransactionKillPoint as TransitionTransactionKillPoint,
963            SetDiskTransitionTransactionKillPointBarrier as TransitionTransactionKillPointBarrier,
964            SetDiskTransitionTransactionMutationKind as TransitionTransactionMutationKind,
965            SetDiskTransitionTransactionMutationObservation as TransitionTransactionMutationObservation,
966            SetDiskTransitionTransactionMutationProbe as TransitionTransactionMutationProbe,
967            SetDiskTransitionUploadedCommitBarrier as TransitionUploadedCommitBarrier,
968        },
969        storage_api_contracts::list::ListOperations as _,
970    };
971    #[cfg(feature = "test-util")]
972    use crate::{bucket::replication::ReplicationObjectBridge, storage_api_contracts::bucket::DeleteBucketOptions};
973    use crate::{
974        bucket::replication::{ReplicationState, ReplicationStatusType, replication_statuses_map},
975        core::pools::{
976            DecommissionErasureLayout, DecommissionPoolCapacityInfo, POOL_META_IDENTITY_NAME, POOL_META_NAME, POOL_META_VERSION,
977            PoolDecommissionInfo, PoolMeta, PoolStatus, pending_pool_meta_identity_for_test,
978            pool_meta_identity_initialized_for_test, pool_meta_v3_commit_state_for_test,
979            set_decommission_capacity_info_overrides_for_test,
980        },
981        disk::endpoint::Endpoint,
982        error::{Error, Result, StorageError},
983        io_support::rio::{WritePlan, compression_metadata_value},
984        layout::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints},
985        object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader},
986        services::rebalance::{RebalStatus, RebalanceInfo, RebalanceMeta, RebalanceStats},
987        storage_api_contracts::{
988            bucket::{BucketOperations as _, MakeBucketOptions},
989            multipart::MultipartOperations as _,
990            object::{ObjectIO, ObjectOperations as _, ObjectToDelete},
991            range::HTTPRangeSpec,
992        },
993    };
994    #[cfg(feature = "test-util")]
995    use futures::{StreamExt as _, TryStreamExt as _};
996    use http::HeaderMap;
997    use rustfs_config::server_config::KVS;
998    #[cfg(feature = "test-util")]
999    use rustfs_filemeta::{FileInfo, FileMeta};
1000    use rustfs_filemeta::{FileInfoVersions, MetaCacheEntry};
1001    #[cfg(feature = "test-util")]
1002    use rustfs_protos::{TIER_MUTATION_RPC_PROTOCOL_VERSION, TierMutationRpcPhase};
1003    use rustfs_rio::{Checksum, ChecksumType};
1004    #[cfg(feature = "test-util")]
1005    use rustfs_s3_client::transition_api::ReaderImpl;
1006    use rustfs_utils::{
1007        CompressionAlgorithm,
1008        http::{SUFFIX_COMPRESSION, insert_str},
1009    };
1010    use std::{
1011        collections::HashMap,
1012        future::Future,
1013        io::Cursor,
1014        sync::{
1015            Arc, Mutex,
1016            atomic::{AtomicBool, AtomicUsize, Ordering},
1017        },
1018        time::Duration,
1019    };
1020    use time::OffsetDateTime;
1021    use tokio::io::AsyncReadExt;
1022    use tokio_util::sync::CancellationToken;
1023
1024    fn run_large_stack_async_test<C, F>(name: &str, case: C)
1025    where
1026        C: FnOnce() -> F + Send + 'static,
1027        F: Future<Output = ()> + 'static,
1028    {
1029        const STACK_SIZE: usize = if cfg!(debug_assertions) {
1030            8 * rustfs_config::DEFAULT_THREAD_STACK_SIZE
1031        } else if cfg!(target_os = "macos") {
1032            2 * rustfs_config::DEFAULT_THREAD_STACK_SIZE
1033        } else {
1034            rustfs_config::DEFAULT_THREAD_STACK_SIZE
1035        };
1036        std::thread::Builder::new()
1037            .name(name.to_string())
1038            .stack_size(STACK_SIZE)
1039            .spawn(move || {
1040                let runtime = tokio::runtime::Builder::new_multi_thread()
1041                    .enable_all()
1042                    .worker_threads(2)
1043                    .thread_stack_size(STACK_SIZE)
1044                    .build()
1045                    .expect("large-stack store test runtime should build");
1046                runtime.block_on(case());
1047            })
1048            .expect("large-stack store test thread should spawn")
1049            .join()
1050            .expect("large-stack store test thread should complete");
1051    }
1052    use uuid::Uuid;
1053
1054    #[test]
1055    fn new_with_instance_ctx_future_remains_stack_bounded() {
1056        let future = crate::store::ECStore::new_with_instance_ctx(
1057            std::net::SocketAddr::from(([127, 0, 0, 1], 0)),
1058            EndpointServerPools(Vec::new()),
1059            CancellationToken::new(),
1060            Arc::new(crate::runtime::instance::InstanceContext::new()),
1061        );
1062        let future_size = std::mem::size_of_val(&future);
1063
1064        assert!(
1065            future_size <= 4 * 1024,
1066            "ECStore constructor future must remain stack-bounded; measured {future_size} bytes"
1067        );
1068    }
1069
1070    fn startup_pool_meta_payload(meta: &PoolMeta) -> Vec<u8> {
1071        meta.encode_config_data_for_test().expect("pool metadata should encode")
1072    }
1073
1074    #[derive(Debug)]
1075    struct StartupPoolMetaStorage {
1076        read_payload: Vec<u8>,
1077        read_error: bool,
1078        read_without_lock: AtomicBool,
1079        wrote_without_lock: AtomicBool,
1080        wrote_with_max_parity: AtomicBool,
1081        written_payload: Mutex<Option<Vec<u8>>>,
1082        revision: AtomicUsize,
1083        pool_meta_write_attempts: AtomicUsize,
1084        fail_pool_meta_write_at: AtomicUsize,
1085        pool_meta_written_versions: Mutex<Vec<u16>>,
1086        objects: Mutex<HashMap<String, (Vec<u8>, String)>>,
1087    }
1088
1089    impl StartupPoolMetaStorage {
1090        fn new(read_payload: Vec<u8>) -> Self {
1091            let mut objects = HashMap::new();
1092            if !read_payload.is_empty() {
1093                objects.insert(POOL_META_NAME.to_string(), (read_payload.clone(), "startup-pool-meta-0".to_string()));
1094            }
1095            Self {
1096                read_payload,
1097                read_error: false,
1098                read_without_lock: AtomicBool::new(false),
1099                wrote_without_lock: AtomicBool::new(false),
1100                wrote_with_max_parity: AtomicBool::new(false),
1101                written_payload: Mutex::new(None),
1102                revision: AtomicUsize::new(0),
1103                pool_meta_write_attempts: AtomicUsize::new(0),
1104                fail_pool_meta_write_at: AtomicUsize::new(0),
1105                pool_meta_written_versions: Mutex::new(Vec::new()),
1106                objects: Mutex::new(objects),
1107            }
1108        }
1109
1110        fn unreadable() -> Self {
1111            Self {
1112                read_payload: Vec::new(),
1113                read_error: true,
1114                read_without_lock: AtomicBool::new(false),
1115                wrote_without_lock: AtomicBool::new(false),
1116                wrote_with_max_parity: AtomicBool::new(false),
1117                written_payload: Mutex::new(None),
1118                revision: AtomicUsize::new(0),
1119                pool_meta_write_attempts: AtomicUsize::new(0),
1120                fail_pool_meta_write_at: AtomicUsize::new(0),
1121                pool_meta_written_versions: Mutex::new(Vec::new()),
1122                objects: Mutex::new(HashMap::new()),
1123            }
1124        }
1125
1126        fn object_info(&self, bucket: &str, object: &str, size: usize, etag: String) -> ObjectInfo {
1127            ObjectInfo {
1128                bucket: bucket.to_string(),
1129                name: object.to_string(),
1130                size: size as i64,
1131                actual_size: size as i64,
1132                etag: Some(etag),
1133                ..Default::default()
1134            }
1135        }
1136    }
1137
1138    #[async_trait::async_trait]
1139    impl ObjectIO for StartupPoolMetaStorage {
1140        type Error = Error;
1141        type RangeSpec = HTTPRangeSpec;
1142        type HeaderMap = HeaderMap;
1143        type ObjectOptions = ObjectOptions;
1144        type ObjectInfo = ObjectInfo;
1145        type GetObjectReader = GetObjectReader;
1146        type PutObjectReader = PutObjReader;
1147
1148        async fn get_object_reader(
1149            &self,
1150            bucket: &str,
1151            object: &str,
1152            _range: Option<HTTPRangeSpec>,
1153            _h: HeaderMap,
1154            opts: &ObjectOptions,
1155        ) -> Result<GetObjectReader> {
1156            assert!(opts.no_lock, "store init pool metadata load must not require namespace locks");
1157            self.read_without_lock.store(true, Ordering::SeqCst);
1158            if self.read_error {
1159                return Err(Error::other("pool metadata read quorum unavailable"));
1160            }
1161            let Some((payload, etag)) = self
1162                .objects
1163                .lock()
1164                .unwrap_or_else(std::sync::PoisonError::into_inner)
1165                .get(object)
1166                .cloned()
1167            else {
1168                return Err(Error::FileNotFound);
1169            };
1170
1171            Ok(GetObjectReader {
1172                stream: Box::new(Cursor::new(payload.clone())),
1173                object_info: self.object_info(bucket, object, payload.len(), etag),
1174                buffered_body: None,
1175                body_source: Default::default(),
1176            })
1177        }
1178
1179        async fn put_object(
1180            &self,
1181            bucket: &str,
1182            object: &str,
1183            data: &mut PutObjReader,
1184            opts: &ObjectOptions,
1185        ) -> Result<ObjectInfo> {
1186            assert!(opts.no_lock, "store init pool metadata save must not require namespace locks");
1187            self.wrote_without_lock.store(true, Ordering::SeqCst);
1188            self.wrote_with_max_parity.store(opts.max_parity, Ordering::SeqCst);
1189            if object == POOL_META_NAME {
1190                let attempt = self.pool_meta_write_attempts.fetch_add(1, Ordering::SeqCst) + 1;
1191                if self.fail_pool_meta_write_at.load(Ordering::SeqCst) == attempt {
1192                    return Err(Error::Timeout);
1193                }
1194            }
1195            let current_etag = self
1196                .objects
1197                .lock()
1198                .unwrap_or_else(std::sync::PoisonError::into_inner)
1199                .get(object)
1200                .map(|(_, etag)| etag.clone());
1201            if opts
1202                .http_preconditions
1203                .as_ref()
1204                .and_then(|preconditions| preconditions.if_none_match_value())
1205                == Some("*")
1206                && current_etag.is_some()
1207            {
1208                return Err(Error::PreconditionFailed);
1209            }
1210            if let Some(expected) = opts
1211                .http_preconditions
1212                .as_ref()
1213                .and_then(|preconditions| preconditions.if_match_value())
1214                && current_etag.as_deref() != Some(expected)
1215            {
1216                return Err(Error::PreconditionFailed);
1217            }
1218            let mut payload = Vec::new();
1219            data.stream.read_to_end(&mut payload).await?;
1220            let size = payload.len();
1221            if object == POOL_META_NAME {
1222                *self.written_payload.lock().unwrap_or_else(std::sync::PoisonError::into_inner) = Some(payload.clone());
1223                if payload.len() >= 4 {
1224                    self.pool_meta_written_versions
1225                        .lock()
1226                        .unwrap_or_else(std::sync::PoisonError::into_inner)
1227                        .push(u16::from_le_bytes([payload[2], payload[3]]));
1228                }
1229            }
1230            let etag = format!("startup-pool-meta-{}", self.revision.fetch_add(1, Ordering::SeqCst) + 1);
1231            self.objects
1232                .lock()
1233                .unwrap_or_else(std::sync::PoisonError::into_inner)
1234                .insert(object.to_string(), (payload, etag.clone()));
1235            Ok(self.object_info(bucket, object, size, etag))
1236        }
1237    }
1238
1239    fn init_test_pool_meta(decommission: Option<PoolDecommissionInfo>) -> PoolMeta {
1240        PoolMeta {
1241            version: POOL_META_VERSION,
1242            pools: vec![PoolStatus {
1243                id: 0,
1244                cmd_line: "pool-0".to_string(),
1245                last_update: OffsetDateTime::UNIX_EPOCH,
1246                decommission,
1247            }],
1248            dont_save: false,
1249        }
1250    }
1251
1252    #[tokio::test]
1253    async fn test_fresh_bootstrap_pool_meta_save_reaches_v3_cas_without_blocking_itself() {
1254        let storage = Arc::new(StartupPoolMetaStorage::new(Vec::new()));
1255        let mut write_state = PoolMetaWriteState::for_startup(Uuid::new_v4(), true);
1256        establish_pool_meta_bootstrap_identity_if_proven(vec![storage.clone()], &mut write_state, true)
1257            .await
1258            .expect("the elected fresh bootstrap should persist its identity before loading pool metadata");
1259
1260        let (loaded, replica_state) = load_pool_meta_for_startup(vec![storage.clone()], &mut write_state)
1261            .await
1262            .expect("startup pool metadata load should tolerate missing metadata without locks");
1263        assert!(loaded.pools.is_empty());
1264        assert!(!replica_state.needs_repair);
1265        assert!(replica_state.repair_write_safe);
1266        assert!(storage.read_without_lock.load(Ordering::SeqCst));
1267
1268        let meta = PoolMeta {
1269            version: POOL_META_VERSION,
1270            pools: Vec::new(),
1271            dont_save: false,
1272        };
1273        save_validated_pool_meta_for_startup(&meta, vec![storage.clone()], &mut write_state)
1274            .await
1275            .expect("startup pool metadata save should bypass locks");
1276        assert!(storage.wrote_without_lock.load(Ordering::SeqCst));
1277        assert!(storage.wrote_with_max_parity.load(Ordering::SeqCst));
1278        assert_eq!(
1279            storage.pool_meta_write_attempts.load(Ordering::SeqCst),
1280            2,
1281            "fresh bootstrap should reach both V3 prepare and commit CAS writes"
1282        );
1283        assert_eq!(
1284            *storage
1285                .pool_meta_written_versions
1286                .lock()
1287                .unwrap_or_else(std::sync::PoisonError::into_inner),
1288            vec![3, 3]
1289        );
1290        write_state
1291            .ensure_write_safe("fresh bootstrap publication")
1292            .expect("successful startup publication must disarm the transaction guard");
1293    }
1294
1295    #[tokio::test]
1296    async fn test_legacy_adoption_pool_meta_bootstrap_reaches_v3_cas() {
1297        let deployment_id = Uuid::new_v4();
1298        let storage = Arc::new(StartupPoolMetaStorage::new(Vec::new()));
1299        let mut write_state =
1300            PoolMetaWriteState::for_startup_with_bootstrap_authority(deployment_id, PoolMetaBootstrapAuthority::LegacyAdoption);
1301
1302        establish_pool_meta_bootstrap_identity_if_proven(vec![storage.clone()], &mut write_state, true)
1303            .await
1304            .expect("verified legacy adoption should persist a nonce-bound identity before loading pool metadata");
1305        let (loaded, replica_state) = load_pool_meta_for_startup(vec![storage.clone()], &mut write_state)
1306            .await
1307            .expect("verified legacy adoption should authorize initially missing pool metadata");
1308        assert!(loaded.pools.is_empty());
1309
1310        let committed = persist_pool_meta_for_startup_if_safe(
1311            &init_test_pool_meta(None),
1312            vec![storage.clone()],
1313            replica_state,
1314            &mut write_state,
1315            true,
1316            true,
1317        )
1318        .await
1319        .expect("verified legacy adoption should publish initial pool metadata");
1320        assert_eq!(committed.pools[0].cmd_line, "pool-0");
1321
1322        let objects = storage.objects.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
1323        assert!(objects.contains_key(POOL_META_NAME));
1324        let identity = objects
1325            .get(POOL_META_IDENTITY_NAME)
1326            .map(|(payload, _)| payload.clone())
1327            .expect("legacy adoption should commit the bootstrap identity");
1328        assert!(pool_meta_identity_initialized_for_test(&identity).expect("decode committed identity"));
1329    }
1330
1331    #[tokio::test]
1332    async fn test_non_elected_legacy_adoption_cannot_initialize_pool_meta() {
1333        let storage = Arc::new(StartupPoolMetaStorage::new(Vec::new()));
1334        let mut write_state =
1335            PoolMetaWriteState::for_startup_with_bootstrap_authority(Uuid::new_v4(), PoolMetaBootstrapAuthority::LegacyAdoption);
1336
1337        establish_pool_meta_bootstrap_identity_if_proven(vec![storage.clone()], &mut write_state, false)
1338            .await
1339            .expect("a non-elected distributed node must not create legacy adoption authority");
1340        let err = load_pool_meta_for_startup(vec![storage.clone()], &mut write_state)
1341            .await
1342            .expect_err("legacy adoption still requires the elected writer to create a durable identity");
1343        assert!(err.to_string().contains("no durable bootstrap identity"));
1344        assert!(
1345            !storage
1346                .objects
1347                .lock()
1348                .unwrap_or_else(std::sync::PoisonError::into_inner)
1349                .contains_key(POOL_META_NAME),
1350            "classification must not create pool metadata on non-elected nodes"
1351        );
1352    }
1353
1354    #[tokio::test]
1355    async fn test_unproven_pending_identity_cannot_authorize_all_missing_pool_meta() {
1356        let deployment_id = Uuid::new_v4();
1357        let storage = Arc::new(StartupPoolMetaStorage::new(Vec::new()));
1358        let mut proven_bootstrap = PoolMetaWriteState::for_startup(deployment_id, true);
1359        establish_pool_meta_bootstrap_identity_if_proven(vec![storage.clone()], &mut proven_bootstrap, true)
1360            .await
1361            .expect("fresh topology proof should persist a nonce-bound pending identity");
1362
1363        let identity = storage
1364            .objects
1365            .lock()
1366            .unwrap_or_else(std::sync::PoisonError::into_inner)
1367            .get(POOL_META_IDENTITY_NAME)
1368            .map(|(payload, _)| payload.clone())
1369            .expect("pending identity should be durable");
1370        assert!(!pool_meta_identity_initialized_for_test(&identity).expect("decode pending identity"));
1371
1372        let mut unproven_restart = PoolMetaWriteState::for_startup(deployment_id, false);
1373        let err = load_pool_meta_for_startup(vec![storage.clone()], &mut unproven_restart)
1374            .await
1375            .expect_err("a pending identity alone must not authorize an all-missing restart");
1376        assert!(err.to_string().contains("no verified fresh-bootstrap proof"));
1377        unproven_restart
1378            .ensure_write_safe("unproven pending identity restart")
1379            .expect_err("the rejected restart must latch the write gate");
1380        assert!(
1381            !storage
1382                .objects
1383                .lock()
1384                .unwrap_or_else(std::sync::PoisonError::into_inner)
1385                .contains_key(POOL_META_NAME),
1386            "classification must not create pool metadata"
1387        );
1388    }
1389
1390    #[tokio::test]
1391    async fn test_nonfresh_identity_repair_crash_never_persists_pending_bootstrap_authority() {
1392        let deployment_id = Uuid::new_v4();
1393        let initial = init_test_pool_meta(None);
1394        let storage = Arc::new(StartupPoolMetaStorage::new(startup_pool_meta_payload(&initial)));
1395        let mut write_state = PoolMetaWriteState::for_startup(deployment_id, false);
1396        let (loaded, replica_state) = load_pool_meta_for_startup(vec![storage.clone()], &mut write_state)
1397            .await
1398            .expect("an existing pool metadata snapshot may repair its missing identity");
1399
1400        storage.fail_pool_meta_write_at.store(1, Ordering::SeqCst);
1401        persist_pool_meta_for_startup_if_safe(&loaded, vec![storage.clone()], replica_state, &mut write_state, true, true)
1402            .await
1403            .expect_err("the injected crash boundary should stop before the pool metadata replacement");
1404
1405        let identity = storage
1406            .objects
1407            .lock()
1408            .unwrap_or_else(std::sync::PoisonError::into_inner)
1409            .get(POOL_META_IDENTITY_NAME)
1410            .map(|(payload, _)| payload.clone())
1411            .expect("identity repair should be durable before the failed topology write");
1412        assert!(
1413            pool_meta_identity_initialized_for_test(&identity).expect("decode repaired identity"),
1414            "an existing cluster must never leave pending bootstrap authority at this crash boundary"
1415        );
1416
1417        storage
1418            .objects
1419            .lock()
1420            .unwrap_or_else(std::sync::PoisonError::into_inner)
1421            .remove(POOL_META_NAME);
1422        let mut restarted = PoolMetaWriteState::for_startup(deployment_id, false);
1423        let err = load_pool_meta_for_startup(vec![storage], &mut restarted)
1424            .await
1425            .expect_err("wiping pool metadata after the crash must require recovery");
1426        assert!(err.to_string().contains("initialized cluster identity exists"));
1427    }
1428
1429    #[tokio::test]
1430    #[serial_test::serial(pool_meta_version_env)]
1431    async fn test_existing_legacy_identity_and_replica_repair_respects_disabled_v3_gates() {
1432        temp_env::async_with_vars(
1433            [
1434                (rustfs_config::ENV_POOL_META_V3_WRITE, None::<&str>),
1435                (rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, None::<&str>),
1436            ],
1437            async {
1438                let deployment_id = Uuid::new_v4();
1439                let corrupt = Arc::new(StartupPoolMetaStorage::new(vec![0, 1, 2]));
1440                let legacy = init_test_pool_meta(None);
1441                let valid = Arc::new(StartupPoolMetaStorage::new(startup_pool_meta_payload(&legacy)));
1442                let mut write_state = PoolMetaWriteState::for_startup(deployment_id, false);
1443                let (loaded, replica_state) = load_pool_meta_for_startup(vec![corrupt.clone(), valid.clone()], &mut write_state)
1444                    .await
1445                    .expect("existing V2 metadata should remain readable while its identity is missing");
1446                assert!(replica_state.needs_repair);
1447
1448                let committed = persist_pool_meta_for_startup_if_safe(
1449                    &loaded,
1450                    vec![corrupt.clone(), valid.clone()],
1451                    replica_state,
1452                    &mut write_state,
1453                    true,
1454                    true,
1455                )
1456                .await
1457                .expect("legacy replica and identity repair should succeed without crossing the V3 gate");
1458                assert_eq!(committed.version, POOL_META_VERSION);
1459                for storage in [corrupt, valid] {
1460                    assert_eq!(
1461                        *storage
1462                            .pool_meta_written_versions
1463                            .lock()
1464                            .unwrap_or_else(std::sync::PoisonError::into_inner),
1465                        vec![POOL_META_VERSION],
1466                        "legacy repair must write exactly one V2 snapshot"
1467                    );
1468                    let identity = storage
1469                        .objects
1470                        .lock()
1471                        .unwrap_or_else(std::sync::PoisonError::into_inner)
1472                        .get(POOL_META_IDENTITY_NAME)
1473                        .map(|(payload, _)| payload.clone())
1474                        .expect("identity repair should persist on every pool");
1475                    assert!(pool_meta_identity_initialized_for_test(&identity).expect("decode repaired identity"));
1476                }
1477            },
1478        )
1479        .await;
1480    }
1481
1482    fn startup_object(storage: &StartupPoolMetaStorage, object: &str) -> Option<Vec<u8>> {
1483        storage
1484            .objects
1485            .lock()
1486            .unwrap_or_else(std::sync::PoisonError::into_inner)
1487            .get(object)
1488            .map(|(payload, _)| payload.clone())
1489    }
1490
1491    /// Startup errors wrap their cause in context whose `Display` hides the
1492    /// source, so assertions walk the chain the same way
1493    /// `Error::pool_metadata_failure` does.
1494    fn error_chain_text(err: &Error) -> String {
1495        let mut parts = vec![err.to_string()];
1496        let mut current: Option<&(dyn std::error::Error + 'static)> = Some(err);
1497        while let Some(error) = current {
1498            current = if let Some(io) = error.downcast_ref::<std::io::Error>() {
1499                io.get_ref().map(|inner| inner as &(dyn std::error::Error + 'static))
1500            } else {
1501                error.source()
1502            };
1503            if let Some(next) = current {
1504                parts.push(next.to_string());
1505            }
1506        }
1507        parts.join(" <- ")
1508    }
1509
1510    fn inject_startup_object(storage: &StartupPoolMetaStorage, object: &str, payload: Vec<u8>) {
1511        storage
1512            .objects
1513            .lock()
1514            .unwrap_or_else(std::sync::PoisonError::into_inner)
1515            .insert(object.to_string(), (payload, format!("injected-{object}")));
1516    }
1517
1518    fn init_test_pool_meta_with_pools(pool_count: usize) -> PoolMeta {
1519        PoolMeta {
1520            version: POOL_META_VERSION,
1521            pools: (0..pool_count)
1522                .map(|id| PoolStatus {
1523                    id,
1524                    cmd_line: format!("pool-{id}"),
1525                    last_update: OffsetDateTime::UNIX_EPOCH,
1526                    decommission: None,
1527                })
1528                .collect(),
1529            dont_save: false,
1530        }
1531    }
1532
1533    /// Two single-node pools whose formats were created by different nodes:
1534    /// node0 formatted pool0 and only read pool1's format, node1 the reverse.
1535    fn two_pool_creator_states(deployment_id: Uuid) -> (PoolMetaWriteState, PoolMetaWriteState) {
1536        let node0 = PoolMetaWriteState::for_startup_with_pool_bootstrap_authorities(
1537            deployment_id,
1538            vec![PoolMetaBootstrapAuthority::Fresh, PoolMetaBootstrapAuthority::None],
1539            true,
1540        );
1541        let node1 = PoolMetaWriteState::for_startup_with_pool_bootstrap_authorities(
1542            deployment_id,
1543            vec![PoolMetaBootstrapAuthority::None, PoolMetaBootstrapAuthority::Fresh],
1544            false,
1545        );
1546        (node0, node1)
1547    }
1548
1549    #[tokio::test]
1550    async fn test_two_pool_bootstrap_with_distinct_format_creators_converges_through_creator_attestation() {
1551        let deployment_id = Uuid::new_v4();
1552        let pool0 = Arc::new(StartupPoolMetaStorage::new(Vec::new()));
1553        let pool1 = Arc::new(StartupPoolMetaStorage::new(Vec::new()));
1554        let pools = vec![pool0.clone(), pool1.clone()];
1555        let (mut node0, mut node1) = two_pool_creator_states(deployment_id);
1556        assert!(!node0.bootstrap_identity_proven(), "reading pool1's format is not deployment-wide proof");
1557        assert!(!node1.bootstrap_identity_proven());
1558
1559        // node1 (pool1 creator, non-elected) starts first: no durable nonce exists
1560        // yet, so it must neither mint one nor latch its write gate while waiting.
1561        establish_pool_meta_bootstrap_identity_if_proven(pools.clone(), &mut node1, false)
1562            .await
1563            .expect("a non-first creator without a durable nonce writes nothing");
1564        assert!(startup_object(&pool0, POOL_META_IDENTITY_NAME).is_none());
1565        assert!(startup_object(&pool1, POOL_META_IDENTITY_NAME).is_none());
1566        let err = load_pool_meta_for_startup(pools.clone(), &mut node1)
1567            .await
1568            .expect_err("nothing durable authorizes a non-elected node");
1569        assert!(err.to_string().contains("bootstrap pending"), "{err}");
1570        node1
1571            .ensure_write_safe("waiting non-elected creator")
1572            .expect("waiting for the elected writer must not latch the write gate");
1573
1574        // node0 (pool0 creator, elected) mints the nonce on the pool it created;
1575        // pool1 is still unattested, so it cannot publish pool.bin and must not latch.
1576        establish_pool_meta_bootstrap_identity_if_proven(pools.clone(), &mut node0, true)
1577            .await
1578            .expect("the first pool's creator mints the pending identity");
1579        let minted = startup_object(&pool0, POOL_META_IDENTITY_NAME).expect("pool0 pending identity");
1580        assert!(!pool_meta_identity_initialized_for_test(&minted).expect("decode pending identity"));
1581        assert!(
1582            startup_object(&pool1, POOL_META_IDENTITY_NAME).is_none(),
1583            "node0 holds no first-hand proof for pool1 and must not attest it"
1584        );
1585        let err = load_pool_meta_for_startup(pools.clone(), &mut node0)
1586            .await
1587            .expect_err("an unattested pool keeps the elected writer from publishing");
1588        assert!(err.to_string().contains("waiting for every pool creator"), "{err}");
1589        node0
1590            .ensure_write_safe("waiting elected writer")
1591            .expect("waiting for creators must not latch the write gate");
1592        assert!(startup_object(&pool0, POOL_META_NAME).is_none());
1593
1594        // node1 retries: it copies pool0's pending identity (same nonce) onto the
1595        // pool it created, then keeps waiting for the elected writer's pool.bin.
1596        establish_pool_meta_bootstrap_identity_if_proven(pools.clone(), &mut node1, false)
1597            .await
1598            .expect("the pool1 creator attests with the durable nonce");
1599        assert_eq!(startup_object(&pool1, POOL_META_IDENTITY_NAME).as_deref(), Some(minted.as_slice()));
1600        let err = load_pool_meta_for_startup(pools.clone(), &mut node1)
1601            .await
1602            .expect_err("a complete pending set never unlocks a non-elected node");
1603        assert!(err.to_string().contains("waiting for the elected writer to publish"), "{err}");
1604        node1
1605            .ensure_write_safe("attested non-elected creator")
1606            .expect("waiting for pool.bin must not latch the write gate");
1607        assert!(startup_object(&pool0, POOL_META_NAME).is_none());
1608
1609        // node0 retries: every pool is attested under one nonce, so it publishes
1610        // pool.bin and commits the identity on both pools.
1611        establish_pool_meta_bootstrap_identity_if_proven(pools.clone(), &mut node0, true)
1612            .await
1613            .expect("re-establishing an already minted identity is idempotent");
1614        let (_, replica_state) = load_pool_meta_for_startup(pools.clone(), &mut node0)
1615            .await
1616            .expect("complete creator attestation authorizes the initial pool metadata write");
1617        persist_pool_meta_for_startup_if_safe(
1618            &init_test_pool_meta_with_pools(2),
1619            pools.clone(),
1620            replica_state,
1621            &mut node0,
1622            true,
1623            true,
1624        )
1625        .await
1626        .expect("the elected writer publishes pool.bin and commits the identity");
1627        for pool in [&pool0, &pool1] {
1628            assert!(startup_object(pool, POOL_META_NAME).is_some());
1629            let identity = startup_object(pool, POOL_META_IDENTITY_NAME).expect("committed identity");
1630            assert!(pool_meta_identity_initialized_for_test(&identity).expect("decode committed identity"));
1631        }
1632
1633        // node1 retries once more: pool.bin exists and nothing is rewritten.
1634        let before = startup_object(&pool1, POOL_META_IDENTITY_NAME);
1635        establish_pool_meta_bootstrap_identity_if_proven(pools.clone(), &mut node1, false)
1636            .await
1637            .expect("an initialized deployment never reopens bootstrap");
1638        assert_eq!(startup_object(&pool1, POOL_META_IDENTITY_NAME), before);
1639        load_pool_meta_for_startup(pools, &mut node1)
1640            .await
1641            .expect("published pool metadata admits the non-elected node");
1642        node1
1643            .ensure_write_safe("converged non-elected creator")
1644            .expect("no latch remains after convergence");
1645    }
1646
1647    #[tokio::test]
1648    async fn test_two_pool_bootstrap_rejects_pending_replicas_from_different_bootstraps() {
1649        let deployment_id = Uuid::new_v4();
1650        let pool0 = Arc::new(StartupPoolMetaStorage::new(Vec::new()));
1651        let pool1 = Arc::new(StartupPoolMetaStorage::new(Vec::new()));
1652        let pools = vec![pool0.clone(), pool1.clone()];
1653        let (mut node0, _) = two_pool_creator_states(deployment_id);
1654        establish_pool_meta_bootstrap_identity_if_proven(pools.clone(), &mut node0, true)
1655            .await
1656            .expect("the first pool's creator mints the pending identity");
1657        inject_startup_object(
1658            &pool1,
1659            POOL_META_IDENTITY_NAME,
1660            pending_pool_meta_identity_for_test(deployment_id, 1, Uuid::new_v4()).expect("encode foreign pending identity"),
1661        );
1662
1663        let err = load_pool_meta_for_startup(pools.clone(), &mut node0)
1664            .await
1665            .expect_err("a pending replica bound to another bootstrap nonce must fail closed");
1666        let chain = error_chain_text(&err);
1667        assert!(chain.contains("disagree on fresh-bootstrap proof"), "{chain}");
1668        node0
1669            .ensure_write_safe("split bootstrap")
1670            .expect_err("a split bootstrap latches the write gate");
1671        assert!(startup_object(&pool0, POOL_META_NAME).is_none());
1672    }
1673
1674    #[tokio::test]
1675    async fn test_two_pool_bootstrap_treats_corrupt_creator_replica_as_recovery_not_waiting() {
1676        let deployment_id = Uuid::new_v4();
1677        let pool0 = Arc::new(StartupPoolMetaStorage::new(Vec::new()));
1678        let pool1 = Arc::new(StartupPoolMetaStorage::new(Vec::new()));
1679        let pools = vec![pool0.clone(), pool1.clone()];
1680        let (mut node0, _) = two_pool_creator_states(deployment_id);
1681        establish_pool_meta_bootstrap_identity_if_proven(pools.clone(), &mut node0, true)
1682            .await
1683            .expect("the first pool's creator mints the pending identity");
1684        // Keep the on-disk format/version header so the replica classifies as
1685        // corrupt (undecodable payload) rather than as an incompatible format.
1686        let mut corrupt = pending_pool_meta_identity_for_test(deployment_id, 1, Uuid::new_v4()).expect("encode identity");
1687        corrupt.truncate(4);
1688        corrupt.extend_from_slice(b"not a cluster identity");
1689        inject_startup_object(&pool1, POOL_META_IDENTITY_NAME, corrupt);
1690
1691        let err = load_pool_meta_for_startup(pools.clone(), &mut node0)
1692            .await
1693            .expect_err("a corrupt replica is not a creator that is still catching up");
1694        let chain = error_chain_text(&err);
1695        assert!(chain.contains("no verified fresh-bootstrap proof"), "{chain}");
1696        node0
1697            .ensure_write_safe("corrupt attestation")
1698            .expect_err("a corrupt attestation latches the write gate");
1699        assert!(startup_object(&pool0, POOL_META_NAME).is_none());
1700    }
1701
1702    #[tokio::test]
1703    async fn test_elected_restart_without_first_hand_proof_cannot_reuse_a_complete_pending_set() {
1704        let deployment_id = Uuid::new_v4();
1705        let pool0 = Arc::new(StartupPoolMetaStorage::new(Vec::new()));
1706        let pool1 = Arc::new(StartupPoolMetaStorage::new(Vec::new()));
1707        let pools = vec![pool0.clone(), pool1.clone()];
1708        let (mut node0, mut node1) = two_pool_creator_states(deployment_id);
1709        establish_pool_meta_bootstrap_identity_if_proven(pools.clone(), &mut node0, true)
1710            .await
1711            .expect("mint");
1712        establish_pool_meta_bootstrap_identity_if_proven(pools.clone(), &mut node1, false)
1713            .await
1714            .expect("attest");
1715        assert_eq!(
1716            startup_object(&pool0, POOL_META_IDENTITY_NAME),
1717            startup_object(&pool1, POOL_META_IDENTITY_NAME),
1718            "both creators attested the same pending identity"
1719        );
1720
1721        // The elected node restarts before publishing: it now merely reads both
1722        // formats, so the complete pending set alone must not reopen bootstrap.
1723        let mut restarted = PoolMetaWriteState::for_startup_with_pool_bootstrap_authorities(
1724            deployment_id,
1725            vec![PoolMetaBootstrapAuthority::None, PoolMetaBootstrapAuthority::None],
1726            true,
1727        );
1728        establish_pool_meta_bootstrap_identity_if_proven(pools.clone(), &mut restarted, true)
1729            .await
1730            .expect("a restart without first-hand proof writes nothing");
1731        let err = load_pool_meta_for_startup(pools.clone(), &mut restarted)
1732            .await
1733            .expect_err("a pending set alone never authorizes a writer without first-hand proof");
1734        assert!(err.to_string().contains("no verified fresh-bootstrap proof"), "{err}");
1735        restarted
1736            .ensure_write_safe("unproven restart")
1737            .expect_err("the rejected restart latches the write gate");
1738        assert!(startup_object(&pool0, POOL_META_NAME).is_none());
1739    }
1740
1741    #[tokio::test]
1742    async fn test_fresh_pool_joining_an_initialized_deployment_never_reopens_bootstrap() {
1743        let deployment_id = Uuid::new_v4();
1744        let pool0 = Arc::new(StartupPoolMetaStorage::new(Vec::new()));
1745        let mut founder = PoolMetaWriteState::for_startup(deployment_id, true);
1746        establish_pool_meta_bootstrap_identity_if_proven(vec![pool0.clone()], &mut founder, true)
1747            .await
1748            .expect("the founder mints");
1749        let (_, replica_state) = load_pool_meta_for_startup(vec![pool0.clone()], &mut founder)
1750            .await
1751            .expect("the founder may initialize");
1752        persist_pool_meta_for_startup_if_safe(
1753            &init_test_pool_meta(None),
1754            vec![pool0.clone()],
1755            replica_state,
1756            &mut founder,
1757            true,
1758            true,
1759        )
1760        .await
1761        .expect("the founder commits");
1762        let founded = startup_object(&pool0, POOL_META_IDENTITY_NAME).expect("committed identity");
1763        assert!(pool_meta_identity_initialized_for_test(&founded).expect("decode committed identity"));
1764
1765        // Expansion: pool1 is fresh and was formatted first-hand by the node
1766        // hosting its first endpoint, whether or not that node is elected.
1767        let pool1 = Arc::new(StartupPoolMetaStorage::new(Vec::new()));
1768        let pools = vec![pool0.clone(), pool1.clone()];
1769        for elected in [false, true] {
1770            let mut joiner = PoolMetaWriteState::for_startup_with_pool_bootstrap_authorities(
1771                deployment_id,
1772                vec![PoolMetaBootstrapAuthority::None, PoolMetaBootstrapAuthority::Fresh],
1773                elected,
1774            );
1775            establish_pool_meta_bootstrap_identity_if_proven(pools.clone(), &mut joiner, elected)
1776                .await
1777                .expect("an initialized deployment ignores first-hand proof for a new pool");
1778            assert!(
1779                startup_object(&pool1, POOL_META_IDENTITY_NAME).is_none(),
1780                "no pending identity may be written to an expansion pool"
1781            );
1782            assert_eq!(startup_object(&pool0, POOL_META_IDENTITY_NAME).as_deref(), Some(founded.as_slice()));
1783            let (_, replica_state) = load_pool_meta_for_startup(pools.clone(), &mut joiner)
1784                .await
1785                .expect("published pool metadata admits the joiner");
1786            joiner
1787                .ensure_write_safe("expansion joiner")
1788                .expect("joining never latches the write gate");
1789            if elected {
1790                persist_pool_meta_for_startup_if_safe(
1791                    &init_test_pool_meta_with_pools(2),
1792                    pools.clone(),
1793                    replica_state,
1794                    &mut joiner,
1795                    true,
1796                    true,
1797                )
1798                .await
1799                .expect("the topology update repairs the new pool's replicas");
1800                let identity = startup_object(&pool1, POOL_META_IDENTITY_NAME).expect("expansion pool identity");
1801                assert!(pool_meta_identity_initialized_for_test(&identity).expect("decode repaired identity"));
1802                assert!(startup_object(&pool1, POOL_META_NAME).is_some());
1803            }
1804        }
1805    }
1806
1807    #[tokio::test]
1808    async fn test_store_init_distinguishes_fresh_deployment_from_wiped_lagging_node() {
1809        let deployment_id = Uuid::new_v4();
1810        let storage = Arc::new(StartupPoolMetaStorage::new(Vec::new()));
1811        let mut fresh_state = PoolMetaWriteState::for_startup(deployment_id, true);
1812        establish_pool_meta_bootstrap_identity_if_proven(vec![storage.clone()], &mut fresh_state, true)
1813            .await
1814            .expect("fresh topology proof should become a durable pending identity");
1815        let (_, replica_state) = load_pool_meta_for_startup(vec![storage.clone()], &mut fresh_state)
1816            .await
1817            .expect("new formats with no identity may initialize pool metadata exactly once");
1818        let committed = persist_pool_meta_for_startup_if_safe(
1819            &init_test_pool_meta(None),
1820            vec![storage.clone()],
1821            replica_state,
1822            &mut fresh_state,
1823            true,
1824            true,
1825        )
1826        .await
1827        .expect("fresh deployment should durably commit identity and pool metadata");
1828        assert_eq!(committed.pools[0].cmd_line, "pool-0");
1829        {
1830            let objects = storage.objects.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
1831            assert!(objects.contains_key(POOL_META_NAME));
1832            assert!(objects.contains_key(POOL_META_IDENTITY_NAME));
1833        }
1834
1835        storage
1836            .objects
1837            .lock()
1838            .unwrap_or_else(std::sync::PoisonError::into_inner)
1839            .remove(POOL_META_NAME);
1840        let mut wiped_pool_state = PoolMetaWriteState::for_startup(deployment_id, false);
1841        let err = load_pool_meta_for_startup(vec![storage.clone()], &mut wiped_pool_state)
1842            .await
1843            .expect_err("an initialized identity must prevent an empty pool metadata rebuild");
1844        assert!(err.to_string().contains("initialized cluster identity exists"));
1845
1846        storage
1847            .objects
1848            .lock()
1849            .unwrap_or_else(std::sync::PoisonError::into_inner)
1850            .remove(POOL_META_IDENTITY_NAME);
1851        let mut wiped_all_state = PoolMetaWriteState::for_startup(deployment_id, false);
1852        let err = load_pool_meta_for_startup(vec![storage], &mut wiped_all_state)
1853            .await
1854            .expect_err("existing formats without identity or pool metadata must require recovery");
1855        assert!(err.to_string().contains("no durable bootstrap identity"));
1856
1857        let distributed_node = Arc::new(StartupPoolMetaStorage::new(Vec::new()));
1858        let mut non_elected_state = PoolMetaWriteState::for_startup(Uuid::new_v4(), true);
1859        establish_pool_meta_bootstrap_identity_if_proven(vec![distributed_node.clone()], &mut non_elected_state, false)
1860            .await
1861            .expect("a non-elected distributed node must not create bootstrap authority");
1862        let err = load_pool_meta_for_startup(vec![distributed_node], &mut non_elected_state)
1863            .await
1864            .expect_err("a distributed node without durable identity or pool metadata must require recovery");
1865        assert!(err.to_string().contains("no durable bootstrap identity"));
1866    }
1867
1868    #[tokio::test]
1869    async fn test_store_init_resumes_pending_v3_bootstrap_without_legacy_overwrite() {
1870        let deployment_id = Uuid::new_v4();
1871        let storage = Arc::new(StartupPoolMetaStorage::new(Vec::new()));
1872        let mut first_bootstrap = PoolMetaWriteState::for_startup(deployment_id, true);
1873        establish_pool_meta_bootstrap_identity_if_proven(vec![storage.clone()], &mut first_bootstrap, true)
1874            .await
1875            .expect("fresh bootstrap should persist a pending identity");
1876        let (_, replica_state) = load_pool_meta_for_startup(vec![storage.clone()], &mut first_bootstrap)
1877            .await
1878            .expect("the durable pending identity should authorize the initial pool metadata write");
1879
1880        storage.fail_pool_meta_write_at.store(2, Ordering::SeqCst);
1881        persist_pool_meta_for_startup_if_safe(
1882            &init_test_pool_meta(None),
1883            vec![storage.clone()],
1884            replica_state,
1885            &mut first_bootstrap,
1886            true,
1887            true,
1888        )
1889        .await
1890        .expect_err("the injected crash boundary should leave only the V3 prepare record");
1891
1892        let (pending_pool, pending_identity) = {
1893            let objects = storage.objects.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
1894            (
1895                objects
1896                    .get(POOL_META_NAME)
1897                    .map(|(payload, _)| payload.clone())
1898                    .expect("the V3 prepare record should be durable"),
1899                objects
1900                    .get(POOL_META_IDENTITY_NAME)
1901                    .map(|(payload, _)| payload.clone())
1902                    .expect("the bootstrap identity should be durable"),
1903            )
1904        };
1905        assert_eq!(pool_meta_v3_commit_state_for_test(pending_pool).expect("decode pending V3"), (1, false));
1906        assert!(!pool_meta_identity_initialized_for_test(&pending_identity).expect("decode pending identity"));
1907
1908        storage.fail_pool_meta_write_at.store(0, Ordering::SeqCst);
1909        let mut restarted = PoolMetaWriteState::for_startup(deployment_id, false);
1910        let (loaded, replica_state) = load_pool_meta_for_startup(vec![storage.clone()], &mut restarted)
1911            .await
1912            .expect("restart should recover the predecessor embedded in the pending V3 record");
1913        assert!(loaded.pools.is_empty());
1914        assert!(replica_state.needs_repair);
1915        let committed = persist_pool_meta_for_startup_if_safe(
1916            &init_test_pool_meta(None),
1917            vec![storage.clone()],
1918            replica_state,
1919            &mut restarted,
1920            true,
1921            true,
1922        )
1923        .await
1924        .expect("restart should finish generation 1 before promoting the identity");
1925        assert_eq!(committed.version, 3);
1926
1927        let (committed_pool, committed_identity) = {
1928            let objects = storage.objects.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
1929            (
1930                objects
1931                    .get(POOL_META_NAME)
1932                    .map(|(payload, _)| payload.clone())
1933                    .expect("the committed V3 record should be durable"),
1934                objects
1935                    .get(POOL_META_IDENTITY_NAME)
1936                    .map(|(payload, _)| payload.clone())
1937                    .expect("the committed identity should be durable"),
1938            )
1939        };
1940        assert_eq!(
1941            pool_meta_v3_commit_state_for_test(committed_pool).expect("decode committed V3"),
1942            (1, true)
1943        );
1944        assert!(pool_meta_identity_initialized_for_test(&committed_identity).expect("decode committed identity"));
1945        assert!(
1946            storage
1947                .pool_meta_written_versions
1948                .lock()
1949                .unwrap_or_else(std::sync::PoisonError::into_inner)
1950                .iter()
1951                .all(|version| *version == 3),
1952            "bootstrap recovery must never overwrite a pending V3 record with legacy metadata"
1953        );
1954    }
1955
1956    #[tokio::test]
1957    async fn test_store_init_pool_meta_falls_back_from_corrupt_first_replica() {
1958        let corrupt = Arc::new(StartupPoolMetaStorage::new(vec![0, 1, 2]));
1959        let expected = init_test_pool_meta(None);
1960        let backup = Arc::new(StartupPoolMetaStorage::new(startup_pool_meta_payload(&expected)));
1961        let mut write_state = PoolMetaWriteState::default();
1962
1963        let (loaded, replica_state) = load_pool_meta_for_startup(vec![corrupt.clone(), backup.clone()], &mut write_state)
1964            .await
1965            .expect("startup should select the validated backup replica");
1966
1967        assert!(replica_state.needs_repair);
1968        assert!(replica_state.repair_write_safe);
1969        assert_eq!(loaded.pools.len(), 1);
1970        assert_eq!(loaded.pools[0].cmd_line, expected.pools[0].cmd_line);
1971        assert!(corrupt.read_without_lock.load(Ordering::SeqCst));
1972        assert!(backup.read_without_lock.load(Ordering::SeqCst));
1973
1974        persist_pool_meta_for_startup_if_safe(
1975            &loaded,
1976            vec![corrupt.clone(), backup.clone()],
1977            replica_state,
1978            &mut write_state,
1979            false,
1980            true,
1981        )
1982        .await
1983        .expect("the elected startup writer should repair validated corrupt replicas");
1984
1985        let corrupt_write = corrupt
1986            .written_payload
1987            .lock()
1988            .unwrap_or_else(std::sync::PoisonError::into_inner)
1989            .clone()
1990            .expect("corrupt replica should be repaired");
1991        let backup_write = backup
1992            .written_payload
1993            .lock()
1994            .unwrap_or_else(std::sync::PoisonError::into_inner)
1995            .clone()
1996            .expect("backup replica should receive the same canonical snapshot");
1997        assert_eq!(corrupt_write, backup_write);
1998        assert_ne!(corrupt_write, corrupt.read_payload);
1999    }
2000
2001    #[tokio::test]
2002    async fn test_store_init_pool_meta_does_not_repair_unreadable_replica() {
2003        let valid = Arc::new(StartupPoolMetaStorage::new(startup_pool_meta_payload(&init_test_pool_meta(None))));
2004        let unreadable = Arc::new(StartupPoolMetaStorage::unreadable());
2005        let mut write_state = PoolMetaWriteState::default();
2006
2007        let (loaded, replica_state) = load_pool_meta_for_startup(vec![valid.clone(), unreadable.clone()], &mut write_state)
2008            .await
2009            .expect("startup should use a validated replica without overwriting an unreadable copy");
2010        assert!(replica_state.needs_repair);
2011        assert!(!replica_state.repair_write_safe);
2012
2013        persist_pool_meta_for_startup_if_safe(
2014            &loaded,
2015            vec![valid.clone(), unreadable.clone()],
2016            replica_state,
2017            &mut write_state,
2018            false,
2019            true,
2020        )
2021        .await
2022        .expect("an unreadable copy should defer repair when no topology write is needed");
2023        assert!(!valid.wrote_without_lock.load(Ordering::SeqCst));
2024        assert!(!unreadable.wrote_without_lock.load(Ordering::SeqCst));
2025
2026        let err = persist_pool_meta_for_startup_if_safe(
2027            &loaded,
2028            vec![valid.clone(), unreadable.clone()],
2029            replica_state,
2030            &mut write_state,
2031            true,
2032            true,
2033        )
2034        .await
2035        .expect_err("a topology update must not overwrite an unreadable replica");
2036        assert!(err.to_string().contains("cannot overwrite an unreadable replica"));
2037        assert!(!valid.wrote_without_lock.load(Ordering::SeqCst));
2038        assert!(!unreadable.wrote_without_lock.load(Ordering::SeqCst));
2039    }
2040
2041    #[tokio::test]
2042    async fn test_store_init_pool_meta_stays_blocked_after_all_replicas_were_unreadable() {
2043        let unreadable_a = Arc::new(StartupPoolMetaStorage::unreadable());
2044        let unreadable_b = Arc::new(StartupPoolMetaStorage::unreadable());
2045        let mut write_state = PoolMetaWriteState::default();
2046
2047        load_pool_meta_for_startup(vec![unreadable_a, unreadable_b], &mut write_state)
2048            .await
2049            .expect_err("startup must fail when no readable pool metadata replica exists");
2050
2051        let repaired = Arc::new(StartupPoolMetaStorage::new(startup_pool_meta_payload(&init_test_pool_meta(None))));
2052        let (loaded, replica_state) = load_pool_meta_for_startup(vec![repaired.clone()], &mut write_state)
2053            .await
2054            .expect("a later startup retry may read the repaired replica");
2055        assert!(replica_state.repair_write_safe);
2056
2057        let err =
2058            persist_pool_meta_for_startup_if_safe(&loaded, vec![repaired.clone()], replica_state, &mut write_state, true, true)
2059                .await
2060                .expect_err("the same store instance must not write after observing unreadable replicas");
2061        assert!(
2062            err.to_string()
2063                .contains("restart after all replicas are readable and consistent")
2064        );
2065        assert!(!repaired.wrote_without_lock.load(Ordering::SeqCst));
2066    }
2067
2068    #[test]
2069    fn test_pool_first_endpoint_is_local_respects_local_flag() {
2070        let mut local_endpoint = Endpoint::try_from("http://127.0.0.1:9000/data").expect("endpoint should parse");
2071        local_endpoint.is_local = true;
2072        let pool = PoolEndpoints {
2073            legacy: false,
2074            set_count: 1,
2075            drives_per_set: 1,
2076            endpoints: Endpoints::from(vec![local_endpoint]),
2077            cmd_line: "pool-0".to_string(),
2078            platform: String::new(),
2079        };
2080
2081        assert!(pool_first_endpoint_is_local(&pool));
2082    }
2083
2084    #[test]
2085    fn test_pool_first_endpoint_is_local_rejects_missing_endpoint() {
2086        let pool = PoolEndpoints {
2087            legacy: false,
2088            set_count: 1,
2089            drives_per_set: 1,
2090            endpoints: Endpoints::from(Vec::<Endpoint>::new()),
2091            cmd_line: "pool-0".to_string(),
2092            platform: String::new(),
2093        };
2094
2095        assert!(!pool_first_endpoint_is_local(&pool));
2096    }
2097
2098    #[test]
2099    fn test_local_decommission_watchdog_retry_delay_is_bounded() {
2100        assert_eq!(local_decommission_watchdog_retry_delay(1), LOCAL_DECOMMISSION_RESUME_RETRY_DELAY);
2101        assert_eq!(
2102            local_decommission_watchdog_retry_delay(u32::MAX),
2103            LOCAL_DECOMMISSION_WATCHDOG_MAX_RETRY_DELAY
2104        );
2105    }
2106
2107    #[tokio::test(start_paused = true)]
2108    async fn test_local_decommission_watchdog_cancelled_start_does_not_reconcile() {
2109        let rx = CancellationToken::new();
2110        rx.cancel();
2111        run_local_decommission_watchdog(rx, || async {
2112            panic!("cancelled startup must not schedule persisted work");
2113        })
2114        .await;
2115    }
2116
2117    #[tokio::test]
2118    #[serial_test::serial]
2119    async fn test_local_decommission_recovery_waits_for_live_fleet_proof_before_reserving_worker() {
2120        let (_temp_dirs, store, _other_store) = crate::services::rebalance::test_two_pool_stores(None).await;
2121        mark_test_pool_decommissioning(&store, 0).await;
2122        assert!(store.ctx.is_dist_erasure().await);
2123        let worker_rx = CancellationToken::new();
2124
2125        {
2126            let _proof_guard = crate::services::notification_sys::without_cross_pool_fence_fleet_proof_for_test();
2127            let err = super::reconcile_local_decommission_after_init(&store, worker_rx.clone())
2128                .await
2129                .expect_err("cold distributed recovery must wait for live fleet proof");
2130            assert!(
2131                crate::core::pools::is_pool_activation_fleet_proof_error(&err),
2132                "recovery must reach the live fleet proof gate: {err:?}"
2133            );
2134            assert!(store.decommission_cancelers.read().await.iter().all(Option::is_none));
2135            assert!(pool_meta_has_active_decommission(&*store.pool_meta.read().await));
2136        }
2137
2138        super::reconcile_local_decommission_after_init(&store, worker_rx.clone())
2139            .await
2140            .expect("restored fleet proof should admit the persisted worker");
2141        assert!(store.has_active_local_decommission_worker().await);
2142        worker_rx.cancel();
2143    }
2144
2145    #[tokio::test(start_paused = true)]
2146    async fn test_local_decommission_watchdog_retries_general_failures_until_cancelled() {
2147        let rx = CancellationToken::new();
2148        let attempts = Arc::new(AtomicUsize::new(0));
2149        let task = tokio::spawn(run_local_decommission_watchdog(rx.clone(), {
2150            let attempts = attempts.clone();
2151            let rx = rx.clone();
2152            move || {
2153                let attempts = attempts.clone();
2154                let rx = rx.clone();
2155                async move {
2156                    match attempts.fetch_add(1, Ordering::SeqCst) {
2157                        0 => Err(StorageError::other("pool activation requires a live fleet capability proof")),
2158                        1 => Err(StorageError::SlowDown),
2159                        _ => {
2160                            rx.cancel();
2161                            Ok(())
2162                        }
2163                    }
2164                }
2165            }
2166        }));
2167
2168        tokio::task::yield_now().await;
2169        assert_eq!(attempts.load(Ordering::SeqCst), 1);
2170        tokio::time::advance(LOCAL_DECOMMISSION_RESUME_RETRY_DELAY).await;
2171        tokio::task::yield_now().await;
2172        assert_eq!(attempts.load(Ordering::SeqCst), 2);
2173        tokio::time::advance(local_decommission_watchdog_retry_delay(2)).await;
2174        task.await.expect("watchdog task should exit after cancellation");
2175        assert_eq!(attempts.load(Ordering::SeqCst), 3);
2176    }
2177
2178    #[tokio::test(start_paused = true)]
2179    async fn test_local_decommission_watchdog_rescans_after_success() {
2180        let rx = CancellationToken::new();
2181        let attempts = Arc::new(AtomicUsize::new(0));
2182        let task = tokio::spawn(run_local_decommission_watchdog(rx.clone(), {
2183            let attempts = attempts.clone();
2184            let rx = rx.clone();
2185            move || {
2186                let attempts = attempts.clone();
2187                let rx = rx.clone();
2188                async move {
2189                    if attempts.fetch_add(1, Ordering::SeqCst) == 1 {
2190                        rx.cancel();
2191                    }
2192                    Ok(())
2193                }
2194            }
2195        }));
2196
2197        tokio::task::yield_now().await;
2198        assert_eq!(attempts.load(Ordering::SeqCst), 1);
2199        tokio::time::advance(super::LOCAL_DECOMMISSION_WATCHDOG_INTERVAL).await;
2200        task.await.expect("watchdog task should exit after cancellation");
2201        assert_eq!(attempts.load(Ordering::SeqCst), 2);
2202    }
2203
2204    #[test]
2205    fn test_should_retry_format_load_rejects_permanent_corruption() {
2206        assert!(!should_retry_format_load(&StorageError::CorruptedFormat));
2207        assert!(should_retry_format_load(&StorageError::ErasureReadQuorum));
2208        assert!(should_retry_format_load(&StorageError::FirstDiskWait));
2209    }
2210
2211    #[test]
2212    fn test_should_retry_format_load_rejects_permanent_topology_errors() {
2213        for error in [
2214            StorageError::UnsupportedSnsdExpansion { configured_drives: 4 },
2215            StorageError::PoolTopologyMismatch {
2216                stored_drives: 4,
2217                stored_set_drive_count: 4,
2218                configured_drives: 8,
2219                configured_set_drive_count: 8,
2220            },
2221        ] {
2222            assert!(!should_retry_format_load(&error), "topology errors require operator action: {error}");
2223        }
2224        for error in [
2225            StorageError::DiskNotFound,
2226            StorageError::Timeout,
2227            StorageError::RemoteNotInitialized,
2228            StorageError::NotFirstDisk,
2229            StorageError::other(std::io::Error::from(std::io::ErrorKind::ConnectionRefused)),
2230        ] {
2231            assert!(
2232                should_retry_format_load(&error),
2233                "transient failures retain their existing retry path: {error}"
2234            );
2235        }
2236    }
2237
2238    #[test]
2239    fn test_should_auto_start_rebalance_after_init_allows_active_rebalance_without_decommission() {
2240        assert!(should_auto_start_rebalance_after_init(false, true));
2241    }
2242
2243    #[test]
2244    fn test_should_auto_start_rebalance_after_init_rejects_active_decommission() {
2245        assert!(!should_auto_start_rebalance_after_init(true, true));
2246    }
2247
2248    #[test]
2249    fn test_should_auto_start_rebalance_after_init_rejects_terminal_or_missing_rebalance() {
2250        assert!(!should_auto_start_rebalance_after_init(false, false));
2251    }
2252
2253    #[test]
2254    fn test_should_defer_rebalance_auto_start_only_without_distributed_fleet_proof() {
2255        assert!(should_defer_rebalance_auto_start(true, false));
2256        assert!(!should_defer_rebalance_auto_start(true, true));
2257        assert!(!should_defer_rebalance_auto_start(false, false));
2258    }
2259
2260    #[test]
2261    fn test_store_init_recovery_skips_rebalance_when_decommission_metadata_is_active() {
2262        let pool_meta = init_test_pool_meta(Some(PoolDecommissionInfo {
2263            start_time: Some(OffsetDateTime::UNIX_EPOCH),
2264            complete: false,
2265            failed: false,
2266            canceled: false,
2267            ..Default::default()
2268        }));
2269        let rebalance_meta = Some(RebalanceMeta {
2270            pool_stats: vec![RebalanceStats {
2271                participating: true,
2272                info: RebalanceInfo {
2273                    status: RebalStatus::Started,
2274                    ..Default::default()
2275                },
2276                ..Default::default()
2277            }],
2278            ..Default::default()
2279        });
2280
2281        assert!(!should_auto_start_rebalance_after_init(
2282            pool_meta_has_active_decommission(&pool_meta),
2283            rebalance_meta
2284                .as_ref()
2285                .is_some_and(crate::services::rebalance::rebalance_requires_worker_activation)
2286        ));
2287    }
2288
2289    #[test]
2290    fn test_store_init_recovery_allows_active_rebalance_without_decommission() {
2291        let pool_meta = init_test_pool_meta(None);
2292        let rebalance_meta = Some(RebalanceMeta {
2293            pool_stats: vec![RebalanceStats {
2294                participating: true,
2295                info: RebalanceInfo {
2296                    status: RebalStatus::Started,
2297                    ..Default::default()
2298                },
2299                ..Default::default()
2300            }],
2301            ..Default::default()
2302        });
2303
2304        assert!(should_auto_start_rebalance_after_init(
2305            pool_meta_has_active_decommission(&pool_meta),
2306            rebalance_meta
2307                .as_ref()
2308                .is_some_and(crate::services::rebalance::rebalance_requires_worker_activation)
2309        ));
2310    }
2311
2312    #[test]
2313    fn test_store_init_recovery_skips_completed_rebalance_metadata() {
2314        let pool_meta = init_test_pool_meta(None);
2315        let rebalance_meta = Some(RebalanceMeta {
2316            pool_stats: vec![RebalanceStats {
2317                participating: true,
2318                info: RebalanceInfo {
2319                    status: RebalStatus::Completed,
2320                    ..Default::default()
2321                },
2322                ..Default::default()
2323            }],
2324            ..Default::default()
2325        });
2326
2327        assert!(!should_auto_start_rebalance_after_init(
2328            pool_meta_has_active_decommission(&pool_meta),
2329            rebalance_meta
2330                .as_ref()
2331                .is_some_and(crate::services::rebalance::rebalance_requires_worker_activation)
2332        ));
2333    }
2334
2335    #[test]
2336    fn test_resolve_store_init_stage_result_passthrough_ok() {
2337        resolve_store_init_stage_result(Ok(()), "load_rebalance_meta").expect("successful stage should pass through");
2338    }
2339
2340    #[test]
2341    fn test_resolve_store_init_stage_result_wraps_error_context() {
2342        let err = resolve_store_init_stage_result(Err(StorageError::SlowDown), "start_rebalance")
2343            .expect_err("failed stage should be wrapped");
2344        let err_message = err.to_string();
2345        assert!(err_message.contains("store init failed during start_rebalance"));
2346        assert!(err_message.contains(&StorageError::SlowDown.to_string()));
2347    }
2348
2349    #[tokio::test]
2350    async fn test_wait_for_local_decommission_resume_delay_returns_true_after_delay() {
2351        let rx = CancellationToken::new();
2352        assert!(wait_for_local_decommission_resume_delay(&rx, Duration::from_millis(1)).await);
2353    }
2354
2355    #[tokio::test]
2356    async fn test_wait_for_local_decommission_resume_delay_returns_false_when_cancelled() {
2357        let rx = CancellationToken::new();
2358        rx.cancel();
2359        assert!(!wait_for_local_decommission_resume_delay(&rx, Duration::from_secs(1)).await);
2360    }
2361
2362    #[test]
2363    fn test_pool_first_endpoint_is_local_uses_pool_scope_for_expansion() {
2364        let mut remote_endpoint = Endpoint::try_from("http://127.0.0.2:9000/data1").expect("remote endpoint should parse");
2365        remote_endpoint.is_local = false;
2366
2367        let mut local_endpoint = Endpoint::try_from("http://127.0.0.1:9000/data1").expect("local endpoint should parse");
2368        local_endpoint.is_local = true;
2369
2370        let endpoints = EndpointServerPools::from(vec![
2371            PoolEndpoints {
2372                legacy: false,
2373                set_count: 1,
2374                drives_per_set: 1,
2375                endpoints: Endpoints::from(vec![remote_endpoint]),
2376                cmd_line: "pool-0".to_string(),
2377                platform: String::new(),
2378            },
2379            PoolEndpoints {
2380                legacy: false,
2381                set_count: 1,
2382                drives_per_set: 1,
2383                endpoints: Endpoints::from(vec![local_endpoint]),
2384                cmd_line: "pool-1".to_string(),
2385                platform: String::new(),
2386            },
2387        ]);
2388
2389        assert!(!endpoints.first_local(), "cluster first endpoint is intentionally remote");
2390        assert!(
2391            pool_first_endpoint_is_local(endpoints.as_ref().get(1).expect("second pool should exist")),
2392            "the expanded pool should be initialized by its own first local endpoint"
2393        );
2394    }
2395
2396    fn rpc_preflight_pools(pools: Vec<Vec<Endpoint>>) -> EndpointServerPools {
2397        EndpointServerPools::from(
2398            pools
2399                .into_iter()
2400                .enumerate()
2401                .map(|(pool_index, endpoints)| PoolEndpoints {
2402                    legacy: false,
2403                    set_count: 1,
2404                    drives_per_set: endpoints.len(),
2405                    endpoints: Endpoints::from(endpoints),
2406                    cmd_line: format!("pool-{pool_index}"),
2407                    platform: String::new(),
2408                })
2409                .collect::<Vec<_>>(),
2410        )
2411    }
2412
2413    fn parsed_local_endpoint() -> Endpoint {
2414        let mut endpoint = Endpoint::try_from("http://127.0.0.1:9000/data").expect("local endpoint should parse");
2415        endpoint.is_local = true;
2416        endpoint
2417    }
2418
2419    fn parsed_remote_endpoint() -> Endpoint {
2420        let mut endpoint = Endpoint::try_from("http://10.0.0.2:9000/data").expect("remote endpoint should parse");
2421        endpoint.is_local = false;
2422        endpoint
2423    }
2424
2425    #[test]
2426    fn test_rpc_secret_preflight_aborts_for_remote_endpoints_without_secret() {
2427        // The remote endpoint intentionally sits behind a local one: the
2428        // preflight must scan every endpoint, not just the first.
2429        let pools = rpc_preflight_pools(vec![vec![parsed_local_endpoint(), parsed_remote_endpoint()]]);
2430
2431        let err = preflight_startup_rpc_secret_with(&pools, || {
2432            Err(std::io::Error::other(rustfs_credentials::RPC_SECRET_REQUIRED_MESSAGE))
2433        })
2434        .expect_err("remote endpoints without an RPC secret must abort startup before the format retry loop");
2435
2436        let message = err.to_string();
2437        assert!(message.contains("store init aborted"), "unexpected error: {message}");
2438        assert!(
2439            message.contains(rustfs_credentials::RPC_SECRET_REQUIRED_MESSAGE),
2440            "error must state the resolution failure: {message}"
2441        );
2442        assert!(
2443            message.contains(rustfs_credentials::RPC_SECRET_REQUIRED_OPERATOR_MESSAGE),
2444            "error must carry the operator remediation guidance: {message}"
2445        );
2446    }
2447
2448    #[test]
2449    fn test_rpc_secret_preflight_aborts_for_remote_endpoint_in_later_pool() {
2450        // The remote endpoint hides in a later, otherwise-local pool: a
2451        // first-pool shortcut (the `first_local` shape) must not pass.
2452        let pools = rpc_preflight_pools(vec![vec![parsed_local_endpoint()], vec![parsed_remote_endpoint()]]);
2453
2454        let err = preflight_startup_rpc_secret_with(&pools, || {
2455            Err(std::io::Error::other(rustfs_credentials::RPC_SECRET_REQUIRED_MESSAGE))
2456        })
2457        .expect_err("a remote endpoint in a later pool must abort startup without an RPC secret");
2458
2459        let message = err.to_string();
2460        assert!(message.contains("store init aborted"), "unexpected error: {message}");
2461    }
2462
2463    #[test]
2464    fn test_rpc_secret_preflight_skips_resolution_for_local_only_endpoints() {
2465        preflight_startup_rpc_secret_with(&rpc_preflight_pools(vec![vec![parsed_local_endpoint()]]), || {
2466            panic!("single-node topology must not resolve an RPC secret")
2467        })
2468        .expect("local-only topology must start without an RPC secret");
2469    }
2470
2471    #[test]
2472    fn test_rpc_secret_preflight_accepts_remote_endpoints_with_resolved_secret() {
2473        preflight_startup_rpc_secret_with(&rpc_preflight_pools(vec![vec![parsed_remote_endpoint()]]), || {
2474            Ok("resolved-secret".to_string())
2475        })
2476        .expect("a resolvable RPC secret must not block startup");
2477    }
2478
2479    #[tokio::test]
2480    #[serial_test::serial(storage_class_env)]
2481    async fn test_new_with_instance_ctx_aborts_before_format_retry_loop_without_rpc_secret() {
2482        // Guard: under a shared-process `cargo test` run another test may have
2483        // already pinned the process-global RPC secret (ensure_test_rpc_secret),
2484        // which would let init proceed past the preflight into remote disk
2485        // init. A failing probe pins nothing, so it cannot poison later tests;
2486        // under nextest's process-per-test model (CI) the guard never fires.
2487        if rustfs_credentials::try_get_rpc_token().is_ok() {
2488            return;
2489        }
2490
2491        let mut remote_endpoint = parsed_remote_endpoint();
2492        remote_endpoint.set_pool_index(0);
2493        remote_endpoint.set_set_index(0);
2494        remote_endpoint.set_disk_index(0);
2495
2496        let err = crate::store::ECStore::new_with_instance_ctx(
2497            "127.0.0.1:0".parse().expect("test address"),
2498            rpc_preflight_pools(vec![vec![remote_endpoint]]),
2499            CancellationToken::new(),
2500            Arc::new(crate::runtime::instance::InstanceContext::new()),
2501        )
2502        .await
2503        .expect_err("distributed init without an RPC secret must abort before the format retry loop");
2504
2505        let message = err.to_string();
2506        assert!(message.contains("store init aborted"), "unexpected error: {message}");
2507        assert!(
2508            message.contains(rustfs_credentials::RPC_SECRET_REQUIRED_OPERATOR_MESSAGE),
2509            "abort must carry the operator remediation guidance: {message}"
2510        );
2511    }
2512
2513    fn endpoint_pools_with_drive_counts(counts: &[usize]) -> EndpointServerPools {
2514        EndpointServerPools::from(
2515            counts
2516                .iter()
2517                .enumerate()
2518                .map(|(pool_index, &drives_per_set)| PoolEndpoints {
2519                    legacy: false,
2520                    set_count: 1,
2521                    drives_per_set,
2522                    endpoints: Endpoints::from(Vec::new()),
2523                    cmd_line: format!("pool-{pool_index}"),
2524                    platform: String::new(),
2525                })
2526                .collect::<Vec<_>>(),
2527        )
2528    }
2529
2530    #[test]
2531    fn startup_pool_defaults_are_resolved_per_pool() {
2532        let validate = |pools: &EndpointServerPools| {
2533            let drive_counts: Vec<_> = pools.as_ref().iter().map(|pool| pool.drives_per_set).collect();
2534            crate::config::storageclass::lookup_config_for_pools_without_env(&KVS::new(), &drive_counts).map(|_| ())
2535        };
2536        let defaults = resolve_startup_pool_defaults_with(&endpoint_pools_with_drive_counts(&[4, 2]), validate)
2537            .expect("heterogeneous topology should resolve");
2538        assert_eq!(defaults, vec![2, 1]);
2539
2540        let defaults = resolve_startup_pool_defaults_with(&endpoint_pools_with_drive_counts(&[4, 6]), validate)
2541            .expect("heterogeneous topology should resolve");
2542        assert_eq!(defaults, vec![2, 3]);
2543    }
2544
2545    #[test]
2546    fn startup_pool_defaults_validate_explicit_environment_for_every_pool() {
2547        let validate = |pools: &EndpointServerPools| {
2548            let drive_counts: Vec<_> = pools.as_ref().iter().map(|pool| pool.drives_per_set).collect();
2549            let mut kvs = KVS::new();
2550            kvs.insert(crate::config::storageclass::CLASS_STANDARD.to_string(), "EC:2".to_string());
2551            crate::config::storageclass::lookup_config_for_pools_without_env(&kvs, &drive_counts).map(|_| ())
2552        };
2553        let err = resolve_startup_pool_defaults_with(&endpoint_pools_with_drive_counts(&[4, 2]), validate)
2554            .expect_err("explicit EC:2 must fail before any two-drive pool I/O");
2555        assert!(err.to_string().contains("pool 1") && err.to_string().contains("2 drives"));
2556    }
2557
2558    #[test]
2559    #[serial_test::serial(storage_class_env)]
2560    fn startup_pool_defaults_validate_environment_without_changing_metadata_fallback() {
2561        temp_env::with_vars(
2562            [
2563                (crate::config::storageclass::STANDARD_ENV, Some("EC:1")),
2564                (crate::config::storageclass::RRS_ENV, None),
2565                (crate::config::storageclass::OPTIMIZE_ENV, None),
2566                (crate::config::storageclass::INLINE_BLOCK_ENV, None),
2567            ],
2568            || {
2569                let runtime = crate::config::storageclass::lookup_config_for_pools(&KVS::new(), &[6, 4])
2570                    .expect("explicit standard parity must resolve the runtime candidate");
2571                assert_eq!(runtime.parities_for_sc(crate::config::storageclass::STANDARD), Some(vec![1, 1]));
2572
2573                let defaults = super::resolve_startup_pool_defaults(&endpoint_pools_with_drive_counts(&[6, 4]))
2574                    .expect("explicit standard parity must validate for every pool");
2575                assert_eq!(defaults, vec![3, 2]);
2576            },
2577        );
2578    }
2579
2580    async fn without_storage_class_env<F: Future>(future: F) -> F::Output {
2581        temp_env::async_with_vars(
2582            [
2583                (crate::config::storageclass::STANDARD_ENV, None::<&str>),
2584                (crate::config::storageclass::RRS_ENV, None::<&str>),
2585                (crate::config::storageclass::OPTIMIZE_ENV, None::<&str>),
2586                (crate::config::storageclass::INLINE_BLOCK_ENV, None::<&str>),
2587            ],
2588            future,
2589        )
2590        .await
2591    }
2592
2593    // Build a real local store over a temp dir around a fresh instance context.
2594    async fn build_isolated_test_store(
2595        temp_dir: &std::path::Path,
2596        cmd_line: &str,
2597        pool_drive_counts: &[usize],
2598    ) -> (
2599        Arc<crate::runtime::instance::InstanceContext>,
2600        Arc<crate::store::ECStore>,
2601        CancellationToken,
2602    ) {
2603        build_isolated_test_store_with_shutdown(temp_dir, cmd_line, pool_drive_counts, CancellationToken::new()).await
2604    }
2605
2606    async fn build_isolated_test_store_with_shutdown(
2607        temp_dir: &std::path::Path,
2608        cmd_line: &str,
2609        pool_drive_counts: &[usize],
2610        shutdown: CancellationToken,
2611    ) -> (
2612        Arc<crate::runtime::instance::InstanceContext>,
2613        Arc<crate::store::ECStore>,
2614        CancellationToken,
2615    ) {
2616        let pool_layouts = pool_drive_counts
2617            .iter()
2618            .map(|&drives_per_set| (1, drives_per_set))
2619            .collect::<Vec<_>>();
2620        build_isolated_test_store_with_layout(temp_dir, cmd_line, &pool_layouts, shutdown, None).await
2621    }
2622
2623    async fn build_isolated_test_store_with_layout(
2624        temp_dir: &std::path::Path,
2625        cmd_line: &str,
2626        pool_layouts: &[(usize, usize)],
2627        shutdown: CancellationToken,
2628        instance_ctx: Option<Arc<crate::runtime::instance::InstanceContext>>,
2629    ) -> (
2630        Arc<crate::runtime::instance::InstanceContext>,
2631        Arc<crate::store::ECStore>,
2632        CancellationToken,
2633    ) {
2634        let mut pools = Vec::with_capacity(pool_layouts.len());
2635        for (pool_index, &(set_count, drives_per_set)) in pool_layouts.iter().enumerate() {
2636            let mut endpoints = Vec::with_capacity(set_count * drives_per_set);
2637            for set_index in 0..set_count {
2638                for disk_index in 0..drives_per_set {
2639                    let path = temp_dir.join(format!("pool{pool_index}/set{set_index}/disk{disk_index}"));
2640                    tokio::fs::create_dir_all(&path).await.expect("create disk dir");
2641                    let mut endpoint =
2642                        Endpoint::try_from(path.to_str().expect("disk path should be utf-8")).expect("local endpoint");
2643                    endpoint.set_pool_index(pool_index);
2644                    endpoint.set_set_index(set_index);
2645                    endpoint.set_disk_index(disk_index);
2646                    endpoints.push(endpoint);
2647                }
2648            }
2649            pools.push(PoolEndpoints {
2650                legacy: false,
2651                set_count,
2652                drives_per_set,
2653                endpoints: Endpoints::from(endpoints),
2654                cmd_line: format!("{cmd_line}-pool-{pool_index}"),
2655                platform: "test".to_string(),
2656            });
2657        }
2658        let endpoint_pools = EndpointServerPools(pools);
2659        crate::services::notification_sys::install_cross_pool_fence_fleet_proof_for_test();
2660
2661        let instance_ctx = instance_ctx.unwrap_or_else(|| Arc::new(crate::runtime::instance::InstanceContext::new()));
2662        crate::store::init_local_disks_with_instance_ctx(&instance_ctx, endpoint_pools.clone())
2663            .await
2664            .expect("register local disks into the fresh context");
2665
2666        let store = crate::store::ECStore::new_with_instance_ctx(
2667            "127.0.0.1:0".parse().expect("test address"),
2668            endpoint_pools,
2669            shutdown.clone(),
2670            instance_ctx.clone(),
2671        )
2672        .await
2673        .expect("store should build around the fresh context");
2674
2675        // Capacity admission in these local fixtures must not depend on the
2676        // host volume's statvfs values. Keep enough identical snapshots for
2677        // startup, recovery, and the mutation probes exercised by each test.
2678        let layout = DecommissionErasureLayout { data: 2, parity: 2 };
2679        let snapshot: Vec<DecommissionPoolCapacityInfo> = store
2680            .pools
2681            .iter()
2682            .enumerate()
2683            .map(|(pool_index, _)| DecommissionPoolCapacityInfo::for_test(pool_index, layout, 1 << 40, 1 << 40, 1 << 30))
2684            .collect();
2685        set_decommission_capacity_info_overrides_for_test(store.id, (0..128).map(|_| snapshot.clone()).collect());
2686
2687        (instance_ctx, store, shutdown)
2688    }
2689
2690    #[cfg(feature = "test-util")]
2691    #[tokio::test]
2692    #[serial_test::serial(storage_class_env)]
2693    async fn cross_key_materialized_copy_does_not_publish_remote_tuple_ownership() {
2694        let temp_dir = tempfile::tempdir().expect("create materialized-copy store dir");
2695        let (ctx, store, shutdown) =
2696            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "copy-materialized", &[4])).await;
2697        crate::bucket::metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
2698
2699        let bucket = format!("copy-materialized-{}", Uuid::new_v4());
2700        let source_object = "source-restored.bin";
2701        let target_object = "target-local.bin";
2702        let payload = b"materialized copy must own only its new local bytes".to_vec();
2703        store
2704            .make_bucket(&bucket, &MakeBucketOptions::default())
2705            .await
2706            .expect("copy test bucket should be created");
2707
2708        let tier_name = "COPY-MATERIALIZED-TIER";
2709        register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
2710        let lease = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
2711            .await
2712            .expect("mock tier generation should be available");
2713        let backend_identity = lease.backend_identity();
2714        drop(lease);
2715
2716        let mut source_metadata = HashMap::from([
2717            ("content-type".to_string(), "application/octet-stream".to_string()),
2718            (
2719                "x-amz-restore".to_string(),
2720                "ongoing-request=\"false\", expiry-date=\"2099-01-01T00:00:00Z\"".to_string(),
2721            ),
2722        ]);
2723        for (suffix, value) in [
2724            (
2725                rustfs_utils::http::SUFFIX_TRANSITION_STATUS,
2726                crate::bucket::lifecycle::core::TRANSITION_COMPLETE.to_string(),
2727            ),
2728            (
2729                rustfs_utils::http::SUFFIX_TRANSITIONED_OBJECTNAME,
2730                "remote/source-restored.bin".to_string(),
2731            ),
2732            (rustfs_utils::http::SUFFIX_TRANSITION_TIER, tier_name.to_string()),
2733            (rustfs_utils::http::SUFFIX_TRANSITIONED_VERSION_ID, "remote-source-version".to_string()),
2734            (
2735                rustfs_utils::http::SUFFIX_TRANSITIONED_VERSION_STATE,
2736                rustfs_filemeta::TransitionVersionState::Exact.as_str().to_string(),
2737            ),
2738            (
2739                rustfs_utils::http::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
2740                rustfs_utils::crypto::hex(backend_identity),
2741            ),
2742            (rustfs_utils::http::SUFFIX_TRANSITION_TRANSACTION_ID, Uuid::new_v4().to_string()),
2743        ] {
2744            rustfs_utils::http::metadata_compat::insert_str(&mut source_metadata, suffix, value);
2745        }
2746
2747        let mut source_body = PutObjReader::from_vec(payload.clone());
2748        store
2749            .put_object(
2750                &bucket,
2751                source_object,
2752                &mut source_body,
2753                &ObjectOptions {
2754                    user_defined: source_metadata,
2755                    ..Default::default()
2756                },
2757            )
2758            .await
2759            .expect("restored transitioned source should be written");
2760
2761        let source_opts = ObjectOptions::default();
2762        let mut source_info = store
2763            .get_object_info(&bucket, source_object, &source_opts)
2764            .await
2765            .expect("source metadata should be readable");
2766        assert_eq!(
2767            source_info.transitioned_object.status,
2768            crate::bucket::lifecycle::core::TRANSITION_COMPLETE
2769        );
2770        assert_eq!(source_info.transitioned_object.tier, tier_name);
2771        assert!(source_info.data_dir.is_some(), "restored source must retain local data");
2772        source_info.put_object_reader = Some(PutObjReader::from_vec(payload.clone()));
2773
2774        let copied = store
2775            .copy_object(
2776                &bucket,
2777                source_object,
2778                &bucket,
2779                target_object,
2780                &mut source_info,
2781                &source_opts,
2782                &ObjectOptions::default(),
2783            )
2784            .await
2785            .expect("cross-key materialized copy should succeed");
2786        assert!(copied.transitioned_object.status.is_empty());
2787        assert!(copied.transitioned_object.name.is_empty());
2788        assert!(copied.transitioned_object.version_id.is_empty());
2789        assert!(copied.transitioned_object.tier.is_empty());
2790        assert!(!copied.transitioned_object.free_version);
2791        for suffix in [
2792            rustfs_utils::http::SUFFIX_TRANSITION_STATUS,
2793            rustfs_utils::http::SUFFIX_TRANSITIONED_OBJECTNAME,
2794            rustfs_utils::http::SUFFIX_TRANSITIONED_VERSION_ID,
2795            rustfs_utils::http::SUFFIX_TRANSITIONED_VERSION_STATE,
2796            rustfs_utils::http::SUFFIX_TRANSITION_TIER,
2797            rustfs_utils::http::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
2798            rustfs_utils::http::SUFFIX_TRANSITION_TRANSACTION_ID,
2799        ] {
2800            assert!(
2801                !rustfs_utils::http::metadata_compat::contains_key_str(copied.user_defined.as_ref(), suffix),
2802                "target retained protected source suffix {suffix}"
2803            );
2804        }
2805        assert_eq!(
2806            copied.user_defined.get("content-type").map(String::as_str),
2807            Some("application/octet-stream")
2808        );
2809
2810        let mut target_reader = store
2811            .get_object_reader(&bucket, target_object, None, HeaderMap::new(), &ObjectOptions::default())
2812            .await
2813            .expect("target object should be readable");
2814        let mut target_body = Vec::new();
2815        target_reader
2816            .stream
2817            .read_to_end(&mut target_body)
2818            .await
2819            .expect("target body should stream");
2820        assert_eq!(target_body, payload);
2821
2822        let source_after = store
2823            .get_object_info(&bucket, source_object, &source_opts)
2824            .await
2825            .expect("source metadata should remain readable after copy");
2826        assert_eq!(
2827            source_after.transitioned_object.status,
2828            crate::bucket::lifecycle::core::TRANSITION_COMPLETE
2829        );
2830        assert_eq!(source_after.transitioned_object.tier, tier_name);
2831        assert_eq!(source_after.transitioned_object.name, "remote/source-restored.bin");
2832        shutdown.cancel();
2833    }
2834
2835    #[cfg(feature = "test-util")]
2836    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2837    #[serial_test::serial(storage_class_env)]
2838    async fn tier_overwrite_put_and_self_copy_recover_persisted_cleanup_owners() {
2839        use crate::bucket::lifecycle::bucket_lifecycle_ops::ExpiryState;
2840        use crate::bucket::lifecycle::tier_free_version_recovery::recover_tier_free_versions;
2841        use rustfs_filemeta::TransitionVersionState::{Exact, KnownDisabled, SuspendedNull};
2842        use rustfs_s3_client::transition_api::ReaderImpl;
2843        use rustfs_utils::http::{
2844            SUFFIX_TRANSITION_STATUS, SUFFIX_TRANSITION_TIER, SUFFIX_TRANSITION_TIER_DESTINATION_ID,
2845            SUFFIX_TRANSITIONED_OBJECTNAME, SUFFIX_TRANSITIONED_VERSION_ID, SUFFIX_TRANSITIONED_VERSION_STATE, insert_str,
2846        };
2847
2848        let temp_dir = tempfile::tempdir().expect("create tier overwrite store");
2849        let (mut ctx, mut store, mut shutdown) =
2850            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "tier-overwrite", &[4])).await;
2851        crate::bucket::metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
2852        let tier = "OVERWRITE-TIER";
2853        let backend = register_mock_tier(&ctx.tier_config_mgr(), tier).await;
2854        let lease = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier)
2855            .await
2856            .expect("tier identity");
2857        let identity = rustfs_utils::crypto::hex(lease.backend_identity());
2858        drop(lease);
2859
2860        for state in [Exact, KnownDisabled, SuspendedNull] {
2861            for suspended in [false, true] {
2862                for self_copy in [false, true] {
2863                    let bucket = format!("tier-overwrite-{}", Uuid::new_v4());
2864                    let object = "object";
2865                    let remote = format!("remote/{bucket}");
2866                    let version = match state {
2867                        Exact => "opaque-overwrite-version",
2868                        SuspendedNull => "null",
2869                        _ => "",
2870                    };
2871                    let payload = vec![0x5b; if suspended { 512 * 1024 } else { 257 }];
2872                    store
2873                        .make_bucket(&bucket, &MakeBucketOptions::default())
2874                        .await
2875                        .expect("create bucket");
2876                    backend.set_put_remote_version(Some(version.to_string())).await;
2877                    let lease = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier)
2878                        .await
2879                        .expect("seed tier lease");
2880                    lease
2881                        .put(
2882                            &remote,
2883                            ReaderImpl::Body(bytes::Bytes::from(payload.clone())),
2884                            payload.len().try_into().expect("payload size"),
2885                        )
2886                        .await
2887                        .expect("seed remote bytes");
2888                    drop(lease);
2889                    let mut metadata = HashMap::from([
2890                        ("content-type".to_string(), "application/octet-stream".to_string()),
2891                        (
2892                            "x-amz-restore".to_string(),
2893                            "ongoing-request=\"false\", expiry-date=\"2099-01-01T00:00:00Z\"".to_string(),
2894                        ),
2895                    ]);
2896                    for (suffix, value) in [
2897                        (SUFFIX_TRANSITION_STATUS, "complete"),
2898                        (SUFFIX_TRANSITION_TIER, tier),
2899                        (SUFFIX_TRANSITION_TIER_DESTINATION_ID, identity.as_str()),
2900                        (SUFFIX_TRANSITIONED_OBJECTNAME, remote.as_str()),
2901                        (SUFFIX_TRANSITIONED_VERSION_STATE, state.as_str()),
2902                    ] {
2903                        insert_str(&mut metadata, suffix, value.to_string());
2904                    }
2905                    if !version.is_empty() {
2906                        insert_str(&mut metadata, SUFFIX_TRANSITIONED_VERSION_ID, version.to_string());
2907                    }
2908                    // The commit path acknowledges at write quorum and drains
2909                    // the remaining rename fan-out in the background. This
2910                    // fixture "crashes" the store right after the overwrite,
2911                    // so it must wait for that tail: a disk left with the old
2912                    // live transitioned source makes exact cleanup fail closed
2913                    // (a minority live owner still references the remote
2914                    // tuple) until heal repairs it, which this fixture never
2915                    // runs (rustfs#7921).
2916                    let options = ObjectOptions {
2917                        version_suspended: suspended,
2918                        write_completion: crate::object_api::WriteCompletion::TailDrained,
2919                        ..Default::default()
2920                    };
2921                    store
2922                        .put_object(
2923                            &bucket,
2924                            object,
2925                            &mut PutObjReader::from_vec(payload.clone()),
2926                            &ObjectOptions {
2927                                user_defined: metadata,
2928                                ..options.clone()
2929                            },
2930                        )
2931                        .await
2932                        .expect("seed transitioned source with locally restored bytes");
2933                    // The overwrite queues its committed cleanup owner right
2934                    // away, and from the second iteration on the restarted
2935                    // store already runs expiry workers. Fail that first remote
2936                    // DELETE so the owner stays durable and the restart below
2937                    // still has to rediscover it from xl.meta.
2938                    backend.set_remove_failure(true);
2939                    let expected = if self_copy {
2940                        payload.clone()
2941                    } else {
2942                        vec![0x73; payload.len()]
2943                    };
2944                    let new_metadata = HashMap::from([
2945                        ("content-type".to_string(), "text/plain".to_string()),
2946                        ("x-amz-meta-replacement".to_string(), "kept".to_string()),
2947                    ]);
2948                    if self_copy {
2949                        let mut source = store
2950                            .get_object_info(&bucket, object, &options)
2951                            .await
2952                            .expect("self-copy source");
2953                        source.metadata_only = false;
2954                        source.user_defined = Arc::new(new_metadata);
2955                        source.put_object_reader = Some(PutObjReader::from_vec(expected.clone()));
2956                        store
2957                            .copy_object(&bucket, object, &bucket, object, &mut source, &options, &options)
2958                            .await
2959                            .expect("materialized self-copy");
2960                    } else {
2961                        store
2962                            .put_object(
2963                                &bucket,
2964                                object,
2965                                &mut PutObjReader::from_vec(expected.clone()),
2966                                &ObjectOptions {
2967                                    user_defined: new_metadata,
2968                                    ..options.clone()
2969                                },
2970                            )
2971                            .await
2972                            .expect("overwrite transitioned null version");
2973                    }
2974
2975                    let set = store.pools[0].get_disks_by_key(object);
2976                    let versions = set
2977                        .load_file_info_versions_exact(&bucket, object)
2978                        .await
2979                        .expect("read committed disk metadata")
2980                        .expect("replacement metadata exists");
2981                    let free: Vec<_> = versions
2982                        .versions
2983                        .iter()
2984                        .chain(versions.free_versions.iter())
2985                        .filter(|fi| fi.tier_free_version())
2986                        .collect();
2987                    assert_eq!(free.len(), 1, "{state:?}, suspended={suspended}, copy={self_copy}");
2988                    assert_eq!(free[0].transitioned_objname, remote);
2989                    assert_eq!(free[0].transition_version_state, state);
2990                    assert_every_disk_holds_only_the_cleanup_owner(&set, &bucket, object, &remote).await;
2991                    assert!(backend.contains(&remote).await, "commit must not delete remote bytes before cleanup");
2992                    let removed_before = backend.remove_count().await;
2993
2994                    // Later matrix cases reuse this restarted store. If its
2995                    // expiry worker has already dequeued this failed cleanup,
2996                    // drain it before cancelling the context so the next
2997                    // store cannot inherit object locks that block recovery.
2998                    wait_for_expiry_workers_idle(&store).await;
2999
3000                    // Restart after the failed cleanup attempt. The new
3001                    // runtime must reconstruct ownership from committed
3002                    // xl.meta regardless of whether the old queue delivered it.
3003                    let tier_config = ctx
3004                        .tier_config_mgr()
3005                        .read()
3006                        .await
3007                        .tiers
3008                        .get(tier)
3009                        .expect("tier configuration survives restart")
3010                        .clone_with_credentials();
3011                    drop(set);
3012                    shutdown.cancel();
3013                    drop(store);
3014                    drop(ctx);
3015                    (ctx, store, shutdown) =
3016                        without_storage_class_env(build_isolated_test_store(temp_dir.path(), "tier-overwrite-restart", &[4]))
3017                            .await;
3018                    crate::bucket::metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
3019                    {
3020                        let manager = ctx.tier_config_mgr();
3021                        let mut manager = manager.write().await;
3022                        manager.tiers.insert(tier.to_string(), tier_config);
3023                        manager
3024                            .install_test_driver(tier, Box::new(backend.clone()))
3025                            .expect("rebind the same remote destination after restart");
3026                    }
3027                    let set = store.pools[0].get_disks_by_key(object);
3028                    // A deferred first cleanup must retain its durable owner
3029                    // until a later recovery scan can retry the operation.
3030                    backend.set_remove_failure(true);
3031                    ExpiryState::resize_workers(1, Arc::clone(&store)).await;
3032                    let recovered = recover_tier_free_versions(Arc::clone(&store), 100, None, None)
3033                        .await
3034                        .expect("recover persisted cleanup owner");
3035                    assert!(recovered.enqueued >= 1);
3036                    wait_for_expiry_workers_idle(&store).await;
3037                    assert!(backend.contains(&remote).await, "failed cleanup must retain remote bytes");
3038                    assert_eq!(backend.remove_count().await, removed_before);
3039                    backend.set_remove_failure(false);
3040                    // This fixture starts expiry workers without the runtime's
3041                    // recovery loop, so drive its durable rescan explicitly.
3042                    eprintln!("Recovering overwrite cleanup: state={state:?}, suspended={suspended}, copy={self_copy}");
3043                    wait_for_tier_free_version_recovery(Arc::clone(&store), &backend, removed_before + 1).await;
3044                    let versions = set
3045                        .load_file_info_versions_exact(&bucket, object)
3046                        .await
3047                        .expect("read cleanup progress")
3048                        .expect("new object must survive cleanup");
3049                    assert!(
3050                        versions
3051                            .versions
3052                            .iter()
3053                            .chain(versions.free_versions.iter())
3054                            .all(|fi| !fi.tier_free_version()),
3055                        "cleanup must remove its owner: {state:?}, suspended={suspended}, copy={self_copy}"
3056                    );
3057                    assert!(!backend.contains(&remote).await);
3058                    assert_eq!(backend.remove_count().await, removed_before + 1, "one successful remote DELETE per owner");
3059                    assert_eq!(backend.remove_versions().await.last(), Some(&(remote.clone(), version.to_string())));
3060                    let mut reader = store
3061                        .get_object_reader(&bucket, object, None, HeaderMap::new(), &options)
3062                        .await
3063                        .expect("replacement remains readable");
3064                    let mut actual = Vec::new();
3065                    reader.stream.read_to_end(&mut actual).await.expect("read replacement bytes");
3066                    assert_eq!(actual, expected);
3067                    let current = store
3068                        .get_object_info(&bucket, object, &options)
3069                        .await
3070                        .expect("replacement metadata");
3071                    assert_eq!(current.user_defined.get("content-type").map(String::as_str), Some("text/plain"));
3072                    assert_eq!(current.user_defined.get("x-amz-meta-replacement").map(String::as_str), Some("kept"));
3073                    assert!(current.transitioned_object.status.is_empty());
3074                }
3075            }
3076        }
3077        shutdown.cancel();
3078    }
3079
3080    #[cfg(feature = "test-util")]
3081    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
3082    #[serial_test::serial(storage_class_env)]
3083    async fn copy_object_immediately_reads_small_completed_multipart_source() {
3084        let temp_dir = tempfile::tempdir().expect("create small multipart copy store dir");
3085        let (_ctx, store, shutdown) =
3086            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "small-multipart-copy", &[1])).await;
3087        crate::bucket::metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
3088
3089        let bucket = format!("small-multipart-copy-{}", Uuid::new_v4());
3090        let source_object = "docker/registry/v2/repositories/example/_uploads/upload-id/data";
3091        let target_object = "docker/registry/v2/blobs/sha256/c0/digest/data";
3092        let payload = vec![0xAB; 273];
3093
3094        store
3095            .make_bucket(&bucket, &MakeBucketOptions::default())
3096            .await
3097            .expect("create bucket for small multipart copy");
3098        let upload = store
3099            .new_multipart_upload(&bucket, source_object, &ObjectOptions::default())
3100            .await
3101            .expect("create source multipart upload");
3102        let mut part_reader = PutObjReader::from_vec(payload.clone());
3103        let part = store
3104            .put_object_part(&bucket, source_object, &upload.upload_id, 1, &mut part_reader, &ObjectOptions::default())
3105            .await
3106            .expect("stage small multipart source part");
3107        let completed = store
3108            .clone()
3109            .complete_multipart_upload(
3110                &bucket,
3111                source_object,
3112                &upload.upload_id,
3113                vec![crate::storage_api_contracts::multipart::CompletePart {
3114                    part_num: part.part_num,
3115                    etag: part.etag,
3116                    ..Default::default()
3117                }],
3118                &ObjectOptions::default(),
3119            )
3120            .await
3121            .expect("complete the small multipart source");
3122        assert_eq!(completed.get_actual_size().expect("completed object logical size"), payload.len() as i64);
3123
3124        let source_reader = store
3125            .get_object_reader(&bucket, source_object, None, HeaderMap::new(), &ObjectOptions::default())
3126            .await
3127            .expect("completed multipart source should be immediately readable");
3128        let mut copy_info = source_reader.object_info.clone();
3129        let actual_size = copy_info.get_actual_size().expect("copy source logical size should resolve");
3130        assert_eq!(actual_size, payload.len() as i64);
3131        let copy_reader = rustfs_rio::HashReader::from_stream(source_reader.stream, actual_size, actual_size, None, None, false)
3132            .expect("copy source hash reader should build");
3133        copy_info.put_object_reader = Some(PutObjReader::new(copy_reader));
3134
3135        store
3136            .copy_object(
3137                &bucket,
3138                source_object,
3139                &bucket,
3140                target_object,
3141                &mut copy_info,
3142                &ObjectOptions::default(),
3143                &ObjectOptions::default(),
3144            )
3145            .await
3146            .expect("CopyObject should accept a freshly completed multipart source");
3147
3148        let mut target_reader = store
3149            .get_object_reader(&bucket, target_object, None, HeaderMap::new(), &ObjectOptions::default())
3150            .await
3151            .expect("copied target should be readable");
3152        let mut target_body = Vec::new();
3153        target_reader
3154            .stream
3155            .read_to_end(&mut target_body)
3156            .await
3157            .expect("target body should stream");
3158        assert_eq!(target_body, payload);
3159        shutdown.cancel();
3160    }
3161
3162    /// rustfs/rustfs#7674: a bucket carrying a legacy snapshot-protocol quota
3163    /// (written before durable reservations existed, so `quota.json` has no
3164    /// `reservation_protocol`) must accept a cross-key CopyObject whose
3165    /// destination options carry the handler's quota admission. The storage
3166    /// layer fails closed with `PartMissingOrCorrupt` when a quota-enforced
3167    /// bucket sees a write without admission, so dropping the admission while
3168    /// rebuilding the destination options turns every copy into a
3169    /// deterministic "part missing or corrupt" failure even though the source
3170    /// object is perfectly readable.
3171    #[cfg(feature = "test-util")]
3172    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
3173    #[serial_test::serial(storage_class_env)]
3174    async fn copy_object_forwards_quota_admission_on_legacy_snapshot_quota_bucket() {
3175        let temp_dir = tempfile::tempdir().expect("create legacy quota copy store dir");
3176        let (ctx, store, shutdown) =
3177            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "legacy-quota-copy", &[1])).await;
3178        crate::bucket::metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
3179
3180        let bucket = format!("legacy-quota-copy-{}", Uuid::new_v4());
3181        let source_object = "docker/registry/v2/repositories/example/_uploads/upload-id/data";
3182        let target_object = "docker/registry/v2/blobs/sha256/a5/digest/data";
3183        let payload = vec![0x5A; 8178];
3184        let quota_limit = 1u64 << 30;
3185
3186        store
3187            .make_bucket(&bucket, &MakeBucketOptions::default())
3188            .await
3189            .expect("create bucket for legacy quota copy");
3190        // Legacy quota shape: no `reservation_protocol`, so the storage layer
3191        // requires the handler-supplied snapshot admission on every write.
3192        let legacy_quota = format!(r#"{{"quota":{quota_limit},"quota_type":"Hard"}}"#);
3193        crate::bucket::metadata_sys::update_in(
3194            &ctx,
3195            &bucket,
3196            crate::bucket::metadata::BUCKET_QUOTA_CONFIG_FILE,
3197            legacy_quota.into_bytes(),
3198        )
3199        .await
3200        .expect("persist legacy snapshot quota");
3201        let (quota, _, _) = crate::bucket::metadata_sys::get_quota_config_and_incarnation_from_disk_in(&ctx, &bucket)
3202            .await
3203            .expect("legacy quota should load");
3204        let quota = quota.expect("legacy quota must be persisted");
3205        assert_eq!(quota.quota, Some(quota_limit));
3206        assert!(!quota.uses_durable_reservations(), "fixture must stay on the snapshot protocol");
3207
3208        let mut write_opts = ObjectOptions::default();
3209        assert!(write_opts.set_quota_admission(0, quota_limit));
3210
3211        let upload = store
3212            .new_multipart_upload(&bucket, source_object, &write_opts)
3213            .await
3214            .expect("create source multipart upload");
3215        let mut part_reader = PutObjReader::from_vec(payload.clone());
3216        let part = store
3217            .put_object_part(&bucket, source_object, &upload.upload_id, 1, &mut part_reader, &write_opts)
3218            .await
3219            .expect("stage multipart source part");
3220        store
3221            .clone()
3222            .complete_multipart_upload(
3223                &bucket,
3224                source_object,
3225                &upload.upload_id,
3226                vec![crate::storage_api_contracts::multipart::CompletePart {
3227                    part_num: part.part_num,
3228                    etag: part.etag,
3229                    ..Default::default()
3230                }],
3231                &write_opts,
3232            )
3233            .await
3234            .expect("complete the multipart source under the legacy quota");
3235
3236        let source_reader = store
3237            .get_object_reader(&bucket, source_object, None, HeaderMap::new(), &ObjectOptions::default())
3238            .await
3239            .expect("completed multipart source should be readable");
3240        let mut copy_info = source_reader.object_info.clone();
3241        let actual_size = copy_info.get_actual_size().expect("copy source logical size should resolve");
3242        assert_eq!(actual_size, payload.len() as i64);
3243        let copy_reader = rustfs_rio::HashReader::from_stream(source_reader.stream, actual_size, actual_size, None, None, false)
3244            .expect("copy source hash reader should build");
3245        copy_info.put_object_reader = Some(PutObjReader::new(copy_reader));
3246
3247        let mut dst_opts = ObjectOptions::default();
3248        assert!(dst_opts.set_quota_admission(payload.len() as u64, quota_limit));
3249        store
3250            .copy_object(
3251                &bucket,
3252                source_object,
3253                &bucket,
3254                target_object,
3255                &mut copy_info,
3256                &ObjectOptions::default(),
3257                &dst_opts,
3258            )
3259            .await
3260            .expect("CopyObject must forward the handler quota admission to the destination write");
3261
3262        let mut target_reader = store
3263            .get_object_reader(&bucket, target_object, None, HeaderMap::new(), &ObjectOptions::default())
3264            .await
3265            .expect("copied target should be readable");
3266        let mut target_body = Vec::new();
3267        target_reader
3268            .stream
3269            .read_to_end(&mut target_body)
3270            .await
3271            .expect("target body should stream");
3272        assert_eq!(target_body, payload);
3273        shutdown.cancel();
3274    }
3275
3276    #[cfg(feature = "test-util")]
3277    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
3278    #[serial_test::serial(storage_class_env)]
3279    async fn unfinished_multipart_upload_is_not_copy_source_readable() {
3280        let temp_dir = tempfile::tempdir().expect("create unfinished multipart copy store dir");
3281        let (_ctx, store, shutdown) =
3282            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "unfinished-multipart-copy", &[1])).await;
3283        crate::bucket::metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
3284
3285        let bucket = format!("unfinished-multipart-copy-{}", Uuid::new_v4());
3286        let source_object = "docker/registry/v2/repositories/example/_uploads/upload-id/data";
3287        let payload = vec![0xCD; 273];
3288
3289        store
3290            .make_bucket(&bucket, &MakeBucketOptions::default())
3291            .await
3292            .expect("create bucket for unfinished multipart copy source");
3293        let upload = store
3294            .new_multipart_upload(&bucket, source_object, &ObjectOptions::default())
3295            .await
3296            .expect("create source multipart upload");
3297        let mut part_reader = PutObjReader::from_vec(payload);
3298        store
3299            .put_object_part(&bucket, source_object, &upload.upload_id, 1, &mut part_reader, &ObjectOptions::default())
3300            .await
3301            .expect("stage unfinished multipart source part");
3302
3303        let source_err = match store
3304            .get_object_reader(&bucket, source_object, None, HeaderMap::new(), &ObjectOptions::default())
3305            .await
3306        {
3307            Ok(_) => panic!("an uncompleted multipart upload must not be readable as a CopyObject source"),
3308            Err(err) => err,
3309        };
3310        assert!(
3311            matches!(
3312                source_err,
3313                StorageError::ObjectNotFound(_, _) | StorageError::FileNotFound | StorageError::VersionNotFound(_, _, _)
3314            ),
3315            "unexpected unfinished multipart source error: {source_err:?}"
3316        );
3317
3318        shutdown.cancel();
3319    }
3320
3321    #[tokio::test]
3322    #[serial_test::serial(storage_class_env)]
3323    async fn pool_metadata_preflight_recovery_preserves_single_and_multi_pool_public_mutations() {
3324        for layout in [vec![4], vec![4, 4]] {
3325            let temp_dir = tempfile::tempdir().unwrap();
3326            let (_ctx, store, shutdown) =
3327                without_storage_class_env(build_isolated_test_store(temp_dir.path(), "pool-meta-retry", &layout)).await;
3328            crate::bucket::metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
3329            let bucket = format!("pool-meta-retry-{}", Uuid::new_v4());
3330            store.make_bucket(&bucket, &MakeBucketOptions::default()).await.unwrap();
3331            let mut saved_disks = Vec::new();
3332            for set in &store.pools[0].disk_set {
3333                let mut disks = set.disks.write().await;
3334                let count = disks.len();
3335                saved_disks.push((set.clone(), std::mem::replace(&mut *disks, vec![None; count])));
3336            }
3337            let indices = (0..layout.len()).collect::<Vec<_>>();
3338            let err = store.save_current_pool_meta_for_test(&indices).await.unwrap_err();
3339            assert_eq!(
3340                err.pool_metadata_failure().unwrap().kind,
3341                crate::error::PoolMetadataFailure::ReadUnavailable
3342            );
3343            for (set, disks) in saved_disks {
3344                *set.disks.write().await = disks;
3345            }
3346            store.save_current_pool_meta_for_test(&indices).await.unwrap();
3347            assert!(store.pool_meta_writes_ready().await);
3348
3349            let payload = b"pool metadata recovery payload".to_vec();
3350            store
3351                .put_object(&bucket, "put", &mut PutObjReader::from_vec(payload.clone()), &ObjectOptions::default())
3352                .await
3353                .unwrap();
3354            let mut reader = store
3355                .get_object_reader(&bucket, "put", None, HeaderMap::new(), &ObjectOptions::default())
3356                .await
3357                .unwrap();
3358            let mut actual = Vec::new();
3359            reader.stream.read_to_end(&mut actual).await.unwrap();
3360            assert_eq!(actual, payload);
3361            drop(reader);
3362            store.delete_object(&bucket, "put", ObjectOptions::default()).await.unwrap();
3363            assert!(crate::error::is_err_object_not_found(
3364                &store
3365                    .get_object_info(&bucket, "put", &ObjectOptions::default())
3366                    .await
3367                    .unwrap_err()
3368            ));
3369
3370            let upload = store
3371                .new_multipart_upload(&bucket, "multipart", &ObjectOptions::default())
3372                .await
3373                .unwrap();
3374            let part = store
3375                .put_object_part(
3376                    &bucket,
3377                    "multipart",
3378                    &upload.upload_id,
3379                    1,
3380                    &mut PutObjReader::from_vec(payload.clone()),
3381                    &ObjectOptions::default(),
3382                )
3383                .await
3384                .unwrap();
3385            store
3386                .clone()
3387                .complete_multipart_upload(
3388                    &bucket,
3389                    "multipart",
3390                    &upload.upload_id,
3391                    vec![crate::storage_api_contracts::multipart::CompletePart {
3392                        part_num: part.part_num,
3393                        etag: part.etag,
3394                        ..Default::default()
3395                    }],
3396                    &ObjectOptions::default(),
3397                )
3398                .await
3399                .unwrap();
3400            let mut reader = store
3401                .get_object_reader(&bucket, "multipart", None, HeaderMap::new(), &ObjectOptions::default())
3402                .await
3403                .unwrap();
3404            actual.clear();
3405            reader.stream.read_to_end(&mut actual).await.unwrap();
3406            assert_eq!(actual, payload);
3407            drop(reader);
3408            let upload = store
3409                .new_multipart_upload(&bucket, "abort", &ObjectOptions::default())
3410                .await
3411                .unwrap();
3412            store
3413                .abort_multipart_upload(&bucket, "abort", &upload.upload_id, &ObjectOptions::default())
3414                .await
3415                .unwrap();
3416            assert!(store.pool_meta_writes_ready().await);
3417            shutdown.cancel();
3418        }
3419    }
3420
3421    #[cfg(feature = "test-util")]
3422    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
3423    #[serial_test::serial(storage_class_env)]
3424    async fn complete_multipart_waits_for_tail_rename_before_copy_source_visibility() {
3425        let temp_dir = tempfile::tempdir().expect("create early-ack multipart copy store dir");
3426        let (_ctx, store, shutdown) =
3427            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "early-ack-multipart-copy", &[4])).await;
3428        crate::bucket::metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
3429
3430        let bucket = format!("early-ack-multipart-copy-{}", Uuid::new_v4());
3431        let source_object = "docker/registry/v2/repositories/example/_uploads/upload-id/data";
3432        let target_object = "docker/registry/v2/blobs/sha256/c0/digest/data";
3433        let payload = vec![0xCD; 273];
3434
3435        store
3436            .make_bucket(&bucket, &MakeBucketOptions::default())
3437            .await
3438            .expect("create bucket for early-ack multipart copy");
3439        let upload = store
3440            .new_multipart_upload(&bucket, source_object, &ObjectOptions::default())
3441            .await
3442            .expect("create source multipart upload");
3443        let mut part_reader = PutObjReader::from_vec(payload.clone());
3444        let part = store
3445            .put_object_part(&bucket, source_object, &upload.upload_id, 1, &mut part_reader, &ObjectOptions::default())
3446            .await
3447            .expect("stage small multipart source part");
3448        let completed_parts = vec![crate::storage_api_contracts::multipart::CompletePart {
3449            part_num: part.part_num,
3450            etag: part.etag,
3451            ..Default::default()
3452        }];
3453
3454        temp_env::async_with_vars([(crate::set_disk::ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async {
3455            let rename_tasks = crate::set_disk::rename_fanout_barrier::observe_tasks(source_object);
3456            let rename_barrier = crate::set_disk::rename_fanout_barrier::arm(
3457                source_object,
3458                0,
3459                crate::set_disk::rename_fanout_barrier::PHASE_RENAME,
3460            );
3461            let complete_store = Arc::clone(&store);
3462            let complete_bucket = bucket.clone();
3463            let complete_upload_id = upload.upload_id.clone();
3464            let mut complete = tokio::spawn(async move {
3465                complete_store
3466                    .complete_multipart_upload(
3467                        &complete_bucket,
3468                        source_object,
3469                        &complete_upload_id,
3470                        completed_parts,
3471                        &ObjectOptions::default(),
3472                    )
3473                    .await
3474            });
3475            tokio::time::timeout(Duration::from_secs(30), rename_barrier.wait_until_paused())
3476                .await
3477                .expect("multipart completion should pause one tail disk during rename");
3478            assert!(
3479                rename_tasks.running() >= 1,
3480                "the paused multipart tail disk must remain in flight before completion returns"
3481            );
3482            assert!(
3483                tokio::time::timeout(Duration::from_millis(100), &mut complete).await.is_err(),
3484                "CompleteMultipartUpload must not return while a copy source rename tail is still pending"
3485            );
3486            rename_barrier.release();
3487            complete
3488                .await
3489                .expect("multipart completion task should join")
3490                .expect("multipart completion should return after every rename tail finishes");
3491
3492            let source_reader = store
3493                .get_object_reader(&bucket, source_object, None, HeaderMap::new(), &ObjectOptions::default())
3494                .await
3495                .expect("completed multipart source should be immediately readable after success");
3496            let mut copy_info = source_reader.object_info.clone();
3497            let actual_size = copy_info.get_actual_size().expect("copy source logical size should resolve");
3498            assert_eq!(actual_size, payload.len() as i64);
3499            let copy_reader =
3500                rustfs_rio::HashReader::from_stream(source_reader.stream, actual_size, actual_size, None, None, false)
3501                    .expect("copy source hash reader should build");
3502            copy_info.put_object_reader = Some(PutObjReader::new(copy_reader));
3503
3504            store
3505                .copy_object(
3506                    &bucket,
3507                    source_object,
3508                    &bucket,
3509                    target_object,
3510                    &mut copy_info,
3511                    &ObjectOptions::default(),
3512                    &ObjectOptions::default(),
3513                )
3514                .await
3515                .expect("CopyObject should accept a freshly completed multipart source");
3516        })
3517        .await;
3518
3519        let mut target_reader = store
3520            .get_object_reader(&bucket, target_object, None, HeaderMap::new(), &ObjectOptions::default())
3521            .await
3522            .expect("copied target should be readable after tail release");
3523        let mut target_body = Vec::new();
3524        target_reader
3525            .stream
3526            .read_to_end(&mut target_body)
3527            .await
3528            .expect("target body should stream");
3529        assert_eq!(target_body, payload);
3530        shutdown.cancel();
3531    }
3532
3533    #[cfg(feature = "test-util")]
3534    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
3535    #[serial_test::serial(storage_class_env)]
3536    async fn early_ack_put_tails_block_scanner_publication_until_all_renames_finish() {
3537        use crate::storage_api_contracts::namespace::NamespaceLocking as _;
3538
3539        let temp_dir = tempfile::tempdir().expect("create scanner PUT tail store dir");
3540        let (ctx, store, shutdown) =
3541            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "scanner-put-tails", &[4])).await;
3542        crate::bucket::metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
3543        let bucket = format!("scanner-put-tails-{}", Uuid::new_v4());
3544        store
3545            .make_bucket(&bucket, &MakeBucketOptions::default())
3546            .await
3547            .expect("create scanner PUT tail bucket");
3548        let set = &store.pools[0].disk_set[0];
3549        let objects = [("scanner-tail-a", vec![0xA1; 273]), ("scanner-tail-b", vec![0xB2; 379])];
3550
3551        temp_env::async_with_vars([(crate::set_disk::ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async {
3552            let (active, blocked, movement_generation) = store.scanner_data_movement_activity().await;
3553            assert!(!active && !blocked);
3554            assert!(ctx.scanner_publication_state_allowed(), "the set admission cache should start allowed");
3555            let (old_lease, _) = store
3556                .acquire_scanner_publication_lease(movement_generation, crate::runtime::instance::SCANNER_PUBLICATION_LEASE_TTL)
3557                .await
3558                .expect("publication lease should be admitted before either PUT starts");
3559
3560            let barriers: Vec<_> = objects
3561                .iter()
3562                .map(|(object, _)| {
3563                    crate::set_disk::rename_fanout_barrier::arm(object, 0, crate::set_disk::rename_fanout_barrier::PHASE_RENAME)
3564                })
3565                .collect();
3566            let trackers: Vec<_> = objects
3567                .iter()
3568                .map(|(object, _)| crate::set_disk::rename_fanout_barrier::observe_tasks(object))
3569                .collect();
3570            let puts: Vec<_> = objects
3571                .iter()
3572                .map(|(object, body)| {
3573                    let put_store = Arc::clone(&store);
3574                    let put_bucket = bucket.clone();
3575                    let object = *object;
3576                    let body = body.clone();
3577                    tokio::spawn(async move {
3578                        let mut reader = PutObjReader::from_vec(body);
3579                        put_store
3580                            .put_object(&put_bucket, object, &mut reader, &ObjectOptions::default())
3581                            .await
3582                    })
3583                })
3584                .collect();
3585            let committed = tokio::time::timeout(Duration::from_secs(30), async {
3586                for barrier in &barriers {
3587                    barrier.wait_until_paused().await;
3588                }
3589                let mut committed = Vec::with_capacity(puts.len());
3590                for put in puts {
3591                    committed.push(
3592                        put.await
3593                            .expect("early-ACK PUT task should join while its tail is paused")
3594                            .expect("root PUT should return after quorum without waiting for its tail"),
3595                    );
3596                }
3597                committed
3598            })
3599            .await
3600            .expect("both root PUTs must quorum-ACK while their tail disks remain paused");
3601
3602            assert!(trackers.iter().all(|tracker| tracker.running() >= 1));
3603            assert!(ctx.namespace_commits_pending());
3604            assert!(
3605                ctx.scanner_publication_state_allowed(),
3606                "pending PUT tails must not disable scanner namespace walks"
3607            );
3608            let (active, blocked, observed_movement_generation) = store.scanner_data_movement_activity().await;
3609            assert!(!active, "ordinary PUT tails are not decommission or rebalance work");
3610            assert!(!blocked, "ordinary PUT tails must not block the movement-only scan baseline");
3611            assert_eq!(observed_movement_generation, movement_generation);
3612            assert!(store.scanner_data_usage_publication_blocked().await);
3613            assert!(store.scanner_data_usage_publication_admission_guard().await.is_some());
3614            assert!(set.scanner_data_usage_publication_admission_guard().await.is_some());
3615            for error in [
3616                store
3617                    .acquire_scanner_publication_lease(
3618                        movement_generation,
3619                        crate::runtime::instance::SCANNER_PUBLICATION_LEASE_TTL,
3620                    )
3621                    .await
3622                    .expect_err("a new remote publication lease must reject pending PUT tails"),
3623                store
3624                    .validate_scanner_publication_lease(old_lease, movement_generation)
3625                    .await
3626                    .expect_err("an existing remote lease must not bypass pending PUT tails"),
3627                store
3628                    .acquire_scanner_publication_lease_guard(old_lease)
3629                    .await
3630                    .expect_err("target-side publication admission must reject pending PUT tails"),
3631            ] {
3632                assert!(
3633                    error.to_string().contains("blocked"),
3634                    "publication must fail because of active tails: {error}"
3635                );
3636            }
3637            store.release_scanner_publication_lease(old_lease).await;
3638
3639            for (index, barrier) in barriers.iter().enumerate() {
3640                let commit_generation = ctx.namespace_commit_generation();
3641                let namespace_generation = store.scanner_namespace_mutation_generation();
3642                barrier.release();
3643                tokio::time::timeout(Duration::from_secs(30), async {
3644                    while trackers[index].running() != 0 || ctx.namespace_commit_generation() <= commit_generation {
3645                        tokio::task::yield_now().await;
3646                    }
3647                    if index + 1 == barriers.len() {
3648                        while ctx.namespace_commits_pending() {
3649                            tokio::task::yield_now().await;
3650                        }
3651                    }
3652                })
3653                .await
3654                .expect("released tail must drain and publish its terminal namespace generation");
3655                assert!(store.scanner_namespace_mutation_generation() > namespace_generation);
3656                let pending = index + 1 < barriers.len();
3657                assert_eq!(ctx.namespace_commits_pending(), pending);
3658                assert_eq!(store.scanner_data_usage_publication_blocked().await, pending);
3659                assert!(!store.scanner_data_movement_activity().await.1);
3660                assert!(store.scanner_data_usage_publication_admission_guard().await.is_some());
3661                assert!(set.scanner_data_usage_publication_admission_guard().await.is_some());
3662            }
3663
3664            let (lease, generation) = store
3665                .acquire_scanner_publication_lease(movement_generation, crate::runtime::instance::SCANNER_PUBLICATION_LEASE_TTL)
3666                .await
3667                .expect("remote publication lease should resume after both tails drain");
3668            store
3669                .validate_scanner_publication_lease(lease, generation)
3670                .await
3671                .expect("a resumed remote publication lease should validate");
3672            drop(
3673                store
3674                    .acquire_scanner_publication_lease_guard(lease)
3675                    .await
3676                    .expect("target-side publication admission should resume after both tails drain"),
3677            );
3678            assert!(store.release_scanner_publication_lease(lease).await);
3679
3680            let disks = set.disk_inventory().await;
3681            assert_eq!(disks.len(), 4);
3682            for ((object, body), committed) in objects.iter().zip(&committed) {
3683                let logical_size = i64::try_from(body.len()).expect("fixture payload size should fit i64");
3684                let etag = committed.etag.as_ref().expect("root PUT should return a committed ETag");
3685                for (disk_index, disk) in disks.iter().enumerate() {
3686                    let file_info = disk
3687                        .as_ref()
3688                        .expect("every fixture disk should remain online")
3689                        .read_version(
3690                            "",
3691                            &bucket,
3692                            object,
3693                            "",
3694                            &crate::disk::ReadOptions {
3695                                read_data: true,
3696                                ..Default::default()
3697                            },
3698                        )
3699                        .await
3700                        .unwrap_or_else(|err| panic!("disk {disk_index} should publish {object} after its tail finishes: {err}"));
3701                    assert_eq!(file_info.size, logical_size);
3702                    assert_eq!(file_info.metadata.get(http::header::ETAG.as_str()), Some(etag));
3703                    assert!(
3704                        file_info.inline_data(),
3705                        "small fixture payloads should have an inline shard on every disk"
3706                    );
3707                    let inline_data = file_info.data.as_ref().expect("every disk should retain its inline shard");
3708                    let erasure = crate::erasure::coding::Erasure::try_new_with_options(
3709                        file_info.erasure.data_blocks,
3710                        file_info.erasure.parity_blocks,
3711                        file_info.erasure.block_size,
3712                        file_info.uses_legacy_checksum,
3713                    )
3714                    .expect("persisted erasure geometry should be valid");
3715                    let shard_size =
3716                        usize::try_from(erasure.shard_file_size(logical_size)).expect("fixture shard size should fit usize");
3717                    crate::erasure::coding::bitrot_verify(
3718                        Cursor::new(inline_data.clone()),
3719                        inline_data.len(),
3720                        shard_size,
3721                        rustfs_utils::HashAlgorithm::HighwayHash256S,
3722                        erasure.shard_size(),
3723                    )
3724                    .await
3725                    .unwrap_or_else(|err| panic!("disk {disk_index} should retain a complete valid shard for {object}: {err}"));
3726                }
3727                let mut reader = store
3728                    .get_object_reader(&bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
3729                    .await
3730                    .expect("fully drained PUT should be readable");
3731                let mut actual = Vec::new();
3732                reader.stream.read_to_end(&mut actual).await.expect("PUT body should drain");
3733                assert_eq!(&actual, body);
3734            }
3735
3736            let generation_before_internal_put = ctx.namespace_commit_generation();
3737            let internal_object = "scanner-tail-regression/internal-metadata";
3738            let internal_body = b"scanner metadata must not invalidate its own publication";
3739            let mut internal_reader = PutObjReader::from_vec(internal_body.to_vec());
3740            store
3741                .put_object(RUSTFS_META_BUCKET, internal_object, &mut internal_reader, &ObjectOptions::default())
3742                .await
3743                .expect("internal metadata PUT should commit without scanner self-invalidation");
3744            let internal_lock = set
3745                .new_ns_lock(RUSTFS_META_BUCKET, internal_object)
3746                .await
3747                .expect("internal metadata tail lock should be available");
3748            drop(
3749                internal_lock
3750                    .get_write_lock(Duration::from_secs(30))
3751                    .await
3752                    .expect("internal metadata tail should drain"),
3753            );
3754            assert_eq!(ctx.namespace_commit_generation(), generation_before_internal_put);
3755            assert!(!ctx.namespace_commits_pending());
3756            assert!(store.scanner_data_usage_publication_admission_guard().await.is_some());
3757            assert!(set.scanner_data_usage_publication_admission_guard().await.is_some());
3758            let mut internal_reader = store
3759                .get_object_reader(RUSTFS_META_BUCKET, internal_object, None, HeaderMap::new(), &ObjectOptions::default())
3760                .await
3761                .expect("internal metadata should remain readable");
3762            let mut actual = Vec::new();
3763            internal_reader
3764                .stream
3765                .read_to_end(&mut actual)
3766                .await
3767                .expect("internal metadata body should drain");
3768            assert_eq!(actual, internal_body);
3769        })
3770        .await;
3771        shutdown.cancel();
3772    }
3773
3774    #[cfg(feature = "test-util")]
3775    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
3776    #[serial_test::serial(storage_class_env)]
3777    async fn cancelled_early_ack_put_keeps_scanner_publication_blocked_until_tail_finishes() {
3778        let temp_dir = tempfile::tempdir().expect("create cancelled scanner PUT tail store dir");
3779        let (ctx, store, shutdown) =
3780            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "scanner-cancelled-put-tail", &[4])).await;
3781        crate::bucket::metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
3782        let bucket = format!("scanner-cancelled-put-tail-{}", Uuid::new_v4());
3783        let object = "scanner-cancelled-tail";
3784        let body = vec![0xC3; 273];
3785        store
3786            .make_bucket(&bucket, &MakeBucketOptions::default())
3787            .await
3788            .expect("create cancelled scanner PUT tail bucket");
3789
3790        temp_env::async_with_vars([(crate::set_disk::ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async {
3791            let tracker = crate::set_disk::rename_fanout_barrier::observe_tasks(object);
3792            let tail =
3793                crate::set_disk::rename_fanout_barrier::arm(object, 0, crate::set_disk::rename_fanout_barrier::PHASE_RENAME);
3794            let quorum = crate::set_disk::PutObjectCommitBarrier::install(
3795                &bucket,
3796                object,
3797                crate::set_disk::PutObjectCommitPause::AfterRenameQuorum,
3798            );
3799            let handoff = crate::set_disk::PutObjectCommitBarrier::install(
3800                &bucket,
3801                object,
3802                crate::set_disk::PutObjectCommitPause::AfterRenameHandoff,
3803            );
3804            let put_store = Arc::clone(&store);
3805            let put_bucket = bucket.clone();
3806            let put_body = body.clone();
3807            let put = tokio::spawn(async move {
3808                let mut reader = PutObjReader::from_vec(put_body);
3809                put_store
3810                    .put_object(&put_bucket, object, &mut reader, &ObjectOptions::default())
3811                    .await
3812            });
3813            tokio::time::timeout(Duration::from_secs(30), tail.wait_until_paused())
3814                .await
3815                .expect("cancelled PUT should pause one disk before rename");
3816            quorum.wait_until_paused().await;
3817            put.abort();
3818            assert!(
3819                put.await
3820                    .expect_err("caller should be cancelled after rename quorum")
3821                    .is_cancelled()
3822            );
3823            quorum.release();
3824            handoff.wait_until_paused().await;
3825            assert!(tracker.running() >= 1);
3826            assert!(ctx.namespace_commits_pending());
3827            assert!(!store.scanner_data_movement_activity().await.1);
3828            assert!(store.scanner_data_usage_publication_blocked().await);
3829            assert!(store.scanner_data_usage_publication_admission_guard().await.is_some());
3830            assert!(
3831                store.pools[0].disk_set[0]
3832                    .scanner_data_usage_publication_admission_guard()
3833                    .await
3834                    .is_some()
3835            );
3836            let generation = store.scanner_namespace_mutation_generation();
3837
3838            handoff.release();
3839            tail.release();
3840            tokio::time::timeout(Duration::from_secs(30), async {
3841                while tracker.running() != 0 || ctx.namespace_commits_pending() {
3842                    tokio::task::yield_now().await;
3843                }
3844            })
3845            .await
3846            .expect("cancelled request's detached fanout must release scanner admission after finishing");
3847            assert!(store.scanner_namespace_mutation_generation() > generation);
3848            assert!(!store.scanner_data_usage_publication_blocked().await);
3849            assert!(store.scanner_data_usage_publication_admission_guard().await.is_some());
3850            for (disk_index, disk) in store.pools[0].disk_set[0].disk_inventory().await.iter().enumerate() {
3851                let file_info = disk
3852                    .as_ref()
3853                    .expect("cancelled PUT fixture disk should remain online")
3854                    .read_version("", &bucket, object, "", &crate::disk::ReadOptions::default())
3855                    .await
3856                    .unwrap_or_else(|err| panic!("cancelled PUT must still publish on disk {disk_index}: {err}"));
3857                assert_eq!(file_info.size, i64::try_from(body.len()).expect("fixture body size should fit i64"));
3858            }
3859            let mut reader = store
3860                .get_object_reader(&bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
3861                .await
3862                .expect("a cancelled caller must not discard its quorum-committed object");
3863            let mut actual = Vec::new();
3864            reader
3865                .stream
3866                .read_to_end(&mut actual)
3867                .await
3868                .expect("cancelled PUT body should drain");
3869            assert_eq!(actual, body);
3870        })
3871        .await;
3872        shutdown.cancel();
3873    }
3874
3875    #[cfg(feature = "test-util")]
3876    #[test]
3877    #[serial_test::serial(storage_class_env)]
3878    fn data_movement_multipart_part_staging_holds_no_publication_lock_or_tier_lease() {
3879        run_large_stack_async_test(
3880            "multipart-part-staging-publication-fence",
3881            data_movement_multipart_part_staging_holds_no_publication_lock_or_tier_lease_case,
3882        );
3883    }
3884
3885    #[cfg(feature = "test-util")]
3886    async fn data_movement_multipart_part_staging_holds_no_publication_lock_or_tier_lease_case() {
3887        let temp_dir = tempfile::tempdir().expect("create multipart staging-fence store dir");
3888        let (ctx, store, shutdown) =
3889            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "mpu-staging-publication", &[4, 4])).await;
3890        crate::bucket::metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
3891
3892        let bucket = format!("mpu-staging-publication-{}", Uuid::new_v4());
3893        let object = "remote-owned.bin";
3894        let payload = b"multipart part staging must not publish an object".to_vec();
3895        store
3896            .make_bucket(&bucket, &MakeBucketOptions::default())
3897            .await
3898            .expect("multipart staging test bucket should be created");
3899
3900        let tier_name = "MPU-STAGING-TIER";
3901        register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
3902        let lease = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
3903            .await
3904            .expect("mock tier generation should be available");
3905        let backend_identity = lease.backend_identity();
3906        drop(lease);
3907
3908        let mut source_metadata = HashMap::from([(
3909            "x-amz-restore".to_string(),
3910            "ongoing-request=\"false\", expiry-date=\"2099-01-01T00:00:00Z\"".to_string(),
3911        )]);
3912        for (suffix, value) in [
3913            (
3914                rustfs_utils::http::SUFFIX_TRANSITION_STATUS,
3915                crate::bucket::lifecycle::core::TRANSITION_COMPLETE.to_string(),
3916            ),
3917            (rustfs_utils::http::SUFFIX_TRANSITIONED_OBJECTNAME, "remote/mpu.bin".to_string()),
3918            (rustfs_utils::http::SUFFIX_TRANSITION_TIER, tier_name.to_string()),
3919            (rustfs_utils::http::SUFFIX_TRANSITIONED_VERSION_ID, "remote-mpu-version".to_string()),
3920            (
3921                rustfs_utils::http::SUFFIX_TRANSITIONED_VERSION_STATE,
3922                rustfs_filemeta::TransitionVersionState::Exact.as_str().to_string(),
3923            ),
3924            (
3925                rustfs_utils::http::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
3926                rustfs_utils::crypto::hex(backend_identity),
3927            ),
3928        ] {
3929            rustfs_utils::http::metadata_compat::insert_str(&mut source_metadata, suffix, value);
3930        }
3931        let mut source_body = PutObjReader::from_vec(payload.clone());
3932        store.pools[0]
3933            .put_object(
3934                &bucket,
3935                object,
3936                &mut source_body,
3937                &ObjectOptions {
3938                    user_defined: source_metadata,
3939                    ..Default::default()
3940                },
3941            )
3942            .await
3943            .expect("remote-owned source should be written to the source pool");
3944        let source = store.pools[0]
3945            .get_object_info(&bucket, object, &ObjectOptions::default())
3946            .await
3947            .expect("remote-owned source metadata should be readable");
3948        let publication = store
3949            .acquire_remote_tuple_publication_fence(&bucket, 0, &source, true)
3950            .await
3951            .expect("multipart migration should capture its publication capability");
3952        assert_eq!(
3953            TierConfigMgr::active_operation_lease_count(&ctx.tier_config_mgr(), tier_name).await,
3954            0,
3955            "capturing the commit-late capability must not pin the tier generation"
3956        );
3957
3958        let bucket_incarnation_id = store
3959            .bucket_incarnation_id(&bucket)
3960            .await
3961            .expect("bucket incarnation should resolve");
3962        mark_test_pool_decommissioning(&store, 0).await;
3963        let capacity_owner = test_decommission_capacity_owner(store.as_ref(), 0)
3964            .await
3965            .with_mutation_id(Uuid::new_v4());
3966        let upload_metadata = source.user_defined.as_ref().clone();
3967        let mut staging_opts = ObjectOptions {
3968            data_movement: true,
3969            src_pool_idx: 0,
3970            versioned: source.version_id.is_some(),
3971            version_id: source.version_id.map(|version_id| version_id.to_string()),
3972            mod_time: source.mod_time,
3973            user_defined: upload_metadata,
3974            expected_bucket_incarnation_id: Some(bucket_incarnation_id),
3975            ..Default::default()
3976        };
3977        let upload_identity = crate::data_movement::data_movement_upload_identity_from_options(&staging_opts);
3978        rustfs_utils::http::insert_str(
3979            &mut staging_opts.user_defined,
3980            rustfs_utils::http::SUFFIX_DATA_MOVEMENT_UPLOAD,
3981            upload_identity,
3982        );
3983        capacity_owner.apply_to(&mut staging_opts);
3984        let (upload, target_pool_idx, staged_incarnation_id) = store
3985            .handle_new_multipart_upload_with_pool_idx(&bucket, object, &staging_opts, None)
3986            .await
3987            .expect("target multipart upload should be staged");
3988        assert_eq!(target_pool_idx, 1, "the active pool must receive the staged upload");
3989        assert_eq!(staged_incarnation_id, Some(bucket_incarnation_id));
3990
3991        let barrier = crate::set_disk::MultipartCommitBarrier::install(
3992            &bucket,
3993            object,
3994            crate::set_disk::MultipartCommitPause::PutPartBeforeLockLost,
3995        );
3996        let part_store = Arc::clone(&store);
3997        let part_bucket = bucket.clone();
3998        let upload_id = upload.upload_id.clone();
3999        let part_payload = payload.clone();
4000        let part = tokio::spawn(async move {
4001            let mut reader = PutObjReader::from_vec(part_payload);
4002            let mut part_opts = ObjectOptions {
4003                data_movement: true,
4004                src_pool_idx: 0,
4005                part_number: Some(1),
4006                expected_bucket_incarnation_id: Some(bucket_incarnation_id),
4007                ..Default::default()
4008            };
4009            capacity_owner.apply_to(&mut part_opts);
4010            part_store
4011                .put_object_part_for_data_movement(1, &part_bucket, object, &upload_id, &mut reader, &part_opts)
4012                .await
4013        });
4014        barrier.wait_until_paused().await;
4015
4016        assert_eq!(
4017            TierConfigMgr::active_operation_lease_count(&ctx.tier_config_mgr(), tier_name).await,
4018            0,
4019            "UploadPart staging must not pin the tier generation"
4020        );
4021        let encoded = rustfs_utils::path::encode_dir_object(object);
4022        let mut proof_opts = ObjectOptions::default();
4023        let proof = tokio::time::timeout(
4024            Duration::from_millis(250),
4025            store.acquire_all_physical_object_read_locks("tier_delete_journal_recovery", &bucket, &encoded, &mut proof_opts),
4026        )
4027        .await
4028        .expect("UploadPart staging must not hold any object publication write lock")
4029        .expect("the all-physical recovery proof should acquire every object namespace");
4030        let source_head = store.pools[0]
4031            .get_object_info(&bucket, object, &ObjectOptions::default())
4032            .await
4033            .expect("source HEAD should remain available while the part commit is paused");
4034        assert_eq!(source_head.etag, source.etag);
4035        assert!(
4036            store.pools[1]
4037                .get_object_info(&bucket, object, &ObjectOptions::default())
4038                .await
4039                .is_err(),
4040            "UploadPart staging must not publish target object metadata"
4041        );
4042        drop(proof);
4043
4044        barrier.release();
4045        drop(barrier);
4046        part.await
4047            .expect("UploadPart staging task should join")
4048            .expect("UploadPart staging should commit after the observation releases");
4049        let mut abort_opts = ObjectOptions {
4050            data_movement: true,
4051            src_pool_idx: 0,
4052            versioned: source.version_id.is_some(),
4053            version_id: source.version_id.map(|version_id| version_id.to_string()),
4054            mod_time: source.mod_time,
4055            expected_bucket_incarnation_id: Some(bucket_incarnation_id),
4056            ..Default::default()
4057        };
4058        capacity_owner.apply_to(&mut abort_opts);
4059        store
4060            .abort_multipart_upload_for_data_movement(1, &bucket, object, &upload.upload_id, &abort_opts)
4061            .await
4062            .expect("staged target upload should be removable without publishing");
4063        drop(publication);
4064        assert_eq!(TierConfigMgr::active_operation_lease_count(&ctx.tier_config_mgr(), tier_name).await, 0);
4065        shutdown.cancel();
4066    }
4067
4068    #[cfg(feature = "test-util")]
4069    struct OfflineTestDisks {
4070        disks: Vec<crate::disk::DiskStore>,
4071    }
4072
4073    #[cfg(feature = "test-util")]
4074    impl Drop for OfflineTestDisks {
4075        fn drop(&mut self) {
4076            for disk in &self.disks {
4077                disk.reset_health_for_store_init_retry();
4078            }
4079        }
4080    }
4081
4082    #[cfg(feature = "test-util")]
4083    async fn force_set_disks_offline_for_test(set: &Arc<crate::set_disk::SetDisks>) -> OfflineTestDisks {
4084        let disks = set
4085            .disks
4086            .read()
4087            .await
4088            .iter()
4089            .map(|disk| disk.clone().expect("fault-injection set should start fully online"))
4090            .collect::<Vec<_>>();
4091        for disk in &disks {
4092            disk.close().await.expect("fault injection should stop per-disk monitoring");
4093            disk.force_offline_for_test();
4094        }
4095
4096        // Sets has an independent endpoint monitor that renews missing slots.
4097        // Keep each slot populated by the original, deliberately faulty handle
4098        // and prove an explicit reconnect pass cannot heal this test fault.
4099        set.connect_disks().await;
4100        let current = set.disks.read().await.clone();
4101        assert_eq!(current.len(), disks.len());
4102        for (slot, expected) in current.iter().zip(&disks) {
4103            let disk = slot.as_ref().expect("offline test slot must remain populated");
4104            assert!(Arc::ptr_eq(disk, expected), "endpoint monitor must not replace an offline test handle");
4105            assert_eq!(disk.runtime_state(), crate::disk::health_state::RuntimeDriveHealthState::Offline);
4106            assert!(
4107                disk.is_online().await,
4108                "the valid disk identity must prevent endpoint renewal while the IO health gate remains offline"
4109            );
4110        }
4111        OfflineTestDisks { disks }
4112    }
4113
4114    #[cfg(feature = "test-util")]
4115    async fn force_set_disk_range_offline_for_test(
4116        set: &Arc<crate::set_disk::SetDisks>,
4117        range: std::ops::Range<usize>,
4118    ) -> OfflineTestDisks {
4119        let disks = set
4120            .disks
4121            .read()
4122            .await
4123            .get(range)
4124            .expect("offline test range must fit the set")
4125            .iter()
4126            .map(|disk| disk.clone().expect("fault-injection disk should start online"))
4127            .collect::<Vec<_>>();
4128        for disk in &disks {
4129            disk.close().await.expect("fault injection should stop per-disk monitoring");
4130            disk.force_offline_for_test();
4131        }
4132        set.connect_disks().await;
4133        for disk in &disks {
4134            assert_eq!(disk.runtime_state(), crate::disk::health_state::RuntimeDriveHealthState::Offline);
4135        }
4136        OfflineTestDisks { disks }
4137    }
4138
4139    fn active_rebalance_meta_for_pool(pool_count: usize, active_pool_idx: usize) -> RebalanceMeta {
4140        let now = OffsetDateTime::now_utc();
4141        let mut pool_stats = vec![RebalanceStats::default(); pool_count];
4142        pool_stats[active_pool_idx] = RebalanceStats {
4143            participating: true,
4144            info: RebalanceInfo {
4145                start_time: Some(now),
4146                status: RebalStatus::Started,
4147                ..Default::default()
4148            },
4149            ..Default::default()
4150        };
4151
4152        RebalanceMeta {
4153            id: uuid::Uuid::new_v4().to_string(),
4154            pool_stats,
4155            ..Default::default()
4156        }
4157    }
4158
4159    #[tokio::test]
4160    #[serial_test::serial(storage_class_env)]
4161    async fn quota_object_fence_ignores_an_unrelated_offline_pool() {
4162        let temp_dir = tempfile::tempdir().expect("create quota fence store dir");
4163        let (_ctx, store, shutdown) =
4164            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "quota-object-fence", &[4, 4])).await;
4165        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
4166        let bucket = format!("quota-object-fence-{}", uuid::Uuid::new_v4());
4167        let object = "object.bin";
4168        store
4169            .make_bucket(&bucket, &MakeBucketOptions::default())
4170            .await
4171            .expect("create quota fence bucket");
4172        store.pools[1].disk_set[0].disks.write().await.fill(None);
4173
4174        crate::bucket::quota::reservation::fence_namespace_mutations_for_test(&store, &bucket, object, Some((0, 0)))
4175            .await
4176            .expect("the selected pool fence should ignore an unrelated offline pool");
4177        let err = crate::bucket::quota::reservation::fence_namespace_mutations_for_test(&store, &bucket, object, None)
4178            .await
4179            .expect_err("legacy reservations must conservatively fence every pool");
4180        assert!(matches!(err, StorageError::ErasureWriteQuorum));
4181
4182        shutdown.cancel();
4183    }
4184
4185    async fn migrate_versioned_decommission_test_object(
4186        store: &Arc<crate::store::ECStore>,
4187        bucket: &str,
4188        object: &str,
4189        payload: &[u8],
4190        op_label: &'static str,
4191    ) -> (uuid::Uuid, FileInfoVersions) {
4192        let mut source = PutObjReader::from_vec(payload.to_vec());
4193        let source_info = store.pools[0]
4194            .put_object(
4195                bucket,
4196                object,
4197                &mut source,
4198                &ObjectOptions {
4199                    versioned: true,
4200                    ..Default::default()
4201                },
4202            )
4203            .await
4204            .expect("write versioned source to the pool being decommissioned");
4205        let source_version = source_info.version_id.expect("versioned source must have a version ID");
4206        let expected_source_versions = store.pools[0]
4207            .get_disks_by_key(object)
4208            .load_file_info_versions_exact(bucket, object)
4209            .await
4210            .expect("source versions should be readable before migration")
4211            .expect("source versions should exist before migration");
4212        {
4213            let mut pool_meta = store.pool_meta.write().await;
4214            pool_meta.pools[0].decommission = Some(PoolDecommissionInfo {
4215                start_time: Some(OffsetDateTime::now_utc()),
4216                ..Default::default()
4217            });
4218        }
4219
4220        let barrier = crate::set_disk::PutObjectCommitBarrier::install(
4221            bucket,
4222            object,
4223            crate::set_disk::PutObjectCommitPause::AfterNamespace,
4224        );
4225        let migration_store = Arc::clone(store);
4226        let migration_bucket = bucket.to_string();
4227        let migration_object = object.to_string();
4228        let migration = tokio::spawn(async move {
4229            let source_reader = migration_store.pools[0]
4230                .get_object_reader(
4231                    &migration_bucket,
4232                    &migration_object,
4233                    None,
4234                    HeaderMap::new(),
4235                    &ObjectOptions {
4236                        versioned: true,
4237                        version_id: Some(source_version.to_string()),
4238                        no_lock: true,
4239                        data_movement: true,
4240                        raw_data_movement_read: true,
4241                        ..Default::default()
4242                    },
4243                )
4244                .await?;
4245            crate::data_movement::migrate_decommission_object(
4246                migration_store,
4247                0,
4248                migration_bucket,
4249                source_reader,
4250                None,
4251                op_label,
4252                None,
4253            )
4254            .await
4255        });
4256        barrier.wait_until_paused().await;
4257        barrier.release();
4258        migration
4259            .await
4260            .expect("versioned decommission migration task should join")
4261            .expect("versioned decommission migration should commit");
4262
4263        (source_version, expected_source_versions)
4264    }
4265
4266    fn set_test_decommission_capacity_override(store: &Arc<crate::store::ECStore>, pool_idx: usize) {
4267        let layout = DecommissionErasureLayout { data: 2, parity: 2 };
4268        let source_physical_bytes = 1024 * 1024 * 1024;
4269        let target_physical_bytes = source_physical_bytes * 8;
4270        let capacity: Vec<DecommissionPoolCapacityInfo> = store
4271            .pools
4272            .iter()
4273            .enumerate()
4274            .map(|(index, _)| {
4275                if index == pool_idx {
4276                    DecommissionPoolCapacityInfo::for_test(index, layout, 0, source_physical_bytes, source_physical_bytes)
4277                } else {
4278                    DecommissionPoolCapacityInfo::for_test(index, layout, target_physical_bytes, target_physical_bytes, 0)
4279                }
4280            })
4281            .collect();
4282        set_decommission_capacity_info_overrides_for_test(store.id, (0..128).map(|_| capacity.clone()).collect());
4283    }
4284
4285    async fn mark_test_pool_decommissioning(store: &Arc<crate::store::ECStore>, pool_idx: usize) {
4286        set_test_decommission_capacity_override(store, pool_idx);
4287        store
4288            .save_current_pool_meta_for_decommission_start(&[pool_idx], Vec::new())
4289            .await
4290            .expect("test decommission capacity reservation should activate");
4291    }
4292
4293    #[cfg(feature = "test-util")]
4294    async fn mark_test_pool_decommissioning_with_split_targets(store: &Arc<crate::store::ECStore>, pool_idx: usize) {
4295        let layout = DecommissionErasureLayout { data: 2, parity: 2 };
4296        let source_physical_bytes = 1024 * 1024 * 1024;
4297        let capacity = store
4298            .pools
4299            .iter()
4300            .enumerate()
4301            .map(|(index, _)| {
4302                if index == pool_idx {
4303                    DecommissionPoolCapacityInfo::for_test(index, layout, 0, source_physical_bytes, source_physical_bytes)
4304                } else {
4305                    // The reservation peak is twice the predicted target size.
4306                    // Give each target only half so the real allocator must
4307                    // authorize both receipt-bearing pools.
4308                    DecommissionPoolCapacityInfo::for_test(index, layout, source_physical_bytes, source_physical_bytes, 0)
4309                }
4310            })
4311            .collect::<Vec<_>>();
4312        set_decommission_capacity_info_overrides_for_test(store.id, (0..128).map(|_| capacity.clone()).collect());
4313        store
4314            .save_current_pool_meta_for_decommission_start(&[pool_idx], Vec::new())
4315            .await
4316            .expect("split-target decommission capacity reservation should activate");
4317    }
4318
4319    #[cfg(feature = "test-util")]
4320    async fn test_decommission_capacity_owner(
4321        store: &crate::store::ECStore,
4322        source_pool_index: usize,
4323    ) -> crate::core::pools::DecommissionCapacityOwner {
4324        let pool_meta = store.pool_meta.read().await;
4325        let reservation = pool_meta.pools[source_pool_index]
4326            .decommission
4327            .as_ref()
4328            .and_then(|info| info.capacity_reservation.as_ref())
4329            .expect("test decommission capacity reservation should exist");
4330        crate::core::pools::DecommissionCapacityOwner {
4331            source_pool_index,
4332            operation_id: reservation.operation_id,
4333            generation: reservation.generation,
4334            owner_nonce: reservation.owner_nonce,
4335            mutation_id: None,
4336        }
4337    }
4338
4339    const DECOMMISSION_TEST_FAULT_STAGE_DELETE_MARKER: &str = "delete_marker_copy";
4340    const DECOMMISSION_TEST_FAULT_STAGE_MIGRATE_OBJECT: &str = "migrate_object";
4341    #[cfg(feature = "test-util")]
4342    const DECOMMISSION_TEST_FAULT_STAGE_TIERED: &str = "decommission_tiered_object";
4343
4344    fn decommission_retry_fault_hook(
4345        bucket: &str,
4346        object: &str,
4347        faults: Arc<AtomicUsize>,
4348    ) -> crate::core::pools::DecommissionTestFaultDecision {
4349        let target_bucket = bucket.to_string();
4350        let target_object = object.to_string();
4351        Arc::new(move |stage, bucket, object, attempt, succeeded| {
4352            if !succeeded
4353                || attempt >= crate::core::pools::DECOMMISSION_VERSION_COPY_ATTEMPTS
4354                || stage != DECOMMISSION_TEST_FAULT_STAGE_MIGRATE_OBJECT
4355                || bucket != target_bucket
4356                || object != target_object
4357            {
4358                return false;
4359            }
4360
4361            // Entry retries reset the local attempt; real copy errors can skip
4362            // successful attempts. Only injected faults spend this global budget.
4363            // A real failure may consume an attempt, so preserve the final chance.
4364            faults
4365                .try_update(Ordering::SeqCst, Ordering::SeqCst, |faults| {
4366                    (faults < crate::core::pools::DECOMMISSION_VERSION_COPY_ATTEMPTS.saturating_sub(1))
4367                        .then_some(faults.saturating_add(1))
4368                })
4369                .is_ok()
4370        })
4371    }
4372
4373    async fn seed_decommission_source(
4374        store: &Arc<crate::store::ECStore>,
4375        bucket: &str,
4376        object: &str,
4377        body: Vec<u8>,
4378        opts: &ObjectOptions,
4379    ) {
4380        let mut reader = PutObjReader::from_vec(body);
4381        store.pools[0]
4382            .put_object(bucket, object, &mut reader, opts)
4383            .await
4384            .expect("seed decommission source object");
4385    }
4386
4387    async fn run_decommission_entry_retry_test(
4388        store: &Arc<crate::store::ECStore>,
4389        rx: CancellationToken,
4390        bucket: &str,
4391        object: &str,
4392        expected_bucket_incarnation_id: Option<uuid::Uuid>,
4393        source_changed_exhaustions: Arc<AtomicUsize>,
4394    ) -> crate::error::Result<()> {
4395        let source_set = store.pools[0].get_disks_by_key(object);
4396        store
4397            .decommission_entry_with_retry_state_for_test(
4398                rx,
4399                0,
4400                MetaCacheEntry {
4401                    name: object.to_string(),
4402                    ..Default::default()
4403                },
4404                bucket.to_string(),
4405                source_set,
4406                expected_bucket_incarnation_id,
4407                source_changed_exhaustions,
4408            )
4409            .await
4410    }
4411
4412    async fn read_decommission_target_body(
4413        store: &Arc<crate::store::ECStore>,
4414        bucket: &str,
4415        object: &str,
4416        opts: &ObjectOptions,
4417    ) -> Vec<u8> {
4418        let mut reader = store.pools[1]
4419            .get_object_reader(bucket, object, None, HeaderMap::new(), opts)
4420            .await
4421            .expect("read decommission target object");
4422        let mut body = Vec::new();
4423        reader
4424            .stream
4425            .read_to_end(&mut body)
4426            .await
4427            .expect("drain decommission target body");
4428        body
4429    }
4430
4431    async fn assert_decommission_source_absent(
4432        store: &Arc<crate::store::ECStore>,
4433        bucket: &str,
4434        object: &str,
4435        opts: &ObjectOptions,
4436    ) {
4437        let err = store.pools[0]
4438            .get_object_info(bucket, object, opts)
4439            .await
4440            .expect_err("decommission source must be retained until target commit, then removed");
4441        assert!(
4442            matches!(err, StorageError::ObjectNotFound(_, _) | StorageError::VersionNotFound(_, _, _)),
4443            "unexpected decommission source result: {err:?}"
4444        );
4445    }
4446
4447    #[cfg(feature = "test-util")]
4448    async fn seed_transitioned_free_version(
4449        ctx: &Arc<crate::runtime::instance::InstanceContext>,
4450        store: &Arc<crate::store::ECStore>,
4451        bucket: &str,
4452        object: &str,
4453    ) -> (uuid::Uuid, uuid::Uuid) {
4454        let tier_name = format!("DECOMFREE{}", uuid::Uuid::new_v4().simple());
4455        register_mock_tier(&ctx.tier_config_mgr(), &tier_name).await;
4456
4457        let mut reader = PutObjReader::from_vec(b"transitioned source bytes".to_vec());
4458        let source = store.pools[0]
4459            .put_object(
4460                bucket,
4461                object,
4462                &mut reader,
4463                &ObjectOptions {
4464                    versioned: true,
4465                    ..Default::default()
4466                },
4467            )
4468            .await
4469            .expect("write transitioned decommission source");
4470        let source_version = source.version_id.expect("transitioned source must be versioned");
4471        store.pools[0]
4472            .transition_object(
4473                bucket,
4474                object,
4475                &ObjectOptions {
4476                    versioned: true,
4477                    version_id: Some(source_version.to_string()),
4478                    transition: TransitionOptions {
4479                        status: TRANSITION_PENDING.to_string(),
4480                        tier: tier_name,
4481                        etag: source.etag.clone().expect("transitioned source must have an ETag"),
4482                        ..Default::default()
4483                    },
4484                    mod_time: source.mod_time,
4485                    ..Default::default()
4486                },
4487            )
4488            .await
4489            .expect("transition source before decommission");
4490        store.pools[0]
4491            .delete_object(
4492                bucket,
4493                object,
4494                ObjectOptions {
4495                    versioned: true,
4496                    version_id: Some(source_version.to_string()),
4497                    ..Default::default()
4498                },
4499            )
4500            .await
4501            .expect("delete transitioned source version");
4502
4503        let versions = store.pools[0]
4504            .get_disks_by_key(object)
4505            .load_file_info_versions_exact(bucket, object)
4506            .await
4507            .expect("source versions should decode after transition delete")
4508            .expect("source free version should remain after transition delete");
4509        let free_version = versions
4510            .versions
4511            .iter()
4512            .find(|version| version.tier_free_version())
4513            .and_then(|version| version.version_id)
4514            .expect("transition delete should create a free version");
4515        (source_version, free_version)
4516    }
4517
4518    async fn write_decommission_test_multipart_source(
4519        store: &Arc<crate::store::ECStore>,
4520        pool_idx: usize,
4521        bucket: &str,
4522        object: &str,
4523    ) {
4524        let pool = &store.pools[pool_idx];
4525        let upload = pool
4526            .new_multipart_upload(bucket, object, &ObjectOptions::default())
4527            .await
4528            .expect("create decommission multipart source upload");
4529        let first_part = vec![b'm'; 5 * 1024 * 1024];
4530        let second_part = b"decommission multipart tail".to_vec();
4531        let mut completed_parts = Vec::with_capacity(2);
4532        for (part_number, body) in [(1, first_part), (2, second_part)] {
4533            let mut reader = PutObjReader::from_vec(body);
4534            let part = pool
4535                .put_object_part(bucket, object, &upload.upload_id, part_number, &mut reader, &ObjectOptions::default())
4536                .await
4537                .expect("write decommission multipart source part");
4538            completed_parts.push(crate::storage_api_contracts::multipart::CompletePart {
4539                part_num: part.part_num,
4540                etag: part.etag,
4541                ..Default::default()
4542            });
4543        }
4544        pool.clone()
4545            .complete_multipart_upload(bucket, object, &upload.upload_id, completed_parts, &ObjectOptions::default())
4546            .await
4547            .expect("complete decommission multipart source object");
4548    }
4549
4550    struct DataMovementSourceSeed {
4551        pool_idx: usize,
4552        version_id: Option<Uuid>,
4553        body: Vec<u8>,
4554        multipart_split: Option<usize>,
4555        mod_time: OffsetDateTime,
4556        user_defined: HashMap<String, String>,
4557    }
4558
4559    async fn seed_data_movement_source_reader(
4560        store: &Arc<crate::store::ECStore>,
4561        bucket: &str,
4562        object: &str,
4563        seed: DataMovementSourceSeed,
4564    ) -> GetObjectReader {
4565        let DataMovementSourceSeed {
4566            pool_idx,
4567            version_id,
4568            body,
4569            multipart_split,
4570            mod_time,
4571            user_defined,
4572        } = seed;
4573        let pool = &store.pools[pool_idx];
4574        let write_opts = ObjectOptions {
4575            versioned: version_id.is_some(),
4576            version_id: version_id.map(|version_id| version_id.to_string()),
4577            mod_time: Some(mod_time),
4578            user_defined,
4579            ..Default::default()
4580        };
4581        if let Some(split) = multipart_split {
4582            assert!(split > 0 && split < body.len(), "multipart source split must create two non-empty parts");
4583            let upload = pool
4584                .new_multipart_upload(bucket, object, &write_opts)
4585                .await
4586                .expect("create physical data-movement source upload");
4587            let mut completed_parts = Vec::with_capacity(2);
4588            for (part_number, bytes) in [(1, &body[..split]), (2, &body[split..])] {
4589                let mut reader = PutObjReader::from_vec(bytes.to_vec());
4590                let part = pool
4591                    .put_object_part(bucket, object, &upload.upload_id, part_number, &mut reader, &ObjectOptions::default())
4592                    .await
4593                    .expect("write physical data-movement source part");
4594                completed_parts.push(crate::storage_api_contracts::multipart::CompletePart {
4595                    part_num: part.part_num,
4596                    etag: part.etag,
4597                    ..Default::default()
4598                });
4599            }
4600            pool.clone()
4601                .complete_multipart_upload(bucket, object, &upload.upload_id, completed_parts, &write_opts)
4602                .await
4603                .expect("complete physical data-movement source object");
4604        } else {
4605            let mut reader = PutObjReader::from_vec(body);
4606            pool.put_object(bucket, object, &mut reader, &write_opts)
4607                .await
4608                .expect("write physical data-movement source object");
4609        }
4610
4611        pool.get_object_reader(
4612            bucket,
4613            object,
4614            None,
4615            HeaderMap::new(),
4616            &ObjectOptions {
4617                versioned: version_id.is_some(),
4618                version_id: version_id.map(|version_id| version_id.to_string()),
4619                raw_data_movement_read: true,
4620                include_part_checksums: true,
4621                ..Default::default()
4622            },
4623        )
4624        .await
4625        .expect("read the physical data-movement source snapshot")
4626    }
4627
4628    async fn assert_pool_object_present(pool: &Arc<crate::core::sets::Sets>, bucket: &str, object: &str) {
4629        pool.get_object_info(bucket, object, &ObjectOptions::default())
4630            .await
4631            .expect("expected object generation must remain present");
4632    }
4633
4634    async fn assert_pool_object_absent(pool: &Arc<crate::core::sets::Sets>, bucket: &str, object: &str) {
4635        let err = pool
4636            .get_object_info(bucket, object, &ObjectOptions::default())
4637            .await
4638            .expect_err("fenced decommission target must remain absent");
4639        assert!(
4640            matches!(err, StorageError::ObjectNotFound(_, _) | StorageError::VersionNotFound(_, _, _)),
4641            "unexpected fenced target result: {err:?}"
4642        );
4643    }
4644
4645    async fn write_suspended_decommission_source(store: &Arc<crate::store::ECStore>, bucket: &str, object: &str) {
4646        let mut reader = PutObjReader::from_vec(b"suspended source generation".to_vec());
4647        let source = store.pools[0]
4648            .put_object(
4649                bucket,
4650                object,
4651                &mut reader,
4652                &ObjectOptions {
4653                    version_suspended: true,
4654                    mod_time: Some(OffsetDateTime::UNIX_EPOCH + time::Duration::SECOND),
4655                    ..Default::default()
4656                },
4657            )
4658            .await
4659            .expect("write suspended null source version");
4660        assert!(
4661            source.version_id.is_none_or(|version_id| version_id.is_nil()),
4662            "suspended source must use the null version identity"
4663        );
4664    }
4665
4666    async fn assert_suspended_null_source_present(store: &Arc<crate::store::ECStore>, bucket: &str, object: &str) {
4667        let versions = store.pools[0]
4668            .get_disks_by_key(object)
4669            .load_file_info_versions_exact(bucket, object)
4670            .await
4671            .expect("suspended source versions should be readable")
4672            .expect("suspended source must exist before worker convergence");
4673        assert_eq!(versions.versions.len(), 1, "DELETE must not add a marker to the retiring source");
4674        let source = &versions.versions[0];
4675        assert!(
4676            !source.deleted && source.version_id.is_none_or(|version_id| version_id.is_nil()),
4677            "the source pool must retain its null data version until worker convergence"
4678        );
4679        assert_eq!(source.mod_time, Some(OffsetDateTime::UNIX_EPOCH + time::Duration::SECOND));
4680
4681        let mut reader = store.pools[0]
4682            .get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
4683            .await
4684            .expect("the retiring source must remain directly readable before worker convergence");
4685        let mut body = Vec::new();
4686        reader
4687            .stream
4688            .read_to_end(&mut body)
4689            .await
4690            .expect("read retained source bytes");
4691        assert_eq!(body, b"suspended source generation");
4692    }
4693
4694    async fn assert_suspended_null_delete_marker_visible(
4695        store: &Arc<crate::store::ECStore>,
4696        bucket: &str,
4697        object: &str,
4698        marker_mod_time: OffsetDateTime,
4699    ) {
4700        let versions = store.pools[1]
4701            .get_disks_by_key(object)
4702            .load_file_info_versions_exact(bucket, object)
4703            .await
4704            .expect("healthy target versions should be readable")
4705            .expect("the healthy target must retain the DELETE marker");
4706        assert_eq!(versions.versions.len(), 1, "the target must contain only the null delete marker");
4707        let marker = &versions.versions[0];
4708        assert!(marker.deleted, "migration must not replace the DELETE marker with source data");
4709        assert!(marker.version_id.is_none_or(|version_id| version_id.is_nil()));
4710        assert_eq!(marker.size, 0);
4711        assert_eq!(marker.mod_time, Some(marker_mod_time), "migration must preserve the marker generation");
4712        assert!(marker_mod_time > OffsetDateTime::UNIX_EPOCH + time::Duration::SECOND);
4713
4714        let head_err = store
4715            .get_object_info(bucket, object, &ObjectOptions::default())
4716            .await
4717            .expect_err("HEAD must observe the DELETE marker instead of the old null source");
4718        assert!(matches!(head_err, Error::ObjectNotFound(_, _)), "unexpected HEAD result: {head_err:?}");
4719        let get_err = match store
4720            .get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
4721            .await
4722        {
4723            Ok(_) => panic!("GET must not resurrect the deleted null source"),
4724            Err(err) => err,
4725        };
4726        assert!(matches!(get_err, Error::ObjectNotFound(_, _)), "unexpected GET result: {get_err:?}");
4727    }
4728
4729    #[tokio::test]
4730    #[serial_test::serial(storage_class_env)]
4731    async fn tag_updates_skip_active_rebalance_source_pool() {
4732        let temp_dir = tempfile::tempdir().expect("create writer-fencing store dir");
4733        let (_ctx, store, shutdown) =
4734            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "writer-fencing-tags", &[4, 4])).await;
4735        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
4736
4737        let bucket = format!("writer-fencing-tags-{}", uuid::Uuid::new_v4());
4738        let object = "tagged-object.bin";
4739        store
4740            .make_bucket(&bucket, &MakeBucketOptions::default())
4741            .await
4742            .expect("create writer fencing bucket");
4743
4744        let old_time = OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("fixed timestamp should be valid");
4745        let newer_time = old_time + time::Duration::seconds(10);
4746        let mut source_reader = PutObjReader::from_vec(b"source-body".to_vec());
4747        store.pools[0]
4748            .put_object(
4749                &bucket,
4750                object,
4751                &mut source_reader,
4752                &ObjectOptions {
4753                    mod_time: Some(newer_time),
4754                    ..Default::default()
4755                },
4756            )
4757            .await
4758            .expect("write newer source object");
4759        let mut target_reader = PutObjReader::from_vec(b"target-body".to_vec());
4760        store.pools[1]
4761            .put_object(
4762                &bucket,
4763                object,
4764                &mut target_reader,
4765                &ObjectOptions {
4766                    mod_time: Some(old_time),
4767                    ..Default::default()
4768                },
4769            )
4770            .await
4771            .expect("write older target object");
4772
4773        *store.rebalance_meta.write().await = Some(active_rebalance_meta_for_pool(store.pools.len(), 0));
4774        assert!(store.is_pool_rebalancing(0).await, "pool 0 must be marked as an active rebalance source");
4775
4776        let tags = "rebalance=target";
4777        assert_ne!(
4778            store.pools[0]
4779                .get_object_tags(&bucket, object, &ObjectOptions::default())
4780                .await
4781                .expect("source object tags should be readable before update"),
4782            tags,
4783            "source object must start without the target tag"
4784        );
4785        assert_ne!(
4786            store.pools[1]
4787                .get_object_tags(&bucket, object, &ObjectOptions::default())
4788                .await
4789                .expect("target object tags should be readable before update"),
4790            tags,
4791            "target object must start without the target tag"
4792        );
4793        let selected_pool = store
4794            .get_pool_idx_existing_with_opts(
4795                &bucket,
4796                object,
4797                &ObjectOptions {
4798                    no_lock: true,
4799                    metadata_chg: true,
4800                    skip_decommissioned: true,
4801                    skip_rebalancing: true,
4802                    ..Default::default()
4803                },
4804            )
4805            .await
4806            .expect("writer lookup should select an existing non-rebalancing pool");
4807        assert_eq!(selected_pool, 1, "writer lookup must skip active rebalance pool 0");
4808
4809        let updated = store
4810            .put_object_tags(&bucket, object, tags, &ObjectOptions::default())
4811            .await
4812            .expect("tag update should use the non-rebalancing target pool");
4813        assert_eq!(
4814            updated.mod_time,
4815            Some(old_time),
4816            "tag update must return the non-rebalancing pool object rather than the newer active source"
4817        );
4818
4819        let target_tags = store.pools[1]
4820            .get_object_tags(&bucket, object, &ObjectOptions::default())
4821            .await
4822            .expect("target object tags should be readable");
4823        assert_eq!(target_tags, tags, "non-rebalancing pool must receive writer tag updates");
4824
4825        shutdown.cancel();
4826    }
4827
4828    #[tokio::test]
4829    #[serial_test::serial(storage_class_env)]
4830    async fn multipart_listing_skips_active_rebalance_source_pool() {
4831        let temp_dir = tempfile::tempdir().expect("create multipart writer-fencing store dir");
4832        let (_ctx, store, shutdown) =
4833            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "writer-fencing-multipart", &[4, 4])).await;
4834        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
4835
4836        let bucket = format!("writer-fencing-multipart-{}", uuid::Uuid::new_v4());
4837        store
4838            .make_bucket(&bucket, &MakeBucketOptions::default())
4839            .await
4840            .expect("create multipart writer fencing bucket");
4841
4842        let incarnation = store.bucket_incarnation_id(&bucket).await.expect("read bucket incarnation");
4843        let lifecycle_guard = store
4844            .acquire_bucket_lifecycle_read_lock(&bucket)
4845            .await
4846            .expect("acquire multipart test lifecycle fence");
4847        let mut upload_opts = ObjectOptions {
4848            expected_bucket_incarnation_id: Some(incarnation),
4849            ..Default::default()
4850        };
4851        upload_opts.add_bucket_lifecycle_lock_guard(&lifecycle_guard);
4852        let source_upload = store.pools[0]
4853            .new_multipart_upload(&bucket, "source-only.bin", &upload_opts)
4854            .await
4855            .expect("create source upload");
4856        let target_upload = store.pools[1]
4857            .new_multipart_upload(&bucket, "target-visible.bin", &upload_opts)
4858            .await
4859            .expect("create target upload");
4860
4861        *store.rebalance_meta.write().await = Some(active_rebalance_meta_for_pool(store.pools.len(), 0));
4862        assert!(store.is_pool_rebalancing(0).await, "pool 0 must be marked as an active rebalance source");
4863
4864        let listed = store
4865            .list_multipart_uploads(&bucket, "", None, None, None, 100)
4866            .await
4867            .expect("list multipart uploads");
4868        let listed_uploads: Vec<(&str, &str)> = listed
4869            .uploads
4870            .iter()
4871            .map(|upload| (upload.object.as_str(), upload.upload_id.as_str()))
4872            .collect();
4873
4874        assert!(
4875            !listed_uploads.contains(&("source-only.bin", source_upload.upload_id.as_str())),
4876            "active source pool upload must be hidden from multipart listing"
4877        );
4878        assert!(
4879            listed_uploads.contains(&("target-visible.bin", target_upload.upload_id.as_str())),
4880            "non-rebalancing pool upload must remain visible"
4881        );
4882
4883        shutdown.cancel();
4884    }
4885
4886    #[tokio::test]
4887    #[serial_test::serial(storage_class_env)]
4888    async fn suspended_decommission_source_multipart_remains_operable_until_drained() {
4889        let temp_dir = tempfile::tempdir().expect("create decommission multipart drain store dir");
4890        let (_ctx, store, shutdown) =
4891            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "decommission-multipart-drain", &[4, 4])).await;
4892        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
4893
4894        let bucket = format!("decommission-mp-drain-{}", uuid::Uuid::new_v4());
4895        let complete_object = "complete.bin";
4896        let abort_object = "abort.bin";
4897        store
4898            .make_bucket(&bucket, &MakeBucketOptions::default())
4899            .await
4900            .expect("create decommission multipart drain bucket");
4901
4902        let incarnation = store.bucket_incarnation_id(&bucket).await.expect("read bucket incarnation");
4903        let lifecycle_guard = store
4904            .acquire_bucket_lifecycle_read_lock(&bucket)
4905            .await
4906            .expect("acquire multipart creation lifecycle fence");
4907        let mut upload_opts = ObjectOptions {
4908            expected_bucket_incarnation_id: Some(incarnation),
4909            ..Default::default()
4910        };
4911        upload_opts.add_bucket_lifecycle_lock_guard(&lifecycle_guard);
4912        let complete_upload = store.pools[0]
4913            .new_multipart_upload(&bucket, complete_object, &upload_opts)
4914            .await
4915            .expect("create source upload to complete");
4916        let abort_upload = store.pools[0]
4917            .new_multipart_upload(&bucket, abort_object, &upload_opts)
4918            .await
4919            .expect("create source upload to abort");
4920        drop(lifecycle_guard);
4921
4922        mark_test_pool_decommissioning(&store, 0).await;
4923        let err = store
4924            .ensure_decommission_multipart_uploads_drained_for_test(0)
4925            .await
4926            .expect_err("an unresolved source multipart upload must block final decommission");
4927        let drain_error = err.to_string();
4928        assert!(
4929            drain_error.contains("still contains multipart upload") && drain_error.contains(&bucket),
4930            "the drain error must identify both the upload path and user bucket: {drain_error}"
4931        );
4932
4933        let listed = store
4934            .list_multipart_uploads(&bucket, "", None, None, None, 100)
4935            .await
4936            .expect("list uploads from suspended decommission source");
4937        assert!(
4938            listed
4939                .uploads
4940                .iter()
4941                .any(|upload| upload.upload_id.as_str() == complete_upload.upload_id.as_str()),
4942            "the upload selected before suspension must remain visible"
4943        );
4944        store
4945            .get_multipart_info(&bucket, complete_object, &complete_upload.upload_id, &ObjectOptions::default())
4946            .await
4947            .expect("read upload metadata from suspended decommission source");
4948
4949        let mut part_reader = PutObjReader::from_vec(b"multipart body".to_vec());
4950        let part = store
4951            .put_object_part(
4952                &bucket,
4953                complete_object,
4954                &complete_upload.upload_id,
4955                1,
4956                &mut part_reader,
4957                &ObjectOptions::default(),
4958            )
4959            .await
4960            .expect("write part to suspended decommission source");
4961        let parts = store
4962            .list_object_parts(&bucket, complete_object, &complete_upload.upload_id, None, 100, &ObjectOptions::default())
4963            .await
4964            .expect("list parts from suspended decommission source");
4965        assert_eq!(parts.parts.len(), 1);
4966        assert_eq!(parts.parts[0].etag.as_deref(), part.etag.as_deref());
4967
4968        store
4969            .clone()
4970            .complete_multipart_upload(
4971                &bucket,
4972                complete_object,
4973                &complete_upload.upload_id,
4974                vec![crate::storage_api_contracts::multipart::CompletePart {
4975                    part_num: part.part_num,
4976                    etag: part.etag,
4977                    ..Default::default()
4978                }],
4979                &ObjectOptions::default(),
4980            )
4981            .await
4982            .expect("complete upload on suspended decommission source");
4983        store
4984            .abort_multipart_upload(&bucket, abort_object, &abort_upload.upload_id, &ObjectOptions::default())
4985            .await
4986            .expect("abort upload on suspended decommission source");
4987
4988        tokio::time::timeout(Duration::from_secs(5), async {
4989            loop {
4990                match store.ensure_decommission_multipart_uploads_drained_for_test(0).await {
4991                    Ok(()) => break,
4992                    Err(err) if err.to_string().contains("still contains multipart upload") => {
4993                        tokio::time::sleep(Duration::from_millis(10)).await;
4994                    }
4995                    Err(err) => panic!("unexpected final decommission drain error: {err:?}"),
4996                }
4997            }
4998        })
4999        .await
5000        .expect("final decommission gate should open after upload cleanup converges");
5001        assert_pool_object_present(&store.pools[0], &bucket, complete_object).await;
5002
5003        shutdown.cancel();
5004    }
5005
5006    #[tokio::test]
5007    #[serial_test::serial(storage_class_env)]
5008    async fn active_multipart_upload_routes_before_faulted_suspended_source() {
5009        use sha2::Digest;
5010
5011        let temp_dir = tempfile::tempdir().expect("create active-first multipart routing store dir");
5012        let (_ctx, store, shutdown) =
5013            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "active-first-multipart-routing", &[4, 4]))
5014                .await;
5015        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
5016
5017        let bucket = format!("active-first-mp-route-{}", uuid::Uuid::new_v4());
5018        let object = "target-upload.bin";
5019        store
5020            .make_bucket(&bucket, &MakeBucketOptions::default())
5021            .await
5022            .expect("create active-first multipart routing bucket");
5023
5024        let incarnation = store.bucket_incarnation_id(&bucket).await.expect("read bucket incarnation");
5025        let lifecycle_guard = store
5026            .acquire_bucket_lifecycle_read_lock(&bucket)
5027            .await
5028            .expect("acquire multipart creation lifecycle fence");
5029        let mut upload_opts = ObjectOptions {
5030            expected_bucket_incarnation_id: Some(incarnation),
5031            ..Default::default()
5032        };
5033        upload_opts.add_bucket_lifecycle_lock_guard(&lifecycle_guard);
5034        let upload = store.pools[1]
5035            .new_multipart_upload(&bucket, object, &upload_opts)
5036            .await
5037            .expect("create upload in active target pool");
5038        drop(lifecycle_guard);
5039
5040        mark_test_pool_decommissioning(&store, 0).await;
5041        // Corrupt only the source pool's entry for this UploadID. Taking the
5042        // whole source set offline would also make the bucket-incarnation
5043        // sidecar unreadable before multipart routing is reached.
5044        let upload_sha =
5045            hex_simd::encode_to_string(sha2::Sha256::digest(format!("{bucket}/{object}").as_bytes()), hex_simd::AsciiCase::Lower);
5046        let upload_uuid = crate::runtime::sources::upload_uuid_suffix(&upload.upload_id);
5047        for disk_index in 0..4 {
5048            let metadata_path = temp_dir
5049                .path()
5050                .join(format!("pool0/set0/disk{disk_index}"))
5051                .join(crate::disk::RUSTFS_META_MULTIPART_BUCKET)
5052                .join(&upload_sha)
5053                .join(&upload_uuid)
5054                .join(crate::disk::STORAGE_FORMAT_FILE);
5055            tokio::fs::create_dir_all(metadata_path.parent().expect("multipart metadata path should have a parent"))
5056                .await
5057                .expect("create corrupt source upload directory");
5058            tokio::fs::write(metadata_path, b"not-xl-meta")
5059                .await
5060                .expect("inject source upload metadata read failure");
5061        }
5062
5063        let source_result = store.pools[0]
5064            .get_multipart_info(&bucket, object, &upload.upload_id, &ObjectOptions::default())
5065            .await;
5066        let routed_result = store
5067            .get_multipart_info(&bucket, object, &upload.upload_id, &ObjectOptions::default())
5068            .await;
5069
5070        assert!(
5071            source_result
5072                .as_ref()
5073                .is_err_and(|err| !crate::error::is_err_invalid_upload_id(err)),
5074            "the suspended source must expose the injected hard read failure: {source_result:?}"
5075        );
5076        let routed = routed_result.expect("the active target UploadID must be resolved before the faulted suspended source");
5077        assert_eq!(routed.upload_id, upload.upload_id);
5078
5079        shutdown.cancel();
5080    }
5081
5082    #[tokio::test]
5083    #[serial_test::serial(storage_class_env)]
5084    async fn delete_objects_skips_active_rebalance_source_pool() {
5085        let temp_dir = tempfile::tempdir().expect("create batch-delete writer-fencing store dir");
5086        let (_ctx, store, shutdown) =
5087            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "batch-delete-rebalance", &[4, 4])).await;
5088        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
5089
5090        let bucket = format!("batch-delete-rebalance-{}", uuid::Uuid::new_v4());
5091        let object = "delete-me.bin";
5092        store
5093            .make_bucket(&bucket, &MakeBucketOptions::default())
5094            .await
5095            .expect("create batch delete rebalance bucket");
5096
5097        let mut source_reader = PutObjReader::from_vec(b"source-body".to_vec());
5098        store.pools[0]
5099            .put_object(&bucket, object, &mut source_reader, &ObjectOptions::default())
5100            .await
5101            .expect("write object on active source pool");
5102        let mut target_reader = PutObjReader::from_vec(b"target-body".to_vec());
5103        store.pools[1]
5104            .put_object(&bucket, object, &mut target_reader, &ObjectOptions::default())
5105            .await
5106            .expect("write object on non-rebalancing target pool");
5107
5108        let mut pool_stats = vec![RebalanceStats::default(); store.pools.len()];
5109        pool_stats[0] = RebalanceStats {
5110            participating: true,
5111            info: RebalanceInfo {
5112                start_time: Some(OffsetDateTime::now_utc()),
5113                status: RebalStatus::Started,
5114                ..Default::default()
5115            },
5116            ..Default::default()
5117        };
5118        *store.rebalance_meta.write().await = Some(RebalanceMeta {
5119            id: uuid::Uuid::new_v4().to_string(),
5120            pool_stats,
5121            ..Default::default()
5122        });
5123        assert!(store.is_pool_rebalancing(0).await, "pool 0 must be marked as an active rebalance source");
5124
5125        let (deleted, errs) = store
5126            .delete_objects(
5127                &bucket,
5128                vec![crate::storage_api_contracts::object::ObjectToDelete {
5129                    object_name: object.to_string(),
5130                    ..Default::default()
5131                }],
5132                ObjectOptions::default(),
5133            )
5134            .await;
5135        assert!(matches!(errs.as_slice(), [None]), "batch delete must not fail: {errs:?}");
5136        assert!(
5137            matches!(deleted.as_slice(), [deleted] if deleted.found && deleted.object_name == object),
5138            "batch delete must report the non-rebalancing pool deletion"
5139        );
5140
5141        store.pools[0]
5142            .get_object_info(&bucket, object, &ObjectOptions::default())
5143            .await
5144            .expect("active source pool object must not be deleted by DeleteObjects");
5145        let target_err = store.pools[1]
5146            .get_object_info(&bucket, object, &ObjectOptions::default())
5147            .await
5148            .expect_err("non-rebalancing target pool object must be deleted");
5149        assert!(
5150            matches!(target_err, StorageError::ObjectNotFound(_, _)),
5151            "target pool should report object not found after DeleteObjects, got {target_err:?}"
5152        );
5153
5154        shutdown.cancel();
5155    }
5156
5157    #[test]
5158    #[serial_test::serial(storage_class_env)]
5159    fn data_movement_conflicts_and_multipart_retries_converge_safely() {
5160        // Run this large async scenario on a larger dedicated stack so debug
5161        // test threads do not overflow before the regression assertions.
5162        std::thread::Builder::new()
5163            .stack_size(16 * 1024 * 1024)
5164            .spawn(|| {
5165                let runtime = tokio::runtime::Builder::new_multi_thread()
5166                    .enable_all()
5167                    .worker_threads(2)
5168                    .build()
5169                    .expect("build data movement test runtime");
5170                runtime.block_on(async move {
5171                    let temp_dir = tempfile::tempdir().expect("create data movement store dir");
5172                    let (_ctx, store, _shutdown) = without_storage_class_env(build_isolated_test_store(
5173                        temp_dir.path(),
5174                        "data-movement-conflict-convergence",
5175                        &[4, 4],
5176                    ))
5177                    .await;
5178                    crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
5179
5180                    let bucket = format!("data-movement-conflict-{}", uuid::Uuid::new_v4());
5181                    store
5182                        .make_bucket(&bucket, &MakeBucketOptions::default())
5183                        .await
5184                        .expect("create data movement bucket");
5185                    let source_mod_time = OffsetDateTime::UNIX_EPOCH + time::Duration::SECOND;
5186                    let target_mod_time = source_mod_time + time::Duration::SECOND;
5187
5188                    let object = "single-object";
5189                    let target_body = b"newer client body".to_vec();
5190                    let mut target_reader = PutObjReader::from_vec(target_body.clone());
5191                    store.pools[1]
5192                        .put_object(
5193                            &bucket,
5194                            object,
5195                            &mut target_reader,
5196                            &ObjectOptions {
5197                                mod_time: Some(target_mod_time),
5198                                ..Default::default()
5199                            },
5200                        )
5201                        .await
5202                        .expect("write newer single-part target");
5203
5204                    let source_body = b"stale migration body".to_vec();
5205                    let source_reader = seed_data_movement_source_reader(
5206                        &store,
5207                        &bucket,
5208                        object,
5209                        DataMovementSourceSeed {
5210                            pool_idx: 0,
5211                            version_id: None,
5212                            body: source_body,
5213                            multipart_split: None,
5214                            mod_time: source_mod_time,
5215                            user_defined: HashMap::new(),
5216                        },
5217                    )
5218                    .await;
5219                    crate::data_movement::migrate_object(
5220                        store.clone(),
5221                        0,
5222                        bucket.clone(),
5223                        source_reader,
5224                        None,
5225                        "test_data_movement",
5226                    )
5227                    .await
5228                    .expect("newer single-part target should converge migration");
5229
5230                    let mut reader = store.pools[1]
5231                        .get_object_reader(&bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
5232                        .await
5233                        .expect("read converged single-part target");
5234                    let mut body = Vec::new();
5235                    reader.stream.read_to_end(&mut body).await.expect("drain single-part target");
5236                    assert_eq!(body, target_body);
5237
5238                    let multipart_object = "multipart-object";
5239                    let multipart_target_body = b"newer multipart client body".to_vec();
5240                    let mut multipart_target_reader = PutObjReader::from_vec(multipart_target_body.clone());
5241                    store.pools[1]
5242                        .put_object(
5243                            &bucket,
5244                            multipart_object,
5245                            &mut multipart_target_reader,
5246                            &ObjectOptions {
5247                                mod_time: Some(target_mod_time),
5248                                ..Default::default()
5249                            },
5250                        )
5251                        .await
5252                        .expect("write newer multipart target");
5253
5254                    let first_part_size = 5 * 1024 * 1024;
5255                    let mut multipart_source_body = vec![b'a'; first_part_size];
5256                    multipart_source_body.push(b'b');
5257                    let multipart_source_reader = seed_data_movement_source_reader(
5258                        &store,
5259                        &bucket,
5260                        multipart_object,
5261                        DataMovementSourceSeed {
5262                            pool_idx: 0,
5263                            version_id: None,
5264                            body: multipart_source_body,
5265                            multipart_split: Some(first_part_size),
5266                            mod_time: source_mod_time,
5267                            user_defined: HashMap::new(),
5268                        },
5269                    )
5270                    .await;
5271                    crate::data_movement::migrate_object(
5272                        store.clone(),
5273                        0,
5274                        bucket.clone(),
5275                        multipart_source_reader,
5276                        None,
5277                        "test_data_movement",
5278                    )
5279                    .await
5280                    .expect("newer multipart target should converge migration");
5281
5282                    let uploads = store.pools[1]
5283                        .list_multipart_uploads(&bucket, multipart_object, None, None, None, 100)
5284                        .await
5285                        .expect("list target pool multipart uploads");
5286                    assert!(uploads.uploads.is_empty(), "superseded migration staging must be aborted");
5287
5288                    let mut reader = store.pools[1]
5289                        .get_object_reader(&bucket, multipart_object, None, HeaderMap::new(), &ObjectOptions::default())
5290                        .await
5291                        .expect("read converged multipart target");
5292                    let mut body = Vec::new();
5293                    reader.stream.read_to_end(&mut body).await.expect("drain multipart target");
5294                    assert_eq!(body, multipart_target_body);
5295
5296                    let retry_object = "multipart-retry-object";
5297                    let retry_object_etag = "0123456789abcdef0123456789abcdef".to_string();
5298                    let retry_first_part_size = 5 * 1024 * 1024;
5299                    let retry_object_mod_time =
5300                        OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("fixed retry timestamp should be valid");
5301                    let mut retry_source_body = vec![b'c'; retry_first_part_size];
5302                    retry_source_body.push(b'd');
5303                    let retry_first_checksum =
5304                        Checksum::new_from_data(ChecksumType::CRC32C, &retry_source_body[..retry_first_part_size])
5305                            .expect("first source part checksum should be created");
5306                    let retry_second_checksum =
5307                        Checksum::new_from_data(ChecksumType::CRC32C, &retry_source_body[retry_first_part_size..])
5308                            .expect("second source part checksum should be created");
5309                    let mut retry_checksum_parts = retry_first_checksum.raw.clone();
5310                    retry_checksum_parts.extend_from_slice(&retry_second_checksum.raw);
5311                    let mut retry_checksum_type = ChecksumType::CRC32C;
5312                    retry_checksum_type
5313                        .merge(ChecksumType::MULTIPART)
5314                        .merge(ChecksumType::INCLUDES_MULTIPART);
5315                    let retry_object_checksum = Checksum::new_from_data(retry_checksum_type, &retry_checksum_parts)
5316                        .expect("source multipart checksum should be created");
5317                    let retry_object_checksum_bytes = retry_object_checksum.to_bytes(&retry_checksum_parts);
5318                    let mut retry_metadata = HashMap::from([("x-amz-meta-retry".to_string(), "stable".to_string())]);
5319                    insert_str(
5320                        &mut retry_metadata,
5321                        SUFFIX_COMPRESSION,
5322                        compression_metadata_value(CompressionAlgorithm::default()),
5323                    );
5324                    let retry_upload = store.pools[0]
5325                        .new_multipart_upload(
5326                            &bucket,
5327                            retry_object,
5328                            &ObjectOptions {
5329                                user_defined: retry_metadata,
5330                                want_checksum: Some(retry_object_checksum.clone()),
5331                                ..Default::default()
5332                            },
5333                        )
5334                        .await
5335                        .expect("create source multipart upload");
5336                    let mut retry_parts = Vec::with_capacity(2);
5337                    for (part_number, plaintext, checksum) in [
5338                        (1, &retry_source_body[..retry_first_part_size], &retry_first_checksum),
5339                        (2, &retry_source_body[retry_first_part_size..], &retry_second_checksum),
5340                    ] {
5341                        let part_size = i64::try_from(plaintext.len()).expect("source part size should fit i64");
5342                        let mut plaintext_reader = rustfs_rio::HashReader::from_stream(
5343                            Cursor::new(plaintext.to_vec()),
5344                            part_size,
5345                            part_size,
5346                            None,
5347                            None,
5348                            false,
5349                        )
5350                        .expect("create source part plaintext reader");
5351                        plaintext_reader
5352                            .add_non_trailing_checksum(Some(checksum.clone()), false)
5353                            .expect("set source part checksum");
5354                        let compressed = WritePlan::new()
5355                            .with_compression(CompressionAlgorithm::default())
5356                            .apply(plaintext_reader, part_size)
5357                            .expect("compress source part");
5358                        let mut reader = PutObjReader::new(compressed);
5359                        let part = store.pools[0]
5360                            .put_object_part(
5361                                &bucket,
5362                                retry_object,
5363                                &retry_upload.upload_id,
5364                                part_number,
5365                                &mut reader,
5366                                &ObjectOptions::default(),
5367                            )
5368                            .await
5369                            .expect("write source multipart part");
5370                        retry_parts.push(crate::storage_api_contracts::multipart::CompletePart {
5371                            part_num: part.part_num,
5372                            etag: part.etag,
5373                            checksum_crc32c: Some(checksum.encoded.clone()),
5374                            ..Default::default()
5375                        });
5376                    }
5377                    store.pools[0]
5378                        .clone()
5379                        .complete_multipart_upload(
5380                            &bucket,
5381                            retry_object,
5382                            &retry_upload.upload_id,
5383                            retry_parts,
5384                            &ObjectOptions {
5385                                mod_time: Some(retry_object_mod_time),
5386                                want_checksum: Some(retry_object_checksum),
5387                                preserve_etag: Some(retry_object_etag.clone()),
5388                                ..Default::default()
5389                            },
5390                        )
5391                        .await
5392                        .expect("complete source multipart object");
5393
5394                    *store.rebalance_meta.write().await = Some(active_rebalance_meta_for_pool(store.pools.len(), 0));
5395                    assert!(store.is_pool_rebalancing(0).await, "pool 0 must be marked as an active rebalance source");
5396
5397                    let mut retry_reader = store.pools[0]
5398                        .get_object_reader(
5399                            &bucket,
5400                            retry_object,
5401                            None,
5402                            HeaderMap::new(),
5403                            &ObjectOptions {
5404                                raw_data_movement_read: true,
5405                                ..Default::default()
5406                            },
5407                        )
5408                        .await
5409                        .expect("read source multipart object for first migration");
5410                    let mut retry_source_info = retry_reader.object_info.clone();
5411                    let mut retry_source_parts = retry_source_info.parts.as_ref().clone();
5412                    for (part, checksum) in retry_source_parts
5413                        .iter_mut()
5414                        .zip([&retry_first_checksum, &retry_second_checksum])
5415                    {
5416                        part.mod_time = Some(OffsetDateTime::UNIX_EPOCH);
5417                        part.checksums = Some(HashMap::from([(ChecksumType::CRC32C.to_string(), checksum.encoded.clone())]));
5418                    }
5419                    retry_source_info.parts = Arc::new(retry_source_parts);
5420                    assert_eq!(retry_source_info.etag.as_deref(), Some(retry_object_etag.as_str()));
5421                    assert!(retry_source_info.is_multipart());
5422                    assert!(retry_source_info.parts.iter().all(|part| part.checksums.is_some()));
5423                    assert_eq!(retry_source_info.checksum.as_deref(), Some(retry_object_checksum_bytes.as_ref()));
5424                    assert!(
5425                        !retry_source_info
5426                            .user_defined
5427                            .contains_key(rustfs_rio::RUSTFS_MULTIPART_CHECKSUM)
5428                    );
5429                    assert!(
5430                        !retry_source_info
5431                            .user_defined
5432                            .contains_key(rustfs_rio::RUSTFS_MULTIPART_CHECKSUM_TYPE)
5433                    );
5434                    retry_reader.object_info = retry_source_info.clone();
5435                    temp_env::async_with_vars(
5436                        [
5437                            (rustfs_config::ENV_DATA_MOVEMENT_PART_CHECKSUMS_WRITE, None::<&str>),
5438                            (rustfs_config::ENV_DATA_MOVEMENT_PART_CHECKSUMS_FLEET_CONFIRMED, None::<&str>),
5439                        ],
5440                        crate::data_movement::migrate_object(
5441                            store.clone(),
5442                            0,
5443                            bucket.clone(),
5444                            retry_reader,
5445                            None,
5446                            "test_data_movement_compatible_retry",
5447                        ),
5448                    )
5449                    .await
5450                    .expect("default multipart migration should not require the checksum sidecar capability");
5451
5452                    let compatible_target = store.pools[1]
5453                        .get_object_info(
5454                            &bucket,
5455                            retry_object,
5456                            &ObjectOptions {
5457                                include_part_checksums: true,
5458                                ..Default::default()
5459                            },
5460                        )
5461                        .await
5462                        .expect("read compatible multipart target metadata");
5463                    assert_eq!(compatible_target.checksum, retry_source_info.checksum);
5464                    assert!(compatible_target.parts.iter().all(|part| part.checksums.is_none()));
5465                    assert_eq!(
5466                        rustfs_utils::http::get_consistent_str(
5467                            &compatible_target.user_defined,
5468                            rustfs_utils::http::SUFFIX_DATA_MOVED
5469                        ),
5470                        Some("true")
5471                    );
5472                    assert!(!rustfs_utils::http::contains_key_str(
5473                        &compatible_target.user_defined,
5474                        rustfs_utils::http::SUFFIX_PART_CHECKSUMS
5475                    ));
5476                    let mut compatible_reader = store.pools[1]
5477                        .get_object_reader(&bucket, retry_object, None, HeaderMap::new(), &ObjectOptions::default())
5478                        .await
5479                        .expect("read compatible multipart target body");
5480                    let mut compatible_body = Vec::new();
5481                    compatible_reader
5482                        .stream
5483                        .read_to_end(&mut compatible_body)
5484                        .await
5485                        .expect("drain compatible multipart target body");
5486                    assert_eq!(compatible_body, retry_source_body);
5487
5488                    let mut compatible_retry_reader = store.pools[0]
5489                        .get_object_reader(
5490                            &bucket,
5491                            retry_object,
5492                            None,
5493                            HeaderMap::new(),
5494                            &ObjectOptions {
5495                                raw_data_movement_read: true,
5496                                ..Default::default()
5497                            },
5498                        )
5499                        .await
5500                        .expect("read source multipart object for compatible retry");
5501                    compatible_retry_reader.object_info = retry_source_info.clone();
5502                    temp_env::async_with_vars(
5503                        [
5504                            (rustfs_config::ENV_DATA_MOVEMENT_PART_CHECKSUMS_WRITE, Some("true")),
5505                            (rustfs_config::ENV_DATA_MOVEMENT_PART_CHECKSUMS_FLEET_CONFIRMED, Some("true")),
5506                        ],
5507                        crate::data_movement::migrate_object(
5508                            store.clone(),
5509                            0,
5510                            bucket.clone(),
5511                            compatible_retry_reader,
5512                            None,
5513                            "test_data_movement_compatible_retry",
5514                        ),
5515                    )
5516                    .await
5517                    .expect("pre-gate multipart target should converge after enabling checksum persistence");
5518                    let compatible_target_after_retry = store.pools[1]
5519                        .get_object_info(
5520                            &bucket,
5521                            retry_object,
5522                            &ObjectOptions {
5523                                include_part_checksums: true,
5524                                ..Default::default()
5525                            },
5526                        )
5527                        .await
5528                        .expect("read pre-gate target after fleet-confirmed retry");
5529                    assert_eq!(compatible_target_after_retry.data_dir, compatible_target.data_dir);
5530                    assert_eq!(compatible_target_after_retry.parts, compatible_target.parts);
5531                    let compatible_uploads = store.pools[1]
5532                        .list_multipart_uploads(&bucket, retry_object, None, None, None, 100)
5533                        .await
5534                        .expect("list compatible target multipart uploads");
5535                    assert!(compatible_uploads.uploads.is_empty(), "compatible retry staging must be aborted");
5536
5537                    store.pools[1]
5538                        .delete_object(&bucket, retry_object, ObjectOptions::default())
5539                        .await
5540                        .expect("remove compatible target before fleet-confirmed migration");
5541                    let mut retry_reader = store.pools[0]
5542                        .get_object_reader(
5543                            &bucket,
5544                            retry_object,
5545                            None,
5546                            HeaderMap::new(),
5547                            &ObjectOptions {
5548                                raw_data_movement_read: true,
5549                                ..Default::default()
5550                            },
5551                        )
5552                        .await
5553                        .expect("read source multipart object for fleet-confirmed migration");
5554                    retry_reader.object_info = retry_source_info.clone();
5555                    temp_env::async_with_vars(
5556                        [
5557                            (rustfs_config::ENV_DATA_MOVEMENT_PART_CHECKSUMS_WRITE, Some("true")),
5558                            (rustfs_config::ENV_DATA_MOVEMENT_PART_CHECKSUMS_FLEET_CONFIRMED, Some("true")),
5559                        ],
5560                        crate::data_movement::migrate_object(
5561                            store.clone(),
5562                            0,
5563                            bucket.clone(),
5564                            retry_reader,
5565                            None,
5566                            "test_data_movement_retry",
5567                        ),
5568                    )
5569                    .await
5570                    .expect("first multipart migration should succeed");
5571
5572                    let target_before_retry = store.pools[1]
5573                        .get_object_info(
5574                            &bucket,
5575                            retry_object,
5576                            &ObjectOptions {
5577                                include_part_checksums: true,
5578                                ..Default::default()
5579                            },
5580                        )
5581                        .await
5582                        .expect("read target metadata before retry");
5583                    assert_eq!(target_before_retry.checksum, retry_source_info.checksum);
5584                    assert_eq!(
5585                        target_before_retry
5586                            .parts
5587                            .iter()
5588                            .map(|part| (&part.number, &part.checksums))
5589                            .collect::<Vec<_>>(),
5590                        retry_source_info
5591                            .parts
5592                            .iter()
5593                            .map(|part| (&part.number, &part.checksums))
5594                            .collect::<Vec<_>>()
5595                    );
5596                    assert_eq!(retry_source_info.mod_time, target_before_retry.mod_time);
5597                    assert!(
5598                        retry_source_info.parts.iter().any(|source_part| {
5599                            target_before_retry
5600                                .parts
5601                                .iter()
5602                                .find(|target_part| target_part.number == source_part.number)
5603                                .is_some_and(|target_part| target_part.mod_time != source_part.mod_time)
5604                        }),
5605                        "multipart migration must rewrite at least one target part timestamp"
5606                    );
5607                    let mut retry_reader = store.pools[0]
5608                        .get_object_reader(
5609                            &bucket,
5610                            retry_object,
5611                            None,
5612                            HeaderMap::new(),
5613                            &ObjectOptions {
5614                                raw_data_movement_read: true,
5615                                ..Default::default()
5616                            },
5617                        )
5618                        .await
5619                        .expect("read source multipart object for retry");
5620                    retry_reader.object_info = retry_source_info;
5621                    temp_env::async_with_vars(
5622                        [
5623                            (rustfs_config::ENV_DATA_MOVEMENT_PART_CHECKSUMS_WRITE, Some("true")),
5624                            (rustfs_config::ENV_DATA_MOVEMENT_PART_CHECKSUMS_FLEET_CONFIRMED, Some("true")),
5625                        ],
5626                        crate::data_movement::migrate_object(
5627                            store.clone(),
5628                            0,
5629                            bucket.clone(),
5630                            retry_reader,
5631                            None,
5632                            "test_data_movement_retry",
5633                        ),
5634                    )
5635                    .await
5636                    .expect("equivalent multipart migration retry should converge");
5637
5638                    let target_after_retry = store.pools[1]
5639                        .get_object_info(&bucket, retry_object, &ObjectOptions::default())
5640                        .await
5641                        .expect("read target metadata after retry");
5642                    assert_eq!(target_after_retry.data_dir, target_before_retry.data_dir);
5643                    assert_eq!(target_after_retry.etag, target_before_retry.etag);
5644                    assert_eq!(target_after_retry.checksum, target_before_retry.checksum);
5645                    assert_eq!(target_after_retry.mod_time, target_before_retry.mod_time);
5646                    assert!(!rustfs_utils::http::contains_key_str(
5647                        &target_after_retry.user_defined,
5648                        rustfs_utils::http::SUFFIX_PART_CHECKSUMS
5649                    ));
5650                    assert!(target_after_retry.parts.iter().all(|part| part.checksums.is_none()));
5651                    let cached_target_after_retry = store.pools[1]
5652                        .get_object_info(&bucket, retry_object, &ObjectOptions::default())
5653                        .await
5654                        .expect("read target metadata from the ordinary cache path");
5655                    assert!(!rustfs_utils::http::contains_key_str(
5656                        &cached_target_after_retry.user_defined,
5657                        rustfs_utils::http::SUFFIX_PART_CHECKSUMS
5658                    ));
5659                    assert!(cached_target_after_retry.parts.iter().all(|part| part.checksums.is_none()));
5660                    let hydrated_target_after_retry = store.pools[1]
5661                        .get_object_info(
5662                            &bucket,
5663                            retry_object,
5664                            &ObjectOptions {
5665                                include_part_checksums: true,
5666                                ..Default::default()
5667                            },
5668                        )
5669                        .await
5670                        .expect("hydrate target metadata after ordinary cache reads");
5671                    assert_eq!(hydrated_target_after_retry.user_defined, target_before_retry.user_defined);
5672                    assert_eq!(hydrated_target_after_retry.parts, target_before_retry.parts);
5673
5674                    let mut target_reader = store.pools[1]
5675                        .get_object_reader(&bucket, retry_object, None, HeaderMap::new(), &ObjectOptions::default())
5676                        .await
5677                        .expect("read target multipart body after retry");
5678                    let mut target_body = Vec::new();
5679                    target_reader
5680                        .stream
5681                        .read_to_end(&mut target_body)
5682                        .await
5683                        .expect("drain target multipart body after retry");
5684                    assert_eq!(target_body, retry_source_body);
5685
5686                    let uploads = store.pools[1]
5687                        .list_multipart_uploads(&bucket, retry_object, None, None, None, 100)
5688                        .await
5689                        .expect("list retry target multipart uploads");
5690                    assert!(uploads.uploads.is_empty(), "retry migration staging must be aborted");
5691
5692                    Box::pin(async {
5693                        store.pools[0]
5694                            .delete_object(&bucket, retry_object, ObjectOptions::default())
5695                            .await
5696                            .expect("remove first-hop source before second migration");
5697                        *store.rebalance_meta.write().await = Some(active_rebalance_meta_for_pool(store.pools.len(), 1));
5698                        let second_hop_reader = store.pools[1]
5699                            .get_object_reader(
5700                                &bucket,
5701                                retry_object,
5702                                None,
5703                                HeaderMap::new(),
5704                                &ObjectOptions {
5705                                    data_movement: true,
5706                                    raw_data_movement_read: true,
5707                                    ..Default::default()
5708                                },
5709                            )
5710                            .await
5711                            .expect("read first target as the second migration source");
5712                        assert!(
5713                            second_hop_reader
5714                                .object_info
5715                                .parts
5716                                .iter()
5717                                .all(|part| part.checksums.is_some()),
5718                            "data-movement source reads must hydrate persisted part checksums"
5719                        );
5720                        temp_env::async_with_vars(
5721                            [
5722                                (rustfs_config::ENV_DATA_MOVEMENT_PART_CHECKSUMS_WRITE, Some("true")),
5723                                (rustfs_config::ENV_DATA_MOVEMENT_PART_CHECKSUMS_FLEET_CONFIRMED, Some("true")),
5724                            ],
5725                            crate::data_movement::migrate_object(
5726                                store.clone(),
5727                                1,
5728                                bucket.clone(),
5729                                second_hop_reader,
5730                                None,
5731                                "test_data_movement_second_hop",
5732                            ),
5733                        )
5734                        .await
5735                        .expect("second multipart migration should preserve part checksums");
5736
5737                        let second_hop_target = store.pools[0]
5738                            .get_object_info(
5739                                &bucket,
5740                                retry_object,
5741                                &ObjectOptions {
5742                                    include_part_checksums: true,
5743                                    ..Default::default()
5744                                },
5745                            )
5746                            .await
5747                            .expect("read second-hop target metadata");
5748                        assert_eq!(second_hop_target.checksum, target_before_retry.checksum);
5749                        assert_eq!(
5750                            second_hop_target
5751                                .parts
5752                                .iter()
5753                                .map(|part| (&part.number, &part.checksums))
5754                                .collect::<Vec<_>>(),
5755                            target_before_retry
5756                                .parts
5757                                .iter()
5758                                .map(|part| (&part.number, &part.checksums))
5759                                .collect::<Vec<_>>()
5760                        );
5761                    })
5762                    .await;
5763                });
5764            })
5765            .expect("spawn data movement test thread")
5766            .join()
5767            .expect("join data movement test thread");
5768    }
5769
5770    #[test]
5771    #[serial_test::serial(storage_class_env)]
5772    fn data_movement_multipart_replaces_only_unlocked_owned_generation() {
5773        std::thread::Builder::new()
5774            .stack_size(16 * 1024 * 1024)
5775            .spawn(|| {
5776                let runtime = tokio::runtime::Builder::new_multi_thread()
5777                    .enable_all()
5778                    .worker_threads(2)
5779                    .build()
5780                    .expect("build data movement replacement test runtime");
5781                runtime.block_on(async move {
5782                    let temp_dir = tempfile::tempdir().expect("create data movement replacement store dir");
5783                    let (_ctx, store, _shutdown) = without_storage_class_env(build_isolated_test_store(
5784                        temp_dir.path(),
5785                        "data-movement-stale-replacement",
5786                        &[4, 4],
5787                    ))
5788                    .await;
5789                    crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
5790
5791                    let bucket = format!("dm-stale-replacement-{}", uuid::Uuid::new_v4());
5792                    store
5793                        .make_bucket(
5794                            &bucket,
5795                            &MakeBucketOptions {
5796                                lock_enabled: true,
5797                                ..Default::default()
5798                            },
5799                        )
5800                        .await
5801                        .expect("create data movement replacement bucket");
5802                    *store.rebalance_meta.write().await = Some(active_rebalance_meta_for_pool(store.pools.len(), 0));
5803
5804                    let version_id = uuid::Uuid::new_v4();
5805                    let old_time = OffsetDateTime::UNIX_EPOCH + time::Duration::SECOND;
5806                    let new_time = old_time + time::Duration::SECOND;
5807                    let first_part_size = 5 * 1024 * 1024;
5808                    let mut old_body = vec![b'a'; first_part_size];
5809                    old_body.push(b'b');
5810                    let mut new_body = vec![b'c'; first_part_size];
5811                    new_body.push(b'd');
5812
5813                    let source_reader = |name: &'static str, body: Vec<u8>, mod_time, metadata: HashMap<String, String>| {
5814                        seed_data_movement_source_reader(
5815                            &store,
5816                            &bucket,
5817                            name,
5818                            DataMovementSourceSeed {
5819                                pool_idx: 0,
5820                                version_id: Some(version_id),
5821                                body,
5822                                multipart_split: Some(first_part_size),
5823                                mod_time,
5824                                user_defined: metadata,
5825                            },
5826                        )
5827                    };
5828
5829                    let replaceable = "replaceable.bin";
5830                    crate::data_movement::migrate_object(
5831                        store.clone(),
5832                        0,
5833                        bucket.clone(),
5834                        source_reader(
5835                            replaceable,
5836                            old_body.clone(),
5837                            old_time,
5838                            HashMap::from([("x-amz-meta-generation".to_string(), "old".to_string())]),
5839                        )
5840                        .await,
5841                        None,
5842                        "test_stale_target_seed",
5843                    )
5844                    .await
5845                    .expect("seed old migrated target");
5846                    let seeded_target = store.pools[1]
5847                        .get_object_info(
5848                            &bucket,
5849                            replaceable,
5850                            &ObjectOptions {
5851                                versioned: true,
5852                                version_id: Some(version_id.to_string()),
5853                                ..Default::default()
5854                            },
5855                        )
5856                        .await
5857                        .expect("read old migrated target");
5858                    let replacement_opts = ObjectOptions {
5859                        data_movement: true,
5860                        versioned: true,
5861                        version_id: Some(version_id.to_string()),
5862                        mod_time: Some(new_time),
5863                        http_preconditions: Some(crate::data_movement::data_movement_target_precondition()),
5864                        ..Default::default()
5865                    };
5866                    assert!(
5867                        crate::data_movement::can_replace_stale_data_movement_target(&seeded_target, &replacement_opts),
5868                        "seeded target should be replaceable: {seeded_target:?}"
5869                    );
5870                    crate::data_movement::migrate_object(
5871                        store.clone(),
5872                        0,
5873                        bucket.clone(),
5874                        source_reader(
5875                            replaceable,
5876                            new_body.clone(),
5877                            new_time,
5878                            HashMap::from([("x-amz-meta-generation".to_string(), "new".to_string())]),
5879                        )
5880                        .await,
5881                        None,
5882                        "test_stale_target_replace",
5883                    )
5884                    .await
5885                    .expect("newer source generation should replace the old migrated target");
5886
5887                    let replacement = store.pools[1]
5888                        .get_object_info(
5889                            &bucket,
5890                            replaceable,
5891                            &ObjectOptions {
5892                                versioned: true,
5893                                version_id: Some(version_id.to_string()),
5894                                ..Default::default()
5895                            },
5896                        )
5897                        .await
5898                        .expect("read replaced target metadata");
5899                    assert_eq!(replacement.mod_time, Some(new_time));
5900                    assert_eq!(replacement.user_defined.get("x-amz-meta-generation").map(String::as_str), Some("new"));
5901                    let mut replacement_reader = store.pools[1]
5902                        .get_object_reader(
5903                            &bucket,
5904                            replaceable,
5905                            None,
5906                            HeaderMap::new(),
5907                            &ObjectOptions {
5908                                versioned: true,
5909                                version_id: Some(version_id.to_string()),
5910                                ..Default::default()
5911                            },
5912                        )
5913                        .await
5914                        .expect("read replaced target body");
5915                    let mut replacement_body = Vec::new();
5916                    replacement_reader
5917                        .stream
5918                        .read_to_end(&mut replacement_body)
5919                        .await
5920                        .expect("drain replaced target body");
5921                    assert_eq!(replacement_body, new_body);
5922
5923                    let client_target = "client-target.bin";
5924                    let client_body = b"client-owned exact version".to_vec();
5925                    let mut client_reader = PutObjReader::from_vec(client_body.clone());
5926                    store.pools[1]
5927                        .put_object(
5928                            &bucket,
5929                            client_target,
5930                            &mut client_reader,
5931                            &ObjectOptions {
5932                                versioned: true,
5933                                version_id: Some(version_id.to_string()),
5934                                mod_time: Some(old_time),
5935                                user_defined: HashMap::from([("x-amz-meta-owner".to_string(), "client".to_string())]),
5936                                ..Default::default()
5937                            },
5938                        )
5939                        .await
5940                        .expect("seed client-owned exact version");
5941                    let err = crate::data_movement::migrate_object(
5942                        store.clone(),
5943                        0,
5944                        bucket.clone(),
5945                        source_reader(client_target, new_body.clone(), new_time, HashMap::new()).await,
5946                        None,
5947                        "test_client_target_reject",
5948                    )
5949                    .await
5950                    .expect_err("data movement must not replace a client-owned exact version");
5951                    assert!(err.to_string().contains("complete_multipart_upload"), "unexpected migration error: {err}");
5952                    let mut preserved_reader = store.pools[1]
5953                        .get_object_reader(
5954                            &bucket,
5955                            client_target,
5956                            None,
5957                            HeaderMap::new(),
5958                            &ObjectOptions {
5959                                versioned: true,
5960                                version_id: Some(version_id.to_string()),
5961                                ..Default::default()
5962                            },
5963                        )
5964                        .await
5965                        .expect("read preserved client target");
5966                    let mut preserved_body = Vec::new();
5967                    preserved_reader
5968                        .stream
5969                        .read_to_end(&mut preserved_body)
5970                        .await
5971                        .expect("drain preserved client target");
5972                    assert_eq!(preserved_body, client_body);
5973
5974                    let acknowledged_target = "acknowledged-target.bin";
5975                    crate::data_movement::migrate_object(
5976                        store.clone(),
5977                        0,
5978                        bucket.clone(),
5979                        source_reader(
5980                            acknowledged_target,
5981                            old_body.clone(),
5982                            old_time,
5983                            HashMap::from([("x-amz-meta-generation".to_string(), "old".to_string())]),
5984                        )
5985                        .await,
5986                        None,
5987                        "test_acknowledged_target_seed",
5988                    )
5989                    .await
5990                    .expect("seed migrated target before tag acknowledgement");
5991                    let target_version_opts = ObjectOptions {
5992                        versioned: true,
5993                        version_id: Some(version_id.to_string()),
5994                        ..Default::default()
5995                    };
5996                    store
5997                        .put_object_tags(&bucket, acknowledged_target, "acknowledged=true", &target_version_opts)
5998                        .await
5999                        .expect("tag update should acknowledge the migrated target");
6000                    let acknowledged = store.pools[1]
6001                        .get_object_info(&bucket, acknowledged_target, &target_version_opts)
6002                        .await
6003                        .expect("read acknowledged migrated target");
6004                    let rustfs_data_moved_key = rustfs_utils::http::internal_key_rustfs(rustfs_utils::http::SUFFIX_DATA_MOVED);
6005                    let minio_data_moved_key =
6006                        format!("{}{}", rustfs_utils::http::MINIO_INTERNAL_PREFIX, rustfs_utils::http::SUFFIX_DATA_MOVED);
6007                    assert_eq!(acknowledged.user_defined.get(&rustfs_data_moved_key).map(String::as_str), Some(""));
6008                    assert_eq!(acknowledged.user_defined.get(&minio_data_moved_key).map(String::as_str), Some(""));
6009                    assert_eq!(
6010                        rustfs_utils::http::get_consistent_str(&acknowledged.user_defined, rustfs_utils::http::SUFFIX_DATA_MOVED),
6011                        None
6012                    );
6013
6014                    let err = crate::data_movement::migrate_object(
6015                        store.clone(),
6016                        0,
6017                        bucket.clone(),
6018                        source_reader(
6019                            acknowledged_target,
6020                            new_body.clone(),
6021                            new_time,
6022                            HashMap::from([("x-amz-meta-generation".to_string(), "new".to_string())]),
6023                        )
6024                        .await,
6025                        None,
6026                        "test_acknowledged_target_reject",
6027                    )
6028                    .await
6029                    .expect_err("data movement must not replace a target acknowledged by a tag update");
6030                    assert!(err.to_string().contains("complete_multipart_upload"), "unexpected migration error: {err}");
6031
6032                    let acknowledged_tags = store.pools[1]
6033                        .get_object_tags(&bucket, acknowledged_target, &target_version_opts)
6034                        .await
6035                        .expect("read preserved acknowledged target tags");
6036                    assert_eq!(acknowledged_tags, "acknowledged=true");
6037                    let mut acknowledged_reader = store.pools[1]
6038                        .get_object_reader(&bucket, acknowledged_target, None, HeaderMap::new(), &target_version_opts)
6039                        .await
6040                        .expect("read preserved acknowledged target body");
6041                    let mut acknowledged_body = Vec::new();
6042                    acknowledged_reader
6043                        .stream
6044                        .read_to_end(&mut acknowledged_body)
6045                        .await
6046                        .expect("drain preserved acknowledged target body");
6047                    assert_eq!(acknowledged_body, old_body);
6048
6049                    let metadata_acknowledged_target = "metadata-acknowledged-target.bin";
6050                    crate::data_movement::migrate_object(
6051                        store.clone(),
6052                        0,
6053                        bucket.clone(),
6054                        source_reader(
6055                            metadata_acknowledged_target,
6056                            old_body.clone(),
6057                            old_time,
6058                            HashMap::from([("x-amz-meta-generation".to_string(), "old".to_string())]),
6059                        )
6060                        .await,
6061                        None,
6062                        "test_metadata_acknowledged_target_seed",
6063                    )
6064                    .await
6065                    .expect("seed migrated target before metadata acknowledgement");
6066                    store
6067                        .put_object_metadata(
6068                            &bucket,
6069                            metadata_acknowledged_target,
6070                            &ObjectOptions {
6071                                eval_metadata: Some(HashMap::from([(
6072                                    rustfs_filemeta::metadata_keys::OBJECT_LOCK_LEGAL_HOLD.to_string(),
6073                                    s3s::dto::ObjectLockLegalHoldStatus::OFF.to_string(),
6074                                )])),
6075                                ..target_version_opts.clone()
6076                            },
6077                        )
6078                        .await
6079                        .expect("legal hold metadata update should acknowledge the migrated target");
6080                    let metadata_acknowledged = store.pools[1]
6081                        .get_object_info(&bucket, metadata_acknowledged_target, &target_version_opts)
6082                        .await
6083                        .expect("read metadata-acknowledged migrated target");
6084                    assert_eq!(
6085                        metadata_acknowledged
6086                            .user_defined
6087                            .get(&rustfs_data_moved_key)
6088                            .map(String::as_str),
6089                        Some("")
6090                    );
6091                    assert_eq!(
6092                        metadata_acknowledged
6093                            .user_defined
6094                            .get(&minio_data_moved_key)
6095                            .map(String::as_str),
6096                        Some("")
6097                    );
6098                    assert_eq!(
6099                        metadata_acknowledged
6100                            .user_defined
6101                            .get(rustfs_filemeta::metadata_keys::OBJECT_LOCK_LEGAL_HOLD)
6102                            .map(String::as_str),
6103                        Some("OFF")
6104                    );
6105                    let err = crate::data_movement::migrate_object(
6106                        store.clone(),
6107                        0,
6108                        bucket.clone(),
6109                        source_reader(
6110                            metadata_acknowledged_target,
6111                            new_body.clone(),
6112                            new_time,
6113                            HashMap::from([("x-amz-meta-generation".to_string(), "new".to_string())]),
6114                        )
6115                        .await,
6116                        None,
6117                        "test_metadata_acknowledged_target_reject",
6118                    )
6119                    .await
6120                    .expect_err("data movement must not replace a target acknowledged by a metadata update");
6121                    assert!(err.to_string().contains("complete_multipart_upload"), "unexpected migration error: {err}");
6122
6123                    let retain_until = (OffsetDateTime::now_utc() + time::Duration::days(1))
6124                        .format(&time::format_description::well_known::Rfc3339)
6125                        .expect("retain-until date should format");
6126                    for (object, mode) in [
6127                        ("compliance-target.bin", s3s::dto::ObjectLockRetentionMode::COMPLIANCE),
6128                        ("governance-target.bin", s3s::dto::ObjectLockRetentionMode::GOVERNANCE),
6129                    ] {
6130                        let retained_metadata = HashMap::from([
6131                            (rustfs_filemeta::metadata_keys::OBJECT_LOCK_MODE.to_string(), mode.to_string()),
6132                            (
6133                                rustfs_filemeta::metadata_keys::OBJECT_LOCK_RETAIN_UNTIL_DATE.to_string(),
6134                                retain_until.clone(),
6135                            ),
6136                        ]);
6137                        crate::data_movement::migrate_object(
6138                            store.clone(),
6139                            0,
6140                            bucket.clone(),
6141                            source_reader(object, old_body.clone(), old_time, retained_metadata.clone()).await,
6142                            None,
6143                            "test_retained_target_seed",
6144                        )
6145                        .await
6146                        .expect("seed retained migrated target");
6147                        let err = crate::data_movement::migrate_object(
6148                            store.clone(),
6149                            0,
6150                            bucket.clone(),
6151                            source_reader(object, new_body.clone(), new_time, retained_metadata).await,
6152                            None,
6153                            "test_retained_target_replace",
6154                        )
6155                        .await
6156                        .expect_err("active retention must block stale target replacement");
6157                        assert!(err.to_string().contains("complete_multipart_upload"), "unexpected retention error: {err}");
6158
6159                        let mut retained_reader = store.pools[1]
6160                            .get_object_reader(
6161                                &bucket,
6162                                object,
6163                                None,
6164                                HeaderMap::new(),
6165                                &ObjectOptions {
6166                                    versioned: true,
6167                                    version_id: Some(version_id.to_string()),
6168                                    ..Default::default()
6169                                },
6170                            )
6171                            .await
6172                            .expect("read retained target");
6173                        let mut retained_body = Vec::new();
6174                        retained_reader
6175                            .stream
6176                            .read_to_end(&mut retained_body)
6177                            .await
6178                            .expect("drain retained target");
6179                        assert_eq!(retained_body, old_body);
6180                    }
6181
6182                    for object in [
6183                        replaceable,
6184                        client_target,
6185                        acknowledged_target,
6186                        metadata_acknowledged_target,
6187                        "compliance-target.bin",
6188                        "governance-target.bin",
6189                    ] {
6190                        let uploads = store.pools[1]
6191                            .list_multipart_uploads(&bucket, object, None, None, None, 100)
6192                            .await
6193                            .expect("list target multipart staging");
6194                        assert!(uploads.uploads.is_empty(), "data movement staging must be cleaned for {object}");
6195                    }
6196                });
6197            })
6198            .expect("spawn data movement replacement test thread")
6199            .join()
6200            .expect("join data movement replacement test thread");
6201    }
6202
6203    #[tokio::test]
6204    #[serial_test::serial(storage_class_env)]
6205    async fn data_movement_delete_marker_retries_converge_safely() {
6206        let temp_dir = tempfile::tempdir().expect("create delete-marker data movement store dir");
6207        let (_ctx, store, _shutdown) =
6208            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "data-movement-delete-marker", &[4, 4])).await;
6209        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
6210
6211        let bucket = format!("dm-delete-{}", uuid::Uuid::new_v4());
6212        store
6213            .make_bucket(&bucket, &MakeBucketOptions::default())
6214            .await
6215            .expect("create delete-marker data movement bucket");
6216        let object = "delete-marker-retry";
6217        let version = uuid::Uuid::new_v4();
6218        let mod_time = OffsetDateTime::UNIX_EPOCH + time::Duration::SECOND;
6219        let replica_timestamp = mod_time + time::Duration::SECOND;
6220        let replication_timestamp = mod_time;
6221        let replica_timestamp_string = replica_timestamp
6222            .format(&time::format_description::well_known::Rfc3339)
6223            .expect("replica timestamp should format as RFC3339");
6224        let replication_timestamp_string = replication_timestamp
6225            .format(&time::format_description::well_known::Rfc3339)
6226            .expect("replication timestamp should format as RFC3339");
6227        let replication_status = "arn:minio:replication::TenantA:bucket=COMPLETED;";
6228        store.pools[0]
6229            .delete_object(
6230                &bucket,
6231                object,
6232                ObjectOptions {
6233                    versioned: true,
6234                    version_id: Some(version.to_string()),
6235                    delete_marker: true,
6236                    mod_time: Some(mod_time),
6237                    delete_replication: Some(ReplicationState {
6238                        replica_status: ReplicationStatusType::Replica,
6239                        replica_timestamp: Some(replica_timestamp),
6240                        replication_status_internal: Some(replication_status.to_string()),
6241                        replication_timestamp: Some(replication_timestamp),
6242                        targets: replication_statuses_map(replication_status),
6243                        delete_marker: true,
6244                        ..Default::default()
6245                    }),
6246                    ..Default::default()
6247                },
6248            )
6249            .await
6250            .expect("write source delete marker");
6251        let source = store.pools[0]
6252            .get_object_info(
6253                &bucket,
6254                object,
6255                &ObjectOptions {
6256                    versioned: true,
6257                    version_id: Some(version.to_string()),
6258                    ..Default::default()
6259                },
6260            )
6261            .await
6262            .expect("read source delete marker replication metadata");
6263        assert_eq!(
6264            rustfs_utils::http::get_str(&source.user_defined, rustfs_utils::http::SUFFIX_REPLICA_STATUS).as_deref(),
6265            Some("REPLICA"),
6266            "source metadata: {:?}",
6267            source.user_defined
6268        );
6269
6270        *store.rebalance_meta.write().await = Some(active_rebalance_meta_for_pool(store.pools.len(), 0));
6271        let missing_object = "missing-delete-marker";
6272        let missing_version = uuid::Uuid::new_v4();
6273        let missing_opts = ObjectOptions {
6274            versioned: true,
6275            version_id: Some(missing_version.to_string()),
6276            delete_marker: true,
6277            mod_time: Some(mod_time),
6278            src_pool_idx: 0,
6279            data_movement: true,
6280            skip_decommissioned: true,
6281            ..Default::default()
6282        };
6283        let missing_source_err = store
6284            .delete_object(&bucket, missing_object, missing_opts.clone())
6285            .await
6286            .expect_err("data movement must not recreate a delete marker missing from the source");
6287        assert!(matches!(missing_source_err, StorageError::DataMovementOverwriteErr(_, _, _)));
6288        let missing_target_err = store.pools[1]
6289            .get_object_info(&bucket, missing_object, &missing_opts)
6290            .await
6291            .expect_err("source-missing preflight must not write a target delete marker");
6292        assert!(matches!(
6293            missing_target_err,
6294            StorageError::ObjectNotFound(_, _) | StorageError::VersionNotFound(_, _, _)
6295        ));
6296
6297        let movement_opts = ObjectOptions {
6298            versioned: true,
6299            version_id: Some(version.to_string()),
6300            delete_marker: true,
6301            mod_time: Some(mod_time),
6302            src_pool_idx: 0,
6303            data_movement: true,
6304            skip_decommissioned: true,
6305            ..Default::default()
6306        };
6307        for attempt in 1..=2 {
6308            let moved = store
6309                .delete_object(&bucket, object, movement_opts.clone())
6310                .await
6311                .unwrap_or_else(|err| panic!("delete marker migration attempt {attempt} should converge: {err}"));
6312            assert!(moved.delete_marker);
6313            assert_eq!(moved.version_id, Some(version));
6314        }
6315        let target = store.pools[1]
6316            .get_object_info(&bucket, object, &movement_opts)
6317            .await
6318            .expect("retry must retain the target delete marker");
6319        assert!(target.delete_marker);
6320        assert_eq!(target.mod_time, Some(mod_time));
6321        assert_eq!(
6322            rustfs_utils::http::get_str(&target.user_defined, rustfs_utils::http::SUFFIX_REPLICA_STATUS).as_deref(),
6323            Some("REPLICA")
6324        );
6325        assert_eq!(
6326            rustfs_utils::http::get_str(&target.user_defined, rustfs_utils::http::SUFFIX_REPLICA_TIMESTAMP).as_deref(),
6327            Some(replica_timestamp_string.as_str())
6328        );
6329        assert_eq!(
6330            rustfs_utils::http::get_str(&target.user_defined, rustfs_utils::http::SUFFIX_REPLICATION_STATUS).as_deref(),
6331            Some(replication_status)
6332        );
6333        assert_eq!(
6334            rustfs_utils::http::get_str(&target.user_defined, rustfs_utils::http::SUFFIX_REPLICATION_TIMESTAMP).as_deref(),
6335            Some(replication_timestamp_string.as_str())
6336        );
6337
6338        let null_object = "null-delete-marker-retry";
6339        let null_mod_time = mod_time + time::Duration::seconds(3);
6340        let null_marker = store.pools[0]
6341            .delete_object(
6342                &bucket,
6343                null_object,
6344                ObjectOptions {
6345                    version_suspended: true,
6346                    version_id: Some(uuid::Uuid::nil().to_string()),
6347                    data_movement: true,
6348                    delete_marker: true,
6349                    mod_time: Some(null_mod_time),
6350                    ..Default::default()
6351                },
6352            )
6353            .await
6354            .expect("write source null delete marker");
6355        assert_eq!(null_marker.version_id, Some(uuid::Uuid::nil()));
6356        let null_opts = ObjectOptions {
6357            version_suspended: true,
6358            version_id: Some(uuid::Uuid::nil().to_string()),
6359            delete_marker: true,
6360            mod_time: Some(null_mod_time),
6361            src_pool_idx: 0,
6362            data_movement: true,
6363            skip_decommissioned: true,
6364            ..Default::default()
6365        };
6366        for attempt in 1..=2 {
6367            let moved = store
6368                .delete_object(&bucket, null_object, null_opts.clone())
6369                .await
6370                .unwrap_or_else(|err| panic!("null delete marker migration attempt {attempt} should converge: {err}"));
6371            assert!(moved.delete_marker);
6372            assert_eq!(moved.version_id, Some(uuid::Uuid::nil()));
6373        }
6374        let target_null_marker = store.pools[1]
6375            .get_object_info(&bucket, null_object, &null_opts)
6376            .await
6377            .expect("target must retain the null delete marker identity");
6378        assert!(target_null_marker.delete_marker);
6379        assert_eq!(target_null_marker.version_id, Some(uuid::Uuid::nil()));
6380
6381        let explicit_object = "suspended-explicit-version-delete";
6382        let explicit_version = uuid::Uuid::new_v4();
6383        let mut explicit_reader = PutObjReader::from_vec(b"explicit version".to_vec());
6384        store.pools[0]
6385            .put_object(
6386                &bucket,
6387                explicit_object,
6388                &mut explicit_reader,
6389                &ObjectOptions {
6390                    versioned: true,
6391                    version_id: Some(explicit_version.to_string()),
6392                    ..Default::default()
6393                },
6394            )
6395            .await
6396            .expect("write explicit version before suspended delete");
6397        store.pools[0]
6398            .delete_object(
6399                &bucket,
6400                explicit_object,
6401                ObjectOptions {
6402                    version_suspended: true,
6403                    version_id: Some(explicit_version.to_string()),
6404                    ..Default::default()
6405                },
6406            )
6407            .await
6408            .expect("delete explicit version from suspended bucket");
6409        store.pools[0]
6410            .get_object_info(
6411                &bucket,
6412                explicit_object,
6413                &ObjectOptions {
6414                    versioned: true,
6415                    version_id: Some(explicit_version.to_string()),
6416                    ..Default::default()
6417                },
6418            )
6419            .await
6420            .expect_err("suspended delete must remove the requested UUID version");
6421    }
6422
6423    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6424    #[serial_test::serial(storage_class_env)]
6425    async fn decommission_entry_carries_migration_and_cleanup_mutation_fences() {
6426        let temp_dir = tempfile::tempdir().expect("create decommission delete-fence store dir");
6427        let (_ctx, store, shutdown) = without_storage_class_env(build_isolated_test_store_with_layout(
6428            temp_dir.path(),
6429            "decommission-delete-fence",
6430            &[(2, 4), (1, 4)],
6431            CancellationToken::new(),
6432            None,
6433        ))
6434        .await;
6435        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
6436
6437        let bucket = format!("decom-delete-fence-{}", uuid::Uuid::new_v4());
6438        let object = (0..128)
6439            .map(|index| format!("object-{index}.bin"))
6440            .find(|candidate| store.pools[0].get_disks_by_key(candidate).set_index == 1)
6441            .expect("the deterministic object search should select source set 1");
6442        store
6443            .make_bucket(&bucket, &MakeBucketOptions::default())
6444            .await
6445            .expect("create decommission delete-fence bucket");
6446        let mut source = PutObjReader::from_vec(b"source generation".to_vec());
6447        store.pools[0]
6448            .put_object(&bucket, &object, &mut source, &ObjectOptions::default())
6449            .await
6450            .expect("write source object to the pool being decommissioned");
6451        mark_test_pool_decommissioning(&store, 0).await;
6452        assert!(store.is_suspended(0).await, "pool 0 must be a suspended decommission source");
6453
6454        let barrier = crate::set_disk::PutObjectCommitBarrier::install(
6455            &bucket,
6456            &object,
6457            crate::set_disk::PutObjectCommitPause::AfterNamespace,
6458        );
6459        let cleanup_barrier = crate::data_movement::SourceCleanupDeleteBarrier::install(&bucket, &object);
6460        let source_set = store.pools[0].get_disks_by_key(&object);
6461        assert_eq!(source_set.set_index, 1, "the source entry must exercise the non-fixed set cleanup lock");
6462        let worker_store = Arc::clone(&store);
6463        let worker_bucket = bucket.clone();
6464        let worker_object = object.clone();
6465        let worker = tokio::spawn(async move {
6466            worker_store
6467                .decommission_entry_for_test(
6468                    0,
6469                    MetaCacheEntry {
6470                        name: worker_object,
6471                        ..Default::default()
6472                    },
6473                    worker_bucket,
6474                    source_set,
6475                )
6476                .await
6477        });
6478        barrier.wait_until_paused().await;
6479
6480        let delete_barrier = crate::store::object::DeleteAfterObjectLockSnapshotBarrier::install(&bucket);
6481        let delete_store = Arc::clone(&store);
6482        let delete_bucket = bucket.clone();
6483        let delete_object = object.clone();
6484        let delete = tokio::spawn(async move {
6485            delete_store
6486                .delete_object(&delete_bucket, &delete_object, ObjectOptions::default())
6487                .await
6488        });
6489        delete_barrier.wait_until_paused().await;
6490        delete_barrier.release_and_wait_until_namespace_pending().await;
6491        assert!(
6492            !delete_barrier.namespace_acquired() && !delete.is_finished(),
6493            "DELETE must remain before namespace acquisition behind the decommission worker's target-commit mutation fence"
6494        );
6495
6496        barrier.release();
6497        cleanup_barrier.wait_until_paused().await;
6498        drop(barrier);
6499
6500        let fixed_set = Arc::clone(&store.pools[0].disk_set[0]);
6501        let fixed_mutation_barrier = crate::set_disk::PutObjectCommitBarrier::install(
6502            &bucket,
6503            &object,
6504            crate::set_disk::PutObjectCommitPause::BeforeNamespace,
6505        );
6506        let mutation_bucket = bucket.clone();
6507        let mutation_object = object.clone();
6508        let fixed_mutation = tokio::spawn(async move {
6509            let mut reader = PutObjReader::from_vec(b"fixed-domain replacement".to_vec());
6510            fixed_set
6511                .put_object(&mutation_bucket, &mutation_object, &mut reader, &ObjectOptions::default())
6512                .await
6513        });
6514        fixed_mutation_barrier.wait_until_paused().await;
6515        fixed_mutation_barrier.release_and_wait_until_namespace_pending().await;
6516        assert!(
6517            !fixed_mutation_barrier.namespace_acquired() && !fixed_mutation.is_finished(),
6518            "the source cleanup must retain the fixed mutation fence before the set-0 mutation acquires its namespace"
6519        );
6520        fixed_mutation.abort();
6521        assert!(
6522            fixed_mutation
6523                .await
6524                .expect_err("the fixed-domain mutation should be canceled")
6525                .is_cancelled(),
6526            "the competing fixed-domain mutation must remain cancelable while blocked"
6527        );
6528        drop(fixed_mutation_barrier);
6529
6530        cleanup_barrier.release();
6531        worker
6532            .await
6533            .expect("decommission entry worker should join")
6534            .expect("decommission entry should migrate and clean its source");
6535        delete
6536            .await
6537            .expect("DELETE task should join")
6538            .expect("DELETE should remove the source and migrated target generations");
6539
6540        for pool in &store.pools {
6541            let err = pool
6542                .get_object_info(&bucket, &object, &ObjectOptions::default())
6543                .await
6544                .expect_err("DELETE must remove the source and migrated target copies");
6545            assert!(
6546                matches!(err, StorageError::ObjectNotFound(_, _)),
6547                "unexpected post-delete pool result: {err:?}"
6548            );
6549        }
6550        let err = store
6551            .get_object_info(&bucket, &object, &ObjectOptions::default())
6552            .await
6553            .expect_err("the deleted generation must not become visible again");
6554        assert!(
6555            matches!(err, StorageError::ObjectNotFound(_, _)),
6556            "unexpected post-delete store result: {err:?}"
6557        );
6558
6559        shutdown.cancel();
6560    }
6561
6562    #[test]
6563    fn decommission_retry_fault_budget_counts_successes_across_attempt_changes() {
6564        let cases: &[&[(usize, bool, bool)]] = &[
6565            &[(1, true, true), (2, true, true), (3, true, false)],
6566            &[(1, true, true), (1, true, true), (2, true, false)],
6567            &[(1, true, true), (3, true, false), (3, true, false)],
6568            &[(1, true, true), (2, false, false), (1, true, true), (2, true, false)],
6569            &[(1, true, true), (2, false, false), (3, true, false)],
6570            &[(3, true, false), (4, true, false)],
6571        ];
6572        for case in cases {
6573            let faults = Arc::new(AtomicUsize::new(0));
6574            let hook = decommission_retry_fault_hook("bucket", "object", Arc::clone(&faults));
6575
6576            for (stage, bucket, object, succeeded) in [
6577                ("other-stage", "bucket", "object", true),
6578                (DECOMMISSION_TEST_FAULT_STAGE_MIGRATE_OBJECT, "other-bucket", "object", true),
6579                (DECOMMISSION_TEST_FAULT_STAGE_MIGRATE_OBJECT, "bucket", "other-object", true),
6580                (DECOMMISSION_TEST_FAULT_STAGE_MIGRATE_OBJECT, "bucket", "object", false),
6581            ] {
6582                assert!(!hook(stage, bucket, object, 1, succeeded));
6583            }
6584            assert_eq!(faults.load(Ordering::SeqCst), 0, "unrelated or failed copies must not consume faults");
6585
6586            let mut expected_faults = 0;
6587            for &(attempt, succeeded, expected) in *case {
6588                assert_eq!(
6589                    hook(DECOMMISSION_TEST_FAULT_STAGE_MIGRATE_OBJECT, "bucket", "object", attempt, succeeded),
6590                    expected,
6591                    "fault plan {case:?} at attempt {attempt}"
6592                );
6593                expected_faults += usize::from(expected);
6594                assert_eq!(faults.load(Ordering::SeqCst), expected_faults);
6595            }
6596        }
6597    }
6598
6599    #[test]
6600    #[serial_test::serial(storage_class_env)]
6601    fn decommission_entry_retries_source_changed_without_canceling_other_bucket() {
6602        let handle = std::thread::Builder::new()
6603            .name("decommission_entry_retries_source_changed_without_canceling_other_bucket".to_string())
6604            .stack_size(32 * 1024 * 1024)
6605            .spawn(|| {
6606                let runtime = tokio::runtime::Builder::new_multi_thread()
6607                    .enable_all()
6608                    .worker_threads(2)
6609                    .build()
6610                    .expect("test runtime should build");
6611                runtime.block_on(async {
6612                    let temp_dir = tempfile::tempdir().expect("create decommission retry store dir");
6613                    let (_ctx, store, shutdown) = without_storage_class_env(build_isolated_test_store(
6614                        temp_dir.path(),
6615                        "decommission-entry-retry",
6616                        &[4, 4],
6617                    ))
6618                    .await;
6619                    crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
6620
6621                    let changed_bucket = format!("decom-retry-a-{}", uuid::Uuid::new_v4());
6622                    let other_bucket = format!("decom-retry-b-{}", uuid::Uuid::new_v4());
6623                    for bucket in [&changed_bucket, &other_bucket] {
6624                        store
6625                            .make_bucket(bucket, &MakeBucketOptions::default())
6626                            .await
6627                            .expect("create decommission retry bucket");
6628                    }
6629
6630                    let changed_object = "changed.bin";
6631                    let first_version = uuid::Uuid::new_v4();
6632                    let second_version = uuid::Uuid::new_v4();
6633                    let base_time = OffsetDateTime::now_utc();
6634                    seed_decommission_source(
6635                        &store,
6636                        &changed_bucket,
6637                        changed_object,
6638                        b"first generation".to_vec(),
6639                        &ObjectOptions {
6640                            versioned: true,
6641                            version_id: Some(first_version.to_string()),
6642                            mod_time: Some(base_time),
6643                            ..Default::default()
6644                        },
6645                    )
6646                    .await;
6647                    let other_object = "other.bin";
6648                    seed_decommission_source(
6649                        &store,
6650                        &other_bucket,
6651                        other_object,
6652                        b"other bucket generation".to_vec(),
6653                        &ObjectOptions::default(),
6654                    )
6655                    .await;
6656                    mark_test_pool_decommissioning(&store, 0).await;
6657
6658                    let mutation_calls = Arc::new(AtomicUsize::new(0));
6659                    let mutation_calls_for_hook = Arc::clone(&mutation_calls);
6660                    let mutation_store = Arc::clone(&store);
6661                    let mutation_bucket = changed_bucket.clone();
6662                    let _mutation_guard = crate::core::pools::DecommissionCleanupMutationGuard::install(Arc::new(
6663                        move |bucket, object, attempt| {
6664                            let is_target = bucket == mutation_bucket.as_str() && object == changed_object;
6665                            let calls = Arc::clone(&mutation_calls_for_hook);
6666                            let store = Arc::clone(&mutation_store);
6667                            let bucket = mutation_bucket.clone();
6668                            Box::pin(async move {
6669                                if !is_target {
6670                                    return;
6671                                }
6672                                calls.fetch_add(1, Ordering::SeqCst);
6673                                if attempt == 1 {
6674                                    seed_decommission_source(
6675                                        &store,
6676                                        &bucket,
6677                                        changed_object,
6678                                        b"second generation".to_vec(),
6679                                        &ObjectOptions {
6680                                            versioned: true,
6681                                            version_id: Some(second_version.to_string()),
6682                                            mod_time: Some(base_time + time::Duration::seconds(1)),
6683                                            ..Default::default()
6684                                        },
6685                                    )
6686                                    .await;
6687                                }
6688                            })
6689                        },
6690                    ));
6691
6692                    let ordinary_faults = Arc::new(AtomicUsize::new(0));
6693                    let fault_hook = decommission_retry_fault_hook(&other_bucket, other_object, Arc::clone(&ordinary_faults));
6694                    let _fault_guard = crate::core::pools::DecommissionTestFaultGuard::install(fault_hook);
6695
6696                    let rx = CancellationToken::new();
6697                    let source_changed_exhaustions = Arc::new(AtomicUsize::new(0));
6698                    let changed_incarnation = Some(
6699                        store
6700                            .bucket_incarnation_id(&changed_bucket)
6701                            .await
6702                            .expect("changed bucket incarnation"),
6703                    );
6704                    let other_incarnation = Some(
6705                        store
6706                            .bucket_incarnation_id(&other_bucket)
6707                            .await
6708                            .expect("other bucket incarnation"),
6709                    );
6710                    let (changed_result, other_result) = tokio::join!(
6711                        run_decommission_entry_retry_test(
6712                            &store,
6713                            rx.clone(),
6714                            &changed_bucket,
6715                            changed_object,
6716                            changed_incarnation,
6717                            Arc::clone(&source_changed_exhaustions),
6718                        ),
6719                        run_decommission_entry_retry_test(
6720                            &store,
6721                            rx.clone(),
6722                            &other_bucket,
6723                            other_object,
6724                            other_incarnation,
6725                            Arc::clone(&source_changed_exhaustions),
6726                        )
6727                    );
6728                    changed_result.expect("SourceChanged entry retry must converge");
6729                    other_result.expect("other bucket entry must continue through ordinary copy retries");
6730
6731                    assert_eq!(
6732                        store.pool_meta.read().await.pools[0]
6733                            .decommission
6734                            .as_ref()
6735                            .expect("decommission progress should remain available")
6736                            .items_decommission_failed,
6737                        0,
6738                        "entry completion must not hide an exhausted copy failure"
6739                    );
6740                    assert!(!rx.is_cancelled(), "entry-level SourceChanged must not cancel the shared worker token");
6741                    assert_eq!(mutation_calls.load(Ordering::SeqCst), 2, "entry must be re-listed after SourceChanged");
6742                    assert_eq!(ordinary_faults.load(Ordering::SeqCst), 2, "ordinary copy must consume the retry budget");
6743                    assert_eq!(source_changed_exhaustions.load(Ordering::SeqCst), 0);
6744
6745                    for (version_id, expected_body) in [
6746                        (first_version, b"first generation".as_slice()),
6747                        (second_version, b"second generation".as_slice()),
6748                    ] {
6749                        let opts = ObjectOptions {
6750                            versioned: true,
6751                            version_id: Some(version_id.to_string()),
6752                            ..Default::default()
6753                        };
6754                        assert_decommission_source_absent(&store, &changed_bucket, changed_object, &opts).await;
6755                        assert_eq!(
6756                            read_decommission_target_body(&store, &changed_bucket, changed_object, &opts).await,
6757                            expected_body
6758                        );
6759                    }
6760                    assert_decommission_source_absent(&store, &other_bucket, other_object, &ObjectOptions::default()).await;
6761                    assert_eq!(
6762                        read_decommission_target_body(&store, &other_bucket, other_object, &ObjectOptions::default()).await,
6763                        b"other bucket generation"
6764                    );
6765
6766                    shutdown.cancel();
6767                });
6768            })
6769            .expect("spawn decommission retry test thread");
6770        if let Err(payload) = handle.join() {
6771            std::panic::resume_unwind(payload);
6772        }
6773    }
6774
6775    #[test]
6776    #[serial_test::serial(storage_class_env)]
6777    fn decommission_entry_exhausted_source_changed_retains_source_and_records_failure() {
6778        let handle = std::thread::Builder::new()
6779            .name("decommission_entry_exhausted_source_changed_retains_source_and_records_failure".to_string())
6780            .stack_size(32 * 1024 * 1024)
6781            .spawn(|| {
6782                let runtime = tokio::runtime::Builder::new_multi_thread()
6783                    .enable_all()
6784                    .worker_threads(2)
6785                    .build()
6786                    .expect("test runtime should build");
6787                runtime.block_on(async {
6788                    let temp_dir = tempfile::tempdir().expect("create decommission exhaustion store dir");
6789                    let (_ctx, store, shutdown) = without_storage_class_env(build_isolated_test_store(
6790                        temp_dir.path(),
6791                        "decommission-entry-exhaustion",
6792                        &[4, 4],
6793                    ))
6794                    .await;
6795                    crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
6796
6797                    let bucket = format!("decom-exhausted-{}", uuid::Uuid::new_v4());
6798                    let object = "exhausted.bin";
6799                    store
6800                        .make_bucket(&bucket, &MakeBucketOptions::default())
6801                        .await
6802                        .expect("create decommission exhaustion bucket");
6803                    let original_version = uuid::Uuid::new_v4();
6804                    let base_time = OffsetDateTime::now_utc();
6805                    seed_decommission_source(
6806                        &store,
6807                        &bucket,
6808                        object,
6809                        b"original generation".to_vec(),
6810                        &ObjectOptions {
6811                            versioned: true,
6812                            version_id: Some(original_version.to_string()),
6813                            mod_time: Some(base_time),
6814                            ..Default::default()
6815                        },
6816                    )
6817                    .await;
6818                    mark_test_pool_decommissioning(&store, 0).await;
6819
6820                    let mutation_calls = Arc::new(AtomicUsize::new(0));
6821                    let mutation_calls_for_hook = Arc::clone(&mutation_calls);
6822                    let mutation_store = Arc::clone(&store);
6823                    let mutation_bucket = bucket.clone();
6824                    let _mutation_guard = crate::core::pools::DecommissionCleanupMutationGuard::install(Arc::new(
6825                        move |called_bucket, called_object, _attempt| {
6826                            let is_target = called_bucket == mutation_bucket.as_str() && called_object == object;
6827                            let calls = Arc::clone(&mutation_calls_for_hook);
6828                            let store = Arc::clone(&mutation_store);
6829                            let bucket = mutation_bucket.clone();
6830                            Box::pin(async move {
6831                                if !is_target {
6832                                    return;
6833                                }
6834                                let call = calls.fetch_add(1, Ordering::SeqCst) + 1;
6835                                let offset = i64::try_from(call).expect("entry retry count should fit i64");
6836                                seed_decommission_source(
6837                                    &store,
6838                                    &bucket,
6839                                    object,
6840                                    format!("concurrent generation {call}").into_bytes(),
6841                                    &ObjectOptions {
6842                                        versioned: true,
6843                                        version_id: Some(uuid::Uuid::new_v4().to_string()),
6844                                        mod_time: Some(base_time + time::Duration::seconds(offset)),
6845                                        ..Default::default()
6846                                    },
6847                                )
6848                                .await;
6849                            })
6850                        },
6851                    ));
6852
6853                    let rx = CancellationToken::new();
6854                    let source_changed_exhaustions = Arc::new(AtomicUsize::new(0));
6855                    let incarnation = Some(store.bucket_incarnation_id(&bucket).await.expect("bucket incarnation"));
6856                    run_decommission_entry_retry_test(
6857                        &store,
6858                        rx.clone(),
6859                        &bucket,
6860                        object,
6861                        incarnation,
6862                        Arc::clone(&source_changed_exhaustions),
6863                    )
6864                    .await
6865                    .expect("entry-level exhaustion must stay local below the pool threshold");
6866
6867                    assert!(!rx.is_cancelled(), "one exhausted entry must not cancel other bucket workers");
6868                    assert_eq!(mutation_calls.load(Ordering::SeqCst), crate::core::pools::DECOMMISSION_ENTRY_MAX_ATTEMPTS);
6869                    assert_eq!(source_changed_exhaustions.load(Ordering::SeqCst), 1);
6870                    store.pools[0]
6871                        .get_object_info(
6872                            &bucket,
6873                            object,
6874                            &ObjectOptions {
6875                                versioned: true,
6876                                version_id: Some(original_version.to_string()),
6877                                ..Default::default()
6878                            },
6879                        )
6880                        .await
6881                        .expect("retry exhaustion must retain the original source version");
6882                    let pool_meta = store.pool_meta.read().await;
6883                    let info = pool_meta.pools[0]
6884                        .decommission
6885                        .as_ref()
6886                        .expect("decommission progress must be initialized");
6887                    assert_eq!(info.items_decommission_failed, 1, "exhausted entry must be visible as failed");
6888                    drop(pool_meta);
6889
6890                    shutdown.cancel();
6891                });
6892            })
6893            .expect("spawn decommission retry test thread");
6894        if let Err(payload) = handle.join() {
6895            std::panic::resume_unwind(payload);
6896        }
6897    }
6898
6899    #[test]
6900    #[serial_test::serial(storage_class_env)]
6901    fn decommission_entry_delete_marker_copy_retries_real_path() {
6902        let handle = std::thread::Builder::new()
6903            .name("decommission_entry_delete_marker_copy_retries_real_path".to_string())
6904            .stack_size(32 * 1024 * 1024)
6905            .spawn(|| {
6906                let runtime = tokio::runtime::Builder::new_multi_thread()
6907                    .enable_all()
6908                    .worker_threads(2)
6909                    .build()
6910                    .expect("test runtime should build");
6911                runtime.block_on(async {
6912                    let temp_dir = tempfile::tempdir().expect("create delete marker retry store dir");
6913                    let (_ctx, store, shutdown) = without_storage_class_env(build_isolated_test_store(
6914                        temp_dir.path(),
6915                        "decommission-delete-marker-retry",
6916                        &[4, 4],
6917                    ))
6918                    .await;
6919                    crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
6920
6921                    let bucket = format!("decom-marker-{}", uuid::Uuid::new_v4());
6922                    let object = "marker.bin";
6923                    store
6924                        .make_bucket(&bucket, &MakeBucketOptions::default())
6925                        .await
6926                        .expect("create delete marker retry bucket");
6927                    let data_version = uuid::Uuid::new_v4();
6928                    let marker_version = uuid::Uuid::new_v4();
6929                    let base_time = OffsetDateTime::now_utc();
6930                    seed_decommission_source(
6931                        &store,
6932                        &bucket,
6933                        object,
6934                        b"delete marker data".to_vec(),
6935                        &ObjectOptions {
6936                            versioned: true,
6937                            version_id: Some(data_version.to_string()),
6938                            mod_time: Some(base_time),
6939                            ..Default::default()
6940                        },
6941                    )
6942                    .await;
6943                    store.pools[0]
6944                        .delete_object(
6945                            &bucket,
6946                            object,
6947                            ObjectOptions {
6948                                versioned: true,
6949                                version_id: Some(marker_version.to_string()),
6950                                delete_marker: true,
6951                                mod_time: Some(base_time + time::Duration::seconds(1)),
6952                                ..Default::default()
6953                            },
6954                        )
6955                        .await
6956                        .expect("seed source delete marker");
6957                    mark_test_pool_decommissioning(&store, 0).await;
6958
6959                    let fault_calls = Arc::new(AtomicUsize::new(0));
6960                    let fault_calls_for_hook = Arc::clone(&fault_calls);
6961                    let fault_bucket = bucket.clone();
6962                    let _fault_guard = crate::core::pools::DecommissionTestFaultGuard::install(Arc::new(
6963                        move |stage, called_bucket, called_object, attempt, succeeded| {
6964                            let injected = succeeded
6965                                && stage == DECOMMISSION_TEST_FAULT_STAGE_DELETE_MARKER
6966                                && called_bucket == fault_bucket.as_str()
6967                                && called_object == object
6968                                && attempt < crate::core::pools::DECOMMISSION_VERSION_COPY_ATTEMPTS;
6969                            if injected {
6970                                fault_calls_for_hook.fetch_add(1, Ordering::SeqCst);
6971                            }
6972                            injected
6973                        },
6974                    ));
6975
6976                    let incarnation = Some(store.bucket_incarnation_id(&bucket).await.expect("bucket incarnation"));
6977                    run_decommission_entry_retry_test(
6978                        &store,
6979                        CancellationToken::new(),
6980                        &bucket,
6981                        object,
6982                        incarnation,
6983                        Arc::new(AtomicUsize::new(0)),
6984                    )
6985                    .await
6986                    .expect("delete marker copy retries must converge");
6987
6988                    assert_eq!(
6989                        fault_calls.load(Ordering::SeqCst),
6990                        crate::core::pools::DECOMMISSION_VERSION_COPY_ATTEMPTS - 1
6991                    );
6992                    let marker_opts = ObjectOptions {
6993                        versioned: true,
6994                        version_id: Some(marker_version.to_string()),
6995                        ..Default::default()
6996                    };
6997                    let target_marker = store.pools[1]
6998                        .get_object_info(&bucket, object, &marker_opts)
6999                        .await
7000                        .expect("target delete marker must exist");
7001                    assert!(target_marker.delete_marker);
7002                    assert_decommission_source_absent(&store, &bucket, object, &marker_opts).await;
7003
7004                    let data_opts = ObjectOptions {
7005                        versioned: true,
7006                        version_id: Some(data_version.to_string()),
7007                        ..Default::default()
7008                    };
7009                    assert_eq!(
7010                        read_decommission_target_body(&store, &bucket, object, &data_opts).await,
7011                        b"delete marker data"
7012                    );
7013                    assert_decommission_source_absent(&store, &bucket, object, &data_opts).await;
7014
7015                    shutdown.cancel();
7016                });
7017            })
7018            .expect("spawn decommission retry test thread");
7019        if let Err(payload) = handle.join() {
7020            std::panic::resume_unwind(payload);
7021        }
7022    }
7023
7024    #[cfg(feature = "test-util")]
7025    #[test]
7026    #[serial_test::serial(storage_class_env)]
7027    fn decommission_entry_tiered_copy_retries_real_path() {
7028        let handle = std::thread::Builder::new()
7029            .name("decommission_entry_tiered_copy_retries_real_path".to_string())
7030            .stack_size(32 * 1024 * 1024)
7031            .spawn(|| {
7032                let runtime = tokio::runtime::Builder::new_multi_thread()
7033                    .enable_all()
7034                    .worker_threads(2)
7035                    .build()
7036                    .expect("test runtime should build");
7037                runtime.block_on(async {
7038                    let temp_dir = tempfile::tempdir().expect("create tiered retry store dir");
7039                    let (ctx, store, shutdown) = without_storage_class_env(build_isolated_test_store(
7040                        temp_dir.path(),
7041                        "decommission-tiered-retry",
7042                        &[4, 4],
7043                    ))
7044                    .await;
7045                    crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
7046
7047                    let bucket = format!("decom-tiered-{}", uuid::Uuid::new_v4());
7048                    let object = "tiered.bin";
7049                    store
7050                        .make_bucket(&bucket, &MakeBucketOptions::default())
7051                        .await
7052                        .expect("create tiered retry bucket");
7053                    let mut reader = PutObjReader::from_vec(b"tiered generation".to_vec());
7054                    let original = store.pools[0]
7055                        .put_object(&bucket, object, &mut reader, &ObjectOptions::default())
7056                        .await
7057                        .expect("seed tiered source object");
7058                    let tier_name = format!("DECOM{}", &uuid::Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
7059                    register_mock_tier(&ctx.tier_config_mgr(), &tier_name).await;
7060                    store.pools[0]
7061                        .transition_object(
7062                            &bucket,
7063                            object,
7064                            &ObjectOptions {
7065                                transition: TransitionOptions {
7066                                    status: TRANSITION_PENDING.to_string(),
7067                                    tier: tier_name,
7068                                    etag: original.etag.clone().expect("tiered source ETag"),
7069                                    ..Default::default()
7070                                },
7071                                version_id: original.version_id.map(|version_id| version_id.to_string()),
7072                                mod_time: original.mod_time,
7073                                ..Default::default()
7074                            },
7075                        )
7076                        .await
7077                        .expect("transition source object to mock tier");
7078                    mark_test_pool_decommissioning(&store, 0).await;
7079
7080                    let fault_calls = Arc::new(AtomicUsize::new(0));
7081                    let fault_calls_for_hook = Arc::clone(&fault_calls);
7082                    let fault_bucket = bucket.clone();
7083                    let _fault_guard = crate::core::pools::DecommissionTestFaultGuard::install(Arc::new(
7084                        move |stage, called_bucket, called_object, attempt, succeeded| {
7085                            let injected = succeeded
7086                                && stage == DECOMMISSION_TEST_FAULT_STAGE_TIERED
7087                                && called_bucket == fault_bucket.as_str()
7088                                && called_object == object
7089                                && attempt < crate::core::pools::DECOMMISSION_VERSION_COPY_ATTEMPTS;
7090                            if injected {
7091                                fault_calls_for_hook.fetch_add(1, Ordering::SeqCst);
7092                            }
7093                            injected
7094                        },
7095                    ));
7096
7097                    let incarnation = Some(store.bucket_incarnation_id(&bucket).await.expect("bucket incarnation"));
7098                    run_decommission_entry_retry_test(
7099                        &store,
7100                        CancellationToken::new(),
7101                        &bucket,
7102                        object,
7103                        incarnation,
7104                        Arc::new(AtomicUsize::new(0)),
7105                    )
7106                    .await
7107                    .expect("tiered copy retries must converge");
7108
7109                    assert_eq!(
7110                        fault_calls.load(Ordering::SeqCst),
7111                        crate::core::pools::DECOMMISSION_VERSION_COPY_ATTEMPTS - 1
7112                    );
7113                    let target = store.pools[1]
7114                        .get_object_info(&bucket, object, &ObjectOptions::default())
7115                        .await
7116                        .expect("tiered target metadata must exist");
7117                    assert_eq!(target.transitioned_object.status, rustfs_filemeta::TRANSITION_COMPLETE);
7118                    assert_decommission_source_absent(&store, &bucket, object, &ObjectOptions::default()).await;
7119
7120                    shutdown.cancel();
7121                });
7122            })
7123            .expect("spawn decommission retry test thread");
7124        if let Err(payload) = handle.join() {
7125            std::panic::resume_unwind(payload);
7126        }
7127    }
7128
7129    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7130    #[serial_test::serial(storage_class_env)]
7131    async fn decommission_outer_fence_loss_blocks_target_put_commit() {
7132        let temp_dir = tempfile::tempdir().expect("create decommission PUT fence-loss store dir");
7133        let (_ctx, store, shutdown) =
7134            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "decommission-put-fence-loss", &[4, 4])).await;
7135        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
7136
7137        let bucket = format!("decom-put-fence-loss-{}", uuid::Uuid::new_v4());
7138        let object = "ordinary.bin";
7139        store
7140            .make_bucket(&bucket, &MakeBucketOptions::default())
7141            .await
7142            .expect("create decommission PUT fence-loss bucket");
7143        let mut source = PutObjReader::from_vec(b"source generation".to_vec());
7144        store.pools[0]
7145            .put_object(&bucket, object, &mut source, &ObjectOptions::default())
7146            .await
7147            .expect("write decommission PUT source");
7148        mark_test_pool_decommissioning(&store, 0).await;
7149
7150        let loss_hook = crate::store::object::DecommissionMutationFenceLossHook::install(
7151            &bucket,
7152            object,
7153            crate::store::object::DecommissionMutationFenceTestPhase::Migration,
7154        );
7155        let barrier = crate::set_disk::PutObjectCommitBarrier::install(
7156            &bucket,
7157            object,
7158            crate::set_disk::PutObjectCommitPause::BeforeQuotaRename,
7159        );
7160        let source_set = store.pools[0].get_disks_by_key(object);
7161        let worker_store = Arc::clone(&store);
7162        let worker_bucket = bucket.clone();
7163        let worker = tokio::spawn(async move {
7164            worker_store
7165                .decommission_entry_for_test(
7166                    0,
7167                    MetaCacheEntry {
7168                        name: object.to_string(),
7169                        ..Default::default()
7170                    },
7171                    worker_bucket,
7172                    source_set,
7173                )
7174                .await
7175        });
7176
7177        barrier.wait_until_paused().await;
7178        loss_hook.mark_lost();
7179        barrier.release();
7180        drop(barrier);
7181        worker
7182            .await
7183            .expect("decommission PUT fence-loss worker should join")
7184            .expect("a fenced migration failure should remain retryable at entry scope");
7185
7186        assert_pool_object_absent(&store.pools[1], &bucket, object).await;
7187        assert_pool_object_present(&store.pools[0], &bucket, object).await;
7188        shutdown.cancel();
7189    }
7190
7191    #[test]
7192    #[serial_test::serial(storage_class_env)]
7193    fn decommission_outer_fence_loss_blocks_multipart_commits() {
7194        run_large_stack_async_test(
7195            "decommission-multipart-outer-fence-loss",
7196            decommission_outer_fence_loss_blocks_multipart_commits_case,
7197        );
7198    }
7199
7200    async fn decommission_outer_fence_loss_blocks_multipart_commits_case() {
7201        for (object, pause) in [
7202            ("complete.bin", crate::set_disk::MultipartCommitPause::BeforeLockLost),
7203            ("new-upload.bin", crate::set_disk::MultipartCommitPause::NewUploadBeforeLockLost),
7204        ] {
7205            // Each injected commit failure intentionally leaves a capacity intent for
7206            // reconciliation.  Isolate the scenarios so one failure cannot turn the
7207            // next scenario into a capacity-recovery test before its barrier is hit.
7208            let temp_dir = tempfile::tempdir().expect("create decommission multipart fence-loss store dir");
7209            let (_ctx, store, shutdown) = without_storage_class_env(build_isolated_test_store(
7210                temp_dir.path(),
7211                "decommission-multipart-fence-loss",
7212                &[4, 4],
7213            ))
7214            .await;
7215            crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
7216
7217            let bucket = format!("decom-mpu-fence-loss-{}", uuid::Uuid::new_v4());
7218            store
7219                .make_bucket(&bucket, &MakeBucketOptions::default())
7220                .await
7221                .expect("create decommission multipart fence-loss bucket");
7222            write_decommission_test_multipart_source(&store, 0, &bucket, object).await;
7223            mark_test_pool_decommissioning(&store, 0).await;
7224
7225            let loss_hook = crate::store::object::DecommissionMutationFenceLossHook::install(
7226                &bucket,
7227                object,
7228                crate::store::object::DecommissionMutationFenceTestPhase::Migration,
7229            );
7230            let commit_observation = (pause == crate::set_disk::MultipartCommitPause::NewUploadBeforeLockLost)
7231                .then(|| crate::set_disk::NewMultipartUploadCommitObservation::install(&bucket, object));
7232            let barrier = crate::set_disk::MultipartCommitBarrier::install(&bucket, object, pause);
7233            let source_set = store.pools[0].get_disks_by_key(object);
7234            let recovery_source_set = Arc::clone(&source_set);
7235            let worker_store = Arc::clone(&store);
7236            let worker_bucket = bucket.clone();
7237            let mut worker = tokio::spawn(async move {
7238                worker_store
7239                    .decommission_entry_for_test(
7240                        0,
7241                        MetaCacheEntry {
7242                            name: object.to_string(),
7243                            ..Default::default()
7244                        },
7245                        worker_bucket,
7246                        source_set,
7247                    )
7248                    .await
7249            });
7250
7251            tokio::select! {
7252                _ = barrier.wait_until_paused() => {}
7253                result = &mut worker => {
7254                    panic!("decommission multipart worker finished before the requested {pause:?} commit barrier: {result:?}");
7255                }
7256            }
7257            loss_hook.mark_lost();
7258            barrier.release();
7259            drop(barrier);
7260            worker
7261                .await
7262                .expect("decommission multipart fence-loss worker should join")
7263                .expect("a fenced multipart migration failure should remain retryable at entry scope");
7264
7265            if let Some(commit_observation) = commit_observation {
7266                assert!(
7267                    !commit_observation.committed(),
7268                    "NewMultipart must reject a lost outer decommission/capacity fence even though it does not acquire the final remote-tuple publication fence"
7269                );
7270            }
7271            assert_pool_object_absent(&store.pools[1], &bucket, object).await;
7272            assert_pool_object_present(&store.pools[0], &bucket, object).await;
7273            let uploads = store.pools[1]
7274                .list_multipart_uploads(&bucket, object, None, None, None, 100)
7275                .await
7276                .expect("list target multipart uploads after fenced migration");
7277            assert!(uploads.uploads.is_empty(), "fenced multipart migration must not retain target staging");
7278            drop(loss_hook);
7279            store
7280                .decommission_entry_for_test(
7281                    0,
7282                    MetaCacheEntry {
7283                        name: object.to_string(),
7284                        ..Default::default()
7285                    },
7286                    bucket.clone(),
7287                    recovery_source_set,
7288                )
7289                .await
7290                .expect("same-mutation retry should recover the durable capacity intent");
7291
7292            shutdown.cancel();
7293        }
7294    }
7295
7296    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7297    #[serial_test::serial(storage_class_env)]
7298    async fn decommission_outer_fence_loss_blocks_source_cleanup_delete_commit() {
7299        let temp_dir = tempfile::tempdir().expect("create decommission cleanup fence-loss store dir");
7300        let (_ctx, store, shutdown) =
7301            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "decommission-cleanup-fence-loss", &[4, 4]))
7302                .await;
7303        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
7304
7305        let bucket = format!("decom-cleanup-fence-loss-{}", uuid::Uuid::new_v4());
7306        let object = "cleanup.bin";
7307        store
7308            .make_bucket(&bucket, &MakeBucketOptions::default())
7309            .await
7310            .expect("create decommission cleanup fence-loss bucket");
7311        let mut source = PutObjReader::from_vec(b"source generation".to_vec());
7312        store.pools[0]
7313            .put_object(&bucket, object, &mut source, &ObjectOptions::default())
7314            .await
7315            .expect("write decommission cleanup source");
7316        mark_test_pool_decommissioning(&store, 0).await;
7317
7318        let loss_hook = crate::store::object::DecommissionMutationFenceLossHook::install(
7319            &bucket,
7320            object,
7321            crate::store::object::DecommissionMutationFenceTestPhase::SourceCleanup,
7322        );
7323        let barrier = crate::data_movement::SourceCleanupDeleteBarrier::install(&bucket, object);
7324        let source_set = store.pools[0].get_disks_by_key(object);
7325        let worker_store = Arc::clone(&store);
7326        let worker_bucket = bucket.clone();
7327        let worker = tokio::spawn(async move {
7328            worker_store
7329                .decommission_entry_for_test(
7330                    0,
7331                    MetaCacheEntry {
7332                        name: object.to_string(),
7333                        ..Default::default()
7334                    },
7335                    worker_bucket,
7336                    source_set,
7337                )
7338                .await
7339        });
7340
7341        barrier.wait_until_paused().await;
7342        loss_hook.mark_lost();
7343        barrier.release();
7344        drop(barrier);
7345        let err = worker
7346            .await
7347            .expect("decommission cleanup fence-loss worker should join")
7348            .expect_err("source cleanup must fail after its outer fence is lost");
7349        assert!(
7350            err.to_string().contains("delete_object_commit"),
7351            "cleanup failure must come from the delete commit fence: {err:?}"
7352        );
7353
7354        assert_pool_object_present(&store.pools[0], &bucket, object).await;
7355        assert_pool_object_present(&store.pools[1], &bucket, object).await;
7356        shutdown.cancel();
7357    }
7358
7359    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7360    #[serial_test::serial(storage_class_env)]
7361    async fn reverse_decommission_reuses_fixed_target_fence_for_put_and_multipart() {
7362        let temp_dir = tempfile::tempdir().expect("create reverse decommission store dir");
7363        let (_ctx, store, shutdown) = without_storage_class_env(build_isolated_test_store_with_layout(
7364            temp_dir.path(),
7365            "reverse-decommission-fixed-target",
7366            &[(1, 4), (1, 4)],
7367            CancellationToken::new(),
7368            None,
7369        ))
7370        .await;
7371        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
7372
7373        let bucket = format!("reverse-decom-fixed-target-{}", uuid::Uuid::new_v4());
7374        let object = "ordinary.bin";
7375        let object_body = b"reverse ordinary generation".to_vec();
7376        let self_copy_object = "source-only-self-copy.bin";
7377        let self_copy_body = b"source only copy generation".to_vec();
7378        let multipart_object = "multipart.bin";
7379        let first_part = vec![b'm'; 5 * 1024 * 1024];
7380        let second_part = b"reverse multipart tail".to_vec();
7381        let mut multipart_body = first_part.clone();
7382        multipart_body.extend_from_slice(&second_part);
7383
7384        store
7385            .make_bucket(&bucket, &MakeBucketOptions::default())
7386            .await
7387            .expect("create reverse decommission bucket");
7388        let mut source = PutObjReader::from_vec(object_body.clone());
7389        store.pools[1]
7390            .put_object(&bucket, object, &mut source, &ObjectOptions::default())
7391            .await
7392            .expect("write ordinary source object to pool 1");
7393        let mut self_copy_source = PutObjReader::from_vec(self_copy_body.clone());
7394        store.pools[1]
7395            .put_object(&bucket, self_copy_object, &mut self_copy_source, &ObjectOptions::default())
7396            .await
7397            .expect("write self-copy source object to pool 1");
7398
7399        let upload = store.pools[1]
7400            .new_multipart_upload(&bucket, multipart_object, &ObjectOptions::default())
7401            .await
7402            .expect("create source multipart upload in pool 1");
7403        let mut completed_parts = Vec::with_capacity(2);
7404        for (part_number, bytes) in [(1, first_part.as_slice()), (2, second_part.as_slice())] {
7405            let mut reader = PutObjReader::from_vec(bytes.to_vec());
7406            let part = store.pools[1]
7407                .put_object_part(
7408                    &bucket,
7409                    multipart_object,
7410                    &upload.upload_id,
7411                    part_number,
7412                    &mut reader,
7413                    &ObjectOptions::default(),
7414                )
7415                .await
7416                .expect("write source multipart part");
7417            completed_parts.push(crate::storage_api_contracts::multipart::CompletePart {
7418                part_num: part.part_num,
7419                etag: part.etag,
7420                ..Default::default()
7421            });
7422        }
7423        store.pools[1]
7424            .clone()
7425            .complete_multipart_upload(&bucket, multipart_object, &upload.upload_id, completed_parts, &ObjectOptions::default())
7426            .await
7427            .expect("complete source multipart object in pool 1");
7428
7429        {
7430            let mut pool_meta = store.pool_meta.write().await;
7431            pool_meta.pools[1].decommission = Some(PoolDecommissionInfo {
7432                start_time: Some(OffsetDateTime::now_utc()),
7433                ..Default::default()
7434            });
7435        }
7436        assert!(store.is_suspended(1).await, "pool 1 must be the reverse decommission source");
7437
7438        let self_copy_opts = ObjectOptions {
7439            no_lock: true,
7440            ..Default::default()
7441        };
7442        let mut self_copy_reader = store
7443            .get_object_reader(&bucket, self_copy_object, None, HeaderMap::new(), &self_copy_opts)
7444            .await
7445            .expect("read the suspended source before its target is committed");
7446        let mut self_copy_info = self_copy_reader.object_info.clone();
7447        let mut self_copy_source_body = Vec::new();
7448        self_copy_reader
7449            .stream
7450            .read_to_end(&mut self_copy_source_body)
7451            .await
7452            .expect("drain the suspended self-copy source");
7453        assert_eq!(self_copy_source_body, self_copy_body);
7454        self_copy_info.metadata_only = true;
7455        self_copy_info.put_object_reader = Some(PutObjReader::from_vec(self_copy_source_body));
7456        store
7457            .copy_object(
7458                &bucket,
7459                self_copy_object,
7460                &bucket,
7461                self_copy_object,
7462                &mut self_copy_info,
7463                &self_copy_opts,
7464                &self_copy_opts,
7465            )
7466            .await
7467            .expect("self-copy should read the suspended source and commit to an active pool");
7468        assert_pool_object_present(&store.pools[0], &bucket, self_copy_object).await;
7469        assert_pool_object_present(&store.pools[1], &bucket, self_copy_object).await;
7470
7471        let mut active_copy_reader = store
7472            .get_object_reader(&bucket, self_copy_object, None, HeaderMap::new(), &self_copy_opts)
7473            .await
7474            .expect("read the active self-copy target while the source remains");
7475        let active_copy_data_dir = active_copy_reader.object_info.data_dir;
7476        let mut active_copy_info = active_copy_reader.object_info.clone();
7477        let mut active_copy_body = Vec::new();
7478        active_copy_reader
7479            .stream
7480            .read_to_end(&mut active_copy_body)
7481            .await
7482            .expect("drain the active self-copy target");
7483        assert_eq!(active_copy_body, self_copy_body);
7484        active_copy_info.metadata_only = true;
7485        active_copy_info.put_object_reader = Some(PutObjReader::from_vec(active_copy_body));
7486        let active_copy_result = store
7487            .copy_object(
7488                &bucket,
7489                self_copy_object,
7490                &bucket,
7491                self_copy_object,
7492                &mut active_copy_info,
7493                &self_copy_opts,
7494                &self_copy_opts,
7495            )
7496            .await
7497            .expect("self-copy should keep using the committed active target");
7498        assert_eq!(active_copy_result.data_dir, active_copy_data_dir);
7499
7500        {
7501            let mut pool_meta = store.pool_meta.write().await;
7502            pool_meta.pools[1].decommission = None;
7503        }
7504        mark_test_pool_decommissioning(&store, 1).await;
7505
7506        let cleanup_barrier = crate::data_movement::SourceCleanupDeleteBarrier::install(&bucket, object);
7507        let commit_barrier = crate::set_disk::PutObjectCommitBarrier::install(
7508            &bucket,
7509            object,
7510            crate::set_disk::PutObjectCommitPause::AfterNamespace,
7511        );
7512        let source_set = store.pools[1].get_disks_by_key(object);
7513        let worker_store = Arc::clone(&store);
7514        let worker_bucket = bucket.clone();
7515        let worker = tokio::spawn(async move {
7516            worker_store
7517                .decommission_entry_for_test(
7518                    1,
7519                    MetaCacheEntry {
7520                        name: object.to_string(),
7521                        ..Default::default()
7522                    },
7523                    worker_bucket,
7524                    source_set,
7525                )
7526                .await
7527        });
7528        commit_barrier.wait_until_paused().await;
7529
7530        let delete_barrier = crate::store::object::DeleteAfterObjectLockSnapshotBarrier::install(&bucket);
7531        let delete_store = Arc::clone(&store);
7532        let delete_bucket = bucket.clone();
7533        let delete = tokio::spawn(async move {
7534            delete_store
7535                .delete_object(&delete_bucket, object, ObjectOptions::default())
7536                .await
7537        });
7538        delete_barrier.wait_until_paused().await;
7539        delete_barrier.release_and_wait_until_namespace_pending().await;
7540        assert!(
7541            !delete_barrier.namespace_acquired() && !delete.is_finished(),
7542            "the reverse target commit must keep DELETE behind the fixed read fence"
7543        );
7544        delete.abort();
7545        assert!(
7546            delete
7547                .await
7548                .expect_err("the blocked DELETE should be canceled")
7549                .is_cancelled(),
7550            "canceling the blocked DELETE must not mutate either pool"
7551        );
7552        drop(delete_barrier);
7553
7554        commit_barrier.release();
7555        cleanup_barrier.wait_until_paused().await;
7556        drop(commit_barrier);
7557
7558        let read_opts = ObjectOptions {
7559            no_lock: true,
7560            ..Default::default()
7561        };
7562        let target_info = store.pools[0]
7563            .get_object_info(&bucket, object, &read_opts)
7564            .await
7565            .expect("read the committed active target before source cleanup");
7566        store.pools[1]
7567            .get_object_info(&bucket, object, &read_opts)
7568            .await
7569            .expect("the source must remain present until cleanup is released");
7570        let routed_info = store
7571            .get_object_info(&bucket, object, &read_opts)
7572            .await
7573            .expect("HEAD routing should prefer the active target before source cleanup");
7574        assert_eq!(routed_info.data_dir, target_info.data_dir);
7575
7576        let mut ranged_reader = store
7577            .get_object_reader(
7578                &bucket,
7579                object,
7580                Some(HTTPRangeSpec {
7581                    is_suffix_length: false,
7582                    start: 8,
7583                    end: 15,
7584                }),
7585                HeaderMap::new(),
7586                &read_opts,
7587            )
7588            .await
7589            .expect("ranged GET routing should prefer the active target before source cleanup");
7590        assert_eq!(ranged_reader.object_info.data_dir, target_info.data_dir);
7591        let mut ranged_body = Vec::new();
7592        ranged_reader
7593            .stream
7594            .read_to_end(&mut ranged_body)
7595            .await
7596            .expect("drain the routed target range");
7597        assert_eq!(ranged_body, object_body[8..=15]);
7598
7599        cleanup_barrier.release();
7600        drop(cleanup_barrier);
7601        tokio::time::timeout(Duration::from_secs(60), worker)
7602            .await
7603            .expect("reverse ordinary decommission must not self-deadlock on the fixed target set")
7604            .expect("reverse ordinary decommission worker should join")
7605            .expect("reverse ordinary decommission should complete");
7606
7607        let mut ordinary_reader = store.pools[0]
7608            .get_object_reader(&bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
7609            .await
7610            .expect("read the ordinary object from the fixed target set");
7611        let mut ordinary_target_body = Vec::new();
7612        ordinary_reader
7613            .stream
7614            .read_to_end(&mut ordinary_target_body)
7615            .await
7616            .expect("drain the ordinary target body");
7617        assert_eq!(ordinary_target_body, object_body, "ordinary migration must preserve the full body");
7618        let ordinary_source_err = store.pools[1]
7619            .get_object_info(&bucket, object, &ObjectOptions::default())
7620            .await
7621            .expect_err("ordinary source generation must be cleaned after migration");
7622        assert!(matches!(ordinary_source_err, StorageError::ObjectNotFound(_, _)));
7623
7624        let multipart_source_set = store.pools[1].get_disks_by_key(multipart_object);
7625        let multipart_store = Arc::clone(&store);
7626        let multipart_bucket = bucket.clone();
7627        let multipart_worker = tokio::spawn(async move {
7628            multipart_store
7629                .decommission_entry_for_test(
7630                    1,
7631                    MetaCacheEntry {
7632                        name: multipart_object.to_string(),
7633                        ..Default::default()
7634                    },
7635                    multipart_bucket,
7636                    multipart_source_set,
7637                )
7638                .await
7639        });
7640        tokio::time::timeout(Duration::from_secs(60), multipart_worker)
7641            .await
7642            .expect("reverse multipart decommission must not self-deadlock on new or complete")
7643            .expect("reverse multipart decommission worker should join")
7644            .expect("reverse multipart decommission should complete");
7645
7646        let target_info = store.pools[0]
7647            .get_object_info(&bucket, multipart_object, &ObjectOptions::default())
7648            .await
7649            .expect("read migrated multipart metadata from the fixed target set");
7650        assert!(target_info.is_multipart(), "migration must retain multipart identity");
7651        let mut multipart_reader = store.pools[0]
7652            .get_object_reader(&bucket, multipart_object, None, HeaderMap::new(), &ObjectOptions::default())
7653            .await
7654            .expect("read migrated multipart object from the fixed target set");
7655        let mut multipart_target_body = Vec::new();
7656        multipart_reader
7657            .stream
7658            .read_to_end(&mut multipart_target_body)
7659            .await
7660            .expect("drain the multipart target body");
7661        assert_eq!(multipart_target_body, multipart_body, "multipart migration must preserve the full body");
7662        let multipart_source_err = store.pools[1]
7663            .get_object_info(&bucket, multipart_object, &ObjectOptions::default())
7664            .await
7665            .expect_err("multipart source generation must be cleaned after migration");
7666        assert!(matches!(multipart_source_err, StorageError::ObjectNotFound(_, _)));
7667
7668        shutdown.cancel();
7669    }
7670
7671    #[tokio::test]
7672    #[serial_test::serial(storage_class_env)]
7673    async fn batch_delete_real_path_preserves_source_pool_errors_in_any_pool_order() {
7674        let temp_dir = tempfile::tempdir().expect("create batch delete pool-error store dir");
7675        let (_ctx, store, shutdown) =
7676            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "batch-delete-pool-errors", &[4, 4])).await;
7677        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
7678
7679        for source_pool_idx in [0, 1] {
7680            {
7681                let mut pool_meta = store.pool_meta.write().await;
7682                for pool in &mut pool_meta.pools {
7683                    pool.decommission = None;
7684                }
7685            }
7686
7687            let bucket = format!("batch-del-pool-error-{source_pool_idx}-{}", uuid::Uuid::new_v4());
7688            let object_names = vec![
7689                format!("third-{source_pool_idx}.bin"),
7690                format!("first-{source_pool_idx}.bin"),
7691                format!("second-{source_pool_idx}.bin"),
7692            ];
7693            store
7694                .make_bucket(&bucket, &MakeBucketOptions::default())
7695                .await
7696                .expect("create batch delete pool-error bucket");
7697            for pool in &store.pools {
7698                for object_name in &object_names {
7699                    let mut reader = PutObjReader::from_vec(format!("pool {} {object_name}", pool.pool_idx).into_bytes());
7700                    pool.put_object(&bucket, object_name, &mut reader, &ObjectOptions::default())
7701                        .await
7702                        .expect("seed each object in both the source and active pools");
7703                }
7704            }
7705            {
7706                let mut pool_meta = store.pool_meta.write().await;
7707                pool_meta.pools[source_pool_idx].decommission = Some(PoolDecommissionInfo {
7708                    start_time: Some(OffsetDateTime::now_utc()),
7709                    ..Default::default()
7710                });
7711            }
7712            assert!(
7713                store.is_suspended(source_pool_idx).await,
7714                "the injected error pool must be the decommission source"
7715            );
7716
7717            let expected_errors = [
7718                StorageError::ErasureWriteQuorum,
7719                StorageError::NamespaceLockQuorumUnavailable {
7720                    mode: "delete_objects_commit",
7721                    bucket: bucket.clone(),
7722                    object: object_names[1].clone(),
7723                    required: 3,
7724                    achieved: 2,
7725                },
7726                StorageError::ErasureWriteQuorum,
7727            ];
7728            let injection = crate::store::object::BatchDeletePoolErrorInjection::install(
7729                &bucket,
7730                source_pool_idx,
7731                object_names.iter().cloned().zip(expected_errors.iter().cloned()).collect(),
7732            );
7733            let requests = object_names
7734                .iter()
7735                .map(|object_name| ObjectToDelete {
7736                    object_name: object_name.clone(),
7737                    ..Default::default()
7738                })
7739                .collect();
7740
7741            let (deleted, errors) = store.delete_objects(&bucket, requests, ObjectOptions::default()).await;
7742
7743            assert_eq!(
7744                injection.observed(),
7745                object_names.len(),
7746                "the source pool must first complete every real delete"
7747            );
7748            assert_eq!(
7749                errors,
7750                expected_errors.iter().cloned().map(Some).collect::<Vec<_>>(),
7751                "a successful pool must not clear a source pool failure at any request index"
7752            );
7753            assert_eq!(
7754                deleted.iter().map(|object| object.object_name.as_str()).collect::<Vec<_>>(),
7755                object_names.iter().map(String::as_str).collect::<Vec<_>>(),
7756                "DeleteObjects must preserve request index mapping while aggregating pool failures"
7757            );
7758            assert!(
7759                deleted.iter().all(|object| object.found),
7760                "the injected source results must retain real delete success data"
7761            );
7762
7763            for pool in &store.pools {
7764                for object_name in &object_names {
7765                    let error = pool
7766                        .get_object_info(&bucket, object_name, &ObjectOptions::default())
7767                        .await
7768                        .expect_err("both the active and source pool delete calls must execute");
7769                    assert!(
7770                        matches!(error, StorageError::ObjectNotFound(_, _)),
7771                        "unexpected residual object: {error:?}"
7772                    );
7773                }
7774            }
7775            drop(injection);
7776        }
7777
7778        shutdown.cancel();
7779    }
7780
7781    #[tokio::test]
7782    #[serial_test::serial(storage_class_env)]
7783    async fn decommission_source_cleanup_holds_hashed_set_lock_across_preflight() {
7784        let temp_dir = tempfile::tempdir().expect("create multi-set decommission cleanup store dir");
7785        let (_ctx, store, shutdown) = without_storage_class_env(build_isolated_test_store_with_layout(
7786            temp_dir.path(),
7787            "multi-set-decommission-source-cleanup",
7788            &[(2, 4)],
7789            CancellationToken::new(),
7790            None,
7791        ))
7792        .await;
7793        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
7794
7795        let bucket = format!("decom-source-cleanup-lock-{}", uuid::Uuid::new_v4());
7796        let object = (0..128)
7797            .map(|index| format!("object-{index}.bin"))
7798            .find(|candidate| store.pools[0].get_disks_by_key(candidate).set_index == 1)
7799            .expect("the deterministic object search should select source set 1");
7800        store
7801            .make_bucket(&bucket, &MakeBucketOptions::default())
7802            .await
7803            .expect("create multi-set decommission cleanup bucket");
7804        let mut source = PutObjReader::from_vec(b"source generation".to_vec());
7805        store.pools[0]
7806            .put_object(&bucket, &object, &mut source, &ObjectOptions::default())
7807            .await
7808            .expect("write the source generation to set 1");
7809        let source_set = store.pools[0].get_disks_by_key(&object);
7810        assert_eq!(source_set.set_index, 1, "the source must not share the fixed set-0 namespace");
7811        let expected_source_versions = source_set
7812            .load_file_info_versions_exact(&bucket, &object)
7813            .await
7814            .expect("source versions should be readable")
7815            .expect("the source generation should exist");
7816
7817        let cleanup_barrier = crate::data_movement::SourceCleanupDeleteBarrier::install(&bucket, &object);
7818        let cleanup_store = Arc::clone(&store);
7819        let cleanup_bucket = bucket.clone();
7820        let cleanup_object = object.clone();
7821        let cleanup = tokio::spawn(async move {
7822            let mutation_fence = cleanup_store
7823                .acquire_decommission_source_cleanup_fence(&cleanup_bucket, &cleanup_object, source_set.as_ref())
7824                .await?;
7825            crate::data_movement::cleanup_source_entry_if_unchanged(
7826                source_set,
7827                &cleanup_bucket,
7828                &cleanup_object,
7829                &expected_source_versions,
7830                &[],
7831                crate::data_movement::SourceCleanupBucketFence {
7832                    object_mutation_fence: Some(&mutation_fence),
7833                    ..Default::default()
7834                },
7835                "test_multi_set_decommission_source_cleanup",
7836            )
7837            .await
7838        });
7839        cleanup_barrier.wait_until_paused().await;
7840
7841        let put_barrier = crate::set_disk::PutObjectCommitBarrier::install(
7842            &bucket,
7843            &object,
7844            crate::set_disk::PutObjectCommitPause::BeforeNamespace,
7845        );
7846        let mutation_pool = Arc::clone(&store.pools[0]);
7847        let mutation_bucket = bucket.clone();
7848        let mutation_object = object.clone();
7849        let replacement = b"replacement generation".to_vec();
7850        let expected_replacement = replacement.clone();
7851        let mutation = tokio::spawn(async move {
7852            let mut reader = PutObjReader::from_vec(replacement);
7853            mutation_pool
7854                .put_object(&mutation_bucket, &mutation_object, &mut reader, &ObjectOptions::default())
7855                .await
7856        });
7857        put_barrier.wait_until_paused().await;
7858        put_barrier.release_and_wait_until_namespace_pending().await;
7859        assert!(!mutation.is_finished(), "a source mutation must wait behind cleanup's set-1 write lock");
7860
7861        cleanup_barrier.release();
7862        cleanup
7863            .await
7864            .expect("source cleanup task should join")
7865            .expect("source cleanup should remove only the preflight generation");
7866        mutation
7867            .await
7868            .expect("source mutation task should join")
7869            .expect("source mutation should commit after cleanup releases the set lock");
7870
7871        let mut reader = store.pools[0]
7872            .get_object_reader(&bucket, &object, None, HeaderMap::new(), &ObjectOptions::default())
7873            .await
7874            .expect("the replacement generation must remain readable");
7875        let mut actual = Vec::new();
7876        reader
7877            .stream
7878            .read_to_end(&mut actual)
7879            .await
7880            .expect("read the replacement generation");
7881        assert_eq!(actual, expected_replacement, "cleanup must not delete the replacement generation");
7882
7883        shutdown.cancel();
7884    }
7885
7886    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7887    #[serial_test::serial(storage_class_env)]
7888    async fn versioned_delete_marker_survives_decommission_source_cleanup() {
7889        let temp_dir = tempfile::tempdir().expect("create versioned decommission delete-fence store dir");
7890        let (_ctx, store, shutdown) =
7891            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "versioned-decommission-delete-fence", &[4, 4]))
7892                .await;
7893        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
7894
7895        let bucket = format!("versioned-decom-delete-{}", uuid::Uuid::new_v4());
7896        let object = "object.bin";
7897        store
7898            .make_bucket(&bucket, &MakeBucketOptions::default())
7899            .await
7900            .expect("create versioned decommission delete-fence bucket");
7901        let (source_version, expected_source_versions) = migrate_versioned_decommission_test_object(
7902            &store,
7903            &bucket,
7904            object,
7905            b"source generation",
7906            "test_versioned_decommission_delete_fence",
7907        )
7908        .await;
7909
7910        let delete_barrier = crate::store::object::VersionedDeleteMarkerCommitBarrier::install(&bucket, object);
7911        let delete_store = Arc::clone(&store);
7912        let delete_bucket = bucket.clone();
7913        let delete = tokio::spawn(async move {
7914            delete_store
7915                .delete_object(
7916                    &delete_bucket,
7917                    object,
7918                    ObjectOptions {
7919                        versioned: true,
7920                        ..Default::default()
7921                    },
7922                )
7923                .await
7924        });
7925        delete_barrier.wait_until_paused().await;
7926        let cleanup_set = store.pools[0].get_disks_by_key(object);
7927        crate::data_movement::ensure_source_cleanup_versions_unchanged(
7928            Arc::clone(&cleanup_set),
7929            &bucket,
7930            object,
7931            &expected_source_versions,
7932            &[],
7933            "test_versioned_decommission_delete_fence",
7934        )
7935        .await
7936        .expect("the committed delete marker must not be published to the suspended source pool");
7937
7938        let cleanup_delete_barrier = crate::data_movement::SourceCleanupDeleteBarrier::install(&bucket, object);
7939        let cleanup_store = Arc::clone(&store);
7940        let cleanup_bucket = bucket.clone();
7941        let cleanup = tokio::spawn(async move {
7942            let mutation_fence = cleanup_store
7943                .acquire_decommission_source_cleanup_fence(&cleanup_bucket, object, cleanup_set.as_ref())
7944                .await?;
7945            crate::data_movement::cleanup_source_entry_if_unchanged(
7946                cleanup_set,
7947                &cleanup_bucket,
7948                object,
7949                &expected_source_versions,
7950                &[],
7951                crate::data_movement::SourceCleanupBucketFence {
7952                    object_mutation_fence: Some(&mutation_fence),
7953                    ..Default::default()
7954                },
7955                "test_versioned_decommission_delete_fence",
7956            )
7957            .await
7958        });
7959        cleanup_delete_barrier.wait_until_fence_pending().await;
7960        assert!(
7961            !cleanup_delete_barrier.is_paused(),
7962            "source cleanup must wait for the versioned DELETE mutation fence"
7963        );
7964
7965        delete_barrier.release();
7966        let marker = delete
7967            .await
7968            .expect("versioned DELETE task should join")
7969            .expect("versioned DELETE should publish a delete marker after migration");
7970        assert!(marker.delete_marker, "versioned DELETE must publish a delete marker");
7971        assert!(
7972            marker.version_id.is_some_and(|version_id| !version_id.is_nil()),
7973            "the delete marker must have a non-nil version ID"
7974        );
7975
7976        cleanup_delete_barrier.wait_until_paused().await;
7977        cleanup_delete_barrier.release();
7978        cleanup
7979            .await
7980            .expect("source cleanup task should join")
7981            .expect("source cleanup should preserve the active-pool delete marker");
7982
7983        let err = store
7984            .get_object_info(
7985                &bucket,
7986                object,
7987                &ObjectOptions {
7988                    versioned: true,
7989                    ..Default::default()
7990                },
7991            )
7992            .await
7993            .expect_err("the post-migration delete marker must hide the migrated version");
7994        assert!(
7995            matches!(err, StorageError::ObjectNotFound(_, _)),
7996            "unexpected latest-version result: {err:?}"
7997        );
7998        store
7999            .get_object_info(
8000                &bucket,
8001                object,
8002                &ObjectOptions {
8003                    versioned: true,
8004                    version_id: Some(source_version.to_string()),
8005                    ..Default::default()
8006                },
8007            )
8008            .await
8009            .expect("the migrated source version must remain addressable below the delete marker");
8010        store.pools[0]
8011            .get_object_info(
8012                &bucket,
8013                object,
8014                &ObjectOptions {
8015                    versioned: true,
8016                    version_id: Some(source_version.to_string()),
8017                    ..Default::default()
8018                },
8019            )
8020            .await
8021            .expect_err("source cleanup must remove the decommissioned source versions");
8022
8023        shutdown.cancel();
8024    }
8025
8026    #[cfg(feature = "test-util")]
8027    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8028    #[serial_test::serial(storage_class_env)]
8029    async fn decommission_entry_skips_cleanup_only_marker_when_free_version_is_present() {
8030        let temp_dir = tempfile::tempdir().expect("create free-version decommission store dir");
8031        let (ctx, store, shutdown) =
8032            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "decommission-free-marker", &[4, 4])).await;
8033        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
8034        let bucket = format!("decom-free-marker-{}", uuid::Uuid::new_v4());
8035        let object = "free-marker-object";
8036        store
8037            .make_bucket(&bucket, &MakeBucketOptions::default())
8038            .await
8039            .expect("create free-version decommission bucket");
8040        let (_, free_version) = seed_transitioned_free_version(&ctx, &store, &bucket, object).await;
8041        let source_free = store.pools[0]
8042            .get_disks_by_key(object)
8043            .load_file_info_versions_exact(&bucket, object)
8044            .await
8045            .expect("source free-version metadata should decode")
8046            .and_then(|versions| {
8047                versions
8048                    .versions
8049                    .into_iter()
8050                    .find(|version| version.version_id == Some(free_version) && version.tier_free_version())
8051            })
8052            .expect("source free-version identity should be present before decommission");
8053        let mut target_reader = PutObjReader::from_vec(b"target ordinary bytes".to_vec());
8054        let target_w = store.pools[1]
8055            .put_object(
8056                &bucket,
8057                object,
8058                &mut target_reader,
8059                &ObjectOptions {
8060                    versioned: true,
8061                    ..Default::default()
8062                },
8063            )
8064            .await
8065            .expect("write unrelated target version");
8066        let target_w_version = target_w.version_id.expect("target version should have an id");
8067        let marker = store.pools[0]
8068            .delete_object(
8069                &bucket,
8070                object,
8071                ObjectOptions {
8072                    versioned: true,
8073                    ..Default::default()
8074                },
8075            )
8076            .await
8077            .expect("write cleanup-only delete marker");
8078        assert!(marker.delete_marker);
8079
8080        mark_test_pool_decommissioning(&store, 0).await;
8081        let source_set = store.pools[0].get_disks_by_key(object);
8082        store
8083            .decommission_entry_for_test_with_bucket_incarnation(
8084                0,
8085                MetaCacheEntry {
8086                    name: object.to_string(),
8087                    ..Default::default()
8088                },
8089                bucket.clone(),
8090                source_set.clone(),
8091            )
8092            .await
8093            .expect("real decommission entry should migrate the free version");
8094
8095        let target_versions = store.pools[1]
8096            .get_disks_by_key(object)
8097            .load_file_info_versions_exact(&bucket, object)
8098            .await
8099            .expect("target versions should decode")
8100            .expect("target free version should be present");
8101        assert!(
8102            target_versions
8103                .versions
8104                .iter()
8105                .any(|version| { version.version_id == Some(free_version) && version.tier_free_version() })
8106        );
8107        let migrated_free = target_versions
8108            .versions
8109            .iter()
8110            .find(|version| version.version_id == Some(free_version) && version.tier_free_version())
8111            .expect("migrated free-version identity should remain readable from target disks");
8112        assert!(
8113            crate::store::tiered_data_movement_source_matches(&source_free, migrated_free)
8114                .expect("migrated free-version identity should decode")
8115        );
8116        let retained_w = target_versions
8117            .versions
8118            .iter()
8119            .find(|version| version.version_id == Some(target_w_version))
8120            .expect("unrelated target version should remain");
8121        assert!(!retained_w.deleted && !retained_w.tier_free_version());
8122        assert_eq!(retained_w.size, target_w.size);
8123        assert_eq!(retained_w.get_etag(), target_w.etag);
8124        assert!(
8125            target_versions
8126                .versions
8127                .iter()
8128                .all(|version| { version.tier_free_version() || !version.deleted })
8129        );
8130        assert!(
8131            source_set
8132                .load_file_info_versions_exact(&bucket, object)
8133                .await
8134                .expect("source versions should be readable after cleanup")
8135                .is_none(),
8136            "successful free-version migration should permit source cleanup"
8137        );
8138        let (heal_versions, _, _) = store
8139            .heal_walk_versions_page(1, 0, &bucket, "", None, 2, 16, true)
8140            .await
8141            .expect("heal walk should decode the migrated free version");
8142        let free_version_string = free_version.to_string();
8143        let healed_free = heal_versions
8144            .iter()
8145            .find(|version| version.version_id.as_deref() == Some(free_version_string.as_str()))
8146            .expect("heal walk should surface the migrated free version");
8147        let healed_info = healed_free
8148            .lifecycle_object_info
8149            .as_ref()
8150            .expect("heal walk should retain lifecycle identity for the migrated free version");
8151        assert!(healed_info.transitioned_object.free_version);
8152        assert_eq!(healed_info.transitioned_object.tier, source_free.transition_tier);
8153        assert_eq!(healed_info.transitioned_object.name, source_free.transitioned_objname);
8154        shutdown.cancel();
8155    }
8156
8157    #[cfg(feature = "test-util")]
8158    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8159    #[serial_test::serial(storage_class_env)]
8160    async fn decommission_entry_allows_free_version_consumed_before_source_lock() {
8161        let temp_dir = tempfile::tempdir().expect("create consumed free-version store dir");
8162        let (ctx, store, shutdown) =
8163            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "decommission-free-consumed", &[4, 4])).await;
8164        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
8165        let bucket = format!("decom-free-consumed-{}", uuid::Uuid::new_v4());
8166        let object = "free-consumed-object";
8167        store
8168            .make_bucket(&bucket, &MakeBucketOptions::default())
8169            .await
8170            .expect("create consumed free-version bucket");
8171        let (_, free_version) = seed_transitioned_free_version(&ctx, &store, &bucket, object).await;
8172
8173        mark_test_pool_decommissioning(&store, 0).await;
8174        let source_set = store.pools[0].get_disks_by_key(object);
8175        let barrier = crate::store::object::DecommissionFreeVersionSourceRaceBarrier::install(&bucket, object);
8176        let decommission = tokio::spawn({
8177            let store = store.clone();
8178            let bucket = bucket.clone();
8179            let source_set = source_set.clone();
8180            async move {
8181                store
8182                    .decommission_entry_for_test_with_bucket_incarnation(
8183                        0,
8184                        MetaCacheEntry {
8185                            name: object.to_string(),
8186                            ..Default::default()
8187                        },
8188                        bucket,
8189                        source_set,
8190                    )
8191                    .await
8192            }
8193        });
8194
8195        barrier.wait_until_paused().await;
8196        store.pools[0]
8197            .delete_object(
8198                &bucket,
8199                object,
8200                ObjectOptions {
8201                    versioned: true,
8202                    version_id: Some(free_version.to_string()),
8203                    incl_free_versions: true,
8204                    ..Default::default()
8205                },
8206            )
8207            .await
8208            .expect("lifecycle should consume the source free version before decommission locks it");
8209        assert!(
8210            source_set
8211                .load_file_info_versions_exact(&bucket, object)
8212                .await
8213                .expect("consumed source metadata should remain readable")
8214                .is_none(),
8215            "the lifecycle delete should remove the source free version"
8216        );
8217
8218        barrier.release();
8219        decommission
8220            .await
8221            .expect("decommission task should join")
8222            .expect("a concurrently consumed free version should not fail source cleanup");
8223        assert!(
8224            store.pools[1]
8225                .get_disks_by_key(object)
8226                .load_file_info_versions_exact(&bucket, object)
8227                .await
8228                .expect("target metadata should remain readable")
8229                .is_none(),
8230            "an already consumed free version should not be recreated on the target"
8231        );
8232        shutdown.cancel();
8233    }
8234
8235    #[cfg(feature = "test-util")]
8236    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8237    #[serial_test::serial(storage_class_env)]
8238    async fn decommission_entry_rejects_subquorum_free_version_conflict_and_retains_source() {
8239        let temp_dir = tempfile::tempdir().expect("create sub-quorum free-version store dir");
8240        let (ctx, store, shutdown) =
8241            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "decommission-free-conflict", &[4, 4])).await;
8242        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
8243        let bucket = format!("decom-free-conflict-{}", uuid::Uuid::new_v4());
8244        let object = "free-conflict-object";
8245        store
8246            .make_bucket(&bucket, &MakeBucketOptions::default())
8247            .await
8248            .expect("create sub-quorum conflict bucket");
8249        let (_, free_version) = seed_transitioned_free_version(&ctx, &store, &bucket, object).await;
8250        let source_free = store.pools[0]
8251            .get_disks_by_key(object)
8252            .load_file_info_versions_exact(&bucket, object)
8253            .await
8254            .expect("source free version should decode before crash replay setup")
8255            .and_then(|versions| {
8256                versions
8257                    .versions
8258                    .into_iter()
8259                    .find(|version| version.version_id == Some(free_version))
8260            })
8261            .expect("source free version should be available for crash replay setup");
8262
8263        let mut target_reader = PutObjReader::from_vec(b"target ordinary bytes".to_vec());
8264        let target = store.pools[1]
8265            .put_object(
8266                &bucket,
8267                object,
8268                &mut target_reader,
8269                &ObjectOptions {
8270                    versioned: true,
8271                    ..Default::default()
8272                },
8273            )
8274            .await
8275            .expect("seed ordinary target version");
8276        let target_version = target.version_id.expect("target version must have an ID");
8277        // A regular PUT may acknowledge write quorum while its rename tail still owns the
8278        // target namespace lock. Drain that tail through a locked read before bypassing
8279        // the namespace API with the per-disk crash-replay fixture writes below.
8280        store.pools[1]
8281            .get_object_info(
8282                &bucket,
8283                object,
8284                &ObjectOptions {
8285                    versioned: true,
8286                    version_id: Some(target_version.to_string()),
8287                    ..Default::default()
8288                },
8289            )
8290            .await
8291            .expect("target PUT tail should drain before per-disk fixture mutation");
8292        let target_disks = store.pools[1].get_disks_by_key(object).disks.read().await.clone();
8293        for disk in target_disks.iter().skip(1) {
8294            disk.as_ref()
8295                .expect("target crash replay quorum disk should be online")
8296                .write_metadata("", &bucket, object, source_free.clone())
8297                .await
8298                .expect("seed an equivalent free version on the target quorum");
8299        }
8300        let conflict_path = temp_dir
8301            .path()
8302            .join(format!("pool1/set0/disk0/{bucket}/{object}/{STORAGE_FORMAT_FILE}"));
8303        let encoded = tokio::fs::read(&conflict_path)
8304            .await
8305            .expect("target metadata should be readable");
8306        let mut metadata = FileMeta::load(&encoded).expect("target metadata should decode");
8307        let target_index = metadata
8308            .versions
8309            .iter()
8310            .position(|version| version.header.version_id == Some(target_version))
8311            .expect("target version should be present on the conflict disk");
8312        let mut target_meta = metadata.versions[target_index]
8313            .parse_version_meta()
8314            .expect("target version metadata should decode");
8315        target_meta
8316            .object
8317            .as_mut()
8318            .expect("target conflict must remain an ordinary object")
8319            .version_id = Some(free_version);
8320        metadata.versions[target_index] = target_meta.try_into().expect("conflict metadata should encode");
8321        let expected_conflict_meta = metadata.versions[target_index].meta.clone();
8322        let expected_conflict = metadata.versions[target_index]
8323            .into_fileinfo(&bucket, object, true)
8324            .expect("conflict metadata should decode as an ordinary object");
8325        let duplicate_free: rustfs_filemeta::FileMetaShallowVersion = rustfs_filemeta::FileMetaVersion::from(source_free.clone())
8326            .try_into()
8327            .expect("duplicate free metadata should encode");
8328        metadata.versions.insert(target_index, duplicate_free);
8329        tokio::fs::write(&conflict_path, metadata.marshal_msg().expect("conflict metadata should encode"))
8330            .await
8331            .expect("write sub-quorum conflict metadata");
8332
8333        let mut expected_target_metadata = Vec::with_capacity(4);
8334        for disk_index in 0..4 {
8335            let target_path = temp_dir
8336                .path()
8337                .join(format!("pool1/set0/disk{disk_index}/{bucket}/{object}/{STORAGE_FORMAT_FILE}"));
8338            let target_encoded = tokio::fs::read(&target_path)
8339                .await
8340                .expect("target fixture metadata should be readable before decommission");
8341            let target_meta = FileMeta::load(&target_encoded).expect("target fixture metadata should decode");
8342            let same_id = target_meta
8343                .versions
8344                .iter()
8345                .filter(|version| version.header.version_id == Some(free_version))
8346                .collect::<Vec<_>>();
8347            if disk_index == 0 {
8348                assert_eq!(same_id.len(), 2, "conflict fixture should contain both same-ID records");
8349                assert!(same_id[0].header.free_version());
8350                assert!(!same_id[1].header.free_version());
8351            } else {
8352                assert_eq!(same_id.len(), 1, "target fixture disk {disk_index} should contain one free version");
8353                assert!(same_id[0].header.free_version());
8354            }
8355            expected_target_metadata.push(target_encoded);
8356        }
8357
8358        mark_test_pool_decommissioning(&store, 0).await;
8359        let source_set = store.pools[0].get_disks_by_key(object);
8360        store
8361            .decommission_entry_for_test_with_bucket_incarnation(
8362                0,
8363                MetaCacheEntry {
8364                    name: object.to_string(),
8365                    ..Default::default()
8366                },
8367                bucket.clone(),
8368                source_set.clone(),
8369            )
8370            .await
8371            .expect("conflicted decommission entry should retain the source and retry later");
8372
8373        let source_versions = source_set
8374            .load_file_info_versions_exact(&bucket, object)
8375            .await
8376            .expect("retained source versions should decode")
8377            .expect("source free version should be retained after conflict");
8378        assert!(
8379            source_versions
8380                .versions
8381                .iter()
8382                .any(|version| { version.version_id == Some(free_version) && version.tier_free_version() })
8383        );
8384        let post_encoded = tokio::fs::read(&conflict_path)
8385            .await
8386            .expect("conflict metadata should remain readable");
8387        let post_metadata = FileMeta::load(&post_encoded).expect("post-conflict metadata should decode");
8388        let same_id = post_metadata
8389            .versions
8390            .iter()
8391            .filter(|version| version.header.version_id == Some(free_version))
8392            .collect::<Vec<_>>();
8393        assert_eq!(same_id.len(), 2, "conflict metadata should retain both same-ID records");
8394        assert_eq!(same_id.iter().filter(|version| version.header.free_version()).count(), 1);
8395        assert_eq!(same_id.iter().filter(|version| !version.header.free_version()).count(), 1);
8396        let post_conflict = same_id
8397            .into_iter()
8398            .find(|version| !version.header.free_version())
8399            .expect("ordinary conflict version must remain addressable by the source ID");
8400        let post_conflict_info = post_conflict
8401            .into_fileinfo(&bucket, object, true)
8402            .expect("post-conflict ordinary metadata should decode");
8403        assert!(!post_conflict_info.deleted && !post_conflict_info.tier_free_version());
8404        assert_eq!(post_conflict.meta, expected_conflict_meta);
8405        assert_eq!(post_conflict_info.size, expected_conflict.size);
8406        assert_eq!(post_conflict_info.data_dir, expected_conflict.data_dir);
8407        assert_eq!(post_conflict_info.metadata, expected_conflict.metadata);
8408        assert_eq!(post_conflict_info.get_etag(), expected_conflict.get_etag());
8409
8410        for (disk_index, expected_target_encoded) in expected_target_metadata.iter().enumerate() {
8411            let target_path = temp_dir
8412                .path()
8413                .join(format!("pool1/set0/disk{disk_index}/{bucket}/{object}/{STORAGE_FORMAT_FILE}"));
8414            let target_encoded = tokio::fs::read(&target_path)
8415                .await
8416                .expect("target metadata should remain readable");
8417            assert_eq!(
8418                &target_encoded, expected_target_encoded,
8419                "conflicted decommission must not mutate target disk {disk_index}"
8420            );
8421            let target_meta = FileMeta::load(&target_encoded).expect("target metadata should decode");
8422            let same_id = target_meta
8423                .versions
8424                .iter()
8425                .filter(|version| version.header.version_id == Some(free_version))
8426                .collect::<Vec<_>>();
8427            if disk_index == 0 {
8428                assert_eq!(same_id.len(), 2);
8429                assert!(same_id[0].header.free_version());
8430                assert!(!same_id[1].header.free_version());
8431            } else {
8432                assert_eq!(same_id.len(), 1);
8433                assert!(same_id[0].header.free_version());
8434            }
8435        }
8436        let sweep_err = store
8437            .check_after_decommission_for_test(0)
8438            .await
8439            .expect_err("final sweep must report the retained free version");
8440        assert!(
8441            sweep_err.to_string().contains("version(s) were found"),
8442            "unexpected final sweep error: {sweep_err}"
8443        );
8444        shutdown.cancel();
8445    }
8446
8447    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8448    #[serial_test::serial(storage_class_env)]
8449    async fn versioned_batch_delete_marker_skips_decommission_source() {
8450        let temp_dir = tempfile::tempdir().expect("create versioned batch decommission store dir");
8451        let (_ctx, store, shutdown) = without_storage_class_env(build_isolated_test_store(
8452            temp_dir.path(),
8453            "versioned-batch-decommission-delete-fence",
8454            &[4, 4, 4],
8455        ))
8456        .await;
8457        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
8458
8459        let bucket = format!("vbatch-decom-delete-{}", uuid::Uuid::new_v4());
8460        let object = "batch-object.bin";
8461        store
8462            .make_bucket(&bucket, &MakeBucketOptions::default())
8463            .await
8464            .expect("create versioned batch decommission bucket");
8465        let (_source_version, expected_source_versions) = migrate_versioned_decommission_test_object(
8466            &store,
8467            &bucket,
8468            object,
8469            b"batch source generation",
8470            "test_versioned_batch_decommission_delete_fence",
8471        )
8472        .await;
8473
8474        let delete_config_snapshot =
8475            Arc::new(crate::bucket::replication::DeleteReplicationConfigSnapshot::from_configs_for_test(
8476                s3s::dto::VersioningConfiguration {
8477                    status: Some(s3s::dto::BucketVersioningStatus::from_static(s3s::dto::BucketVersioningStatus::ENABLED)),
8478                    ..Default::default()
8479                },
8480                None,
8481            ));
8482        let delete_barrier = crate::store::object::VersionedDeleteMarkerCommitBarrier::install(&bucket, object);
8483        let delete_store = Arc::clone(&store);
8484        let delete_bucket = bucket.clone();
8485        let delete = tokio::spawn(async move {
8486            delete_store
8487                .delete_objects(
8488                    &delete_bucket,
8489                    vec![ObjectToDelete {
8490                        object_name: object.to_string(),
8491                        ..Default::default()
8492                    }],
8493                    ObjectOptions {
8494                        delete_replication_config_snapshot: Some(delete_config_snapshot),
8495                        ..Default::default()
8496                    },
8497                )
8498                .await
8499        });
8500        delete_barrier.wait_until_paused().await;
8501
8502        let source_set = store.pools[0].get_disks_by_key(object);
8503        crate::data_movement::ensure_source_cleanup_versions_unchanged(
8504            source_set,
8505            &bucket,
8506            object,
8507            &expected_source_versions,
8508            &[],
8509            "test_versioned_batch_decommission_delete_fence",
8510        )
8511        .await
8512        .expect("batch DELETE must not publish a marker to the suspended source");
8513
8514        delete_barrier.release();
8515        let (deleted, errors) = delete.await.expect("versioned batch DELETE task should join");
8516        assert!(errors.iter().all(Option::is_none), "versioned batch DELETE should succeed: {errors:?}");
8517        assert_eq!(deleted.len(), 1);
8518        assert!(deleted[0].delete_marker, "versioned batch DELETE must return a marker");
8519        assert!(
8520            deleted[0]
8521                .delete_marker_version_id
8522                .is_some_and(|version_id| !version_id.is_nil()),
8523            "versioned batch DELETE marker must have a non-nil version ID"
8524        );
8525
8526        let mut active_marker_count = 0;
8527        for pool in store.pools.iter().skip(1) {
8528            let Some(versions) = pool
8529                .get_disks_by_key(object)
8530                .load_file_info_versions_exact(&bucket, object)
8531                .await
8532                .expect("active-pool versions should be readable")
8533            else {
8534                continue;
8535            };
8536            active_marker_count += versions.versions.iter().filter(|version| version.deleted).count();
8537        }
8538        assert_eq!(active_marker_count, 1, "batch DELETE must publish exactly one active-pool marker");
8539
8540        shutdown.cancel();
8541    }
8542
8543    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8544    #[serial_test::serial(storage_class_env)]
8545    async fn suspended_delete_marker_then_decommission_worker_converges_null_source() {
8546        let temp_dir = tempfile::tempdir().expect("create suspended decommission DELETE store dir");
8547        let (_ctx, store, shutdown) = without_storage_class_env(build_isolated_test_store(
8548            temp_dir.path(),
8549            "suspended-decommission-delete-convergence",
8550            &[4, 4],
8551        ))
8552        .await;
8553        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
8554
8555        let bucket = format!("suspended-decom-delete-{}", uuid::Uuid::new_v4());
8556        let object = "single.bin";
8557        store
8558            .make_bucket(&bucket, &MakeBucketOptions::default())
8559            .await
8560            .expect("create suspended decommission DELETE bucket");
8561        write_suspended_decommission_source(&store, &bucket, object).await;
8562        mark_test_pool_decommissioning(&store, 0).await;
8563
8564        let deleted = store
8565            .delete_object(
8566                &bucket,
8567                object,
8568                ObjectOptions {
8569                    version_suspended: true,
8570                    ..Default::default()
8571                },
8572            )
8573            .await
8574            .expect("a healthy reserved target must accept suspended DELETE");
8575        assert!(deleted.delete_marker);
8576        assert_eq!(deleted.version_id, Some(uuid::Uuid::nil()));
8577        let marker_mod_time = deleted.mod_time.expect("DELETE must return the marker generation");
8578        assert_suspended_null_source_present(&store, &bucket, object).await;
8579        assert_suspended_null_delete_marker_visible(&store, &bucket, object, marker_mod_time).await;
8580
8581        let source_set = store.pools[0].get_disks_by_key(object);
8582        let worker_store = Arc::clone(&store);
8583        let worker_bucket = bucket.clone();
8584        tokio::spawn(async move {
8585            worker_store
8586                .decommission_entry_for_test(
8587                    0,
8588                    MetaCacheEntry {
8589                        name: object.to_string(),
8590                        ..Default::default()
8591                    },
8592                    worker_bucket,
8593                    source_set,
8594                )
8595                .await
8596        })
8597        .await
8598        .expect("suspended decommission worker should join")
8599        .expect("worker must converge the old null source behind the newer DELETE marker");
8600
8601        assert_decommission_source_absent(
8602            &store,
8603            &bucket,
8604            object,
8605            &ObjectOptions {
8606                version_suspended: true,
8607                ..Default::default()
8608            },
8609        )
8610        .await;
8611        assert_suspended_null_delete_marker_visible(&store, &bucket, object, marker_mod_time).await;
8612        shutdown.cancel();
8613    }
8614
8615    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8616    #[serial_test::serial(storage_class_env)]
8617    async fn suspended_batch_delete_marker_then_decommission_worker_converges_null_source() {
8618        let temp_dir = tempfile::tempdir().expect("create suspended batch decommission DELETE store dir");
8619        let (_ctx, store, shutdown) = without_storage_class_env(build_isolated_test_store(
8620            temp_dir.path(),
8621            "suspended-batch-decommission-delete-convergence",
8622            &[4, 4],
8623        ))
8624        .await;
8625        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
8626
8627        let bucket = format!("susp-batch-decom-delete-{}", uuid::Uuid::new_v4());
8628        let object = "batch.bin";
8629        store
8630            .make_bucket(&bucket, &MakeBucketOptions::default())
8631            .await
8632            .expect("create suspended batch decommission DELETE bucket");
8633        write_suspended_decommission_source(&store, &bucket, object).await;
8634        mark_test_pool_decommissioning(&store, 0).await;
8635
8636        let delete_config_snapshot =
8637            Arc::new(crate::bucket::replication::DeleteReplicationConfigSnapshot::from_configs_for_test(
8638                s3s::dto::VersioningConfiguration {
8639                    status: Some(s3s::dto::BucketVersioningStatus::from_static(s3s::dto::BucketVersioningStatus::SUSPENDED)),
8640                    ..Default::default()
8641                },
8642                None,
8643            ));
8644        let (deleted, errors) = store
8645            .delete_objects(
8646                &bucket,
8647                vec![ObjectToDelete {
8648                    object_name: object.to_string(),
8649                    ..Default::default()
8650                }],
8651                ObjectOptions {
8652                    delete_replication_config_snapshot: Some(delete_config_snapshot),
8653                    ..Default::default()
8654                },
8655            )
8656            .await;
8657        assert!(
8658            matches!(errors.as_slice(), [None]),
8659            "unexpected suspended batch DELETE result: {errors:?}"
8660        );
8661        assert_eq!(deleted.len(), 1);
8662        assert!(deleted[0].delete_marker);
8663        assert_eq!(deleted[0].object_name, object);
8664        assert!(
8665            deleted[0]
8666                .delete_marker_version_id
8667                .is_none_or(|version_id| version_id.is_nil()),
8668            "batch DELETE must retain the native null version identity"
8669        );
8670        let marker_mod_time = deleted[0]
8671            .delete_marker_mtime
8672            .expect("batch DELETE must return the marker generation");
8673        assert_suspended_null_source_present(&store, &bucket, object).await;
8674        assert_suspended_null_delete_marker_visible(&store, &bucket, object, marker_mod_time).await;
8675
8676        let source_set = store.pools[0].get_disks_by_key(object);
8677        let worker_store = Arc::clone(&store);
8678        let worker_bucket = bucket.clone();
8679        tokio::spawn(async move {
8680            worker_store
8681                .decommission_entry_for_test(
8682                    0,
8683                    MetaCacheEntry {
8684                        name: object.to_string(),
8685                        ..Default::default()
8686                    },
8687                    worker_bucket,
8688                    source_set,
8689                )
8690                .await
8691        })
8692        .await
8693        .expect("suspended batch decommission worker should join")
8694        .expect("worker must converge the old null source behind the newer batch DELETE marker");
8695
8696        assert_decommission_source_absent(
8697            &store,
8698            &bucket,
8699            object,
8700            &ObjectOptions {
8701                version_suspended: true,
8702                ..Default::default()
8703            },
8704        )
8705        .await;
8706        assert_suspended_null_delete_marker_visible(&store, &bucket, object, marker_mod_time).await;
8707        shutdown.cancel();
8708    }
8709
8710    #[cfg(feature = "test-util")]
8711    #[tokio::test]
8712    #[serial_test::serial(storage_class_env)]
8713    async fn tiered_data_movement_rejects_a_stale_source_snapshot_before_target_write() {
8714        let temp_dir = tempfile::tempdir().expect("create stale-source data movement store dir");
8715        let (ctx, store, _shutdown) =
8716            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "stale-tier-source", &[4, 4])).await;
8717        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
8718        let bucket = "stale-tier-source-bucket";
8719        let object = "stale-tier-source-object";
8720        store
8721            .make_bucket(bucket, &MakeBucketOptions::default())
8722            .await
8723            .expect("bucket should be created");
8724        register_mock_tier(&ctx.tier_config_mgr(), "STALE-TIER").await;
8725        let incarnation = store
8726            .bucket_incarnation_id(bucket)
8727            .await
8728            .expect("bucket incarnation should exist");
8729        let version_id = uuid::Uuid::new_v4();
8730        let stale = FileInfo {
8731            volume: bucket.to_string(),
8732            name: object.to_string(),
8733            version_id: Some(version_id),
8734            transition_status: rustfs_filemeta::TRANSITION_COMPLETE.to_string(),
8735            transitioned_objname: "remote/stale-source".to_string(),
8736            transition_tier: "STALE-TIER".to_string(),
8737            transition_version_id: Some(uuid::Uuid::new_v4()),
8738            transition_version_state: rustfs_filemeta::TransitionVersionState::Exact,
8739            data_dir: Some(uuid::Uuid::new_v4()),
8740            mod_time: Some(OffsetDateTime::UNIX_EPOCH),
8741            size: 1,
8742            metadata: HashMap::new(),
8743            ..Default::default()
8744        };
8745        let err = store
8746            .decommission_tiered_object(
8747                bucket,
8748                object,
8749                &stale,
8750                &ObjectOptions {
8751                    versioned: true,
8752                    version_id: Some(version_id.to_string()),
8753                    src_pool_idx: 0,
8754                    data_movement: true,
8755                    expected_bucket_incarnation_id: Some(incarnation),
8756                    ..Default::default()
8757                },
8758            )
8759            .await
8760            .expect_err("a source removed after queue capture must fail closed");
8761        assert!(matches!(err, Error::ObjectNotFound(_, _) | Error::FileNotFound));
8762        let target_err = store.pools[1]
8763            .get_object_info(bucket, object, &ObjectOptions::default())
8764            .await
8765            .expect_err("stale source rejection must not write target metadata");
8766        assert!(matches!(
8767            target_err,
8768            StorageError::ObjectNotFound(_, _) | StorageError::VersionNotFound(_, _, _)
8769        ));
8770    }
8771
8772    #[tokio::test]
8773    #[serial_test::serial(storage_class_env)]
8774    async fn data_movement_put_conflict_validates_only_selected_target_pool() {
8775        let temp_dir = tempfile::tempdir().expect("create three-pool data movement store dir");
8776        let (_ctx, store, _shutdown) =
8777            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "data-movement-selected-target", &[4, 4, 4]))
8778                .await;
8779        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
8780
8781        let bucket = format!("dm-selected-target-{}", uuid::Uuid::new_v4());
8782        let object = "object.bin";
8783        let source_body = b"source-and-equivalent".to_vec();
8784        let conflicting_body = b"newer-conflicting-target".to_vec();
8785        let source_mod_time = OffsetDateTime::UNIX_EPOCH + time::Duration::SECOND;
8786        let version = uuid::Uuid::new_v4();
8787        store
8788            .make_bucket(&bucket, &MakeBucketOptions::default())
8789            .await
8790            .expect("create selected-target bucket");
8791        for (pool_idx, body, mod_time) in [
8792            (0, source_body.clone(), source_mod_time),
8793            (1, conflicting_body.clone(), source_mod_time + time::Duration::SECOND),
8794            (2, source_body.clone(), source_mod_time),
8795        ] {
8796            let mut reader = PutObjReader::from_vec(body);
8797            store.pools[pool_idx]
8798                .put_object(
8799                    &bucket,
8800                    object,
8801                    &mut reader,
8802                    &ObjectOptions {
8803                        versioned: true,
8804                        version_id: Some(version.to_string()),
8805                        mod_time: Some(mod_time),
8806                        ..Default::default()
8807                    },
8808                )
8809                .await
8810                .expect("seed data movement pool");
8811        }
8812        *store.rebalance_meta.write().await = Some(active_rebalance_meta_for_pool(store.pools.len(), 0));
8813
8814        let source_reader = store.pools[0]
8815            .get_object_reader(
8816                &bucket,
8817                object,
8818                None,
8819                HeaderMap::new(),
8820                &ObjectOptions {
8821                    versioned: true,
8822                    version_id: Some(version.to_string()),
8823                    raw_data_movement_read: true,
8824                    ..Default::default()
8825                },
8826            )
8827            .await
8828            .expect("read source for selected-target migration");
8829        crate::data_movement::migrate_object(store.clone(), 0, bucket.clone(), source_reader, None, "test_selected_target")
8830            .await
8831            .expect_err("an equivalent object in another pool must not mask the selected target conflict");
8832
8833        let mut selected_target = store.pools[1]
8834            .get_object_reader(
8835                &bucket,
8836                object,
8837                None,
8838                HeaderMap::new(),
8839                &ObjectOptions {
8840                    versioned: true,
8841                    version_id: Some(version.to_string()),
8842                    ..Default::default()
8843                },
8844            )
8845            .await
8846            .expect("read selected conflicting target");
8847        let mut selected_body = Vec::new();
8848        selected_target
8849            .stream
8850            .read_to_end(&mut selected_body)
8851            .await
8852            .expect("drain selected conflicting target");
8853        assert_eq!(selected_body, conflicting_body);
8854    }
8855
8856    #[tokio::test]
8857    #[serial_test::serial(storage_class_env)]
8858    async fn data_movement_target_remains_readable_before_source_cleanup() {
8859        let temp_dir = tempfile::tempdir().expect("create data movement read-window store dir");
8860        let (_ctx, store, shutdown) =
8861            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "data-movement-read-window", &[4, 4])).await;
8862        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
8863
8864        let bucket = format!("dm-read-window-{}", uuid::Uuid::new_v4());
8865        let object = "object.bin";
8866        let version = uuid::Uuid::new_v4();
8867        let mod_time = OffsetDateTime::UNIX_EPOCH + time::Duration::SECOND;
8868        store
8869            .make_bucket(&bucket, &MakeBucketOptions::default())
8870            .await
8871            .expect("create data movement read-window bucket");
8872
8873        let mut reader = PutObjReader::from_vec(b"source body".to_vec());
8874        store.pools[0]
8875            .put_object(
8876                &bucket,
8877                object,
8878                &mut reader,
8879                &ObjectOptions {
8880                    versioned: true,
8881                    version_id: Some(version.to_string()),
8882                    mod_time: Some(mod_time),
8883                    ..Default::default()
8884                },
8885            )
8886            .await
8887            .expect("seed data movement source");
8888        *store.rebalance_meta.write().await = Some(active_rebalance_meta_for_pool(store.pools.len(), 0));
8889
8890        let source_reader = store.pools[0]
8891            .get_object_reader(
8892                &bucket,
8893                object,
8894                None,
8895                HeaderMap::new(),
8896                &ObjectOptions {
8897                    versioned: true,
8898                    version_id: Some(version.to_string()),
8899                    raw_data_movement_read: true,
8900                    ..Default::default()
8901                },
8902            )
8903            .await
8904            .expect("read data movement source");
8905        let source_data_dir = source_reader.object_info.data_dir;
8906        crate::data_movement::migrate_object(store.clone(), 0, bucket.clone(), source_reader, None, "test_read_window")
8907            .await
8908            .expect("commit data movement target");
8909
8910        let target = store.pools[1]
8911            .get_object_info(
8912                &bucket,
8913                object,
8914                &ObjectOptions {
8915                    versioned: true,
8916                    version_id: Some(version.to_string()),
8917                    ..Default::default()
8918                },
8919            )
8920            .await
8921            .expect("read committed data movement target");
8922        assert_ne!(source_data_dir, target.data_dir, "the target must use its own data directory");
8923        assert_eq!(
8924            rustfs_utils::http::get_consistent_str(&target.user_defined, rustfs_utils::http::SUFFIX_DATA_MOVED),
8925            Some("true")
8926        );
8927
8928        let resolved = store
8929            .get_object_info(
8930                &bucket,
8931                object,
8932                &ObjectOptions {
8933                    versioned: true,
8934                    version_id: Some(version.to_string()),
8935                    ..Default::default()
8936                },
8937            )
8938            .await
8939            .expect("the equal-time migration window must remain readable");
8940        assert_eq!(resolved.data_dir, target.data_dir);
8941
8942        shutdown.cancel();
8943    }
8944
8945    #[tokio::test]
8946    #[serial_test::serial(storage_class_env)]
8947    async fn data_movement_multipart_conflict_validates_exact_version_target_pool() {
8948        let temp_dir = tempfile::tempdir().expect("create three-pool multipart data movement store dir");
8949        let (_ctx, store, _shutdown) =
8950            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "data-movement-multipart-target", &[4, 4, 4]))
8951                .await;
8952        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
8953
8954        let bucket = format!("dm-multipart-target-{}", uuid::Uuid::new_v4());
8955        let object = "object.bin";
8956        let source_version = uuid::Uuid::new_v4();
8957        let other_version = uuid::Uuid::new_v4();
8958        let source_mod_time = OffsetDateTime::UNIX_EPOCH + time::Duration::SECOND;
8959        store
8960            .make_bucket(&bucket, &MakeBucketOptions::default())
8961            .await
8962            .expect("create multipart selected-target bucket");
8963        let first_part_size = 5 * 1024 * 1024;
8964        let mut source_body = vec![b'a'; first_part_size];
8965        source_body.push(b'b');
8966        for (pool_idx, version_id, body, mod_time) in [
8967            (
8968                1,
8969                other_version,
8970                b"newer different version".to_vec(),
8971                source_mod_time + time::Duration::seconds(2),
8972            ),
8973            (2, source_version, b"conflicting exact version".to_vec(), source_mod_time),
8974        ] {
8975            let mut reader = PutObjReader::from_vec(body);
8976            store.pools[pool_idx]
8977                .put_object(
8978                    &bucket,
8979                    object,
8980                    &mut reader,
8981                    &ObjectOptions {
8982                        versioned: true,
8983                        version_id: Some(version_id.to_string()),
8984                        mod_time: Some(mod_time),
8985                        ..Default::default()
8986                    },
8987                )
8988                .await
8989                .expect("seed multipart data movement pool");
8990        }
8991        let source_reader = seed_data_movement_source_reader(
8992            &store,
8993            &bucket,
8994            object,
8995            DataMovementSourceSeed {
8996                pool_idx: 0,
8997                version_id: Some(source_version),
8998                body: source_body,
8999                multipart_split: Some(first_part_size),
9000                mod_time: source_mod_time,
9001                user_defined: HashMap::new(),
9002            },
9003        )
9004        .await;
9005        *store.rebalance_meta.write().await = Some(active_rebalance_meta_for_pool(store.pools.len(), 0));
9006
9007        let completion_barrier = crate::store::multipart::DataMovementMultipartCompletionBarrier::install(&bucket);
9008        let migration_store = store.clone();
9009        let migration_bucket = bucket.clone();
9010        let migration = tokio::spawn(async move {
9011            crate::data_movement::migrate_object(
9012                migration_store,
9013                0,
9014                migration_bucket,
9015                source_reader,
9016                None,
9017                "test_multipart_selected_target",
9018            )
9019            .await
9020        });
9021        completion_barrier.wait_until_paused().await;
9022        let unrelated_set = store.pools[1].get_disks_by_key(object);
9023        let original_disks = {
9024            let mut disks = unrelated_set.disks.write().await;
9025            let original = disks.clone();
9026            for disk in disks.iter_mut().take(3) {
9027                *disk = None;
9028            }
9029            original
9030        };
9031        drop(completion_barrier);
9032        let err = migration
9033            .await
9034            .expect("multipart migration task should join")
9035            .expect_err("the exact-version target conflict must not be bypassed by a newer different version");
9036        *unrelated_set.disks.write().await = original_disks;
9037        let rendered = err.to_string();
9038        assert!(
9039            rendered.contains("complete_multipart_upload"),
9040            "the selected target should be reached despite an unrelated degraded pool: {rendered}"
9041        );
9042        assert!(
9043            !rendered.contains("put_object_part failed"),
9044            "part upload must not scan the unrelated degraded pool: {rendered}"
9045        );
9046
9047        let unexpected_version = store.pools[1]
9048            .get_object_info(
9049                &bucket,
9050                object,
9051                &ObjectOptions {
9052                    versioned: true,
9053                    version_id: Some(source_version.to_string()),
9054                    ..Default::default()
9055                },
9056            )
9057            .await
9058            .expect_err("multipart migration must not duplicate the source version into the latest-version pool");
9059        assert!(matches!(
9060            unexpected_version,
9061            StorageError::ObjectNotFound(_, _) | StorageError::VersionNotFound(_, _, _)
9062        ));
9063    }
9064
9065    #[cfg(feature = "test-util")]
9066    #[tokio::test]
9067    #[serial_test::serial(storage_class_env)]
9068    async fn decommission_durable_ilm_target_read_error_is_not_masked_by_peer_success() {
9069        let temp_dir = tempfile::tempdir().expect("create temp store dir");
9070        let (_ctx, store, _shutdown) =
9071            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "durable-ilm-target-read-error", &[4, 4, 4]))
9072                .await;
9073        let job_id = uuid::Uuid::new_v4();
9074        let job =
9075            ManualTransitionJobRecord::new(job_id, "manual-target-read-error", &ManualTransitionRunOptions::default(), "owner");
9076        let path = manual_transition_job_record_object_name(job_id).expect("manual job path should build");
9077        let data = job.encode().expect("manual job should encode");
9078        for pool in &store.pools {
9079            com::save_config(pool.clone(), &path, data.clone())
9080                .await
9081                .expect("manual job fixture should persist in every pool");
9082        }
9083        store.pool_meta.write().await.pools[0].decommission = Some(PoolDecommissionInfo {
9084            start_time: Some(OffsetDateTime::now_utc()),
9085            ..Default::default()
9086        });
9087
9088        let failing_target = store.pools[2].get_disks_by_key(&path);
9089        let original_disks = {
9090            let mut disks = failing_target.disks.write().await;
9091            let original = disks.clone();
9092            for disk in disks.iter_mut().take(3) {
9093                *disk = None;
9094            }
9095            original
9096        };
9097        let error = store
9098            .verify_and_cleanup_decommissioned_durable_ilm_record_for_test(0, store.pools[0].get_disks_by_key(&path), &path)
9099            .await
9100            .expect_err("one target read-quorum error must fail closed despite another target success")
9101            .to_string();
9102        *failing_target.disks.write().await = original_disks;
9103
9104        assert!(error.contains(&path));
9105        assert!(error.contains("pool 2"));
9106        assert_eq!(
9107            com::read_config(store.pools[0].clone(), &path)
9108                .await
9109                .expect("target read error must retain the source"),
9110            data
9111        );
9112        assert_eq!(
9113            store
9114                .decommission_durable_ilm_receipt_count_for_test(0)
9115                .await
9116                .expect("failed target verification should not create a receipt"),
9117            0
9118        );
9119    }
9120
9121    #[cfg(feature = "test-util")]
9122    #[tokio::test]
9123    #[serial_test::serial(storage_class_env)]
9124    async fn decommission_durable_ilm_terminal_receipt_recovers_failed_source_cleanup() {
9125        let temp_dir = tempfile::tempdir().expect("create temp store dir");
9126        let (ctx, store, _shutdown) =
9127            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "durable-ilm-terminal-receipt", &[4, 4])).await;
9128        let tier_name = "DECOMMISSION-RECEIPT";
9129        let backend = register_transition_reconcile_test_tier(&ctx.tier_config_mgr(), tier_name).await;
9130        let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
9131            .await
9132            .expect("tier lease should resolve")
9133            .backend_identity();
9134        let entry = Jentry {
9135            persisted_version: 0,
9136            obj_name: "receipt-recovery-object".to_string(),
9137            version_id: "receipt-recovery-version".to_string(),
9138            tier_name: tier_name.to_string(),
9139            backend_identity: Some(backend_identity),
9140            version_id_exact: true,
9141            version_state: rustfs_filemeta::TransitionVersionState::Exact,
9142            state: TierDeleteJournalState::Committed,
9143            source: None,
9144            dispatch: None,
9145        };
9146        let path = tier_delete_journal_object_name(&entry);
9147        let data = encode_tier_delete_journal_entry(&entry).expect("tier journal should encode");
9148        com::save_config(store.pools[0].clone(), &path, data.clone())
9149            .await
9150            .expect("source tier journal should persist");
9151        com::save_config(store.pools[1].clone(), &path, data.clone())
9152            .await
9153            .expect("target tier journal should persist");
9154        let active_pool_meta = {
9155            let mut pool_meta = store.pool_meta.write().await;
9156            pool_meta.pools[0].decommission = Some(PoolDecommissionInfo {
9157                start_time: Some(OffsetDateTime::now_utc()),
9158                ..Default::default()
9159            });
9160            pool_meta.clone()
9161        };
9162        active_pool_meta
9163            .save(store.pools.clone())
9164            .await
9165            .expect("active decommission run identity should persist");
9166
9167        let source_set = store.pools[0].get_disks_by_key(&path);
9168        let barrier = SourceCleanupDeleteBarrier::install(RUSTFS_META_BUCKET, &path);
9169        let cleanup_store = store.clone();
9170        let cleanup_set = source_set.clone();
9171        let cleanup_path = path.clone();
9172        let cleanup = tokio::spawn(async move {
9173            cleanup_store
9174                .verify_and_cleanup_decommissioned_durable_ilm_record_for_test(0, cleanup_set, &cleanup_path)
9175                .await
9176        });
9177        barrier.wait_until_paused().await;
9178        let original_source_disks = {
9179            let mut disks = source_set.disks.write().await;
9180            let original = disks.clone();
9181            for disk in disks.iter_mut().take(3) {
9182                *disk = None;
9183            }
9184            original
9185        };
9186        barrier.release();
9187        let cleanup_error = cleanup
9188            .await
9189            .expect("source cleanup task should not panic")
9190            .expect_err("injected source delete quorum failure must fail cleanup")
9191            .to_string();
9192        *source_set.disks.write().await = original_source_disks;
9193        drop(barrier);
9194
9195        assert!(cleanup_error.contains("source durable ILM cleanup failed"));
9196        assert_eq!(
9197            store
9198                .decommission_durable_ilm_receipt_count_for_test(0)
9199                .await
9200                .expect("receipt should persist before source cleanup"),
9201            1
9202        );
9203        assert_eq!(
9204            com::read_config(store.pools[0].clone(), &path)
9205                .await
9206                .expect("failed cleanup must retain the source"),
9207            data
9208        );
9209
9210        let mut restarted_pool_meta = PoolMeta::default();
9211        restarted_pool_meta
9212            .load(store.pools[0].clone(), store.pools.clone())
9213            .await
9214            .expect("decommission run identity should reload after restart");
9215        *store.pool_meta.write().await = restarted_pool_meta;
9216        let stats = recover_tier_delete_journal_entries(store.clone(), 100, None)
9217            .await
9218            .expect("target recovery should commit terminal proof and delete the target");
9219        assert_eq!((stats.scanned, stats.deleted, stats.failed), (1, 1, 0));
9220        assert!(matches!(
9221            com::read_config(store.pools[1].clone(), &path).await,
9222            Err(Error::ConfigNotFound)
9223        ));
9224        assert_eq!(
9225            com::read_config(store.pools[0].clone(), &path)
9226                .await
9227                .expect("target recovery must not delete the decommission source"),
9228            data
9229        );
9230
9231        store
9232            .verify_and_cleanup_decommissioned_durable_ilm_record_for_test(0, source_set, &path)
9233            .await
9234            .expect("terminal receipt should authorize cleanup after target deletion");
9235        assert!(matches!(
9236            com::read_config(store.pools[0].clone(), &path).await,
9237            Err(Error::ConfigNotFound)
9238        ));
9239        assert!(backend.remove_versions().await.contains(&(entry.obj_name, entry.version_id)));
9240    }
9241
9242    #[cfg(feature = "test-util")]
9243    #[tokio::test]
9244    #[serial_test::serial(storage_class_env)]
9245    async fn decommission_final_sweep_blocks_cancel_until_source_cleanup_finishes() {
9246        let temp_dir = tempfile::tempdir().expect("create final sweep gate store dir");
9247        let (_ctx, store, _shutdown) =
9248            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "durable-ilm-final-sweep-gate", &[4, 4])).await;
9249        let job_id = uuid::Uuid::new_v4();
9250        let job = ManualTransitionJobRecord::new(job_id, "final-sweep-gate", &ManualTransitionRunOptions::default(), "owner");
9251        let path = manual_transition_job_record_object_name(job_id).expect("manual job path should build");
9252        let data = job.encode().expect("manual job should encode");
9253        for pool in &store.pools {
9254            com::save_config(pool.clone(), &path, data.clone())
9255                .await
9256                .expect("manual job fixture should persist in both pools");
9257        }
9258        let active_pool_meta = {
9259            let mut pool_meta = store.pool_meta.write().await;
9260            pool_meta.pools[0].decommission = Some(PoolDecommissionInfo {
9261                start_time: Some(OffsetDateTime::now_utc()),
9262                ..Default::default()
9263            });
9264            pool_meta.clone()
9265        };
9266        active_pool_meta
9267            .save(store.pools.clone())
9268            .await
9269            .expect("active decommission run identity should persist");
9270
9271        let barrier = SourceCleanupDeleteBarrier::install(RUSTFS_META_BUCKET, &path);
9272        let final_sweep = tokio::spawn({
9273            let store = store.clone();
9274            async move { store.check_after_decommission_for_test(0).await }
9275        });
9276        barrier.wait_until_paused().await;
9277
9278        let mut cancel = tokio::spawn({
9279            let store = store.clone();
9280            async move { store.decommission_cancel(0).await }
9281        });
9282        assert!(
9283            tokio::time::timeout(Duration::from_millis(100), &mut cancel).await.is_err(),
9284            "cancel must wait for the final sweep source cleanup"
9285        );
9286        {
9287            let pool_meta = store.pool_meta.read().await;
9288            let decommission = pool_meta.pools[0]
9289                .decommission
9290                .as_ref()
9291                .expect("decommission state should remain present");
9292            assert!(
9293                decommission.canceled,
9294                "cancel must publish its durable terminal state before waiting for the final sweep"
9295            );
9296            assert!(
9297                decommission.start_time.is_none(),
9298                "the durable cancel state must clear the active run identity before quiescence"
9299            );
9300        }
9301
9302        barrier.release();
9303        final_sweep
9304            .await
9305            .expect("final sweep task should not panic")
9306            .expect("final sweep should finish after the barrier releases");
9307        cancel
9308            .await
9309            .expect("cancel task should not panic")
9310            .expect("cancel should complete after the final sweep releases the operation gate");
9311        let pool_meta = store.pool_meta.read().await;
9312        let decommission = pool_meta.pools[0]
9313            .decommission
9314            .as_ref()
9315            .expect("decommission state should remain present");
9316        assert!(decommission.canceled);
9317        assert!(decommission.start_time.is_none());
9318    }
9319
9320    #[cfg(feature = "test-util")]
9321    #[tokio::test]
9322    #[serial_test::serial(storage_class_env)]
9323    async fn decommission_durable_ilm_recovery_keeps_multiple_active_sources() {
9324        let temp_dir = tempfile::tempdir().expect("create multi-source recovery store dir");
9325        let (ctx, store, _shutdown) = without_storage_class_env(build_isolated_test_store(
9326            temp_dir.path(),
9327            "durable-ilm-multi-source-recovery",
9328            &[4, 4, 4],
9329        ))
9330        .await;
9331        let tier_name = "DECOMMISSION-MULTI-SOURCE";
9332        let backend = register_transition_reconcile_test_tier(&ctx.tier_config_mgr(), tier_name).await;
9333        let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
9334            .await
9335            .expect("tier lease should resolve")
9336            .backend_identity();
9337        let entry = Jentry {
9338            persisted_version: 0,
9339            obj_name: "multi-source-recovery-object".to_string(),
9340            version_id: "multi-source-recovery-version".to_string(),
9341            tier_name: tier_name.to_string(),
9342            backend_identity: Some(backend_identity),
9343            version_id_exact: true,
9344            version_state: rustfs_filemeta::TransitionVersionState::Exact,
9345            state: TierDeleteJournalState::Committed,
9346            source: None,
9347            dispatch: None,
9348        };
9349        let path = tier_delete_journal_object_name(&entry);
9350        let data = encode_tier_delete_journal_entry(&entry).expect("tier journal should encode");
9351        for pool in &store.pools {
9352            com::save_config(pool.clone(), &path, data.clone())
9353                .await
9354                .expect("source and target tier journals should persist");
9355        }
9356
9357        let active_pool_meta = {
9358            let mut pool_meta = store.pool_meta.write().await;
9359            let start_time = OffsetDateTime::now_utc();
9360            for pool_idx in [0, 1] {
9361                pool_meta.pools[pool_idx].decommission = Some(PoolDecommissionInfo {
9362                    start_time: Some(start_time),
9363                    ..Default::default()
9364                });
9365            }
9366            pool_meta.clone()
9367        };
9368        active_pool_meta
9369            .save(store.pools.clone())
9370            .await
9371            .expect("multiple active decommission runs should persist");
9372        let mut restarted_pool_meta = PoolMeta::default();
9373        restarted_pool_meta
9374            .load(store.pools[0].clone(), store.pools.clone())
9375            .await
9376            .expect("multiple active decommission runs should reload");
9377        *store.pool_meta.write().await = restarted_pool_meta;
9378
9379        let record = validate_durable_ilm_record(&path, &data).expect("tier journal should validate");
9380        let source_zero_receipt = store
9381            .persist_decommission_durable_ilm_receipt_for_test(0, 1, &path, &record, false)
9382            .await
9383            .expect("source pool zero receipt should persist on the other active source");
9384        let source_one_receipt = store
9385            .persist_decommission_durable_ilm_receipt_for_test(1, 0, &path, &record, false)
9386            .await
9387            .expect("source pool one receipt should persist on the other active source");
9388        assert_ne!(
9389            source_zero_receipt, source_one_receipt,
9390            "active source runs must have distinct receipt paths"
9391        );
9392
9393        let stats = recover_tier_delete_journal_entries(store.clone(), 100, None)
9394            .await
9395            .expect("cross-source receipts should not remove active source journals");
9396        assert_eq!((stats.scanned, stats.deleted, stats.failed), (1, 1, 0));
9397        for pool in &store.pools {
9398            assert_eq!(
9399                com::read_config(pool.clone(), &path)
9400                    .await
9401                    .expect("cross-source receipts alone must retain every journal copy"),
9402                data
9403            );
9404        }
9405
9406        store
9407            .persist_decommission_durable_ilm_receipt_for_test(0, 2, &path, &record, false)
9408            .await
9409            .expect("source pool zero receipt should persist on the target");
9410        store
9411            .persist_decommission_durable_ilm_receipt_for_test(1, 2, &path, &record, false)
9412            .await
9413            .expect("source pool one receipt should persist on the target");
9414
9415        let stats = recover_tier_delete_journal_entries(store.clone(), 100, None)
9416            .await
9417            .expect("multi-source tier journal recovery should complete");
9418        assert_eq!((stats.scanned, stats.deleted, stats.failed), (1, 1, 0));
9419        assert_eq!(
9420            com::read_config(store.pools[0].clone(), &path)
9421                .await
9422                .expect("first active source must remain after target recovery"),
9423            data
9424        );
9425        assert_eq!(
9426            com::read_config(store.pools[1].clone(), &path)
9427                .await
9428                .expect("second active source must remain after target recovery"),
9429            data
9430        );
9431        assert!(matches!(
9432            com::read_config(store.pools[2].clone(), &path).await,
9433            Err(Error::ConfigNotFound)
9434        ));
9435        assert!(backend.remove_versions().await.contains(&(entry.obj_name, entry.version_id)));
9436    }
9437
9438    #[cfg(feature = "test-util")]
9439    #[tokio::test]
9440    #[serial_test::serial(storage_class_env)]
9441    async fn decommission_durable_ilm_receipt_pagination_fails_closed_on_second_page() {
9442        const RECEIPT_COUNT: usize = 1001;
9443
9444        let temp_dir = tempfile::tempdir().expect("create paginated receipt store dir");
9445        let (_ctx, store, _shutdown) =
9446            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "durable-ilm-receipt-pages", &[4, 4])).await;
9447        store.pool_meta.write().await.pools[0].decommission = Some(PoolDecommissionInfo {
9448            start_time: Some(OffsetDateTime::now_utc()),
9449            ..Default::default()
9450        });
9451
9452        futures::stream::iter(0..RECEIPT_COUNT)
9453            .map(|index| {
9454                let store = store.clone();
9455                async move {
9456                    let id = format!("{index:064x}");
9457                    let source_path = format!("ilm/tier-delete-journal/{id}.json");
9458                    let record = ValidatedDurableIlmRecord {
9459                        namespace: "tier-delete-journal",
9460                        id_kind: "operation_id",
9461                        id,
9462                        checkpoint: DurableIlmRecordCheckpoint::TierDeleteJournal {
9463                            content_sha256: format!("{:064x}", index + RECEIPT_COUNT),
9464                            identity_sha256: "f".repeat(64),
9465                            committed: false,
9466                            dispatch_identity_sha256: None,
9467                            state: None,
9468                        },
9469                    };
9470                    store
9471                        .persist_decommission_durable_ilm_receipt_for_test(0, 0, &source_path, &record, true)
9472                        .await?;
9473                    store
9474                        .persist_decommission_durable_ilm_receipt_for_test(0, 1, &source_path, &record, true)
9475                        .await?;
9476                    Ok::<(), Error>(())
9477                }
9478            })
9479            .buffer_unordered(32)
9480            .try_collect::<Vec<_>>()
9481            .await
9482            .expect("more than one receipt page should persist");
9483        store
9484            .persist_decommission_durable_ilm_manifest_for_test(0)
9485            .await
9486            .expect("paginated source receipts should produce a manifest");
9487
9488        let target_receipts = store
9489            .decommission_durable_ilm_receipt_paths_for_test(0)
9490            .await
9491            .expect("paginated target receipts should list");
9492        assert_eq!(target_receipts.len(), RECEIPT_COUNT);
9493        let (target_pool_idx, second_page_path) = target_receipts
9494            .get(1000)
9495            .cloned()
9496            .expect("the real 1000-item page boundary should expose a second page receipt");
9497        let receipt_bytes = com::read_config(store.pools[target_pool_idx].clone(), &second_page_path)
9498            .await
9499            .expect("second page receipt should be readable");
9500
9501        com::delete_config(store.pools[target_pool_idx].clone(), &second_page_path)
9502            .await
9503            .expect("second page receipt should delete");
9504        let missing = store
9505            .complete_decommission(0)
9506            .await
9507            .expect_err("a missing second page receipt must block completion")
9508            .to_string();
9509        assert!(missing.contains(&second_page_path));
9510        assert!(
9511            !store.pool_meta.read().await.pools[0]
9512                .decommission
9513                .as_ref()
9514                .expect("source pool should remain in decommission")
9515                .complete
9516        );
9517        assert!(com::read_config(store.pools[0].clone(), &second_page_path).await.is_ok());
9518
9519        let full_tail = ObjectOptions {
9520            max_parity: true,
9521            write_completion: crate::object_api::WriteCompletion::TailDrained,
9522            ..Default::default()
9523        };
9524        com::save_config_with_opts(store.pools[target_pool_idx].clone(), &second_page_path, receipt_bytes.clone(), &full_tail)
9525            .await
9526            .expect("second page receipt should restore");
9527        com::save_config_with_opts(store.pools[target_pool_idx].clone(), &second_page_path, b"{corrupt".to_vec(), &full_tail)
9528            .await
9529            .expect("second page receipt should corrupt deterministically");
9530        let corrupt = store
9531            .complete_decommission(0)
9532            .await
9533            .expect_err("a corrupt second page receipt must block completion")
9534            .to_string();
9535        assert!(corrupt.contains(&second_page_path));
9536        assert!(corrupt.contains("invalid"));
9537        assert!(
9538            !store.pool_meta.read().await.pools[0]
9539                .decommission
9540                .as_ref()
9541                .expect("source pool should remain in decommission")
9542                .complete
9543        );
9544        assert!(com::read_config(store.pools[0].clone(), &second_page_path).await.is_ok());
9545    }
9546
9547    #[cfg(feature = "test-util")]
9548    #[test]
9549    #[serial_test::serial(storage_class_env)]
9550    fn decommission_migrates_and_verifies_registered_durable_ilm_records() {
9551        std::thread::Builder::new()
9552            .name("durable-ilm-decommission-test".to_string())
9553            .stack_size(16 * 1024 * 1024)
9554            .spawn(|| {
9555                let runtime = tokio::runtime::Builder::new_multi_thread()
9556                    .enable_all()
9557                    .worker_threads(2)
9558                    .build()
9559                    .expect("durable ILM decommission runtime should build");
9560                runtime.block_on(decommission_migrates_and_verifies_registered_durable_ilm_records_scenario());
9561            })
9562            .expect("durable ILM decommission scenario thread should spawn")
9563            .join()
9564            .expect("durable ILM decommission scenario should not panic");
9565    }
9566
9567    #[cfg(feature = "test-util")]
9568    async fn decommission_migrates_and_verifies_registered_durable_ilm_records_scenario() {
9569        let temp_dir = tempfile::tempdir().expect("create temp store dir");
9570        let (ctx, store, _shutdown) =
9571            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "durable-ilm-decommission", &[4, 4])).await;
9572
9573        let tier_name = "DECOMMISSION-ILM";
9574        let backend = register_transition_reconcile_test_tier(&ctx.tier_config_mgr(), tier_name).await;
9575        let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
9576            .await
9577            .expect("tier lease should resolve")
9578            .backend_identity();
9579        let tier_entry = Jentry {
9580            persisted_version: 0,
9581            obj_name: "decommissioned-remote-object".to_string(),
9582            version_id: "decommissioned-remote-version".to_string(),
9583            tier_name: tier_name.to_string(),
9584            backend_identity: Some(backend_identity),
9585            version_id_exact: true,
9586            version_state: rustfs_filemeta::TransitionVersionState::Exact,
9587            state: TierDeleteJournalState::Committed,
9588            source: None,
9589            dispatch: None,
9590        };
9591        let tier_path = tier_delete_journal_object_name(&tier_entry);
9592        let tier_bytes = encode_tier_delete_journal_entry(&tier_entry).expect("tier journal should encode");
9593
9594        let mut transaction = TransitionTransaction::new(TransitionTransactionInit {
9595            deployment_id: ctx.deployment_id().expect("test store should initialize deployment id"),
9596            transaction_id: uuid::Uuid::new_v4(),
9597            owner_epoch: uuid::Uuid::new_v4(),
9598            write_id: uuid::Uuid::new_v4(),
9599            source: TransitionSourceIdentity {
9600                bucket: "source-bucket".to_string(),
9601                object: "source-object".to_string(),
9602                version_id: Some(uuid::Uuid::new_v4()),
9603                data_dir: uuid::Uuid::new_v4(),
9604                mod_time_unix_nanos: 1_770_000_000_000_000_000,
9605                size: 42,
9606                etag: "source-etag".to_string(),
9607                version_mode: TransitionSourceVersionMode::Versioned,
9608            },
9609            tier_name: tier_name.to_string(),
9610            backend_fingerprint: backend_identity,
9611            not_after_unix_nanos: 1_780_000_000_000_000_000,
9612        })
9613        .expect("transition transaction should build");
9614        let regressed_transaction = transaction.clone();
9615        transaction
9616            .advance(transaction.fence(), TransitionTransactionState::UploadOutcomeUnknown, None)
9617            .expect("transition transaction should advance before migration");
9618        let transaction_path = transition_transaction_record_object_name(transaction.transaction_id)
9619            .expect("transition transaction path should build");
9620        let transaction_bytes = transaction.encode().expect("transition transaction should encode");
9621
9622        let manual_job_id = uuid::Uuid::new_v4();
9623        let manual_bucket = format!("manual-decommission-{}", manual_job_id.simple());
9624        let manual_options = ManualTransitionRunOptions {
9625            prefix: "logs/".to_string(),
9626            tier: Some(tier_name.to_string()),
9627            ..Default::default()
9628        };
9629        let mut manual_job = ManualTransitionJobRecord::new(manual_job_id, &manual_bucket, &manual_options, "old-owner");
9630        manual_job.scan_completed = true;
9631        manual_job.report.enqueued = 1;
9632        manual_job.lease_expires_at_unix_nanos = 0;
9633        let manual_scope = ManualTransitionScopeAdmission::from_job(&manual_job);
9634        let task_key = manual_transition_worker_result_task_key(&manual_bucket, "logs/a", None);
9635        let manual_task = ManualTransitionTaskRecord::new(manual_job_id, &task_key, &manual_bucket, "logs/a", None, tier_name);
9636        let manual_result =
9637            ManualTransitionWorkerResultRecord::new(manual_job_id, &task_key, ManualTransitionWorkerResult::Completed);
9638
9639        let manual_job_path = manual_transition_job_record_object_name(manual_job_id).expect("manual job path should build");
9640        let manual_scope_path =
9641            manual_transition_scope_record_object_name(&manual_scope.scope_key).expect("manual scope path should build");
9642        let manual_task_path =
9643            manual_transition_task_object_name(manual_job_id, &task_key).expect("manual task path should build");
9644        let manual_result_path = manual_transition_worker_result_object_name(manual_job_id, &task_key)
9645            .expect("manual worker result path should build");
9646        let manual_job_bytes = manual_job.encode().expect("manual job should encode");
9647        let manual_scope_bytes = serde_json::to_vec(&manual_scope).expect("manual scope should encode");
9648        let manual_task_bytes = manual_task.encode().expect("manual task should encode");
9649        let manual_result_bytes = manual_result.encode().expect("manual result should encode");
9650
9651        let records = vec![
9652            (tier_path.clone(), tier_bytes.clone()),
9653            (transaction_path.clone(), transaction_bytes.clone()),
9654            (manual_job_path.clone(), manual_job_bytes.clone()),
9655            (manual_scope_path.clone(), manual_scope_bytes.clone()),
9656            (manual_task_path.clone(), manual_task_bytes.clone()),
9657            (manual_result_path.clone(), manual_result_bytes.clone()),
9658        ];
9659        for (path, data) in &records {
9660            com::save_config(store.pools[0].clone(), path, data.clone())
9661                .await
9662                .expect("durable ILM source record should persist");
9663        }
9664
9665        let legacy_queue = [com::CONFIG_PREFIX, BUCKET_META_PREFIX]
9666            .into_iter()
9667            .map(|prefix| {
9668                DecomBucketInfo {
9669                    name: RUSTFS_META_BUCKET.to_string(),
9670                    prefix: prefix.to_string(),
9671                }
9672                .to_string()
9673            })
9674            .collect();
9675        let legacy_pool_meta = {
9676            let mut pool_meta = store.pool_meta.write().await;
9677            pool_meta.pools[0].decommission = Some(PoolDecommissionInfo {
9678                queued: true,
9679                queued_buckets: legacy_queue,
9680                ..Default::default()
9681            });
9682            pool_meta.clone()
9683        };
9684        legacy_pool_meta
9685            .save(store.pools.clone())
9686            .await
9687            .expect("legacy decommission queue should persist before restart");
9688        let mut restarted_pool_meta = PoolMeta::default();
9689        restarted_pool_meta
9690            .load(store.pools[0].clone(), store.pools.clone())
9691            .await
9692            .expect("legacy decommission queue should reload after restart");
9693        *store.pool_meta.write().await = restarted_pool_meta;
9694        set_test_decommission_capacity_override(&store, 0);
9695        store
9696            .promote_queued_decommission_for_test(0)
9697            .await
9698            .expect("legacy queued decommission should resume");
9699        let expected_ilm_queue = DecomBucketInfo {
9700            name: RUSTFS_META_BUCKET.to_string(),
9701            prefix: ILM_META_PREFIX.to_string(),
9702        }
9703        .to_string();
9704        {
9705            let pool_meta = store.pool_meta.read().await;
9706            let decommission = pool_meta.pools[0]
9707                .decommission
9708                .as_ref()
9709                .expect("decommission state should remain present");
9710            assert!(!decommission.queued);
9711            assert!(decommission.queued_buckets.contains(&expected_ilm_queue));
9712        }
9713
9714        let ilm_bucket = DecomBucketInfo {
9715            name: RUSTFS_META_BUCKET.to_string(),
9716            prefix: ILM_META_PREFIX.to_string(),
9717        };
9718        for _ in 0..2 {
9719            store
9720                .decommission_pool_for_test(CancellationToken::new(), 0, store.pools[0].clone(), ilm_bucket.clone())
9721                .await
9722                .expect("durable ILM decommission should be idempotent");
9723        }
9724        for (path, expected) in &records {
9725            assert_eq!(
9726                com::read_config(store.pools[0].clone(), path)
9727                    .await
9728                    .expect("source should remain until the final sweep"),
9729                *expected
9730            );
9731            assert_eq!(
9732                com::read_config(store.pools[1].clone(), path)
9733                    .await
9734                    .expect("target should contain the migrated record"),
9735                *expected
9736            );
9737        }
9738        assert_eq!(
9739            store
9740                .decommission_durable_ilm_receipt_count_for_test(0)
9741                .await
9742                .expect("no receipt should exist before the final sweep"),
9743            0
9744        );
9745        let isolated_tier_stats = recover_tier_delete_journal_entries(store.clone(), 100, None)
9746            .await
9747            .expect("tier recovery should retain a terminal record until its receipt is committed");
9748        assert!(isolated_tier_stats.scanned >= 1);
9749        assert_eq!(isolated_tier_stats.deleted, 0);
9750        assert!(isolated_tier_stats.failed >= 1);
9751        assert_eq!(
9752            com::read_config(store.pools[1].clone(), &tier_path)
9753                .await
9754                .expect("receipt isolation must retain the target tier journal"),
9755            tier_bytes
9756        );
9757
9758        com::delete_config(store.pools[1].clone(), &manual_job_path)
9759            .await
9760            .expect("target manual job should delete");
9761        let missing = store
9762            .verify_and_cleanup_decommissioned_durable_ilm_record_for_test(
9763                0,
9764                store.pools[0].get_disks_by_key(&manual_job_path),
9765                &manual_job_path,
9766            )
9767            .await
9768            .expect_err("missing target must block source cleanup");
9769        let missing = missing.to_string();
9770        assert!(missing.contains(&manual_job_path) && missing.contains(&manual_job_id.to_string()));
9771        assert_eq!(
9772            com::read_config(store.pools[0].clone(), &manual_job_path)
9773                .await
9774                .expect("missing target must retain source"),
9775            manual_job_bytes
9776        );
9777        com::save_config(store.pools[1].clone(), &manual_job_path, manual_job_bytes.clone())
9778            .await
9779            .expect("target manual job should restore");
9780
9781        com::save_config(store.pools[1].clone(), &transaction_path, b"{corrupt".to_vec())
9782            .await
9783            .expect("target transaction should corrupt deterministically");
9784        let corrupt = store
9785            .verify_and_cleanup_decommissioned_durable_ilm_record_for_test(
9786                0,
9787                store.pools[0].get_disks_by_key(&transaction_path),
9788                &transaction_path,
9789            )
9790            .await
9791            .expect_err("corrupt target must block source cleanup");
9792        let corrupt = corrupt.to_string();
9793        assert!(corrupt.contains(&transaction_path) && corrupt.contains(&transaction.transaction_id.to_string()));
9794        assert_eq!(
9795            com::read_config(store.pools[0].clone(), &transaction_path)
9796                .await
9797                .expect("corrupt target must retain source"),
9798            transaction_bytes
9799        );
9800        com::save_config(store.pools[1].clone(), &transaction_path, transaction_bytes.clone())
9801            .await
9802            .expect("target transaction should restore");
9803
9804        com::save_config(store.pools[1].clone(), &manual_scope_path, manual_scope_bytes.clone())
9805            .await
9806            .expect("target scope rewrite should invalidate cached metadata before the quorum check");
9807        let target_scope_set = store.pools[1].get_disks_by_key(&manual_scope_path);
9808        let original_target_scope_disks = {
9809            let mut disks = target_scope_set.disks.write().await;
9810            let original = disks.clone();
9811            for disk in disks.iter_mut().take(3) {
9812                *disk = None;
9813            }
9814            original
9815        };
9816        let quorum_error = store
9817            .verify_and_cleanup_decommissioned_durable_ilm_record_for_test(
9818                0,
9819                store.pools[0].get_disks_by_key(&manual_scope_path),
9820                &manual_scope_path,
9821            )
9822            .await
9823            .expect_err("target below read quorum must block source cleanup");
9824        *target_scope_set.disks.write().await = original_target_scope_disks;
9825        let quorum_error = quorum_error.to_string();
9826        assert!(quorum_error.contains(&manual_scope_path) && quorum_error.contains(&manual_job_id.to_string()));
9827        assert!(com::read_config(store.pools[0].clone(), &manual_scope_path).await.is_ok());
9828
9829        com::save_config(store.pools[1].clone(), &manual_task_path, manual_task_bytes.clone())
9830            .await
9831            .expect("target task rewrite should invalidate cached metadata before the quorum check");
9832        let manual_task_record =
9833            validate_durable_ilm_record(&manual_task_path, &manual_task_bytes).expect("manual task should validate");
9834        let manual_task_receipt_path = store
9835            .decommission_durable_ilm_receipt_path_for_test(0, &manual_task_path, &manual_task_record)
9836            .await
9837            .expect("manual task receipt path should resolve");
9838        let target_task_set = store.pools[1].get_disks_by_key(&manual_task_receipt_path);
9839        let offline_target_task_disks = force_set_disk_range_offline_for_test(&target_task_set, 0..2).await;
9840        let receipt_quorum_error = store
9841            .verify_and_cleanup_decommissioned_durable_ilm_record_for_test(
9842                0,
9843                store.pools[0].get_disks_by_key(&manual_task_path),
9844                &manual_task_path,
9845            )
9846            .await
9847            .expect_err("target read quorum without receipt write quorum must retain the source");
9848        drop(offline_target_task_disks);
9849        let receipt_quorum_error = receipt_quorum_error.to_string();
9850        assert!(receipt_quorum_error.contains("receipt"));
9851        assert!(receipt_quorum_error.contains(&manual_task_path));
9852        assert!(receipt_quorum_error.contains(&manual_job_id.to_string()));
9853        assert!(com::read_config(store.pools[0].clone(), &manual_task_path).await.is_ok());
9854        store
9855            .verify_and_cleanup_decommissioned_durable_ilm_record_for_test(
9856                0,
9857                store.pools[0].get_disks_by_key(&manual_task_path),
9858                &manual_task_path,
9859            )
9860            .await
9861            .expect("healthy target should persist the receipt before source cleanup");
9862
9863        let unknown_path = "ilm/future-durable/jobs/one.json";
9864        com::save_config(store.pools[0].clone(), unknown_path, b"{}".to_vec())
9865            .await
9866            .expect("unknown durable ILM record should persist for the guard test");
9867        let unknown_migration = store
9868            .decommission_pool_for_test(CancellationToken::new(), 0, store.pools[0].clone(), ilm_bucket)
9869            .await
9870            .expect_err("unregistered durable ILM namespace must block migration");
9871        assert!(unknown_migration.to_string().contains(unknown_path));
9872        let unknown_final_sweep = store
9873            .check_after_decommission_for_test(0)
9874            .await
9875            .expect_err("unregistered durable ILM namespace must block completion");
9876        assert!(unknown_final_sweep.to_string().contains(unknown_path));
9877        com::delete_config(store.pools[0].clone(), unknown_path)
9878            .await
9879            .expect("unknown guard fixture should be removed before the successful final sweep");
9880
9881        store
9882            .check_after_decommission_for_test(0)
9883            .await
9884            .expect("production final sweep should validate every target before cleanup");
9885        assert_eq!(
9886            store
9887                .decommission_durable_ilm_receipt_count_for_test(0)
9888                .await
9889                .expect("durable ILM receipts should be listable"),
9890            records.len(),
9891            "every cleaned source record must have a durable validation receipt"
9892        );
9893        for (path, expected) in &records {
9894            assert!(
9895                matches!(com::read_config(store.pools[0].clone(), path).await, Err(Error::ConfigNotFound)),
9896                "final sweep should remove the validated source `{path}`"
9897            );
9898            assert_eq!(
9899                com::read_config(store.pools[1].clone(), path)
9900                    .await
9901                    .expect("final sweep must preserve the target"),
9902                *expected
9903            );
9904        }
9905
9906        let mut crash_restarted_pool_meta = PoolMeta::default();
9907        crash_restarted_pool_meta
9908            .load(store.pools[0].clone(), store.pools.clone())
9909            .await
9910            .expect("pool metadata should reload after the simulated pre-complete crash");
9911        *store.pool_meta.write().await = crash_restarted_pool_meta;
9912
9913        let (manual_job_receipt_pool, manual_job_receipt_path) = store
9914            .decommission_durable_ilm_receipt_paths_for_test(0)
9915            .await
9916            .expect("durable ILM receipt paths should be listable")
9917            .into_iter()
9918            .find(|(_, path)| path.contains(&manual_job_path))
9919            .expect("manual job should have one target receipt");
9920        let manual_job_receipt_bytes = com::read_config(store.pools[manual_job_receipt_pool].clone(), &manual_job_receipt_path)
9921            .await
9922            .expect("manual job receipt should be readable before deletion");
9923        com::delete_config(store.pools[manual_job_receipt_pool].clone(), &manual_job_receipt_path)
9924            .await
9925            .expect("manual job receipt should delete after source cleanup");
9926        com::delete_config(store.pools[1].clone(), &manual_job_path)
9927            .await
9928            .expect("post-crash target manual job should delete");
9929        let missing_after_crash = store
9930            .complete_decommission(0)
9931            .await
9932            .expect_err("completion must reject a missing target after source cleanup and restart")
9933            .to_string();
9934        assert!(missing_after_crash.contains("receipt"));
9935        assert!(missing_after_crash.contains(&manual_job_path));
9936        assert!(missing_after_crash.contains(&manual_job_id.to_string()));
9937        assert!(
9938            !store.pool_meta.read().await.pools[0]
9939                .decommission
9940                .as_ref()
9941                .expect("decommission state should survive restart")
9942                .complete
9943        );
9944        com::save_config(store.pools[1].clone(), &manual_job_path, manual_job_bytes.clone())
9945            .await
9946            .expect("post-crash target manual job should restore");
9947        com::save_config(
9948            store.pools[manual_job_receipt_pool].clone(),
9949            &manual_job_receipt_path,
9950            manual_job_receipt_bytes,
9951        )
9952        .await
9953        .expect("manual job receipt should restore after the missing-receipt check");
9954
9955        com::save_config(store.pools[1].clone(), &transaction_path, b"{corrupt".to_vec())
9956            .await
9957            .expect("post-crash target transaction should corrupt deterministically");
9958        let corrupt_after_crash = store
9959            .complete_decommission(0)
9960            .await
9961            .expect_err("completion must reject a corrupt target after source cleanup and restart")
9962            .to_string();
9963        assert!(corrupt_after_crash.contains(&transaction_path));
9964        assert!(corrupt_after_crash.contains(&transaction.transaction_id.to_string()));
9965        com::save_config(store.pools[1].clone(), &transaction_path, transaction_bytes.clone())
9966            .await
9967            .expect("post-crash target transaction should restore");
9968
9969        let mut wrong_manual_job = manual_job.clone();
9970        wrong_manual_job.job_id = uuid::Uuid::new_v4();
9971        com::save_config(
9972            store.pools[1].clone(),
9973            &manual_job_path,
9974            wrong_manual_job.encode().expect("wrong-id job should encode"),
9975        )
9976        .await
9977        .expect("post-crash target manual job should accept the wrong-id fixture");
9978        let wrong_id_after_crash = store
9979            .complete_decommission(0)
9980            .await
9981            .expect_err("completion must reject a target record with the wrong id")
9982            .to_string();
9983        assert!(wrong_id_after_crash.contains(&manual_job_path));
9984        assert!(wrong_id_after_crash.contains(&manual_job_id.to_string()));
9985        com::save_config(store.pools[1].clone(), &manual_job_path, manual_job_bytes.clone())
9986            .await
9987            .expect("post-crash target manual job should restore after the wrong-id check");
9988
9989        com::save_config(
9990            store.pools[1].clone(),
9991            &transaction_path,
9992            regressed_transaction
9993                .encode()
9994                .expect("regressed transition transaction should encode"),
9995        )
9996        .await
9997        .expect("post-crash target transaction should accept the regression fixture");
9998        let regression_after_crash = store
9999            .complete_decommission(0)
10000            .await
10001            .expect_err("completion must reject a lower transition transaction revision")
10002            .to_string();
10003        assert!(regression_after_crash.contains("generation mismatch"));
10004        assert!(regression_after_crash.contains(&transaction_path));
10005        assert!(regression_after_crash.contains(&transaction.transaction_id.to_string()));
10006        com::save_config(store.pools[1].clone(), &transaction_path, transaction_bytes.clone())
10007            .await
10008            .expect("post-crash target transaction should restore after the regression check");
10009
10010        let (manual_task_receipt_pool, manual_task_receipt_path) = store
10011            .decommission_durable_ilm_receipt_paths_for_test(0)
10012            .await
10013            .expect("durable ILM receipt paths should be listable")
10014            .into_iter()
10015            .find(|(_, path)| path.contains(&manual_task_path))
10016            .expect("manual task receipt should retain its reversible source path");
10017        let manual_task_receipt_bytes =
10018            com::read_config(store.pools[manual_task_receipt_pool].clone(), &manual_task_receipt_path)
10019                .await
10020                .expect("manual task receipt should be readable before corruption");
10021        com::save_config(
10022            store.pools[manual_task_receipt_pool].clone(),
10023            &manual_task_receipt_path,
10024            b"{corrupt".to_vec(),
10025        )
10026        .await
10027        .expect("manual task receipt should corrupt deterministically");
10028        let corrupt_receipt = store
10029            .complete_decommission(0)
10030            .await
10031            .expect_err("completion must fail closed on a corrupt receipt")
10032            .to_string();
10033        assert!(corrupt_receipt.contains(&manual_task_path));
10034        assert!(corrupt_receipt.contains(&manual_job_id.to_string()));
10035        com::save_config(
10036            store.pools[manual_task_receipt_pool].clone(),
10037            &manual_task_receipt_path,
10038            manual_task_receipt_bytes,
10039        )
10040        .await
10041        .expect("manual task receipt should restore after the corruption check");
10042
10043        let tier_stats = recover_tier_delete_journal_entries(store.clone(), 100, None)
10044            .await
10045            .expect("tier journal recovery should consume the migrated record before completion");
10046        assert_eq!((tier_stats.scanned, tier_stats.deleted, tier_stats.failed), (1, 1, 0));
10047        assert!(matches!(com::read_config(store.clone(), &tier_path).await, Err(Error::ConfigNotFound)));
10048
10049        let recovered_transition_version = "recovered-transition-version".to_string();
10050        backend
10051            .set_transition_candidate_probe_override(Some(TransitionCandidateProbe::VersionedPresent(
10052                recovered_transition_version.clone(),
10053            )))
10054            .await;
10055        let transaction_stats = {
10056            let _proof = crate::services::notification_sys::install_current_remote_version_state_fleet_proof_for_test();
10057            temp_env::async_with_vars(
10058                [
10059                    (rustfs_config::ENV_TIER_REMOTE_VERSION_STATE_WRITE, Some("true")),
10060                    (rustfs_config::ENV_TIER_REMOTE_VERSION_STATE_FLEET_CONFIRMED, Some("true")),
10061                ],
10062                recover_transition_transaction_records(store.clone(), 100, None),
10063            )
10064            .await
10065            .expect("transition recovery should advance and consume the migrated transaction before completion")
10066        };
10067        backend.set_transition_candidate_probe_override(None).await;
10068        assert_eq!(
10069            (
10070                transaction_stats.scanned,
10071                transaction_stats.recovered,
10072                transaction_stats.retained,
10073                transaction_stats.failed,
10074            ),
10075            (1, 1, 0, 0)
10076        );
10077        assert!(matches!(
10078            com::read_config(store.clone(), &transaction_path).await,
10079            Err(Error::ConfigNotFound)
10080        ));
10081        com::save_config(
10082            store.pools[1].clone(),
10083            &transaction_path,
10084            regressed_transaction
10085                .encode()
10086                .expect("post-terminal transition rollback should encode"),
10087        )
10088        .await
10089        .expect("target should accept the post-terminal rollback fixture");
10090        let post_terminal_regression = store
10091            .complete_decommission(0)
10092            .await
10093            .expect_err("terminal proof must not mask a lower transition revision")
10094            .to_string();
10095        assert!(post_terminal_regression.contains("generation mismatch"));
10096        assert!(post_terminal_regression.contains(&transaction_path));
10097        assert!(post_terminal_regression.contains(&transaction.transaction_id.to_string()));
10098        com::delete_config(store.pools[1].clone(), &transaction_path)
10099            .await
10100            .expect("post-terminal rollback fixture should be removed");
10101
10102        let manual_stats = recover_manual_transition_jobs_once(store.clone(), 100, None)
10103            .await
10104            .expect("manual recovery should advance the migrated job and consume its scope before completion");
10105        assert_eq!(
10106            (manual_stats.scanned, manual_stats.resumed, manual_stats.skipped, manual_stats.failed,),
10107            (1, 1, 0, 0)
10108        );
10109        assert!(matches!(
10110            com::read_config(store.clone(), &manual_scope_path).await,
10111            Err(Error::ConfigNotFound)
10112        ));
10113        let recovered_manual_job_bytes = com::read_config(store.pools[1].clone(), &manual_job_path)
10114            .await
10115            .expect("manual recovery should retain the advanced job record");
10116        assert_ne!(recovered_manual_job_bytes, manual_job_bytes);
10117
10118        com::save_config(store.pools[1].clone(), &manual_job_path, manual_job_bytes.clone())
10119            .await
10120            .expect("target manual job should accept the rollback fixture");
10121        let manual_regression = store
10122            .complete_decommission(0)
10123            .await
10124            .expect_err("completion must reject a manual job generation rollback")
10125            .to_string();
10126        assert!(manual_regression.contains("generation mismatch"));
10127        assert!(manual_regression.contains(&manual_job_path));
10128        assert!(manual_regression.contains(&manual_job_id.to_string()));
10129        com::save_config(store.pools[1].clone(), &manual_job_path, recovered_manual_job_bytes)
10130            .await
10131            .expect("target manual job should restore its recovered generation");
10132
10133        store
10134            .complete_decommission(0)
10135            .await
10136            .expect("completion should persist before receipt cleanup");
10137        assert!(
10138            store.pool_meta.read().await.pools[0]
10139                .decommission
10140                .as_ref()
10141                .expect("completed decommission state should remain present")
10142                .complete
10143        );
10144        assert_eq!(
10145            store
10146                .decommission_durable_ilm_receipt_count_for_test(0)
10147                .await
10148                .expect("receipt cleanup should be observable"),
10149            0
10150        );
10151        store
10152            .cleanup_decommission_durable_ilm_receipts_for_test(0)
10153            .await
10154            .expect("receipt cleanup should be idempotent");
10155        let removed_versions = backend.remove_versions().await;
10156        assert!(removed_versions.contains(&(tier_entry.obj_name.clone(), tier_entry.version_id.clone())));
10157        assert!(removed_versions.contains(&(transaction.remote_object.clone(), recovered_transition_version)));
10158    }
10159
10160    #[cfg(feature = "test-util")]
10161    async fn tier_delete_journal_count(store: Arc<crate::store::ECStore>) -> usize {
10162        store
10163            .list_objects_v2(RUSTFS_META_BUCKET, TIER_DELETE_JOURNAL_PREFIX, None, None, 100, false, None, false)
10164            .await
10165            .expect("tier delete journal should be listable")
10166            .objects
10167            .len()
10168    }
10169
10170    #[cfg(feature = "test-util")]
10171    async fn tier_delete_dispatch_manifest_count(store: Arc<crate::store::ECStore>) -> usize {
10172        store
10173            .list_objects_v2(
10174                RUSTFS_META_BUCKET,
10175                TIER_DELETE_DISPATCH_MANIFEST_PREFIX,
10176                None,
10177                None,
10178                100,
10179                false,
10180                None,
10181                false,
10182            )
10183            .await
10184            .expect("tier delete dispatch manifests should be listable")
10185            .objects
10186            .len()
10187    }
10188
10189    #[cfg(feature = "test-util")]
10190    async fn install_committed_tier_delete_journals(
10191        store: Arc<crate::store::ECStore>,
10192        tier_name: &str,
10193        backend_identity: [u8; 32],
10194        count: usize,
10195    ) -> Vec<Jentry> {
10196        let mut entries = Vec::with_capacity(count);
10197        for index in 0..count {
10198            let entry = Jentry {
10199                persisted_version: 0,
10200                obj_name: format!("remote/pass-drain-{index:06}.bin"),
10201                version_id: uuid::Uuid::new_v4().to_string(),
10202                tier_name: tier_name.to_string(),
10203                backend_identity: Some(backend_identity),
10204                version_id_exact: true,
10205                version_state: rustfs_filemeta::TransitionVersionState::Exact,
10206                state: TierDeleteJournalState::Committed,
10207                source: None,
10208                dispatch: None,
10209            };
10210            persist_tier_delete_journal_entry(store.clone(), &entry)
10211                .await
10212                .expect("committed tier-delete journal fixture should persist");
10213            entries.push(entry);
10214        }
10215        entries
10216    }
10217
10218    #[cfg(feature = "test-util")]
10219    fn synthetic_v6_dispatch_entry(
10220        bucket: &str,
10221        object: &str,
10222        tier_name: &str,
10223        backend_identity: [u8; 32],
10224        remote_version: &str,
10225    ) -> Jentry {
10226        Jentry {
10227            persisted_version: 0,
10228            obj_name: format!("remote/{object}"),
10229            version_id: remote_version.to_string(),
10230            tier_name: tier_name.to_string(),
10231            backend_identity: Some(backend_identity),
10232            version_id_exact: true,
10233            version_state: rustfs_filemeta::TransitionVersionState::Exact,
10234            state: TierDeleteJournalState::Prepared,
10235            source: Some(TierDeleteSourceIdentity {
10236                bucket: bucket.to_string(),
10237                object: object.to_string(),
10238                version_id: Some(uuid::Uuid::new_v4().to_string()),
10239                versioned: true,
10240                version_suspended: false,
10241                data_dir: Some(uuid::Uuid::new_v4().to_string()),
10242                etag: Some(format!("etag-{object}")),
10243                mod_time: Some(OffsetDateTime::UNIX_EPOCH.to_string()),
10244            }),
10245            dispatch: None,
10246        }
10247    }
10248
10249    #[cfg(feature = "test-util")]
10250    async fn install_aborting_dispatch_fixture(
10251        store: Arc<crate::store::ECStore>,
10252        bucket: &str,
10253        incarnation: uuid::Uuid,
10254        prefix: &str,
10255        tier_name: &str,
10256        backend_identity: [u8; 32],
10257        journal_count: usize,
10258    ) -> (String, Vec<Jentry>) {
10259        let entries = (0..journal_count)
10260            .map(|index| {
10261                (
10262                    synthetic_v6_dispatch_entry(
10263                        bucket,
10264                        &format!("{prefix}{index:06}.bin"),
10265                        tier_name,
10266                        backend_identity,
10267                        &uuid::Uuid::new_v4().to_string(),
10268                    ),
10269                    Some(TierDeleteJournalState::Prepared),
10270                )
10271            })
10272            .collect();
10273        install_test_tier_delete_dispatch_fixture(
10274            store,
10275            bucket,
10276            incarnation,
10277            prefix,
10278            entries,
10279            TierDeleteDispatchManifestState::Aborting,
10280        )
10281        .await
10282        .expect("Aborting rollback fixture should persist")
10283    }
10284
10285    #[cfg(feature = "test-util")]
10286    async fn drive_tier_delete_dispatch_restart_to_convergence(store: Arc<crate::store::ECStore>) {
10287        tokio::time::timeout(Duration::from_secs(30), async {
10288            let mut stable_empty_observations = 0;
10289            loop {
10290                recover_tier_delete_dispatch_manifests(store.clone(), 100, None)
10291                    .await
10292                    .expect("restarted manifest recovery pass should finish");
10293                recover_tier_delete_journal_entries(store.clone(), 100, None)
10294                    .await
10295                    .expect("restarted journal recovery pass should finish");
10296                if tier_delete_journal_count(store.clone()).await == 0
10297                    && tier_delete_dispatch_manifest_count(store.clone()).await == 0
10298                    && tier_delete_dispatch_manifest_recovery_count_for_test(&store) == 0
10299                {
10300                    stable_empty_observations += 1;
10301                    if stable_empty_observations == 10 {
10302                        break;
10303                    }
10304                } else {
10305                    stable_empty_observations = 0;
10306                }
10307                tokio::time::sleep(Duration::from_millis(50)).await;
10308            }
10309        })
10310        .await
10311        .expect("same-disk restart recovery should converge without retained dispatch state");
10312    }
10313
10314    #[cfg(feature = "test-util")]
10315    #[tokio::test]
10316    #[serial_test::serial(storage_class_env)]
10317    async fn tier_delete_recovery_pass_drains_multiple_fast_manifest_pages() {
10318        const MANIFEST_COUNT: usize = 10;
10319
10320        let temp_dir = tempfile::tempdir().expect("create fast manifest pass recovery store dir");
10321        let mut instance_ctx = crate::runtime::instance::InstanceContext::new();
10322        instance_ctx.suppress_tier_delete_journal_recovery_for_test();
10323        let (ctx, store, shutdown) = without_storage_class_env(build_isolated_test_store_with_layout(
10324            temp_dir.path(),
10325            "tier-delete-fast-manifest-pass",
10326            &[(1, 4)],
10327            CancellationToken::new(),
10328            Some(Arc::new(instance_ctx)),
10329        ))
10330        .await;
10331        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
10332        let tier_name = "FAST-MANIFEST-PASS";
10333        let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
10334        let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
10335            .await
10336            .expect("fast manifest pass tier lease should resolve")
10337            .backend_identity();
10338        for index in 0..MANIFEST_COUNT {
10339            // Pagination must not depend on same-bucket lock wait deadlines.
10340            let bucket = format!("tier-delete-fast-manifest-pass-{index}");
10341            store
10342                .make_bucket(&bucket, &MakeBucketOptions::default())
10343                .await
10344                .expect("fast manifest pass bucket should be created");
10345            let incarnation = store
10346                .bucket_incarnation_id(&bucket)
10347                .await
10348                .expect("fast manifest pass bucket incarnation should resolve");
10349            install_aborting_dispatch_fixture(
10350                store.clone(),
10351                &bucket,
10352                incarnation,
10353                &format!("manifest-page-{index:06}/"),
10354                tier_name,
10355                backend_identity,
10356                1,
10357            )
10358            .await;
10359        }
10360        assert_eq!(tier_delete_dispatch_manifest_count(store.clone()).await, MANIFEST_COUNT);
10361        assert_eq!(tier_delete_journal_count(store.clone()).await, MANIFEST_COUNT);
10362
10363        let cancel = CancellationToken::new();
10364        let mut manifest_marker = None;
10365        let mut journal_marker = None;
10366        let stats = recover_test_tier_delete_journal_pass_with_budget(
10367            store.clone(),
10368            &cancel,
10369            &mut manifest_marker,
10370            &mut journal_marker,
10371            Duration::from_secs(30),
10372        )
10373        .await;
10374
10375        assert!(!stats.canceled);
10376        assert!(!stats.deadline_exhausted);
10377        assert!(
10378            stats.manifest_pages > 1,
10379            "one production pass must cross the default eight-manifest page limit"
10380        );
10381        assert_eq!(stats.manifests.scanned, MANIFEST_COUNT);
10382        assert_eq!(stats.manifests.deleted, MANIFEST_COUNT, "full recovery result: {stats:?}");
10383        assert_eq!(stats.manifests.failed, 0, "full recovery result: {stats:?}");
10384        assert_eq!(manifest_marker, None);
10385        assert_eq!(tier_delete_dispatch_manifest_count(store.clone()).await, 0);
10386        assert_eq!(tier_delete_journal_count(store).await, 0);
10387        assert_eq!(backend.remove_count().await, 0, "rollback recovery must not call the remote tier");
10388        shutdown.cancel();
10389    }
10390
10391    #[cfg(feature = "test-util")]
10392    #[tokio::test]
10393    #[serial_test::serial(storage_class_env)]
10394    async fn tier_delete_manual_pass_retains_manifest_owned_by_startup_recovery() {
10395        let temp_dir = tempfile::tempdir().expect("create automatic recovery ownership store dir");
10396        let (ctx, store, shutdown) =
10397            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "tier-delete-auto-owner", &[4])).await;
10398        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
10399        let bucket = "tier-delete-auto-owner-bucket";
10400        store
10401            .make_bucket(bucket, &MakeBucketOptions::default())
10402            .await
10403            .expect("automatic recovery bucket should be created");
10404        let incarnation = store.bucket_incarnation_id(bucket).await.expect("bucket incarnation");
10405        let tier_name = "AUTO-OWNER";
10406        let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
10407        let identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
10408            .await
10409            .expect("automatic recovery tier lease")
10410            .backend_identity();
10411
10412        // The automatic worker must not observe a partially installed fixture.
10413        let lifecycle_guard = store
10414            .acquire_bucket_lifecycle_write_lock(bucket)
10415            .await
10416            .expect("fixture lifecycle lock");
10417        let (manifest_name, entries) =
10418            install_aborting_dispatch_fixture(store.clone(), bucket, incarnation, "auto-owner/", tier_name, identity, 1).await;
10419        let journal_name = tier_delete_journal_object_name(&entries[0]);
10420        let hook = TierDeleteDispatchRollbackTestHook::install_slow_delete(&journal_name, &journal_name);
10421        drop(lifecycle_guard);
10422        ctx.wake_tier_delete_journal_recovery();
10423        tokio::time::timeout(Duration::from_secs(30), hook.wait_until_delete_paused())
10424            .await
10425            .expect("startup recovery should own the manifest before a manual pass");
10426        assert!(tier_delete_dispatch_manifest_recovery_inflight_for_test(&store, &manifest_name));
10427
10428        let stats = recover_tier_delete_dispatch_manifests(store.clone(), 8, None)
10429            .await
10430            .expect("manual recovery scan");
10431        assert_eq!(stats.scanned, 1, "{stats:?}");
10432        assert_eq!(stats.retained, 1, "{stats:?}");
10433        assert_eq!(stats.deleted, 0, "{stats:?}");
10434        assert_eq!(stats.failed, 0, "{stats:?}");
10435        assert_eq!(tier_delete_dispatch_manifest_count(store.clone()).await, 1);
10436        assert_eq!(tier_delete_journal_count(store.clone()).await, 1);
10437
10438        hook.release_delete();
10439        tokio::time::timeout(Duration::from_secs(30), async {
10440            loop {
10441                let manifest_gone = matches!(com::read_config(store.clone(), &manifest_name).await, Err(Error::ConfigNotFound));
10442                if manifest_gone && !tier_delete_dispatch_manifest_recovery_inflight_for_test(&store, &manifest_name) {
10443                    break;
10444                }
10445                tokio::time::sleep(Duration::from_millis(10)).await;
10446            }
10447        })
10448        .await
10449        .expect("automatic recovery should converge without a manual retry");
10450        assert_eq!(tier_delete_dispatch_manifest_count(store.clone()).await, 0);
10451        assert_eq!(tier_delete_journal_count(store).await, 0);
10452        assert_eq!(backend.remove_count().await, 0, "rollback must not delete from the remote tier");
10453        shutdown.cancel();
10454    }
10455
10456    #[cfg(feature = "test-util")]
10457    #[tokio::test]
10458    #[serial_test::serial(storage_class_env)]
10459    async fn tier_delete_recovery_pass_drains_multiple_fast_journal_pages() {
10460        const JOURNAL_COUNT: usize = 25;
10461
10462        let temp_dir = tempfile::tempdir().expect("create fast pass recovery store dir");
10463        let (ctx, store, _shutdown) =
10464            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "tier-delete-fast-pass", &[4])).await;
10465        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
10466        let tier_name = "FAST-PASS-RECOVERY";
10467        let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
10468        let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
10469            .await
10470            .expect("fast pass tier lease should resolve")
10471            .backend_identity();
10472        let entries = install_committed_tier_delete_journals(store.clone(), tier_name, backend_identity, JOURNAL_COUNT).await;
10473        assert_eq!(tier_delete_journal_count(store.clone()).await, JOURNAL_COUNT);
10474
10475        let cancel = CancellationToken::new();
10476        let mut manifest_marker = None;
10477        let mut journal_marker = None;
10478        let stats = recover_test_tier_delete_journal_pass_with_budget(
10479            store.clone(),
10480            &cancel,
10481            &mut manifest_marker,
10482            &mut journal_marker,
10483            Duration::from_secs(30),
10484        )
10485        .await;
10486
10487        assert!(!stats.canceled);
10488        assert!(!stats.deadline_exhausted);
10489        assert!(
10490            stats.journal_pages > 1,
10491            "one production pass must cross the default eight-record page limit"
10492        );
10493        assert_eq!(stats.journals.scanned, JOURNAL_COUNT);
10494        assert_eq!(stats.journals.deleted, JOURNAL_COUNT);
10495        assert_eq!(stats.journals.failed, 0);
10496        assert_eq!(journal_marker, None);
10497        assert_eq!(tier_delete_journal_count(store).await, 0);
10498        assert_eq!(backend.exact_remove_count(), JOURNAL_COUNT);
10499        let mut removed = backend.remove_versions().await;
10500        removed.sort();
10501        let mut expected = entries
10502            .into_iter()
10503            .map(|entry| (entry.obj_name, entry.version_id))
10504            .collect::<Vec<_>>();
10505        expected.sort();
10506        assert_eq!(removed, expected);
10507    }
10508
10509    #[cfg(feature = "test-util")]
10510    #[tokio::test]
10511    #[serial_test::serial(storage_class_env)]
10512    async fn tier_delete_recovery_pass_rotates_failed_journal_pages_without_busy_loop() {
10513        const JOURNAL_COUNT: usize = 9;
10514
10515        let temp_dir = tempfile::tempdir().expect("create failed pass recovery store dir");
10516        let (ctx, store, _shutdown) =
10517            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "tier-delete-failed-pass", &[4])).await;
10518        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
10519        let tier_name = "FAILED-PASS-RECOVERY";
10520        let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
10521        let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
10522            .await
10523            .expect("failed pass tier lease should resolve")
10524            .backend_identity();
10525        install_committed_tier_delete_journals(store.clone(), tier_name, backend_identity, JOURNAL_COUNT).await;
10526        backend.set_remove_failure(true);
10527
10528        let cancel = CancellationToken::new();
10529        let mut manifest_marker = None;
10530        let mut journal_marker = None;
10531        let first = recover_test_tier_delete_journal_pass_with_budget(
10532            store.clone(),
10533            &cancel,
10534            &mut manifest_marker,
10535            &mut journal_marker,
10536            Duration::from_secs(30),
10537        )
10538        .await;
10539        assert_eq!(first.journal_pages, 1, "a no-progress failed page must stop the pass");
10540        assert_eq!(first.journals.scanned, 8);
10541        assert_eq!(first.journals.deleted, 0);
10542        assert_eq!(first.journals.failed, 8);
10543        assert!(journal_marker.is_some(), "a truncated failed page must retain its continuation marker");
10544        assert_eq!(tier_delete_journal_count(store.clone()).await, JOURNAL_COUNT);
10545
10546        let second = recover_test_tier_delete_journal_pass_with_budget(
10547            store.clone(),
10548            &cancel,
10549            &mut manifest_marker,
10550            &mut journal_marker,
10551            Duration::from_secs(30),
10552        )
10553        .await;
10554        assert_eq!(second.journal_pages, 1);
10555        assert_eq!(second.journals.scanned, 1);
10556        assert_eq!(second.journals.deleted, 0);
10557        assert_eq!(second.journals.failed, 1);
10558        assert_eq!(
10559            journal_marker, None,
10560            "end-of-list rotation must revisit the failed prefix on a later pass"
10561        );
10562        assert_eq!(tier_delete_journal_count(store.clone()).await, JOURNAL_COUNT);
10563
10564        backend.set_remove_failure(false);
10565        let third = recover_test_tier_delete_journal_pass_with_budget(
10566            store.clone(),
10567            &cancel,
10568            &mut manifest_marker,
10569            &mut journal_marker,
10570            Duration::from_secs(30),
10571        )
10572        .await;
10573        assert_eq!(third.journals.scanned, JOURNAL_COUNT);
10574        assert_eq!(third.journals.deleted, JOURNAL_COUNT);
10575        assert_eq!(third.journals.failed, 0);
10576        assert_eq!(tier_delete_journal_count(store).await, 0);
10577    }
10578
10579    #[cfg(feature = "test-util")]
10580    #[tokio::test]
10581    #[serial_test::serial(storage_class_env)]
10582    async fn tier_delete_journal_rejects_unbound_new_prepare_without_persisting() {
10583        let temp_dir = tempfile::tempdir().expect("create journal admission store dir");
10584        let (_ctx, store, _shutdown) =
10585            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "tier-journal-unbound-prepare", &[4])).await;
10586        let entry = Jentry {
10587            persisted_version: 0,
10588            obj_name: "remote/object".to_string(),
10589            version_id: uuid::Uuid::new_v4().to_string(),
10590            tier_name: "COLD-A".to_string(),
10591            backend_identity: Some([7; 32]),
10592            version_id_exact: true,
10593            version_state: rustfs_filemeta::TransitionVersionState::Exact,
10594            state: TierDeleteJournalState::Prepared,
10595            source: Some(TierDeleteSourceIdentity {
10596                bucket: "source-bucket".to_string(),
10597                object: "source-object".to_string(),
10598                version_id: Some(uuid::Uuid::new_v4().to_string()),
10599                versioned: true,
10600                version_suspended: false,
10601                data_dir: Some(uuid::Uuid::new_v4().to_string()),
10602                etag: Some("source-etag".to_string()),
10603                mod_time: Some(OffsetDateTime::UNIX_EPOCH.to_string()),
10604            }),
10605            dispatch: None,
10606        };
10607        let error = persist_tier_delete_journal_entry(store.clone(), &entry)
10608            .await
10609            .expect_err("a new source-owning journal without a v6 manifest binding must fail closed");
10610        assert!(error.to_string().contains("dispatch manifest binding"), "unexpected error: {error}");
10611        assert_eq!(tier_delete_journal_count(store).await, 0, "rejected prepare must not write a journal");
10612    }
10613
10614    #[cfg(feature = "test-util")]
10615    #[tokio::test]
10616    #[serial_test::serial(storage_class_env)]
10617    async fn dispatch_manifest_recovery_rolls_back_unapproved_crash_states_without_remote_io() {
10618        let temp_dir = tempfile::tempdir().expect("create dispatch rollback store dir");
10619        let (ctx, store, _shutdown) =
10620            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "dispatch-manifest-rollback", &[4])).await;
10621        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
10622        let bucket = "dispatch-manifest-rollback-bucket";
10623        store
10624            .make_bucket(bucket, &MakeBucketOptions::default())
10625            .await
10626            .expect("dispatch rollback bucket should be created");
10627        let incarnation = store
10628            .bucket_incarnation_id(bucket)
10629            .await
10630            .expect("bucket incarnation should resolve");
10631        let tier_name = "DISPATCH-ROLLBACK";
10632        let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
10633        let identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
10634            .await
10635            .expect("tier lease should resolve")
10636            .backend_identity();
10637
10638        let preparing = [
10639            ("preparing/a.bin", TierDeleteJournalState::Prepared),
10640            ("preparing/b.bin", TierDeleteJournalState::Dispatched),
10641        ]
10642        .into_iter()
10643        .map(|(object, state)| {
10644            (
10645                synthetic_v6_dispatch_entry(bucket, object, tier_name, identity, &uuid::Uuid::new_v4().to_string()),
10646                Some(state),
10647            )
10648        })
10649        .collect();
10650        install_test_tier_delete_dispatch_fixture(
10651            store.clone(),
10652            bucket,
10653            incarnation,
10654            "preparing/",
10655            preparing,
10656            TierDeleteDispatchManifestState::Preparing,
10657        )
10658        .await
10659        .expect("Preparing crash fixture should persist");
10660
10661        let aborting = [
10662            ("aborting/missing.bin", None),
10663            ("aborting/prepared.bin", Some(TierDeleteJournalState::Prepared)),
10664            ("aborting/dispatched.bin", Some(TierDeleteJournalState::Dispatched)),
10665        ]
10666        .into_iter()
10667        .map(|(object, state)| {
10668            (
10669                synthetic_v6_dispatch_entry(bucket, object, tier_name, identity, &uuid::Uuid::new_v4().to_string()),
10670                state,
10671            )
10672        })
10673        .collect();
10674        install_test_tier_delete_dispatch_fixture(
10675            store.clone(),
10676            bucket,
10677            incarnation,
10678            "aborting/",
10679            aborting,
10680            TierDeleteDispatchManifestState::Aborting,
10681        )
10682        .await
10683        .expect("Aborting crash fixture should persist");
10684        install_test_tier_delete_dispatch_fixture(
10685            store.clone(),
10686            bucket,
10687            incarnation,
10688            "empty/",
10689            Vec::new(),
10690            TierDeleteDispatchManifestState::Completed,
10691        )
10692        .await
10693        .expect("empty Completed crash fixture should persist");
10694
10695        let deleted = tokio::time::timeout(Duration::from_secs(30), async {
10696            let mut deleted = 0;
10697            loop {
10698                let stats = recover_tier_delete_dispatch_manifests(store.clone(), 100, None)
10699                    .await
10700                    .expect("manifest recovery should reconcile negative-seal crash states");
10701                deleted += stats.deleted;
10702                if tier_delete_dispatch_manifest_count(store.clone()).await == 0 {
10703                    break deleted;
10704                }
10705                // A different process-global test can briefly revoke the
10706                // fleet proof. Recovery must fail closed for that pass and
10707                // converge after the proof generation is republished.
10708                tokio::time::sleep(Duration::from_millis(10)).await;
10709            }
10710        })
10711        .await
10712        .expect("valid negative-seal crash states should eventually converge");
10713        assert_eq!(deleted, 3);
10714        assert_eq!(tier_delete_journal_count(store.clone()).await, 0);
10715        assert_eq!(tier_delete_dispatch_manifest_count(store).await, 0);
10716        assert_eq!(
10717            backend.remove_count().await,
10718            0,
10719            "rollback recovery must perform zero remote delete operations"
10720        );
10721    }
10722
10723    #[cfg(feature = "test-util")]
10724    #[tokio::test]
10725    #[serial_test::serial(storage_class_env)]
10726    async fn dispatch_manifest_recovery_quarantines_impossible_committed_rollback_set() {
10727        let temp_dir = tempfile::tempdir().expect("create dispatch quarantine store dir");
10728        let (ctx, store, _shutdown) =
10729            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "dispatch-manifest-quarantine", &[4])).await;
10730        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
10731        let bucket = "dispatch-manifest-quarantine-bucket";
10732        store
10733            .make_bucket(bucket, &MakeBucketOptions::default())
10734            .await
10735            .expect("dispatch quarantine bucket should be created");
10736        let incarnation = store
10737            .bucket_incarnation_id(bucket)
10738            .await
10739            .expect("bucket incarnation should resolve");
10740        let tier_name = "DISPATCH-QUARANTINE";
10741        let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
10742        let identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
10743            .await
10744            .expect("tier lease should resolve")
10745            .backend_identity();
10746        let entries = [TierDeleteJournalState::Prepared, TierDeleteJournalState::Committed]
10747            .into_iter()
10748            .enumerate()
10749            .map(|(index, state)| {
10750                (
10751                    synthetic_v6_dispatch_entry(
10752                        bucket,
10753                        &format!("quarantine/{index}.bin"),
10754                        tier_name,
10755                        identity,
10756                        &uuid::Uuid::new_v4().to_string(),
10757                    ),
10758                    Some(state),
10759                )
10760            })
10761            .collect();
10762        let (manifest_name, _) = install_test_tier_delete_dispatch_fixture(
10763            store.clone(),
10764            bucket,
10765            incarnation,
10766            "quarantine/",
10767            entries,
10768            TierDeleteDispatchManifestState::Aborting,
10769        )
10770        .await
10771        .expect("impossible rollback fixture should persist");
10772
10773        let stats = recover_tier_delete_dispatch_manifests(store.clone(), 100, None)
10774            .await
10775            .expect("quarantined manifest scan should finish");
10776        assert_eq!((stats.scanned, stats.deleted, stats.failed), (1, 0, 1));
10777        assert_eq!(
10778            tier_delete_journal_count(store.clone()).await,
10779            2,
10780            "validation must precede every deletion"
10781        );
10782        assert_eq!(tier_delete_dispatch_manifest_count(store.clone()).await, 1);
10783        assert_eq!(
10784            test_tier_delete_dispatch_manifest_state(store, &manifest_name)
10785                .await
10786                .expect("quarantined manifest should remain readable"),
10787            Some(TierDeleteDispatchManifestState::Aborting)
10788        );
10789        assert_eq!(backend.remove_count().await, 0);
10790    }
10791
10792    #[cfg(feature = "test-util")]
10793    #[tokio::test]
10794    #[serial_test::serial(storage_class_env)]
10795    async fn dispatch_manifest_rollback_bounded_concurrency_reaches_tail_behind_slow_member() {
10796        const JOURNAL_COUNT: usize = 65;
10797
10798        let temp_dir = tempfile::tempdir().expect("create bounded rollback store dir");
10799        let (ctx, store, _shutdown) =
10800            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "bounded-dispatch-rollback", &[4])).await;
10801        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
10802        let bucket = "bounded-dispatch-rollback-bucket";
10803        store
10804            .make_bucket(bucket, &MakeBucketOptions::default())
10805            .await
10806            .expect("bounded rollback bucket should be created");
10807        let incarnation = store
10808            .bucket_incarnation_id(bucket)
10809            .await
10810            .expect("bounded rollback bucket incarnation should resolve");
10811        let tier_name = "BOUNDED-DISPATCH-ROLLBACK";
10812        let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
10813        let identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
10814            .await
10815            .expect("bounded rollback tier lease should resolve")
10816            .backend_identity();
10817        let (manifest_name, entries) =
10818            install_aborting_dispatch_fixture(store.clone(), bucket, incarnation, "slow/", tier_name, identity, JOURNAL_COUNT)
10819                .await;
10820        let first_name = tier_delete_journal_object_name(entries.first().expect("slow rollback fixture should not be empty"));
10821        let last_name = tier_delete_journal_object_name(entries.last().expect("slow rollback fixture should not be empty"));
10822        let hook = TierDeleteDispatchRollbackTestHook::install_slow_delete(&first_name, &last_name);
10823        let worker_store = store.clone();
10824        let worker_manifest = manifest_name.clone();
10825        let worker =
10826            tokio::spawn(async move { recover_test_tier_delete_dispatch_manifest(worker_store, &worker_manifest).await });
10827        tokio::time::timeout(Duration::from_secs(30), hook.wait_until_delete_paused())
10828            .await
10829            .expect("the first rollback member should reach the slow hook");
10830        tokio::time::timeout(Duration::from_secs(30), hook.wait_until_delete_observed())
10831            .await
10832            .expect("a member beyond the first two concurrency windows should remain reachable");
10833        assert!(
10834            (2..=32).contains(&hook.max_in_flight()),
10835            "rollback delete concurrency must make progress without exceeding its bound; observed {}",
10836            hook.max_in_flight()
10837        );
10838        hook.release_delete();
10839        worker
10840            .await
10841            .expect("bounded rollback recovery task should join")
10842            .expect("bounded rollback recovery should converge after the slow member releases");
10843        drop(hook);
10844
10845        assert_eq!(tier_delete_journal_count(store.clone()).await, 0);
10846        assert_eq!(tier_delete_dispatch_manifest_count(store.clone()).await, 0);
10847        assert_eq!(backend.remove_count().await, 0, "rollback must not call the remote tier");
10848    }
10849
10850    #[cfg(feature = "test-util")]
10851    #[tokio::test]
10852    #[serial_test::serial(storage_class_env)]
10853    async fn aborted_dispatch_retry_defers_residual_scan_to_recovery() {
10854        const JOURNAL_COUNT: usize = 65;
10855
10856        let temp_dir = tempfile::tempdir().expect("create aborted retry store dir");
10857        let (ctx, store, shutdown) =
10858            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "aborted-dispatch-retry", &[4])).await;
10859        shutdown.cancel();
10860        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
10861        let bucket = "aborted-dispatch-retry-bucket";
10862        let prefix = "archive/";
10863        store
10864            .make_bucket(bucket, &MakeBucketOptions::default())
10865            .await
10866            .expect("aborted retry bucket should be created");
10867        let incarnation = store
10868            .bucket_incarnation_id(bucket)
10869            .await
10870            .expect("aborted retry bucket incarnation should resolve");
10871        let tier_name = "ABORTED-DISPATCH-RETRY";
10872        let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
10873        let identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
10874            .await
10875            .expect("aborted retry tier lease should resolve")
10876            .backend_identity();
10877        let aborted_entries = (0..JOURNAL_COUNT)
10878            .map(|index| {
10879                (
10880                    synthetic_v6_dispatch_entry(
10881                        bucket,
10882                        &format!("{prefix}{index:06}.bin"),
10883                        tier_name,
10884                        identity,
10885                        &uuid::Uuid::new_v4().to_string(),
10886                    ),
10887                    Some(TierDeleteJournalState::Prepared),
10888                )
10889            })
10890            .collect::<Vec<_>>();
10891        let entries = aborted_entries.iter().map(|(entry, _)| entry.clone()).collect::<Vec<_>>();
10892        let (manifest_name, _) = install_test_tier_delete_dispatch_fixture(
10893            store.clone(),
10894            bucket,
10895            incarnation,
10896            prefix,
10897            aborted_entries,
10898            TierDeleteDispatchManifestState::Aborted,
10899        )
10900        .await
10901        .expect("Aborted retry fixture should persist");
10902
10903        let lifecycle_guard = store
10904            .acquire_bucket_lifecycle_write_lock(bucket)
10905            .await
10906            .expect("aborted retry should acquire the bucket lifecycle fence");
10907        let mut fence_opts = ObjectOptions::default();
10908        fence_opts.add_bucket_lifecycle_lock_guard(&lifecycle_guard);
10909        let bucket_fence = fence_opts
10910            .bucket_lifecycle_lock_fence
10911            .clone()
10912            .expect("aborted retry should capture the bucket lifecycle fence");
10913        let fleet_proof = acquire_tier_delete_journal_fleet_proof().expect("aborted retry fixture should have a fleet proof");
10914        let hook = TierDeleteDispatchMemberReadTestHook::install_pause(TierDeleteDispatchMemberReadTestStage::Validation);
10915
10916        let error =
10917            match prepare_tier_delete_dispatch(store.clone(), bucket, incarnation, prefix, entries, fleet_proof, &bucket_fence)
10918                .await
10919            {
10920                Ok(_) => panic!("an Aborted manifest must be retained for bounded recovery cleanup"),
10921                Err(err) => err,
10922            };
10923
10924        assert!(
10925            error.to_string().contains("bounded recovery cleanup"),
10926            "unexpected aborted retry error: {error}"
10927        );
10928        assert_eq!(
10929            hook.entry_count(),
10930            0,
10931            "request retry must not scan the retained Aborted journal set under bucket write lock"
10932        );
10933        assert_eq!(
10934            test_tier_delete_dispatch_manifest_state(store.clone(), &manifest_name)
10935                .await
10936                .expect("retained Aborted manifest should remain readable"),
10937            Some(TierDeleteDispatchManifestState::Aborted)
10938        );
10939        assert_eq!(tier_delete_journal_count(store.clone()).await, JOURNAL_COUNT);
10940        assert_eq!(backend.remove_count().await, 0, "retry must not call the remote tier");
10941        drop(hook);
10942        drop(lifecycle_guard);
10943
10944        recover_test_tier_delete_dispatch_manifest(store.clone(), &manifest_name)
10945            .await
10946            .expect("bounded recovery should clean the retained Aborted manifest");
10947        assert_eq!(tier_delete_journal_count(store.clone()).await, 0);
10948        assert_eq!(tier_delete_dispatch_manifest_count(store).await, 0);
10949    }
10950
10951    #[cfg(feature = "test-util")]
10952    #[tokio::test]
10953    #[serial_test::serial(storage_class_env)]
10954    async fn dispatch_manifest_page_timeout_detaches_one_deduplicated_worker_until_convergence() {
10955        let temp_dir = tempfile::tempdir().expect("create detached rollback store dir");
10956        let (ctx, store, _shutdown) =
10957            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "detached-dispatch-rollback", &[4])).await;
10958        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
10959        let bucket = "detached-dispatch-rollback-bucket";
10960        store
10961            .make_bucket(bucket, &MakeBucketOptions::default())
10962            .await
10963            .expect("detached rollback bucket should be created");
10964        let incarnation = store
10965            .bucket_incarnation_id(bucket)
10966            .await
10967            .expect("detached rollback bucket incarnation should resolve");
10968        let tier_name = "DETACHED-DISPATCH-ROLLBACK";
10969        let _backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
10970        let identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
10971            .await
10972            .expect("detached rollback tier lease should resolve")
10973            .backend_identity();
10974        let (manifest_name, entries) =
10975            install_aborting_dispatch_fixture(store.clone(), bucket, incarnation, "detached/", tier_name, identity, 1).await;
10976        let journal_name = tier_delete_journal_object_name(&entries[0]);
10977        let hook = TierDeleteDispatchRollbackTestHook::install_slow_delete(&journal_name, &journal_name);
10978
10979        assert!(
10980            recover_test_tier_delete_dispatch_manifest_with_page_budget(
10981                store.clone(),
10982                &manifest_name,
10983                Duration::from_millis(10),
10984            )
10985            .await,
10986            "the synthetic page budget must elapse while the manifest worker continues"
10987        );
10988        tokio::time::timeout(Duration::from_secs(30), hook.wait_until_delete_paused())
10989            .await
10990            .expect("the detached manifest worker should continue into its delete phase");
10991        assert!(
10992            tier_delete_dispatch_manifest_recovery_inflight_for_test(&store, &manifest_name),
10993            "the detached worker must retain its per-store/object deduplication guard"
10994        );
10995        assert!(
10996            !recover_test_tier_delete_dispatch_manifest_with_page_budget(store.clone(), &manifest_name, Duration::from_secs(30),)
10997                .await,
10998            "a duplicate page observation should be retained without queuing another worker"
10999        );
11000
11001        hook.release_delete();
11002        tokio::time::timeout(Duration::from_secs(30), async {
11003            loop {
11004                let manifest_gone = matches!(com::read_config(store.clone(), &manifest_name).await, Err(Error::ConfigNotFound));
11005                if manifest_gone && !tier_delete_dispatch_manifest_recovery_inflight_for_test(&store, &manifest_name) {
11006                    break;
11007                }
11008                tokio::time::sleep(Duration::from_millis(10)).await;
11009            }
11010        })
11011        .await
11012        .expect("the detached manifest worker should release its guard after convergence");
11013        drop(hook);
11014        assert_eq!(tier_delete_journal_count(store.clone()).await, 0);
11015    }
11016
11017    #[cfg(feature = "test-util")]
11018    #[tokio::test]
11019    #[serial_test::serial(storage_class_env)]
11020    async fn dispatch_manifest_detached_workers_are_store_bounded_and_shutdown_cancellable() {
11021        const MANIFEST_COUNT: usize = 6;
11022        const WORKER_LIMIT: usize = 4;
11023
11024        let temp_dir = tempfile::tempdir().expect("create bounded detached rollback store dir");
11025        let (ctx, store, shutdown) =
11026            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "bounded-detached-dispatch-rollback", &[4]))
11027                .await;
11028        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
11029        let tier_name = "BOUNDED-DETACHED-DISPATCH-ROLLBACK";
11030        let _backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
11031        let identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
11032            .await
11033            .expect("bounded detached rollback tier lease should resolve")
11034            .backend_identity();
11035        let mut manifest_names = Vec::with_capacity(MANIFEST_COUNT);
11036        for index in 0..MANIFEST_COUNT {
11037            let bucket = format!("bounded-detached-dispatch-rollback-{index}");
11038            store
11039                .make_bucket(&bucket, &MakeBucketOptions::default())
11040                .await
11041                .expect("bounded detached rollback bucket should be created");
11042            let incarnation = store
11043                .bucket_incarnation_id(&bucket)
11044                .await
11045                .expect("bounded detached rollback bucket incarnation should resolve");
11046            let (manifest_name, _) = install_aborting_dispatch_fixture(
11047                store.clone(),
11048                &bucket,
11049                incarnation,
11050                &format!("detached-{index}/"),
11051                tier_name,
11052                identity,
11053                1,
11054            )
11055            .await;
11056            manifest_names.push(manifest_name);
11057        }
11058        let hook = TierDeleteDispatchRollbackTestHook::install_pause_all_deletes();
11059
11060        let first_pass = futures::stream::iter(manifest_names.iter().cloned().map(|manifest_name| {
11061            let store = store.clone();
11062            async move {
11063                recover_test_tier_delete_dispatch_manifest_with_page_budget(store, &manifest_name, Duration::from_millis(10))
11064                    .await
11065            }
11066        }))
11067        .buffer_unordered(MANIFEST_COUNT)
11068        .collect::<Vec<_>>()
11069        .await;
11070        tokio::time::timeout(Duration::from_secs(30), hook.wait_until_delete_pause_count(WORKER_LIMIT))
11071            .await
11072            .expect("every admitted detached worker should pause before its first mutation");
11073        assert_eq!(
11074            first_pass.into_iter().filter(|timed_out| *timed_out).count(),
11075            WORKER_LIMIT,
11076            "only the per-store worker budget may detach"
11077        );
11078        assert_eq!(
11079            tier_delete_dispatch_manifest_recovery_count_for_test(&store),
11080            WORKER_LIMIT,
11081            "detached workers must remain bounded per store"
11082        );
11083
11084        let repeated_pass = futures::stream::iter(manifest_names.iter().cloned().map(|manifest_name| {
11085            let store = store.clone();
11086            async move {
11087                recover_test_tier_delete_dispatch_manifest_with_page_budget(store, &manifest_name, Duration::from_secs(30)).await
11088            }
11089        }))
11090        .buffer_unordered(MANIFEST_COUNT)
11091        .collect::<Vec<_>>()
11092        .await;
11093        assert!(
11094            repeated_pass.into_iter().all(|timed_out| !timed_out),
11095            "duplicates and over-budget manifests must be retained without a waiter queue"
11096        );
11097        assert_eq!(tier_delete_dispatch_manifest_recovery_count_for_test(&store), WORKER_LIMIT);
11098
11099        shutdown.cancel();
11100        tokio::time::sleep(Duration::from_millis(50)).await;
11101        assert_eq!(
11102            tier_delete_dispatch_manifest_recovery_count_for_test(&store),
11103            WORKER_LIMIT,
11104            "shutdown must not drop a bounded batch before its admitted work drains"
11105        );
11106        assert_eq!(tier_delete_journal_count(store.clone()).await, MANIFEST_COUNT);
11107        assert_eq!(tier_delete_dispatch_manifest_count(store.clone()).await, MANIFEST_COUNT);
11108        hook.release_all_deletes();
11109        tokio::time::timeout(Duration::from_secs(30), async {
11110            while tier_delete_dispatch_manifest_recovery_count_for_test(&store) != 0 {
11111                tokio::time::sleep(Duration::from_millis(10)).await;
11112            }
11113        })
11114        .await
11115        .expect("shutdown should cancel every detached worker and release its registry permit");
11116        assert_eq!(tier_delete_journal_count(store.clone()).await, MANIFEST_COUNT);
11117        assert_eq!(tier_delete_dispatch_manifest_count(store.clone()).await, MANIFEST_COUNT);
11118        drop(hook);
11119        tokio::time::sleep(Duration::from_millis(50)).await;
11120        assert_eq!(
11121            tier_delete_journal_count(store.clone()).await,
11122            MANIFEST_COUNT,
11123            "releasing the old hook after shutdown must not resume a cancelled mutation"
11124        );
11125        assert_eq!(
11126            tier_delete_dispatch_manifest_recovery_count_for_test(&store),
11127            0,
11128            "the shutdown store must leave no registry permits behind"
11129        );
11130        drop(store);
11131        drop(ctx);
11132
11133        let (_restarted_ctx, restarted_store, restarted_shutdown) = without_storage_class_env(build_isolated_test_store(
11134            temp_dir.path(),
11135            "bounded-detached-dispatch-rollback-restart",
11136            &[4],
11137        ))
11138        .await;
11139        crate::bucket::metadata_sys::init_bucket_metadata_sys(restarted_store.clone(), Vec::new()).await;
11140        assert!(!restarted_shutdown.is_cancelled(), "the restarted store needs a fresh shutdown token");
11141        tokio::time::timeout(Duration::from_secs(30), async {
11142            loop {
11143                let active = tier_delete_dispatch_manifest_recovery_count_for_test(&restarted_store);
11144                let manifests = tier_delete_dispatch_manifest_count(restarted_store.clone()).await;
11145                if active > 0 || manifests < MANIFEST_COUNT {
11146                    break;
11147                }
11148                tokio::task::yield_now().await;
11149            }
11150        })
11151        .await
11152        .expect("the restarted store should start a fresh manifest recovery");
11153        tokio::time::timeout(Duration::from_secs(30), async {
11154            while tier_delete_dispatch_manifest_recovery_count_for_test(&restarted_store) != 0 {
11155                tokio::time::sleep(Duration::from_millis(10)).await;
11156            }
11157        })
11158        .await
11159        .expect("the restarted store recovery should release every worker permit");
11160        if tier_delete_dispatch_manifest_count(restarted_store.clone()).await != 0 {
11161            let stats = recover_tier_delete_dispatch_manifests(restarted_store.clone(), 100, None)
11162                .await
11163                .expect("the restarted store should retry any conservatively retained manifest");
11164            assert_eq!(stats.failed, 0, "the restarted recovery must not quarantine a valid manifest");
11165        }
11166        assert_eq!(tier_delete_journal_count(restarted_store.clone()).await, 0);
11167        assert_eq!(tier_delete_dispatch_manifest_count(restarted_store).await, 0);
11168        restarted_shutdown.cancel();
11169    }
11170
11171    #[cfg(feature = "test-util")]
11172    #[tokio::test]
11173    #[serial_test::serial(storage_class_env)]
11174    async fn dispatch_manifest_shutdown_stops_aborting_validation_tail_reads() {
11175        const JOURNAL_COUNT: usize = 65;
11176        const ADMITTED_READS: usize = 32;
11177
11178        let temp_dir = tempfile::tempdir().expect("create validation-read shutdown store dir");
11179        // The manual worker must own the seeded records and shutdown progress.
11180        let mut instance_ctx = crate::runtime::instance::InstanceContext::new();
11181        instance_ctx.suppress_tier_delete_journal_recovery_for_test();
11182        let (ctx, store, shutdown) = without_storage_class_env(build_isolated_test_store_with_layout(
11183            temp_dir.path(),
11184            "dispatch-validation-read-shutdown",
11185            &[(1, 4)],
11186            CancellationToken::new(),
11187            Some(Arc::new(instance_ctx)),
11188        ))
11189        .await;
11190        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
11191        let bucket = "dispatch-validation-read-shutdown-bucket";
11192        store
11193            .make_bucket(bucket, &MakeBucketOptions::default())
11194            .await
11195            .expect("validation-read shutdown bucket should be created");
11196        let incarnation = store
11197            .bucket_incarnation_id(bucket)
11198            .await
11199            .expect("validation-read shutdown bucket incarnation should resolve");
11200        let tier_name = "VALIDATION-READ-SHUTDOWN";
11201        let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
11202        let identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
11203            .await
11204            .expect("validation-read shutdown tier lease should resolve")
11205            .backend_identity();
11206        let (manifest_name, _) =
11207            install_aborting_dispatch_fixture(store.clone(), bucket, incarnation, "archive/", tier_name, identity, JOURNAL_COUNT)
11208                .await;
11209        let hook = TierDeleteDispatchMemberReadTestHook::install_pause(TierDeleteDispatchMemberReadTestStage::Validation);
11210        let worker_store = store.clone();
11211        let worker_manifest = manifest_name.clone();
11212        let worker = tokio::spawn(async move {
11213            recover_test_tier_delete_dispatch_manifest_with_page_budget(worker_store, &worker_manifest, Duration::from_secs(30))
11214                .await
11215        });
11216        tokio::time::timeout(Duration::from_secs(30), hook.wait_until_read_pause_count(ADMITTED_READS))
11217            .await
11218            .expect("the first validation read window should be admitted");
11219        assert_eq!(hook.entry_count(), ADMITTED_READS);
11220        assert_eq!(hook.in_flight(), ADMITTED_READS);
11221
11222        shutdown.cancel();
11223        tokio::time::sleep(Duration::from_millis(50)).await;
11224        assert_eq!(hook.entry_count(), ADMITTED_READS, "shutdown must not admit validation tail reads");
11225        assert_eq!(tier_delete_dispatch_manifest_recovery_count_for_test(&store), 1);
11226        assert!(tier_delete_dispatch_manifest_operation_lock_held_for_test(store.clone(), &manifest_name).await);
11227        assert!(crate::services::notification_sys::tier_delete_journal_fleet_proof_has_inflight_for_test());
11228        assert_eq!(tier_delete_journal_count(store.clone()).await, JOURNAL_COUNT);
11229        assert_eq!(backend.remove_count().await, 0);
11230
11231        hook.release_all_reads();
11232        assert!(
11233            !worker.await.expect("validation-read shutdown worker should join"),
11234            "cooperative shutdown should retain an unvalidated rollback"
11235        );
11236        assert_eq!(hook.entry_count(), ADMITTED_READS);
11237        assert_eq!(hook.in_flight(), 0);
11238        assert_eq!(tier_delete_dispatch_manifest_recovery_count_for_test(&store), 0);
11239        assert_eq!(
11240            tier_delete_journal_count(store.clone()).await,
11241            JOURNAL_COUNT,
11242            "no rollback DELETE may start"
11243        );
11244        assert_eq!(
11245            test_tier_delete_dispatch_manifest_state(store.clone(), &manifest_name)
11246                .await
11247                .expect("retained Aborting manifest should remain readable"),
11248            Some(TierDeleteDispatchManifestState::Aborting)
11249        );
11250        drop(hook);
11251        drop(store);
11252        drop(ctx);
11253
11254        let (_restarted_ctx, restarted_store, restarted_shutdown) = without_storage_class_env(build_isolated_test_store(
11255            temp_dir.path(),
11256            "dispatch-validation-read-shutdown-restart",
11257            &[4],
11258        ))
11259        .await;
11260        crate::bucket::metadata_sys::init_bucket_metadata_sys(restarted_store.clone(), Vec::new()).await;
11261        drive_tier_delete_dispatch_restart_to_convergence(restarted_store.clone()).await;
11262        restarted_shutdown.cancel();
11263    }
11264
11265    #[cfg(feature = "test-util")]
11266    #[tokio::test]
11267    #[serial_test::serial(storage_class_env)]
11268    async fn dispatch_manifest_shutdown_stops_authorized_tail_reads_before_receipt_or_cas() {
11269        const JOURNAL_COUNT: usize = 65;
11270        const ADMITTED_READS: usize = 32;
11271
11272        let temp_dir = tempfile::tempdir().expect("create authorized-read shutdown store dir");
11273        let (ctx, store, shutdown) =
11274            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "dispatch-authorized-read-shutdown", &[4]))
11275                .await;
11276        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
11277        let bucket = "dispatch-authorized-read-shutdown-bucket";
11278        store
11279            .make_bucket(bucket, &MakeBucketOptions::default())
11280            .await
11281            .expect("authorized-read shutdown bucket should be created");
11282        let incarnation = store
11283            .bucket_incarnation_id(bucket)
11284            .await
11285            .expect("authorized-read shutdown bucket incarnation should resolve");
11286        let tier_name = "AUTHORIZED-READ-SHUTDOWN";
11287        let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
11288        let identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
11289            .await
11290            .expect("authorized-read shutdown tier lease should resolve")
11291            .backend_identity();
11292        let entries = (0..JOURNAL_COUNT)
11293            .map(|index| {
11294                (
11295                    synthetic_v6_dispatch_entry(
11296                        bucket,
11297                        &format!("archive/{index:06}.bin"),
11298                        tier_name,
11299                        identity,
11300                        &uuid::Uuid::new_v4().to_string(),
11301                    ),
11302                    Some(TierDeleteJournalState::Committed),
11303                )
11304            })
11305            .collect();
11306        let (manifest_name, _) = install_test_tier_delete_dispatch_fixture(
11307            store.clone(),
11308            bucket,
11309            incarnation,
11310            "archive/",
11311            entries,
11312            TierDeleteDispatchManifestState::DispatchAuthorized,
11313        )
11314        .await
11315        .expect("authorized-read shutdown fixture should persist");
11316        let tier_config = ctx
11317            .tier_config_mgr()
11318            .read()
11319            .await
11320            .tiers
11321            .get(tier_name)
11322            .cloned()
11323            .expect("authorized-read tier config should remain available for restart");
11324        let hook = TierDeleteDispatchMemberReadTestHook::install_pause(TierDeleteDispatchMemberReadTestStage::Authorized);
11325        let worker_store = store.clone();
11326        let worker_manifest = manifest_name.clone();
11327        let worker = tokio::spawn(async move {
11328            recover_test_tier_delete_dispatch_manifest_with_page_budget(worker_store, &worker_manifest, Duration::from_secs(30))
11329                .await
11330        });
11331        tokio::time::timeout(Duration::from_secs(30), hook.wait_until_read_pause_count(ADMITTED_READS))
11332            .await
11333            .expect("the first authorized read window should be admitted");
11334
11335        shutdown.cancel();
11336        tokio::time::sleep(Duration::from_millis(50)).await;
11337        assert_eq!(hook.entry_count(), ADMITTED_READS, "shutdown must not admit authorized tail reads");
11338        assert_eq!(hook.in_flight(), ADMITTED_READS);
11339        assert_eq!(tier_delete_dispatch_manifest_recovery_count_for_test(&store), 1);
11340        assert!(tier_delete_dispatch_manifest_operation_lock_held_for_test(store.clone(), &manifest_name).await);
11341        assert!(crate::services::notification_sys::tier_delete_journal_fleet_proof_has_inflight_for_test());
11342
11343        hook.release_all_reads();
11344        assert!(
11345            !worker.await.expect("authorized-read shutdown worker should join"),
11346            "cooperative shutdown should retain an unscanned authorized manifest"
11347        );
11348        assert_eq!(hook.entry_count(), ADMITTED_READS);
11349        assert_eq!(hook.in_flight(), 0);
11350        assert_eq!(tier_delete_dispatch_manifest_recovery_count_for_test(&store), 0);
11351        assert_eq!(
11352            test_tier_delete_dispatch_manifest_state(store.clone(), &manifest_name)
11353                .await
11354                .expect("retained Authorized manifest should remain readable"),
11355            Some(TierDeleteDispatchManifestState::DispatchAuthorized),
11356            "cancellation before member reads must prevent the Completed CAS"
11357        );
11358        assert_eq!(
11359            hook.authorized_progress_count(),
11360            0,
11361            "cancellation before member reads must not attempt a decommission progress receipt"
11362        );
11363        assert_eq!(backend.remove_count().await, 0);
11364        drop(hook);
11365        drop(store);
11366        drop(ctx);
11367
11368        let (restarted_ctx, restarted_store, restarted_shutdown) = without_storage_class_env(build_isolated_test_store(
11369            temp_dir.path(),
11370            "dispatch-authorized-read-shutdown-restart",
11371            &[4],
11372        ))
11373        .await;
11374        {
11375            let tier_config_mgr = restarted_ctx.tier_config_mgr();
11376            let mut manager = tier_config_mgr.write().await;
11377            manager.tiers.insert(tier_name.to_string(), tier_config);
11378            manager
11379                .install_test_driver(tier_name, Box::new(backend.clone()))
11380                .expect("the exact authorized-read tier driver should reinstall after restart");
11381        }
11382        crate::bucket::metadata_sys::init_bucket_metadata_sys(restarted_store.clone(), Vec::new()).await;
11383        drive_tier_delete_dispatch_restart_to_convergence(restarted_store.clone()).await;
11384        assert_eq!(backend.remove_count().await, JOURNAL_COUNT);
11385        restarted_shutdown.cancel();
11386    }
11387
11388    #[cfg(feature = "test-util")]
11389    #[tokio::test]
11390    #[serial_test::serial(storage_class_env)]
11391    async fn dispatch_completion_cas_is_bounded_and_reaches_the_tail() {
11392        const JOURNAL_COUNT: usize = 65;
11393        const ADMITTED_COMMITS: usize = 32;
11394
11395        let temp_dir = tempfile::tempdir().expect("create dispatch completion concurrency store dir");
11396        let (ctx, store, shutdown) =
11397            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "dispatch-completion-concurrency", &[4])).await;
11398        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
11399        let bucket = "dispatch-completion-concurrency-bucket";
11400        let prefix = "archive/";
11401        store
11402            .make_bucket(bucket, &MakeBucketOptions::default())
11403            .await
11404            .expect("dispatch completion concurrency bucket should be created");
11405        let incarnation = store
11406            .bucket_incarnation_id(bucket)
11407            .await
11408            .expect("dispatch completion concurrency bucket incarnation should resolve");
11409        let tier_name = "DISPATCH-COMPLETION-CONCURRENCY";
11410        let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
11411        backend.set_remove_failure(true);
11412        let identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
11413            .await
11414            .expect("dispatch completion concurrency tier lease should resolve")
11415            .backend_identity();
11416        let entries = (0..JOURNAL_COUNT)
11417            .map(|index| {
11418                synthetic_v6_dispatch_entry(
11419                    bucket,
11420                    &format!("{prefix}{index:06}.bin"),
11421                    tier_name,
11422                    identity,
11423                    &uuid::Uuid::new_v4().to_string(),
11424                )
11425            })
11426            .collect::<Vec<_>>();
11427        let (manifest_name, _) = install_test_tier_delete_dispatch_fixture(
11428            store.clone(),
11429            bucket,
11430            incarnation,
11431            prefix,
11432            entries
11433                .iter()
11434                .cloned()
11435                .map(|entry| (entry, Some(TierDeleteJournalState::Dispatched)))
11436                .collect(),
11437            TierDeleteDispatchManifestState::DispatchAuthorized,
11438        )
11439        .await
11440        .expect("dispatch completion concurrency fixture should persist");
11441
11442        let lifecycle_guard = store
11443            .acquire_bucket_lifecycle_write_lock(bucket)
11444            .await
11445            .expect("dispatch completion should acquire the bucket lifecycle fence");
11446        let mut fence_opts = ObjectOptions::default();
11447        fence_opts.add_bucket_lifecycle_lock_guard(&lifecycle_guard);
11448        let bucket_fence = fence_opts
11449            .bucket_lifecycle_lock_fence
11450            .clone()
11451            .expect("dispatch completion should capture the bucket lifecycle fence");
11452        let fleet_proof =
11453            acquire_tier_delete_journal_fleet_proof().expect("dispatch completion concurrency fixture should have a fleet proof");
11454        let prepared =
11455            prepare_tier_delete_dispatch(store.clone(), bucket, incarnation, prefix, entries, fleet_proof, &bucket_fence)
11456                .await
11457                .expect("the existing Authorized dispatch should reload");
11458        let active = prepared
11459            .consume(bucket, incarnation, prefix)
11460            .expect("the existing Authorized dispatch permit should be consumable");
11461        active
11462            .authorization()
11463            .mark_mutation_started(bucket, incarnation, prefix)
11464            .expect("the completion test should mark its synthetic local mutation");
11465
11466        let hook = TierDeleteDispatchMemberReadTestHook::install_pause(TierDeleteDispatchMemberReadTestStage::Completion);
11467        let worker_store = store.clone();
11468        let worker = tokio::spawn(async move { complete_tier_delete_dispatch(worker_store, &active, &bucket_fence).await });
11469        tokio::time::timeout(Duration::from_secs(30), hook.wait_until_read_pause_count(ADMITTED_COMMITS))
11470            .await
11471            .expect("the first completion CAS window should be admitted");
11472        tokio::time::sleep(Duration::from_millis(50)).await;
11473        assert_eq!(hook.entry_count(), ADMITTED_COMMITS, "completion must not admit an unbounded CAS tail");
11474        assert_eq!(hook.in_flight(), ADMITTED_COMMITS);
11475
11476        hook.release_all_reads();
11477        worker
11478            .await
11479            .expect("dispatch completion concurrency worker should join")
11480            .expect("every bounded completion CAS should succeed");
11481        assert_eq!(
11482            hook.entry_count(),
11483            JOURNAL_COUNT,
11484            "completion must eventually admit the entire journal set"
11485        );
11486        assert_eq!(hook.in_flight(), 0);
11487        assert_eq!(
11488            test_tier_delete_dispatch_manifest_state(store.clone(), &manifest_name)
11489                .await
11490                .expect("completed concurrency manifest should remain readable"),
11491            Some(TierDeleteDispatchManifestState::Completed)
11492        );
11493        drop(hook);
11494        shutdown.cancel();
11495    }
11496
11497    #[cfg(feature = "test-util")]
11498    #[tokio::test]
11499    #[serial_test::serial(storage_class_env)]
11500    async fn dispatch_manifest_shutdown_stops_completed_missing_journal_tail_reads() {
11501        const JOURNAL_COUNT: usize = 65;
11502        const ADMITTED_READS: usize = 32;
11503
11504        let temp_dir = tempfile::tempdir().expect("create completed-read shutdown store dir");
11505        let (ctx, store, shutdown) =
11506            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "dispatch-completed-read-shutdown", &[4])).await;
11507        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
11508        let bucket = "dispatch-completed-read-shutdown-bucket";
11509        store
11510            .make_bucket(bucket, &MakeBucketOptions::default())
11511            .await
11512            .expect("completed-read shutdown bucket should be created");
11513        let incarnation = store
11514            .bucket_incarnation_id(bucket)
11515            .await
11516            .expect("completed-read shutdown bucket incarnation should resolve");
11517        let tier_name = "COMPLETED-READ-SHUTDOWN";
11518        let _backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
11519        let identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
11520            .await
11521            .expect("completed-read shutdown tier lease should resolve")
11522            .backend_identity();
11523        let entries = (0..JOURNAL_COUNT)
11524            .map(|index| {
11525                (
11526                    synthetic_v6_dispatch_entry(
11527                        bucket,
11528                        &format!("archive/{index:06}.bin"),
11529                        tier_name,
11530                        identity,
11531                        &uuid::Uuid::new_v4().to_string(),
11532                    ),
11533                    None,
11534                )
11535            })
11536            .collect();
11537        let (manifest_name, _) = install_test_tier_delete_dispatch_fixture(
11538            store.clone(),
11539            bucket,
11540            incarnation,
11541            "archive/",
11542            entries,
11543            TierDeleteDispatchManifestState::Completed,
11544        )
11545        .await
11546        .expect("completed-read shutdown fixture should persist");
11547        let hook = TierDeleteDispatchMemberReadTestHook::install_pause(TierDeleteDispatchMemberReadTestStage::Completed);
11548        let worker_store = store.clone();
11549        let worker_manifest = manifest_name.clone();
11550        let worker = tokio::spawn(async move {
11551            recover_test_tier_delete_dispatch_manifest_with_page_budget(worker_store, &worker_manifest, Duration::from_secs(30))
11552                .await
11553        });
11554        tokio::time::timeout(Duration::from_secs(30), hook.wait_until_read_pause_count(ADMITTED_READS))
11555            .await
11556            .expect("the first Completed read window should be admitted");
11557
11558        shutdown.cancel();
11559        tokio::time::sleep(Duration::from_millis(50)).await;
11560        assert_eq!(hook.entry_count(), ADMITTED_READS, "shutdown must not admit later Completed chunks");
11561        assert_eq!(hook.in_flight(), ADMITTED_READS);
11562        assert_eq!(tier_delete_dispatch_manifest_recovery_count_for_test(&store), 1);
11563        assert!(tier_delete_dispatch_manifest_operation_lock_held_for_test(store.clone(), &manifest_name).await);
11564        assert!(crate::services::notification_sys::tier_delete_journal_fleet_proof_has_inflight_for_test());
11565
11566        hook.release_all_reads();
11567        assert!(
11568            !worker.await.expect("completed-read shutdown worker should join"),
11569            "cooperative shutdown should retain a partially scanned Completed manifest"
11570        );
11571        assert_eq!(hook.entry_count(), ADMITTED_READS);
11572        assert_eq!(hook.in_flight(), 0);
11573        assert_eq!(tier_delete_dispatch_manifest_recovery_count_for_test(&store), 0);
11574        assert_eq!(tier_delete_journal_count(store.clone()).await, 0);
11575        assert_eq!(
11576            tier_delete_dispatch_manifest_count(store.clone()).await,
11577            1,
11578            "manifest DELETE must not run"
11579        );
11580        assert_eq!(
11581            test_tier_delete_dispatch_manifest_state(store.clone(), &manifest_name)
11582                .await
11583                .expect("retained Completed manifest should remain readable"),
11584            Some(TierDeleteDispatchManifestState::Completed)
11585        );
11586        drop(hook);
11587        drop(store);
11588        drop(ctx);
11589
11590        let (_restarted_ctx, restarted_store, restarted_shutdown) = without_storage_class_env(build_isolated_test_store(
11591            temp_dir.path(),
11592            "dispatch-completed-read-shutdown-restart",
11593            &[4],
11594        ))
11595        .await;
11596        crate::bucket::metadata_sys::init_bucket_metadata_sys(restarted_store.clone(), Vec::new()).await;
11597        drive_tier_delete_dispatch_restart_to_convergence(restarted_store.clone()).await;
11598        restarted_shutdown.cancel();
11599    }
11600
11601    #[cfg(feature = "test-util")]
11602    #[tokio::test]
11603    #[serial_test::serial(storage_class_env)]
11604    async fn dispatch_manifest_shutdown_drains_preparing_cas_before_releasing_fences() {
11605        let temp_dir = tempfile::tempdir().expect("create preparing CAS shutdown store dir");
11606        let (ctx, store, shutdown) =
11607            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "dispatch-preparing-cas-shutdown", &[4])).await;
11608        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
11609        let bucket = "dispatch-preparing-cas-shutdown-bucket";
11610        store
11611            .make_bucket(bucket, &MakeBucketOptions::default())
11612            .await
11613            .expect("preparing CAS shutdown bucket should be created");
11614        let incarnation = store
11615            .bucket_incarnation_id(bucket)
11616            .await
11617            .expect("preparing CAS shutdown bucket incarnation should resolve");
11618        let entry = synthetic_v6_dispatch_entry(
11619            bucket,
11620            "archive/preparing-cas.bin",
11621            "PREPARING-CAS-SHUTDOWN",
11622            [31; 32],
11623            &uuid::Uuid::new_v4().to_string(),
11624        );
11625        let (manifest_name, _) = install_test_tier_delete_dispatch_fixture(
11626            store.clone(),
11627            bucket,
11628            incarnation,
11629            "archive/",
11630            vec![(entry, Some(TierDeleteJournalState::Prepared))],
11631            TierDeleteDispatchManifestState::Preparing,
11632        )
11633        .await
11634        .expect("preparing CAS shutdown fixture should persist");
11635        let barrier = crate::set_disk::PutObjectCommitBarrier::install(
11636            RUSTFS_META_BUCKET,
11637            &manifest_name,
11638            crate::set_disk::PutObjectCommitPause::BeforeQuotaRename,
11639        );
11640        let worker_store = store.clone();
11641        let worker_manifest = manifest_name.clone();
11642        let worker = tokio::spawn(async move {
11643            recover_test_tier_delete_dispatch_manifest_with_page_budget(worker_store, &worker_manifest, Duration::from_secs(30))
11644                .await
11645        });
11646        barrier.wait_until_paused().await;
11647
11648        shutdown.cancel();
11649        tokio::time::sleep(Duration::from_millis(50)).await;
11650        assert_eq!(
11651            tier_delete_dispatch_manifest_recovery_count_for_test(&store),
11652            1,
11653            "a paused CAS must retain its registry permit after shutdown"
11654        );
11655        assert!(
11656            tier_delete_dispatch_manifest_operation_lock_held_for_test(store.clone(), &manifest_name).await,
11657            "a paused CAS must retain its manifest operation lock after shutdown"
11658        );
11659        assert!(
11660            crate::services::notification_sys::tier_delete_journal_fleet_proof_has_inflight_for_test(),
11661            "a paused CAS must retain its fleet-proof generation permit"
11662        );
11663
11664        barrier.release();
11665        assert!(
11666            !worker.await.expect("preparing CAS shutdown worker should join"),
11667            "cooperative shutdown should retain the partially advanced manifest"
11668        );
11669        assert_eq!(tier_delete_dispatch_manifest_recovery_count_for_test(&store), 0);
11670        assert!(
11671            !tier_delete_dispatch_manifest_operation_lock_held_for_test(store.clone(), &manifest_name).await,
11672            "the operation lock may release only after the admitted CAS drains"
11673        );
11674        assert!(
11675            com::read_config(store.clone(), &manifest_name).await.is_ok(),
11676            "shutdown must retain the durable manifest for restart replay"
11677        );
11678        drop(barrier);
11679        drop(store);
11680        drop(ctx);
11681
11682        let (_restarted_ctx, restarted_store, restarted_shutdown) = without_storage_class_env(build_isolated_test_store(
11683            temp_dir.path(),
11684            "dispatch-preparing-cas-shutdown-restart",
11685            &[4],
11686        ))
11687        .await;
11688        crate::bucket::metadata_sys::init_bucket_metadata_sys(restarted_store.clone(), Vec::new()).await;
11689        tokio::time::timeout(Duration::from_secs(30), async {
11690            let mut stable_empty_observations = 0;
11691            loop {
11692                if tier_delete_journal_count(restarted_store.clone()).await == 0
11693                    && tier_delete_dispatch_manifest_count(restarted_store.clone()).await == 0
11694                    && tier_delete_dispatch_manifest_recovery_count_for_test(&restarted_store) == 0
11695                {
11696                    stable_empty_observations += 1;
11697                    if stable_empty_observations == 10 {
11698                        break;
11699                    }
11700                } else {
11701                    stable_empty_observations = 0;
11702                }
11703                tokio::time::sleep(Duration::from_millis(50)).await;
11704            }
11705        })
11706        .await
11707        .expect("the restarted background recovery should replay the drained Preparing CAS");
11708        assert_eq!(tier_delete_journal_count(restarted_store.clone()).await, 0);
11709        assert_eq!(tier_delete_dispatch_manifest_count(restarted_store).await, 0);
11710        restarted_shutdown.cancel();
11711    }
11712
11713    #[cfg(feature = "test-util")]
11714    #[tokio::test]
11715    #[serial_test::serial(storage_class_env)]
11716    async fn dispatch_manifest_shutdown_drains_admitted_delete_batch_without_tail_admission() {
11717        const JOURNAL_COUNT: usize = 65;
11718        const ADMITTED_BATCH: usize = 32;
11719
11720        let temp_dir = tempfile::tempdir().expect("create delete-batch shutdown store dir");
11721        // The manual worker must own the seeded records and shutdown progress.
11722        let mut instance_ctx = crate::runtime::instance::InstanceContext::new();
11723        instance_ctx.suppress_tier_delete_journal_recovery_for_test();
11724        let (ctx, store, shutdown) = without_storage_class_env(build_isolated_test_store_with_layout(
11725            temp_dir.path(),
11726            "dispatch-delete-batch-shutdown",
11727            &[(1, 4)],
11728            CancellationToken::new(),
11729            Some(Arc::new(instance_ctx)),
11730        ))
11731        .await;
11732        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
11733        let bucket = "dispatch-delete-batch-shutdown-bucket";
11734        store
11735            .make_bucket(bucket, &MakeBucketOptions::default())
11736            .await
11737            .expect("delete-batch shutdown bucket should be created");
11738        let incarnation = store
11739            .bucket_incarnation_id(bucket)
11740            .await
11741            .expect("delete-batch shutdown bucket incarnation should resolve");
11742        let (manifest_name, entries) = install_aborting_dispatch_fixture(
11743            store.clone(),
11744            bucket,
11745            incarnation,
11746            "archive/",
11747            "DELETE-BATCH-SHUTDOWN",
11748            [32; 32],
11749            JOURNAL_COUNT,
11750        )
11751        .await;
11752        let mut journal_names = entries.iter().map(tier_delete_journal_object_name).collect::<Vec<_>>();
11753        journal_names.sort();
11754        let published_delete_name = journal_names[0].clone();
11755        let hook = TierDeleteDispatchRollbackTestHook::install_pause_all_except_delete(&published_delete_name);
11756        let barrier =
11757            crate::set_disk::DeleteObjectCommitBarrier::install_after_publish(RUSTFS_META_BUCKET, &published_delete_name);
11758        let worker_store = store.clone();
11759        let worker_manifest = manifest_name.clone();
11760        let worker = tokio::spawn(async move {
11761            recover_test_tier_delete_dispatch_manifest_with_page_budget(worker_store, &worker_manifest, Duration::from_secs(30))
11762                .await
11763        });
11764        barrier.wait_until_paused().await;
11765        tokio::time::timeout(Duration::from_secs(30), hook.wait_until_delete_pause_count(ADMITTED_BATCH - 1))
11766            .await
11767            .expect("the rest of the bounded delete batch should pause before mutation");
11768        assert_eq!(hook.delete_entry_count(), ADMITTED_BATCH);
11769
11770        shutdown.cancel();
11771        tokio::time::sleep(Duration::from_millis(50)).await;
11772        assert_eq!(tier_delete_dispatch_manifest_recovery_count_for_test(&store), 1);
11773        assert!(
11774            tier_delete_dispatch_manifest_operation_lock_held_for_test(store.clone(), &manifest_name).await,
11775            "the partially published DELETE must retain its operation lock"
11776        );
11777        assert!(
11778            crate::services::notification_sys::tier_delete_journal_fleet_proof_has_inflight_for_test(),
11779            "the partially published DELETE must retain its fleet-proof permit"
11780        );
11781        assert_eq!(
11782            hook.delete_entry_count(),
11783            ADMITTED_BATCH,
11784            "shutdown must stop admitting the tail behind the bounded batch"
11785        );
11786
11787        barrier.release();
11788        tokio::time::sleep(Duration::from_millis(50)).await;
11789        assert_eq!(
11790            tier_delete_dispatch_manifest_recovery_count_for_test(&store),
11791            1,
11792            "the worker must retain its guards until every admitted sibling drains"
11793        );
11794        hook.release_all_deletes();
11795        assert!(
11796            !worker.await.expect("delete-batch shutdown worker should join"),
11797            "cooperative shutdown should retain the partial rollback for restart"
11798        );
11799        assert_eq!(hook.delete_entry_count(), ADMITTED_BATCH);
11800        assert_eq!(tier_delete_dispatch_manifest_recovery_count_for_test(&store), 0);
11801        assert!(
11802            tier_delete_journal_count(store.clone()).await >= JOURNAL_COUNT - 1,
11803            "only the already-published DELETE may commit before restart"
11804        );
11805        drop(barrier);
11806        drop(hook);
11807        drop(store);
11808        drop(ctx);
11809
11810        let (_restarted_ctx, restarted_store, restarted_shutdown) =
11811            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "dispatch-delete-batch-shutdown-restart", &[4]))
11812                .await;
11813        crate::bucket::metadata_sys::init_bucket_metadata_sys(restarted_store.clone(), Vec::new()).await;
11814        tokio::time::timeout(Duration::from_secs(30), async {
11815            let mut stable_empty_observations = 0;
11816            loop {
11817                if tier_delete_journal_count(restarted_store.clone()).await == 0
11818                    && tier_delete_dispatch_manifest_count(restarted_store.clone()).await == 0
11819                    && tier_delete_dispatch_manifest_recovery_count_for_test(&restarted_store) == 0
11820                {
11821                    stable_empty_observations += 1;
11822                    if stable_empty_observations == 10 {
11823                        break;
11824                    }
11825                } else {
11826                    stable_empty_observations = 0;
11827                }
11828                tokio::time::sleep(Duration::from_millis(50)).await;
11829            }
11830        })
11831        .await
11832        .expect("the restarted background recovery should replay the partially drained delete batch");
11833        assert_eq!(tier_delete_journal_count(restarted_store.clone()).await, 0);
11834        assert_eq!(tier_delete_dispatch_manifest_count(restarted_store).await, 0);
11835        restarted_shutdown.cancel();
11836    }
11837
11838    #[cfg(feature = "test-util")]
11839    #[tokio::test]
11840    #[serial_test::serial(storage_class_env)]
11841    async fn dispatch_manifest_rollback_retries_delete_and_confirmation_failures() {
11842        const JOURNAL_COUNT: usize = 40;
11843
11844        let temp_dir = tempfile::tempdir().expect("create rollback retry store dir");
11845        // Manual retries must own progress between fault removal and the next attempt.
11846        let mut instance_ctx = crate::runtime::instance::InstanceContext::new();
11847        instance_ctx.suppress_tier_delete_journal_recovery_for_test();
11848        let (ctx, store, shutdown) = without_storage_class_env(build_isolated_test_store_with_layout(
11849            temp_dir.path(),
11850            "dispatch-rollback-retry",
11851            &[(1, 4)],
11852            CancellationToken::new(),
11853            Some(Arc::new(instance_ctx)),
11854        ))
11855        .await;
11856        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
11857        let bucket = "dispatch-rollback-retry-bucket";
11858        store
11859            .make_bucket(bucket, &MakeBucketOptions::default())
11860            .await
11861            .expect("rollback retry bucket should be created");
11862        let incarnation = store
11863            .bucket_incarnation_id(bucket)
11864            .await
11865            .expect("rollback retry bucket incarnation should resolve");
11866        let tier_name = "DISPATCH-ROLLBACK-RETRY";
11867        let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
11868        let identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
11869            .await
11870            .expect("rollback retry tier lease should resolve")
11871            .backend_identity();
11872
11873        let (delete_manifest, delete_entries) = install_aborting_dispatch_fixture(
11874            store.clone(),
11875            bucket,
11876            incarnation,
11877            "delete-failure/",
11878            tier_name,
11879            identity,
11880            JOURNAL_COUNT,
11881        )
11882        .await;
11883        let delete_failure_name = tier_delete_journal_object_name(&delete_entries[10]);
11884        let delete_hook = TierDeleteDispatchRollbackTestHook::install_delete_failure(&delete_failure_name);
11885        let delete_error = recover_test_tier_delete_dispatch_manifest(store.clone(), &delete_manifest)
11886            .await
11887            .expect_err("an injected member delete failure must retain the manifest")
11888            .to_string();
11889        assert!(delete_error.contains("injected tier delete dispatch rollback delete failure"));
11890        assert!(
11891            tier_delete_journal_count(store.clone()).await > 0,
11892            "the failed member and all work not admitted after the first error must remain retryable"
11893        );
11894        drop(delete_hook);
11895        recover_test_tier_delete_dispatch_manifest(store.clone(), &delete_manifest)
11896            .await
11897            .expect("the delete-stage retry should converge");
11898
11899        let (confirmation_manifest, confirmation_entries) = install_aborting_dispatch_fixture(
11900            store.clone(),
11901            bucket,
11902            incarnation,
11903            "confirmation-failure/",
11904            tier_name,
11905            identity,
11906            JOURNAL_COUNT,
11907        )
11908        .await;
11909        let confirmation_failure_name = tier_delete_journal_object_name(&confirmation_entries[10]);
11910        let confirmation_hook = TierDeleteDispatchRollbackTestHook::install_confirmation_failure(&confirmation_failure_name);
11911        let confirmation_error = recover_test_tier_delete_dispatch_manifest(store.clone(), &confirmation_manifest)
11912            .await
11913            .expect_err("an injected confirmation failure must retain the manifest")
11914            .to_string();
11915        assert!(confirmation_error.contains("injected tier delete dispatch rollback confirmation failure"));
11916        assert_eq!(
11917            tier_delete_journal_count(store.clone()).await,
11918            0,
11919            "confirmation failure occurs only after the bounded delete phase drains"
11920        );
11921        assert_eq!(tier_delete_dispatch_manifest_count(store.clone()).await, 1);
11922        drop(confirmation_hook);
11923        recover_test_tier_delete_dispatch_manifest(store.clone(), &confirmation_manifest)
11924            .await
11925            .expect("the confirmation-stage retry should converge over missing members");
11926
11927        assert_eq!(tier_delete_dispatch_manifest_count(store.clone()).await, 0);
11928        assert_eq!(backend.remove_count().await, 0, "rollback retries must never call the remote tier");
11929        shutdown.cancel();
11930    }
11931
11932    #[cfg(feature = "test-util")]
11933    #[test]
11934    #[serial_test::serial(storage_class_env)]
11935    fn v6_decommission_checkpoint_no_lock_put_rejects_lost_publication_fence() {
11936        run_large_stack_async_test(
11937            "v6-checkpoint-fence-loss",
11938            v6_decommission_checkpoint_no_lock_put_rejects_lost_publication_fence_case,
11939        );
11940    }
11941
11942    #[cfg(feature = "test-util")]
11943    async fn v6_decommission_checkpoint_no_lock_put_rejects_lost_publication_fence_case() {
11944        let temp_dir = tempfile::tempdir().expect("create v6 checkpoint fence-loss store dir");
11945        let (_ctx, store, _shutdown) =
11946            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "v6-checkpoint-fence-loss", &[4, 4])).await;
11947        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
11948        let bucket = "v6-checkpoint-fence-loss-bucket";
11949        store
11950            .make_bucket(bucket, &MakeBucketOptions::default())
11951            .await
11952            .expect("checkpoint fence-loss bucket should be created");
11953        let incarnation = store
11954            .bucket_incarnation_id(bucket)
11955            .await
11956            .expect("checkpoint fence-loss bucket incarnation should resolve");
11957        let entry = synthetic_v6_dispatch_entry(
11958            bucket,
11959            "archive/fence-loss.bin",
11960            "FENCE-LOSS-TIER",
11961            [18; 32],
11962            &uuid::Uuid::new_v4().to_string(),
11963        );
11964        let (manifest_name, entries) = install_test_tier_delete_dispatch_fixture(
11965            store.clone(),
11966            bucket,
11967            incarnation,
11968            "archive/",
11969            vec![(entry, Some(TierDeleteJournalState::Prepared))],
11970            TierDeleteDispatchManifestState::Preparing,
11971        )
11972        .await
11973        .expect("checkpoint fence-loss fixture should persist");
11974        let journal_name = tier_delete_journal_object_name(&entries[0]);
11975        let manifest_data = com::read_config(store.clone(), &manifest_name)
11976            .await
11977            .expect("checkpoint fence-loss manifest should be readable");
11978        let journal_data = com::read_config(store.clone(), &journal_name)
11979            .await
11980            .expect("checkpoint fence-loss journal should be readable");
11981        for pool in &store.pools {
11982            com::save_config(pool.clone(), &manifest_name, manifest_data.clone())
11983                .await
11984                .expect("manifest copy should persist in each checkpoint fence-loss pool");
11985            com::save_config(pool.clone(), &journal_name, journal_data.clone())
11986                .await
11987                .expect("journal copy should persist in each checkpoint fence-loss pool");
11988        }
11989        mark_test_pool_decommissioning(&store, 0).await;
11990        for (path, data) in [
11991            (&manifest_name, manifest_data.as_slice()),
11992            (&journal_name, journal_data.as_slice()),
11993        ] {
11994            let record = validate_durable_ilm_record(path, data).expect("checkpoint fence-loss v6 record should validate");
11995            store
11996                .persist_decommission_durable_ilm_receipt_for_test(0, 1, path, &record, false)
11997                .await
11998                .expect("checkpoint fence-loss receipt should persist");
11999        }
12000
12001        let loss_hook = crate::store::object::DecommissionMutationFenceLossHook::install(
12002            RUSTFS_META_BUCKET,
12003            &manifest_name,
12004            crate::store::object::DecommissionMutationFenceTestPhase::Migration,
12005        );
12006        let barrier = crate::set_disk::PutObjectCommitBarrier::install(
12007            RUSTFS_META_BUCKET,
12008            &manifest_name,
12009            crate::set_disk::PutObjectCommitPause::BeforeQuotaRename,
12010        );
12011        let worker_store = store.clone();
12012        let worker_manifest = manifest_name.clone();
12013        let worker =
12014            tokio::spawn(async move { recover_test_tier_delete_dispatch_manifest(worker_store, &worker_manifest).await });
12015        barrier.wait_until_paused().await;
12016        loss_hook.mark_lost();
12017        barrier.release();
12018        drop(barrier);
12019        let error = worker
12020            .await
12021            .expect("checkpoint fence-loss recovery task should join")
12022            .expect_err("lost publication fence must reject the no-lock checkpoint PUT")
12023            .to_string();
12024        assert!(
12025            error.contains("lock") || error.contains("fence"),
12026            "unexpected checkpoint fence-loss error: {error}"
12027        );
12028        assert_eq!(
12029            com::read_config(store.pools[1].clone(), &manifest_name)
12030                .await
12031                .expect("fenced target manifest should remain readable"),
12032            manifest_data,
12033            "the target checkpoint must not commit after its publication fence is lost"
12034        );
12035
12036        drop(loss_hook);
12037        recover_test_tier_delete_dispatch_manifest(store.clone(), &manifest_name)
12038            .await
12039            .expect("checkpoint recovery should converge after a fresh fence is acquired");
12040    }
12041
12042    #[cfg(feature = "test-util")]
12043    #[tokio::test]
12044    #[serial_test::serial(storage_class_env)]
12045    async fn v6_decommission_checkpoint_partial_target_failure_retries_without_advancing_receipts() {
12046        let temp_dir = tempfile::tempdir().expect("create v6 partial checkpoint store dir");
12047        let (_ctx, store, _shutdown) =
12048            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "v6-partial-checkpoint", &[4, 4, 4])).await;
12049        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
12050        let bucket = "v6-partial-checkpoint-bucket";
12051        store
12052            .make_bucket(bucket, &MakeBucketOptions::default())
12053            .await
12054            .expect("partial checkpoint bucket should be created");
12055        let incarnation = store
12056            .bucket_incarnation_id(bucket)
12057            .await
12058            .expect("partial checkpoint bucket incarnation should resolve");
12059        let entry = synthetic_v6_dispatch_entry(
12060            bucket,
12061            "archive/partial.bin",
12062            "PARTIAL-CHECKPOINT-TIER",
12063            [19; 32],
12064            &uuid::Uuid::new_v4().to_string(),
12065        );
12066        let (manifest_name, entries) = install_test_tier_delete_dispatch_fixture(
12067            store.clone(),
12068            bucket,
12069            incarnation,
12070            "archive/",
12071            vec![(entry, Some(TierDeleteJournalState::Prepared))],
12072            TierDeleteDispatchManifestState::Preparing,
12073        )
12074        .await
12075        .expect("partial checkpoint fixture should persist");
12076        let journal_name = tier_delete_journal_object_name(&entries[0]);
12077        let manifest_data = com::read_config(store.clone(), &manifest_name)
12078            .await
12079            .expect("partial checkpoint manifest should be readable");
12080        let journal_data = com::read_config(store.clone(), &journal_name)
12081            .await
12082            .expect("partial checkpoint journal should be readable");
12083        for pool in &store.pools {
12084            com::save_config(pool.clone(), &manifest_name, manifest_data.clone())
12085                .await
12086                .expect("manifest copy should persist in each partial checkpoint pool");
12087            com::save_config(pool.clone(), &journal_name, journal_data.clone())
12088                .await
12089                .expect("journal copy should persist in each partial checkpoint pool");
12090        }
12091        mark_test_pool_decommissioning_with_split_targets(&store, 0).await;
12092        for target_pool_index in [1, 2] {
12093            for (path, data) in [
12094                (&manifest_name, manifest_data.as_slice()),
12095                (&journal_name, journal_data.as_slice()),
12096            ] {
12097                let record = validate_durable_ilm_record(path, data).expect("partial checkpoint v6 record should validate");
12098                store
12099                    .persist_decommission_durable_ilm_receipt_for_test(0, target_pool_index, path, &record, false)
12100                    .await
12101                    .expect("partial checkpoint receipt should persist on each target");
12102            }
12103        }
12104        let (aborting_data, preparing_etag) = test_tier_delete_dispatch_manifest_checkpoint(
12105            store.clone(),
12106            &manifest_name,
12107            TierDeleteDispatchManifestState::Aborting,
12108        )
12109        .await
12110        .expect("partial checkpoint Aborting generation should encode");
12111        let targets = store
12112            .decommission_durable_ilm_checkpoint_targets(&manifest_name, &aborting_data, &preparing_etag)
12113            .await
12114            .expect("partial checkpoint targets should resolve")
12115            .expect("the active decommission should own the partial checkpoint");
12116        assert_eq!(
12117            targets.iter().map(|target| target.target_pool_index).collect::<Vec<_>>(),
12118            vec![1, 2],
12119            "the capacity reservation must exercise two independently confirmed targets"
12120        );
12121
12122        let failure_hook = DecommissionCheckpointTargetFailureHook::install(2);
12123        let error = recover_test_tier_delete_dispatch_manifest(store.clone(), &manifest_name)
12124            .await
12125            .expect_err("the injected second-target failure must reject the checkpoint");
12126        assert!(
12127            error.to_string().contains("injected decommission checkpoint target failure"),
12128            "unexpected partial checkpoint error: {error}"
12129        );
12130        let mut diagnostic = &error as &(dyn std::error::Error + 'static);
12131        let mut target_context_found = false;
12132        while let Some(source) = diagnostic.source() {
12133            if source.to_string().contains("target pool 2") {
12134                target_context_found = true;
12135                break;
12136            }
12137            diagnostic = source;
12138        }
12139        assert!(
12140            target_context_found,
12141            "the stable checkpoint error must retain the failed target in its diagnostic source chain: {error:?}"
12142        );
12143        assert_eq!(
12144            com::read_config(store.pools[1].clone(), &manifest_name)
12145                .await
12146                .expect("first target checkpoint should be readable"),
12147            aborting_data,
12148            "the first target should model a committed checkpoint before the later failure"
12149        );
12150        assert_eq!(
12151            com::read_config(store.pools[2].clone(), &manifest_name)
12152                .await
12153                .expect("second target checkpoint should be readable"),
12154            manifest_data,
12155            "the failed second target must retain the prior generation"
12156        );
12157        let retry_targets = store
12158            .decommission_durable_ilm_checkpoint_targets(&manifest_name, &aborting_data, &preparing_etag)
12159            .await
12160            .expect("unadvanced receipts must still authorize the exact partial checkpoint retry")
12161            .expect("the partial checkpoint retry should remain decommission-owned");
12162        assert!(retry_targets[0].already_committed);
12163        assert!(!retry_targets[1].already_committed);
12164
12165        drop(failure_hook);
12166        recover_test_tier_delete_dispatch_manifest(store.clone(), &manifest_name)
12167            .await
12168            .expect("the partial checkpoint should converge after the target recovers");
12169    }
12170
12171    #[cfg(feature = "test-util")]
12172    #[tokio::test]
12173    #[serial_test::serial(storage_class_env)]
12174    async fn v6_decommission_checkpoint_shutdown_drains_target_put_and_replays_capacity() {
12175        let temp_dir = tempfile::tempdir().expect("create checkpoint shutdown store dir");
12176        let (ctx, store, shutdown) =
12177            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "v6-checkpoint-shutdown", &[4, 4])).await;
12178        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
12179        let bucket = "v6-checkpoint-shutdown-bucket";
12180        store
12181            .make_bucket(bucket, &MakeBucketOptions::default())
12182            .await
12183            .expect("checkpoint shutdown bucket should be created");
12184        let incarnation = store
12185            .bucket_incarnation_id(bucket)
12186            .await
12187            .expect("checkpoint shutdown bucket incarnation should resolve");
12188        let entry = synthetic_v6_dispatch_entry(
12189            bucket,
12190            "archive/checkpoint-shutdown.bin",
12191            "CHECKPOINT-SHUTDOWN",
12192            [33; 32],
12193            &uuid::Uuid::new_v4().to_string(),
12194        );
12195        let (manifest_name, entries) = install_test_tier_delete_dispatch_fixture(
12196            store.clone(),
12197            bucket,
12198            incarnation,
12199            "archive/",
12200            vec![(entry, Some(TierDeleteJournalState::Prepared))],
12201            TierDeleteDispatchManifestState::Preparing,
12202        )
12203        .await
12204        .expect("checkpoint shutdown fixture should persist");
12205        let journal_name = tier_delete_journal_object_name(&entries[0]);
12206        let manifest_data = com::read_config(store.clone(), &manifest_name)
12207            .await
12208            .expect("checkpoint shutdown manifest should be readable");
12209        let journal_data = com::read_config(store.clone(), &journal_name)
12210            .await
12211            .expect("checkpoint shutdown journal should be readable");
12212        for pool in &store.pools {
12213            com::save_config(pool.clone(), &manifest_name, manifest_data.clone())
12214                .await
12215                .expect("manifest copy should persist in each checkpoint shutdown pool");
12216            com::save_config(pool.clone(), &journal_name, journal_data.clone())
12217                .await
12218                .expect("journal copy should persist in each checkpoint shutdown pool");
12219        }
12220        mark_test_pool_decommissioning(&store, 0).await;
12221        for (path, data) in [
12222            (&manifest_name, manifest_data.as_slice()),
12223            (&journal_name, journal_data.as_slice()),
12224        ] {
12225            let record = validate_durable_ilm_record(path, data).expect("checkpoint shutdown v6 record should validate");
12226            store
12227                .persist_decommission_durable_ilm_receipt_for_test(0, 1, path, &record, false)
12228                .await
12229                .expect("checkpoint shutdown receipt should persist");
12230        }
12231        let (aborting_data, preparing_etag) = test_tier_delete_dispatch_manifest_checkpoint(
12232            store.clone(),
12233            &manifest_name,
12234            TierDeleteDispatchManifestState::Aborting,
12235        )
12236        .await
12237        .expect("checkpoint shutdown Aborting generation should encode");
12238        let targets = store
12239            .decommission_durable_ilm_checkpoint_targets(&manifest_name, &aborting_data, &preparing_etag)
12240            .await
12241            .expect("checkpoint shutdown target should resolve")
12242            .expect("the active decommission should own the checkpoint target");
12243        assert_eq!(targets.len(), 1);
12244        let target = targets[0].clone();
12245        let consumed_before_checkpoint = store.pool_meta.read().await.pools[0]
12246            .decommission
12247            .as_ref()
12248            .and_then(|info| info.capacity_reservation.as_ref())
12249            .expect("checkpoint shutdown capacity reservation should exist")
12250            .consumed_target_physical_bytes;
12251        let precommit_error = store
12252            .run_decommission_capacity_non_growing_replacement_with_capacity_lease(
12253                target.target_pool_index,
12254                Some(target.capacity_owner),
12255                Some(aborting_data.len()),
12256                |_| async { Err::<(), Error>(Error::other("injected checkpoint failure before commit")) },
12257            )
12258            .await
12259            .expect_err("the first checkpoint attempt should fail before writing its target")
12260            .to_string();
12261        assert!(precommit_error.contains("injected checkpoint failure before commit"));
12262        assert!(
12263            store
12264                .has_decommission_capacity_temporary_mutation_state(target.target_pool_index, target.capacity_owner)
12265                .await,
12266            "a failed checkpoint attempt must retain exact retry state"
12267        );
12268        let barrier = crate::set_disk::PutObjectCommitBarrier::install(
12269            RUSTFS_META_BUCKET,
12270            &manifest_name,
12271            crate::set_disk::PutObjectCommitPause::BeforeQuotaRename,
12272        );
12273        let worker_store = store.clone();
12274        let worker_manifest = manifest_name.clone();
12275        let worker = tokio::spawn(async move {
12276            recover_test_tier_delete_dispatch_manifest_with_page_budget(worker_store, &worker_manifest, Duration::from_secs(30))
12277                .await
12278        });
12279        barrier.wait_until_paused().await;
12280        assert!(
12281            store
12282                .has_decommission_capacity_temporary_mutation_state(target.target_pool_index, target.capacity_owner)
12283                .await,
12284            "the target PUT must persist its capacity intent before commit"
12285        );
12286
12287        shutdown.cancel();
12288        tokio::time::sleep(Duration::from_millis(50)).await;
12289        assert_eq!(tier_delete_dispatch_manifest_recovery_count_for_test(&store), 1);
12290        assert!(
12291            tier_delete_dispatch_manifest_operation_lock_held_for_test(store.clone(), &manifest_name).await,
12292            "the target checkpoint PUT must retain its operation lock after shutdown"
12293        );
12294        assert!(
12295            crate::services::notification_sys::tier_delete_journal_fleet_proof_has_inflight_for_test(),
12296            "the target checkpoint PUT must retain its fleet-proof permit after shutdown"
12297        );
12298
12299        barrier.release();
12300        assert!(
12301            !worker.await.expect("checkpoint shutdown worker should join"),
12302            "cooperative shutdown should retain the partially committed checkpoint"
12303        );
12304        assert_eq!(tier_delete_dispatch_manifest_recovery_count_for_test(&store), 0);
12305        assert_eq!(
12306            com::read_config(store.pools[target.target_pool_index].clone(), &manifest_name)
12307                .await
12308                .expect("the committed target checkpoint should be readable"),
12309            aborting_data,
12310            "the admitted target PUT must finish before its fences release"
12311        );
12312        assert!(
12313            !store
12314                .has_decommission_capacity_temporary_mutation_state(target.target_pool_index, target.capacity_owner)
12315                .await,
12316            "the admitted target PUT must drain its capacity transaction before releasing the recovery fences"
12317        );
12318        {
12319            let pool_meta = store.pool_meta.read().await;
12320            let reservation = pool_meta.pools[0]
12321                .decommission
12322                .as_ref()
12323                .and_then(|info| info.capacity_reservation.as_ref())
12324                .expect("checkpoint shutdown capacity reservation should remain active");
12325            let capacity_target = reservation
12326                .targets
12327                .iter()
12328                .find(|candidate| candidate.pool_index == target.target_pool_index)
12329                .expect("checkpoint shutdown capacity target should remain allocated");
12330            assert_eq!(
12331                reservation.consumed_target_physical_bytes, consumed_before_checkpoint,
12332                "a non-growing checkpoint replacement must not consume durable migration capacity"
12333            );
12334            assert_eq!(reservation.pending_target_physical_bytes, 0);
12335            assert_eq!(reservation.inflight_target_physical_bytes, 0);
12336            assert_eq!(capacity_target.pending_physical_bytes, 0);
12337            assert!(capacity_target.temporary_mutations.is_empty());
12338        }
12339        drop(barrier);
12340        let mut reloaded_pool_meta = PoolMeta::default();
12341        reloaded_pool_meta
12342            .load(store.pools[0].clone(), store.pools.clone())
12343            .await
12344            .expect("checkpoint shutdown pool metadata should reload from disk");
12345        *store.pool_meta.write().await = reloaded_pool_meta;
12346        recover_test_tier_delete_dispatch_manifest(store.clone(), &manifest_name)
12347            .await
12348            .expect("a fresh recovery attempt should replay the committed checkpoint and receipt");
12349        assert!(
12350            !store
12351                .has_decommission_capacity_temporary_mutation_state(target.target_pool_index, target.capacity_owner)
12352                .await,
12353            "durable replay must reconcile the exact capacity transaction"
12354        );
12355        assert_eq!(
12356            store
12357                .decommission_durable_ilm_receipt_count_for_test(0)
12358                .await
12359                .expect("checkpoint shutdown receipts should be listable"),
12360            2,
12361            "durable replay must advance both the manifest and journal receipts"
12362        );
12363        drop(ctx);
12364    }
12365
12366    #[cfg(feature = "test-util")]
12367    #[tokio::test]
12368    #[serial_test::serial(storage_class_env)]
12369    async fn v6_dispatch_recovery_advances_decommission_receipts_and_cleans_target_copies() {
12370        let temp_dir = tempfile::tempdir().expect("create v6 decommission receipt store dir");
12371        let (_ctx, store, _shutdown) =
12372            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "v6-dispatch-decommission-receipts", &[4, 4]))
12373                .await;
12374        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
12375        let bucket = "v6-dispatch-decommission-receipts-bucket";
12376        store
12377            .make_bucket(bucket, &MakeBucketOptions::default())
12378            .await
12379            .expect("receipt bucket should be created");
12380        let incarnation = store
12381            .bucket_incarnation_id(bucket)
12382            .await
12383            .expect("receipt bucket incarnation should resolve");
12384        let entry = synthetic_v6_dispatch_entry(
12385            bucket,
12386            "archive/receipt.bin",
12387            "RECEIPT-TIER",
12388            [17; 32],
12389            &uuid::Uuid::new_v4().to_string(),
12390        );
12391        let (manifest_name, entries) = install_test_tier_delete_dispatch_fixture(
12392            store.clone(),
12393            bucket,
12394            incarnation,
12395            "archive/",
12396            vec![(entry, Some(TierDeleteJournalState::Prepared))],
12397            TierDeleteDispatchManifestState::Preparing,
12398        )
12399        .await
12400        .expect("Preparing v6 fixture should persist");
12401        let journal_name = tier_delete_journal_object_name(&entries[0]);
12402        let manifest_data = com::read_config(store.clone(), &manifest_name)
12403            .await
12404            .expect("manifest fixture should be readable");
12405        let journal_data = com::read_config(store.clone(), &journal_name)
12406            .await
12407            .expect("journal fixture should be readable");
12408
12409        // Model an in-flight decommission after both records have been copied
12410        // to its target pool but before their state machines advance.
12411        for pool in &store.pools {
12412            com::save_config(pool.clone(), &manifest_name, manifest_data.clone())
12413                .await
12414                .expect("manifest copy should persist in each pool");
12415            com::save_config(pool.clone(), &journal_name, journal_data.clone())
12416                .await
12417                .expect("journal copy should persist in each pool");
12418        }
12419        // Exercise the same durable capacity reservation and identity that a
12420        // real decommission start installs. A hand-written active flag makes
12421        // every target mutation look external and masks whether recovery is
12422        // correctly charged to the decommission owner.
12423        mark_test_pool_decommissioning(&store, 0).await;
12424
12425        let missing_receipt_error = recover_test_tier_delete_dispatch_manifest(store.clone(), &manifest_name)
12426            .await
12427            .expect_err("an active decommission without receipt coverage must block v6 checkpoint mutation")
12428            .to_string();
12429        assert!(
12430            missing_receipt_error.contains("missing receipt coverage"),
12431            "unexpected missing-receipt error: {missing_receipt_error}"
12432        );
12433        assert_eq!(
12434            com::read_config(store.pools[1].clone(), &manifest_name)
12435                .await
12436                .expect("blocked checkpoint must retain the target manifest generation"),
12437            manifest_data,
12438            "missing receipt coverage must fail before mutating a target copy"
12439        );
12440
12441        for (path, data) in [
12442            (&manifest_name, manifest_data.as_slice()),
12443            (&journal_name, journal_data.as_slice()),
12444        ] {
12445            let record = validate_durable_ilm_record(path, data).expect("v6 durable record should validate");
12446            store
12447                .persist_decommission_durable_ilm_receipt_for_test(0, 1, path, &record, false)
12448                .await
12449                .expect("initial decommission receipt should persist");
12450        }
12451
12452        // Model a crash after the target checkpoint PUT committed but before
12453        // its temporary-capacity intent was resolved. Recovery must recognize
12454        // the exact target bytes and finish that same deterministic intent
12455        // instead of treating the checkpoint as an uncharged success.
12456        let (aborting_data, preparing_etag) = test_tier_delete_dispatch_manifest_checkpoint(
12457            store.clone(),
12458            &manifest_name,
12459            TierDeleteDispatchManifestState::Aborting,
12460        )
12461        .await
12462        .expect("the Aborting checkpoint should encode");
12463        let checkpoint_targets = store
12464            .decommission_durable_ilm_checkpoint_targets(&manifest_name, &aborting_data, &preparing_etag)
12465            .await
12466            .expect("receipt-bearing checkpoint targets should resolve")
12467            .expect("an active decommission should own the checkpoint");
12468        assert_eq!(checkpoint_targets.len(), 1);
12469        let checkpoint_target = checkpoint_targets
12470            .into_iter()
12471            .next()
12472            .expect("one checkpoint target should exist");
12473        let checkpoint_owner = checkpoint_target.capacity_owner;
12474        let target_pool_index = checkpoint_target.target_pool_index;
12475        let injected_write_error = store
12476            .run_decommission_capacity_temporary_mutation_with_capacity_lease(
12477                target_pool_index,
12478                Some(checkpoint_owner),
12479                Some(aborting_data.len()),
12480                |_| {
12481                    let target_pool = store.pools[target_pool_index].clone();
12482                    let manifest_name = manifest_name.clone();
12483                    let aborting_data = aborting_data.clone();
12484                    async move {
12485                        com::save_config(target_pool, &manifest_name, aborting_data)
12486                            .await
12487                            .expect("the injected target checkpoint should commit");
12488                        Err::<(), Error>(Error::other("injected crash after checkpoint target commit"))
12489                    }
12490                },
12491            )
12492            .await
12493            .expect_err("the injected post-commit crash should leave a pending capacity intent")
12494            .to_string();
12495        assert!(injected_write_error.contains("injected crash after checkpoint target commit"));
12496        assert!(
12497            store
12498                .has_decommission_capacity_temporary_mutation_state(target_pool_index, checkpoint_owner)
12499                .await,
12500            "the committed target must retain its exact unresolved capacity intent"
12501        );
12502
12503        recover_test_tier_delete_dispatch_manifest(store.clone(), &manifest_name)
12504            .await
12505            .expect("Preparing rollback should advance receipts and finish terminal cleanup");
12506        assert!(
12507            !store
12508                .has_decommission_capacity_temporary_mutation_state(target_pool_index, checkpoint_owner)
12509                .await,
12510            "already-committed checkpoint recovery must resolve the pending capacity intent"
12511        );
12512        let stats = recover_tier_delete_dispatch_manifests(store.clone(), 100, None)
12513            .await
12514            .expect("the terminal source checkpoint should remain non-actionable");
12515        assert_eq!((stats.scanned, stats.deleted, stats.retained, stats.failed), (1, 0, 1, 0));
12516        assert_eq!(
12517            store
12518                .decommission_durable_ilm_receipt_count_for_test(0)
12519                .await
12520                .expect("v6 terminal receipts should be listable"),
12521            2
12522        );
12523
12524        {
12525            let _proof_guard = crate::services::notification_sys::without_cross_pool_fence_fleet_proof_for_test();
12526            let proof_error = store
12527                .verify_and_cleanup_decommissioned_durable_ilm_record_for_test(
12528                    0,
12529                    store.pools[0].get_disks_by_key(&manifest_name),
12530                    &manifest_name,
12531                )
12532                .await
12533                .expect_err("revoked v6 fleet proof must block final source cleanup")
12534                .to_string();
12535            assert!(
12536                proof_error.contains("fleet capability is unavailable"),
12537                "unexpected revoked-proof cleanup error: {proof_error}"
12538            );
12539            assert!(
12540                com::read_config(store.pools[0].clone(), &manifest_name).await.is_ok(),
12541                "revoked fleet proof must retain the last source manifest copy"
12542            );
12543        }
12544
12545        for path in [&manifest_name, &journal_name] {
12546            store
12547                .verify_and_cleanup_decommissioned_durable_ilm_record_for_test(0, store.pools[0].get_disks_by_key(path), path)
12548                .await
12549                .expect("terminal receipt should authorize verified source cleanup");
12550        }
12551        let stats = recover_tier_delete_dispatch_manifests(store.clone(), 100, None)
12552            .await
12553            .expect("verified source cleanup should remove the last manifest copy");
12554        assert_eq!((stats.scanned, stats.deleted, stats.failed), (0, 0, 0));
12555        for path in [&manifest_name, &journal_name] {
12556            for pool in &store.pools {
12557                assert!(
12558                    matches!(com::read_config(pool.clone(), path).await, Err(Error::ConfigNotFound)),
12559                    "terminal receipt cleanup must remove `{path}` from every staged pool"
12560                );
12561            }
12562        }
12563    }
12564
12565    #[cfg(feature = "test-util")]
12566    #[tokio::test]
12567    #[serial_test::serial(storage_class_env)]
12568    async fn authorized_all_committed_manifest_recovers_after_last_commit_crash() {
12569        let temp_dir = tempfile::tempdir().expect("create authorized manifest recovery store dir");
12570        let (ctx, store, _shutdown) =
12571            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "authorized-manifest-last-commit", &[4])).await;
12572        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
12573        let bucket = "authorized-manifest-last-commit-bucket";
12574        store
12575            .make_bucket(bucket, &MakeBucketOptions::default())
12576            .await
12577            .expect("authorized recovery bucket should be created");
12578        let incarnation = store
12579            .bucket_incarnation_id(bucket)
12580            .await
12581            .expect("bucket incarnation should resolve");
12582        let tier_name = "AUTHORIZED-LAST-COMMIT";
12583        let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
12584        let lease = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
12585            .await
12586            .expect("tier lease should resolve");
12587        let mut entries = Vec::new();
12588        for index in 0..2 {
12589            let remote_version = uuid::Uuid::new_v4().to_string();
12590            let entry = synthetic_v6_dispatch_entry(
12591                bucket,
12592                &format!("archive/{index}.bin"),
12593                tier_name,
12594                lease.backend_identity(),
12595                &remote_version,
12596            );
12597            backend.set_put_remote_version(Some(remote_version)).await;
12598            lease
12599                .put(&entry.obj_name, ReaderImpl::Body(bytes::Bytes::from_static(b"remote candidate")), 16)
12600                .await
12601                .expect("remote candidate should be seeded");
12602            entries.push((entry, Some(TierDeleteJournalState::Committed)));
12603        }
12604        let (manifest_name, _) = install_test_tier_delete_dispatch_fixture(
12605            store.clone(),
12606            bucket,
12607            incarnation,
12608            "archive/",
12609            entries,
12610            TierDeleteDispatchManifestState::DispatchAuthorized,
12611        )
12612        .await
12613        .expect("last-commit crash fixture should persist");
12614
12615        let manifest_stats = recover_tier_delete_dispatch_manifests(store.clone(), 100, None)
12616            .await
12617            .expect("all-Committed Authorized manifest should advance");
12618        assert_eq!((manifest_stats.scanned, manifest_stats.advanced, manifest_stats.failed), (1, 1, 0));
12619        assert_eq!(
12620            test_tier_delete_dispatch_manifest_state(store.clone(), &manifest_name)
12621                .await
12622                .expect("advanced manifest should remain readable"),
12623            Some(TierDeleteDispatchManifestState::Completed)
12624        );
12625        let journal_stats = recover_tier_delete_journal_entries(store.clone(), 100, None)
12626            .await
12627            .expect("Completed journal set should recover remote candidates");
12628        assert_eq!((journal_stats.scanned, journal_stats.deleted, journal_stats.failed), (2, 2, 0));
12629        assert_eq!(tier_delete_journal_count(store.clone()).await, 0);
12630        assert_eq!(tier_delete_dispatch_manifest_count(store.clone()).await, 1);
12631        let gc_stats = recover_tier_delete_dispatch_manifests(store.clone(), 100, None)
12632            .await
12633            .expect("manifest coordinator should remove the empty Completed manifest");
12634        assert_eq!((gc_stats.scanned, gc_stats.deleted, gc_stats.failed), (1, 1, 0));
12635        assert_eq!(tier_delete_dispatch_manifest_count(store).await, 0);
12636        assert_eq!(backend.object_count().await, 0);
12637        assert_eq!(backend.remove_versions().await.len(), 2);
12638    }
12639
12640    #[cfg(feature = "test-util")]
12641    async fn transition_transaction_record_count(store: Arc<crate::store::ECStore>) -> usize {
12642        store
12643            .list_objects_v2(
12644                RUSTFS_META_BUCKET,
12645                TRANSITION_TRANSACTION_RECORD_PREFIX,
12646                None,
12647                None,
12648                100,
12649                false,
12650                None,
12651                false,
12652            )
12653            .await
12654            .expect("transition transaction records should be listable")
12655            .objects
12656            .len()
12657    }
12658
12659    #[cfg(feature = "test-util")]
12660    async fn only_transition_transaction(store: Arc<crate::store::ECStore>) -> TransitionTransaction {
12661        let records = store
12662            .clone()
12663            .list_objects_v2(
12664                RUSTFS_META_BUCKET,
12665                TRANSITION_TRANSACTION_RECORD_PREFIX,
12666                None,
12667                None,
12668                100,
12669                false,
12670                None,
12671                false,
12672            )
12673            .await
12674            .expect("transition transaction records should be listable")
12675            .objects;
12676        assert_eq!(records.len(), 1, "test fixture should have exactly one transition transaction");
12677        let transaction_id = records[0]
12678            .name
12679            .rsplit('/')
12680            .next()
12681            .and_then(|name| name.strip_suffix(".json"))
12682            .and_then(|name| uuid::Uuid::parse_str(name).ok())
12683            .expect("transition transaction path should end in its UUID");
12684        load_transition_transaction_record(store, transaction_id)
12685            .await
12686            .expect("transition transaction should load")
12687    }
12688
12689    #[cfg(feature = "test-util")]
12690    async fn register_transition_reconcile_test_tier(
12691        handle: &Arc<tokio::sync::RwLock<TierConfigMgr>>,
12692        tier_name: &str,
12693    ) -> MockWarmBackend {
12694        let backend = MockWarmBackend::new();
12695        let mut manager = handle.write().await;
12696        manager.tiers.insert(
12697            tier_name.to_string(),
12698            TierConfig {
12699                version: "v1".to_string(),
12700                tier_type: TierType::Wasabi,
12701                name: tier_name.to_string(),
12702                wasabi: Some(TierWasabi {
12703                    name: tier_name.to_string(),
12704                    endpoint: "https://s3.wasabisys.com".to_string(),
12705                    access_key: "test-access-key".to_string(),
12706                    secret_key: "test-secret-key".to_string(),
12707                    bucket: "mock-tier".to_string(),
12708                    prefix: format!("mock/{}/", uuid::Uuid::new_v4()),
12709                    region: "us-east-1".to_string(),
12710                }),
12711                ..Default::default()
12712            },
12713        );
12714        manager
12715            .install_test_driver(tier_name, Box::new(backend.clone()))
12716            .expect("mock fallback tier driver should install");
12717        backend
12718    }
12719
12720    #[cfg(feature = "test-util")]
12721    async fn wait_for_tier_delete_journal_recovery(
12722        store: Arc<crate::store::ECStore>,
12723        backend: &MockWarmBackend,
12724        expected_removes: usize,
12725    ) {
12726        tokio::time::timeout(Duration::from_secs(30), async {
12727            loop {
12728                if backend.remove_versions().await.len() >= expected_removes
12729                    && tier_delete_journal_count(store.clone()).await == 0
12730                {
12731                    return;
12732                }
12733                tokio::time::sleep(Duration::from_millis(10)).await;
12734            }
12735        })
12736        .await
12737        .expect("tier delete journal recovery should complete");
12738    }
12739
12740    #[cfg(feature = "test-util")]
12741    async fn wait_for_tier_free_version_recovery(
12742        store: Arc<crate::store::ECStore>,
12743        backend: &MockWarmBackend,
12744        expected_removes: usize,
12745    ) {
12746        ExpiryState::resize_workers(1, store.clone()).await;
12747        let mut last_progress = None;
12748        let result = tokio::time::timeout(Duration::from_secs(30), async {
12749            loop {
12750                let stats = recover_tier_free_versions(store.clone(), 100, None, None)
12751                    .await
12752                    .expect("tier free-version recovery scan should succeed");
12753                let removes = backend.remove_versions().await.len();
12754                let recovered = removes >= expected_removes && stats.enqueued == 0 && stats.failed == 0;
12755                last_progress = Some((removes, stats));
12756                if recovered {
12757                    return;
12758                }
12759                tokio::time::sleep(Duration::from_millis(50)).await;
12760            }
12761        })
12762        .await;
12763        if result.is_err() {
12764            let expiry_state = store.ctx.expiry_state();
12765            let workers = expiry_state
12766                .try_read()
12767                .ok()
12768                .map(|state| (state.pending_tasks(), state.active_tasks()));
12769            panic!(
12770                "tier free-version recovery should complete: expected_removes={expected_removes}, \
12771                 last_progress={last_progress:?}, workers(pending, active)={workers:?}"
12772            );
12773        }
12774        wait_for_expiry_workers_idle(&store).await;
12775    }
12776
12777    /// Unlocked poll for exact metadata absence while asynchronous free-version
12778    /// cleanup removes the per-disk copies. Mid-cleanup, fewer than a read
12779    /// quorum of disks may still hold the record, so that transient result
12780    /// means "not converged yet"; every other error still fails the test.
12781    #[cfg(feature = "test-util")]
12782    async fn exact_metadata_absent_during_cleanup(store: &crate::store::ECStore, bucket: &str, object: &str) -> bool {
12783        match store.pools[0]
12784            .get_disks_by_key(object)
12785            .load_file_info_versions_exact(bucket, object)
12786            .await
12787        {
12788            Ok(metadata) => metadata.is_none(),
12789            Err(StorageError::InsufficientReadQuorum(_, _)) => false,
12790            Err(error) => panic!("{object} cleanup metadata should remain readable: {error:?}"),
12791        }
12792    }
12793
12794    /// Every physical copy must carry the replacement plus its tier
12795    /// free-version owner, and none may still hold the pre-overwrite live
12796    /// transitioned source. A stale minority copy is exactly what a lost
12797    /// early-ACK rename tail leaves behind, and exact cleanup refuses to
12798    /// delete remote bytes while such a live reference exists.
12799    #[cfg(feature = "test-util")]
12800    async fn assert_every_disk_holds_only_the_cleanup_owner(
12801        set: &crate::set_disk::SetDisks,
12802        bucket: &str,
12803        object: &str,
12804        remote: &str,
12805    ) {
12806        use crate::disk::DiskAPI as _;
12807
12808        let disk_object = rustfs_utils::path::encode_dir_object(object);
12809        for (index, disk) in set.disk_inventory().await.into_iter().enumerate() {
12810            let disk = disk.unwrap_or_else(|| panic!("disk{index} should be online"));
12811            let raw = disk
12812                .read_xl(bucket, &disk_object, false)
12813                .await
12814                .unwrap_or_else(|err| panic!("disk{index} xl.meta should be readable after the overwrite: {err:?}"));
12815            let versions = rustfs_filemeta::FileMeta::load(&raw.buf)
12816                .and_then(|meta| meta.get_all_file_info_versions(bucket, object, true))
12817                .unwrap_or_else(|err| panic!("disk{index} xl.meta should decode: {err:?}"));
12818            let all: Vec<_> = versions.versions.iter().chain(versions.free_versions.iter()).collect();
12819            let owners = all.iter().filter(|fi| fi.tier_free_version()).count();
12820            let live_sources = all
12821                .iter()
12822                .filter(|fi| !fi.tier_free_version() && fi.transitioned_objname == remote)
12823                .count();
12824            assert_eq!(owners, 1, "disk{index} must hold exactly one cleanup owner after the drained overwrite");
12825            assert_eq!(live_sources, 0, "disk{index} must not retain the pre-overwrite live transitioned source");
12826        }
12827    }
12828
12829    #[cfg(feature = "test-util")]
12830    async fn wait_for_expiry_workers_idle(store: &crate::store::ECStore) {
12831        let expiry_state = store.ctx.expiry_state();
12832        let idle = tokio::time::timeout(Duration::from_secs(30), async {
12833            loop {
12834                let idle = {
12835                    let state = expiry_state.read().await;
12836                    state.pending_tasks() == 0 && state.active_tasks() == 0
12837                };
12838                if idle {
12839                    // Recheck after a scheduler turn so the tiny hand-off
12840                    // between dequeue and active-task accounting cannot make
12841                    // the test observe a false idle state.
12842                    tokio::time::sleep(Duration::from_millis(10)).await;
12843                    let state = expiry_state.read().await;
12844                    if state.pending_tasks() == 0 && state.active_tasks() == 0 {
12845                        return;
12846                    }
12847                } else {
12848                    tokio::time::sleep(Duration::from_millis(10)).await;
12849                }
12850            }
12851        })
12852        .await;
12853        if idle.is_err() {
12854            let state = expiry_state.read().await;
12855            panic!(
12856                "lifecycle expiry workers should become idle: pending={} active={}",
12857                state.pending_tasks(),
12858                state.active_tasks()
12859            );
12860        }
12861    }
12862
12863    #[cfg(feature = "test-util")]
12864    async fn run_transitioned_delete_free_version_owner_case(
12865        store_name: &str,
12866        tier_name: &str,
12867        bucket: &str,
12868        object: &str,
12869        restore_before_delete: bool,
12870        causal_enqueue: bool,
12871        delete_with_journal: bool,
12872    ) {
12873        let temp_dir = tempfile::tempdir().expect("create transitioned delete store dir");
12874        let (ctx, store, _shutdown) =
12875            without_storage_class_env(build_isolated_test_store(temp_dir.path(), store_name, &[4])).await;
12876        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
12877
12878        let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
12879        store
12880            .make_bucket(bucket, &MakeBucketOptions::default())
12881            .await
12882            .expect("source bucket should be created");
12883        let mut reader = PutObjReader::from_vec(vec![b'd'; 1024 * 1024]);
12884        let original = store
12885            .put_object(bucket, object, &mut reader, &ObjectOptions::default())
12886            .await
12887            .expect("source object should be written");
12888        store
12889            .transition_object(
12890                bucket,
12891                object,
12892                &ObjectOptions {
12893                    transition: TransitionOptions {
12894                        status: TRANSITION_PENDING.to_string(),
12895                        tier: tier_name.to_string(),
12896                        etag: original.etag.clone().expect("source should have an etag"),
12897                        ..Default::default()
12898                    },
12899                    mod_time: original.mod_time,
12900                    ..Default::default()
12901                },
12902            )
12903            .await
12904            .expect("source transition should commit");
12905        assert_eq!(backend.object_count().await, 1, "transition should create one remote object");
12906        if restore_before_delete {
12907            store
12908                .clone()
12909                .restore_transitioned_object(
12910                    bucket,
12911                    object,
12912                    &ObjectOptions {
12913                        transition: TransitionOptions {
12914                            restore_request: s3s::dto::RestoreRequest {
12915                                days: Some(1),
12916                                ..Default::default()
12917                            },
12918                            ..Default::default()
12919                        },
12920                        ..Default::default()
12921                    },
12922                )
12923                .await
12924                .expect("restore should complete before transitioned delete");
12925            let restored = store
12926                .get_object_info(bucket, object, &ObjectOptions::default())
12927                .await
12928                .expect("restored transitioned source should remain readable");
12929            assert!(
12930                restored.user_defined.contains_key(rustfs_filemeta::metadata_keys::RESTORE),
12931                "restore completion metadata must be present before the delete regression"
12932            );
12933        }
12934
12935        if causal_enqueue {
12936            ExpiryState::resize_workers(1, store.clone()).await;
12937        }
12938        backend.set_remove_failure(!causal_enqueue);
12939        if delete_with_journal {
12940            store
12941                .delete_object_with_tier_delete_journal(bucket, object, ObjectOptions::default())
12942                .await
12943                .expect("transitioned source journal-wrapper delete should commit");
12944        } else {
12945            store
12946                .delete_object(bucket, object, ObjectOptions::default())
12947                .await
12948                .expect("transitioned source plain object-layer delete should commit");
12949        }
12950
12951        if causal_enqueue {
12952            tokio::time::timeout(Duration::from_secs(30), async {
12953                loop {
12954                    let metadata_absent = {
12955                        // Observe one cleanup state, without spanning per-disk marker removal.
12956                        let mut read_opts = ObjectOptions::default();
12957                        let _guards = store
12958                            .acquire_all_physical_object_read_locks(
12959                                "transitioned_delete_cleanup_test",
12960                                bucket,
12961                                object,
12962                                &mut read_opts,
12963                            )
12964                            .await
12965                            .expect("causal free-version cleanup observation should acquire object read locks");
12966                        store.pools[0]
12967                            .get_disks_by_key(object)
12968                            .load_file_info_versions_exact(bucket, object)
12969                            .await
12970                            .expect("causal free-version cleanup metadata should remain readable")
12971                            .is_none()
12972                    };
12973                    if metadata_absent && backend.remove_versions().await.len() == 1 {
12974                        return;
12975                    }
12976                    tokio::time::sleep(Duration::from_millis(10)).await;
12977                }
12978            })
12979            .await
12980            .expect("committed free-version should be cleaned without a recovery scan");
12981            assert_eq!(backend.object_count().await, 0, "causal cleanup should remove the remote object");
12982            assert_eq!(
12983                tier_delete_journal_count(store.clone()).await,
12984                0,
12985                "ordinary causal cleanup must not create a journal"
12986            );
12987            store
12988                .delete_bucket(bucket, &DeleteBucketOptions::default())
12989                .await
12990                .expect("bucket delete should succeed after causal free-version cleanup");
12991            return;
12992        }
12993
12994        let local_versions = store.pools[0]
12995            .get_disks_by_key(object)
12996            .load_file_info_versions_exact(bucket, object)
12997            .await
12998            .expect("post-delete metadata should remain decodable");
12999        let local_versions = local_versions.expect("ordinary delete must leave a durable free-version cleanup owner");
13000        assert_eq!(
13001            local_versions
13002                .versions
13003                .iter()
13004                .chain(local_versions.free_versions.iter())
13005                .filter(|version| version.tier_free_version())
13006                .count(),
13007            1,
13008            "ordinary delete must leave exactly one hidden free-version owner"
13009        );
13010        assert_eq!(
13011            local_versions
13012                .versions
13013                .iter()
13014                .filter(|version| !version.tier_free_version())
13015                .count(),
13016            0,
13017            "ordinary delete must remove the visible transitioned source"
13018        );
13019        let listed = store
13020            .clone()
13021            .list_object_versions(bucket, "", None, None, None, 100)
13022            .await
13023            .expect("source versions should be listable");
13024        assert!(listed.objects.is_empty(), "source must have no visible versions after delete");
13025
13026        assert_eq!(
13027            tier_delete_journal_count(store.clone()).await,
13028            0,
13029            "ordinary delete must not create a journal"
13030        );
13031
13032        backend.set_remove_failure(false);
13033        ExpiryState::resize_workers(1, store.clone()).await;
13034        let recovered = recover_tier_free_versions(store.clone(), 100, None, None)
13035            .await
13036            .expect("free-version recovery should scan the hidden owner");
13037        assert_eq!((recovered.enqueued, recovered.failed), (1, 0));
13038        tokio::time::timeout(Duration::from_secs(30), async {
13039            loop {
13040                let metadata_absent = {
13041                    // Observe one cleanup state, without spanning per-disk marker removal.
13042                    let mut read_opts = ObjectOptions::default();
13043                    let _guards = store
13044                        .acquire_all_physical_object_read_locks(
13045                            "transitioned_delete_cleanup_test",
13046                            bucket,
13047                            object,
13048                            &mut read_opts,
13049                        )
13050                        .await
13051                        .expect("free-version cleanup observation should acquire object read locks");
13052                    store.pools[0]
13053                        .get_disks_by_key(object)
13054                        .load_file_info_versions_exact(bucket, object)
13055                        .await
13056                        .expect("free-version cleanup metadata should remain readable")
13057                        .is_none()
13058                };
13059                if metadata_absent && backend.remove_versions().await.len() == 1 {
13060                    return;
13061                }
13062                tokio::time::sleep(Duration::from_millis(10)).await;
13063            }
13064        })
13065        .await
13066        .expect("free-version recovery should delete the remote and exact local owner");
13067        assert_eq!(backend.object_count().await, 0, "free-version recovery should remove the remote object");
13068        store
13069            .delete_bucket(bucket, &DeleteBucketOptions::default())
13070            .await
13071            .expect("ordinary bucket delete should succeed after the last transitioned object");
13072    }
13073
13074    #[cfg(feature = "test-util")]
13075    #[tokio::test]
13076    #[serial_test::serial(storage_class_env)]
13077    async fn transitioned_delete_uses_free_version_as_cleanup_owner() {
13078        run_transitioned_delete_free_version_owner_case(
13079            "transitioned-delete-journal-owner",
13080            "DELETEOWNER",
13081            "transitioned-delete-journal-owner-bucket",
13082            "transition/archive.bin",
13083            false,
13084            false,
13085            true,
13086        )
13087        .await;
13088    }
13089
13090    #[cfg(feature = "test-util")]
13091    #[tokio::test]
13092    #[serial_test::serial(storage_class_env)]
13093    async fn restored_transitioned_delete_uses_free_version_as_cleanup_owner() {
13094        run_transitioned_delete_free_version_owner_case(
13095            "restored-transitioned-delete-journal-owner",
13096            "RESTOREDELETEOWNER",
13097            "restored-transitioned-delete-journal-owner-bucket",
13098            "transition/archive.bin",
13099            true,
13100            false,
13101            true,
13102        )
13103        .await;
13104    }
13105
13106    #[cfg(feature = "test-util")]
13107    #[tokio::test]
13108    #[serial_test::serial(storage_class_env)]
13109    async fn transitioned_delete_causally_enqueues_free_version() {
13110        run_transitioned_delete_free_version_owner_case(
13111            "transitioned-delete-causal-enqueue",
13112            "DELETE-CAUSAL",
13113            "transitioned-delete-causal-enqueue-bucket",
13114            "transition/archive.bin",
13115            false,
13116            true,
13117            false,
13118        )
13119        .await;
13120    }
13121
13122    #[cfg(feature = "test-util")]
13123    #[tokio::test]
13124    #[serial_test::serial(storage_class_env)]
13125    async fn restored_transitioned_delete_causally_enqueues_free_version() {
13126        run_transitioned_delete_free_version_owner_case(
13127            "restored-transitioned-delete-causal-enqueue",
13128            "RESTORE-DELETE-CAUSAL",
13129            "restored-transitioned-delete-causal-enqueue-bucket",
13130            "transition/archive.bin",
13131            true,
13132            true,
13133            true,
13134        )
13135        .await;
13136    }
13137
13138    #[cfg(feature = "test-util")]
13139    #[tokio::test]
13140    #[serial_test::serial(storage_class_env)]
13141    async fn force_tier_remove_blocks_on_physical_free_version_hidden_by_other_pool() {
13142        let temp_dir = tempfile::tempdir().expect("create tier reference store dir");
13143        let (ctx, store, _shutdown) =
13144            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "tier-free-reference-proof", &[4, 4])).await;
13145        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
13146
13147        let tier_name = "FREE-PROOF";
13148        let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
13149        let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
13150            .await
13151            .expect("test tier identity should resolve")
13152            .backend_identity();
13153        let bucket = "tier-free-reference-proof-bucket";
13154        let object = "archive.bin";
13155        store
13156            .make_bucket(bucket, &MakeBucketOptions::default())
13157            .await
13158            .expect("source bucket should be created");
13159        let mut reader = PutObjReader::from_vec(vec![b'f'; 1024 * 1024]);
13160        let original = store.pools[0]
13161            .put_object(bucket, object, &mut reader, &ObjectOptions::default())
13162            .await
13163            .expect("source object should be written");
13164        store.pools[0]
13165            .transition_object(
13166                bucket,
13167                object,
13168                &ObjectOptions {
13169                    transition: TransitionOptions {
13170                        status: TRANSITION_PENDING.to_string(),
13171                        tier: tier_name.to_string(),
13172                        etag: original.etag.clone().expect("source should have an etag"),
13173                        ..Default::default()
13174                    },
13175                    mod_time: original.mod_time,
13176                    ..Default::default()
13177                },
13178            )
13179            .await
13180            .expect("source transition should commit");
13181        backend.set_remove_failure(true);
13182        store.pools[0]
13183            .delete_object(
13184                bucket,
13185                object,
13186                ObjectOptions {
13187                    tier_delete_journal_api: Some(store.clone()),
13188                    ..Default::default()
13189                },
13190            )
13191            .await
13192            .expect("journal-less compatibility delete should retain a free-version");
13193
13194        let exact = store.pools[0]
13195            .get_disks_by_key(object)
13196            .load_file_info_versions_exact(bucket, object)
13197            .await
13198            .expect("free-version metadata should remain decodable")
13199            .expect("free-version metadata should remain on disk");
13200        assert_eq!(
13201            exact
13202                .versions
13203                .iter()
13204                .chain(exact.free_versions.iter())
13205                .filter(|version| version.tier_free_version())
13206                .count(),
13207            1,
13208            "fixture must contain one real serialized hidden free-version"
13209        );
13210
13211        let mut hot_reader = PutObjReader::from_vec(vec![b'h'; 1024 * 1024]);
13212        store.pools[1]
13213            .put_object(
13214                bucket,
13215                object,
13216                &mut hot_reader,
13217                &ObjectOptions {
13218                    mod_time: Some(OffsetDateTime::now_utc() + time::Duration::hours(1)),
13219                    ..Default::default()
13220                },
13221            )
13222            .await
13223            .expect("second pool hot source should be written");
13224        let merged = store
13225            .get_object_info(bucket, object, &ObjectOptions::default())
13226            .await
13227            .expect("merged lookup should return the second-pool hot source");
13228        assert!(
13229            merged.transitioned_object.status.is_empty() && !merged.transitioned_object.free_version,
13230            "the ordinary second-pool source must hide the first-pool free-version from merged lookup"
13231        );
13232
13233        let err = ensure_no_authoritative_tier_references_for_test(store, tier_name, backend_identity, true)
13234            .await
13235            .expect_err("force tier removal must not bypass a physical free-version cleanup obligation");
13236        assert_eq!(err.code, ERR_TIER_BACKEND_IN_USE.code);
13237    }
13238
13239    #[cfg(feature = "test-util")]
13240    #[tokio::test]
13241    #[serial_test::serial(storage_class_env)]
13242    async fn tier_reference_proof_fails_closed_when_physical_bucket_topology_is_incomplete() {
13243        let temp_dir = tempfile::tempdir().expect("create incomplete tier reference store dir");
13244        let (ctx, store, _shutdown) =
13245            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "tier-reference-incomplete-topology", &[4]))
13246                .await;
13247        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
13248
13249        let tier_name = "INCOMPLETE-PROOF";
13250        register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
13251        let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
13252            .await
13253            .expect("test tier identity should resolve")
13254            .backend_identity();
13255        store.pools[0].disk_set[0].disks.write().await[0] = None;
13256
13257        let err = ensure_no_authoritative_tier_references_for_test(store, tier_name, backend_identity, true)
13258            .await
13259            .expect_err("an incomplete physical bucket enumeration must block tier mutation");
13260        assert_eq!(err.code, ERR_TIER_INVALID_CONFIG.code);
13261        assert!(
13262            err.message.contains("complete set quorum"),
13263            "unexpected reference proof error: {}",
13264            err.message
13265        );
13266    }
13267
13268    #[cfg(feature = "test-util")]
13269    async fn copy_test_xlmeta_between_pools(
13270        root: &std::path::Path,
13271        source_pool: usize,
13272        target_pool: usize,
13273        bucket: &str,
13274        object: &str,
13275    ) {
13276        for disk_index in 0..4 {
13277            let source = root.join(format!("pool{source_pool}/set0/disk{disk_index}/{bucket}/{object}/{STORAGE_FORMAT_FILE}"));
13278            let target = root.join(format!("pool{target_pool}/set0/disk{disk_index}/{bucket}/{object}/{STORAGE_FORMAT_FILE}"));
13279            tokio::fs::create_dir_all(target.parent().expect("target xl.meta should have a parent"))
13280                .await
13281                .expect("target object directory should be created");
13282            tokio::fs::copy(&source, &target)
13283                .await
13284                .expect("xl.meta should copy exactly between pools");
13285        }
13286    }
13287
13288    #[cfg(feature = "test-util")]
13289    async fn rewrite_transitioned_xlmeta_as_legacy_unknown(
13290        root: &std::path::Path,
13291        pool_index: usize,
13292        bucket: &str,
13293        object: &str,
13294        minio_unversioned: bool,
13295    ) {
13296        for disk_index in 0..4 {
13297            let metadata_path =
13298                root.join(format!("pool{pool_index}/set0/disk{disk_index}/{bucket}/{object}/{STORAGE_FORMAT_FILE}"));
13299            let encoded = tokio::fs::read(&metadata_path)
13300                .await
13301                .expect("transitioned xl.meta should be readable");
13302            let mut metadata = FileMeta::load(&encoded).expect("transitioned xl.meta should decode");
13303            let mut found = false;
13304            for shallow in &mut metadata.versions {
13305                let mut version = shallow
13306                    .parse_version_meta()
13307                    .expect("transitioned version record should decode");
13308                let Some(object_meta) = version.object.as_mut() else {
13309                    continue;
13310                };
13311                if rustfs_utils::http::metadata_compat::get_bytes(
13312                    &object_meta.meta_sys,
13313                    rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_STATUS,
13314                )
13315                .as_deref()
13316                    != Some(rustfs_filemeta::TRANSITION_COMPLETE.as_bytes())
13317                {
13318                    continue;
13319                }
13320                found = true;
13321                for suffix in [
13322                    rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
13323                    rustfs_utils::http::metadata_compat::SUFFIX_TRANSITIONED_VERSION_STATE,
13324                    rustfs_utils::http::metadata_compat::SUFFIX_TRANSITIONED_VERSION_ID,
13325                ] {
13326                    rustfs_utils::http::metadata_compat::remove_bytes(&mut object_meta.meta_sys, suffix);
13327                }
13328                if minio_unversioned {
13329                    object_meta
13330                        .meta_sys
13331                        .insert("x-minio-internal-transitioned-versionID".to_string(), Vec::new());
13332                }
13333                *shallow = rustfs_filemeta::FileMetaShallowVersion::try_from(version)
13334                    .expect("legacy transitioned version should re-encode");
13335            }
13336            assert!(found, "transitioned source should exist in serialized xl.meta");
13337            tokio::fs::write(&metadata_path, metadata.marshal_msg().expect("legacy transitioned xl.meta should encode"))
13338                .await
13339                .expect("legacy transitioned xl.meta should persist");
13340        }
13341    }
13342
13343    #[cfg(feature = "test-util")]
13344    async fn read_store_body(
13345        store: &Arc<crate::store::ECStore>,
13346        bucket: &str,
13347        object: &str,
13348        range: Option<HTTPRangeSpec>,
13349        opts: &ObjectOptions,
13350    ) -> Vec<u8> {
13351        let mut reader = store
13352            .get_object_reader(bucket, object, range, HeaderMap::new(), opts)
13353            .await
13354            .expect("object reader should open");
13355        let mut body = Vec::new();
13356        reader.stream.read_to_end(&mut body).await.expect("object body should drain");
13357        body
13358    }
13359
13360    #[cfg(feature = "test-util")]
13361    #[test]
13362    #[serial_test::serial(storage_class_env)]
13363    fn legacy_transition_state_inspection_and_apply_keep_all_disk_copies_unchanged() {
13364        run_large_stack_async_test("legacy-state-reconcile-inspection", || {
13365            legacy_transition_state_inspection_and_apply_case(false)
13366        });
13367    }
13368
13369    #[cfg(feature = "test-util")]
13370    #[cfg(not(windows))]
13371    #[test]
13372    #[serial_test::serial(storage_class_env)]
13373    fn legacy_transition_state_backfill_retries_partial_commits_and_preserves_other_bytes() {
13374        run_large_stack_async_test("legacy-state-reconcile-backfill", || {
13375            legacy_transition_state_inspection_and_apply_case(true)
13376        });
13377    }
13378
13379    #[cfg(feature = "test-util")]
13380    async fn legacy_transition_state_inspection_and_apply_case(write_enabled: bool) {
13381        #[cfg(windows)]
13382        assert!(!write_enabled, "Windows supports inspection but cannot prove repair directory durability");
13383        use crate::bucket::lifecycle::legacy_transition_state_reconcile::{
13384            LegacyTransitionStateReconcileOutcome as Outcome, LegacyTransitionStateReconcileRequest,
13385            LegacyTransitionStateReconcileSelector,
13386        };
13387        for (remote_version, expected_state) in [
13388            ("", rustfs_filemeta::TransitionVersionState::KnownDisabled),
13389            ("null", rustfs_filemeta::TransitionVersionState::SuspendedNull),
13390            ("opaque-version", rustfs_filemeta::TransitionVersionState::Exact),
13391        ] {
13392            let temp_dir = tempfile::tempdir().expect("legacy reconcile store directory");
13393            let (ctx, store, _shutdown) =
13394                without_storage_class_env(build_isolated_test_store(temp_dir.path(), "legacy-state-reconcile-inspect", &[4]))
13395                    .await;
13396            crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
13397            let tier_name = "LEGACY-RECONCILE";
13398            let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
13399            backend.set_put_remote_version(Some(remote_version.to_string())).await;
13400            let bucket = "legacy-state-reconcile-bucket";
13401            let object = "archive.bin";
13402            store
13403                .make_bucket(bucket, &MakeBucketOptions::default())
13404                .await
13405                .expect("create legacy fixture bucket");
13406            let mut reader = PutObjReader::from_vec(b"legacy reconcile body".repeat(1024));
13407            let source = store
13408                .put_object(bucket, object, &mut reader, &ObjectOptions::default())
13409                .await
13410                .expect("write source");
13411            {
13412                // Create the fixture under the existing remote-version writer
13413                // gate. This does not authorize legacy metadata reconciliation.
13414                let _proof = crate::services::notification_sys::install_current_remote_version_state_fleet_proof_for_test();
13415                temp_env::async_with_vars(
13416                    [
13417                        (rustfs_config::ENV_TIER_REMOTE_VERSION_STATE_WRITE, Some("true")),
13418                        (rustfs_config::ENV_TIER_REMOTE_VERSION_STATE_FLEET_CONFIRMED, Some("true")),
13419                    ],
13420                    store.transition_object(
13421                        bucket,
13422                        object,
13423                        &ObjectOptions {
13424                            transition: TransitionOptions {
13425                                status: TRANSITION_PENDING.to_string(),
13426                                tier: tier_name.to_string(),
13427                                etag: source.etag.clone().expect("source ETag"),
13428                                ..Default::default()
13429                            },
13430                            mod_time: source.mod_time,
13431                            ..Default::default()
13432                        },
13433                    ),
13434                )
13435                .await
13436                .expect("transition source");
13437            }
13438            assert!(
13439                crate::services::notification_sys::acquire_legacy_transition_state_reconcile_fleet_proof()
13440                    .await
13441                    .is_none(),
13442                "fixture setup must not grant the missing reconciliation write capability"
13443            );
13444            let selector = LegacyTransitionStateReconcileSelector {
13445                bucket: bucket.to_string(),
13446                object: object.to_string(),
13447                version_id: "null".to_string(),
13448            };
13449            if expected_state == rustfs_filemeta::TransitionVersionState::Exact {
13450                backend
13451                    .set_transition_candidate_probe_override(Some(
13452                        crate::services::tier::warm_backend::TransitionCandidateProbe::Ambiguous,
13453                    ))
13454                    .await;
13455            }
13456            let converged = store
13457                .inspect_legacy_transition_state(selector.clone())
13458                .await
13459                .expect("inspect an already explicit transition");
13460            assert_eq!(converged.outcome, Outcome::Migrated, "{converged:?}");
13461            assert!(!converged.changed);
13462            backend.set_transition_candidate_probe_override(None).await;
13463            rewrite_transitioned_xlmeta_as_legacy_unknown(temp_dir.path(), 0, bucket, object, remote_version.is_empty()).await;
13464            let paths = (0..4)
13465                .map(|disk| {
13466                    temp_dir
13467                        .path()
13468                        .join(format!("pool0/set0/disk{disk}/{bucket}/{object}/{STORAGE_FORMAT_FILE}"))
13469                })
13470                .collect::<Vec<_>>();
13471            let mut original = Vec::new();
13472            for path in &paths {
13473                original.push(tokio::fs::read(path).await.expect("original xl.meta"));
13474            }
13475            backend.clear_op_log().await;
13476            let inspection = store.inspect_legacy_transition_state(selector.clone());
13477            assert!(
13478                std::mem::size_of_val(&inspection) <= 4 * 1024,
13479                "admin inspection future must remain stack-bounded"
13480            );
13481            let inspected = inspection.await.expect("inspect legacy state");
13482            assert_eq!(inspected.outcome, Outcome::ReadyToMigrate, "{inspected:?}");
13483            assert!(!inspected.readiness.post_ready, "current fleet cannot authorize conditional writes");
13484            let target = inspected.target.expect("live probe should establish one model");
13485            assert_eq!(target.state, expected_state);
13486            let request = LegacyTransitionStateReconcileRequest {
13487                confirm: true,
13488                selector,
13489                source: inspected.source.expect("immutable source"),
13490                original_sets: inspected.original_sets,
13491                target,
13492                reconciliation_digest: inspected.reconciliation_digest.expect("expected tuple digest"),
13493            };
13494            #[cfg(not(windows))]
13495            if write_enabled {
13496                crate::services::notification_sys::with_legacy_transition_state_fleet_proof_for_test(async {
13497                    crate::disk::local::bucket_durability::set(bucket, Some(crate::disk::local::DurabilityMode::None));
13498                    let unsynced = store.reconcile_legacy_transition_state(request.clone()).await;
13499                    crate::disk::local::bucket_durability::set(bucket, None);
13500                    let unsynced = unsynced.expect("repair without metadata durability");
13501                    assert_eq!(unsynced.outcome, Outcome::BackendUnavailable);
13502                    assert!(!unsynced.changed);
13503                    let rollback = paths[0].parent().expect("object directory").join(Uuid::new_v4().to_string());
13504                    tokio::fs::create_dir(&rollback).await.expect("pending rollback directory");
13505                    tokio::fs::write(rollback.join(crate::disk::STORAGE_FORMAT_FILE_BACKUP), &original[0])
13506                        .await
13507                        .expect("pending old metadata backup");
13508                    let unsettled = store.reconcile_legacy_transition_state(request.clone()).await;
13509                    tokio::fs::remove_dir_all(&rollback).await.expect("settle fixture rollback");
13510                    let unsettled = unsettled.expect("repair must wait for rollback");
13511                    assert_eq!(unsettled.outcome, Outcome::BackendUnavailable);
13512                    assert!(!unsettled.changed);
13513                    for (path, bytes) in paths.iter().zip(&original) {
13514                        assert_eq!(tokio::fs::read(path).await.expect("blocked repair leaves original bytes"), *bytes);
13515                    }
13516                    // The first disk commits; the second stops after staging.
13517                    // This models an interrupted cross-disk effect without rollback.
13518                    let disks = store.all_set_disks()[0].disk_inventory().await;
13519                    let first_disk = disks[0].as_ref().expect("first physical disk");
13520                    let publication_path = first_disk
13521                        .get_object_path_for_io_if_local(bucket, &format!("{object}/{STORAGE_FORMAT_FILE}"))
13522                        .expect("local disk")
13523                        .expect("publication path");
13524                    let crash_key = format!("{object}/{STORAGE_FORMAT_FILE}");
13525                    let _hook = crate::disk::os::prepared_publication_test_hooks::install_at(
13526                        crate::disk::os::prepared_publication_test_hooks::Stage::Rename,
13527                        &publication_path,
13528                        move || {
13529                            crate::crash_inject::arm(crate::crash_inject::CrashPoint::MetaWriteAfterTmpBeforeRename, &crash_key);
13530                        },
13531                    );
13532                    let partial = store
13533                        .reconcile_legacy_transition_state(request.clone())
13534                        .await
13535                        .expect("partial repair response");
13536                    assert_eq!(partial.outcome, Outcome::BackendUnavailable, "{partial:?}");
13537                    assert!(partial.changed, "first copy was committed: {partial:?}");
13538                    assert!(partial.changes_indeterminate);
13539                    assert_ne!(tokio::fs::read(&paths[0]).await.expect("first committed copy"), original[0]);
13540                    for (path, bytes) in paths[1..].iter().zip(&original[1..]) {
13541                        assert_eq!(tokio::fs::read(path).await.expect("uncommitted copy"), *bytes);
13542                    }
13543                    assert!(
13544                        store.all_set_disks()[0]
13545                            .load_file_info_versions_exact(bucket, object)
13546                            .await
13547                            .is_err(),
13548                        "cleanup cannot select a partial repair subset"
13549                    );
13550                    let repaired = store
13551                        .reconcile_legacy_transition_state(request.clone())
13552                        .await
13553                        .expect("retry original snapshot");
13554                    assert_eq!(repaired.outcome, Outcome::Migrated, "{repaired:?}");
13555                    assert!(repaired.changed);
13556                    assert!(!repaired.changes_indeterminate);
13557                    let mut committed = Vec::new();
13558                    for (path, original) in paths.iter().zip(&original) {
13559                        let raw = tokio::fs::read(path).await.expect("repaired copy");
13560                        let metadata = FileMeta::load(&raw).expect("decode repaired copy");
13561                        let previous = FileMeta::load(original).expect("decode original copy");
13562                        assert_eq!(
13563                            metadata.transition_reconcile_generation(None).unwrap(),
13564                            previous.transition_reconcile_generation(None).unwrap()
13565                        );
13566                        let (_, version) = metadata.find_version(None).expect("selected version");
13567                        let info = version.into_fileinfo(bucket, object, true).expect("repaired FileInfo");
13568                        assert_eq!(info.transition_version_state, expected_state);
13569                        assert_eq!(info.transition_version, request.target.remote_version);
13570                        committed.push(raw);
13571                    }
13572                    assert!(
13573                        store.all_set_disks()[0]
13574                            .load_file_info_versions_exact(bucket, object)
13575                            .await
13576                            .expect("converged cleanup snapshot")
13577                            .is_some()
13578                    );
13579                    let replay = store
13580                        .reconcile_legacy_transition_state(request.clone())
13581                        .await
13582                        .expect("idempotent original request replay");
13583                    assert_eq!(replay.outcome, Outcome::Migrated, "{replay:?}");
13584                    assert!(!replay.changed);
13585                    for (path, expected) in paths.iter().zip(&committed) {
13586                        assert_eq!(
13587                            tokio::fs::read(path).await.expect("replayed copy"),
13588                            *expected,
13589                            "idempotence preserves raw encoding"
13590                        );
13591                    }
13592                    for (path, bytes) in paths.iter().zip(&original) {
13593                        tokio::fs::write(path, bytes)
13594                            .await
13595                            .expect("reset independent cancellation fixture");
13596                    }
13597                    let (entered_tx, entered) = tokio::sync::oneshot::channel();
13598                    let (release, released) = std::sync::mpsc::channel::<()>();
13599                    let _pause = crate::disk::os::prepared_publication_test_hooks::install_at(
13600                        crate::disk::os::prepared_publication_test_hooks::Stage::Rename,
13601                        &publication_path,
13602                        move || {
13603                            let _ = entered_tx.send(());
13604                            let _ = released.recv();
13605                        },
13606                    );
13607                    let mut repair = Box::pin(store.reconcile_legacy_transition_state(request.clone()));
13608                    tokio::select! {
13609                        result = &mut repair => panic!("repair completed before publication pause: {result:?}"),
13610                        result = entered => result.expect("publication executor entered"),
13611                    }
13612                    let update_options = crate::disk::UpdateMetadataOpts::default();
13613                    let mut update = Box::pin(first_disk.update_metadata(
13614                        bucket,
13615                        object,
13616                        FileInfo {
13617                            metadata: HashMap::from([("x-amz-meta-concurrent".to_string(), "kept".to_string())]),
13618                            ..Default::default()
13619                        },
13620                        &update_options,
13621                    ));
13622                    assert!(
13623                        tokio::time::timeout(std::time::Duration::from_millis(25), update.as_mut())
13624                            .await
13625                            .is_err(),
13626                        "another metadata RMW must wait for publication"
13627                    );
13628                    drop(repair);
13629                    assert!(
13630                        tokio::time::timeout(std::time::Duration::from_millis(25), update.as_mut())
13631                            .await
13632                            .is_err(),
13633                        "cancelling the coordinator must not release an in-flight disk mutation"
13634                    );
13635                    release.send(()).expect("resume owned publication");
13636                    update.await.expect("serialized metadata update");
13637                    let raw = tokio::fs::read(&paths[0])
13638                        .await
13639                        .expect("cancelled repair and later metadata update");
13640                    let (_, version) = FileMeta::load(&raw)
13641                        .expect("metadata after cancellation")
13642                        .find_version(None)
13643                        .expect("selected version");
13644                    let info = version
13645                        .into_fileinfo(bucket, object, true)
13646                        .expect("metadata after serialized update");
13647                    assert_eq!(info.transition_version_state, expected_state);
13648                    assert_eq!(info.metadata.get("x-amz-meta-concurrent").map(String::as_str), Some("kept"));
13649                    let stale = store
13650                        .reconcile_legacy_transition_state(request.clone())
13651                        .await
13652                        .expect("stale original request");
13653                    assert_eq!(
13654                        stale.outcome,
13655                        Outcome::Corrupt,
13656                        "unrelated metadata change invalidates the original generation: {stale:?}"
13657                    );
13658                    assert!(!stale.changed);
13659                    assert_eq!(tokio::fs::read(&paths[0]).await.expect("stale write leaves bytes unchanged"), raw);
13660                    assert_eq!(backend.remove_count().await, 0);
13661                    // A pinned version probe uses GET to verify that exact
13662                    // candidate; every backend operation still targets it.
13663                    let operations = backend.op_log().await;
13664                    assert!(
13665                        operations.iter().all(|operation| match operation {
13666                            MockWarmOp::Probe { object } | MockWarmOp::Get { object } => object == &request.source.remote_object,
13667                            _ => false,
13668                        }),
13669                        "unexpected backend effects: {operations:?}"
13670                    );
13671                })
13672                .await;
13673                continue;
13674            }
13675            let mut tampered = request.clone();
13676            tampered.source.remote_object.push_str("-other");
13677            let probes_before = backend.op_log().await.len();
13678            let rejected = store
13679                .reconcile_legacy_transition_state(tampered)
13680                .await
13681                .expect("reject tampered tuple");
13682            assert_eq!(rejected.outcome, Outcome::Corrupt);
13683            assert_eq!(backend.op_log().await.len(), probes_before, "invalid digest must not probe the backend");
13684            let applied = store
13685                .reconcile_legacy_transition_state(request)
13686                .await
13687                .expect("apply must report unavailable write authority");
13688            assert_eq!(applied.outcome, Outcome::BackendUnavailable, "{applied:?}");
13689            assert_eq!(applied.reason_code, "write_fence_unavailable");
13690            assert!(!applied.changed);
13691            for (path, expected) in paths.iter().zip(&original) {
13692                assert_eq!(tokio::fs::read(path).await.expect("xl.meta after inspection"), *expected);
13693            }
13694            assert_eq!(backend.remove_count().await, 0);
13695            assert!(
13696                backend
13697                    .op_log()
13698                    .await
13699                    .iter()
13700                    .all(|operation| matches!(operation, MockWarmOp::Probe { .. }))
13701            );
13702
13703            backend.set_unreachable(true).await;
13704            let unavailable = store
13705                .inspect_legacy_transition_state(LegacyTransitionStateReconcileSelector {
13706                    bucket: bucket.to_string(),
13707                    object: object.to_string(),
13708                    version_id: "null".to_string(),
13709                })
13710                .await
13711                .expect("unreachable tier is a diagnostic outcome");
13712            assert_eq!(unavailable.outcome, Outcome::BackendUnavailable);
13713            assert!(!unavailable.changed);
13714            backend.set_unreachable(false).await;
13715            for candidate in ["", "00000000-0000-0000-0000-000000000000", "bad\nversion"] {
13716                backend
13717                    .set_transition_candidate_probe_override(Some(
13718                        crate::services::tier::warm_backend::TransitionCandidateProbe::VersionedPresent(candidate.to_string()),
13719                    ))
13720                    .await;
13721                let invalid_proof = store
13722                    .inspect_legacy_transition_state(LegacyTransitionStateReconcileSelector {
13723                        bucket: bucket.to_string(),
13724                        object: object.to_string(),
13725                        version_id: "null".to_string(),
13726                    })
13727                    .await
13728                    .expect("invalid backend proof is a diagnostic outcome");
13729                assert_eq!(invalid_proof.outcome, Outcome::BackendUnavailable, "{invalid_proof:?}");
13730                assert!(invalid_proof.target.is_none());
13731            }
13732            backend.set_transition_candidate_probe_override(None).await;
13733
13734            backend.clear_op_log().await;
13735            for path in &paths[1..] {
13736                tokio::fs::remove_file(path)
13737                    .await
13738                    .expect("hide majority metadata copies in fixture");
13739            }
13740            let minority = store
13741                .inspect_legacy_transition_state(LegacyTransitionStateReconcileSelector {
13742                    bucket: bucket.to_string(),
13743                    object: object.to_string(),
13744                    version_id: "null".to_string(),
13745                })
13746                .await
13747                .expect("inspect minority legacy record");
13748            assert_eq!(
13749                minority.outcome,
13750                Outcome::BackendUnavailable,
13751                "a minority owner must remain visible: {minority:?}"
13752            );
13753            assert!(
13754                backend.op_log().await.is_empty(),
13755                "unproven metadata quorum cannot initiate a remote probe"
13756            );
13757            for (path, bytes) in paths.iter().zip(&original) {
13758                tokio::fs::write(path, bytes).await.expect("restore fixture copies");
13759            }
13760            tokio::fs::write(&paths[0], b"corrupt-xl-meta")
13761                .await
13762                .expect("inject corrupt copy");
13763            let corrupt = store
13764                .inspect_legacy_transition_state(LegacyTransitionStateReconcileSelector {
13765                    bucket: bucket.to_string(),
13766                    object: object.to_string(),
13767                    version_id: "null".to_string(),
13768                })
13769                .await
13770                .expect("inspect corrupt legacy record");
13771            assert_eq!(corrupt.outcome, Outcome::Corrupt, "{corrupt:?}");
13772            assert!(backend.op_log().await.is_empty(), "corruption must fail before backend I/O");
13773        }
13774    }
13775
13776    #[cfg(feature = "test-util")]
13777    #[tokio::test]
13778    #[serial_test::serial(storage_class_env)]
13779    async fn legacy_unknown_unversioned_transition_supports_head_get_and_range_without_backfill() {
13780        let temp_dir = tempfile::tempdir().expect("create legacy unknown unversioned store dir");
13781        let (ctx, store, _shutdown) =
13782            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "legacy-unknown-unversioned-read", &[4])).await;
13783        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
13784        let tier_name = "LEGACY-UNKNOWN-UNVERSIONED-READ";
13785        let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
13786        backend.set_put_remote_version(Some(String::new())).await;
13787        let bucket = "legacy-unknown-unversioned-read-bucket";
13788        let object = "object.bin";
13789        let payload = b"legacy unversioned remote tier object remains readable".repeat(1024);
13790        store
13791            .make_bucket(bucket, &MakeBucketOptions::default())
13792            .await
13793            .expect("legacy source bucket should be created");
13794        let mut reader = PutObjReader::from_vec(payload.clone());
13795        let source = store
13796            .put_object(bucket, object, &mut reader, &ObjectOptions::default())
13797            .await
13798            .expect("legacy source should be written");
13799        store
13800            .transition_object(
13801                bucket,
13802                object,
13803                &ObjectOptions {
13804                    transition: TransitionOptions {
13805                        status: TRANSITION_PENDING.to_string(),
13806                        tier: tier_name.to_string(),
13807                        etag: source.etag.clone().expect("legacy source should have an etag"),
13808                        ..Default::default()
13809                    },
13810                    mod_time: source.mod_time,
13811                    ..Default::default()
13812                },
13813            )
13814            .await
13815            .expect("legacy source should transition");
13816        rewrite_transitioned_xlmeta_as_legacy_unknown(temp_dir.path(), 0, bucket, object, true).await;
13817        backend.clear_op_log().await;
13818
13819        let opts = ObjectOptions {
13820            metadata_cache_safe: false,
13821            ..Default::default()
13822        };
13823        let head = store
13824            .get_object_info(bucket, object, &opts)
13825            .await
13826            .expect("legacy transitioned HEAD should use local metadata");
13827        assert_eq!(head.transition_version_state, rustfs_filemeta::TransitionVersionState::Unknown);
13828        assert!(head.transitioned_object.version_id.is_empty());
13829        assert_eq!(
13830            head.user_defined
13831                .get("x-minio-internal-transitioned-versionID")
13832                .map(String::as_str),
13833            Some(""),
13834            "the MinIO empty version-key provenance must survive xl.meta decoding"
13835        );
13836        assert!(
13837            !rustfs_utils::http::metadata_compat::contains_key_str(
13838                &head.user_defined,
13839                rustfs_utils::http::metadata_compat::SUFFIX_TRANSITIONED_VERSION_STATE,
13840            ),
13841            "the compatibility read must not synthesize version-state metadata"
13842        );
13843
13844        let full_body = read_store_body(&store, bucket, object, None, &opts).await;
13845        assert_eq!(full_body, payload);
13846
13847        let range = HTTPRangeSpec {
13848            is_suffix_length: false,
13849            start: 7,
13850            end: 38,
13851        };
13852        let ranged_body = read_store_body(&store, bucket, object, Some(range), &opts).await;
13853        assert_eq!(ranged_body, &payload[7..=38]);
13854
13855        let after_read = store.pools[0]
13856            .get_disks_by_key(object)
13857            .load_file_info_versions_exact(bucket, object)
13858            .await
13859            .expect("legacy metadata should remain readable after GET")
13860            .expect("legacy object metadata should remain on disk")
13861            .versions
13862            .into_iter()
13863            .find(|version| version.transition_status == rustfs_filemeta::TRANSITION_COMPLETE)
13864            .expect("legacy transitioned source should remain visible after GET");
13865        assert_eq!(after_read.transition_version_state, rustfs_filemeta::TransitionVersionState::Unknown);
13866        assert!(after_read.transition_version.is_none());
13867        assert!(after_read.transition_version_id.is_none());
13868        assert_eq!(
13869            after_read
13870                .metadata
13871                .get("x-minio-internal-transitioned-versionID")
13872                .map(String::as_str),
13873            Some(""),
13874            "the MinIO empty version-key provenance must remain after GET and Range GET"
13875        );
13876        assert!(
13877            !rustfs_utils::http::metadata_compat::contains_key_str(
13878                &after_read.metadata,
13879                rustfs_utils::http::metadata_compat::SUFFIX_TRANSITIONED_VERSION_STATE,
13880            ),
13881            "the compatibility read must remain side-effect free"
13882        );
13883
13884        assert_eq!(
13885            backend.op_log().await,
13886            vec![
13887                MockWarmOp::Probe {
13888                    object: after_read.transitioned_objname.clone(),
13889                },
13890                MockWarmOp::Get {
13891                    object: after_read.transitioned_objname.clone(),
13892                },
13893                MockWarmOp::Probe {
13894                    object: after_read.transitioned_objname.clone(),
13895                },
13896                MockWarmOp::Get {
13897                    object: after_read.transitioned_objname,
13898                },
13899            ],
13900            "legacy reads should probe before each unversioned GET and never mutate local metadata"
13901        );
13902        assert_eq!(backend.remove_count().await, 0);
13903    }
13904
13905    #[cfg(feature = "test-util")]
13906    #[tokio::test]
13907    #[serial_test::serial(storage_class_env)]
13908    async fn legacy_unknown_transition_delete_falls_back_for_single_batch_and_blocks_prefix() {
13909        let temp_dir = tempfile::tempdir().expect("create legacy transitioned delete store dir");
13910        let (ctx, store, _shutdown) =
13911            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "legacy-unknown-transitioned-delete", &[4]))
13912                .await;
13913        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
13914        let tier_name = "LEGACY-UNKNOWN-DELETE";
13915        let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
13916        let bucket = "legacy-unknown-transitioned-delete-bucket";
13917        store
13918            .make_bucket(bucket, &MakeBucketOptions::default())
13919            .await
13920            .expect("legacy source bucket should be created");
13921
13922        for (index, object) in ["single.bin", "batch.bin", "prefix/legacy.bin"].into_iter().enumerate() {
13923            let mut reader = PutObjReader::from_vec(vec![b'l' + index as u8; 1024 * 1024]);
13924            let source = store
13925                .put_object(bucket, object, &mut reader, &ObjectOptions::default())
13926                .await
13927                .expect("legacy source should be written");
13928            store
13929                .transition_object(
13930                    bucket,
13931                    object,
13932                    &ObjectOptions {
13933                        transition: TransitionOptions {
13934                            status: TRANSITION_PENDING.to_string(),
13935                            tier: tier_name.to_string(),
13936                            etag: source.etag.clone().expect("legacy source should have an etag"),
13937                            ..Default::default()
13938                        },
13939                        mod_time: source.mod_time,
13940                        ..Default::default()
13941                    },
13942                )
13943                .await
13944                .expect("legacy source should transition");
13945            rewrite_transitioned_xlmeta_as_legacy_unknown(temp_dir.path(), 0, bucket, object, false).await;
13946            let legacy = store.pools[0]
13947                .get_disks_by_key(object)
13948                .load_file_info_versions_exact(bucket, object)
13949                .await
13950                .expect("legacy source metadata should remain decodable")
13951                .expect("legacy source should remain on disk")
13952                .versions
13953                .into_iter()
13954                .find(|version| version.transition_status == rustfs_filemeta::TRANSITION_COMPLETE)
13955                .expect("legacy transitioned source should remain visible before delete");
13956            assert_eq!(legacy.transition_version_state, rustfs_filemeta::TransitionVersionState::Unknown);
13957            assert_eq!(
13958                crate::services::tier::tier::tier_destination_id_from_metadata(&legacy.metadata)
13959                    .expect("legacy metadata identity lookup should not fail"),
13960                None
13961            );
13962        }
13963
13964        backend.set_remove_failure(true);
13965        store
13966            .delete_object_with_tier_delete_journal(bucket, "single.bin", ObjectOptions::default())
13967            .await
13968            .expect("legacy single delete should retain its free-version fallback");
13969        let free_version_prefix_error = store
13970            .delete_object_with_tier_delete_journal(
13971                bucket,
13972                "single.bin",
13973                ObjectOptions {
13974                    delete_prefix: true,
13975                    ..Default::default()
13976                },
13977            )
13978            .await
13979            .expect_err("recursive prefix delete must not discard an existing hidden cleanup owner");
13980        assert!(
13981            free_version_prefix_error
13982                .to_string()
13983                .contains("tier free-version cleanup obligation"),
13984            "unexpected hidden-owner prefix failure: {free_version_prefix_error}"
13985        );
13986        let (_deleted, errors) = store
13987            .delete_objects_with_tier_delete_journal(
13988                bucket,
13989                vec![ObjectToDelete {
13990                    object_name: "batch.bin".to_string(),
13991                    ..Default::default()
13992                }],
13993                ObjectOptions::default(),
13994            )
13995            .await;
13996        assert!(errors.iter().all(Option::is_none), "legacy batch delete should succeed: {errors:?}");
13997        let prefix_error = store
13998            .delete_object_with_tier_delete_journal(
13999                bucket,
14000                "prefix/",
14001                ObjectOptions {
14002                    delete_prefix: true,
14003                    ..Default::default()
14004                },
14005            )
14006            .await
14007            .expect_err("legacy recursive prefix delete must fail closed before physical removal");
14008        assert!(
14009            prefix_error.to_string().contains("legacy Unknown"),
14010            "unexpected prefix failure: {prefix_error}"
14011        );
14012
14013        assert_eq!(
14014            tier_delete_journal_count(store.clone()).await,
14015            0,
14016            "legacy cleanup must not create an unsafe journal"
14017        );
14018        assert_eq!(
14019            backend.object_count().await,
14020            3,
14021            "failed free-version cleanup must retain all remote objects"
14022        );
14023        for object in ["single.bin", "batch.bin"] {
14024            let exact = store.pools[0]
14025                .get_disks_by_key(object)
14026                .load_file_info_versions_exact(bucket, object)
14027                .await
14028                .expect("legacy post-delete metadata should decode")
14029                .expect("legacy cleanup owner should remain on disk");
14030            assert_eq!(
14031                exact
14032                    .versions
14033                    .iter()
14034                    .chain(exact.free_versions.iter())
14035                    .filter(|version| version.tier_free_version())
14036                    .count(),
14037                1,
14038                "legacy {object} must retain exactly one hidden cleanup owner"
14039            );
14040            assert_eq!(
14041                exact.versions.iter().filter(|version| !version.tier_free_version()).count(),
14042                0,
14043                "legacy {object} must have no visible source after delete"
14044            );
14045        }
14046        let retained_prefix = store.pools[0]
14047            .get_disks_by_key("prefix/legacy.bin")
14048            .load_file_info_versions_exact(bucket, "prefix/legacy.bin")
14049            .await
14050            .expect("rejected prefix metadata should decode")
14051            .expect("rejected prefix delete must retain its source");
14052        assert_eq!(
14053            retained_prefix
14054                .versions
14055                .iter()
14056                .filter(|version| !version.tier_free_version())
14057                .count(),
14058            1
14059        );
14060    }
14061
14062    #[cfg(feature = "test-util")]
14063    #[tokio::test]
14064    #[serial_test::serial(storage_class_env)]
14065    async fn stable_transitioned_recursive_prefix_delete_uses_journal_owners() {
14066        let temp_dir = tempfile::tempdir().expect("create stable transitioned prefix-delete store dir");
14067        let (ctx, store, _shutdown) =
14068            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "stable-transitioned-prefix-delete", &[4]))
14069                .await;
14070        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
14071        let tier_name = "STABLE-PREFIX-DELETE";
14072        let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
14073        let bucket = "stable-transitioned-prefix-delete-bucket";
14074        store
14075            .make_bucket(bucket, &MakeBucketOptions::default())
14076            .await
14077            .expect("stable prefix source bucket should be created");
14078
14079        let objects = ["prefix/first.bin", "prefix/second.bin"];
14080        for (index, object) in objects.into_iter().enumerate() {
14081            let mut reader = PutObjReader::from_vec(vec![b'a' + index as u8; 1024 * 1024]);
14082            let source = store
14083                .put_object(bucket, object, &mut reader, &ObjectOptions::default())
14084                .await
14085                .expect("stable prefix source should be written");
14086            store
14087                .transition_object(
14088                    bucket,
14089                    object,
14090                    &ObjectOptions {
14091                        transition: TransitionOptions {
14092                            status: TRANSITION_PENDING.to_string(),
14093                            tier: tier_name.to_string(),
14094                            etag: source.etag.clone().expect("stable prefix source should have an etag"),
14095                            ..Default::default()
14096                        },
14097                        mod_time: source.mod_time,
14098                        ..Default::default()
14099                    },
14100                )
14101                .await
14102                .expect("stable prefix source should transition");
14103        }
14104        assert_eq!(backend.object_count().await, objects.len());
14105        backend.set_remove_failure(true);
14106
14107        store
14108            .delete_object_with_tier_delete_journal(
14109                bucket,
14110                "prefix/",
14111                ObjectOptions {
14112                    delete_prefix: true,
14113                    ..Default::default()
14114                },
14115            )
14116            .await
14117            .expect("stable recursive prefix delete should use journal ownership");
14118
14119        assert_eq!(tier_delete_journal_count(store.clone()).await, objects.len());
14120        assert_eq!(backend.object_count().await, objects.len());
14121        for object in objects {
14122            assert!(
14123                store.pools[0]
14124                    .get_disks_by_key(object)
14125                    .load_file_info_versions_exact(bucket, object)
14126                    .await
14127                    .expect("stable prefix metadata lookup should succeed")
14128                    .is_none(),
14129                "stable recursive delete must leave neither a visible source nor a free-version for {object}"
14130            );
14131        }
14132
14133        backend.set_remove_failure(false);
14134        let stats = recover_tier_delete_journal_entries(store.clone(), 100, None)
14135            .await
14136            .expect("stable prefix journals should recover");
14137        assert_eq!(stats.failed, 0);
14138        assert!(stats.scanned <= objects.len() && stats.deleted <= objects.len());
14139        assert_eq!(tier_delete_journal_count(store).await, 0);
14140        assert_eq!(backend.object_count().await, 0);
14141        assert_eq!(backend.remove_versions().await.len(), objects.len());
14142    }
14143
14144    #[cfg(feature = "test-util")]
14145    #[test]
14146    #[serial_test::serial(storage_class_env)]
14147    fn tier_delete_prefix_limit_and_multi_chunk_batches_converge() {
14148        run_large_stack_async_test(
14149            "tier-delete-prefix-limit-and-multi-chunk",
14150            tier_delete_prefix_limit_and_multi_chunk_batches_converge_case,
14151        );
14152    }
14153
14154    #[cfg(feature = "test-util")]
14155    async fn tier_delete_prefix_limit_and_multi_chunk_batches_converge_case() {
14156        let _batch_limit = TierDeleteDispatchBatchLimitGuard::install(2);
14157        let temp_dir = tempfile::tempdir().expect("create chunked prefix-delete store dir");
14158        let (ctx, store, shutdown) =
14159            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "chunked-prefix-delete", &[4])).await;
14160        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
14161        let tier_name = "CHUNKED-PREFIX-DELETE";
14162        let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
14163        let bucket = "chunked-prefix-delete-bucket";
14164        store
14165            .make_bucket(bucket, &MakeBucketOptions::default())
14166            .await
14167            .expect("chunked prefix source bucket should be created");
14168
14169        for (prefix, count) in [("at-limit", 2), ("limit-plus-one", 3), ("multi-chunk", 5)] {
14170            for index in 0..count {
14171                let object = format!("{prefix}/object-{index}.bin");
14172                let mut reader = PutObjReader::from_vec(vec![b'a' + index as u8; 1024 * 1024]);
14173                let source = store
14174                    .put_object(bucket, &object, &mut reader, &ObjectOptions::default())
14175                    .await
14176                    .expect("chunked prefix source should be written");
14177                store
14178                    .transition_object(
14179                        bucket,
14180                        &object,
14181                        &ObjectOptions {
14182                            transition: TransitionOptions {
14183                                status: TRANSITION_PENDING.to_string(),
14184                                tier: tier_name.to_string(),
14185                                etag: source.etag.clone().expect("chunked prefix source should have an etag"),
14186                                ..Default::default()
14187                            },
14188                            mod_time: source.mod_time,
14189                            ..Default::default()
14190                        },
14191                    )
14192                    .await
14193                    .expect("chunked prefix source should transition");
14194            }
14195        }
14196        backend.set_remove_failure(true);
14197
14198        store
14199            .delete_object_with_tier_delete_journal(
14200                bucket,
14201                "at-limit/",
14202                ObjectOptions {
14203                    delete_prefix: true,
14204                    ..Default::default()
14205                },
14206            )
14207            .await
14208            .expect("exactly one batch must retain the v1 one-shot path");
14209
14210        let limit_plus_one_first = store
14211            .delete_object_with_tier_delete_journal(
14212                bucket,
14213                "limit-plus-one/",
14214                ObjectOptions {
14215                    delete_prefix: true,
14216                    ..Default::default()
14217                },
14218            )
14219            .await
14220            .expect_err("limit plus one must start a bounded parent transaction");
14221        assert!(
14222            limit_plus_one_first.to_string().contains("retry the next durable batch"),
14223            "unexpected first limit-plus-one result: {limit_plus_one_first}"
14224        );
14225        let active_limit_plus_one_records = store
14226            .clone()
14227            .list_objects_v2(
14228                RUSTFS_META_BUCKET,
14229                TIER_DELETE_DISPATCH_MANIFEST_PREFIX,
14230                None,
14231                None,
14232                10,
14233                false,
14234                None,
14235                false,
14236            )
14237            .await
14238            .expect("active limit-plus-one records should be listable");
14239        let mut active_limit_plus_one_parent_seen = false;
14240        for record in active_limit_plus_one_records
14241            .objects
14242            .iter()
14243            .filter(|record| !record.name.contains("/chunks/"))
14244        {
14245            let data = com::read_config(store.clone(), &record.name)
14246                .await
14247                .expect("an active dispatch root should be readable");
14248            let value: serde_json::Value = serde_json::from_slice(&data).expect("an active dispatch root should contain JSON");
14249            if value["prefix"] == "limit-plus-one/" {
14250                assert_eq!(value["record_type"], "chunked_parent");
14251                active_limit_plus_one_parent_seen = true;
14252            }
14253        }
14254        assert!(
14255            active_limit_plus_one_parent_seen,
14256            "limit plus one must establish the fail-closed parent at the legacy root"
14257        );
14258
14259        let mut limit_plus_one_completed = false;
14260        let mut limit_plus_one_retries = 1;
14261        for _ in 0..3 {
14262            match store
14263                .delete_object_with_tier_delete_journal(
14264                    bucket,
14265                    "limit-plus-one/",
14266                    ObjectOptions {
14267                        delete_prefix: true,
14268                        ..Default::default()
14269                    },
14270                )
14271                .await
14272            {
14273                Ok(_) => {
14274                    limit_plus_one_completed = true;
14275                    break;
14276                }
14277                Err(err) if err.to_string().contains("retry the next durable batch") => {
14278                    limit_plus_one_retries += 1;
14279                }
14280                Err(err) => panic!("limit-plus-one delete returned an unexpected error: {err}"),
14281            }
14282        }
14283        assert!(limit_plus_one_completed, "limit plus one must converge through two bounded children");
14284        assert_eq!(limit_plus_one_retries, 2, "limit plus one must require exactly two child batches");
14285
14286        let first_batch = store
14287            .delete_object_with_tier_delete_journal(
14288                bucket,
14289                "multi-chunk/",
14290                ObjectOptions {
14291                    delete_prefix: true,
14292                    ..Default::default()
14293                },
14294            )
14295            .await
14296            .expect_err("the first bounded child must request a successor batch");
14297        assert!(
14298            first_batch.to_string().contains("retry the next durable batch"),
14299            "unexpected first child result: {first_batch}"
14300        );
14301
14302        let newcomer = "multi-chunk/zzz-newcomer.bin";
14303        let mut newcomer_reader = PutObjReader::from_vec(vec![b'n'; 1024 * 1024]);
14304        let newcomer_source = store
14305            .put_object(bucket, newcomer, &mut newcomer_reader, &ObjectOptions::default())
14306            .await
14307            .expect("a source created between chunks should be written");
14308        store
14309            .transition_object(
14310                bucket,
14311                newcomer,
14312                &ObjectOptions {
14313                    transition: TransitionOptions {
14314                        status: TRANSITION_PENDING.to_string(),
14315                        tier: tier_name.to_string(),
14316                        etag: newcomer_source
14317                            .etag
14318                            .clone()
14319                            .expect("the between-chunks source should have an etag"),
14320                        ..Default::default()
14321                    },
14322                    mod_time: newcomer_source.mod_time,
14323                    ..Default::default()
14324                },
14325            )
14326            .await
14327            .expect("a source created between chunks should transition");
14328
14329        let mut completed = false;
14330        let mut durable_batch_retries = 1;
14331        for _ in 0..7 {
14332            match store
14333                .delete_object_with_tier_delete_journal(
14334                    bucket,
14335                    "multi-chunk/",
14336                    ObjectOptions {
14337                        delete_prefix: true,
14338                        ..Default::default()
14339                    },
14340                )
14341                .await
14342            {
14343                Ok(_) => {
14344                    completed = true;
14345                    break;
14346                }
14347                Err(err)
14348                    if err.to_string().contains("retry the next durable batch")
14349                        || err.to_string().contains("retry the next batch") =>
14350                {
14351                    durable_batch_retries += 1;
14352                }
14353                Err(err) => panic!("chunked prefix delete returned an unexpected error: {err}"),
14354            }
14355        }
14356        assert!(completed, "six entries must converge through three bounded child batches");
14357        assert_eq!(durable_batch_retries, 3, "limit two should require exactly three child batches");
14358        assert_eq!(tier_delete_journal_count(store.clone()).await, 11);
14359        assert_eq!(
14360            backend.object_count().await,
14361            11,
14362            "remote cleanup must remain durable while the tier is unavailable"
14363        );
14364        let dispatch_records = store
14365            .clone()
14366            .list_objects_v2(
14367                RUSTFS_META_BUCKET,
14368                TIER_DELETE_DISPATCH_MANIFEST_PREFIX,
14369                None,
14370                None,
14371                20,
14372                false,
14373                None,
14374                false,
14375            )
14376            .await
14377            .expect("bounded dispatch records should be listable");
14378        let mut child_batches = 0;
14379        let mut legacy_at_limit_seen = false;
14380        for record in &dispatch_records.objects {
14381            let data = com::read_config(store.clone(), &record.name)
14382                .await
14383                .expect("a retained dispatch record should be readable");
14384            let value: serde_json::Value = serde_json::from_slice(&data).expect("a retained dispatch record should contain JSON");
14385            if record.name.contains("/chunks/") {
14386                let journal_count = value["journal_count"]
14387                    .as_u64()
14388                    .expect("a retained child should declare its journal count");
14389                assert!(journal_count <= 2, "a child batch exceeded the configured resource bound");
14390                child_batches += 1;
14391            } else if value["prefix"] == "at-limit/" {
14392                assert!(
14393                    value.get("record_type").is_none(),
14394                    "the exact-limit root must remain a legacy v1 manifest"
14395                );
14396                assert_eq!(value["journal_count"], 2);
14397                legacy_at_limit_seen = true;
14398            }
14399        }
14400        assert!(
14401            legacy_at_limit_seen,
14402            "the exact-limit dispatch must retain its byte-compatible root shape"
14403        );
14404        assert_eq!(child_batches, 5, "nine chunked sources should persist exactly five bounded children");
14405
14406        backend.set_remove_failure(false);
14407        drive_tier_delete_dispatch_restart_to_convergence(store.clone()).await;
14408        assert_eq!(tier_delete_journal_count(store.clone()).await, 0);
14409        assert_eq!(tier_delete_dispatch_manifest_count(store.clone()).await, 0);
14410        assert_eq!(backend.object_count().await, 0);
14411        assert_eq!(
14412            backend.remove_versions().await.len(),
14413            11,
14414            "each remote version must be removed exactly once"
14415        );
14416        shutdown.cancel();
14417    }
14418
14419    #[cfg(feature = "test-util")]
14420    #[test]
14421    #[serial_test::serial(storage_class_env)]
14422    fn tier_delete_chunk_crash_boundaries_resume_without_skipping_sources() {
14423        run_large_stack_async_test(
14424            "tier-delete-chunk-crash-boundaries",
14425            tier_delete_chunk_crash_boundaries_resume_without_skipping_sources_case,
14426        );
14427    }
14428
14429    #[cfg(feature = "test-util")]
14430    async fn tier_delete_chunk_crash_boundaries_resume_without_skipping_sources_case() {
14431        let _batch_limit = TierDeleteDispatchBatchLimitGuard::install(1);
14432        let stages = [
14433            TierDeleteChunkTestStage::ParentPersisted,
14434            TierDeleteChunkTestStage::ChildManifestPersisted,
14435            TierDeleteChunkTestStage::ParentBound,
14436            TierDeleteChunkTestStage::DispatchAuthorized,
14437            TierDeleteChunkTestStage::LocalReplayCompleted,
14438            TierDeleteChunkTestStage::ChildCompleted,
14439            TierDeleteChunkTestStage::ParentProgressed,
14440            TierDeleteChunkTestStage::FinalLocalDeletionCompleted,
14441            TierDeleteChunkTestStage::ParentCompleted,
14442        ];
14443        for (case, stage) in stages.into_iter().enumerate() {
14444            let temp_dir = tempfile::tempdir().expect("create chunk crash-boundary store dir");
14445            let initial_name = format!("chunk-crash-boundary-{case}");
14446            let (ctx, store, shutdown) =
14447                without_storage_class_env(build_isolated_test_store(temp_dir.path(), &initial_name, &[4])).await;
14448            crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
14449            let tier_name = format!("CHUNK-CRASH-BOUNDARY-{case}");
14450            let backend = register_mock_tier(&ctx.tier_config_mgr(), &tier_name).await;
14451            backend.set_remove_failure(true);
14452            let bucket = format!("chunk-crash-boundary-{case}-bucket");
14453            let prefix = "prefix/";
14454            store
14455                .make_bucket(&bucket, &MakeBucketOptions::default())
14456                .await
14457                .expect("chunk crash-boundary bucket should be created");
14458            for index in 0..2 {
14459                let object = format!("{prefix}object-{index}.bin");
14460                let mut reader = PutObjReader::from_vec(vec![b'a' + index as u8; 1024 * 1024]);
14461                let source = store
14462                    .put_object(&bucket, &object, &mut reader, &ObjectOptions::default())
14463                    .await
14464                    .expect("chunk crash-boundary source should be written");
14465                store
14466                    .transition_object(
14467                        &bucket,
14468                        &object,
14469                        &ObjectOptions {
14470                            transition: TransitionOptions {
14471                                status: TRANSITION_PENDING.to_string(),
14472                                tier: tier_name.clone(),
14473                                etag: source.etag.clone().expect("chunk crash-boundary source should have an etag"),
14474                                ..Default::default()
14475                            },
14476                            mod_time: source.mod_time,
14477                            ..Default::default()
14478                        },
14479                    )
14480                    .await
14481                    .expect("chunk crash-boundary source should transition");
14482            }
14483            let local_only = format!("{prefix}local-only.bin");
14484            let mut local_reader = PutObjReader::from_vec(vec![b'l'; 1024 * 1024]);
14485            store
14486                .put_object(&bucket, &local_only, &mut local_reader, &ObjectOptions::default())
14487                .await
14488                .expect("chunk crash-boundary local-only object should be written");
14489            let tier_config = ctx
14490                .tier_config_mgr()
14491                .read()
14492                .await
14493                .tiers
14494                .get(&tier_name)
14495                .expect("chunk crash-boundary tier config should remain available for restart")
14496                .clone_with_credentials();
14497
14498            if matches!(
14499                stage,
14500                TierDeleteChunkTestStage::FinalLocalDeletionCompleted | TierDeleteChunkTestStage::ParentCompleted
14501            ) {
14502                for _ in 0..2 {
14503                    let retry = store
14504                        .delete_object_with_tier_delete_journal(
14505                            &bucket,
14506                            prefix,
14507                            ObjectOptions {
14508                                delete_prefix: true,
14509                                ..Default::default()
14510                            },
14511                        )
14512                        .await
14513                        .expect_err("each bounded child must complete before the final crash boundary");
14514                    assert!(
14515                        retry.to_string().contains("retry the next durable batch"),
14516                        "unexpected bounded-child result before {stage:?}: {retry}"
14517                    );
14518                }
14519            }
14520
14521            let barrier = TierDeleteChunkTestBarrier::install(stage);
14522            let worker_store = store.clone();
14523            let worker_bucket = bucket.clone();
14524            let worker = tokio::spawn(async move {
14525                worker_store
14526                    .delete_object_with_tier_delete_journal(
14527                        &worker_bucket,
14528                        prefix,
14529                        ObjectOptions {
14530                            delete_prefix: true,
14531                            ..Default::default()
14532                        },
14533                    )
14534                    .await
14535            });
14536            tokio::time::timeout(Duration::from_secs(30), barrier.wait_until_paused())
14537                .await
14538                .unwrap_or_else(|_| panic!("chunk delete did not reach crash boundary {stage:?}"));
14539            worker.abort();
14540            let _ = worker.await;
14541            drop(barrier);
14542            let released_bucket_guard =
14543                tokio::time::timeout(Duration::from_secs(5), store.acquire_bucket_lifecycle_write_lock(&bucket))
14544                    .await
14545                    .unwrap_or_else(|_| panic!("canceling at {stage:?} did not release the bucket lifecycle lock"))
14546                    .unwrap_or_else(|err| {
14547                        panic!("bucket lifecycle lock reacquire failed after cancellation at {stage:?}: {err}")
14548                    });
14549            drop(released_bucket_guard);
14550            shutdown.cancel();
14551            drop(store);
14552            drop(ctx);
14553
14554            let restarted_name = format!("chunk-crash-boundary-{case}-restart");
14555            let (restarted_ctx, restarted_store, restarted_shutdown) =
14556                without_storage_class_env(build_isolated_test_store(temp_dir.path(), &restarted_name, &[4])).await;
14557            {
14558                let tier_config_mgr = restarted_ctx.tier_config_mgr();
14559                let mut manager = tier_config_mgr.write().await;
14560                manager.tiers.insert(tier_name.clone(), tier_config);
14561                manager
14562                    .install_test_driver(&tier_name, Box::new(backend.clone()))
14563                    .expect("the exact chunk crash-boundary tier driver should reinstall after restart");
14564            }
14565            crate::bucket::metadata_sys::init_bucket_metadata_sys(restarted_store.clone(), Vec::new()).await;
14566
14567            let mut completed = false;
14568            for _ in 0..20 {
14569                let recovery = recover_tier_delete_dispatch_manifests(restarted_store.clone(), 100, None)
14570                    .await
14571                    .expect("chunk crash-boundary manifest recovery should remain readable");
14572                assert_eq!(recovery.failed, 0, "recovery must not quarantine a valid chunk boundary");
14573                match restarted_store
14574                    .delete_object_with_tier_delete_journal(
14575                        &bucket,
14576                        prefix,
14577                        ObjectOptions {
14578                            delete_prefix: true,
14579                            ..Default::default()
14580                        },
14581                    )
14582                    .await
14583                {
14584                    Ok(_) => {
14585                        completed = true;
14586                        break;
14587                    }
14588                    Err(err)
14589                        if err.to_string().contains("retry")
14590                            || err.to_string().contains("rollback")
14591                            || err.to_string().contains("durable cleanup") => {}
14592                    Err(err) => panic!("chunk crash-boundary retry returned an unexpected error at {stage:?}: {err}"),
14593                }
14594            }
14595            assert!(completed, "chunk state must converge after cancellation at {stage:?}");
14596            for index in 0..2 {
14597                let object = format!("{prefix}object-{index}.bin");
14598                assert!(
14599                    restarted_store.pools[0]
14600                        .get_disks_by_key(&object)
14601                        .load_file_info_versions_exact(&bucket, &object)
14602                        .await
14603                        .expect("chunk crash-boundary source lookup should succeed")
14604                        .is_none(),
14605                    "source {object} must not be skipped after cancellation at {stage:?}"
14606                );
14607            }
14608            assert!(
14609                restarted_store.pools[0]
14610                    .get_disks_by_key(&local_only)
14611                    .load_file_info_versions_exact(&bucket, &local_only)
14612                    .await
14613                    .expect("chunk crash-boundary local-only source lookup should succeed")
14614                    .is_none(),
14615                "the final raw delete must remove the local-only source after cancellation at {stage:?}"
14616            );
14617            assert_eq!(backend.object_count().await, 2);
14618            backend.set_remove_failure(false);
14619            drive_tier_delete_dispatch_restart_to_convergence(restarted_store.clone()).await;
14620            assert_eq!(tier_delete_journal_count(restarted_store.clone()).await, 0);
14621            assert_eq!(tier_delete_dispatch_manifest_count(restarted_store.clone()).await, 0);
14622            assert_eq!(backend.object_count().await, 0);
14623            assert_eq!(
14624                backend.remove_versions().await.len(),
14625                2,
14626                "each crash case must retain exactly one cleanup owner per remote version"
14627            );
14628            restarted_shutdown.cancel();
14629        }
14630    }
14631
14632    #[cfg(feature = "test-util")]
14633    #[test]
14634    #[serial_test::serial(storage_class_env)]
14635    fn tier_delete_chunk_missing_child_with_journals_fails_closed() {
14636        run_large_stack_async_test(
14637            "tier-delete-chunk-missing-child",
14638            tier_delete_chunk_missing_child_with_journals_fails_closed_case,
14639        );
14640    }
14641
14642    #[cfg(feature = "test-util")]
14643    async fn tier_delete_chunk_missing_child_with_journals_fails_closed_case() {
14644        let _batch_limit = TierDeleteDispatchBatchLimitGuard::install(1);
14645        let temp_dir = tempfile::tempdir().expect("create missing-child store dir");
14646        let (ctx, store, shutdown) =
14647            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "chunk-missing-child", &[4])).await;
14648        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
14649        let tier_name = "CHUNK-MISSING-CHILD";
14650        let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
14651        backend.set_remove_failure(true);
14652        let bucket = "chunk-missing-child-bucket";
14653        let prefix = "prefix/";
14654        store
14655            .make_bucket(bucket, &MakeBucketOptions::default())
14656            .await
14657            .expect("missing-child bucket should be created");
14658        for index in 0..2 {
14659            let object = format!("{prefix}object-{index}.bin");
14660            let mut reader = PutObjReader::from_vec(vec![b'm' + index as u8; 1024 * 1024]);
14661            let source = store
14662                .put_object(bucket, &object, &mut reader, &ObjectOptions::default())
14663                .await
14664                .expect("missing-child source should be written");
14665            store
14666                .transition_object(
14667                    bucket,
14668                    &object,
14669                    &ObjectOptions {
14670                        transition: TransitionOptions {
14671                            status: TRANSITION_PENDING.to_string(),
14672                            tier: tier_name.to_string(),
14673                            etag: source.etag.clone().expect("missing-child source should have an etag"),
14674                            ..Default::default()
14675                        },
14676                        mod_time: source.mod_time,
14677                        ..Default::default()
14678                    },
14679                )
14680                .await
14681                .expect("missing-child source should transition");
14682        }
14683
14684        let barrier = TierDeleteChunkTestBarrier::install(TierDeleteChunkTestStage::DispatchAuthorized);
14685        let worker_store = store.clone();
14686        let worker = tokio::spawn(async move {
14687            worker_store
14688                .delete_object_with_tier_delete_journal(
14689                    bucket,
14690                    prefix,
14691                    ObjectOptions {
14692                        delete_prefix: true,
14693                        ..Default::default()
14694                    },
14695                )
14696                .await
14697        });
14698        tokio::time::timeout(Duration::from_secs(30), barrier.wait_until_paused())
14699            .await
14700            .expect("chunk delete should persist its authorization before corruption injection");
14701        worker.abort();
14702        let _ = worker.await;
14703        drop(barrier);
14704
14705        let records = store
14706            .clone()
14707            .list_objects_v2(
14708                RUSTFS_META_BUCKET,
14709                TIER_DELETE_DISPATCH_MANIFEST_PREFIX,
14710                None,
14711                None,
14712                10,
14713                false,
14714                None,
14715                false,
14716            )
14717            .await
14718            .expect("parent and child records should be listable");
14719        let child = records
14720            .objects
14721            .iter()
14722            .find(|object| object.name.contains("/chunks/"))
14723            .expect("the bound child manifest should exist")
14724            .name
14725            .clone();
14726        com::delete_config(store.clone(), &child)
14727            .await
14728            .expect("the test should remove only the child manifest");
14729        assert_eq!(tier_delete_journal_count(store.clone()).await, 1);
14730
14731        let error = store
14732            .delete_object_with_tier_delete_journal(
14733                bucket,
14734                prefix,
14735                ObjectOptions {
14736                    delete_prefix: true,
14737                    ..Default::default()
14738                },
14739            )
14740            .await
14741            .expect_err("a missing child with retained journals must quarantine the parent");
14742        assert!(
14743            error.to_string().contains("missing child with retained journals"),
14744            "unexpected missing-child result: {error}"
14745        );
14746        assert_eq!(backend.object_count().await, 2, "fail-closed inspection must not remove a remote version");
14747        shutdown.cancel();
14748    }
14749
14750    #[cfg(feature = "test-util")]
14751    #[test]
14752    #[serial_test::serial(storage_class_env)]
14753    fn authorized_prefix_retry_replays_predecessor_before_newcomer() {
14754        let handle = std::thread::Builder::new()
14755            .name("authorized-prefix-predecessor-replay-test".to_string())
14756            .stack_size(32 * 1024 * 1024)
14757            .spawn(|| {
14758                let runtime = tokio::runtime::Builder::new_multi_thread()
14759                    .enable_all()
14760                    .worker_threads(2)
14761                    .build()
14762                    .expect("authorized predecessor replay test runtime should build");
14763                runtime.block_on(authorized_prefix_retry_replays_predecessor_before_newcomer_case());
14764            })
14765            .expect("authorized predecessor replay test thread should spawn");
14766        if let Err(payload) = handle.join() {
14767            std::panic::resume_unwind(payload);
14768        }
14769    }
14770
14771    #[cfg(feature = "test-util")]
14772    async fn authorized_prefix_retry_replays_predecessor_before_newcomer_case() {
14773        let _batch_limit = TierDeleteDispatchBatchLimitGuard::install(1);
14774        let temp_dir = tempfile::tempdir().expect("create authorized predecessor replay store dir");
14775        let (ctx, store, _shutdown) =
14776            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "authorized-prefix-predecessor-replay", &[4]))
14777                .await;
14778        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
14779        let tier_name = "AUTHORIZED-PREFIX-REPLAY";
14780        let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
14781        let lease = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
14782            .await
14783            .expect("authorized replay tier lease should resolve");
14784        let bucket = "authorized-prefix-predecessor-replay-bucket";
14785        let prefix = "prefix/";
14786        let predecessor = "prefix/000-predecessor.bin";
14787        let newcomer = "prefix/zzz-newcomer.bin";
14788        store
14789            .make_bucket(bucket, &MakeBucketOptions::default())
14790            .await
14791            .expect("authorized replay bucket should be created");
14792
14793        let mut predecessor_reader = PutObjReader::from_vec(vec![b'p'; 1024 * 1024]);
14794        let predecessor_source = store
14795            .put_object(bucket, predecessor, &mut predecessor_reader, &ObjectOptions::default())
14796            .await
14797            .expect("predecessor source should be written");
14798        store
14799            .transition_object(
14800                bucket,
14801                predecessor,
14802                &ObjectOptions {
14803                    transition: TransitionOptions {
14804                        status: TRANSITION_PENDING.to_string(),
14805                        tier: tier_name.to_string(),
14806                        etag: predecessor_source.etag.clone().expect("predecessor should have an etag"),
14807                        ..Default::default()
14808                    },
14809                    mod_time: predecessor_source.mod_time,
14810                    ..Default::default()
14811                },
14812            )
14813            .await
14814            .expect("predecessor should transition");
14815        let predecessor_info = store
14816            .get_object_info(bucket, predecessor, &ObjectOptions::default())
14817            .await
14818            .expect("transitioned predecessor should be readable");
14819        let mut predecessor_entry =
14820            transitioned_delete_journal_entry_for_source(None, false, false, bucket, predecessor, &predecessor_info)
14821                .expect("transitioned predecessor should produce a dispatch journal");
14822        predecessor_entry.backend_identity = Some(lease.backend_identity());
14823        let incarnation = store
14824            .bucket_incarnation_id(bucket)
14825            .await
14826            .expect("authorized replay bucket incarnation should resolve");
14827        let (manifest_name, _) = install_test_tier_delete_dispatch_fixture(
14828            store.clone(),
14829            bucket,
14830            incarnation,
14831            prefix,
14832            vec![(predecessor_entry, Some(TierDeleteJournalState::Dispatched))],
14833            TierDeleteDispatchManifestState::DispatchAuthorized,
14834        )
14835        .await
14836        .expect("authorized predecessor fixture should persist");
14837
14838        let mut newcomer_reader = PutObjReader::from_vec(vec![b'n'; 1024 * 1024]);
14839        let newcomer_source = store
14840            .put_object(bucket, newcomer, &mut newcomer_reader, &ObjectOptions::default())
14841            .await
14842            .expect("newcomer source should be written after authorization");
14843        store
14844            .transition_object(
14845                bucket,
14846                newcomer,
14847                &ObjectOptions {
14848                    transition: TransitionOptions {
14849                        status: TRANSITION_PENDING.to_string(),
14850                        tier: tier_name.to_string(),
14851                        etag: newcomer_source.etag.clone().expect("newcomer should have an etag"),
14852                        ..Default::default()
14853                    },
14854                    mod_time: newcomer_source.mod_time,
14855                    ..Default::default()
14856                },
14857            )
14858            .await
14859            .expect("newcomer should transition");
14860        backend.set_remove_failure(true);
14861
14862        let publication_epoch = store
14863            .scanner_data_usage_publication_epoch()
14864            .await
14865            .expect("authorized replay should observe the current scanner publication epoch");
14866        let publication_scope = store
14867            .scanner_data_usage_publication_commit_scope(
14868                publication_epoch,
14869                tokio::time::Instant::now() + Duration::from_secs(30),
14870                Vec::new(),
14871            )
14872            .await
14873            .expect("authorized replay should acquire a scanner publication scope");
14874
14875        let retry = store
14876            .delete_object_with_tier_delete_journal(
14877                bucket,
14878                prefix,
14879                ObjectOptions {
14880                    delete_prefix: true,
14881                    scanner_publication_commit_scope: Some(publication_scope.clone()),
14882                    ..Default::default()
14883                },
14884            )
14885            .await
14886            .expect_err("the predecessor-only replay must request a successor retry");
14887        assert!(
14888            retry.to_string().contains("retry the successor dispatch"),
14889            "unexpected predecessor replay result: {retry}"
14890        );
14891        assert_eq!(
14892            publication_scope.state(),
14893            crate::object_api::ScannerPublicationCommitState::Committed,
14894            "an exact predecessor replay must report its local mutation as committed"
14895        );
14896        assert!(publication_scope.release_movement_permit().await);
14897        assert!(
14898            store.pools[0]
14899                .get_disks_by_key(predecessor)
14900                .load_file_info_versions_exact(bucket, predecessor)
14901                .await
14902                .expect("predecessor metadata should remain readable")
14903                .is_none(),
14904            "the authorized predecessor must be removed exactly"
14905        );
14906        assert!(
14907            store.pools[0]
14908                .get_disks_by_key(newcomer)
14909                .load_file_info_versions_exact(bucket, newcomer)
14910                .await
14911                .expect("newcomer metadata should remain readable")
14912                .is_some(),
14913            "the predecessor authorization must not delete the newcomer"
14914        );
14915        assert_eq!(
14916            test_tier_delete_dispatch_manifest_state(store.clone(), &manifest_name)
14917                .await
14918                .expect("completed predecessor manifest should remain readable"),
14919            Some(TierDeleteDispatchManifestState::Completed)
14920        );
14921
14922        backend.set_remove_failure(false);
14923        let predecessor_journals = recover_tier_delete_journal_entries(store.clone(), 100, None)
14924            .await
14925            .expect("completed predecessor journal should recover");
14926        assert_eq!(predecessor_journals.failed, 0);
14927        assert!(predecessor_journals.scanned <= 1 && predecessor_journals.deleted <= 1);
14928        assert_eq!(tier_delete_journal_count(store.clone()).await, 0);
14929        assert_eq!(backend.object_count().await, 1, "only the newcomer remote object should remain");
14930        assert_eq!(backend.remove_versions().await.len(), 1);
14931        let predecessor_manifest = recover_tier_delete_dispatch_manifests(store.clone(), 100, None)
14932            .await
14933            .expect("completed predecessor manifest should be collected");
14934        assert_eq!(predecessor_manifest.failed, 0);
14935        assert!(predecessor_manifest.deleted <= 1);
14936        assert_eq!(tier_delete_dispatch_manifest_count(store.clone()).await, 0);
14937
14938        store
14939            .delete_object_with_tier_delete_journal(
14940                bucket,
14941                prefix,
14942                ObjectOptions {
14943                    delete_prefix: true,
14944                    ..Default::default()
14945                },
14946            )
14947            .await
14948            .expect("a successor dispatch should delete the remaining newcomer");
14949        assert!(
14950            store.pools[0]
14951                .get_disks_by_key(newcomer)
14952                .load_file_info_versions_exact(bucket, newcomer)
14953                .await
14954                .expect("newcomer metadata lookup should succeed after successor")
14955                .is_none()
14956        );
14957        drive_tier_delete_dispatch_restart_to_convergence(store.clone()).await;
14958        assert_eq!(tier_delete_journal_count(store.clone()).await, 0);
14959        assert_eq!(tier_delete_dispatch_manifest_count(store.clone()).await, 0);
14960        assert_eq!(backend.object_count().await, 0);
14961        assert_eq!(backend.remove_versions().await.len(), 2);
14962    }
14963
14964    #[cfg(feature = "test-util")]
14965    #[tokio::test]
14966    #[serial_test::serial(storage_class_env)]
14967    async fn multi_pool_same_tuple_recursive_prefix_uses_one_journal_owner() {
14968        let temp_dir = tempfile::tempdir().expect("create shared-tuple prefix-delete store dir");
14969        let (ctx, store, _shutdown) = without_storage_class_env(build_isolated_test_store(
14970            temp_dir.path(),
14971            "multi-pool-shared-tuple-prefix-delete",
14972            &[4, 4],
14973        ))
14974        .await;
14975        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
14976        let tier_name = "SHARED-PREFIX-DELETE";
14977        let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
14978        let bucket = "multi-pool-shared-tuple-prefix-delete-bucket";
14979        let object = "prefix/shared.bin";
14980        store
14981            .make_bucket(bucket, &MakeBucketOptions::default())
14982            .await
14983            .expect("shared-tuple prefix bucket should be created");
14984        let mut reader = PutObjReader::from_vec(vec![b's'; 1024 * 1024]);
14985        let source = store.pools[0]
14986            .put_object(bucket, object, &mut reader, &ObjectOptions::default())
14987            .await
14988            .expect("shared-tuple prefix source should be written");
14989        store.pools[0]
14990            .transition_object(
14991                bucket,
14992                object,
14993                &ObjectOptions {
14994                    transition: TransitionOptions {
14995                        status: TRANSITION_PENDING.to_string(),
14996                        tier: tier_name.to_string(),
14997                        etag: source.etag.clone().expect("shared-tuple prefix source should have an etag"),
14998                        ..Default::default()
14999                    },
15000                    mod_time: source.mod_time,
15001                    ..Default::default()
15002                },
15003            )
15004            .await
15005            .expect("shared-tuple prefix source should transition");
15006        copy_test_xlmeta_between_pools(temp_dir.path(), 0, 1, bucket, object).await;
15007        assert_eq!(backend.object_count().await, 1);
15008        backend.set_remove_failure(true);
15009
15010        store
15011            .delete_object_with_tier_delete_journal(
15012                bucket,
15013                "prefix/",
15014                ObjectOptions {
15015                    delete_prefix: true,
15016                    ..Default::default()
15017                },
15018            )
15019            .await
15020            .expect("shared-tuple recursive prefix delete should commit");
15021
15022        assert_eq!(tier_delete_journal_count(store.clone()).await, 1);
15023        for pool_idx in 0..2 {
15024            assert!(
15025                store.pools[pool_idx]
15026                    .get_disks_by_key(object)
15027                    .load_file_info_versions_exact(bucket, object)
15028                    .await
15029                    .expect("shared-tuple post-delete metadata should decode")
15030                    .is_none(),
15031                "pool {pool_idx} must be physically empty"
15032            );
15033        }
15034        backend.set_remove_failure(false);
15035        ctx.wake_tier_delete_journal_recovery();
15036        wait_for_tier_delete_journal_recovery(store.clone(), &backend, 1).await;
15037        assert_eq!(backend.object_count().await, 0);
15038        assert_eq!(backend.remove_versions().await.len(), 1);
15039    }
15040
15041    #[cfg(feature = "test-util")]
15042    #[tokio::test]
15043    #[serial_test::serial(storage_class_env)]
15044    async fn multi_pool_recursive_prefix_rejects_legacy_or_hidden_merge_loser_before_delete() {
15045        let temp_dir = tempfile::tempdir().expect("create merge-loser prefix-delete store dir");
15046        let (ctx, store, _shutdown) = without_storage_class_env(build_isolated_test_store(
15047            temp_dir.path(),
15048            "multi-pool-merge-loser-prefix-delete",
15049            &[4, 4],
15050        ))
15051        .await;
15052        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
15053        let tier_name = "MERGE-LOSER-PREFIX-DELETE";
15054        let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
15055        let bucket = "multi-pool-merge-loser-prefix-delete-bucket";
15056        store
15057            .make_bucket(bucket, &MakeBucketOptions::default())
15058            .await
15059            .expect("merge-loser prefix bucket should be created");
15060
15061        for (object, byte) in [("legacy/item.bin", b'l'), ("hidden/item.bin", b'h')] {
15062            let mut reader = PutObjReader::from_vec(vec![byte; 1024 * 1024]);
15063            let source = store.pools[0]
15064                .put_object(bucket, object, &mut reader, &ObjectOptions::default())
15065                .await
15066                .expect("merge-loser source should be written");
15067            store.pools[0]
15068                .transition_object(
15069                    bucket,
15070                    object,
15071                    &ObjectOptions {
15072                        transition: TransitionOptions {
15073                            status: TRANSITION_PENDING.to_string(),
15074                            tier: tier_name.to_string(),
15075                            etag: source.etag.clone().expect("merge-loser source should have an etag"),
15076                            ..Default::default()
15077                        },
15078                        mod_time: source.mod_time,
15079                        ..Default::default()
15080                    },
15081                )
15082                .await
15083                .expect("merge-loser source should transition");
15084            copy_test_xlmeta_between_pools(temp_dir.path(), 0, 1, bucket, object).await;
15085        }
15086        rewrite_transitioned_xlmeta_as_legacy_unknown(temp_dir.path(), 1, bucket, "legacy/item.bin", false).await;
15087        backend.set_remove_failure(true);
15088        store.pools[1]
15089            .delete_object(bucket, "hidden/item.bin", ObjectOptions::default())
15090            .await
15091            .expect("second-pool source delete should retain a hidden free-version");
15092
15093        let legacy_error = store
15094            .delete_object_with_tier_delete_journal(
15095                bucket,
15096                "legacy/",
15097                ObjectOptions {
15098                    delete_prefix: true,
15099                    ..Default::default()
15100                },
15101            )
15102            .await
15103            .expect_err("a legacy merge loser must reject recursive deletion");
15104        assert!(legacy_error.to_string().contains("legacy Unknown"), "unexpected error: {legacy_error}");
15105        let hidden_error = store
15106            .delete_object_with_tier_delete_journal(
15107                bucket,
15108                "hidden/",
15109                ObjectOptions {
15110                    delete_prefix: true,
15111                    ..Default::default()
15112                },
15113            )
15114            .await
15115            .expect_err("a hidden free-version merge loser must reject recursive deletion");
15116        assert!(
15117            hidden_error.to_string().contains("free-version cleanup obligation"),
15118            "unexpected error: {hidden_error}"
15119        );
15120
15121        assert_eq!(tier_delete_journal_count(store.clone()).await, 0);
15122        assert_eq!(backend.object_count().await, 2);
15123        assert_eq!(backend.remove_count().await, 0);
15124        for object in ["legacy/item.bin", "hidden/item.bin"] {
15125            let pool0 = store.pools[0]
15126                .get_disks_by_key(object)
15127                .load_file_info_versions_exact(bucket, object)
15128                .await
15129                .expect("winner metadata should decode")
15130                .expect("winner source must remain before destructive deletion");
15131            assert_eq!(pool0.versions.iter().filter(|version| !version.tier_free_version()).count(), 1);
15132        }
15133        let hidden_pool1 = store.pools[1]
15134            .get_disks_by_key("hidden/item.bin")
15135            .load_file_info_versions_exact(bucket, "hidden/item.bin")
15136            .await
15137            .expect("hidden loser metadata should decode")
15138            .expect("hidden loser cleanup owner must remain");
15139        assert_eq!(
15140            hidden_pool1
15141                .versions
15142                .iter()
15143                .chain(hidden_pool1.free_versions.iter())
15144                .filter(|version| version.tier_free_version())
15145                .count(),
15146            1
15147        );
15148    }
15149
15150    #[cfg(feature = "test-util")]
15151    #[tokio::test]
15152    #[serial_test::serial(storage_class_env)]
15153    async fn recursive_prefix_partial_pool_failure_keeps_prepared_cleanup_owners() {
15154        let temp_dir = tempfile::tempdir().expect("create partial prefix-delete store dir");
15155        let (ctx, store, _shutdown) =
15156            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "partial-pool-prefix-delete", &[4, 4])).await;
15157        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
15158        let tier_name = "PARTIAL-PREFIX-DELETE";
15159        let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
15160        let bucket = "partial-pool-prefix-delete-bucket";
15161        let object = "prefix/item.bin";
15162        store
15163            .make_bucket(bucket, &MakeBucketOptions::default())
15164            .await
15165            .expect("partial prefix bucket should be created");
15166        for (pool_idx, byte) in [(0, b'a'), (1, b'b')] {
15167            let mut reader = PutObjReader::from_vec(vec![byte; 1024 * 1024]);
15168            let source = store.pools[pool_idx]
15169                .put_object(bucket, object, &mut reader, &ObjectOptions::default())
15170                .await
15171                .expect("pool-local prefix source should be written");
15172            store.pools[pool_idx]
15173                .transition_object(
15174                    bucket,
15175                    object,
15176                    &ObjectOptions {
15177                        transition: TransitionOptions {
15178                            status: TRANSITION_PENDING.to_string(),
15179                            tier: tier_name.to_string(),
15180                            etag: source.etag.clone().expect("pool-local source should have an etag"),
15181                            ..Default::default()
15182                        },
15183                        mod_time: source.mod_time,
15184                        ..Default::default()
15185                    },
15186                )
15187                .await
15188                .expect("pool-local prefix source should transition");
15189        }
15190        assert_eq!(backend.object_count().await, 2);
15191        backend.set_remove_failure(true);
15192
15193        let barrier = crate::store::object::PrefixDeleteAfterJournalPrepareBarrier::install(bucket, "prefix/");
15194        let delete_store = store.clone();
15195        let delete = tokio::spawn(async move {
15196            delete_store
15197                .delete_object_with_tier_delete_journal(
15198                    bucket,
15199                    "prefix/",
15200                    ObjectOptions {
15201                        delete_prefix: true,
15202                        ..Default::default()
15203                    },
15204                )
15205                .await
15206        });
15207        tokio::time::timeout(Duration::from_secs(10), barrier.wait_until_paused())
15208            .await
15209            .expect("prefix delete should pause after every journal is Prepared");
15210        assert_eq!(tier_delete_journal_count(store.clone()).await, 2);
15211
15212        let failed_set = store.pools[1].get_disks_by_key(object);
15213        let offline_disks = force_set_disks_offline_for_test(&failed_set).await;
15214        barrier.release();
15215        let error = delete
15216            .await
15217            .expect("partial prefix delete task should join")
15218            .expect_err("an offline second pool must fail the recursive prefix delete");
15219        assert!(
15220            matches!(
15221                error,
15222                StorageError::InsufficientWriteQuorum(_, _) | StorageError::InsufficientReadQuorum(_, _)
15223            ),
15224            "unexpected error: {error:?}"
15225        );
15226        drop(offline_disks);
15227        drop(barrier);
15228
15229        assert_eq!(
15230            tier_delete_journal_count(store.clone()).await,
15231            2,
15232            "an ambiguous partial mutation must retain every Dispatched owner"
15233        );
15234        let first_pool_state = store.pools[0]
15235            .get_disks_by_key(object)
15236            .load_file_info_versions_exact(bucket, object)
15237            .await
15238            .expect("first-pool metadata should remain decodable after an ambiguous failure");
15239        if let Some(first_pool_state) = first_pool_state {
15240            assert_eq!(
15241                first_pool_state
15242                    .versions
15243                    .iter()
15244                    .filter(|version| !version.tier_free_version())
15245                    .count(),
15246                1,
15247                "a retained first-pool source must remain a visible source, not an unjournaled fallback owner"
15248            );
15249        }
15250        assert!(
15251            store.pools[1]
15252                .get_disks_by_key(object)
15253                .load_file_info_versions_exact(bucket, object)
15254                .await
15255                .expect("failed-pool metadata should decode")
15256                .is_some(),
15257            "the failed pool should retain its live source"
15258        );
15259
15260        let _ = recover_tier_delete_journal_entries(store.clone(), 100, None)
15261            .await
15262            .expect("Dispatched owners should reconcile after the failed pool returns");
15263        assert_eq!(
15264            tier_delete_journal_count(store.clone()).await,
15265            2,
15266            "recovery must never abort or delete a Dispatched journal while any source is still live"
15267        );
15268        store
15269            .delete_object_with_tier_delete_journal(
15270                bucket,
15271                "prefix/",
15272                ObjectOptions {
15273                    delete_prefix: true,
15274                    ..Default::default()
15275                },
15276            )
15277            .await
15278            .expect("retry should delete the remaining pool source");
15279        backend.set_remove_failure(false);
15280        for _ in 0..3 {
15281            let _ = recover_tier_delete_journal_entries(store.clone(), 100, None)
15282                .await
15283                .expect("remaining pool journals should settle after backend recovery");
15284            if tier_delete_journal_count(store.clone()).await == 0 {
15285                break;
15286            }
15287        }
15288        assert_eq!(tier_delete_journal_count(store.clone()).await, 0);
15289        assert_eq!(backend.object_count().await, 0);
15290        assert_eq!(backend.remove_versions().await.len(), 2);
15291    }
15292
15293    #[cfg(feature = "test-util")]
15294    #[tokio::test]
15295    #[serial_test::serial(storage_class_env)]
15296    async fn recursive_prefix_partial_set_failure_keeps_prepared_cleanup_owners() {
15297        let temp_dir = tempfile::tempdir().expect("create partial-set prefix-delete store dir");
15298        let (ctx, store, _shutdown) = without_storage_class_env(build_isolated_test_store_with_layout(
15299            temp_dir.path(),
15300            "partial-set-prefix-delete",
15301            &[(2, 4)],
15302            CancellationToken::new(),
15303            None,
15304        ))
15305        .await;
15306        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
15307        let tier_name = "PARTIAL-SET-PREFIX-DELETE";
15308        let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
15309        let bucket = "partial-set-prefix-delete-bucket";
15310        store
15311            .make_bucket(bucket, &MakeBucketOptions::default())
15312            .await
15313            .expect("partial-set prefix bucket should be created");
15314
15315        let objects = (0..2)
15316            .map(|target_set| {
15317                (0..512)
15318                    .map(|index| format!("prefix/set-{target_set}-{index}.bin"))
15319                    .find(|candidate| store.pools[0].get_disks_by_key(candidate).set_index == target_set)
15320                    .expect("a deterministic key should map to each test set")
15321            })
15322            .collect::<Vec<_>>();
15323        for (index, object) in objects.iter().enumerate() {
15324            let mut reader = PutObjReader::from_vec(vec![b'0' + index as u8; 1024 * 1024]);
15325            let source = store.pools[0]
15326                .put_object(bucket, object, &mut reader, &ObjectOptions::default())
15327                .await
15328                .expect("set-local prefix source should be written");
15329            store.pools[0]
15330                .transition_object(
15331                    bucket,
15332                    object,
15333                    &ObjectOptions {
15334                        transition: TransitionOptions {
15335                            status: TRANSITION_PENDING.to_string(),
15336                            tier: tier_name.to_string(),
15337                            etag: source.etag.clone().expect("set-local source should have an etag"),
15338                            ..Default::default()
15339                        },
15340                        mod_time: source.mod_time,
15341                        ..Default::default()
15342                    },
15343                )
15344                .await
15345                .expect("set-local prefix source should transition");
15346        }
15347        assert_eq!(backend.object_count().await, 2);
15348        backend.set_remove_failure(true);
15349
15350        let barrier = crate::store::object::PrefixDeleteAfterJournalPrepareBarrier::install(bucket, "prefix/");
15351        let delete_store = store.clone();
15352        let delete = tokio::spawn(async move {
15353            delete_store
15354                .delete_object_with_tier_delete_journal(
15355                    bucket,
15356                    "prefix/",
15357                    ObjectOptions {
15358                        delete_prefix: true,
15359                        ..Default::default()
15360                    },
15361                )
15362                .await
15363        });
15364        tokio::time::timeout(Duration::from_secs(10), barrier.wait_until_paused())
15365            .await
15366            .expect("set-scoped prefix delete should pause after journal Prepare");
15367        assert_eq!(tier_delete_journal_count(store.clone()).await, 2);
15368
15369        let failed_set = store.pools[0].disk_set[1].clone();
15370        let offline_disks = force_set_disks_offline_for_test(&failed_set).await;
15371        barrier.release();
15372        let error = delete
15373            .await
15374            .expect("partial-set prefix delete task should join")
15375            .expect_err("an offline second set must fail recursive prefix deletion");
15376        assert!(
15377            matches!(
15378                error,
15379                StorageError::InsufficientWriteQuorum(_, _) | StorageError::InsufficientReadQuorum(_, _)
15380            ),
15381            "unexpected error: {error:?}"
15382        );
15383        drop(offline_disks);
15384        drop(barrier);
15385
15386        assert_eq!(tier_delete_journal_count(store.clone()).await, 2);
15387        let first_set_state = store.pools[0]
15388            .get_disks_by_key(&objects[0])
15389            .load_file_info_versions_exact(bucket, &objects[0])
15390            .await
15391            .expect("first-set metadata should remain decodable after an ambiguous failure");
15392        if let Some(first_set_state) = first_set_state {
15393            assert_eq!(
15394                first_set_state
15395                    .versions
15396                    .iter()
15397                    .filter(|version| !version.tier_free_version())
15398                    .count(),
15399                1,
15400                "a retained first-set source must remain visible under its Dispatched journal"
15401            );
15402        }
15403        assert!(
15404            store.pools[0]
15405                .get_disks_by_key(&objects[1])
15406                .load_file_info_versions_exact(bucket, &objects[1])
15407                .await
15408                .expect("failed-set metadata should decode")
15409                .is_some()
15410        );
15411
15412        let _ = recover_tier_delete_journal_entries(store.clone(), 100, None)
15413            .await
15414            .expect("set-scoped Dispatched owners should reconcile");
15415        assert_eq!(
15416            tier_delete_journal_count(store.clone()).await,
15417            2,
15418            "recovery must retain the complete Authorized dispatch until every source is absent"
15419        );
15420        store
15421            .delete_object_with_tier_delete_journal(
15422                bucket,
15423                "prefix/",
15424                ObjectOptions {
15425                    delete_prefix: true,
15426                    ..Default::default()
15427                },
15428            )
15429            .await
15430            .expect("set-scoped retry should delete the remaining source");
15431        backend.set_remove_failure(false);
15432        for _ in 0..3 {
15433            let _ = recover_tier_delete_journal_entries(store.clone(), 100, None)
15434                .await
15435                .expect("remaining set journals should settle after backend recovery");
15436            if tier_delete_journal_count(store.clone()).await == 0 {
15437                break;
15438            }
15439        }
15440        assert_eq!(tier_delete_journal_count(store.clone()).await, 0);
15441        assert_eq!(backend.object_count().await, 0);
15442        assert_eq!(backend.remove_versions().await.len(), 2);
15443    }
15444
15445    #[cfg(feature = "test-util")]
15446    #[tokio::test]
15447    #[serial_test::serial(storage_class_env)]
15448    async fn versioned_explicit_transition_delete_preserves_other_version_then_allows_bucket_delete() {
15449        let temp_dir = tempfile::tempdir().expect("create versioned transitioned delete store dir");
15450        let (ctx, store, _shutdown) =
15451            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "versioned-transitioned-delete-owner", &[4]))
15452                .await;
15453        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
15454        let tier_name = "VERSIONED-DELETE";
15455        let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
15456        let bucket = "versioned-transitioned-delete-owner-bucket";
15457        let object = "archive.bin";
15458        store
15459            .make_bucket(bucket, &MakeBucketOptions::default())
15460            .await
15461            .expect("versioned source bucket should be created");
15462        crate::bucket::metadata_sys::update_in(
15463            &ctx,
15464            bucket,
15465            BUCKET_VERSIONING_CONFIG,
15466            b"<VersioningConfiguration><Status>Enabled</Status></VersioningConfiguration>".to_vec(),
15467        )
15468        .await
15469        .expect("bucket versioning should be enabled");
15470
15471        let mut first_reader = PutObjReader::from_vec(vec![b'1'; 1024 * 1024]);
15472        let first = store
15473            .put_object(
15474                bucket,
15475                object,
15476                &mut first_reader,
15477                &ObjectOptions {
15478                    versioned: true,
15479                    ..Default::default()
15480                },
15481            )
15482            .await
15483            .expect("first version should be written");
15484        let first_version = first.version_id.expect("versioned PUT should return an identity");
15485        store
15486            .transition_object(
15487                bucket,
15488                object,
15489                &ObjectOptions {
15490                    version_id: Some(first_version.to_string()),
15491                    versioned: true,
15492                    transition: TransitionOptions {
15493                        status: TRANSITION_PENDING.to_string(),
15494                        tier: tier_name.to_string(),
15495                        etag: first.etag.clone().expect("first version should have an etag"),
15496                        ..Default::default()
15497                    },
15498                    mod_time: first.mod_time,
15499                    ..Default::default()
15500                },
15501            )
15502            .await
15503            .expect("first version should transition");
15504        let mut second_reader = PutObjReader::from_vec(vec![b'2'; 1024 * 1024]);
15505        let second = store
15506            .put_object(
15507                bucket,
15508                object,
15509                &mut second_reader,
15510                &ObjectOptions {
15511                    versioned: true,
15512                    ..Default::default()
15513                },
15514            )
15515            .await
15516            .expect("second version should be written");
15517        let second_version = second.version_id.expect("second PUT should return an identity");
15518
15519        backend.set_remove_failure(true);
15520        store
15521            .delete_object_with_tier_delete_journal(
15522                bucket,
15523                object,
15524                ObjectOptions {
15525                    version_id: Some(first_version.to_string()),
15526                    versioned: true,
15527                    ..Default::default()
15528                },
15529            )
15530            .await
15531            .expect("explicit transitioned version delete should commit");
15532        store
15533            .get_object_info(
15534                bucket,
15535                object,
15536                &ObjectOptions {
15537                    version_id: Some(second_version.to_string()),
15538                    versioned: true,
15539                    ..Default::default()
15540                },
15541            )
15542            .await
15543            .expect("the unrelated second version must remain readable");
15544        let exact = store.pools[0]
15545            .get_disks_by_key(object)
15546            .load_file_info_versions_exact(bucket, object)
15547            .await
15548            .expect("remaining version metadata should decode")
15549            .expect("the second version should remain");
15550        assert_eq!(
15551            exact
15552                .versions
15553                .iter()
15554                .chain(exact.free_versions.iter())
15555                .filter(|version| version.tier_free_version())
15556                .count(),
15557            1,
15558            "ordinary exact-version delete must leave one hidden free-version owner"
15559        );
15560        assert!(
15561            exact
15562                .versions
15563                .iter()
15564                .any(|version| version.version_id == Some(second_version)),
15565            "the explicit delete must not remove the other version"
15566        );
15567        assert_eq!(
15568            tier_delete_journal_count(store.clone()).await,
15569            0,
15570            "ordinary exact-version delete must not create a journal"
15571        );
15572        backend.set_remove_failure(false);
15573        wait_for_tier_free_version_recovery(store.clone(), &backend, 1).await;
15574        let exact = store.pools[0]
15575            .get_disks_by_key(object)
15576            .load_file_info_versions_exact(bucket, object)
15577            .await
15578            .expect("post-cleanup metadata should decode")
15579            .expect("the unrelated second version should remain after cleanup");
15580        assert_eq!(
15581            exact
15582                .versions
15583                .iter()
15584                .chain(exact.free_versions.iter())
15585                .filter(|version| version.tier_free_version())
15586                .count(),
15587            0,
15588            "free-version recovery must remove only its exact owner"
15589        );
15590        assert!(
15591            exact
15592                .versions
15593                .iter()
15594                .any(|version| version.version_id == Some(second_version)),
15595            "free-version recovery must preserve the unrelated version"
15596        );
15597
15598        store
15599            .delete_object_with_tier_delete_journal(
15600                bucket,
15601                object,
15602                ObjectOptions {
15603                    version_id: Some(second_version.to_string()),
15604                    versioned: true,
15605                    ..Default::default()
15606                },
15607            )
15608            .await
15609            .expect("last ordinary version should be removed");
15610        store
15611            .delete_bucket(bucket, &DeleteBucketOptions::default())
15612            .await
15613            .expect("bucket delete should succeed after the last version is removed");
15614    }
15615
15616    #[cfg(feature = "test-util")]
15617    #[tokio::test]
15618    #[serial_test::serial(storage_class_env)]
15619    async fn suspended_null_transition_delete_uses_free_version_as_sole_owner() {
15620        let temp_dir = tempfile::tempdir().expect("create suspended transitioned delete store dir");
15621        let (ctx, store, _shutdown) =
15622            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "suspended-transitioned-delete-owner", &[4]))
15623                .await;
15624        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
15625        let tier_name = "SUSPENDED-DELETE";
15626        let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
15627        let bucket = "suspended-transitioned-delete-owner-bucket";
15628        let object = "archive.bin";
15629        store
15630            .make_bucket(bucket, &MakeBucketOptions::default())
15631            .await
15632            .expect("suspended source bucket should be created");
15633        crate::bucket::metadata_sys::update_in(
15634            &ctx,
15635            bucket,
15636            BUCKET_VERSIONING_CONFIG,
15637            b"<VersioningConfiguration><Status>Suspended</Status></VersioningConfiguration>".to_vec(),
15638        )
15639        .await
15640        .expect("bucket versioning should be suspended");
15641
15642        let mut reader = PutObjReader::from_vec(vec![b's'; 1024 * 1024]);
15643        let source = store
15644            .put_object(
15645                bucket,
15646                object,
15647                &mut reader,
15648                &ObjectOptions {
15649                    version_suspended: true,
15650                    ..Default::default()
15651                },
15652            )
15653            .await
15654            .expect("suspended null source should be written");
15655        let null_version = source.version_id.expect("suspended source should expose the null identity");
15656        assert!(null_version.is_nil(), "suspended source identity must be null");
15657        store
15658            .transition_object(
15659                bucket,
15660                object,
15661                &ObjectOptions {
15662                    version_id: Some(null_version.to_string()),
15663                    version_suspended: true,
15664                    transition: TransitionOptions {
15665                        status: TRANSITION_PENDING.to_string(),
15666                        tier: tier_name.to_string(),
15667                        etag: source.etag.clone().expect("suspended source should have an etag"),
15668                        ..Default::default()
15669                    },
15670                    mod_time: source.mod_time,
15671                    ..Default::default()
15672                },
15673            )
15674            .await
15675            .expect("suspended null source should transition");
15676
15677        backend.set_remove_failure(true);
15678        store
15679            .delete_object_with_tier_delete_journal(
15680                bucket,
15681                object,
15682                ObjectOptions {
15683                    version_id: Some(null_version.to_string()),
15684                    version_suspended: true,
15685                    ..Default::default()
15686                },
15687            )
15688            .await
15689            .expect("explicit null transitioned version delete should commit");
15690        let exact = store.pools[0]
15691            .get_disks_by_key(object)
15692            .load_file_info_versions_exact(bucket, object)
15693            .await
15694            .expect("post-delete null metadata should decode")
15695            .expect("ordinary null-version delete must retain a cleanup owner");
15696        assert_eq!(
15697            exact
15698                .versions
15699                .iter()
15700                .chain(exact.free_versions.iter())
15701                .filter(|version| version.tier_free_version())
15702                .count(),
15703            1,
15704            "ordinary null-version delete must retain exactly one hidden free-version owner"
15705        );
15706        assert_eq!(
15707            exact.versions.iter().filter(|version| !version.tier_free_version()).count(),
15708            0,
15709            "ordinary null-version delete must remove the visible source"
15710        );
15711        assert_eq!(tier_delete_journal_count(store.clone()).await, 0);
15712        backend.set_remove_failure(false);
15713        wait_for_tier_free_version_recovery(store.clone(), &backend, 1).await;
15714        store
15715            .delete_bucket(bucket, &DeleteBucketOptions::default())
15716            .await
15717            .expect("suspended bucket should be empty after exact null-version cleanup");
15718    }
15719
15720    #[cfg(feature = "test-util")]
15721    #[tokio::test]
15722    #[serial_test::serial(storage_class_env)]
15723    async fn versioned_delete_marker_keeps_transitioned_source_and_remote_object() {
15724        let temp_dir = tempfile::tempdir().expect("create versioned delete-marker store dir");
15725        let (ctx, store, _shutdown) =
15726            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "transitioned-delete-marker", &[4])).await;
15727        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
15728        let tier_name = "DELETE-MARKER";
15729        let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
15730        let bucket = "transitioned-delete-marker-bucket";
15731        let object = "archive.bin";
15732        store
15733            .make_bucket(bucket, &MakeBucketOptions::default())
15734            .await
15735            .expect("versioned source bucket should be created");
15736        crate::bucket::metadata_sys::update_in(
15737            &ctx,
15738            bucket,
15739            BUCKET_VERSIONING_CONFIG,
15740            b"<VersioningConfiguration><Status>Enabled</Status></VersioningConfiguration>".to_vec(),
15741        )
15742        .await
15743        .expect("bucket versioning should be enabled");
15744
15745        let mut reader = PutObjReader::from_vec(vec![b'm'; 1024 * 1024]);
15746        let source = store
15747            .put_object(
15748                bucket,
15749                object,
15750                &mut reader,
15751                &ObjectOptions {
15752                    versioned: true,
15753                    ..Default::default()
15754                },
15755            )
15756            .await
15757            .expect("versioned source should be written");
15758        let source_version = source.version_id.expect("versioned source should have an identity");
15759        store
15760            .transition_object(
15761                bucket,
15762                object,
15763                &ObjectOptions {
15764                    version_id: Some(source_version.to_string()),
15765                    versioned: true,
15766                    transition: TransitionOptions {
15767                        status: TRANSITION_PENDING.to_string(),
15768                        tier: tier_name.to_string(),
15769                        etag: source.etag.clone().expect("versioned source should have an etag"),
15770                        ..Default::default()
15771                    },
15772                    mod_time: source.mod_time,
15773                    ..Default::default()
15774                },
15775            )
15776            .await
15777            .expect("versioned source should transition");
15778
15779        let marker = store
15780            .delete_object_with_tier_delete_journal(
15781                bucket,
15782                object,
15783                ObjectOptions {
15784                    versioned: true,
15785                    ..Default::default()
15786                },
15787            )
15788            .await
15789            .expect("versioned DELETE without versionId should create a marker");
15790        assert!(marker.delete_marker);
15791        let marker_version = marker.version_id.expect("delete marker should have an identity");
15792        assert_eq!(backend.object_count().await, 1, "delete-marker creation must retain the remote object");
15793        assert_eq!(
15794            tier_delete_journal_count(store.clone()).await,
15795            0,
15796            "delete-marker creation must not schedule cleanup"
15797        );
15798        let exact = store.pools[0]
15799            .get_disks_by_key(object)
15800            .load_file_info_versions_exact(bucket, object)
15801            .await
15802            .expect("versioned delete-marker metadata should decode")
15803            .expect("source and delete marker should remain on disk");
15804        assert!(
15805            exact.versions.iter().any(|version| {
15806                version.version_id == Some(source_version) && version.transition_status == rustfs_filemeta::TRANSITION_COMPLETE
15807            }),
15808            "the transitioned source must remain behind the delete marker"
15809        );
15810        assert_eq!(
15811            exact
15812                .versions
15813                .iter()
15814                .chain(exact.free_versions.iter())
15815                .filter(|version| version.tier_free_version())
15816                .count(),
15817            0,
15818            "delete-marker creation must not create a cleanup free-version"
15819        );
15820
15821        store
15822            .delete_object_with_tier_delete_journal(
15823                bucket,
15824                object,
15825                ObjectOptions {
15826                    version_id: Some(marker_version.to_string()),
15827                    versioned: true,
15828                    ..Default::default()
15829                },
15830            )
15831            .await
15832            .expect("delete marker should be removable by exact identity");
15833        store
15834            .delete_object_with_tier_delete_journal(
15835                bucket,
15836                object,
15837                ObjectOptions {
15838                    version_id: Some(source_version.to_string()),
15839                    versioned: true,
15840                    ..Default::default()
15841                },
15842            )
15843            .await
15844            .expect("transitioned source should be removable by exact identity");
15845        wait_for_tier_free_version_recovery(store.clone(), &backend, 1).await;
15846        if let Some(exact) = store.pools[0]
15847            .get_disks_by_key(object)
15848            .load_file_info_versions_exact(bucket, object)
15849            .await
15850            .expect("post-recovery version metadata should decode")
15851        {
15852            assert!(
15853                exact
15854                    .versions
15855                    .iter()
15856                    .chain(exact.free_versions.iter())
15857                    .all(|version| !version.tier_free_version()),
15858                "free-version recovery must remove the all-physical cleanup owner"
15859            );
15860        }
15861        store
15862            .delete_bucket(bucket, &DeleteBucketOptions::default())
15863            .await
15864            .expect("versioned source bucket should be empty after explicit cleanup");
15865    }
15866
15867    #[cfg(feature = "test-util")]
15868    #[tokio::test]
15869    #[serial_test::serial(storage_class_env)]
15870    async fn batch_transitioned_delete_uses_free_version_per_item() {
15871        let temp_dir = tempfile::tempdir().expect("create batch transitioned delete store dir");
15872        let (ctx, store, _shutdown) =
15873            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "batch-transitioned-delete-owner", &[4])).await;
15874        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
15875        let tier_name = "BATCH-DELETE";
15876        let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
15877        let bucket = "batch-transitioned-delete-owner-bucket";
15878        let transitioned = "transitioned.bin";
15879        let ordinary = "ordinary.bin";
15880        store
15881            .make_bucket(bucket, &MakeBucketOptions::default())
15882            .await
15883            .expect("batch source bucket should be created");
15884        let mut transitioned_reader = PutObjReader::from_vec(vec![b't'; 1024 * 1024]);
15885        let transitioned_source = store
15886            .put_object(bucket, transitioned, &mut transitioned_reader, &ObjectOptions::default())
15887            .await
15888            .expect("transitioned batch source should be written");
15889        store
15890            .transition_object(
15891                bucket,
15892                transitioned,
15893                &ObjectOptions {
15894                    transition: TransitionOptions {
15895                        status: TRANSITION_PENDING.to_string(),
15896                        tier: tier_name.to_string(),
15897                        etag: transitioned_source
15898                            .etag
15899                            .clone()
15900                            .expect("transitioned batch source should have an etag"),
15901                        ..Default::default()
15902                    },
15903                    mod_time: transitioned_source.mod_time,
15904                    ..Default::default()
15905                },
15906            )
15907            .await
15908            .expect("batch source transition should commit");
15909        let mut ordinary_reader = PutObjReader::from_vec(vec![b'o'; 1024 * 1024]);
15910        store
15911            .put_object(bucket, ordinary, &mut ordinary_reader, &ObjectOptions::default())
15912            .await
15913            .expect("ordinary batch source should be written");
15914
15915        backend.set_remove_failure(true);
15916        let (_deleted, errors) = store
15917            .delete_objects_with_tier_delete_journal(
15918                bucket,
15919                vec![
15920                    ObjectToDelete {
15921                        object_name: transitioned.to_string(),
15922                        ..Default::default()
15923                    },
15924                    ObjectToDelete {
15925                        object_name: ordinary.to_string(),
15926                        ..Default::default()
15927                    },
15928                    ObjectToDelete {
15929                        object_name: "missing.bin".to_string(),
15930                        ..Default::default()
15931                    },
15932                ],
15933                ObjectOptions::default(),
15934            )
15935            .await;
15936        assert!(errors.iter().all(Option::is_none), "S3 batch delete should be idempotent: {errors:?}");
15937        let transitioned_versions = store.pools[0]
15938            .get_disks_by_key(transitioned)
15939            .load_file_info_versions_exact(bucket, transitioned)
15940            .await
15941            .expect("transitioned batch metadata should remain decodable")
15942            .expect("transitioned batch item must retain a cleanup owner");
15943        assert_eq!(
15944            transitioned_versions
15945                .versions
15946                .iter()
15947                .chain(transitioned_versions.free_versions.iter())
15948                .filter(|version| version.tier_free_version())
15949                .count(),
15950            1,
15951            "transitioned batch item must retain exactly one hidden free-version owner"
15952        );
15953        assert_eq!(
15954            transitioned_versions
15955                .versions
15956                .iter()
15957                .filter(|version| !version.tier_free_version())
15958                .count(),
15959            0,
15960            "transitioned batch item must have no visible source"
15961        );
15962        assert!(
15963            store.pools[0]
15964                .get_disks_by_key(ordinary)
15965                .load_file_info_versions_exact(bucket, ordinary)
15966                .await
15967                .expect("ordinary batch metadata should remain decodable")
15968                .is_none(),
15969            "ordinary batch item should have no local metadata"
15970        );
15971        assert_eq!(
15972            tier_delete_journal_count(store.clone()).await,
15973            0,
15974            "ordinary batch delete must not create a tier-delete journal"
15975        );
15976        assert_eq!(backend.object_count().await, 1, "failed remote cleanup must retain its object");
15977
15978        backend.set_remove_failure(false);
15979        wait_for_tier_free_version_recovery(store.clone(), &backend, 1).await;
15980        assert!(
15981            store.pools[0]
15982                .get_disks_by_key(transitioned)
15983                .load_file_info_versions_exact(bucket, transitioned)
15984                .await
15985                .expect("cleaned batch metadata should remain decodable")
15986                .is_none(),
15987            "free-version recovery must remove the exact cleanup owner"
15988        );
15989
15990        let causal = "causal.bin";
15991        let mut causal_reader = PutObjReader::from_vec(vec![b'c'; 1024 * 1024]);
15992        let causal_source = store
15993            .put_object(bucket, causal, &mut causal_reader, &ObjectOptions::default())
15994            .await
15995            .expect("causal batch source should be written");
15996        store
15997            .transition_object(
15998                bucket,
15999                causal,
16000                &ObjectOptions {
16001                    transition: TransitionOptions {
16002                        status: TRANSITION_PENDING.to_string(),
16003                        tier: tier_name.to_string(),
16004                        etag: causal_source.etag.clone().expect("causal batch source should have an etag"),
16005                        ..Default::default()
16006                    },
16007                    mod_time: causal_source.mod_time,
16008                    ..Default::default()
16009                },
16010            )
16011            .await
16012            .expect("causal batch source transition should commit");
16013        let (_deleted, errors) = store
16014            .delete_objects(
16015                bucket,
16016                vec![
16017                    ObjectToDelete {
16018                        object_name: causal.to_string(),
16019                        ..Default::default()
16020                    },
16021                    ObjectToDelete {
16022                        object_name: causal.to_string(),
16023                        ..Default::default()
16024                    },
16025                ],
16026                ObjectOptions::default(),
16027            )
16028            .await;
16029        assert!(
16030            errors.iter().all(Option::is_none),
16031            "duplicate causal batch deletes should remain idempotent: {errors:?}"
16032        );
16033        tokio::time::timeout(Duration::from_secs(30), async {
16034            loop {
16035                let metadata_absent = exact_metadata_absent_during_cleanup(&store, bucket, causal).await;
16036                if metadata_absent && backend.remove_versions().await.len() >= 2 {
16037                    return;
16038                }
16039                tokio::time::sleep(Duration::from_millis(10)).await;
16040            }
16041        })
16042        .await
16043        .expect("batch free-version receipt should converge without another recovery scan");
16044        assert_eq!(
16045            backend.remove_versions().await.len(),
16046            2,
16047            "duplicate batch requests must cause only one remote delete for the causal object"
16048        );
16049
16050        crate::bucket::metadata_sys::update_in(
16051            &ctx,
16052            bucket,
16053            BUCKET_VERSIONING_CONFIG,
16054            b"<VersioningConfiguration><Status>Enabled</Status></VersioningConfiguration>".to_vec(),
16055        )
16056        .await
16057        .expect("causal batch bucket versioning should be enabled");
16058        let versioned_causal = "versioned-causal.bin";
16059        let mut versioned_reader = PutObjReader::from_vec(vec![b'v'; 1024 * 1024]);
16060        let versioned_source = store
16061            .put_object(
16062                bucket,
16063                versioned_causal,
16064                &mut versioned_reader,
16065                &ObjectOptions {
16066                    versioned: true,
16067                    ..Default::default()
16068                },
16069            )
16070            .await
16071            .expect("versioned causal batch source should be written");
16072        let versioned_source_id = versioned_source
16073            .version_id
16074            .expect("versioned causal batch source should have an identity");
16075        store
16076            .transition_object(
16077                bucket,
16078                versioned_causal,
16079                &ObjectOptions {
16080                    version_id: Some(versioned_source_id.to_string()),
16081                    versioned: true,
16082                    transition: TransitionOptions {
16083                        status: TRANSITION_PENDING.to_string(),
16084                        tier: tier_name.to_string(),
16085                        etag: versioned_source
16086                            .etag
16087                            .clone()
16088                            .expect("versioned causal batch source should have an etag"),
16089                        ..Default::default()
16090                    },
16091                    mod_time: versioned_source.mod_time,
16092                    ..Default::default()
16093                },
16094            )
16095            .await
16096            .expect("versioned causal batch source transition should commit");
16097        let (_deleted, errors) = store
16098            .delete_objects(
16099                bucket,
16100                vec![ObjectToDelete {
16101                    object_name: versioned_causal.to_string(),
16102                    version_id: Some(versioned_source_id),
16103                    ..Default::default()
16104                }],
16105                ObjectOptions::default(),
16106            )
16107            .await;
16108        assert!(
16109            errors.iter().all(Option::is_none),
16110            "explicit-version causal batch delete should commit: {errors:?}"
16111        );
16112        tokio::time::timeout(Duration::from_secs(30), async {
16113            loop {
16114                let metadata_absent = exact_metadata_absent_during_cleanup(&store, bucket, versioned_causal).await;
16115                if metadata_absent && backend.remove_versions().await.len() == 3 {
16116                    return;
16117                }
16118                tokio::time::sleep(Duration::from_millis(10)).await;
16119            }
16120        })
16121        .await
16122        .expect("explicit-version batch receipt should converge without a recovery scan");
16123        store
16124            .delete_bucket(bucket, &DeleteBucketOptions::default())
16125            .await
16126            .expect("batch source bucket should be physically empty");
16127    }
16128
16129    #[cfg(feature = "test-util")]
16130    #[tokio::test]
16131    #[serial_test::serial(storage_class_env)]
16132    async fn batch_transitioned_delete_aggregate_error_still_enqueues_committed_free_version() {
16133        let temp_dir = tempfile::tempdir().expect("create aggregate-error batch delete store dir");
16134        let (ctx, store, shutdown) =
16135            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "batch-transitioned-aggregate-error", &[4, 4]))
16136                .await;
16137        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
16138        let tier_name = "BATCH-AGGREGATE-ERROR";
16139        let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
16140        let bucket = "batch-transitioned-aggregate-error-bucket";
16141        let object = "archive.bin";
16142        store
16143            .make_bucket(bucket, &MakeBucketOptions::default())
16144            .await
16145            .expect("aggregate-error source bucket should be created");
16146        let mut reader = PutObjReader::from_vec(vec![b'a'; 1024 * 1024]);
16147        let source = store.pools[0]
16148            .put_object(bucket, object, &mut reader, &ObjectOptions::default())
16149            .await
16150            .expect("aggregate-error source should be written");
16151        store.pools[0]
16152            .transition_object(
16153                bucket,
16154                object,
16155                &ObjectOptions {
16156                    transition: TransitionOptions {
16157                        status: TRANSITION_PENDING.to_string(),
16158                        tier: tier_name.to_string(),
16159                        etag: source.etag.clone().expect("aggregate-error source should have an etag"),
16160                        ..Default::default()
16161                    },
16162                    mod_time: source.mod_time,
16163                    ..Default::default()
16164                },
16165            )
16166            .await
16167            .expect("aggregate-error source should transition");
16168
16169        // Model a data-movement copy: both pools own the same logical source
16170        // and exact remote tuple, but each batch delete creates its own local
16171        // free-version UUID in the shared request sink.
16172        for disk_index in 0..4 {
16173            let source_meta = temp_dir
16174                .path()
16175                .join(format!("pool0/set0/disk{disk_index}/{bucket}/{object}/{STORAGE_FORMAT_FILE}"));
16176            let target_meta = temp_dir
16177                .path()
16178                .join(format!("pool1/set0/disk{disk_index}/{bucket}/{object}/{STORAGE_FORMAT_FILE}"));
16179            tokio::fs::create_dir_all(target_meta.parent().expect("target xl.meta should have a parent"))
16180                .await
16181                .expect("second-pool object directory should be created");
16182            tokio::fs::copy(&source_meta, &target_meta)
16183                .await
16184                .expect("transitioned xl.meta should copy exactly to the second pool");
16185        }
16186
16187        ExpiryState::resize_workers(1, store.clone()).await;
16188        let injection = crate::store::object::BatchDeletePoolErrorInjection::install(
16189            bucket,
16190            1,
16191            vec![(object.to_string(), StorageError::ErasureWriteQuorum)],
16192        );
16193        let (deleted, errors) = store
16194            .delete_objects(
16195                bucket,
16196                vec![ObjectToDelete {
16197                    object_name: object.to_string(),
16198                    ..Default::default()
16199                }],
16200                ObjectOptions::default(),
16201            )
16202            .await;
16203        assert_eq!(injection.observed(), 1, "the second pool should inject one post-commit aggregate error");
16204        assert_eq!(errors, vec![Some(StorageError::ErasureWriteQuorum)]);
16205        assert!(deleted[0].found, "the aggregate error must retain the committed pool result");
16206        drop(injection);
16207
16208        // Exact reads can see subquorum metadata while workers remove each
16209        // disk's free version. Inspect the final state after cleanup drains.
16210        wait_for_expiry_workers_idle(&store).await;
16211        for pool in &store.pools {
16212            assert!(
16213                pool.get_disks_by_key(object)
16214                    .load_file_info_versions_exact(bucket, object)
16215                    .await
16216                    .expect("aggregate-error cleanup metadata should remain readable")
16217                    .is_none(),
16218                "aggregate failure must not suppress committed receipt cleanup"
16219            );
16220        }
16221        assert_eq!(
16222            backend.remove_count().await,
16223            1,
16224            "committed receipts must remove the shared remote object once"
16225        );
16226        assert_eq!(backend.object_count().await, 0, "the shared remote object should be removed exactly once");
16227        store
16228            .delete_bucket(bucket, &DeleteBucketOptions::default())
16229            .await
16230            .expect("aggregate-error bucket should be physically empty");
16231        shutdown.cancel();
16232    }
16233
16234    #[cfg(feature = "test-util")]
16235    #[tokio::test]
16236    #[serial_test::serial(storage_class_env)]
16237    async fn transitioned_delete_free_version_replays_after_store_restart() {
16238        let temp_dir = tempfile::tempdir().expect("create restarted free-version store dir");
16239        let (ctx, store, shutdown) =
16240            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "transitioned-delete-restart-before", &[4]))
16241                .await;
16242        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
16243        let tier_name = "DELETE-RESTART";
16244        let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
16245        let tier_config = ctx
16246            .tier_config_mgr()
16247            .read()
16248            .await
16249            .tiers
16250            .get(tier_name)
16251            .expect("mock tier config should exist")
16252            .clone_with_credentials();
16253        let bucket = "transitioned-delete-restart-bucket";
16254        let object = "archive.bin";
16255        store
16256            .make_bucket(bucket, &MakeBucketOptions::default())
16257            .await
16258            .expect("restart source bucket should be created");
16259        let mut reader = PutObjReader::from_vec(vec![b'r'; 1024 * 1024]);
16260        let original = store
16261            .put_object(bucket, object, &mut reader, &ObjectOptions::default())
16262            .await
16263            .expect("restart source object should be written");
16264        store
16265            .transition_object(
16266                bucket,
16267                object,
16268                &ObjectOptions {
16269                    transition: TransitionOptions {
16270                        status: TRANSITION_PENDING.to_string(),
16271                        tier: tier_name.to_string(),
16272                        etag: original.etag.clone().expect("restart source should have an etag"),
16273                        ..Default::default()
16274                    },
16275                    mod_time: original.mod_time,
16276                    ..Default::default()
16277                },
16278            )
16279            .await
16280            .expect("restart source transition should commit");
16281        backend.set_remove_failure(true);
16282        store
16283            .delete_object_with_tier_delete_journal(bucket, object, ObjectOptions::default())
16284            .await
16285            .expect("local source delete should commit while remote cleanup is retryable");
16286        let retained = store.pools[0]
16287            .get_disks_by_key(object)
16288            .load_file_info_versions_exact(bucket, object)
16289            .await
16290            .expect("post-delete local metadata should decode")
16291            .expect("ordinary delete must retain a free-version across restart");
16292        assert_eq!(
16293            retained
16294                .versions
16295                .iter()
16296                .chain(retained.free_versions.iter())
16297                .filter(|version| version.tier_free_version())
16298                .count(),
16299            1,
16300            "ordinary delete must retain exactly one free-version across restart"
16301        );
16302        assert_eq!(
16303            retained
16304                .versions
16305                .iter()
16306                .filter(|version| !version.tier_free_version())
16307                .count(),
16308            0,
16309            "ordinary delete must remove the visible source before restart"
16310        );
16311        assert_eq!(tier_delete_journal_count(store.clone()).await, 0);
16312        assert_eq!(backend.object_count().await, 1);
16313
16314        shutdown.cancel();
16315        drop(store);
16316        drop(ctx);
16317        let (restarted_ctx, restarted_store, restarted_shutdown) =
16318            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "transitioned-delete-restart-after", &[4]))
16319                .await;
16320        crate::bucket::metadata_sys::init_bucket_metadata_sys(restarted_store.clone(), Vec::new()).await;
16321        {
16322            let restarted_tier_config_mgr = restarted_ctx.tier_config_mgr();
16323            let mut manager = restarted_tier_config_mgr.write().await;
16324            manager.tiers.insert(tier_name.to_string(), tier_config);
16325            manager
16326                .install_test_driver(tier_name, Box::new(backend.clone()))
16327                .expect("restarted store should bind the original tier destination");
16328        }
16329        backend.set_remove_failure(false);
16330        wait_for_tier_free_version_recovery(restarted_store.clone(), &backend, 1).await;
16331        assert_eq!(tier_delete_journal_count(restarted_store.clone()).await, 0);
16332        assert_eq!(backend.object_count().await, 0);
16333        assert!(
16334            restarted_store.pools[0]
16335                .get_disks_by_key(object)
16336                .load_file_info_versions_exact(bucket, object)
16337                .await
16338                .expect("restarted cleanup metadata should decode")
16339                .is_none(),
16340            "restarted recovery must remove the exact free-version owner"
16341        );
16342        restarted_store
16343            .delete_bucket(bucket, &DeleteBucketOptions::default())
16344            .await
16345            .expect("source bucket should be empty after restarted free-version recovery");
16346        restarted_shutdown.cancel();
16347    }
16348
16349    #[cfg(feature = "test-util")]
16350    #[tokio::test]
16351    #[serial_test::serial(storage_class_env)]
16352    async fn transitioned_delete_local_quorum_failure_rolls_back_without_cleanup_owner() {
16353        let temp_dir = tempfile::tempdir().expect("create failed local delete store dir");
16354        let (ctx, store, _shutdown) =
16355            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "transitioned-delete-local-failure", &[4]))
16356                .await;
16357        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
16358        let tier_name = "DELETE-LOCAL-FAIL";
16359        let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
16360        let bucket = "transitioned-delete-local-failure-bucket";
16361        let object = "archive.bin";
16362        store
16363            .make_bucket(bucket, &MakeBucketOptions::default())
16364            .await
16365            .expect("failed-delete source bucket should be created");
16366        let mut reader = PutObjReader::from_vec(vec![b'q'; 1024 * 1024]);
16367        let source = store
16368            .put_object(bucket, object, &mut reader, &ObjectOptions::default())
16369            .await
16370            .expect("failed-delete source should be written");
16371        store
16372            .transition_object(
16373                bucket,
16374                object,
16375                &ObjectOptions {
16376                    transition: TransitionOptions {
16377                        status: TRANSITION_PENDING.to_string(),
16378                        tier: tier_name.to_string(),
16379                        etag: source.etag.clone().expect("failed-delete source should have an etag"),
16380                        ..Default::default()
16381                    },
16382                    mod_time: source.mod_time,
16383                    ..Default::default()
16384                },
16385            )
16386            .await
16387            .expect("failed-delete source should transition");
16388
16389        let set = store.pools[0].get_disks_by_key(object);
16390        let saved_disk = {
16391            let mut disks = set.disks.write().await;
16392            let first = disks[0].as_ref().expect("first disk should be online");
16393            crate::disk::local::set_delete_version_fail_after_commit(first.path().as_path(), object);
16394            disks[1].take().expect("second disk should be online before fault injection")
16395        };
16396        let err = store
16397            .delete_object_with_tier_delete_journal(bucket, object, ObjectOptions::default())
16398            .await
16399            .expect_err("one post-commit error plus one offline disk must miss delete quorum");
16400        assert!(matches!(err, StorageError::Io(_)), "unexpected local delete failure: {err:?}");
16401        assert!(
16402            err.to_string()
16403                .contains("Storage resources are insufficient for the write operation"),
16404            "the caller must observe the wrapped quorum failure: {err}"
16405        );
16406        set.disks.write().await[1] = Some(saved_disk);
16407
16408        let retained = store
16409            .get_object_info(bucket, object, &ObjectOptions::default())
16410            .await
16411            .expect("failed local delete must roll back the transitioned source");
16412        assert_eq!(retained.transitioned_object.status, rustfs_filemeta::TRANSITION_COMPLETE);
16413        let retained_versions = set
16414            .load_file_info_versions_exact(bucket, object)
16415            .await
16416            .expect("rolled-back metadata should decode")
16417            .expect("rolled-back source should remain on disk");
16418        assert_eq!(
16419            retained_versions
16420                .versions
16421                .iter()
16422                .chain(retained_versions.free_versions.iter())
16423                .filter(|version| version.tier_free_version())
16424                .count(),
16425            0,
16426            "failed local quorum must not create a cleanup owner"
16427        );
16428        assert_eq!(
16429            tier_delete_journal_count(store.clone()).await,
16430            0,
16431            "ordinary delete must not create a journal, including on rollback"
16432        );
16433        assert_eq!(backend.object_count().await, 1, "failed local commit must retain the remote object");
16434        assert_eq!(backend.remove_count().await, 0, "failed local commit must not dispatch remote cleanup");
16435
16436        backend.set_remove_failure(true);
16437        store
16438            .delete_object_with_tier_delete_journal(bucket, object, ObjectOptions::default())
16439            .await
16440            .expect("retry after disk recovery should commit");
16441        let cleanup_owner = set
16442            .load_file_info_versions_exact(bucket, object)
16443            .await
16444            .expect("retry metadata should decode")
16445            .expect("successful retry must leave a free-version owner");
16446        assert_eq!(
16447            cleanup_owner
16448                .versions
16449                .iter()
16450                .chain(cleanup_owner.free_versions.iter())
16451                .filter(|version| version.tier_free_version())
16452                .count(),
16453            1,
16454            "successful retry must leave exactly one free-version owner"
16455        );
16456        assert_eq!(tier_delete_journal_count(store.clone()).await, 0);
16457        backend.set_remove_failure(false);
16458        wait_for_tier_free_version_recovery(store.clone(), &backend, 1).await;
16459        store
16460            .delete_bucket(bucket, &DeleteBucketOptions::default())
16461            .await
16462            .expect("retry should leave the source bucket empty");
16463    }
16464
16465    #[cfg(feature = "test-util")]
16466    #[tokio::test]
16467    #[serial_test::serial(storage_class_env)]
16468    async fn batch_transitioned_delete_post_commit_failures_roll_back_without_free_version_receipt() {
16469        let temp_dir = tempfile::tempdir().expect("create failed batch delete store dir");
16470        let (ctx, store, shutdown) =
16471            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "batch-delete-local-failure", &[4])).await;
16472        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
16473        let tier_name = "BATCH-DELETE-LOCAL-FAIL";
16474        let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
16475        let bucket = "batch-delete-local-failure-bucket";
16476        let object = "archive.bin";
16477        store
16478            .make_bucket(bucket, &MakeBucketOptions::default())
16479            .await
16480            .expect("failed batch source bucket should be created");
16481        crate::bucket::metadata_sys::update_in(
16482            &ctx,
16483            bucket,
16484            BUCKET_VERSIONING_CONFIG,
16485            b"<VersioningConfiguration><Status>Enabled</Status></VersioningConfiguration>".to_vec(),
16486        )
16487        .await
16488        .expect("failed batch bucket versioning should be enabled");
16489        let mut transitioned_reader = PutObjReader::from_vec(vec![b't'; 1024 * 1024]);
16490        let transitioned_source = store
16491            .put_object(
16492                bucket,
16493                object,
16494                &mut transitioned_reader,
16495                &ObjectOptions {
16496                    versioned: true,
16497                    ..Default::default()
16498                },
16499            )
16500            .await
16501            .expect("failed batch transitioned version should be written");
16502        let transitioned_version_id = transitioned_source
16503            .version_id
16504            .expect("failed batch transitioned source should have a version identity");
16505        store
16506            .transition_object(
16507                bucket,
16508                object,
16509                &ObjectOptions {
16510                    version_id: Some(transitioned_version_id.to_string()),
16511                    versioned: true,
16512                    transition: TransitionOptions {
16513                        status: TRANSITION_PENDING.to_string(),
16514                        tier: tier_name.to_string(),
16515                        etag: transitioned_source
16516                            .etag
16517                            .clone()
16518                            .expect("failed batch transitioned source should have an etag"),
16519                        ..Default::default()
16520                    },
16521                    mod_time: transitioned_source.mod_time,
16522                    ..Default::default()
16523                },
16524            )
16525            .await
16526            .expect("failed batch source version should transition");
16527        let mut ordinary_reader = PutObjReader::from_vec(vec![b'o'; 1024 * 1024]);
16528        let ordinary_source = store
16529            .put_object(
16530                bucket,
16531                object,
16532                &mut ordinary_reader,
16533                &ObjectOptions {
16534                    versioned: true,
16535                    ..Default::default()
16536                },
16537            )
16538            .await
16539            .expect("failed batch ordinary sibling should be written");
16540        let ordinary_version_id = ordinary_source
16541            .version_id
16542            .expect("failed batch ordinary sibling should have a version identity");
16543        let delete_requests = || {
16544            vec![
16545                ObjectToDelete {
16546                    object_name: object.to_string(),
16547                    version_id: Some(transitioned_version_id),
16548                    ..Default::default()
16549                },
16550                ObjectToDelete {
16551                    object_name: object.to_string(),
16552                    version_id: Some(ordinary_version_id),
16553                    ..Default::default()
16554                },
16555            ]
16556        };
16557
16558        let set = store.pools[0].get_disks_by_key(object);
16559        let disks = set.disks.read().await;
16560        assert_eq!(disks.len(), 4, "the rollback fixture must use four disks");
16561        // Keep discovery fully online, then make a quorum of disks report an
16562        // error only after their batch metadata commit has completed.
16563        for disk in disks.iter().take(3) {
16564            let disk = disk.as_ref().expect("injected rollback disks should be online");
16565            crate::disk::local::set_delete_version_fail_after_commit(disk.path().as_path(), object);
16566        }
16567        drop(disks);
16568        let receipt_sink = crate::object_api::TierFreeVersionReceiptSink::new();
16569        let (_deleted, errors) = store
16570            .delete_objects(
16571                bucket,
16572                delete_requests(),
16573                ObjectOptions {
16574                    tier_free_version_receipt_sink: Some(receipt_sink.clone()),
16575                    ..Default::default()
16576                },
16577            )
16578            .await;
16579        assert_eq!(
16580            errors,
16581            vec![Some(StorageError::Unexpected), Some(StorageError::Unexpected),],
16582            "three post-commit disk errors must fail batch delete before receipts publish"
16583        );
16584
16585        assert!(
16586            receipt_sink
16587                .drain()
16588                .expect("the test-owned failed-batch sink should drain exactly once")
16589                .is_empty(),
16590            "a rolled-back physical group must publish no cleanup receipt"
16591        );
16592        let retained_transitioned = store
16593            .get_object_info(
16594                bucket,
16595                object,
16596                &ObjectOptions {
16597                    version_id: Some(transitioned_version_id.to_string()),
16598                    versioned: true,
16599                    ..Default::default()
16600                },
16601            )
16602            .await
16603            .expect("failed batch delete must restore the transitioned sibling");
16604        assert_eq!(retained_transitioned.transitioned_object.status, rustfs_filemeta::TRANSITION_COMPLETE);
16605        let retained_ordinary = store
16606            .get_object_info(
16607                bucket,
16608                object,
16609                &ObjectOptions {
16610                    version_id: Some(ordinary_version_id.to_string()),
16611                    versioned: true,
16612                    ..Default::default()
16613                },
16614            )
16615            .await
16616            .expect("failed batch delete must restore the ordinary sibling");
16617        assert_ne!(retained_ordinary.transitioned_object.status, rustfs_filemeta::TRANSITION_COMPLETE);
16618        let retained_versions = set
16619            .load_file_info_versions_exact(bucket, object)
16620            .await
16621            .expect("rolled-back batch metadata should decode")
16622            .expect("rolled-back batch source should remain on disk");
16623        assert_eq!(
16624            retained_versions
16625                .versions
16626                .iter()
16627                .chain(retained_versions.free_versions.iter())
16628                .filter(|version| version.tier_free_version())
16629                .count(),
16630            0,
16631            "failed batch quorum must not retain a free-version owner"
16632        );
16633        let retained_version_ids = retained_versions
16634            .versions
16635            .iter()
16636            .filter_map(|version| version.version_id)
16637            .collect::<std::collections::HashSet<_>>();
16638        assert_eq!(
16639            retained_version_ids,
16640            std::collections::HashSet::from([transitioned_version_id, ordinary_version_id]),
16641            "the physical-group rollback must restore both explicit siblings"
16642        );
16643        assert_eq!(backend.object_count().await, 1, "failed batch commit must retain the remote object");
16644        assert_eq!(backend.remove_count().await, 0, "failed batch commit must not dispatch remote cleanup");
16645
16646        ExpiryState::resize_workers(1, store.clone()).await;
16647        let (_deleted, retry_errors) = store
16648            .delete_objects(bucket, delete_requests(), ObjectOptions::default())
16649            .await;
16650        assert!(
16651            retry_errors.iter().all(Option::is_none),
16652            "retry after disk recovery should commit: {retry_errors:?}"
16653        );
16654        tokio::time::timeout(Duration::from_secs(30), async {
16655            loop {
16656                // Cleanup rewrites xl.meta disk by disk, so a read racing it
16657                // can briefly miss quorum; any other error is a real failure.
16658                let metadata_absent = match set.load_file_info_versions_exact(bucket, object).await {
16659                    Ok(versions) => versions.is_none(),
16660                    Err(StorageError::InsufficientReadQuorum(..)) => false,
16661                    Err(err) => panic!("retry cleanup metadata should remain readable: {err:?}"),
16662                };
16663                if metadata_absent && backend.remove_count().await == 1 {
16664                    return;
16665                }
16666                tokio::time::sleep(Duration::from_millis(10)).await;
16667            }
16668        })
16669        .await
16670        .expect("successful batch retry should converge without a recovery scan");
16671        store
16672            .delete_bucket(bucket, &DeleteBucketOptions::default())
16673            .await
16674            .expect("successful batch retry should leave the bucket empty");
16675        shutdown.cancel();
16676    }
16677
16678    #[cfg(feature = "test-util")]
16679    async fn run_multi_pool_same_remote_tuple_delete_case(batch: bool) {
16680        let temp_dir = tempfile::tempdir().expect("create shared-tuple multi-pool store dir");
16681        let case = if batch { "batch" } else { "single" };
16682        let (ctx, store, _shutdown) = without_storage_class_env(build_isolated_test_store(
16683            temp_dir.path(),
16684            &format!("multi-pool-same-tuple-{case}"),
16685            &[4, 4],
16686        ))
16687        .await;
16688        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
16689        let tier_name = if batch { "SAME-TUPLE-BATCH" } else { "SAME-TUPLE-SINGLE" };
16690        let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
16691        let bucket = format!("multi-pool-same-tuple-{case}-bucket");
16692        let object = "archive.bin";
16693        store
16694            .make_bucket(&bucket, &MakeBucketOptions::default())
16695            .await
16696            .expect("shared-tuple bucket should be created");
16697        let mut reader = PutObjReader::from_vec(vec![b's'; 1024 * 1024]);
16698        let source = store.pools[0]
16699            .put_object(&bucket, object, &mut reader, &ObjectOptions::default())
16700            .await
16701            .expect("first pool source should be written");
16702        store.pools[0]
16703            .transition_object(
16704                &bucket,
16705                object,
16706                &ObjectOptions {
16707                    transition: TransitionOptions {
16708                        status: TRANSITION_PENDING.to_string(),
16709                        tier: tier_name.to_string(),
16710                        etag: source.etag.clone().expect("shared source should have an etag"),
16711                        ..Default::default()
16712                    },
16713                    mod_time: source.mod_time,
16714                    ..Default::default()
16715                },
16716            )
16717            .await
16718            .expect("first pool source should transition");
16719        let first = store.pools[0]
16720            .get_object_info(&bucket, object, &ObjectOptions::default())
16721            .await
16722            .expect("first pool transitioned source should be readable");
16723
16724        // Model the exact metadata duplication produced by data movement:
16725        // both pools refer to the same remote object/version/backend and carry
16726        // the same stable source identity.
16727        for disk_index in 0..4 {
16728            let source_meta = temp_dir
16729                .path()
16730                .join(format!("pool0/set0/disk{disk_index}/{bucket}/{object}/{STORAGE_FORMAT_FILE}"));
16731            let target_meta = temp_dir
16732                .path()
16733                .join(format!("pool1/set0/disk{disk_index}/{bucket}/{object}/{STORAGE_FORMAT_FILE}"));
16734            tokio::fs::create_dir_all(target_meta.parent().expect("target xl.meta should have a parent"))
16735                .await
16736                .expect("second pool object directory should be created");
16737            tokio::fs::copy(&source_meta, &target_meta)
16738                .await
16739                .expect("transitioned xl.meta should copy exactly to the second pool");
16740        }
16741        let second = store.pools[1]
16742            .get_object_info(&bucket, object, &ObjectOptions::default())
16743            .await
16744            .expect("copied second-pool source should be readable");
16745        assert_eq!(
16746            (first.transitioned_object.name.clone(), first.transitioned_object.version_id.clone()),
16747            (second.transitioned_object.name.clone(), second.transitioned_object.version_id.clone())
16748        );
16749        assert_eq!(backend.object_count().await, 1);
16750
16751        let receipt_sink = crate::object_api::TierFreeVersionReceiptSink::new();
16752        let mut delete_opts = ObjectOptions {
16753            tier_delete_journal_api: Some(store.clone()),
16754            tier_free_version_receipt_sink: Some(receipt_sink.clone()),
16755            ..Default::default()
16756        };
16757        if batch {
16758            delete_opts.delete_replication_config_snapshot = Some(Arc::new(
16759                ReplicationObjectBridge::delete_request_config_in(&store.ctx, &bucket)
16760                    .await
16761                    .expect("batch delete replication snapshot should load"),
16762            ));
16763            let (_deleted, errors, _accounting) = store.pools[0]
16764                .delete_objects_with_accounting(
16765                    &bucket,
16766                    vec![ObjectToDelete {
16767                        object_name: object.to_string(),
16768                        ..Default::default()
16769                    }],
16770                    delete_opts.clone(),
16771                )
16772                .await;
16773            assert!(errors.iter().all(Option::is_none), "first pool batch delete should commit: {errors:?}");
16774        } else {
16775            store.pools[0]
16776                .delete_object(&bucket, object, delete_opts.clone())
16777                .await
16778                .expect("first pool single delete should commit");
16779        }
16780
16781        let first_pool_versions = store.pools[0]
16782            .get_disks_by_key(object)
16783            .load_file_info_versions_exact(&bucket, object)
16784            .await
16785            .expect("first-pool metadata should decode")
16786            .expect("first pool must retain a free-version owner");
16787        assert_eq!(
16788            first_pool_versions
16789                .versions
16790                .iter()
16791                .chain(first_pool_versions.free_versions.iter())
16792                .filter(|version| version.tier_free_version())
16793                .count(),
16794            1,
16795            "first pool must retain exactly one free-version owner"
16796        );
16797        assert_eq!(
16798            first_pool_versions
16799                .versions
16800                .iter()
16801                .filter(|version| !version.tier_free_version())
16802                .count(),
16803            0,
16804            "first pool must remove its visible source"
16805        );
16806        assert_eq!(
16807            tier_delete_journal_count(store.clone()).await,
16808            0,
16809            "ordinary pool-local delete must not create a journal"
16810        );
16811
16812        ExpiryState::resize_workers(1, store.clone()).await;
16813        let first_recovery = recover_tier_free_versions(store.clone(), 100, None, None)
16814            .await
16815            .expect("first-pool free-version should be recoverable");
16816        assert_eq!(
16817            (first_recovery.enqueued, first_recovery.failed),
16818            (0, 0),
16819            "the live second-pool source is the merged walk winner, so its hidden loser must not dispatch cleanup"
16820        );
16821        wait_for_expiry_workers_idle(&store).await;
16822        assert_eq!(backend.remove_count().await, 0, "live second-pool source must gate remote DELETE");
16823        assert_eq!(backend.object_count().await, 1, "live source must retain its remote body");
16824        assert!(
16825            store.pools[0]
16826                .get_disks_by_key(object)
16827                .load_file_info_versions_exact(&bucket, object)
16828                .await
16829                .expect("blocked cleanup metadata should decode")
16830                .is_some(),
16831            "blocked cleanup must retain the first-pool free-version"
16832        );
16833        let surviving = store.pools[1]
16834            .get_object_info(&bucket, object, &ObjectOptions::default())
16835            .await
16836            .expect("second-pool source must remain GET-addressable");
16837        assert_eq!(surviving.transitioned_object.name, first.transitioned_object.name);
16838
16839        backend.set_remove_failure(true);
16840        if batch {
16841            let (_deleted, errors, _accounting) = store.pools[1]
16842                .delete_objects_with_accounting(
16843                    &bucket,
16844                    vec![ObjectToDelete {
16845                        object_name: object.to_string(),
16846                        ..Default::default()
16847                    }],
16848                    delete_opts,
16849                )
16850                .await;
16851            assert!(errors.iter().all(Option::is_none), "second pool batch retry should commit: {errors:?}");
16852        } else {
16853            store.pools[1]
16854                .delete_object(&bucket, object, delete_opts)
16855                .await
16856                .expect("second pool single retry should commit");
16857        }
16858        wait_for_expiry_workers_idle(&store).await;
16859        let mut free_version_ids = Vec::new();
16860        for pool_idx in 0..2 {
16861            let retained = store.pools[pool_idx]
16862                .get_disks_by_key(object)
16863                .load_file_info_versions_exact(&bucket, object)
16864                .await
16865                .expect("shared-tuple free-version metadata should decode")
16866                .expect("remote failure must retain every shared-tuple cleanup owner");
16867            let owners = retained
16868                .versions
16869                .iter()
16870                .chain(retained.free_versions.iter())
16871                .filter(|version| version.tier_free_version())
16872                .collect::<Vec<_>>();
16873            assert_eq!(owners.len(), 1, "pool {pool_idx} must retain one free-version owner");
16874            free_version_ids.push(owners[0].version_id);
16875        }
16876        assert_ne!(
16877            free_version_ids[0], free_version_ids[1],
16878            "ordinary pool-local deletes must model distinct local free-version identities"
16879        );
16880
16881        backend.set_remove_failure(false);
16882        let receipts = receipt_sink
16883            .drain()
16884            .expect("the simulated outer multi-pool wrapper should drain exactly once");
16885        assert_eq!(
16886            receipts.len(),
16887            1,
16888            "the same physical key and remote tuple must collapse to one causal task"
16889        );
16890        assert_eq!(
16891            crate::bucket::lifecycle::bucket_lifecycle_ops::enqueue_committed_free_versions(&store, receipts).await,
16892            1,
16893            "the committed shared-tuple task should enter the running worker"
16894        );
16895        wait_for_expiry_workers_idle(&store).await;
16896        assert_eq!(backend.remove_count().await, 1, "shared remote tuple should be deleted exactly once");
16897        for pool_idx in 0..2 {
16898            assert!(
16899                store.pools[pool_idx]
16900                    .get_disks_by_key(object)
16901                    .load_file_info_versions_exact(&bucket, object)
16902                    .await
16903                    .expect("post-delete metadata should decode")
16904                    .is_none(),
16905                "pool {pool_idx} must not retain source or free-version metadata"
16906            );
16907        }
16908        store
16909            .delete_bucket(&bucket, &DeleteBucketOptions::default())
16910            .await
16911            .expect("shared-tuple bucket should be physically empty");
16912    }
16913
16914    #[cfg(feature = "test-util")]
16915    #[tokio::test]
16916    #[serial_test::serial(storage_class_env)]
16917    async fn multi_pool_same_remote_tuple_single_delete_waits_for_all_sources() {
16918        run_multi_pool_same_remote_tuple_delete_case(false).await;
16919    }
16920
16921    #[cfg(feature = "test-util")]
16922    #[tokio::test]
16923    #[serial_test::serial(storage_class_env)]
16924    async fn multi_pool_same_remote_tuple_batch_delete_waits_for_all_sources() {
16925        run_multi_pool_same_remote_tuple_delete_case(true).await;
16926    }
16927
16928    #[cfg(feature = "test-util")]
16929    #[tokio::test]
16930    #[serial_test::serial(storage_class_env)]
16931    async fn multi_pool_transitioned_delete_persists_one_free_version_per_remote_tuple() {
16932        let temp_dir = tempfile::tempdir().expect("create multi-pool transitioned delete store dir");
16933        let (ctx, store, _shutdown) = without_storage_class_env(build_isolated_test_store(
16934            temp_dir.path(),
16935            "multi-pool-transitioned-delete-owner",
16936            &[4, 4],
16937        ))
16938        .await;
16939        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
16940        let tier_name = "MULTIPOOL-DELETE";
16941        let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
16942        let bucket = "multi-pool-transitioned-delete-owner-bucket";
16943        let object = "archive.bin";
16944        store
16945            .make_bucket(bucket, &MakeBucketOptions::default())
16946            .await
16947            .expect("multi-pool source bucket should be created");
16948
16949        let mut remote_tuples = Vec::new();
16950        for pool_idx in 0..2 {
16951            let mut reader = PutObjReader::from_vec(vec![b'a' + pool_idx as u8; 1024 * 1024]);
16952            let source = store.pools[pool_idx]
16953                .put_object(
16954                    bucket,
16955                    object,
16956                    &mut reader,
16957                    &ObjectOptions {
16958                        mod_time: Some(OffsetDateTime::now_utc() + time::Duration::seconds(pool_idx as i64)),
16959                        ..Default::default()
16960                    },
16961                )
16962                .await
16963                .expect("pool-local source should be written");
16964            store.pools[pool_idx]
16965                .transition_object(
16966                    bucket,
16967                    object,
16968                    &ObjectOptions {
16969                        transition: TransitionOptions {
16970                            status: TRANSITION_PENDING.to_string(),
16971                            tier: tier_name.to_string(),
16972                            etag: source.etag.clone().expect("pool-local source should have an etag"),
16973                            ..Default::default()
16974                        },
16975                        mod_time: source.mod_time,
16976                        ..Default::default()
16977                    },
16978                )
16979                .await
16980                .expect("pool-local source should transition");
16981            let transitioned = store.pools[pool_idx]
16982                .get_object_info(bucket, object, &ObjectOptions::default())
16983                .await
16984                .expect("pool-local transitioned source should be readable");
16985            remote_tuples.push((
16986                transitioned.transitioned_object.name.clone(),
16987                transitioned.transitioned_object.version_id.clone(),
16988            ));
16989        }
16990        assert_ne!(remote_tuples[0], remote_tuples[1], "fixture must use distinct remote cleanup tuples");
16991        assert_eq!(backend.object_count().await, 2);
16992
16993        backend.set_remove_failure(true);
16994        store
16995            .delete_object_with_tier_delete_journal(bucket, object, ObjectOptions::default())
16996            .await
16997            .expect("store-level delete should remove both pool copies");
16998        for pool_idx in 0..2 {
16999            let retained = store.pools[pool_idx]
17000                .get_disks_by_key(object)
17001                .load_file_info_versions_exact(bucket, object)
17002                .await
17003                .expect("pool-local post-delete metadata should decode")
17004                .expect("each pool-specific remote tuple must retain a cleanup owner");
17005            assert_eq!(
17006                retained
17007                    .versions
17008                    .iter()
17009                    .chain(retained.free_versions.iter())
17010                    .filter(|version| version.tier_free_version())
17011                    .count(),
17012                1,
17013                "pool {pool_idx} must retain exactly one free-version owner"
17014            );
17015            assert_eq!(
17016                retained
17017                    .versions
17018                    .iter()
17019                    .filter(|version| !version.tier_free_version())
17020                    .count(),
17021                0,
17022                "pool {pool_idx} must not retain a visible source"
17023            );
17024        }
17025        assert_eq!(
17026            tier_delete_journal_count(store.clone()).await,
17027            0,
17028            "ordinary multi-pool delete must not create tier-delete journals"
17029        );
17030        assert_eq!(backend.object_count().await, 2, "failed remote cleanup must retain both remote objects");
17031
17032        backend.set_remove_failure(false);
17033        wait_for_tier_free_version_recovery(store.clone(), &backend, 2).await;
17034        let mut removed = backend.remove_versions().await;
17035        removed.sort();
17036        remote_tuples.sort();
17037        assert_eq!(removed, remote_tuples, "recovery must use each pool's exact remote tuple");
17038        for pool_idx in 0..2 {
17039            assert!(
17040                store.pools[pool_idx]
17041                    .get_disks_by_key(object)
17042                    .load_file_info_versions_exact(bucket, object)
17043                    .await
17044                    .expect("post-recovery pool metadata should decode")
17045                    .is_none(),
17046                "pool {pool_idx} must not retain source or free-version metadata"
17047            );
17048        }
17049        store
17050            .delete_bucket(bucket, &DeleteBucketOptions::default())
17051            .await
17052            .expect("multi-pool source bucket should be physically empty");
17053    }
17054
17055    // Phase 5 follow-up (backlog#1052): building a real store through the
17056    // ctx-explicit constructor lands every construction-time write — object
17057    // graph adoption, local-disk registry, deployment id — on the passed
17058    // context, not on the process bootstrap one. This is the storage-layer
17059    // seam a future second embedded server needs to stay isolated.
17060    #[tokio::test]
17061    #[serial_test::serial(storage_class_env)]
17062    async fn new_with_instance_ctx_threads_context_through_store_graph() {
17063        let temp_dir = tempfile::tempdir().expect("create temp store dir");
17064        let (instance_ctx, store, _shutdown) =
17065            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "instance-ctx-store-graph-test", &[4])).await;
17066
17067        assert!(
17068            Arc::ptr_eq(&store.ctx, &instance_ctx),
17069            "the store must adopt the explicitly passed instance context"
17070        );
17071        for sets in &store.pools {
17072            assert!(
17073                Arc::ptr_eq(sets.instance_ctx(), &instance_ctx),
17074                "every pool's Sets must carry the passed instance context"
17075            );
17076        }
17077        assert_eq!(
17078            instance_ctx.deployment_id(),
17079            Some(store.id),
17080            "the deployment id must land on the passed context and mirror the store id"
17081        );
17082
17083        let registered: Vec<String> = instance_ctx.local_disk_map().read().await.keys().cloned().collect();
17084        assert_eq!(registered.len(), 4, "the passed context must register all four local disks");
17085        let registered_disk_ids = instance_ctx.local_disk_id_map();
17086        let registered_disk_ids = registered_disk_ids.read().await;
17087        assert_eq!(registered_disk_ids.len(), 4, "the passed context must publish all four disk IDs");
17088        for endpoint in registered_disk_ids.values() {
17089            assert!(
17090                registered.contains(endpoint),
17091                "every disk ID in the passed context must resolve to one of its registered endpoints"
17092            );
17093        }
17094        drop(registered_disk_ids);
17095        let bootstrap = crate::runtime::instance::bootstrap_ctx();
17096        assert_ne!(
17097            bootstrap.deployment_id(),
17098            Some(store.id),
17099            "the bootstrap context must not absorb the fresh store's deployment id"
17100        );
17101        let bootstrap_map = bootstrap.local_disk_map();
17102        let bootstrap_map = bootstrap_map.read().await;
17103        for key in &registered {
17104            assert!(
17105                !bootstrap_map.contains_key(key),
17106                "the bootstrap context must not absorb the fresh store's disks"
17107            );
17108        }
17109        drop(bootstrap_map);
17110        let bootstrap_disk_ids = bootstrap.local_disk_id_map();
17111        let bootstrap_disk_ids = bootstrap_disk_ids.read().await;
17112        for endpoint in bootstrap_disk_ids.values() {
17113            assert!(
17114                !registered.contains(endpoint),
17115                "the bootstrap context must not absorb the fresh store's disk IDs"
17116            );
17117        }
17118    }
17119
17120    #[tokio::test]
17121    #[serial_test::serial(storage_class_env)]
17122    async fn new_with_instance_ctx_applies_default_parity_to_each_real_pool() {
17123        let temp_dir = tempfile::tempdir().expect("create multi-pool store dir");
17124        let (_, store, _shutdown) =
17125            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "pool-parity-regression", &[4, 2])).await;
17126
17127        assert_eq!(store.pools.len(), 2);
17128        assert_eq!(store.pools[0].default_parity_count, 2);
17129        assert_eq!(store.pools[0].disk_set[0].default_parity_count, 2);
17130        assert_eq!(store.pools[1].default_parity_count, 1);
17131        assert_eq!(store.pools[1].disk_set[0].default_parity_count, 1);
17132    }
17133
17134    // backlog#1052 S3: two stores in one process each initialize their own
17135    // bucket metadata system on their own instance context. Before this, the
17136    // second `init_bucket_metadata_sys` panicked on the process-global
17137    // OnceLock — the hard blocker for a second embedded server's services.
17138    #[tokio::test]
17139    #[serial_test::serial(storage_class_env)]
17140    async fn two_stores_initialize_their_own_bucket_metadata_sys() {
17141        let temp_a = tempfile::tempdir().expect("create temp store dir a");
17142        let temp_b = tempfile::tempdir().expect("create temp store dir b");
17143        let (ctx_a, store_a, _shutdown_a) =
17144            without_storage_class_env(build_isolated_test_store(temp_a.path(), "bucket-metadata-isolation-a", &[4])).await;
17145        let (ctx_b, store_b, _shutdown_b) =
17146            without_storage_class_env(build_isolated_test_store(temp_b.path(), "bucket-metadata-isolation-b", &[4])).await;
17147
17148        crate::bucket::metadata_sys::init_bucket_metadata_sys(store_a.clone(), Vec::new()).await;
17149        // The old process-global cell would panic right here.
17150        crate::bucket::metadata_sys::init_bucket_metadata_sys(store_b.clone(), Vec::new()).await;
17151
17152        let sys_a = ctx_a
17153            .bucket_metadata_sys()
17154            .expect("store A's context must hold its metadata system");
17155        let sys_b = ctx_b
17156            .bucket_metadata_sys()
17157            .expect("store B's context must hold its metadata system");
17158        assert!(!Arc::ptr_eq(&sys_a, &sys_b), "each store must own a distinct bucket metadata system");
17159    }
17160
17161    #[tokio::test]
17162    #[serial_test::serial(storage_class_env)]
17163    async fn same_name_bucket_recursive_delete_uses_owning_instance_object_lock_state() {
17164        let temp_a = tempfile::tempdir().expect("create temp store dir a");
17165        let temp_b = tempfile::tempdir().expect("create temp store dir b");
17166        let (_ctx_a, store_a, _shutdown_a) =
17167            without_storage_class_env(build_isolated_test_store(temp_a.path(), "delete-scope-a", &[4])).await;
17168        let (_ctx_b, store_b, _shutdown_b) =
17169            without_storage_class_env(build_isolated_test_store(temp_b.path(), "delete-scope-b", &[4])).await;
17170        crate::bucket::metadata_sys::init_bucket_metadata_sys(store_a.clone(), Vec::new()).await;
17171        crate::bucket::metadata_sys::init_bucket_metadata_sys(store_b.clone(), Vec::new()).await;
17172
17173        let bucket = format!("same-name-delete-scope-{}", uuid::Uuid::new_v4());
17174        store_a
17175            .make_bucket(&bucket, &MakeBucketOptions::default())
17176            .await
17177            .expect("create plain bucket in store A");
17178        store_b
17179            .make_bucket(
17180                &bucket,
17181                &MakeBucketOptions {
17182                    lock_enabled: true,
17183                    ..Default::default()
17184                },
17185            )
17186            .await
17187            .expect("create Object Lock bucket in store B");
17188
17189        for store in [&store_a, &store_b] {
17190            let mut reader = PutObjReader::from_vec(b"payload".to_vec());
17191            store
17192                .put_object(
17193                    &bucket,
17194                    "prefix/object.bin",
17195                    &mut reader,
17196                    &ObjectOptions {
17197                        versioned: Arc::ptr_eq(store, &store_b),
17198                        ..Default::default()
17199                    },
17200                )
17201                .await
17202                .expect("put same-name object");
17203        }
17204
17205        store_a
17206            .delete_object(
17207                &bucket,
17208                "prefix",
17209                ObjectOptions {
17210                    delete_prefix: true,
17211                    ..Default::default()
17212                },
17213            )
17214            .await
17215            .expect("plain store prefix delete should remain allowed");
17216
17217        let err = store_b
17218            .delete_object(
17219                &bucket,
17220                "prefix",
17221                ObjectOptions {
17222                    delete_prefix: true,
17223                    ..Default::default()
17224                },
17225            )
17226            .await
17227            .expect_err("store B must not consult store A's plain bucket metadata");
17228        assert!(matches!(err, StorageError::InvalidArgument(_, _, _)));
17229        store_b
17230            .get_object_info(&bucket, "prefix/object.bin", &ObjectOptions::default())
17231            .await
17232            .expect("locked store object must survive the rejected prefix delete");
17233    }
17234
17235    #[tokio::test]
17236    #[serial_test::serial(storage_class_env)]
17237    async fn recursive_delete_checks_object_lock_on_every_version_page() {
17238        let temp = tempfile::tempdir().expect("create temp store dir");
17239        let (_ctx, store, _shutdown) =
17240            without_storage_class_env(build_isolated_test_store(temp.path(), "recursive-delete-pagination", &[4])).await;
17241        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
17242
17243        let bucket = format!("recursive-page-{}", uuid::Uuid::new_v4());
17244        store
17245            .make_bucket(&bucket, &MakeBucketOptions::default())
17246            .await
17247            .expect("create plain bucket");
17248        for object in ["prefix/a.bin", "prefix/b.bin"] {
17249            let mut reader = PutObjReader::from_vec(b"plain".to_vec());
17250            store
17251                .put_object(&bucket, object, &mut reader, &ObjectOptions::default())
17252                .await
17253                .expect("write first-page object");
17254        }
17255        let locked_object = "prefix/z-locked.bin";
17256        let mut reader = PutObjReader::from_vec(b"locked".to_vec());
17257        store
17258            .put_object(
17259                &bucket,
17260                locked_object,
17261                &mut reader,
17262                &ObjectOptions {
17263                    user_defined: HashMap::from([
17264                        (
17265                            rustfs_filemeta::metadata_keys::OBJECT_LOCK_MODE.to_string(),
17266                            s3s::dto::ObjectLockRetentionMode::COMPLIANCE.to_string(),
17267                        ),
17268                        (
17269                            rustfs_filemeta::metadata_keys::OBJECT_LOCK_RETAIN_UNTIL_DATE.to_string(),
17270                            "2099-01-01T00:00:00Z".to_string(),
17271                        ),
17272                    ]),
17273                    ..Default::default()
17274                },
17275            )
17276            .await
17277            .expect("write second-page locked object");
17278
17279        let first_page = store.pools[0].disk_set[0]
17280            .clone()
17281            .inner_list_object_versions_for_recursive_delete(&bucket, "prefix/", None, None, 2)
17282            .await
17283            .expect("list the first recursive-delete validation page");
17284        assert!(first_page.is_truncated, "the fixture must require a continuation page");
17285        assert!(
17286            first_page.objects.iter().all(|object| object.name != locked_object),
17287            "the locked object must not be visible on the first page"
17288        );
17289        let second_page = store.pools[0].disk_set[0]
17290            .clone()
17291            .inner_list_object_versions_for_recursive_delete(
17292                &bucket,
17293                "prefix/",
17294                first_page.next_marker,
17295                first_page.next_version_idmarker,
17296                2,
17297            )
17298            .await
17299            .expect("list the recursive-delete continuation page");
17300        assert!(
17301            second_page.objects.iter().any(|object| object.name == locked_object),
17302            "the continuation page must contain the locked object"
17303        );
17304
17305        let err = store
17306            .delete_object(
17307                &bucket,
17308                "prefix/",
17309                ObjectOptions {
17310                    delete_prefix: true,
17311                    ..Default::default()
17312                },
17313            )
17314            .await
17315            .expect_err("recursive delete must scan the continuation page and find the locked object");
17316        assert!(
17317            matches!(err, StorageError::PrefixAccessDenied(ref name, ref object) if name == &bucket && object == locked_object),
17318            "unexpected recursive delete error: {err:?}"
17319        );
17320        for object in ["prefix/a.bin", "prefix/b.bin", locked_object] {
17321            store
17322                .get_object_info(&bucket, object, &ObjectOptions::default())
17323                .await
17324                .expect("a rejected recursive delete must leave every page intact");
17325        }
17326    }
17327
17328    #[tokio::test]
17329    #[serial_test::serial(storage_class_env)]
17330    async fn delete_holds_bucket_incarnation_sentinel_through_object_commit() {
17331        let temp = tempfile::tempdir().expect("create temp store dir");
17332        let (_ctx, store, _shutdown) =
17333            without_storage_class_env(build_isolated_test_store(temp.path(), "delete-bucket-incarnation", &[4])).await;
17334        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
17335
17336        let bucket = format!("delete-bucket-incarnation-{}", uuid::Uuid::new_v4());
17337        let object = "same-key.bin";
17338        store
17339            .make_bucket(&bucket, &MakeBucketOptions::default())
17340            .await
17341            .expect("create initial plain bucket");
17342        let old_incarnation = store
17343            .bucket_incarnation_id(&bucket)
17344            .await
17345            .expect("read initial bucket incarnation");
17346        let mut reader = PutObjReader::from_vec(b"old generation".to_vec());
17347        store
17348            .put_object(&bucket, object, &mut reader, &ObjectOptions::default())
17349            .await
17350            .expect("put old-generation object");
17351
17352        let barrier = crate::store::object::DeleteAfterObjectLockSnapshotBarrier::install(&bucket);
17353        let delete_store = Arc::clone(&store);
17354        let delete_bucket = bucket.clone();
17355        let delete = tokio::spawn(async move {
17356            delete_store
17357                .delete_object(&delete_bucket, object, ObjectOptions::default())
17358                .await
17359        });
17360        barrier.wait_until_paused().await;
17361
17362        let recreate_store = Arc::clone(&store);
17363        let recreate_bucket = bucket.clone();
17364        let mut recreate = tokio::spawn(async move {
17365            recreate_store
17366                .delete_bucket(
17367                    &recreate_bucket,
17368                    &crate::storage_api_contracts::bucket::DeleteBucketOptions {
17369                        force: true,
17370                        ..Default::default()
17371                    },
17372                )
17373                .await?;
17374            recreate_store
17375                .make_bucket(
17376                    &recreate_bucket,
17377                    &MakeBucketOptions {
17378                        lock_enabled: true,
17379                        ..Default::default()
17380                    },
17381                )
17382                .await?;
17383            let mut reader = PutObjReader::from_vec(b"new protected generation".to_vec());
17384            recreate_store
17385                .put_object(
17386                    &recreate_bucket,
17387                    object,
17388                    &mut reader,
17389                    &ObjectOptions {
17390                        versioned: true,
17391                        ..Default::default()
17392                    },
17393                )
17394                .await?;
17395            Ok::<_, Error>(())
17396        });
17397
17398        assert!(
17399            tokio::time::timeout(Duration::from_millis(100), &mut recreate).await.is_err(),
17400            "bucket delete and same-name recreation must wait for the in-flight object delete"
17401        );
17402        barrier.release();
17403        delete
17404            .await
17405            .expect("delete task should join")
17406            .expect("old-generation delete should complete");
17407        tokio::time::timeout(Duration::from_secs(10), recreate)
17408            .await
17409            .expect("bucket recreation should finish after delete releases the sentinel")
17410            .expect("bucket recreation task should join")
17411            .expect("bucket recreation should succeed");
17412
17413        let stale_delete_err = store
17414            .delete_object(
17415                &bucket,
17416                object,
17417                ObjectOptions {
17418                    expected_bucket_incarnation_id: Some(old_incarnation),
17419                    versioned: true,
17420                    ..Default::default()
17421                },
17422            )
17423            .await
17424            .expect_err("an authorization snapshot from the old bucket incarnation must be rejected");
17425        assert!(matches!(stale_delete_err, StorageError::BucketNotFound(name) if name == bucket));
17426
17427        let (_deleted, stale_batch_errors) = store
17428            .delete_objects(
17429                &bucket,
17430                vec![crate::storage_api_contracts::object::ObjectToDelete {
17431                    object_name: object.to_string(),
17432                    ..Default::default()
17433                }],
17434                ObjectOptions {
17435                    expected_bucket_incarnation_id: Some(old_incarnation),
17436                    versioned: true,
17437                    ..Default::default()
17438                },
17439            )
17440            .await;
17441        assert!(
17442            matches!(stale_batch_errors.as_slice(), [Some(StorageError::BucketNotFound(name))] if name.as_str() == bucket),
17443            "batch delete must reject an authorization snapshot from the old bucket incarnation"
17444        );
17445
17446        store
17447            .get_object_info(
17448                &bucket,
17449                object,
17450                &ObjectOptions {
17451                    versioned: true,
17452                    ..Default::default()
17453                },
17454            )
17455            .await
17456            .expect("the new locked-bucket object must survive the old-generation delete");
17457    }
17458
17459    #[tokio::test]
17460    #[serial_test::serial(storage_class_env)]
17461    async fn delete_objects_holds_bucket_incarnation_sentinel_through_commit() {
17462        let temp = tempfile::tempdir().expect("create temp store dir");
17463        let (_ctx, store, _shutdown) =
17464            without_storage_class_env(build_isolated_test_store(temp.path(), "batch-delete-bucket-incarnation", &[4])).await;
17465        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
17466
17467        let bucket = format!("batch-incarnation-{}", uuid::Uuid::new_v4());
17468        let object = "same-key.bin";
17469        store
17470            .make_bucket(&bucket, &MakeBucketOptions::default())
17471            .await
17472            .expect("create initial bucket");
17473        let mut reader = PutObjReader::from_vec(b"old generation".to_vec());
17474        store
17475            .put_object(&bucket, object, &mut reader, &ObjectOptions::default())
17476            .await
17477            .expect("put old-generation object");
17478
17479        let barrier = crate::store::object::DeleteAfterObjectLockSnapshotBarrier::install(&bucket);
17480        let delete_store = Arc::clone(&store);
17481        let delete_bucket = bucket.clone();
17482        let delete = tokio::spawn(async move {
17483            delete_store
17484                .delete_objects(
17485                    &delete_bucket,
17486                    vec![crate::storage_api_contracts::object::ObjectToDelete {
17487                        object_name: object.to_string(),
17488                        ..Default::default()
17489                    }],
17490                    ObjectOptions::default(),
17491                )
17492                .await
17493        });
17494        barrier.wait_until_paused().await;
17495
17496        let recreate_store = Arc::clone(&store);
17497        let recreate_bucket = bucket.clone();
17498        let mut recreate = tokio::spawn(async move {
17499            recreate_store
17500                .delete_bucket(
17501                    &recreate_bucket,
17502                    &crate::storage_api_contracts::bucket::DeleteBucketOptions {
17503                        force: true,
17504                        ..Default::default()
17505                    },
17506                )
17507                .await?;
17508            recreate_store
17509                .make_bucket(&recreate_bucket, &MakeBucketOptions::default())
17510                .await?;
17511            let mut reader = PutObjReader::from_vec(b"new generation".to_vec());
17512            recreate_store
17513                .put_object(&recreate_bucket, object, &mut reader, &ObjectOptions::default())
17514                .await?;
17515            Ok::<_, Error>(())
17516        });
17517
17518        assert!(
17519            tokio::time::timeout(Duration::from_millis(100), &mut recreate).await.is_err(),
17520            "bucket recreation must wait for the in-flight batch delete"
17521        );
17522        barrier.release();
17523        let (_deleted, errors) = delete.await.expect("batch delete task should join");
17524        assert!(
17525            errors.iter().all(Option::is_none),
17526            "old-generation batch delete should complete: {errors:?}"
17527        );
17528        tokio::time::timeout(Duration::from_secs(10), recreate)
17529            .await
17530            .expect("bucket recreation should finish after batch delete releases the sentinel")
17531            .expect("bucket recreation task should join")
17532            .expect("bucket recreation should succeed");
17533        store
17534            .get_object_info(&bucket, object, &ObjectOptions::default())
17535            .await
17536            .expect("new-generation object must survive the old batch delete");
17537    }
17538
17539    #[tokio::test]
17540    #[serial_test::serial(storage_class_env)]
17541    async fn prefix_delete_serializes_with_concurrent_object_lock_metadata_updates() {
17542        let temp = tempfile::tempdir().expect("create temp store dir");
17543        let (_ctx, store, _shutdown) =
17544            without_storage_class_env(build_isolated_test_store(temp.path(), "prefix-delete-lock-metadata", &[4])).await;
17545        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
17546
17547        let cases = [
17548            (
17549                "compliance",
17550                HashMap::from([
17551                    ("x-amz-object-lock-mode".to_string(), "COMPLIANCE".to_string()),
17552                    ("x-amz-object-lock-retain-until-date".to_string(), "2099-01-01T00:00:00Z".to_string()),
17553                ]),
17554            ),
17555            (
17556                "legal-hold",
17557                HashMap::from([("x-amz-object-lock-legal-hold".to_string(), "ON".to_string())]),
17558            ),
17559        ];
17560
17561        for (case, eval_metadata) in cases {
17562            let bucket = format!("prefix-lock-{case}-{}", uuid::Uuid::new_v4());
17563            let object = "protected-after-scan.bin";
17564            store
17565                .make_bucket(
17566                    &bucket,
17567                    &MakeBucketOptions {
17568                        lock_enabled: true,
17569                        ..Default::default()
17570                    },
17571                )
17572                .await
17573                .expect("create Object Lock bucket");
17574            let mut reader = PutObjReader::from_vec(b"body".to_vec());
17575            let object_info = store
17576                .put_object(
17577                    &bucket,
17578                    object,
17579                    &mut reader,
17580                    &ObjectOptions {
17581                        versioned: true,
17582                        ..Default::default()
17583                    },
17584                )
17585                .await
17586                .expect("write unprotected version");
17587            let version_id = object_info.version_id.expect("versioned PUT should return an ID").to_string();
17588
17589            let barrier = crate::store::object::DeleteAfterObjectLockSnapshotBarrier::install(&bucket);
17590            let delete_store = Arc::clone(&store);
17591            let delete_bucket = bucket.clone();
17592            let delete = tokio::spawn(async move {
17593                delete_store
17594                    .delete_object(
17595                        &delete_bucket,
17596                        object,
17597                        ObjectOptions {
17598                            delete_prefix: true,
17599                            delete_prefix_object: true,
17600                            versioned: true,
17601                            ..Default::default()
17602                        },
17603                    )
17604                    .await
17605            });
17606            barrier.wait_until_paused().await;
17607
17608            let update_store = Arc::clone(&store);
17609            let update_bucket = bucket.clone();
17610            let mut update = tokio::spawn(async move {
17611                update_store
17612                    .put_object_metadata(
17613                        &update_bucket,
17614                        object,
17615                        &ObjectOptions {
17616                            version_id: Some(version_id),
17617                            versioned: true,
17618                            eval_metadata: Some(eval_metadata),
17619                            ..Default::default()
17620                        },
17621                    )
17622                    .await
17623            });
17624            assert!(
17625                tokio::time::timeout(Duration::from_millis(100), &mut update).await.is_err(),
17626                "{case} metadata update must wait behind the prefix-delete lifecycle write fence"
17627            );
17628
17629            barrier.release();
17630            delete
17631                .await
17632                .expect("prefix delete task should join")
17633                .expect("prefix delete should remove the unprotected version");
17634            update
17635                .await
17636                .expect("metadata update task should join")
17637                .expect_err("metadata update must not succeed after the version was deleted");
17638            assert!(
17639                store
17640                    .get_object_info(
17641                        &bucket,
17642                        object,
17643                        &ObjectOptions {
17644                            versioned: true,
17645                            ..Default::default()
17646                        },
17647                    )
17648                    .await
17649                    .is_err(),
17650                "{case} case must not report a successful protected write followed by deletion"
17651            );
17652        }
17653    }
17654
17655    #[tokio::test]
17656    #[serial_test::serial(storage_class_env)]
17657    async fn put_rejects_same_name_bucket_recreated_after_request_snapshot() {
17658        let temp = tempfile::tempdir().expect("create temp store dir");
17659        let (_ctx, store, _shutdown) =
17660            without_storage_class_env(build_isolated_test_store(temp.path(), "put-bucket-incarnation", &[4])).await;
17661        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
17662
17663        let bucket = format!("put-bucket-incarnation-{}", uuid::Uuid::new_v4());
17664        let object = "object.bin";
17665        store
17666            .make_bucket(&bucket, &MakeBucketOptions::default())
17667            .await
17668            .expect("create initial bucket");
17669        let old_incarnation = store
17670            .bucket_incarnation_id(&bucket)
17671            .await
17672            .expect("read initial bucket incarnation");
17673
17674        let barrier = crate::set_disk::PutObjectCommitBarrier::install(
17675            &bucket,
17676            object,
17677            crate::set_disk::PutObjectCommitPause::BeforeNamespace,
17678        );
17679        let put_store = Arc::clone(&store);
17680        let put_bucket = bucket.clone();
17681        let put = tokio::spawn(async move {
17682            let mut reader = PutObjReader::from_vec(b"old generation".to_vec());
17683            put_store
17684                .put_object(
17685                    &put_bucket,
17686                    object,
17687                    &mut reader,
17688                    &ObjectOptions {
17689                        versioned: true,
17690                        ..Default::default()
17691                    },
17692                )
17693                .await
17694        });
17695        barrier.wait_until_paused().await;
17696
17697        store
17698            .delete_bucket(&bucket, &crate::storage_api_contracts::bucket::DeleteBucketOptions::default())
17699            .await
17700            .expect("delete initial empty bucket while PUT is staged");
17701        store
17702            .make_bucket(&bucket, &MakeBucketOptions::default())
17703            .await
17704            .expect("recreate same-name bucket");
17705        let new_incarnation = store
17706            .bucket_incarnation_id(&bucket)
17707            .await
17708            .expect("read recreated bucket incarnation");
17709        assert_ne!(old_incarnation, new_incarnation);
17710
17711        barrier.release();
17712        let err = put
17713            .await
17714            .expect("PUT task should join")
17715            .expect_err("old-generation PUT must not commit into the recreated bucket");
17716        assert!(
17717            matches!(err, StorageError::BucketNotFound(ref name) if name == &bucket),
17718            "unexpected stale PUT error: {err:?}"
17719        );
17720        assert!(
17721            store
17722                .get_object_info(&bucket, object, &ObjectOptions::default())
17723                .await
17724                .is_err(),
17725            "the recreated bucket must not contain the staged old-generation object"
17726        );
17727    }
17728
17729    #[tokio::test]
17730    #[serial_test::serial(storage_class_env)]
17731    async fn exact_version_put_uses_only_its_own_bucket_object_lock_snapshot() {
17732        let target_temp = tempfile::tempdir().expect("create target store dir");
17733        let foreign_temp = tempfile::tempdir().expect("create foreign store dir");
17734        let (target_ctx, target_store, _target_shutdown) =
17735            without_storage_class_env(build_isolated_test_store(target_temp.path(), "put-object-lock-target", &[4])).await;
17736        let (_foreign_ctx, foreign_store, _foreign_shutdown) =
17737            without_storage_class_env(build_isolated_test_store(foreign_temp.path(), "put-object-lock-foreign", &[4])).await;
17738        crate::bucket::metadata_sys::init_bucket_metadata_sys(target_store.clone(), Vec::new()).await;
17739        crate::bucket::metadata_sys::init_bucket_metadata_sys(foreign_store.clone(), Vec::new()).await;
17740
17741        let bucket = format!("put-object-lock-snapshot-{}", uuid::Uuid::new_v4());
17742        let object = "object.bin";
17743        target_store
17744            .make_bucket(
17745                &bucket,
17746                &MakeBucketOptions {
17747                    lock_enabled: true,
17748                    ..Default::default()
17749                },
17750            )
17751            .await
17752            .expect("create target Object Lock bucket");
17753        foreign_store
17754            .make_bucket(&bucket, &MakeBucketOptions::default())
17755            .await
17756            .expect("create same-name plain bucket in foreign store");
17757        let stale_snapshot = target_store
17758            .object_lock_config_snapshot(&bucket)
17759            .await
17760            .expect("load target snapshot before changing its default retention");
17761
17762        let mut metadata = crate::bucket::metadata_sys::get_in(&target_ctx, &bucket)
17763            .await
17764            .expect("load target bucket metadata")
17765            .as_ref()
17766            .clone();
17767        metadata
17768            .update_config(
17769                crate::bucket::metadata::OBJECT_LOCK_CONFIG,
17770                b"<ObjectLockConfiguration><ObjectLockEnabled>Enabled</ObjectLockEnabled><Rule><DefaultRetention><Mode>COMPLIANCE</Mode><Days>1</Days></DefaultRetention></Rule></ObjectLockConfiguration>".to_vec(),
17771            )
17772            .expect("set default COMPLIANCE metadata");
17773        let update_started = Arc::new(tokio::sync::Barrier::new(2));
17774        let update_ctx = Arc::clone(&target_ctx);
17775        let update_started_task = Arc::clone(&update_started);
17776        let mut update = tokio::spawn(async move {
17777            update_started_task.wait().await;
17778            let _transaction_guard =
17779                crate::bucket::metadata_sys::acquire_bucket_metadata_transaction_lock_in(&update_ctx, &metadata.name).await?;
17780            crate::bucket::metadata_sys::set_bucket_metadata_in(&update_ctx, metadata).await
17781        });
17782        update_started.wait().await;
17783        assert!(
17784            tokio::time::timeout(Duration::from_millis(100), &mut update).await.is_err(),
17785            "an active Object Lock snapshot must hold the metadata revision read fence"
17786        );
17787        drop(stale_snapshot);
17788        update
17789            .await
17790            .expect("Object Lock metadata update task should join")
17791            .expect("persist target Object Lock metadata after snapshot release");
17792
17793        let original_body = b"protected body".to_vec();
17794        let mut original_reader = PutObjReader::from_vec(original_body.clone());
17795        let original = target_store
17796            .put_object(
17797                &bucket,
17798                object,
17799                &mut original_reader,
17800                &ObjectOptions {
17801                    versioned: true,
17802                    ..Default::default()
17803                },
17804            )
17805            .await
17806            .expect("write protected target version");
17807        let version_id = original
17808            .version_id
17809            .expect("versioned PUT should return a version ID")
17810            .to_string();
17811
17812        let mut replacement = PutObjReader::from_vec(b"replacement".to_vec());
17813        let err = target_store
17814            .put_object(
17815                &bucket,
17816                object,
17817                &mut replacement,
17818                &ObjectOptions {
17819                    version_id: Some(version_id.clone()),
17820                    versioned: true,
17821                    ..Default::default()
17822                },
17823            )
17824            .await
17825            .expect_err("ECStore must load default COMPLIANCE before an exact-version overwrite");
17826        assert!(matches!(err, StorageError::PrefixAccessDenied(_, _)));
17827
17828        let foreign_snapshot = foreign_store
17829            .object_lock_config_snapshot(&bucket)
17830            .await
17831            .expect("load same-name foreign bucket snapshot");
17832        let mut foreign_replacement = PutObjReader::from_vec(b"foreign replacement".to_vec());
17833        target_store
17834            .put_object(
17835                &bucket,
17836                object,
17837                &mut foreign_replacement,
17838                &ObjectOptions {
17839                    version_id: Some(version_id.clone()),
17840                    versioned: true,
17841                    object_lock_config_snapshot: Some(foreign_snapshot),
17842                    ..Default::default()
17843                },
17844            )
17845            .await
17846            .expect_err("a snapshot from another store must fail closed");
17847
17848        let mut reader = target_store
17849            .get_object_reader(
17850                &bucket,
17851                object,
17852                None,
17853                HeaderMap::new(),
17854                &ObjectOptions {
17855                    version_id: Some(version_id),
17856                    versioned: true,
17857                    ..Default::default()
17858                },
17859            )
17860            .await
17861            .expect("protected target version should remain readable");
17862        let mut body = Vec::new();
17863        reader
17864            .stream
17865            .read_to_end(&mut body)
17866            .await
17867            .expect("protected body should drain");
17868        assert_eq!(body, original_body);
17869    }
17870
17871    #[cfg(feature = "test-util")]
17872    #[tokio::test]
17873    #[serial_test::serial(storage_class_env)]
17874    async fn object_lock_snapshot_uses_read_quorum_bucket_existence_probe() {
17875        let temp = tempfile::tempdir().expect("create degraded snapshot store dir");
17876        let (ctx, store, _shutdown) = without_storage_class_env(build_isolated_test_store_with_layout(
17877            temp.path(),
17878            "degraded-object-lock-snapshot",
17879            &[(2, 12)],
17880            CancellationToken::new(),
17881            None,
17882        ))
17883        .await;
17884        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
17885
17886        let bucket = format!("degraded-ol-{}", uuid::Uuid::new_v4());
17887        store
17888            .make_bucket(&bucket, &MakeBucketOptions::default())
17889            .await
17890            .expect("create snapshot bucket");
17891        let expected_incarnation = store
17892            .bucket_incarnation_id(&bucket)
17893            .await
17894            .expect("read bucket incarnation before degrading sets");
17895
17896        let mut offline_disks = Vec::new();
17897        for set in store.all_set_disks() {
17898            offline_disks.push(force_set_disk_range_offline_for_test(&set, 6..12).await);
17899        }
17900
17901        let snapshot = store
17902            .object_lock_config_snapshot(&bucket)
17903            .await
17904            .expect("read-quorum bucket existence should admit guarded Object Lock snapshot");
17905        assert!(matches!(
17906            snapshot.state(),
17907            crate::bucket::metadata_sys::ObjectLockConfigState::ConfirmedAbsent
17908        ));
17909        assert!(snapshot.is_valid_for_destructive_put(store.id, &bucket, expected_incarnation));
17910
17911        let current_incarnation = crate::bucket::metadata_sys::get_object_lock_config_and_incarnation_from_disk_in(&ctx, &bucket)
17912            .await
17913            .expect("authoritative metadata read should also survive at read quorum")
17914            .1;
17915        assert_eq!(current_incarnation, expected_incarnation);
17916
17917        drop(offline_disks);
17918    }
17919
17920    #[tokio::test]
17921    #[serial_test::serial(storage_class_env)]
17922    async fn force_create_existing_bucket_preserves_incarnation_and_inflight_request() {
17923        let temp = tempfile::tempdir().expect("create temp store dir");
17924        let (_ctx, store, _shutdown) =
17925            without_storage_class_env(build_isolated_test_store(temp.path(), "force-create-incarnation", &[4])).await;
17926        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
17927
17928        let bucket = format!("force-create-incarnation-{}", uuid::Uuid::new_v4());
17929        store
17930            .make_bucket(&bucket, &MakeBucketOptions::default())
17931            .await
17932            .expect("create initial bucket");
17933        let incarnation = store.bucket_incarnation_id(&bucket).await.expect("read initial incarnation");
17934
17935        store
17936            .make_bucket(
17937                &bucket,
17938                &MakeBucketOptions {
17939                    force_create: true,
17940                    ..Default::default()
17941                },
17942            )
17943            .await
17944            .expect("idempotent force-create should succeed");
17945        assert_eq!(
17946            store.bucket_incarnation_id(&bucket).await.unwrap(),
17947            incarnation,
17948            "idempotent force-create must not rotate the bucket generation"
17949        );
17950
17951        let mut reader = PutObjReader::from_vec(b"authorized before force-create".to_vec());
17952        store
17953            .put_object(
17954                &bucket,
17955                "inflight.bin",
17956                &mut reader,
17957                &ObjectOptions {
17958                    expected_bucket_incarnation_id: Some(incarnation),
17959                    ..Default::default()
17960                },
17961            )
17962            .await
17963            .expect("a request authorized before idempotent force-create must still commit");
17964    }
17965
17966    #[tokio::test]
17967    #[serial_test::serial(storage_class_env)]
17968    async fn lock_enabled_create_retries_metadata_intent_without_unlocking() {
17969        let temp = tempfile::tempdir().expect("create temp store dir");
17970        let (ctx, store, _shutdown) =
17971            without_storage_class_env(build_isolated_test_store(temp.path(), "lock-create-intent-retry", &[4])).await;
17972        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
17973
17974        let bucket = format!("lock-create-intent-retry-{}", uuid::Uuid::new_v4());
17975        let mut intent = crate::bucket::metadata::BucketMetadata::new(&bucket);
17976        intent.lock_enabled = true;
17977        crate::bucket::metadata_sys::set_new_bucket_metadata_in(&ctx, intent)
17978            .await
17979            .expect("persist pre-visibility lock intent");
17980        assert!(
17981            store
17982                .peer_sys
17983                .get_bucket_info(&bucket, &crate::storage_api_contracts::bucket::BucketOptions::default())
17984                .await
17985                .is_err(),
17986            "the intent must not make the physical bucket visible"
17987        );
17988
17989        store
17990            .make_bucket(
17991                &bucket,
17992                &MakeBucketOptions {
17993                    lock_enabled: true,
17994                    ..Default::default()
17995                },
17996            )
17997            .await
17998            .expect("retry lock-enabled creation");
17999        assert!(matches!(
18000            crate::bucket::metadata_sys::get_object_lock_config_state_in(&ctx, &bucket)
18001                .await
18002                .expect("read retried lock state"),
18003            crate::bucket::metadata_sys::ObjectLockConfigState::Configured { .. }
18004        ));
18005    }
18006
18007    #[tokio::test]
18008    #[serial_test::serial(storage_class_env)]
18009    async fn physical_lock_bucket_with_incomplete_metadata_fails_closed() {
18010        let temp = tempfile::tempdir().expect("create temp store dir");
18011        let (ctx, store, _shutdown) =
18012            without_storage_class_env(build_isolated_test_store(temp.path(), "lock-create-incomplete-metadata", &[4])).await;
18013        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
18014
18015        let bucket = format!("lock-create-incomplete-metadata-{}", uuid::Uuid::new_v4());
18016        let mut metadata = crate::bucket::metadata::BucketMetadata::new(&bucket);
18017        metadata.lock_enabled = true;
18018        metadata
18019            .save_with_store(store.clone())
18020            .await
18021            .expect("persist lock metadata before sidecar");
18022        store
18023            .peer_sys
18024            .make_bucket(&bucket, &MakeBucketOptions::default())
18025            .await
18026            .expect("expose physical bucket after metadata phase");
18027
18028        let err = crate::bucket::metadata_sys::get_object_lock_config_state_in(&ctx, &bucket)
18029            .await
18030            .expect_err("missing sidecar must not fabricate an unlocked bucket");
18031        assert!(err.to_string().contains("sidecar is missing"), "unexpected error: {err}");
18032    }
18033
18034    #[tokio::test]
18035    #[serial_test::serial(storage_class_env)]
18036    async fn successful_lock_enabled_create_has_complete_metadata_intent() {
18037        let temp = tempfile::tempdir().expect("create temp store dir");
18038        let (ctx, store, _shutdown) =
18039            without_storage_class_env(build_isolated_test_store(temp.path(), "lock-create-complete-metadata", &[4])).await;
18040        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
18041
18042        let bucket = format!("lock-create-{}", uuid::Uuid::new_v4());
18043        store
18044            .make_bucket(
18045                &bucket,
18046                &MakeBucketOptions {
18047                    lock_enabled: true,
18048                    ..Default::default()
18049                },
18050            )
18051            .await
18052            .expect("create lock-enabled bucket");
18053        let (metadata, persisted) = crate::bucket::metadata_sys::get_config_from_disk_with_presence_in(&ctx, &bucket)
18054            .await
18055            .expect("read completed metadata intent");
18056        assert!(persisted && metadata.bucket_incarnation_sidecar && metadata.lock_enabled);
18057        assert!(matches!(
18058            crate::bucket::metadata_sys::get_object_lock_config_state_in(&ctx, &bucket)
18059                .await
18060                .expect("read completed lock state"),
18061            crate::bucket::metadata_sys::ObjectLockConfigState::Configured { .. }
18062        ));
18063    }
18064
18065    #[tokio::test]
18066    #[serial_test::serial(storage_class_env)]
18067    async fn lifecycle_expiry_rejects_task_from_recreated_bucket_incarnation() {
18068        let temp = tempfile::tempdir().expect("create temp store dir");
18069        let (_ctx, store, _shutdown) =
18070            without_storage_class_env(build_isolated_test_store(temp.path(), "lifecycle-expiry-incarnation", &[4])).await;
18071        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
18072
18073        let bucket = format!("lifecycle-incarnation-{}", uuid::Uuid::new_v4());
18074        let object = "same-key.bin";
18075        store
18076            .make_bucket(&bucket, &MakeBucketOptions::default())
18077            .await
18078            .expect("create initial bucket");
18079        let old_incarnation = store.bucket_incarnation_id(&bucket).await.expect("read old incarnation");
18080        let mut old_reader = PutObjReader::from_vec(b"old-generation".to_vec());
18081        store
18082            .put_object(&bucket, object, &mut old_reader, &ObjectOptions::default())
18083            .await
18084            .expect("put old-generation object");
18085        let old_object = store
18086            .get_object_info(&bucket, object, &ObjectOptions::default())
18087            .await
18088            .expect("snapshot old-generation object");
18089
18090        store
18091            .delete_object(&bucket, object, ObjectOptions::default())
18092            .await
18093            .expect("delete old-generation object");
18094        store
18095            .delete_bucket(&bucket, &crate::storage_api_contracts::bucket::DeleteBucketOptions::default())
18096            .await
18097            .expect("delete old bucket");
18098        store
18099            .make_bucket(&bucket, &MakeBucketOptions::default())
18100            .await
18101            .expect("recreate bucket");
18102        let mut new_reader = PutObjReader::from_vec(b"new-generation".to_vec());
18103        store
18104            .put_object(&bucket, object, &mut new_reader, &ObjectOptions::default())
18105            .await
18106            .expect("put replacement object");
18107
18108        let applied = crate::bucket::lifecycle::bucket_lifecycle_ops::apply_expiry_on_non_transitioned_objects(
18109            store.clone(),
18110            &old_object,
18111            &crate::bucket::lifecycle::lifecycle::Event {
18112                action: crate::bucket::lifecycle::lifecycle::IlmAction::DeleteAction,
18113                ..Default::default()
18114            },
18115            &crate::bucket::lifecycle::bucket_lifecycle_audit::LcEventSrc::Scanner,
18116            old_incarnation,
18117        )
18118        .await;
18119        assert!(!applied, "an expiry task from the old bucket incarnation must fail closed");
18120
18121        let mut reader = store
18122            .get_object_reader(&bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
18123            .await
18124            .expect("new-generation object must survive stale expiry");
18125        let mut body = Vec::new();
18126        reader.stream.read_to_end(&mut body).await.expect("read replacement object");
18127        assert_eq!(body, b"new-generation");
18128    }
18129
18130    #[tokio::test]
18131    #[serial_test::serial(storage_class_env)]
18132    async fn pre_upgrade_multipart_upload_can_complete_and_abort_in_the_same_bucket_lifetime() {
18133        let temp = tempfile::tempdir().expect("create temp store dir");
18134        let (_ctx, store, _shutdown) =
18135            without_storage_class_env(build_isolated_test_store(temp.path(), "multipart-upgrade-compat", &[4])).await;
18136        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
18137
18138        let bucket = format!("multipart-upgrade-compat-{}", uuid::Uuid::new_v4());
18139        let created = OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("fixed timestamp should be valid");
18140        store
18141            .make_bucket(
18142                &bucket,
18143                &MakeBucketOptions {
18144                    created_at: Some(created),
18145                    ..Default::default()
18146                },
18147            )
18148            .await
18149            .expect("create bucket");
18150
18151        let complete_object = "complete.bin";
18152        let complete_upload = store.pools[0]
18153            .new_multipart_upload(
18154                &bucket,
18155                complete_object,
18156                &ObjectOptions {
18157                    mod_time: Some(created + time::Duration::seconds(1)),
18158                    ..Default::default()
18159                },
18160            )
18161            .await
18162            .expect("create a pre-upgrade multipart upload without an incarnation stamp");
18163        let mut part_reader = PutObjReader::from_vec(b"pre-upgrade multipart body".to_vec());
18164        let part = store.pools[0]
18165            .put_object_part(
18166                &bucket,
18167                complete_object,
18168                &complete_upload.upload_id,
18169                1,
18170                &mut part_reader,
18171                &ObjectOptions::default(),
18172            )
18173            .await
18174            .expect("stage a pre-upgrade multipart part");
18175        store
18176            .clone()
18177            .complete_multipart_upload(
18178                &bucket,
18179                complete_object,
18180                &complete_upload.upload_id,
18181                vec![crate::storage_api_contracts::multipart::CompletePart {
18182                    part_num: part.part_num,
18183                    etag: part.etag,
18184                    ..Default::default()
18185                }],
18186                &ObjectOptions::default(),
18187            )
18188            .await
18189            .expect("a pre-upgrade upload from the current bucket lifetime should complete");
18190        store
18191            .get_object_info(&bucket, complete_object, &ObjectOptions::default())
18192            .await
18193            .expect("completed pre-upgrade upload should be visible");
18194
18195        let abort_object = "abort.bin";
18196        let abort_upload = store.pools[0]
18197            .new_multipart_upload(
18198                &bucket,
18199                abort_object,
18200                &ObjectOptions {
18201                    mod_time: Some(created + time::Duration::seconds(2)),
18202                    ..Default::default()
18203                },
18204            )
18205            .await
18206            .expect("create another pre-upgrade multipart upload");
18207        store
18208            .abort_multipart_upload(&bucket, abort_object, &abort_upload.upload_id, &ObjectOptions::default())
18209            .await
18210            .expect("a pre-upgrade upload from the current bucket lifetime should abort");
18211        let err = store
18212            .get_multipart_info(&bucket, abort_object, &abort_upload.upload_id, &ObjectOptions::default())
18213            .await
18214            .expect_err("aborted pre-upgrade upload should be removed");
18215        assert!(matches!(err, StorageError::InvalidUploadID(..)));
18216    }
18217
18218    #[tokio::test]
18219    #[serial_test::serial(storage_class_env)]
18220    async fn complete_multipart_rejects_legacy_upload_from_recreated_bucket_incarnation() {
18221        let temp = tempfile::tempdir().expect("create temp store dir");
18222        let (_ctx, store, _shutdown) =
18223            without_storage_class_env(build_isolated_test_store(temp.path(), "multipart-bucket-incarnation", &[4])).await;
18224        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
18225
18226        let bucket = format!("multipart-incarnation-{}", uuid::Uuid::new_v4());
18227        let object = "object.bin";
18228        let created = OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("fixed timestamp should be valid");
18229        store
18230            .make_bucket(
18231                &bucket,
18232                &MakeBucketOptions {
18233                    created_at: Some(created),
18234                    ..Default::default()
18235                },
18236            )
18237            .await
18238            .expect("create initial bucket");
18239        let old_incarnation = store
18240            .bucket_incarnation_id(&bucket)
18241            .await
18242            .expect("read initial bucket incarnation");
18243        let upload = store.pools[0]
18244            .new_multipart_upload(
18245                &bucket,
18246                object,
18247                &ObjectOptions {
18248                    mod_time: Some(created + time::Duration::seconds(1)),
18249                    ..Default::default()
18250                },
18251            )
18252            .await
18253            .expect("create old-generation legacy multipart upload");
18254
18255        store
18256            .delete_bucket(&bucket, &crate::storage_api_contracts::bucket::DeleteBucketOptions::default())
18257            .await
18258            .expect("delete bucket with an incomplete multipart upload");
18259        store
18260            .make_bucket(
18261                &bucket,
18262                &MakeBucketOptions {
18263                    created_at: Some(created + time::Duration::seconds(2)),
18264                    ..Default::default()
18265                },
18266            )
18267            .await
18268            .expect("recreate same-name bucket");
18269        let new_incarnation = store
18270            .bucket_incarnation_id(&bucket)
18271            .await
18272            .expect("read recreated bucket incarnation");
18273        assert_ne!(old_incarnation, new_incarnation);
18274
18275        let err = store
18276            .clone()
18277            .complete_multipart_upload(&bucket, object, &upload.upload_id, Vec::new(), &ObjectOptions::default())
18278            .await
18279            .expect_err("an upload initiated in the old bucket incarnation must not commit into the replacement");
18280        assert!(
18281            match &err {
18282                StorageError::BucketNotFound(name) => name == &bucket,
18283                StorageError::InvalidUploadID(name, key, _) => name == &bucket && key == object,
18284                _ => false,
18285            },
18286            "unexpected stale multipart completion error: {err:?}"
18287        );
18288        assert!(
18289            store
18290                .get_object_info(&bucket, object, &ObjectOptions::default())
18291                .await
18292                .is_err(),
18293            "the recreated bucket must not contain the old-generation multipart upload"
18294        );
18295    }
18296
18297    #[tokio::test]
18298    #[serial_test::serial(storage_class_env)]
18299    async fn delete_after_bucket_validation_preserves_bucket_not_found() {
18300        let temp = tempfile::tempdir().expect("create temp store dir");
18301        let (_ctx, store, _shutdown) =
18302            without_storage_class_env(build_isolated_test_store(temp.path(), "delete-missing-bucket", &[4])).await;
18303        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
18304
18305        let bucket = format!("delete-missing-bucket-{}", uuid::Uuid::new_v4());
18306        store
18307            .make_bucket(&bucket, &MakeBucketOptions::default())
18308            .await
18309            .expect("create bucket before validation");
18310        store
18311            .get_bucket_info(&bucket, &crate::storage_api_contracts::bucket::BucketOptions::default())
18312            .await
18313            .expect("request-level bucket validation should succeed");
18314        store
18315            .delete_bucket(&bucket, &crate::storage_api_contracts::bucket::DeleteBucketOptions::default())
18316            .await
18317            .expect("delete bucket after validation");
18318
18319        let err = store
18320            .delete_object(&bucket, "missing.bin", ObjectOptions::default())
18321            .await
18322            .expect_err("post-validation bucket deletion must not become an object-not-found error");
18323
18324        assert!(matches!(err, StorageError::BucketNotFound(name) if name == bucket));
18325    }
18326
18327    #[tokio::test]
18328    #[serial_test::serial(storage_class_env)]
18329    async fn object_lock_metadata_read_error_blocks_every_destructive_delete_shape() {
18330        let temp = tempfile::tempdir().expect("create temp store dir");
18331        let (ctx, store, _shutdown) =
18332            without_storage_class_env(build_isolated_test_store(temp.path(), "delete-config-read-error", &[4])).await;
18333        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
18334
18335        let bucket = format!("delete-config-read-error-{}", uuid::Uuid::new_v4());
18336        store
18337            .make_bucket(&bucket, &MakeBucketOptions::default())
18338            .await
18339            .expect("create bucket");
18340        for object in ["single.bin", "batch.bin", "prefix/child.bin", "lifecycle.bin"] {
18341            let mut reader = PutObjReader::from_vec(b"must survive".to_vec());
18342            store
18343                .put_object(&bucket, object, &mut reader, &ObjectOptions::default())
18344                .await
18345                .expect("put object before metadata corruption");
18346        }
18347
18348        crate::bucket::metadata_sys::inject_object_lock_disk_read_error_in(&ctx, &bucket)
18349            .await
18350            .expect("inject single-delete metadata read failure");
18351
18352        let single_err = store
18353            .delete_object(&bucket, "single.bin", ObjectOptions::default())
18354            .await
18355            .expect_err("single delete must propagate the metadata read failure");
18356        assert!(
18357            single_err
18358                .to_string()
18359                .contains("injected Object Lock metadata disk read failure")
18360        );
18361
18362        crate::bucket::metadata_sys::inject_object_lock_disk_read_error_in(&ctx, &bucket)
18363            .await
18364            .expect("inject batch-delete metadata read failure");
18365        let (_deleted, batch_errs) = store
18366            .delete_objects(
18367                &bucket,
18368                vec![crate::storage_api_contracts::object::ObjectToDelete {
18369                    object_name: "batch.bin".to_string(),
18370                    ..Default::default()
18371                }],
18372                ObjectOptions::default(),
18373            )
18374            .await;
18375        assert!(
18376            batch_errs[0]
18377                .as_ref()
18378                .is_some_and(|err| err.to_string().contains("injected Object Lock metadata disk read failure"))
18379        );
18380
18381        crate::bucket::metadata_sys::inject_object_lock_disk_read_error_in(&ctx, &bucket)
18382            .await
18383            .expect("inject prefix-delete metadata read failure");
18384        let prefix_err = store
18385            .delete_object(
18386                &bucket,
18387                "prefix",
18388                ObjectOptions {
18389                    delete_prefix: true,
18390                    ..Default::default()
18391                },
18392            )
18393            .await
18394            .expect_err("prefix delete must propagate the metadata read failure");
18395        assert!(
18396            prefix_err
18397                .to_string()
18398                .contains("injected Object Lock metadata disk read failure")
18399        );
18400
18401        crate::bucket::metadata_sys::inject_object_lock_disk_read_error_in(&ctx, &bucket)
18402            .await
18403            .expect("inject lifecycle-style delete-all metadata read failure");
18404        let lifecycle_err = store
18405            .delete_object(
18406                &bucket,
18407                "lifecycle.bin",
18408                ObjectOptions {
18409                    delete_prefix: true,
18410                    delete_prefix_object: true,
18411                    ..Default::default()
18412                },
18413            )
18414            .await
18415            .expect_err("lifecycle-style delete-all must propagate the metadata read failure");
18416        assert!(
18417            lifecycle_err
18418                .to_string()
18419                .contains("injected Object Lock metadata disk read failure")
18420        );
18421
18422        for object in ["single.bin", "batch.bin", "prefix/child.bin", "lifecycle.bin"] {
18423            store
18424                .get_object_info(&bucket, object, &ObjectOptions::default())
18425                .await
18426                .expect("metadata failure must preserve every object");
18427        }
18428    }
18429
18430    #[cfg(feature = "test-util")]
18431    #[tokio::test]
18432    #[serial_test::serial(storage_class_env)]
18433    async fn legacy_tier_delete_journals_create_redacted_recovery_controls_without_remote_calls() {
18434        let temp_dir = tempfile::tempdir().expect("create legacy journal recovery store dir");
18435        let (ctx, store, _shutdown) =
18436            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "legacy-tier-journal-recovery", &[4])).await;
18437        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
18438
18439        let tier_name = "LEGACY-RECOVERY";
18440        let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
18441        let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
18442            .await
18443            .expect("legacy recovery tier lease should resolve")
18444            .backend_identity();
18445        let fixtures = [
18446            serde_json::json!({
18447                "version": 1,
18448                "obj_name": "legacy/remote-v1",
18449                "version_id": "opaque-v1",
18450                "tier_name": tier_name,
18451            }),
18452            serde_json::json!({
18453                "version": 2,
18454                "obj_name": "legacy/remote-v2",
18455                "version_id": "opaque-v2",
18456                "tier_name": tier_name,
18457                "backend_identity": backend_identity,
18458            }),
18459        ];
18460        let mut journal_paths = Vec::new();
18461        for fixture in &fixtures {
18462            let data = serde_json::to_vec(&fixture).expect("legacy journal fixture should encode");
18463            let entry = crate::bucket::lifecycle::tier_delete_journal::decode_tier_delete_journal_entry(&data)
18464                .expect("legacy journal fixture should decode");
18465            let path = tier_delete_journal_object_name(&entry);
18466            com::save_config(store.clone(), &path, data)
18467                .await
18468                .expect("legacy journal fixture should persist");
18469            journal_paths.push(path);
18470        }
18471        let corrupt_path = format!(
18472            "{TIER_DELETE_JOURNAL_PREFIX}/{}.json",
18473            rustfs_utils::crypto::hex_sha256(b"corrupt legacy tier journal", ToOwned::to_owned)
18474        );
18475        com::save_config(store.clone(), &corrupt_path, b"{corrupt".to_vec())
18476            .await
18477            .expect("corrupt legacy journal fixture should persist");
18478
18479        let (first, concurrent) = tokio::join!(
18480            recover_tier_delete_journal_entries(store.clone(), 100, None),
18481            recover_tier_delete_journal_entries(store.clone(), 100, None),
18482        );
18483        for stats in [first, concurrent] {
18484            let stats = stats.expect("concurrent legacy journal recovery scan should finish");
18485            assert_eq!((stats.scanned, stats.deleted, stats.failed), (3, 0, 0));
18486        }
18487        assert_eq!(tier_delete_journal_count(store.clone()).await, 3);
18488        assert_eq!(backend.remove_count().await, 0, "legacy recovery must not call the remote tier");
18489        assert_eq!(backend.exact_remove_count(), 0, "legacy recovery must not issue exact remote DELETE");
18490        assert!(backend.op_log().await.is_empty(), "legacy recovery must not invoke any backend operation");
18491
18492        let mut first_controls = list_recovery_controls(store.clone(), IlmRecoveryProtocol::TierDeleteJournal, None, 100, None)
18493            .await
18494            .expect("legacy recovery controls should be listable")
18495            .records;
18496        first_controls.sort_by(|left, right| left.control_id.cmp(&right.control_id));
18497        assert_eq!(first_controls.len(), 3);
18498        assert_eq!(
18499            first_controls
18500                .iter()
18501                .filter(|control| control.classification == IlmRecoveryClassification::RetainedAmbiguous)
18502                .count(),
18503            2
18504        );
18505        assert_eq!(
18506            first_controls
18507                .iter()
18508                .filter(|control| control.classification == IlmRecoveryClassification::Corrupt)
18509                .count(),
18510            1
18511        );
18512        for view in &first_controls {
18513            assert_eq!(view.protocol, IlmRecoveryProtocol::TierDeleteJournal);
18514            assert_eq!(view.revision, 1);
18515            assert_eq!(view.attempt_count, 0);
18516            let encoded = serde_json::to_string(view).expect("recovery control view should encode");
18517            for secret in ["legacy/remote-v1", "legacy/remote-v2", "opaque-v1", "opaque-v2", tier_name] {
18518                assert!(!encoded.contains(secret), "recovery control view must redact `{secret}`");
18519            }
18520            let persisted = load_recovery_control(store.clone(), IlmRecoveryProtocol::TierDeleteJournal, &view.control_id)
18521                .await
18522                .expect("legacy recovery control should load");
18523            match view.source_schema.as_str() {
18524                "rustfs-tier-delete-journal-v1" => {
18525                    assert_eq!(persisted.control.identity.record_class, "tier_delete_journal_v1");
18526                    assert_eq!(view.last_error_code, IlmRecoveryErrorCode::RemoteVersionUnknown);
18527                }
18528                "rustfs-tier-delete-journal-v2" => {
18529                    assert_eq!(persisted.control.identity.record_class, "tier_delete_journal_v2");
18530                    assert_eq!(view.last_error_code, IlmRecoveryErrorCode::RemoteVersionUnknown);
18531                }
18532                "rustfs-tier-delete-journal-unknown" => {
18533                    assert_eq!(persisted.control.identity.record_class, "tier_delete_journal_corrupt");
18534                    assert_eq!(view.last_error_code, IlmRecoveryErrorCode::SourceCorrupt);
18535                }
18536                schema => panic!("unexpected legacy recovery source schema: {schema}"),
18537            }
18538        }
18539
18540        let creator_sha256 = rustfs_utils::crypto::hex_sha256(b"legacy-export-actor", ToOwned::to_owned);
18541        let mut created_exports = Vec::new();
18542        for exportable in first_controls
18543            .iter()
18544            .filter(|control| control.classification == IlmRecoveryClassification::RetainedAmbiguous)
18545        {
18546            let observation = inspect_recovery_export_observation(store.clone(), &exportable.control_id)
18547                .await
18548                .expect("fresh legacy recovery observation should be exportable");
18549            let created = create_recovery_export(store.clone(), &observation, &creator_sha256)
18550                .await
18551                .expect("legacy recovery export should be created exactly once");
18552            assert!(!created.replayed);
18553            let loaded = load_recovery_export(store.clone(), &created.export_id)
18554                .await
18555                .expect("created legacy recovery export should load");
18556            assert_eq!(loaded.encoded, created.encoded, "export readback must preserve the exact committed bytes");
18557            let replayed = create_recovery_export(store.clone(), &observation, &creator_sha256)
18558                .await
18559                .expect("the same observed generation should replay its immutable export");
18560            assert!(replayed.replayed);
18561            assert_eq!(replayed.encoded, created.encoded);
18562            created_exports.push(created);
18563        }
18564        assert_eq!(created_exports.len(), 2, "both v1 and v2 legacy journals must have an export path");
18565
18566        let corrupt_export_id = &created_exports[0].export_id;
18567        let export_path = recovery_export_record_object_name(IlmRecoveryProtocol::TierDeleteJournal, corrupt_export_id)
18568            .expect("export path should build");
18569        com::save_config(store.clone(), &export_path, Vec::new())
18570            .await
18571            .expect("zero-byte corruption fixture should persist");
18572        let corrupt_export = load_recovery_export(store.clone(), corrupt_export_id)
18573            .await
18574            .expect_err("an existing zero-byte export must fail closed");
18575        assert!(!matches!(corrupt_export, Error::ConfigNotFound));
18576
18577        com::save_config(
18578            store.clone(),
18579            &journal_paths[0],
18580            serde_json::to_vec_pretty(&fixtures[0]).expect("rewritten legacy journal fixture should encode"),
18581        )
18582        .await
18583        .expect("equivalent legacy journal rewrite should persist");
18584        let second = recover_tier_delete_journal_entries(store.clone(), 100, None)
18585            .await
18586            .expect("repeated legacy journal recovery scan should finish");
18587        assert_eq!((second.scanned, second.deleted, second.failed), (3, 0, 0));
18588        let mut second_controls = list_recovery_controls(store, IlmRecoveryProtocol::TierDeleteJournal, None, 100, None)
18589            .await
18590            .expect("repeated legacy recovery controls should remain listable")
18591            .records;
18592        second_controls.sort_by(|left, right| left.control_id.cmp(&right.control_id));
18593        assert_eq!(second_controls, first_controls, "repeated scans must not reset durable controls");
18594        assert_eq!(backend.remove_count().await, 0, "repeated recovery must remain remote-call free");
18595        assert_eq!(backend.exact_remove_count(), 0);
18596        assert!(
18597            backend.op_log().await.is_empty(),
18598            "repeated recovery must not invoke any backend operation"
18599        );
18600    }
18601
18602    #[cfg(feature = "test-util")]
18603    #[tokio::test]
18604    #[serial_test::serial(storage_class_env)]
18605    async fn legacy_recovery_disposition_removes_only_local_journals_and_replays() {
18606        Box::pin(legacy_recovery_disposition_removes_only_local_journals_and_replays_case()).await;
18607    }
18608
18609    #[cfg(feature = "test-util")]
18610    async fn legacy_recovery_disposition_removes_only_local_journals_and_replays_case() {
18611        let temp_dir = tempfile::tempdir().expect("create legacy disposition store dir");
18612        let (ctx, store, _shutdown) =
18613            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "legacy-recovery-disposition", &[4])).await;
18614        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
18615
18616        let tier_name = "LEGACY-DISPOSITION";
18617        let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
18618        let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
18619            .await
18620            .expect("legacy disposition tier lease should resolve")
18621            .backend_identity();
18622        let fixtures = [
18623            serde_json::json!({
18624                "version": 1,
18625                "obj_name": "legacy/disposition-v1",
18626                "version_id": "opaque-disposition-v1",
18627                "tier_name": tier_name,
18628            }),
18629            serde_json::json!({
18630                "version": 2,
18631                "obj_name": "legacy/disposition-v2",
18632                "version_id": "opaque-disposition-v2",
18633                "tier_name": tier_name,
18634                "backend_identity": backend_identity,
18635            }),
18636        ];
18637        let mut journal_paths = Vec::new();
18638        for fixture in &fixtures {
18639            let data = serde_json::to_vec(fixture).expect("legacy disposition fixture should encode");
18640            let entry = crate::bucket::lifecycle::tier_delete_journal::decode_tier_delete_journal_entry(&data)
18641                .expect("legacy disposition fixture should decode");
18642            let path = tier_delete_journal_object_name(&entry);
18643            com::save_config(store.clone(), &path, data)
18644                .await
18645                .expect("legacy disposition fixture should persist");
18646            journal_paths.push(path);
18647        }
18648
18649        let recovered = recover_tier_delete_journal_entries(store.clone(), 100, None)
18650            .await
18651            .expect("legacy disposition recovery scan should finish");
18652        assert_eq!((recovered.scanned, recovered.deleted, recovered.failed), (2, 0, 0));
18653        assert_eq!(tier_delete_journal_count(store.clone()).await, 2);
18654
18655        let mut controls = list_recovery_controls(
18656            store.clone(),
18657            IlmRecoveryProtocol::TierDeleteJournal,
18658            Some(IlmRecoveryClassification::RetainedAmbiguous),
18659            100,
18660            None,
18661        )
18662        .await
18663        .expect("legacy disposition controls should be listable")
18664        .records;
18665        controls.sort_by(|left, right| left.control_id.cmp(&right.control_id));
18666        assert_eq!(controls.len(), 2, "both legacy schemas must support disposition");
18667
18668        let actor_sha256 = rustfs_utils::crypto::hex_sha256(b"legacy-disposition-actor", ToOwned::to_owned);
18669        let wrong_actor_sha256 = rustfs_utils::crypto::hex_sha256(b"different-disposition-actor", ToOwned::to_owned);
18670        let wrong_export_sha256 = "ff".repeat(32);
18671        for (index, control) in controls.iter().enumerate() {
18672            let observation = inspect_recovery_export_observation(store.clone(), &control.control_id)
18673                .await
18674                .expect("legacy disposition source should be observable");
18675            let export = create_recovery_export(store.clone(), &observation, &actor_sha256)
18676                .await
18677                .expect("legacy disposition export should persist");
18678            let confirmed_at_unix_nanos = i64::try_from(OffsetDateTime::now_utc().unix_timestamp_nanos())
18679                .expect("legacy disposition timestamp should fit i64");
18680
18681            if index == 0 {
18682                let wrong_hash = Box::pin(execute_recovery_disposition(
18683                    store.clone(),
18684                    &observation,
18685                    &export.export_id,
18686                    &wrong_export_sha256,
18687                    &actor_sha256,
18688                    confirmed_at_unix_nanos,
18689                ))
18690                .await
18691                .expect_err("a mismatched export checksum must fail before local deletion");
18692                assert_eq!(wrong_hash, Error::PreconditionFailed);
18693                assert_eq!(tier_delete_journal_count(store.clone()).await, 2);
18694            }
18695
18696            let dry_run = dry_run_recovery_disposition(
18697                store.clone(),
18698                &observation,
18699                &export.export_id,
18700                &export.content_sha256,
18701                &actor_sha256,
18702                confirmed_at_unix_nanos,
18703            )
18704            .await
18705            .expect("legacy disposition dry-run should validate exact local state");
18706            assert_eq!(dry_run.source_copy_count, observation.source_generation.copies.len());
18707            assert_eq!(
18708                tier_delete_journal_count(store.clone()).await,
18709                fixtures.len() - index,
18710                "dry-run must not delete a legacy journal"
18711            );
18712            assert!(
18713                matches!(
18714                    load_recovery_disposition(store.clone(), IlmRecoveryProtocol::TierDeleteJournal, &dry_run.disposition_id,)
18715                        .await,
18716                    Err(Error::ConfigNotFound)
18717                ),
18718                "dry-run must not persist a disposition record"
18719            );
18720
18721            if index == 0 {
18722                inject_recovery_disposition_crash_once(RecoveryDispositionCrashStage::AfterLocalDelete);
18723                Box::pin(execute_recovery_disposition(
18724                    store.clone(),
18725                    &observation,
18726                    &export.export_id,
18727                    &export.content_sha256,
18728                    &actor_sha256,
18729                    confirmed_at_unix_nanos,
18730                ))
18731                .await
18732                .expect_err("the injected crash must stop after local delete commits");
18733                assert_eq!(tier_delete_journal_count(store.clone()).await, 1);
18734                let interrupted =
18735                    load_recovery_disposition(store.clone(), IlmRecoveryProtocol::TierDeleteJournal, &dry_run.disposition_id)
18736                        .await
18737                        .expect("the applying disposition must survive the post-delete crash");
18738                assert_eq!(interrupted.disposition.state, IlmRecoveryDispositionState::Applying);
18739                assert!(
18740                    interrupted.disposition.confirmed_absent.is_empty(),
18741                    "the crash must occur before absence progress is persisted"
18742                );
18743            } else {
18744                inject_recovery_disposition_crash_once(RecoveryDispositionCrashStage::AfterControlAbandon);
18745                Box::pin(execute_recovery_disposition(
18746                    store.clone(),
18747                    &observation,
18748                    &export.export_id,
18749                    &export.content_sha256,
18750                    &actor_sha256,
18751                    confirmed_at_unix_nanos,
18752                ))
18753                .await
18754                .expect_err("the injected crash must stop after control abandonment commits");
18755                let interrupted =
18756                    load_recovery_disposition(store.clone(), IlmRecoveryProtocol::TierDeleteJournal, &dry_run.disposition_id)
18757                        .await
18758                        .expect("the applying disposition must survive the post-control crash");
18759                assert_eq!(interrupted.disposition.state, IlmRecoveryDispositionState::Applying);
18760                assert_eq!(
18761                    interrupted.disposition.confirmed_absent.len(),
18762                    interrupted.disposition.identity.source_generation.copies.len()
18763                );
18764
18765                let abandoned =
18766                    load_recovery_control(store.clone(), IlmRecoveryProtocol::TierDeleteJournal, &observation.control_id)
18767                        .await
18768                        .expect("the abandoned control must survive the injected crash");
18769                let exact_abandoned = abandoned.control.encode().expect("the exact abandoned control should encode");
18770                let mut wrong_history = abandoned.control;
18771                wrong_history.last_error_code = IlmRecoveryErrorCode::CleanupFailed;
18772                let control_path = recovery_control_record_object_name(observation.protocol, &observation.control_id)
18773                    .expect("control path should remain canonical");
18774                com::save_config(
18775                    store.clone(),
18776                    &control_path,
18777                    wrong_history
18778                        .encode()
18779                        .expect("the alternate valid control history should encode"),
18780                )
18781                .await
18782                .expect("the alternate control history fixture should persist");
18783                let wrong_history_err = Box::pin(execute_recovery_disposition(
18784                    store.clone(),
18785                    &observation,
18786                    &export.export_id,
18787                    &export.content_sha256,
18788                    &actor_sha256,
18789                    confirmed_at_unix_nanos + 1,
18790                ))
18791                .await
18792                .expect_err("a different abandoned control history must not bridge to completion");
18793                assert_eq!(wrong_history_err, Error::PreconditionFailed);
18794                com::save_config(store.clone(), &control_path, exact_abandoned)
18795                    .await
18796                    .expect("the exact abandoned control fixture should be restored");
18797            }
18798
18799            let replay_confirmed_at_unix_nanos = confirmed_at_unix_nanos + 2;
18800            let executed = Box::pin(execute_recovery_disposition(
18801                store.clone(),
18802                &observation,
18803                &export.export_id,
18804                &export.content_sha256,
18805                &actor_sha256,
18806                replay_confirmed_at_unix_nanos,
18807            ))
18808            .await
18809            .expect("a later request must resume and complete the interrupted disposition");
18810            assert_eq!(executed.state, IlmRecoveryDispositionState::Completed);
18811            assert_eq!(executed.outcome, IlmRecoveryDispositionExecutionOutcome::Completed);
18812            assert_eq!(executed.confirmed_absent_copy_count, executed.source_copy_count);
18813            assert_eq!(tier_delete_journal_count(store.clone()).await, fixtures.len() - index - 1);
18814            assert!(matches!(
18815                com::read_config(store.clone(), &observation.canonical_source_path).await,
18816                Err(Error::ConfigNotFound)
18817            ));
18818            if index == 0 {
18819                let untouched = journal_paths
18820                    .iter()
18821                    .find(|path| *path != &observation.canonical_source_path)
18822                    .expect("the other legacy journal should remain");
18823                com::read_config(store.clone(), untouched)
18824                    .await
18825                    .expect("disposition must not remove a different legacy journal");
18826            }
18827
18828            let abandoned = load_recovery_control(store.clone(), IlmRecoveryProtocol::TierDeleteJournal, &observation.control_id)
18829                .await
18830                .expect("abandoned recovery control should remain inspectable");
18831            assert_eq!(abandoned.control.classification, IlmRecoveryClassification::Abandoned);
18832            assert_eq!(abandoned.control.revision, observation.control_revision + 1);
18833            assert_eq!(abandoned.control.observed_source_generation, observation.source_generation);
18834
18835            let persisted =
18836                load_recovery_disposition(store.clone(), IlmRecoveryProtocol::TierDeleteJournal, &executed.disposition_id)
18837                    .await
18838                    .expect("completed disposition should remain durable");
18839            assert_eq!(persisted.disposition.state, IlmRecoveryDispositionState::Completed);
18840
18841            let replayed = Box::pin(execute_recovery_disposition(
18842                store.clone(),
18843                &observation,
18844                &export.export_id,
18845                &export.content_sha256,
18846                &actor_sha256,
18847                replay_confirmed_at_unix_nanos + 1,
18848            ))
18849            .await
18850            .expect("same actor should replay the completed disposition");
18851            assert_eq!(replayed.state, IlmRecoveryDispositionState::Completed);
18852            assert_eq!(replayed.outcome, IlmRecoveryDispositionExecutionOutcome::Replayed);
18853
18854            let wrong_actor = Box::pin(execute_recovery_disposition(
18855                store.clone(),
18856                &observation,
18857                &export.export_id,
18858                &export.content_sha256,
18859                &wrong_actor_sha256,
18860                replay_confirmed_at_unix_nanos + 2,
18861            ))
18862            .await
18863            .expect_err("a different actor must not replay a completed disposition");
18864            assert_eq!(wrong_actor, Error::PreconditionFailed);
18865
18866            assert!(
18867                !Box::pin(garbage_collect_completed_recovery_disposition(
18868                    store.clone(),
18869                    &persisted,
18870                    persisted.disposition.retain_until_unix_nanos - 1,
18871                ))
18872                .await
18873                .expect("completed disposition should remain before retention expires")
18874            );
18875            assert!(
18876                Box::pin(garbage_collect_completed_recovery_disposition(
18877                    store.clone(),
18878                    &persisted,
18879                    persisted.disposition.retain_until_unix_nanos,
18880                ))
18881                .await
18882                .expect("expired completed disposition should be garbage collected")
18883            );
18884            assert!(matches!(
18885                load_recovery_disposition(store.clone(), IlmRecoveryProtocol::TierDeleteJournal, &executed.disposition_id).await,
18886                Err(Error::ConfigNotFound)
18887            ));
18888            assert_eq!(backend.remove_count().await, 0, "legacy disposition must not call the remote tier");
18889            assert_eq!(backend.exact_remove_count(), 0, "legacy disposition must not issue exact remote DELETE");
18890            assert!(
18891                backend.op_log().await.is_empty(),
18892                "legacy disposition must not invoke any backend operation"
18893            );
18894        }
18895    }
18896
18897    #[cfg(feature = "test-util")]
18898    #[tokio::test]
18899    #[serial_test::serial(storage_class_env)]
18900    async fn v4_tier_delete_journal_recovery_is_exact_and_store_scoped() {
18901        let temp_a = tempfile::tempdir().expect("create temp store dir a");
18902        let temp_b = tempfile::tempdir().expect("create temp store dir b");
18903        let (ctx_a, store_a, _shutdown_a) =
18904            without_storage_class_env(build_isolated_test_store(temp_a.path(), "tier-journal-recovery-a", &[4])).await;
18905        let (ctx_b, store_b, _shutdown_b) =
18906            without_storage_class_env(build_isolated_test_store(temp_b.path(), "tier-journal-recovery-b", &[4])).await;
18907        crate::bucket::metadata_sys::init_bucket_metadata_sys(store_a.clone(), Vec::new()).await;
18908        crate::bucket::metadata_sys::init_bucket_metadata_sys(store_b.clone(), Vec::new()).await;
18909
18910        let tier_a = "JOURNAL-A";
18911        let tier_b = "JOURNAL-B";
18912        let backend_a = register_mock_tier(&ctx_a.tier_config_mgr(), tier_a).await;
18913        let backend_b = register_mock_tier(&ctx_b.tier_config_mgr(), tier_b).await;
18914        let identity_a = TierConfigMgr::acquire_operation_lease(&ctx_a.tier_config_mgr(), tier_a)
18915            .await
18916            .expect("store A tier lease should resolve")
18917            .backend_identity();
18918        let identity_b = TierConfigMgr::acquire_operation_lease(&ctx_b.tier_config_mgr(), tier_b)
18919            .await
18920            .expect("store B tier lease should resolve")
18921            .backend_identity();
18922        let entry_a = Jentry {
18923            persisted_version: 0,
18924            obj_name: "remote-a".to_string(),
18925            version_id: "version-a".to_string(),
18926            tier_name: tier_a.to_string(),
18927            backend_identity: Some(identity_a),
18928            version_id_exact: true,
18929            version_state: rustfs_filemeta::TransitionVersionState::Exact,
18930            state: crate::bucket::lifecycle::tier_sweeper::TierDeleteJournalState::Committed,
18931            source: None,
18932            dispatch: None,
18933        };
18934        let entry_b = Jentry {
18935            persisted_version: 0,
18936            obj_name: "remote-b".to_string(),
18937            version_id: "version-b".to_string(),
18938            tier_name: tier_b.to_string(),
18939            backend_identity: Some(identity_b),
18940            version_id_exact: true,
18941            version_state: rustfs_filemeta::TransitionVersionState::Exact,
18942            state: crate::bucket::lifecycle::tier_sweeper::TierDeleteJournalState::Committed,
18943            source: None,
18944            dispatch: None,
18945        };
18946        persist_tier_delete_journal_entry(store_a.clone(), &entry_a)
18947            .await
18948            .expect("store A v4 compatibility fixture should persist");
18949        persist_tier_delete_journal_entry(store_b.clone(), &entry_b)
18950            .await
18951            .expect("store B v4 compatibility fixture should persist");
18952        let entry_a_v3 = Jentry {
18953            obj_name: "remote-a-v3".to_string(),
18954            version_id: "version-a-v3".to_string(),
18955            ..entry_a.clone()
18956        };
18957        let mut raw_v3: serde_json::Value = serde_json::from_slice(
18958            &encode_tier_delete_journal_entry(&entry_a_v3).expect("store A v3 fixture should encode from the v4 shape"),
18959        )
18960        .expect("store A v3 fixture should be JSON");
18961        raw_v3["version"] = serde_json::json!(3);
18962        raw_v3
18963            .as_object_mut()
18964            .expect("store A v3 fixture should be an object")
18965            .remove("version_state");
18966        let path_a_v3 = tier_delete_journal_object_name(&entry_a_v3);
18967        com::save_config(
18968            store_a.clone(),
18969            &path_a_v3,
18970            serde_json::to_vec(&raw_v3).expect("store A raw v3 fixture should encode"),
18971        )
18972        .await
18973        .expect("store A raw v3 compatibility fixture should persist");
18974        let path_a = tier_delete_journal_object_name(&entry_a);
18975        let path_b = tier_delete_journal_object_name(&entry_b);
18976        let recovered_a = recover_tier_delete_journal_entries(store_a.clone(), 100, None)
18977            .await
18978            .expect("store A recovery scan should finish");
18979        let recovered_b = recover_tier_delete_journal_entries(store_b.clone(), 100, None)
18980            .await
18981            .expect("store B recovery scan should finish");
18982        assert_eq!((recovered_a.scanned, recovered_a.deleted, recovered_a.failed), (2, 2, 0));
18983        assert_eq!((recovered_b.scanned, recovered_b.deleted, recovered_b.failed), (1, 1, 0));
18984        let mut removed_a = backend_a.remove_versions().await;
18985        removed_a.sort();
18986        assert_eq!(
18987            removed_a,
18988            vec![
18989                ("remote-a".to_string(), "version-a".to_string()),
18990                ("remote-a-v3".to_string(), "version-a-v3".to_string()),
18991            ]
18992        );
18993        assert_eq!(backend_b.remove_versions().await, vec![("remote-b".to_string(), "version-b".to_string())]);
18994        assert!(matches!(com::read_config(store_a.clone(), &path_a).await, Err(Error::ConfigNotFound)));
18995        assert!(matches!(com::read_config(store_a.clone(), &path_a_v3).await, Err(Error::ConfigNotFound)));
18996        assert!(matches!(com::read_config(store_b.clone(), &path_b).await, Err(Error::ConfigNotFound)));
18997        assert_eq!(tier_delete_journal_count(store_a).await, 0);
18998        assert_eq!(tier_delete_journal_count(store_b).await, 0);
18999    }
19000
19001    #[cfg(feature = "test-util")]
19002    #[tokio::test]
19003    #[serial_test::serial(storage_class_env)]
19004    async fn v5_tier_delete_recovery_retains_live_sources_and_aborts_only_prepared() {
19005        let temp_dir = tempfile::tempdir().expect("create v5 live-source store dir");
19006        let (ctx, store, _shutdown) =
19007            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "v5-live-source-recovery", &[4])).await;
19008        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
19009
19010        let tier_name = "V5-LIVE-SOURCE";
19011        let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
19012        let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
19013            .await
19014            .expect("v5 live-source tier lease should resolve")
19015            .backend_identity();
19016        let bucket = "v5-live-source-recovery-bucket";
19017        store
19018            .make_bucket(bucket, &MakeBucketOptions::default())
19019            .await
19020            .expect("v5 live-source bucket should be created");
19021
19022        let mut paths = Vec::new();
19023        for (index, state) in [
19024            TierDeleteJournalState::Prepared,
19025            TierDeleteJournalState::Dispatched,
19026            TierDeleteJournalState::Committed,
19027        ]
19028        .into_iter()
19029        .enumerate()
19030        {
19031            let object = format!("live-{index}.bin");
19032            let remote_version = uuid::Uuid::new_v4().to_string();
19033            backend.set_put_remote_version(Some(remote_version)).await;
19034            let mut reader = PutObjReader::from_vec(format!("v5 live source {index}").into_bytes());
19035            let source = store
19036                .put_object(bucket, &object, &mut reader, &ObjectOptions::default())
19037                .await
19038                .expect("v5 live source should be written");
19039            store
19040                .transition_object(
19041                    bucket,
19042                    &object,
19043                    &ObjectOptions {
19044                        transition: TransitionOptions {
19045                            status: TRANSITION_PENDING.to_string(),
19046                            tier: tier_name.to_string(),
19047                            etag: source.etag.clone().expect("v5 live source should have an ETag"),
19048                            ..Default::default()
19049                        },
19050                        version_id: source.version_id.map(|version| version.to_string()),
19051                        mod_time: source.mod_time,
19052                        ..Default::default()
19053                    },
19054                )
19055                .await
19056                .expect("v5 live source should transition");
19057            let transitioned = store
19058                .get_object_info(
19059                    bucket,
19060                    &object,
19061                    &ObjectOptions {
19062                        no_lock: true,
19063                        metadata_cache_safe: false,
19064                        ..Default::default()
19065                    },
19066                )
19067                .await
19068                .expect("v5 transitioned source should be readable");
19069            let mut entry = transitioned_delete_journal_entry_for_source(None, false, false, bucket, &object, &transitioned)
19070                .expect("v5 transitioned source should produce a stable journal identity");
19071            entry.persisted_version = 5;
19072            entry.backend_identity = Some(backend_identity);
19073            entry.state = state;
19074            entry.dispatch = None;
19075            let path = tier_delete_journal_object_name(&entry);
19076            com::save_config(
19077                store.clone(),
19078                &path,
19079                encode_tier_delete_journal_entry(&entry).expect("raw v5 live-source journal should encode"),
19080            )
19081            .await
19082            .expect("raw v5 live-source journal should persist");
19083            paths.push((state, path));
19084        }
19085
19086        let stats = recover_tier_delete_journal_entries(store.clone(), 100, None)
19087            .await
19088            .expect("v5 live-source recovery should finish");
19089        assert_eq!((stats.scanned, stats.deleted, stats.failed), (3, 1, 2));
19090        assert_eq!(backend.remove_count().await, 0, "a live source must block every remote DELETE");
19091        for (state, path) in paths {
19092            let observed = com::read_config(store.clone(), &path).await;
19093            if state == TierDeleteJournalState::Prepared {
19094                assert!(matches!(observed, Err(Error::ConfigNotFound)), "Prepared v5 evidence should abort");
19095            } else {
19096                assert!(observed.is_ok(), "{state:?} v5 evidence must remain while its exact source exists");
19097            }
19098        }
19099    }
19100
19101    #[cfg(feature = "test-util")]
19102    #[tokio::test]
19103    #[serial_test::serial(storage_class_env)]
19104    async fn v5_tier_delete_recovery_replays_all_crash_states_without_sources() {
19105        let temp_dir = tempfile::tempdir().expect("create v5 crash-replay store dir");
19106        let (ctx, store, _shutdown) =
19107            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "v5-crash-replay", &[4])).await;
19108        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
19109
19110        let tier_name = "V5-CRASH-REPLAY";
19111        let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
19112        let lease = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
19113            .await
19114            .expect("v5 crash-replay tier lease should resolve");
19115        let backend_identity = lease.backend_identity();
19116        let bucket = "v5-crash-replay-bucket";
19117        store
19118            .make_bucket(bucket, &MakeBucketOptions::default())
19119            .await
19120            .expect("v5 crash-replay bucket should be created");
19121
19122        for (index, state, seed_remote) in [
19123            (0, TierDeleteJournalState::Prepared, true),
19124            (1, TierDeleteJournalState::Dispatched, true),
19125            (2, TierDeleteJournalState::Committed, true),
19126            // Models a restart after exact remote DELETE succeeded but before
19127            // the local Committed journal was removed.
19128            (3, TierDeleteJournalState::Committed, false),
19129        ] {
19130            let remote_version = uuid::Uuid::new_v4().to_string();
19131            let mut entry = synthetic_v6_dispatch_entry(
19132                bucket,
19133                &format!("missing-{index}.bin"),
19134                tier_name,
19135                backend_identity,
19136                &remote_version,
19137            );
19138            entry.persisted_version = 5;
19139            entry.state = state;
19140            entry.dispatch = None;
19141            if seed_remote {
19142                backend.set_put_remote_version(Some(remote_version)).await;
19143                let body = bytes::Bytes::from(format!("v5 remote candidate {index}"));
19144                lease
19145                    .put(
19146                        &entry.obj_name,
19147                        ReaderImpl::Body(body.clone()),
19148                        i64::try_from(body.len()).expect("v5 candidate length should fit i64"),
19149                    )
19150                    .await
19151                    .expect("v5 remote candidate should be seeded");
19152            }
19153            let path = tier_delete_journal_object_name(&entry);
19154            com::save_config(
19155                store.clone(),
19156                &path,
19157                encode_tier_delete_journal_entry(&entry).expect("raw v5 crash journal should encode"),
19158            )
19159            .await
19160            .expect("raw v5 crash journal should persist");
19161        }
19162
19163        let stats = recover_tier_delete_journal_entries(store.clone(), 100, None)
19164            .await
19165            .expect("v5 crash-state recovery should finish");
19166        assert_eq!((stats.scanned, stats.deleted, stats.failed), (4, 4, 0));
19167        assert_eq!(tier_delete_journal_count(store).await, 0);
19168        assert_eq!(backend.object_count().await, 0);
19169        assert_eq!(backend.exact_remove_count(), 4, "every v5 state must use exact-version cleanup");
19170    }
19171
19172    #[cfg(feature = "test-util")]
19173    #[tokio::test]
19174    #[serial_test::serial(storage_class_env)]
19175    async fn dispatched_tier_delete_recovery_finds_directory_source_on_encoded_set() {
19176        let temp_dir = tempfile::tempdir().expect("create temp store dir");
19177        let shutdown = CancellationToken::new();
19178        let (ctx, store, _shutdown) = without_storage_class_env(build_isolated_test_store_with_layout(
19179            temp_dir.path(),
19180            "prepared-directory-recovery",
19181            &[(2, 4)],
19182            shutdown,
19183            None,
19184        ))
19185        .await;
19186        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
19187
19188        let pool = &store.pools[0];
19189        let object = (0..10_000)
19190            .map(|index| format!("directory-{index}/"))
19191            .find(|candidate| {
19192                let encoded = rustfs_utils::path::encode_dir_object(candidate);
19193                !Arc::ptr_eq(&pool.get_disks_by_key(candidate), &pool.get_disks_by_key(&encoded))
19194            })
19195            .expect("test topology should have a directory key whose encoded form hashes to another set");
19196        let encoded = rustfs_utils::path::encode_dir_object(&object);
19197        assert!(!Arc::ptr_eq(&pool.get_disks_by_key(&object), &pool.get_disks_by_key(&encoded)));
19198
19199        let tier_name = "PREPAREDDIRECTORY";
19200        let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
19201        let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
19202            .await
19203            .expect("tier lease should resolve")
19204            .backend_identity();
19205        let bucket = "prepared-directory-recovery-bucket";
19206        store
19207            .make_bucket(bucket, &MakeBucketOptions::default())
19208            .await
19209            .expect("bucket should be created");
19210        let mut reader = PutObjReader::from_vec(b"directory source".to_vec());
19211        let original = store
19212            .put_object(bucket, &object, &mut reader, &ObjectOptions::default())
19213            .await
19214            .expect("directory source should be written");
19215        store
19216            .transition_object(
19217                bucket,
19218                &object,
19219                &ObjectOptions {
19220                    transition: TransitionOptions {
19221                        status: TRANSITION_PENDING.to_string(),
19222                        tier: tier_name.to_string(),
19223                        etag: original.etag.clone().expect("source should have an ETag"),
19224                        ..Default::default()
19225                    },
19226                    mod_time: original.mod_time,
19227                    ..Default::default()
19228                },
19229            )
19230            .await
19231            .expect("directory transition should commit");
19232        let committed = store
19233            .get_object_info(
19234                bucket,
19235                &object,
19236                &ObjectOptions {
19237                    no_lock: true,
19238                    metadata_cache_safe: false,
19239                    ..Default::default()
19240                },
19241            )
19242            .await
19243            .expect("transitioned directory source should be readable");
19244        let mut entry = transitioned_delete_journal_entry_for_source(None, false, false, bucket, &object, &committed)
19245            .expect("transitioned source should produce a prepared journal entry");
19246        entry.backend_identity = Some(backend_identity);
19247        let incarnation = store
19248            .bucket_incarnation_id(bucket)
19249            .await
19250            .expect("bucket incarnation should resolve");
19251        install_test_tier_delete_dispatch_fixture(
19252            store.clone(),
19253            bucket,
19254            incarnation,
19255            &object,
19256            vec![(entry, Some(TierDeleteJournalState::Dispatched))],
19257            TierDeleteDispatchManifestState::DispatchAuthorized,
19258        )
19259        .await
19260        .expect("authorized dispatched journal should persist");
19261
19262        let stats = recover_tier_delete_journal_entries(store.clone(), 100, None)
19263            .await
19264            .expect("prepared recovery should complete");
19265
19266        assert_eq!((stats.scanned, stats.deleted, stats.failed), (1, 0, 1));
19267        assert_eq!(tier_delete_journal_count(store).await, 1);
19268        assert_eq!(backend.remove_count().await, 0);
19269        assert_eq!(backend.object_count().await, 1, "live directory source must retain its remote object");
19270    }
19271
19272    #[cfg(feature = "test-util")]
19273    #[tokio::test]
19274    #[serial_test::serial(storage_class_env)]
19275    async fn dispatched_tier_delete_recovery_checks_later_pool_then_commits_after_source_removal() {
19276        let temp_dir = tempfile::tempdir().expect("create cross-pool recovery store dir");
19277        let (ctx, store, _shutdown) =
19278            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "prepared-cross-pool-recovery", &[4, 4])).await;
19279        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
19280        let tier_name = "PREPAREDCROSSPOOL";
19281        let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
19282        let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
19283            .await
19284            .expect("tier lease should resolve")
19285            .backend_identity();
19286        let bucket = "prepared-cross-pool-recovery-bucket";
19287        let object = "object";
19288        store
19289            .make_bucket(bucket, &MakeBucketOptions::default())
19290            .await
19291            .expect("bucket should be created");
19292        let mut reader = PutObjReader::from_vec(b"later pool source".to_vec());
19293        let original = store.pools[1]
19294            .put_object(bucket, object, &mut reader, &ObjectOptions::default())
19295            .await
19296            .expect("source should be written only to the later pool");
19297        store.pools[1]
19298            .transition_object(
19299                bucket,
19300                object,
19301                &ObjectOptions {
19302                    transition: TransitionOptions {
19303                        status: TRANSITION_PENDING.to_string(),
19304                        tier: tier_name.to_string(),
19305                        etag: original.etag.clone().expect("source should have an ETag"),
19306                        ..Default::default()
19307                    },
19308                    mod_time: original.mod_time,
19309                    ..Default::default()
19310                },
19311            )
19312            .await
19313            .expect("later-pool transition should commit");
19314        let committed = store.pools[1]
19315            .get_object_info(bucket, object, &ObjectOptions::default())
19316            .await
19317            .expect("transitioned source should be readable");
19318        let mut entry = transitioned_delete_journal_entry_for_source(None, false, false, bucket, object, &committed)
19319            .expect("transitioned source should produce a prepared journal");
19320        entry.backend_identity = Some(backend_identity);
19321        let incarnation = store
19322            .bucket_incarnation_id(bucket)
19323            .await
19324            .expect("bucket incarnation should resolve");
19325        install_test_tier_delete_dispatch_fixture(
19326            store.clone(),
19327            bucket,
19328            incarnation,
19329            object,
19330            vec![(entry, Some(TierDeleteJournalState::Dispatched))],
19331            TierDeleteDispatchManifestState::DispatchAuthorized,
19332        )
19333        .await
19334        .expect("authorized dispatched journal should persist");
19335
19336        let retained = recover_tier_delete_journal_entries(store.clone(), 100, None)
19337            .await
19338            .expect("recovery should scan the later pool");
19339        assert_eq!((retained.scanned, retained.deleted, retained.failed), (1, 0, 1));
19340        assert_eq!(backend.remove_count().await, 0);
19341        assert_eq!(backend.object_count().await, 1);
19342        assert_eq!(
19343            tier_delete_journal_count(store.clone()).await,
19344            1,
19345            "an authorized journal must remain while its live source is durable"
19346        );
19347
19348        store.pools[1]
19349            .delete_object(
19350                bucket,
19351                object,
19352                ObjectOptions {
19353                    version_id: committed.version_id.map(|version| version.to_string()),
19354                    expiration: crate::storage_api_contracts::lifecycle::ExpirationOptions { expire: true },
19355                    skip_free_version: true,
19356                    ..Default::default()
19357                },
19358            )
19359            .await
19360            .expect("source version should be removed before recovery retry");
19361        let committed_stats = recover_tier_delete_journal_entries(store.clone(), 100, None)
19362            .await
19363            .expect("journal recovery should commit an absent stable source");
19364        assert_eq!((committed_stats.scanned, committed_stats.deleted, committed_stats.failed), (1, 0, 1));
19365        assert_eq!(backend.remove_count().await, 0);
19366
19367        let manifest_stats = recover_tier_delete_dispatch_manifests(store.clone(), 100, None)
19368            .await
19369            .expect("manifest recovery should authorize committed remote cleanup");
19370        assert_eq!((manifest_stats.advanced, manifest_stats.failed), (1, 0));
19371
19372        let deleted = recover_tier_delete_journal_entries(store.clone(), 100, None)
19373            .await
19374            .expect("completed journal should delete the exact remote version");
19375        assert_eq!((deleted.scanned, deleted.deleted, deleted.failed), (1, 1, 0));
19376        assert_eq!(tier_delete_journal_count(store).await, 0);
19377        assert_eq!(backend.remove_count().await, 1);
19378        assert_eq!(backend.object_count().await, 0);
19379    }
19380
19381    #[cfg(feature = "test-util")]
19382    #[tokio::test]
19383    #[serial_test::serial(storage_class_env)]
19384    async fn dispatched_tier_delete_recovery_retains_journal_on_source_metadata_error() {
19385        let temp_dir = tempfile::tempdir().expect("create metadata-error recovery store dir");
19386        let (ctx, store, _shutdown) =
19387            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "prepared-metadata-error-recovery", &[4])).await;
19388        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
19389        let tier_name = "PREPAREDMETADATAERROR";
19390        let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
19391        let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
19392            .await
19393            .expect("tier lease should resolve")
19394            .backend_identity();
19395        let bucket = "prepared-metadata-error-recovery-bucket";
19396        let object = "object";
19397        store
19398            .make_bucket(bucket, &MakeBucketOptions::default())
19399            .await
19400            .expect("bucket should be created");
19401        let mut reader = PutObjReader::from_vec(b"source with unreadable metadata".to_vec());
19402        let original = store
19403            .put_object(bucket, object, &mut reader, &ObjectOptions::default())
19404            .await
19405            .expect("source should be written");
19406        store
19407            .transition_object(
19408                bucket,
19409                object,
19410                &ObjectOptions {
19411                    transition: TransitionOptions {
19412                        status: TRANSITION_PENDING.to_string(),
19413                        tier: tier_name.to_string(),
19414                        etag: original.etag.clone().expect("source should have an ETag"),
19415                        ..Default::default()
19416                    },
19417                    mod_time: original.mod_time,
19418                    ..Default::default()
19419                },
19420            )
19421            .await
19422            .expect("source transition should commit");
19423        let committed = store
19424            .get_object_info(bucket, object, &ObjectOptions::default())
19425            .await
19426            .expect("transitioned source should be readable");
19427        let mut entry = transitioned_delete_journal_entry_for_source(None, false, false, bucket, object, &committed)
19428            .expect("transitioned source should produce a prepared journal");
19429        entry.backend_identity = Some(backend_identity);
19430        let incarnation = store
19431            .bucket_incarnation_id(bucket)
19432            .await
19433            .expect("bucket incarnation should resolve");
19434        install_test_tier_delete_dispatch_fixture(
19435            store.clone(),
19436            bucket,
19437            incarnation,
19438            object,
19439            vec![(entry, Some(TierDeleteJournalState::Dispatched))],
19440            TierDeleteDispatchManifestState::DispatchAuthorized,
19441        )
19442        .await
19443        .expect("authorized dispatched journal should persist");
19444
19445        for disk_index in 0..4 {
19446            let metadata_path = temp_dir
19447                .path()
19448                .join(format!("pool0/set0/disk{disk_index}/{bucket}/{object}/{STORAGE_FORMAT_FILE}"));
19449            tokio::fs::write(metadata_path, b"not-xl-meta")
19450                .await
19451                .expect("source metadata should be corrupted");
19452        }
19453
19454        let stats = recover_tier_delete_journal_entries(store.clone(), 100, None)
19455            .await
19456            .expect("recovery scan should complete despite the entry failure");
19457
19458        assert_eq!((stats.scanned, stats.deleted, stats.failed), (1, 0, 1));
19459        assert_eq!(tier_delete_journal_count(store).await, 1, "journal must remain dispatched for retry");
19460        assert_eq!(backend.remove_count().await, 0, "unreadable source metadata must block remote deletion");
19461        assert_eq!(backend.object_count().await, 1, "remote source must remain intact");
19462    }
19463
19464    #[cfg(feature = "test-util")]
19465    #[tokio::test]
19466    #[serial_test::serial(storage_class_env)]
19467    async fn tier_delete_recovery_retains_content_that_does_not_match_its_object_name() {
19468        let temp_dir = tempfile::tempdir().expect("create mismatched journal store dir");
19469        let (ctx, store, _shutdown) =
19470            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "mismatched-tier-journal", &[4])).await;
19471        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
19472        let tier_name = "MISMATCHEDJOURNAL";
19473        let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
19474        let lease = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
19475            .await
19476            .expect("tier lease should resolve");
19477        let remote_version = uuid::Uuid::new_v4().to_string();
19478        backend.set_put_remote_version(Some(remote_version.clone())).await;
19479        lease
19480            .put("remote/original", ReaderImpl::Body(bytes::Bytes::from_static(b"remote body")), 11)
19481            .await
19482            .expect("remote body should be seeded");
19483        let entry = Jentry {
19484            persisted_version: 0,
19485            obj_name: "remote/original".to_string(),
19486            version_id: remote_version,
19487            tier_name: tier_name.to_string(),
19488            backend_identity: Some(lease.backend_identity()),
19489            version_id_exact: true,
19490            version_state: rustfs_filemeta::TransitionVersionState::Exact,
19491            state: TierDeleteJournalState::Prepared,
19492            source: Some(TierDeleteSourceIdentity {
19493                bucket: "absent-source-bucket".to_string(),
19494                object: "absent-source-object".to_string(),
19495                version_id: Some(uuid::Uuid::new_v4().to_string()),
19496                versioned: true,
19497                version_suspended: false,
19498                data_dir: Some(uuid::Uuid::new_v4().to_string()),
19499                etag: Some("etag".to_string()),
19500                mod_time: Some(OffsetDateTime::UNIX_EPOCH.to_string()),
19501            }),
19502            dispatch: None,
19503        };
19504        let (_manifest_name, bound) = install_test_tier_delete_dispatch_fixture(
19505            store.clone(),
19506            "absent-source-bucket",
19507            uuid::Uuid::new_v4(),
19508            "absent-source-object",
19509            vec![(entry, Some(TierDeleteJournalState::Dispatched))],
19510            TierDeleteDispatchManifestState::DispatchAuthorized,
19511        )
19512        .await
19513        .expect("authorized dispatched journal should persist");
19514        let journal_name = tier_delete_journal_object_name(&bound[0]);
19515        let data = com::read_config(store.clone(), &journal_name)
19516            .await
19517            .expect("prepared journal should be readable");
19518        let mut value: serde_json::Value = serde_json::from_slice(&data).expect("journal should contain JSON");
19519        value["obj_name"] = serde_json::json!("remote/replaced");
19520        com::save_config(
19521            store.clone(),
19522            &journal_name,
19523            serde_json::to_vec(&value).expect("mismatched journal should encode"),
19524        )
19525        .await
19526        .expect("mismatched journal content should be written under the original name");
19527
19528        let stats = recover_tier_delete_journal_entries(store.clone(), 100, None)
19529            .await
19530            .expect("recovery scan should complete");
19531
19532        assert_eq!((stats.scanned, stats.deleted, stats.failed), (1, 0, 1));
19533        assert_eq!(tier_delete_journal_count(store).await, 1, "mismatched journal must be retained");
19534        assert_eq!(backend.remove_count().await, 0);
19535        assert_eq!(backend.object_count().await, 1);
19536    }
19537
19538    #[cfg(feature = "test-util")]
19539    #[test]
19540    #[serial_test::serial(storage_class_env)]
19541    fn transitioned_history_expiry_uses_journal_or_legacy_free_version() {
19542        std::thread::Builder::new()
19543            .name("transitioned-delete-all-test".to_string())
19544            .stack_size(32 * 1024 * 1024)
19545            .spawn(|| {
19546                let runtime = tokio::runtime::Builder::new_multi_thread()
19547                    .enable_all()
19548                    .worker_threads(2)
19549                    .build()
19550                    .expect("test runtime should build");
19551                runtime.block_on(async {
19552                    let temp_dir = tempfile::tempdir().expect("create transitioned delete-all store dir");
19553                    let (ctx, store, _shutdown) =
19554                        without_storage_class_env(build_isolated_test_store(temp_dir.path(), "transitioned-delete-all", &[4]))
19555                            .await;
19556                    crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
19557                    let tier_name = "DELETEALLTRANSITIONED";
19558                    let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
19559                    let bucket = "transitioned-delete-all-bucket";
19560                    let object = "object";
19561                    store
19562                        .make_bucket(bucket, &MakeBucketOptions::default())
19563                        .await
19564                        .expect("bucket should be created");
19565                    crate::bucket::metadata_sys::update_in(
19566                        &ctx,
19567                        bucket,
19568                        BUCKET_VERSIONING_CONFIG,
19569                        b"<VersioningConfiguration><Status>Enabled</Status></VersioningConfiguration>".to_vec(),
19570                    )
19571                    .await
19572                    .expect("bucket versioning should be enabled");
19573                    crate::bucket::metadata_sys::update_in(
19574                        &ctx,
19575                        bucket,
19576                        BUCKET_LIFECYCLE_CONFIG,
19577                        br#"<LifecycleConfiguration>
19578  <Rule>
19579    <ID>delete-all-versions</ID>
19580    <Status>Enabled</Status>
19581    <Filter><Prefix></Prefix></Filter>
19582    <Expiration><Days>1</Days><ExpiredObjectAllVersions>true</ExpiredObjectAllVersions></Expiration>
19583  </Rule>
19584</LifecycleConfiguration>"#
19585                            .to_vec(),
19586                    )
19587                    .await
19588                    .expect("delete-all lifecycle should be configured");
19589
19590                    let old_time = OffsetDateTime::now_utc() - time::Duration::days(3);
19591                    let mut history_reader = PutObjReader::from_vec(b"transitioned history".to_vec());
19592                    let history = store
19593                        .put_object(
19594                            bucket,
19595                            object,
19596                            &mut history_reader,
19597                            &ObjectOptions {
19598                                versioned: true,
19599                                mod_time: Some(old_time - time::Duration::hours(1)),
19600                                ..Default::default()
19601                            },
19602                        )
19603                        .await
19604                        .expect("historical version should be written");
19605                    let mut current_reader = PutObjReader::from_vec(b"current version".to_vec());
19606                    let current = store
19607                        .put_object(
19608                            bucket,
19609                            object,
19610                            &mut current_reader,
19611                            &ObjectOptions {
19612                                versioned: true,
19613                                mod_time: Some(old_time),
19614                                ..Default::default()
19615                            },
19616                        )
19617                        .await
19618                        .expect("current version should be written");
19619                    store
19620                        .transition_object(
19621                            bucket,
19622                            object,
19623                            &ObjectOptions {
19624                                versioned: true,
19625                                version_id: history.version_id.map(|version_id| version_id.to_string()),
19626                                transition: TransitionOptions {
19627                                    status: TRANSITION_PENDING.to_string(),
19628                                    tier: tier_name.to_string(),
19629                                    etag: history.etag.clone().expect("history should have an ETag"),
19630                                    ..Default::default()
19631                                },
19632                                mod_time: history.mod_time,
19633                                ..Default::default()
19634                            },
19635                        )
19636                        .await
19637                        .expect("historical version should transition");
19638                    assert_eq!(backend.object_count().await, 1);
19639                    assert_eq!(backend.put_versions().await.len(), 1);
19640                    backend.set_remove_failure(true);
19641
19642                    let incarnation = store
19643                        .bucket_incarnation_id_from_disk(bucket)
19644                        .await
19645                        .expect("bucket incarnation should be available");
19646                    for disk_index in 0..4 {
19647                        let metadata_path = temp_dir
19648                            .path()
19649                            .join(format!("pool0/set0/disk{disk_index}/{bucket}/{object}/{STORAGE_FORMAT_FILE}"));
19650                        let encoded = tokio::fs::read(&metadata_path)
19651                            .await
19652                            .expect("transition metadata should be readable");
19653                        let mut metadata = FileMeta::load(&encoded).expect("transition metadata should decode");
19654                        let (version_index, mut transitioned) = metadata
19655                            .find_version(history.version_id)
19656                            .expect("transitioned history should exist");
19657                        // Rewrite the serialized record to model legacy metadata;
19658                        // ordinary writes preserve an already reconciled state.
19659                        rustfs_utils::http::metadata_compat::remove_bytes(
19660                            &mut transitioned.object.as_mut().expect("history should be an object").meta_sys,
19661                            rustfs_utils::http::metadata_compat::SUFFIX_TRANSITIONED_VERSION_STATE,
19662                        );
19663                        metadata.versions[version_index] = rustfs_filemeta::FileMetaShallowVersion::try_from(transitioned)
19664                            .expect("legacy history should re-encode");
19665                        tokio::fs::write(
19666                            &metadata_path,
19667                            metadata.marshal_msg().expect("unknown transition metadata should encode"),
19668                        )
19669                        .await
19670                        .expect("unknown transition metadata should be written");
19671                        let encoded = tokio::fs::read(&metadata_path)
19672                            .await
19673                            .expect("legacy transition metadata should be readable");
19674                        let legacy = FileMeta::load(&encoded)
19675                            .expect("legacy transition metadata should decode")
19676                            .find_version(history.version_id)
19677                            .expect("legacy history should exist")
19678                            .1
19679                            .into_fileinfo(bucket, object, true)
19680                            .expect("legacy history should decode");
19681                        assert_eq!(legacy.transition_version_state, rustfs_filemeta::TransitionVersionState::Unknown);
19682                    }
19683                    let lifecycle_event = crate::bucket::lifecycle::lifecycle::Event {
19684                        action: rustfs_scanner_metrics::metrics::IlmAction::DeleteAllVersionsAction,
19685                        rule_id: "delete-all-versions".to_string(),
19686                        ..Default::default()
19687                    };
19688                    let legacy_applied =
19689                        crate::bucket::lifecycle::bucket_lifecycle_ops::apply_expiry_on_non_transitioned_objects(
19690                            store.clone(),
19691                            &current,
19692                            &lifecycle_event,
19693                            &crate::bucket::lifecycle::bucket_lifecycle_audit::LcEventSrc::Scanner,
19694                            incarnation,
19695                        )
19696                        .await;
19697                    assert!(legacy_applied, "legacy Unknown transition metadata should use the free-version fallback");
19698                    assert_eq!(tier_delete_journal_count(store.clone()).await, 0);
19699                    assert_eq!(backend.remove_count().await, 0);
19700                    let retained = store.pools[0].disk_set[0]
19701                        .load_file_info_versions_exact(bucket, object)
19702                        .await
19703                        .expect("fallback delete-all metadata should remain readable")
19704                        .expect("fallback delete-all should retain a hidden cleanup owner");
19705                    assert_eq!(
19706                        retained
19707                            .versions
19708                            .iter()
19709                            .filter(|version| !version.tier_free_version())
19710                            .count(),
19711                        0
19712                    );
19713                    assert_eq!(
19714                        retained.versions.iter().filter(|version| version.tier_free_version()).count(),
19715                        1,
19716                        "legacy cleanup must retain exactly one hidden free-version"
19717                    );
19718                    assert_eq!(backend.object_count().await, 1, "legacy remote cleanup must remain pending");
19719
19720                    // Use a distinct tier/backend so the legacy worker can be
19721                    // held in a deterministic remote-failure state while the
19722                    // exact journal path independently proves convergence.
19723                    let exact_tier_name = "DELETEALLTRANSITIONEDEXACT";
19724                    let exact_backend = register_mock_tier(&ctx.tier_config_mgr(), exact_tier_name).await;
19725                    let exact_object = "exact-object";
19726                    let mut exact_history_reader = PutObjReader::from_vec(b"exact transitioned history".to_vec());
19727                    let exact_history = store
19728                        .put_object(
19729                            bucket,
19730                            exact_object,
19731                            &mut exact_history_reader,
19732                            &ObjectOptions {
19733                                versioned: true,
19734                                mod_time: Some(old_time - time::Duration::hours(1)),
19735                                ..Default::default()
19736                            },
19737                        )
19738                        .await
19739                        .expect("second historical version should be written");
19740                    let mut exact_current_reader = PutObjReader::from_vec(b"exact current version".to_vec());
19741                    let exact_current = store
19742                        .put_object(
19743                            bucket,
19744                            exact_object,
19745                            &mut exact_current_reader,
19746                            &ObjectOptions {
19747                                versioned: true,
19748                                mod_time: Some(old_time),
19749                                ..Default::default()
19750                            },
19751                        )
19752                        .await
19753                        .expect("second current version should be written");
19754                    store
19755                        .transition_object(
19756                            bucket,
19757                            exact_object,
19758                            &ObjectOptions {
19759                                versioned: true,
19760                                version_id: exact_history.version_id.map(|version_id| version_id.to_string()),
19761                                transition: TransitionOptions {
19762                                    status: TRANSITION_PENDING.to_string(),
19763                                    tier: exact_tier_name.to_string(),
19764                                    etag: exact_history.etag.clone().expect("second history should have an ETag"),
19765                                    ..Default::default()
19766                                },
19767                                mod_time: exact_history.mod_time,
19768                                ..Default::default()
19769                            },
19770                        )
19771                        .await
19772                        .expect("second historical version should transition exactly");
19773                    let exact_remote_version = exact_backend
19774                        .put_versions()
19775                        .await
19776                        .last()
19777                        .cloned()
19778                        .expect("second transition should publish a remote version");
19779                    exact_backend.set_remove_failure(true);
19780                    let applied = crate::bucket::lifecycle::bucket_lifecycle_ops::apply_expiry_on_non_transitioned_objects(
19781                        store.clone(),
19782                        &exact_current,
19783                        &lifecycle_event,
19784                        &crate::bucket::lifecycle::bucket_lifecycle_audit::LcEventSrc::Scanner,
19785                        incarnation,
19786                    )
19787                    .await;
19788
19789                    assert!(applied, "delete-all should remove current and transitioned history");
19790                    assert!(
19791                        store.pools[0].disk_set[0]
19792                            .load_file_info_versions_exact(bucket, exact_object)
19793                            .await
19794                            .expect("exact journal-owned metadata lookup should succeed")
19795                            .is_none(),
19796                        "exact journal ownership must remove the source key without leaving a free-version"
19797                    );
19798                    let versions = store.pools[0].disk_set[0]
19799                        .load_file_info_versions_exact(bucket, object)
19800                        .await
19801                        .expect("remaining fallback metadata should be readable")
19802                        .expect("the prior legacy cleanup owner should remain");
19803                    assert_eq!(
19804                        versions
19805                            .versions
19806                            .iter()
19807                            .filter(|version| !version.tier_free_version())
19808                            .count(),
19809                        0,
19810                        "exact delete-all must remove every visible version"
19811                    );
19812                    assert_eq!(
19813                        versions.versions.iter().filter(|version| version.tier_free_version()).count(),
19814                        1,
19815                        "the exact v6 journal must not add another free-version"
19816                    );
19817                    assert_eq!(tier_delete_journal_count(store.clone()).await, 1);
19818                    assert_eq!(backend.object_count().await, 1, "legacy remote cleanup must remain pending");
19819                    assert_eq!(exact_backend.object_count().await, 1, "exact remote cleanup must remain journal-owned");
19820
19821                    exact_backend.set_remove_failure(false);
19822                    let stats = recover_tier_delete_journal_entries(store.clone(), 100, None)
19823                        .await
19824                        .expect("committed journal should recover");
19825                    assert_eq!(stats.failed, 0);
19826                    assert!(stats.scanned <= 1 && stats.deleted <= 1);
19827                    assert_eq!(tier_delete_journal_count(store).await, 0);
19828                    assert_eq!(
19829                        backend.object_count().await,
19830                        1,
19831                        "journal recovery must leave the legacy free-version remote"
19832                    );
19833                    assert_eq!(exact_backend.object_count().await, 0);
19834                    assert!(exact_backend.exact_remove_count() >= 1);
19835                    assert_eq!(exact_backend.remove_versions().await, vec![exact_remote_version]);
19836                });
19837            })
19838            .expect("test thread should spawn")
19839            .join()
19840            .expect("test thread should complete");
19841    }
19842
19843    #[cfg(feature = "test-util")]
19844    #[tokio::test]
19845    #[serial_test::serial(storage_class_env)]
19846    async fn odm_write_back_requires_one_set_and_enabled_namespace_locking() {
19847        for (layout, locking, supported) in [
19848            (&[(1, 4)][..], true, true),
19849            (&[(1, 4), (1, 4)][..], true, false),
19850            (&[(2, 4)][..], true, false),
19851            (&[(1, 4)][..], false, false),
19852        ] {
19853            temp_env::async_with_vars([("RUSTFS_LOCK_ENABLED", Some(if locking { "true" } else { "false" }))], async {
19854                let dir = tempfile::tempdir().expect("isolated topology");
19855                let shutdown = CancellationToken::new();
19856                let (_ctx, store, _) = without_storage_class_env(build_isolated_test_store_with_layout(
19857                    dir.path(),
19858                    "odm-topology",
19859                    layout,
19860                    shutdown.clone(),
19861                    None,
19862                ))
19863                .await;
19864                assert_eq!(
19865                    store.supports_atomic_create_only_write_back(),
19866                    supported,
19867                    "layout={layout:?}, locking={locking}"
19868                );
19869                shutdown.cancel();
19870            })
19871            .await;
19872        }
19873    }
19874
19875    #[cfg(feature = "test-util")]
19876    #[tokio::test]
19877    #[serial_test::serial(storage_class_env)]
19878    async fn dispatched_tier_delete_recovery_requires_namespace_locking() {
19879        temp_env::async_with_vars([("RUSTFS_LOCK_ENABLED", Some("false"))], async {
19880            let temp_dir = tempfile::tempdir().expect("create lock-disabled store dir");
19881            let (ctx, store, _shutdown) =
19882                without_storage_class_env(build_isolated_test_store(temp_dir.path(), "prepared-recovery-lock-disabled", &[4]))
19883                    .await;
19884            crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
19885            assert!(ctx.lock_manager().is_disabled());
19886
19887            let tier_name = "PREPAREDLOCKDISABLED";
19888            let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
19889            let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
19890                .await
19891                .expect("tier lease should resolve")
19892                .backend_identity();
19893            let entry = Jentry {
19894                persisted_version: 0,
19895                obj_name: "remote/lock-disabled".to_string(),
19896                version_id: uuid::Uuid::new_v4().to_string(),
19897                tier_name: tier_name.to_string(),
19898                backend_identity: Some(backend_identity),
19899                version_id_exact: true,
19900                version_state: rustfs_filemeta::TransitionVersionState::Exact,
19901                state: TierDeleteJournalState::Prepared,
19902                source: Some(TierDeleteSourceIdentity {
19903                    bucket: "absent-source-bucket".to_string(),
19904                    object: "absent-source-object".to_string(),
19905                    version_id: Some(uuid::Uuid::new_v4().to_string()),
19906                    versioned: true,
19907                    version_suspended: false,
19908                    data_dir: Some(uuid::Uuid::new_v4().to_string()),
19909                    etag: Some("etag".to_string()),
19910                    mod_time: Some(OffsetDateTime::UNIX_EPOCH.to_string()),
19911                }),
19912                dispatch: None,
19913            };
19914            install_test_tier_delete_dispatch_fixture(
19915                store.clone(),
19916                "absent-source-bucket",
19917                uuid::Uuid::new_v4(),
19918                "absent-source-object",
19919                vec![(entry, Some(TierDeleteJournalState::Dispatched))],
19920                TierDeleteDispatchManifestState::DispatchAuthorized,
19921            )
19922            .await
19923            .expect("authorized dispatched journal should persist");
19924
19925            let stats = recover_tier_delete_journal_entries(store.clone(), 100, None)
19926                .await
19927                .expect("recovery scan should complete");
19928
19929            assert_eq!((stats.scanned, stats.deleted, stats.failed), (1, 0, 1));
19930            assert_eq!(tier_delete_journal_count(store).await, 1, "journal must remain dispatched for retry");
19931            assert_eq!(backend.remove_count().await, 0, "lock-disabled recovery must not delete remotely");
19932        })
19933        .await;
19934    }
19935
19936    #[cfg(feature = "test-util")]
19937    #[tokio::test]
19938    #[serial_test::serial(storage_class_env)]
19939    async fn lifecycle_delete_all_requires_namespace_locking_before_mutation() {
19940        temp_env::async_with_vars([("RUSTFS_LOCK_ENABLED", Some("false"))], async {
19941            let temp_dir = tempfile::tempdir().expect("create lock-disabled delete-all store dir");
19942            let (ctx, store, _shutdown) =
19943                without_storage_class_env(build_isolated_test_store(temp_dir.path(), "delete-all-lock-disabled", &[4])).await;
19944            crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
19945            assert!(ctx.lock_manager().is_disabled());
19946            let bucket = "delete-all-lock-disabled-bucket";
19947            let object = "object";
19948            store
19949                .make_bucket(bucket, &MakeBucketOptions::default())
19950                .await
19951                .expect("bucket should be created");
19952            let mut reader = PutObjReader::from_vec(b"must survive".to_vec());
19953            let no_lock_opts = ObjectOptions {
19954                no_lock: true,
19955                ..Default::default()
19956            };
19957            let original = store.pools[0]
19958                .put_object(bucket, object, &mut reader, &no_lock_opts)
19959                .await
19960                .expect("source should be written");
19961
19962            let mut delete_opts = ObjectOptions {
19963                delete_prefix: true,
19964                delete_prefix_object: true,
19965                lifecycle_delete_all: Some(crate::object_api::LifecycleDeleteAllRequest {
19966                    version_id: original.version_id,
19967                    delete_marker: false,
19968                    action: rustfs_scanner_metrics::metrics::IlmAction::DeleteAllVersionsAction,
19969                    rule_id: "rule".to_string(),
19970                    phase: crate::object_api::LifecycleDeleteAllPhase::Preflight,
19971                }),
19972                delete_replication_config_snapshot: Some(Arc::new(
19973                    crate::bucket::replication::DeleteReplicationConfigSnapshot::default(),
19974                )),
19975                ..Default::default()
19976            };
19977            delete_opts.ensure_lifecycle_delete_all_journal();
19978
19979            let err = store
19980                .delete_object_with_tier_delete_journal(bucket, object, delete_opts)
19981                .await
19982                .expect_err("delete-all must reject disabled namespace locking");
19983
19984            assert!(err.to_string().contains("requires namespace locking"));
19985            let retained = store.pools[0]
19986                .get_object_info(bucket, object, &no_lock_opts)
19987                .await
19988                .expect("rejected delete-all must retain the source");
19989            assert_eq!(retained.etag, original.etag);
19990            assert_eq!(tier_delete_journal_count(store).await, 0, "rejected delete-all must not prepare journals");
19991        })
19992        .await;
19993    }
19994
19995    #[cfg(feature = "test-util")]
19996    #[tokio::test]
19997    #[serial_test::serial(storage_class_env)]
19998    async fn tier_mutation_intent_record_round_trips_through_config_store() {
19999        let temp_dir = tempfile::tempdir().expect("create temp store dir");
20000        let (_ctx, store, _shutdown) =
20001            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "tier-mutation-intent-record", &[4])).await;
20002        let mutation_id = uuid::Uuid::new_v4();
20003        let intent = TierMutationIntent {
20004            mutation_id,
20005            revision: 1,
20006            kind: TierMutationIntentKind::Edit,
20007            state: TierMutationIntentState::Prepared,
20008            old_config_etag: Some("old-etag".to_string()),
20009            committed_config_etag: None,
20010            candidate_digest: [3; 32],
20011            affected_targets: vec![TierMutationIntentTarget {
20012                tier_name: "COLD-A".to_string(),
20013                old_backend_identity: Some([1; 32]),
20014                new_backend_identity: Some([2; 32]),
20015            }],
20016            expires_at_unix_nanos: 1_780_000_000_000_000_000,
20017        };
20018
20019        save_tier_mutation_intent_record(store.clone(), &intent)
20020            .await
20021            .expect("tier mutation intent record should persist");
20022        let loaded = load_tier_mutation_intent_record(store.clone(), mutation_id)
20023            .await
20024            .expect("tier mutation intent record should load");
20025
20026        assert_eq!(loaded, intent);
20027
20028        delete_tier_mutation_intent_record(store.clone(), mutation_id)
20029            .await
20030            .expect("tier mutation intent record delete should be idempotent");
20031        delete_tier_mutation_intent_record(store.clone(), mutation_id)
20032            .await
20033            .expect("tier mutation intent record delete should tolerate missing records");
20034        let err = load_tier_mutation_intent_record(store, mutation_id)
20035            .await
20036            .expect_err("deleted tier mutation intent record should not load");
20037        assert!(matches!(err, Error::ConfigNotFound));
20038    }
20039
20040    #[cfg(feature = "test-util")]
20041    #[tokio::test]
20042    #[serial_test::serial(storage_class_env)]
20043    async fn tier_probe_intent_store_enforces_create_cas_and_terminal_delete_preconditions() {
20044        let temp_dir = tempfile::tempdir().expect("create temp store dir");
20045        let (_ctx, store, _shutdown) =
20046            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "tier-probe-intent-cas", &[4])).await;
20047        let probe_id = uuid::Uuid::new_v4();
20048        let creator_epoch = uuid::Uuid::new_v4();
20049        let initial = TierProbeIntent {
20050            probe_id,
20051            revision: 1,
20052            state: TierProbeIntentState::UploadOutcomeUnknown,
20053            operation: TierProbeOperationIdentity::Verify {
20054                config_etag: "config-etag".to_string(),
20055                backend_identity: [1; 32],
20056            },
20057            tier_name: "COLD-A".to_string(),
20058            destination_id: [1; 32],
20059            probe_object: format!("rustfs-tier-probe-{probe_id}"),
20060            creator_id: "node-a".to_string(),
20061            creator_epoch,
20062            created_at_unix_nanos: 1_780_000_000_000_000_000,
20063            owner: TierProbeOwnerFence {
20064                owner_id: "node-a".to_string(),
20065                owner_epoch: creator_epoch,
20066                not_after_unix_nanos: 1_780_000_900_000_000_000,
20067            },
20068            remote_version: TierProbeRemoteVersion::default(),
20069        };
20070
20071        save_tier_probe_intent_record_if_absent(store.clone(), &initial)
20072            .await
20073            .expect("initial probe intent should persist with create-only semantics");
20074        let duplicate = save_tier_probe_intent_record_if_absent(store.clone(), &initial)
20075            .await
20076            .expect_err("duplicate create must fail closed");
20077        assert!(matches!(duplicate, Error::PreconditionFailed));
20078
20079        let observed_initial = load_tier_probe_intent_record(store.clone(), probe_id)
20080            .await
20081            .expect("initial probe intent should load with an ETag");
20082        assert_eq!(observed_initial.intent(), &initial);
20083
20084        let nonterminal_delete = delete_tier_probe_intent_record_if_current(store.clone(), &observed_initial)
20085            .await
20086            .expect_err("nonterminal evidence must not be deleted");
20087        assert!(nonterminal_delete.to_string().contains("must be terminal"));
20088
20089        let mut fabricated_current_intent = initial.clone();
20090        fabricated_current_intent.tier_name = "COLD-B".to_string();
20091        let mut fabricated_successor = fabricated_current_intent.clone();
20092        fabricated_successor
20093            .advance(
20094                TierProbeIntentState::Uploaded,
20095                TierProbeRemoteVersion::versioned(uuid::Uuid::new_v4().to_string()),
20096            )
20097            .expect("fabricated successor should be internally valid");
20098        let fabricated_current = observed_initial.with_intent_for_test(fabricated_current_intent.clone());
20099        let crossed_cas = save_tier_probe_intent_record_if_current(store.clone(), &fabricated_current, &fabricated_successor)
20100            .await
20101            .expect_err("a live ETag must not authorize a different caller record");
20102        assert!(matches!(crossed_cas, Error::PreconditionFailed));
20103        assert_eq!(
20104            load_tier_probe_intent_record(store.clone(), probe_id)
20105                .await
20106                .expect("crossed CAS must retain the authoritative record")
20107                .intent(),
20108            &initial
20109        );
20110
20111        let mut fabricated_terminal_intent = fabricated_current_intent;
20112        fabricated_terminal_intent
20113            .advance(TierProbeIntentState::AbortedNoRemote, TierProbeRemoteVersion::default())
20114            .expect("fabricated terminal should be internally valid");
20115        let fabricated_terminal = observed_initial.with_intent_for_test(fabricated_terminal_intent);
20116        let crossed_delete = delete_tier_probe_intent_record_if_current(store.clone(), &fabricated_terminal)
20117            .await
20118            .expect_err("a live ETag must not delete for a different caller record");
20119        assert!(matches!(crossed_delete, Error::PreconditionFailed));
20120        assert_eq!(
20121            load_tier_probe_intent_record(store.clone(), probe_id)
20122                .await
20123                .expect("crossed delete must retain the authoritative record")
20124                .intent(),
20125            &initial
20126        );
20127
20128        let remote_version = TierProbeRemoteVersion::versioned(uuid::Uuid::new_v4().to_string());
20129        let mut uploaded = observed_initial.intent().clone();
20130        uploaded
20131            .advance(TierProbeIntentState::Uploaded, remote_version.clone())
20132            .expect("known PUT result should advance");
20133        save_tier_probe_intent_record_if_current(store.clone(), &observed_initial, &uploaded)
20134            .await
20135            .expect("the matching initial ETag should admit one successor");
20136
20137        let stale_cas = save_tier_probe_intent_record_if_current(store.clone(), &observed_initial, &uploaded)
20138            .await
20139            .expect_err("a consumed ETag must not overwrite the current generation");
20140        assert!(matches!(stale_cas, Error::PreconditionFailed));
20141
20142        let observed_uploaded = load_tier_probe_intent_record(store.clone(), probe_id)
20143            .await
20144            .expect("uploaded generation should load");
20145        assert_eq!(observed_uploaded.intent(), &uploaded);
20146        let mut cleanup = observed_uploaded.intent().clone();
20147        cleanup
20148            .advance(TierProbeIntentState::CleanupPending, remote_version.clone())
20149            .expect("known candidate should become cleanup-pending");
20150        save_tier_probe_intent_record_if_current(store.clone(), &observed_uploaded, &cleanup)
20151            .await
20152            .expect("cleanup generation should persist by exact ETag");
20153
20154        let observed_cleanup = load_tier_probe_intent_record(store.clone(), probe_id)
20155            .await
20156            .expect("cleanup generation should load");
20157        let mut completed = observed_cleanup.intent().clone();
20158        completed
20159            .advance(TierProbeIntentState::Completed, remote_version)
20160            .expect("exact cleanup should become terminal");
20161        save_tier_probe_intent_record_if_current(store.clone(), &observed_cleanup, &completed)
20162            .await
20163            .expect("terminal generation should persist by exact ETag");
20164
20165        let stale_terminal = observed_cleanup.with_intent_for_test(completed.clone());
20166        let stale_delete = delete_tier_probe_intent_record_if_current(store.clone(), &stale_terminal)
20167            .await
20168            .expect_err("a stale ETag must not delete terminal evidence");
20169        assert!(matches!(stale_delete, Error::PreconditionFailed));
20170
20171        let observed_completed = load_tier_probe_intent_record(store.clone(), probe_id)
20172            .await
20173            .expect("terminal generation should remain after stale delete");
20174        assert_eq!(observed_completed.intent(), &completed);
20175        delete_tier_probe_intent_record_if_current(store.clone(), &observed_completed)
20176            .await
20177            .expect("the exact terminal ETag should delete the record");
20178        assert!(matches!(load_tier_probe_intent_record(store, probe_id).await, Err(Error::ConfigNotFound)));
20179    }
20180
20181    #[cfg(feature = "test-util")]
20182    #[tokio::test]
20183    #[serial_test::serial(storage_class_env)]
20184    async fn tier_mutation_intent_record_scan_retains_good_records_and_counts_bad_records() {
20185        let temp_dir = tempfile::tempdir().expect("create temp store dir");
20186        let (_ctx, store, _shutdown) =
20187            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "tier-mutation-intent-scan", &[4])).await;
20188        let build_intent = |mutation_id: uuid::Uuid, tier_name: &str| TierMutationIntent {
20189            mutation_id,
20190            revision: 1,
20191            kind: TierMutationIntentKind::Edit,
20192            state: TierMutationIntentState::Prepared,
20193            old_config_etag: Some("old-etag".to_string()),
20194            committed_config_etag: None,
20195            candidate_digest: [3; 32],
20196            affected_targets: vec![TierMutationIntentTarget {
20197                tier_name: tier_name.to_string(),
20198                old_backend_identity: Some([1; 32]),
20199                new_backend_identity: Some([2; 32]),
20200            }],
20201            expires_at_unix_nanos: 1_780_000_000_000_000_000,
20202        };
20203        let first_id = uuid::Uuid::parse_str("12345678-1234-5678-9abc-def012345678").expect("first uuid should parse");
20204        let second_id = uuid::Uuid::parse_str("22345678-1234-5678-9abc-def012345678").expect("second uuid should parse");
20205        let first = build_intent(first_id, "COLD-A");
20206        let second = build_intent(second_id, "COLD-B");
20207        save_tier_mutation_intent_record(store.clone(), &first)
20208            .await
20209            .expect("first tier mutation intent record should persist");
20210        save_tier_mutation_intent_record(store.clone(), &second)
20211            .await
20212            .expect("second tier mutation intent record should persist");
20213        com::save_config(
20214            store.clone(),
20215            &format!("{TIER_MUTATION_INTENT_RECORD_PREFIX}/00/00/33345678123456789abcdef012345678.json"),
20216            b"{}".to_vec(),
20217        )
20218        .await
20219        .expect("malformed-shard intent record should persist");
20220        com::save_config(
20221            store.clone(),
20222            &format!("{TIER_MUTATION_INTENT_RECORD_PREFIX}/44/44/44445678123456789abcdef012345678.json"),
20223            b"{".to_vec(),
20224        )
20225        .await
20226        .expect("corrupt-json intent record should persist");
20227
20228        let scan = list_tier_mutation_intent_records(store, 100, None)
20229            .await
20230            .expect("tier mutation intent records should scan");
20231        let mut loaded_ids: Vec<_> = scan.intents.into_iter().map(|intent| intent.mutation_id).collect();
20232        loaded_ids.sort();
20233
20234        assert_eq!(scan.scanned, 4);
20235        assert_eq!(scan.failed, 2);
20236        assert_eq!(loaded_ids, vec![first_id, second_id]);
20237        assert!(!scan.truncated);
20238        assert_eq!(scan.next_marker, None);
20239    }
20240
20241    #[cfg(feature = "test-util")]
20242    #[tokio::test]
20243    #[serial_test::serial(storage_class_env)]
20244    async fn tier_mutation_intent_record_advance_is_idempotent_in_config_store() {
20245        let temp_dir = tempfile::tempdir().expect("create temp store dir");
20246        let (_ctx, store, _shutdown) =
20247            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "tier-mutation-intent-advance", &[4])).await;
20248        let mutation_id = uuid::Uuid::new_v4();
20249        let intent = TierMutationIntent {
20250            mutation_id,
20251            revision: 1,
20252            kind: TierMutationIntentKind::Edit,
20253            state: TierMutationIntentState::Prepared,
20254            old_config_etag: Some("old-etag".to_string()),
20255            committed_config_etag: None,
20256            candidate_digest: [3; 32],
20257            affected_targets: vec![TierMutationIntentTarget {
20258                tier_name: "COLD-A".to_string(),
20259                old_backend_identity: Some([1; 32]),
20260                new_backend_identity: Some([2; 32]),
20261            }],
20262            expires_at_unix_nanos: 1_780_000_000_000_000_000,
20263        };
20264        save_tier_mutation_intent_record(store.clone(), &intent)
20265            .await
20266            .expect("prepared tier mutation intent record should persist");
20267        let (loaded_before_advance, stale_etag) = load_tier_mutation_intent_record_with_etag(store.clone(), mutation_id)
20268            .await
20269            .expect("prepared tier mutation intent record should load with etag");
20270        assert_eq!(loaded_before_advance, intent);
20271        assert!(!stale_etag.is_empty());
20272
20273        let (committed, first_advanced) = advance_tier_mutation_intent_record_idempotent(
20274            store.clone(),
20275            mutation_id,
20276            TierMutationIntentState::Committed,
20277            Some("new-etag".to_string()),
20278        )
20279        .await
20280        .expect("first commit should advance the record");
20281        assert!(first_advanced);
20282        assert_eq!(committed.state, TierMutationIntentState::Committed);
20283        assert_eq!(committed.revision, 2);
20284        assert_eq!(committed.committed_config_etag.as_deref(), Some("new-etag"));
20285
20286        let (retried, retry_advanced) = advance_tier_mutation_intent_record_idempotent(
20287            store.clone(),
20288            mutation_id,
20289            TierMutationIntentState::Committed,
20290            Some("new-etag".to_string()),
20291        )
20292        .await
20293        .expect("same commit retry should be idempotent");
20294        assert!(!retry_advanced);
20295        assert_eq!(retried, committed);
20296
20297        let conflict = advance_tier_mutation_intent_record_idempotent(
20298            store.clone(),
20299            mutation_id,
20300            TierMutationIntentState::Committed,
20301            Some("other-etag".to_string()),
20302        )
20303        .await
20304        .expect_err("conflicting commit retry should fail closed");
20305        assert!(matches!(conflict, Error::Io(_)));
20306        assert!(conflict.to_string().contains("committed config etag does not match"));
20307
20308        let mut stale_conflict = intent;
20309        stale_conflict
20310            .advance_idempotent(TierMutationIntentState::Committed, Some("other-etag".to_string()))
20311            .expect("stale conflicting intent should advance locally");
20312        let stale_save = save_tier_mutation_intent_record_if_current(store.clone(), &stale_conflict, &stale_etag)
20313            .await
20314            .expect_err("stale etag must fail closed instead of overwriting the committed record");
20315        assert!(matches!(stale_save, Error::PreconditionFailed));
20316
20317        let loaded = load_tier_mutation_intent_record(store.clone(), mutation_id)
20318            .await
20319            .expect("conflicting retry must not overwrite the durable record");
20320        assert_eq!(loaded, committed);
20321
20322        let abort_id = uuid::Uuid::new_v4();
20323        let abort_intent = TierMutationIntent {
20324            mutation_id: abort_id,
20325            revision: 1,
20326            kind: TierMutationIntentKind::Edit,
20327            state: TierMutationIntentState::Prepared,
20328            old_config_etag: Some("old-etag".to_string()),
20329            committed_config_etag: None,
20330            candidate_digest: [4; 32],
20331            affected_targets: vec![TierMutationIntentTarget {
20332                tier_name: "COLD-B".to_string(),
20333                old_backend_identity: Some([1; 32]),
20334                new_backend_identity: Some([2; 32]),
20335            }],
20336            expires_at_unix_nanos: 1_780_000_000_000_000_000,
20337        };
20338        save_tier_mutation_intent_record(store.clone(), &abort_intent)
20339            .await
20340            .expect("prepared abort intent record should persist");
20341
20342        let (aborted, first_abort_advanced) =
20343            advance_tier_mutation_intent_record_idempotent(store.clone(), abort_id, TierMutationIntentState::Aborted, None)
20344                .await
20345                .expect("first abort should advance the record");
20346        assert!(first_abort_advanced);
20347        assert_eq!(aborted.state, TierMutationIntentState::Aborted);
20348        assert_eq!(aborted.revision, 2);
20349        assert_eq!(aborted.committed_config_etag, None);
20350
20351        let (aborted_retry, retry_abort_advanced) =
20352            advance_tier_mutation_intent_record_idempotent(store, abort_id, TierMutationIntentState::Aborted, None)
20353                .await
20354                .expect("same abort retry should be idempotent");
20355        assert!(!retry_abort_advanced);
20356        assert_eq!(aborted_retry, aborted);
20357    }
20358
20359    #[cfg(feature = "test-util")]
20360    fn tier_mutation_peer_test_intent(
20361        mutation_id: uuid::Uuid,
20362        tier_name: &str,
20363        candidate_digest: [u8; 32],
20364    ) -> TierMutationIntent {
20365        TierMutationIntent {
20366            mutation_id,
20367            revision: 1,
20368            kind: TierMutationIntentKind::Edit,
20369            state: TierMutationIntentState::Prepared,
20370            old_config_etag: Some("old-etag".to_string()),
20371            committed_config_etag: None,
20372            candidate_digest,
20373            affected_targets: vec![TierMutationIntentTarget {
20374                tier_name: tier_name.to_string(),
20375                old_backend_identity: Some([1; 32]),
20376                new_backend_identity: Some([2; 32]),
20377            }],
20378            expires_at_unix_nanos: i64::MAX,
20379        }
20380    }
20381
20382    #[cfg(feature = "test-util")]
20383    #[tokio::test]
20384    #[serial_test::serial(storage_class_env)]
20385    async fn tier_config_initial_reload_lock_failure_keeps_recovery_worker() {
20386        use crate::storage_api_contracts::namespace::NamespaceLocking as _;
20387
20388        let temp_dir = tempfile::tempdir().expect("create contended tier startup store dir");
20389        let (ctx, store, shutdown) =
20390            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "tier-startup-contention", &[4])).await;
20391        let manager = ctx.tier_config_mgr();
20392        register_mock_tier(&manager, "COLD-A").await;
20393        let candidate = TierConfigMgr::new();
20394        let candidate = candidate.read().await;
20395        let candidate_digest = tier_config_candidate_digest(&candidate).expect("empty candidate digest should resolve");
20396        candidate
20397            .save_tiering_config(store.clone())
20398            .await
20399            .expect("committed removal config should persist");
20400        let config_info = store
20401            .get_object_info(
20402                RUSTFS_META_BUCKET,
20403                &format!("{}/{TIER_CONFIG_FILE}", com::CONFIG_PREFIX),
20404                &ObjectOptions::default(),
20405            )
20406            .await
20407            .expect("committed removal config metadata should load");
20408        let mutation_id = uuid::Uuid::new_v4();
20409        let mut intent = tier_mutation_peer_test_intent(mutation_id, "COLD-A", candidate_digest);
20410        intent.kind = TierMutationIntentKind::Remove;
20411        intent.affected_targets[0].new_backend_identity = None;
20412        intent
20413            .advance(
20414                TierMutationIntentState::Committed,
20415                Some(config_info.etag.expect("committed config should have an ETag")),
20416            )
20417            .expect("removal intent should commit");
20418        save_tier_mutation_intent_record(store.clone(), &intent)
20419            .await
20420            .expect("committed removal intent should persist");
20421
20422        let config_lock = format!("{}/{TIER_CONFIG_FILE}.lock", com::CONFIG_PREFIX);
20423        let namespace = store
20424            .new_ns_lock(RUSTFS_META_BUCKET, &config_lock)
20425            .await
20426            .expect("tier config namespace should resolve");
20427        let owner = namespace
20428            .get_write_lock(crate::set_disk::get_lock_acquire_timeout())
20429            .await
20430            .expect("competing recovery should hold the config lock");
20431        let initial_error = runtime_sources::init_tier_config_mgr_handle(manager.clone(), store.clone())
20432            .await
20433            .expect_err("startup must still report the initial recovery lock failure");
20434        assert!(initial_error.to_string().contains(&config_lock));
20435        match TierConfigMgr::acquire_operation_lease(&manager, "COLD-A").await {
20436            Err(err) => assert_eq!(err.message, "Remote tier configuration is being replaced"),
20437            Ok(_) => panic!("failed initial recovery must retain its mutation fence"),
20438        }
20439        drop(owner);
20440
20441        tokio::time::timeout(Duration::from_secs(30), async {
20442            loop {
20443                if manager.read().await.tiers.is_empty()
20444                    && matches!(
20445                        TierConfigMgr::acquire_operation_lease(&manager, "COLD-A").await,
20446                        Err(err) if err.code == crate::services::tier::tier_handlers::ERR_TIER_NOT_FOUND.code
20447                    )
20448                {
20449                    break;
20450                }
20451                tokio::time::sleep(Duration::from_millis(10)).await;
20452            }
20453        })
20454        .await
20455        .expect("recovery worker must publish and clear its fence without another admin mutation");
20456        assert_eq!(
20457            load_tier_mutation_intent_record(store.clone(), mutation_id)
20458                .await
20459                .expect("unexpired committed tombstone must remain durable"),
20460            intent
20461        );
20462        shutdown.cancel();
20463    }
20464
20465    #[cfg(feature = "test-util")]
20466    #[tokio::test]
20467    #[serial_test::serial(storage_class_env)]
20468    async fn tier_mutation_peer_prepare_reloads_after_save_and_load_lock_timeouts() {
20469        use crate::storage_api_contracts::namespace::NamespaceLocking as _;
20470        use std::sync::atomic::Ordering;
20471
20472        let temp_dir = tempfile::tempdir().expect("create contended tier prepare store dir");
20473        let (ctx, store, shutdown) =
20474            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "tier-prepare-contention", &[4])).await;
20475        let mutation_id = uuid::Uuid::new_v4();
20476        let intent = tier_mutation_peer_test_intent(mutation_id, "COLD-A", [9; 32]);
20477        let object = crate::services::tier::tier_mutation_intent::tier_mutation_intent_record_object_name(mutation_id)
20478            .expect("intent record object should resolve");
20479        let barrier = crate::set_disk::PutObjectCommitBarrier::install(
20480            RUSTFS_META_BUCKET,
20481            &object,
20482            crate::set_disk::PutObjectCommitPause::BeforeNamespace,
20483        );
20484        let prepare_store = store.clone();
20485        let payload = intent.encode().expect("prepare intent should encode");
20486        let mut prepare = tokio::spawn(async move {
20487            handle_tier_mutation_peer_request(
20488                prepare_store,
20489                TIER_MUTATION_RPC_PROTOCOL_VERSION,
20490                TierMutationRpcPhase::Prepare,
20491                mutation_id,
20492                &payload,
20493            )
20494            .await
20495        });
20496        barrier.wait_until_paused().await;
20497        let namespace = store.pools[0].disk_set[0]
20498            .new_ns_lock(RUSTFS_META_BUCKET, &object)
20499            .await
20500            .expect("intent record namespace should resolve");
20501        let owner = namespace
20502            .get_write_lock(crate::set_disk::get_lock_acquire_timeout())
20503            .await
20504            .expect("competing publisher should hold the intent namespace");
20505        // The competing publisher already owns the namespace. Bypass only its
20506        // recursive acquisition and drain the durable tail before releasing it.
20507        com::save_config_with_opts(
20508            store.clone(),
20509            &object,
20510            intent.encode().expect("winning intent should encode"),
20511            &ObjectOptions {
20512                no_lock: true,
20513                max_parity: true,
20514                write_completion: crate::object_api::WriteCompletion::TailDrained,
20515                ..Default::default()
20516            },
20517        )
20518        .await
20519        .expect("competing publisher should persist the matching intent under its lock");
20520        let manager = ctx
20521            .lock_manager()
20522            .as_fast_lock_manager()
20523            .expect("fixture should use real locks");
20524        let key = rustfs_lock::fast_lock::types::ObjectKey {
20525            bucket: Arc::from(RUSTFS_META_BUCKET),
20526            object: Arc::from(object.as_str()),
20527            version: None,
20528        };
20529        let timeouts = &manager.get_shard(&key).metrics().timeouts;
20530        let before = timeouts.load(Ordering::Relaxed);
20531        barrier.release_and_wait_until_namespace_pending().await;
20532        drop(barrier);
20533        tokio::select! {
20534            result = &mut prepare => panic!("Prepare must retry the contended save and load before completing: {result:?}"),
20535            result = tokio::time::timeout(Duration::from_secs(30), async {
20536                while timeouts.load(Ordering::Relaxed) - before < 2 {
20537                    tokio::time::sleep(Duration::from_millis(10)).await;
20538                }
20539            }) => result.expect("the save and subsequent load should each exhaust their lock wait"),
20540        }
20541        assert_eq!(timeouts.load(Ordering::Relaxed) - before, 2);
20542        drop(owner);
20543        let outcome = tokio::time::timeout(Duration::from_secs(30), prepare)
20544            .await
20545            .expect("third prepare round should complete after the competing publisher releases its lock")
20546            .expect("prepare task should join")
20547            .expect("third prepare round should reload the matching durable intent");
20548        assert_eq!(outcome.state, TierMutationPeerState::Prepared);
20549        assert!(!outcome.applied, "the retry must reuse the winning record rather than recreate it");
20550        assert_eq!(
20551            load_tier_mutation_intent_record(store.clone(), mutation_id)
20552                .await
20553                .expect("winning record should remain readable"),
20554            intent
20555        );
20556        assert_eq!(timeouts.load(Ordering::Relaxed) - before, 2);
20557        shutdown.cancel();
20558    }
20559
20560    #[cfg(feature = "test-util")]
20561    #[tokio::test]
20562    #[serial_test::serial(storage_class_env)]
20563    async fn tier_mutation_peer_prepare_preserves_lock_timeout_after_three_rounds() {
20564        use crate::storage_api_contracts::namespace::NamespaceLocking as _;
20565        use std::sync::atomic::Ordering;
20566
20567        let temp_dir = tempfile::tempdir().expect("create exhausted tier prepare store dir");
20568        let (ctx, store, shutdown) =
20569            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "tier-prepare-exhaustion", &[4])).await;
20570        let mutation_id = uuid::Uuid::new_v4();
20571        let intent = tier_mutation_peer_test_intent(mutation_id, "COLD-A", [9; 32]);
20572        let object = crate::services::tier::tier_mutation_intent::tier_mutation_intent_record_object_name(mutation_id)
20573            .expect("intent record object should resolve");
20574        let barrier = crate::set_disk::PutObjectCommitBarrier::install(
20575            RUSTFS_META_BUCKET,
20576            &object,
20577            crate::set_disk::PutObjectCommitPause::BeforeNamespace,
20578        );
20579        let prepare_store = store.clone();
20580        let payload = intent.encode().expect("prepare intent should encode");
20581        let prepare = tokio::spawn(async move {
20582            handle_tier_mutation_peer_request(
20583                prepare_store,
20584                TIER_MUTATION_RPC_PROTOCOL_VERSION,
20585                TierMutationRpcPhase::Prepare,
20586                mutation_id,
20587                &payload,
20588            )
20589            .await
20590        });
20591        barrier.wait_until_paused().await;
20592        let namespace = store.pools[0].disk_set[0]
20593            .new_ns_lock(RUSTFS_META_BUCKET, &object)
20594            .await
20595            .expect("intent record namespace should resolve");
20596        let lock_timeout = crate::set_disk::get_lock_acquire_timeout();
20597        let owner = namespace
20598            .get_write_lock(lock_timeout)
20599            .await
20600            .expect("competing publisher should hold the intent namespace");
20601        let manager = ctx
20602            .lock_manager()
20603            .as_fast_lock_manager()
20604            .expect("fixture should use real locks");
20605        let key = rustfs_lock::fast_lock::types::ObjectKey {
20606            bucket: Arc::from(RUSTFS_META_BUCKET),
20607            object: Arc::from(object.as_str()),
20608            version: None,
20609        };
20610        let timeouts = &manager.get_shard(&key).metrics().timeouts;
20611        let before = timeouts.load(Ordering::Relaxed);
20612        barrier.release_and_wait_until_namespace_pending().await;
20613        drop(barrier);
20614        let error = tokio::time::timeout(Duration::from_secs(30), prepare)
20615            .await
20616            .expect("three contended prepare rounds should exhaust the bounded budget")
20617            .expect("prepare task should join")
20618            .expect_err("Prepare must fail while the competing publisher keeps the namespace");
20619        match error {
20620            TierMutationPeerError::Store(Error::Lock(rustfs_lock::LockError::Timeout { resource, timeout })) => {
20621                assert_eq!(resource, key.to_string());
20622                assert_eq!(timeout, lock_timeout);
20623            }
20624            other => panic!("Prepare must preserve the final typed lock timeout: {other:?}"),
20625        }
20626        assert_eq!(timeouts.load(Ordering::Relaxed) - before, 3, "save and load must share three rounds");
20627        drop(owner);
20628        assert!(matches!(
20629            load_tier_mutation_intent_record(store, mutation_id).await,
20630            Err(Error::ConfigNotFound)
20631        ));
20632        shutdown.cancel();
20633    }
20634
20635    #[cfg(feature = "test-util")]
20636    #[tokio::test]
20637    #[serial_test::serial(storage_class_env)]
20638    async fn tier_mutation_abort_tombstone_rejects_delayed_prepare() {
20639        let temp_dir = tempfile::tempdir().expect("create temp store dir");
20640        let (_ctx, store, _shutdown) =
20641            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "tier-mutation-abort-tombstone", &[4])).await;
20642        register_mock_tier(&store.tier_config_mgr(), "COLD-A").await;
20643        store
20644            .tier_config_mgr()
20645            .read()
20646            .await
20647            .save_tiering_config(store.clone())
20648            .await
20649            .expect("base tier config should persist");
20650        let config_info = store
20651            .get_object_info(
20652                RUSTFS_META_BUCKET,
20653                &format!("{}/{}", com::CONFIG_PREFIX, TIER_CONFIG_FILE),
20654                &ObjectOptions::default(),
20655            )
20656            .await
20657            .expect("base tier config metadata should load");
20658        let mutation_id = uuid::Uuid::new_v4();
20659        let mut intent = tier_mutation_peer_test_intent(mutation_id, "COLD-A", [7; 32]);
20660        intent.old_config_etag = config_info.etag;
20661        intent.expires_at_unix_nanos =
20662            i64::try_from((OffsetDateTime::now_utc() + time::Duration::hours(1)).unix_timestamp_nanos())
20663                .expect("future tombstone expiry should fit i64");
20664        let payload = intent.encode().expect("prepare identity should encode");
20665
20666        let mismatch_id = uuid::Uuid::new_v4();
20667        let mut mismatch = intent.clone();
20668        mismatch.mutation_id = mismatch_id;
20669        mismatch.old_config_etag = Some("not-the-current-config-etag".to_string());
20670        let mismatch_error = handle_tier_mutation_peer_request(
20671            store.clone(),
20672            TIER_MUTATION_RPC_PROTOCOL_VERSION,
20673            TierMutationRpcPhase::Abort,
20674            mismatch_id,
20675            &mismatch.encode().expect("proof-mismatch Abort should encode"),
20676        )
20677        .await
20678        .expect_err("a missing-record Abort with the wrong config proof must fail closed");
20679        assert!(matches!(mismatch_error, TierMutationPeerError::AbortProofMismatch));
20680        assert!(matches!(
20681            load_tier_mutation_intent_record(store.clone(), mismatch_id).await,
20682            Err(Error::ConfigNotFound)
20683        ));
20684
20685        let aborted = handle_tier_mutation_peer_request(
20686            store.clone(),
20687            TIER_MUTATION_RPC_PROTOCOL_VERSION,
20688            TierMutationRpcPhase::Abort,
20689            mutation_id,
20690            &payload,
20691        )
20692        .await
20693        .expect("missing-record abort should persist a tombstone");
20694        assert!(aborted.applied);
20695        assert_eq!(aborted.state, TierMutationPeerState::Aborted);
20696        let tombstone = load_tier_mutation_intent_record(store.clone(), mutation_id)
20697            .await
20698            .expect("abort tombstone should be durable");
20699        assert_eq!(tombstone.state, TierMutationIntentState::Aborted);
20700        assert!(tombstone.same_identity_as(&intent));
20701
20702        let manager = store.tier_config_mgr();
20703        TierConfigMgr::reload_handle(&manager, store.clone())
20704            .await
20705            .expect("reload should retain an unexpired abort tombstone");
20706        let tombstone_after_reload = load_tier_mutation_intent_record(store.clone(), mutation_id)
20707            .await
20708            .expect("reload must not clean an unexpired abort tombstone");
20709        assert_eq!(tombstone_after_reload.state, TierMutationIntentState::Aborted);
20710        assert!(tombstone_after_reload.same_identity_as(&intent));
20711
20712        register_mock_tier(&store.tier_config_mgr(), "COLD-B").await;
20713        store
20714            .tier_config_mgr()
20715            .read()
20716            .await
20717            .save_tiering_config(store.clone())
20718            .await
20719            .expect("later config change should persist before idempotent Abort replay");
20720        let retried_abort = handle_tier_mutation_peer_request(
20721            store.clone(),
20722            TIER_MUTATION_RPC_PROTOCOL_VERSION,
20723            TierMutationRpcPhase::Abort,
20724            mutation_id,
20725            &payload,
20726        )
20727        .await
20728        .expect("terminal Abort retry must not depend on the mutable current config proof");
20729        assert!(!retried_abort.applied);
20730        assert_eq!(retried_abort.state, TierMutationPeerState::Aborted);
20731
20732        let delayed = handle_tier_mutation_peer_request(
20733            store.clone(),
20734            TIER_MUTATION_RPC_PROTOCOL_VERSION,
20735            TierMutationRpcPhase::Prepare,
20736            mutation_id,
20737            &payload,
20738        )
20739        .await
20740        .expect("delayed matching prepare should converge on the tombstone");
20741        assert!(!delayed.applied);
20742        assert_eq!(delayed.state, TierMutationPeerState::Aborted);
20743        TierConfigMgr::acquire_operation_lease(&store.tier_config_mgr(), "COLD-A")
20744            .await
20745            .expect("delayed prepare must not reinstall the runtime block");
20746
20747        let mut conflicting = intent.clone();
20748        conflicting.candidate_digest = [8; 32];
20749        let conflict = handle_tier_mutation_peer_request(
20750            store,
20751            TIER_MUTATION_RPC_PROTOCOL_VERSION,
20752            TierMutationRpcPhase::Prepare,
20753            mutation_id,
20754            &conflicting.encode().expect("conflicting prepare should encode"),
20755        )
20756        .await
20757        .expect_err("different identity must not reuse an abort tombstone");
20758        assert!(matches!(conflict, TierMutationPeerError::ConflictingIntent));
20759    }
20760
20761    #[cfg(feature = "test-util")]
20762    #[tokio::test]
20763    #[serial_test::serial(storage_class_env)]
20764    async fn tier_mutation_expired_prepare_cannot_recreate_a_cleaned_tombstone() {
20765        let temp_dir = tempfile::tempdir().expect("create temp store dir");
20766        let (_ctx, store, _shutdown) =
20767            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "tier-mutation-expired-replay", &[4])).await;
20768        register_mock_tier(&store.tier_config_mgr(), "COLD-A").await;
20769        let mutation_id = uuid::Uuid::new_v4();
20770        let mut intent = tier_mutation_peer_test_intent(mutation_id, "COLD-A", [11; 32]);
20771        intent.expires_at_unix_nanos =
20772            i64::try_from((OffsetDateTime::now_utc() - time::Duration::seconds(1)).unix_timestamp_nanos())
20773                .expect("expired timestamp should fit i64");
20774        let payload = intent.encode().expect("expired signed Prepare should still encode");
20775
20776        let error = handle_tier_mutation_peer_request(
20777            store.clone(),
20778            TIER_MUTATION_RPC_PROTOCOL_VERSION,
20779            TierMutationRpcPhase::Prepare,
20780            mutation_id,
20781            &payload,
20782        )
20783        .await
20784        .expect_err("a missing-record expired Prepare must fail closed");
20785        assert!(matches!(error, TierMutationPeerError::ExpiredIntent));
20786        assert!(matches!(
20787            load_tier_mutation_intent_record(store.clone(), mutation_id).await,
20788            Err(Error::ConfigNotFound)
20789        ));
20790        TierConfigMgr::acquire_operation_lease(&store.tier_config_mgr(), "COLD-A")
20791            .await
20792            .expect("expired replay must not install a peer-only runtime fence");
20793    }
20794
20795    #[cfg(feature = "test-util")]
20796    #[tokio::test]
20797    #[serial_test::serial(storage_class_env)]
20798    async fn tier_mutation_peer_prepare_waits_for_inflight_reference_lease() {
20799        let temp_dir = tempfile::tempdir().expect("create temp store dir");
20800        let (_ctx, store, _shutdown) =
20801            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "tier-mutation-peer-drain", &[4])).await;
20802        register_mock_tier(&store.tier_config_mgr(), "COLD-A").await;
20803        let old_lease = TierConfigMgr::acquire_operation_lease(&store.tier_config_mgr(), "COLD-A")
20804            .await
20805            .expect("old tier operation lease should be available");
20806        let mutation_id = uuid::Uuid::new_v4();
20807        let mut intent = tier_mutation_peer_test_intent(mutation_id, "COLD-A", [9; 32]);
20808        intent.affected_targets[0].old_backend_identity = Some(old_lease.backend_identity());
20809        let payload = intent.encode().expect("prepare intent should encode");
20810        let prepare_store = store.clone();
20811        let prepare = tokio::spawn(async move {
20812            handle_tier_mutation_peer_request(
20813                prepare_store,
20814                TIER_MUTATION_RPC_PROTOCOL_VERSION,
20815                TierMutationRpcPhase::Prepare,
20816                mutation_id,
20817                &payload,
20818            )
20819            .await
20820        });
20821
20822        tokio::time::timeout(Duration::from_secs(5), async {
20823            loop {
20824                if TierConfigMgr::acquire_operation_lease(&store.tier_config_mgr(), "COLD-A")
20825                    .await
20826                    .is_err()
20827                {
20828                    return;
20829                }
20830                tokio::task::yield_now().await;
20831            }
20832        })
20833        .await
20834        .expect("peer Prepare should install its runtime block");
20835        assert!(
20836            !prepare.is_finished(),
20837            "peer Prepare must not acknowledge while a pre-existing reference creator still holds a lease"
20838        );
20839
20840        drop(old_lease);
20841        let outcome = prepare
20842            .await
20843            .expect("peer Prepare task should join")
20844            .expect("peer Prepare should finish after the lease drains");
20845        assert_eq!(outcome.state, TierMutationPeerState::Prepared);
20846        TierConfigMgr::clear_prepared_mutation_intent_block(&store.tier_config_mgr(), mutation_id)
20847            .await
20848            .expect("test should clear the prepared runtime block");
20849    }
20850
20851    #[cfg(feature = "test-util")]
20852    #[test]
20853    #[serial_test::serial(storage_class_env)]
20854    fn tier_mutation_peer_handler_applies_prepare_commit_and_abort_idempotently() {
20855        // This end-to-end state-machine scenario intentionally keeps many
20856        // durable fixtures live at once. Give its debug future the same
20857        // dedicated stack used by the other large ecstore scenarios so the
20858        // assertions run under the default test command without RUST_MIN_STACK.
20859        std::thread::Builder::new()
20860            .name("tier-mutation-peer-handler-test".to_string())
20861            .stack_size(32 * 1024 * 1024)
20862            .spawn(|| {
20863                let runtime = tokio::runtime::Builder::new_multi_thread()
20864                    .enable_all()
20865                    .worker_threads(2)
20866                    .build()
20867                    .expect("tier mutation peer handler test runtime should build");
20868                runtime.block_on(tier_mutation_peer_handler_case());
20869            })
20870            .expect("tier mutation peer handler test thread should spawn")
20871            .join()
20872            .expect("tier mutation peer handler test thread should complete");
20873    }
20874
20875    #[cfg(feature = "test-util")]
20876    async fn tier_mutation_peer_handler_case() {
20877        let temp_dir = tempfile::tempdir().expect("create temp store dir");
20878        let (_ctx, store, _shutdown) =
20879            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "tier-mutation-peer-handler", &[4])).await;
20880        register_mock_tier(&store.tier_config_mgr(), "COLD-A").await;
20881        let (candidate_digest, config_etag) = {
20882            let tier_config_mgr = store.tier_config_mgr();
20883            let manager = tier_config_mgr.read().await;
20884            let candidate_digest = tier_config_candidate_digest(&manager).expect("peer commit candidate digest should build");
20885            manager
20886                .save_tiering_config(store.clone())
20887                .await
20888                .expect("peer commit config fixture should persist");
20889            let config_info = store
20890                .get_object_info(
20891                    RUSTFS_META_BUCKET,
20892                    &format!("{}/{}", com::CONFIG_PREFIX, TIER_CONFIG_FILE),
20893                    &ObjectOptions::default(),
20894                )
20895                .await
20896                .expect("peer commit config fixture should load");
20897            (candidate_digest, config_info.etag.expect("peer commit config should carry an ETag"))
20898        };
20899        let mutation_id = uuid::Uuid::new_v4();
20900        let mut intent = tier_mutation_peer_test_intent(mutation_id, "COLD-A", candidate_digest);
20901        intent.old_config_etag = Some(config_etag.clone());
20902        let prepare_payload = intent.encode().expect("prepare intent should encode");
20903
20904        let prepared = handle_tier_mutation_peer_request(
20905            store.clone(),
20906            TIER_MUTATION_RPC_PROTOCOL_VERSION,
20907            TierMutationRpcPhase::Prepare,
20908            mutation_id,
20909            &prepare_payload,
20910        )
20911        .await
20912        .expect("first prepare should create the peer intent");
20913        assert!(prepared.applied);
20914        assert_eq!(prepared.state, TierMutationPeerState::Prepared);
20915        let blocked = match TierConfigMgr::acquire_operation_lease(&store.tier_config_mgr(), "COLD-A").await {
20916            Ok(_) => panic!("prepared peer mutation should block new tier operation leases"),
20917            Err(err) => err,
20918        };
20919        assert!(
20920            blocked.message.contains("being replaced"),
20921            "prepared peer mutation should reuse the existing blocked-tier error: {blocked}"
20922        );
20923
20924        let retried_prepare = handle_tier_mutation_peer_request(
20925            store.clone(),
20926            TIER_MUTATION_RPC_PROTOCOL_VERSION,
20927            TierMutationRpcPhase::Prepare,
20928            mutation_id,
20929            &prepare_payload,
20930        )
20931        .await
20932        .expect("same prepare retry should be idempotent");
20933        assert!(!retried_prepare.applied);
20934        assert_eq!(retried_prepare.state, TierMutationPeerState::Prepared);
20935        let retried_blocked = match TierConfigMgr::acquire_operation_lease(&store.tier_config_mgr(), "COLD-A").await {
20936            Ok(_) => panic!("prepared retry should keep blocking new tier operation leases"),
20937            Err(err) => err,
20938        };
20939        assert!(
20940            retried_blocked.message.contains("being replaced"),
20941            "prepared retry should keep the existing blocked-tier error: {retried_blocked}"
20942        );
20943
20944        let mismatched_commit = handle_tier_mutation_peer_request(
20945            store.clone(),
20946            TIER_MUTATION_RPC_PROTOCOL_VERSION,
20947            TierMutationRpcPhase::Commit,
20948            mutation_id,
20949            b"not-the-current-etag",
20950        )
20951        .await
20952        .expect_err("commit with a mismatched config proof must fail closed");
20953        assert!(matches!(mismatched_commit, TierMutationPeerError::CommitProofMismatch));
20954
20955        register_mock_tier(&store.tier_config_mgr(), "COLD-C").await;
20956        let bad_digest_id = uuid::Uuid::new_v4();
20957        let mut bad_digest_intent = tier_mutation_peer_test_intent(bad_digest_id, "COLD-C", [9; 32]);
20958        bad_digest_intent.old_config_etag = Some(config_etag.clone());
20959        handle_tier_mutation_peer_request(
20960            store.clone(),
20961            TIER_MUTATION_RPC_PROTOCOL_VERSION,
20962            TierMutationRpcPhase::Prepare,
20963            bad_digest_id,
20964            &bad_digest_intent.encode().expect("bad digest prepare intent should encode"),
20965        )
20966        .await
20967        .expect("bad digest prepare should install a prepared intent");
20968        let mismatched_digest_commit = handle_tier_mutation_peer_request(
20969            store.clone(),
20970            TIER_MUTATION_RPC_PROTOCOL_VERSION,
20971            TierMutationRpcPhase::Commit,
20972            bad_digest_id,
20973            config_etag.as_bytes(),
20974        )
20975        .await
20976        .expect_err("a correct ETag with a mismatched candidate digest must fail closed");
20977        assert!(matches!(mismatched_digest_commit, TierMutationPeerError::CommitProofMismatch));
20978        let bad_digest_loaded = load_tier_mutation_intent_record(store.clone(), bad_digest_id)
20979            .await
20980            .expect("mismatched digest must leave the prepared intent durable");
20981        assert_eq!(bad_digest_loaded.state, TierMutationIntentState::Prepared);
20982        let bad_digest_blocked = match TierConfigMgr::acquire_operation_lease(&store.tier_config_mgr(), "COLD-C").await {
20983            Ok(_) => panic!("mismatched digest must retain the prepared runtime fence"),
20984            Err(err) => err,
20985        };
20986        assert!(bad_digest_blocked.message.contains("being replaced"), "{bad_digest_blocked}");
20987        handle_tier_mutation_peer_request(
20988            store.clone(),
20989            TIER_MUTATION_RPC_PROTOCOL_VERSION,
20990            TierMutationRpcPhase::Abort,
20991            bad_digest_id,
20992            &bad_digest_intent.encode().expect("bad digest abort identity should encode"),
20993        )
20994        .await
20995        .expect("the negative digest proof fixture should clean up through abort");
20996
20997        let committed = handle_tier_mutation_peer_request(
20998            store.clone(),
20999            TIER_MUTATION_RPC_PROTOCOL_VERSION,
21000            TierMutationRpcPhase::Commit,
21001            mutation_id,
21002            config_etag.as_bytes(),
21003        )
21004        .await
21005        .expect("commit should advance the prepared peer intent");
21006        assert!(committed.applied);
21007        assert_eq!(committed.state, TierMutationPeerState::Committed);
21008        let committed_blocked = match TierConfigMgr::acquire_operation_lease(&store.tier_config_mgr(), "COLD-A").await {
21009            Ok(_) => panic!("committed peer mutation must remain blocked until local reload publishes the config"),
21010            Err(err) => err,
21011        };
21012        assert!(
21013            committed_blocked.message.contains("being replaced"),
21014            "committed peer mutation should keep the existing blocked-tier error: {committed_blocked}"
21015        );
21016
21017        let retried_commit = handle_tier_mutation_peer_request(
21018            store.clone(),
21019            TIER_MUTATION_RPC_PROTOCOL_VERSION,
21020            TierMutationRpcPhase::Commit,
21021            mutation_id,
21022            config_etag.as_bytes(),
21023        )
21024        .await
21025        .expect("same commit retry should be idempotent");
21026        assert!(!retried_commit.applied);
21027        assert_eq!(retried_commit.state, TierMutationPeerState::Committed);
21028        let retried_commit_blocked = match TierConfigMgr::acquire_operation_lease(&store.tier_config_mgr(), "COLD-A").await {
21029            Ok(_) => panic!("committed retry must keep the tier blocked until local reload"),
21030            Err(err) => err,
21031        };
21032        assert!(
21033            retried_commit_blocked.message.contains("being replaced"),
21034            "committed retry should keep the existing blocked-tier error: {retried_commit_blocked}"
21035        );
21036
21037        let delayed_prepare_retry = handle_tier_mutation_peer_request(
21038            store.clone(),
21039            TIER_MUTATION_RPC_PROTOCOL_VERSION,
21040            TierMutationRpcPhase::Prepare,
21041            mutation_id,
21042            &prepare_payload,
21043        )
21044        .await
21045        .expect("delayed duplicate prepare should report the durable committed state");
21046        assert!(!delayed_prepare_retry.applied);
21047        assert_eq!(delayed_prepare_retry.state, TierMutationPeerState::Committed);
21048        let delayed_prepare_blocked = match TierConfigMgr::acquire_operation_lease(&store.tier_config_mgr(), "COLD-A").await {
21049            Ok(_) => panic!("delayed committed prepare retry must preserve the committed runtime block"),
21050            Err(err) => err,
21051        };
21052        assert!(
21053            delayed_prepare_blocked.message.contains("being replaced"),
21054            "delayed committed prepare retry should keep the existing blocked-tier error: {delayed_prepare_blocked}"
21055        );
21056
21057        let loaded = load_tier_mutation_intent_record(store.clone(), mutation_id)
21058            .await
21059            .expect("committed peer intent should remain durable");
21060        assert_eq!(loaded.state, TierMutationIntentState::Committed);
21061        assert_eq!(loaded.committed_config_etag.as_deref(), Some(config_etag.as_str()));
21062
21063        store
21064            .tier_config_mgr()
21065            .read()
21066            .await
21067            .save_tiering_config(store.clone())
21068            .await
21069            .expect("tier config should persist for cleaned intent commit proof");
21070        let tier_config_info = store
21071            .get_object_info(
21072                RUSTFS_META_BUCKET,
21073                &format!("{}/{}", com::CONFIG_PREFIX, TIER_CONFIG_FILE),
21074                &ObjectOptions::default(),
21075            )
21076            .await
21077            .expect("tier config object info should load");
21078        let tier_config_etag = tier_config_info.etag.expect("tier config should carry an ETag");
21079        delete_tier_mutation_intent_record(store.clone(), mutation_id)
21080            .await
21081            .expect("simulate another node cleaning the shared committed peer intent");
21082        let cleaned_commit_retry = handle_tier_mutation_peer_request(
21083            store.clone(),
21084            TIER_MUTATION_RPC_PROTOCOL_VERSION,
21085            TierMutationRpcPhase::Commit,
21086            mutation_id,
21087            tier_config_etag.as_bytes(),
21088        )
21089        .await
21090        .expect("commit retry after durable cleanup should be idempotently terminal");
21091        assert!(!cleaned_commit_retry.applied);
21092        assert_eq!(cleaned_commit_retry.state, TierMutationPeerState::Committed);
21093        let cleaned_commit_blocked = match TierConfigMgr::acquire_operation_lease(&store.tier_config_mgr(), "COLD-A").await {
21094            Ok(_) => panic!("shared intent cleanup must not clear this node's committed runtime block"),
21095            Err(err) => err,
21096        };
21097        assert!(
21098            cleaned_commit_blocked.message.contains("being replaced"),
21099            "commit retry after shared cleanup should keep the existing blocked-tier error: {cleaned_commit_blocked}"
21100        );
21101        let mismatched_cleaned_commit = handle_tier_mutation_peer_request(
21102            store.clone(),
21103            TIER_MUTATION_RPC_PROTOCOL_VERSION,
21104            TierMutationRpcPhase::Commit,
21105            mutation_id,
21106            b"not-the-current-etag",
21107        )
21108        .await
21109        .expect_err("missing intent without a matching committed config ETag must fail closed");
21110        assert!(matches!(mismatched_cleaned_commit, TierMutationPeerError::Store(Error::ConfigNotFound)));
21111        let refresh_store = store.clone();
21112        let refresh_manager = store.tier_config_mgr();
21113        let refresh_worker = tokio::spawn(async move {
21114            TierConfigMgr::refresh_tier_config_handle_with(refresh_manager, refresh_store).await;
21115        });
21116        tokio::time::timeout(Duration::from_secs(5), async {
21117            loop {
21118                if let Ok(lease) = TierConfigMgr::acquire_operation_lease(&store.tier_config_mgr(), "COLD-A").await {
21119                    drop(lease);
21120                    break;
21121                }
21122                tokio::task::yield_now().await;
21123            }
21124        })
21125        .await
21126        .expect("matching cleaned commit should wake the refresh worker and clear the committed fence");
21127        refresh_worker.abort();
21128        let _ = refresh_worker.await;
21129
21130        let abort_id = uuid::Uuid::new_v4();
21131        register_mock_tier(&store.tier_config_mgr(), "COLD-B").await;
21132        store
21133            .tier_config_mgr()
21134            .read()
21135            .await
21136            .save_tiering_config(store.clone())
21137            .await
21138            .expect("abort target tier config should persist");
21139        let abort_config_info = store
21140            .get_object_info(
21141                RUSTFS_META_BUCKET,
21142                &format!("{}/{}", com::CONFIG_PREFIX, TIER_CONFIG_FILE),
21143                &ObjectOptions::default(),
21144            )
21145            .await
21146            .expect("abort target config metadata should load");
21147        let abort_config_etag = abort_config_info.etag.expect("abort target config should carry an ETag");
21148        let mut abort_intent = tier_mutation_peer_test_intent(abort_id, "COLD-B", [4; 32]);
21149        abort_intent.old_config_etag = Some(abort_config_etag);
21150        let abort_prepare_payload = abort_intent.encode().expect("abort prepare intent should encode");
21151        handle_tier_mutation_peer_request(
21152            store.clone(),
21153            TIER_MUTATION_RPC_PROTOCOL_VERSION,
21154            TierMutationRpcPhase::Prepare,
21155            abort_id,
21156            &abort_prepare_payload,
21157        )
21158        .await
21159        .expect("abort target prepare should create the peer intent");
21160        let abort_blocked = match TierConfigMgr::acquire_operation_lease(&store.tier_config_mgr(), "COLD-B").await {
21161            Ok(_) => panic!("abort target prepare should block new tier operation leases"),
21162            Err(err) => err,
21163        };
21164        assert!(
21165            abort_blocked.message.contains("being replaced"),
21166            "abort target prepare should reuse the existing blocked-tier error: {abort_blocked}"
21167        );
21168
21169        let aborted = handle_tier_mutation_peer_request(
21170            store.clone(),
21171            TIER_MUTATION_RPC_PROTOCOL_VERSION,
21172            TierMutationRpcPhase::Abort,
21173            abort_id,
21174            &abort_prepare_payload,
21175        )
21176        .await
21177        .expect("abort should advance the prepared peer intent");
21178        assert!(aborted.applied);
21179        assert_eq!(aborted.state, TierMutationPeerState::Aborted);
21180        TierConfigMgr::acquire_operation_lease(&store.tier_config_mgr(), "COLD-B")
21181            .await
21182            .expect("durable abort must clear the peer runtime fence immediately");
21183
21184        let retried_abort = handle_tier_mutation_peer_request(
21185            store.clone(),
21186            TIER_MUTATION_RPC_PROTOCOL_VERSION,
21187            TierMutationRpcPhase::Abort,
21188            abort_id,
21189            &abort_prepare_payload,
21190        )
21191        .await
21192        .expect("same abort retry should be idempotent before recovery cleanup");
21193        assert!(!retried_abort.applied);
21194        assert_eq!(retried_abort.state, TierMutationPeerState::Aborted);
21195    }
21196
21197    #[cfg(feature = "test-util")]
21198    #[tokio::test]
21199    #[serial_test::serial(storage_class_env)]
21200    async fn late_abort_after_config_commit_keeps_fence_until_commit_recovery() {
21201        let temp_dir = tempfile::tempdir().expect("create temp store dir");
21202        let (_ctx, store, _shutdown) =
21203            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "tier-mutation-late-abort", &[4])).await;
21204        register_mock_tier(&store.tier_config_mgr(), "COLD-A").await;
21205        store
21206            .tier_config_mgr()
21207            .read()
21208            .await
21209            .save_tiering_config(store.clone())
21210            .await
21211            .expect("base tier config should persist");
21212        let base_config_info = store
21213            .get_object_info(
21214                RUSTFS_META_BUCKET,
21215                &format!("{}/{}", com::CONFIG_PREFIX, TIER_CONFIG_FILE),
21216                &ObjectOptions::default(),
21217            )
21218            .await
21219            .expect("base tier config metadata should load");
21220        let base_etag = base_config_info.etag.expect("base tier config should carry an ETag");
21221
21222        register_mock_tier(&store.tier_config_mgr(), "COLD-B").await;
21223        let candidate_digest = {
21224            let manager = store.tier_config_mgr();
21225            let manager = manager.read().await;
21226            tier_config_candidate_digest(&manager).expect("candidate digest should build")
21227        };
21228        let mutation_id = uuid::Uuid::new_v4();
21229        let mut intent = tier_mutation_peer_test_intent(mutation_id, "COLD-B", candidate_digest);
21230        intent.old_config_etag = Some(base_etag.clone());
21231        let prepare_payload = intent.encode().expect("late abort prepare intent should encode");
21232        handle_tier_mutation_peer_request(
21233            store.clone(),
21234            TIER_MUTATION_RPC_PROTOCOL_VERSION,
21235            TierMutationRpcPhase::Prepare,
21236            mutation_id,
21237            &prepare_payload,
21238        )
21239        .await
21240        .expect("prepare should install the runtime fence");
21241
21242        store
21243            .tier_config_mgr()
21244            .read()
21245            .await
21246            .save_tiering_config(store.clone())
21247            .await
21248            .expect("candidate tier config should persist before the late abort");
21249        let committed_config_info = store
21250            .get_object_info(
21251                RUSTFS_META_BUCKET,
21252                &format!("{}/{}", com::CONFIG_PREFIX, TIER_CONFIG_FILE),
21253                &ObjectOptions::default(),
21254            )
21255            .await
21256            .expect("committed tier config metadata should load");
21257        let committed_etag = committed_config_info
21258            .etag
21259            .expect("committed tier config should carry an ETag");
21260        assert_ne!(committed_etag, base_etag);
21261
21262        let late_abort = handle_tier_mutation_peer_request(
21263            store.clone(),
21264            TIER_MUTATION_RPC_PROTOCOL_VERSION,
21265            TierMutationRpcPhase::Abort,
21266            mutation_id,
21267            &prepare_payload,
21268        )
21269        .await
21270        .expect_err("an abort after the candidate config commit must fail closed");
21271        assert!(matches!(late_abort, TierMutationPeerError::AbortProofMismatch));
21272        let prepared = load_tier_mutation_intent_record(store.clone(), mutation_id)
21273            .await
21274            .expect("rejected late abort must retain the prepared intent");
21275        assert_eq!(prepared.state, TierMutationIntentState::Prepared);
21276        let blocked = match TierConfigMgr::acquire_operation_lease(&store.tier_config_mgr(), "COLD-B").await {
21277            Ok(_) => panic!("rejected late abort must retain the runtime fence"),
21278            Err(err) => err,
21279        };
21280        assert!(blocked.message.contains("being replaced"), "{blocked}");
21281
21282        let committed = handle_tier_mutation_peer_request(
21283            store.clone(),
21284            TIER_MUTATION_RPC_PROTOCOL_VERSION,
21285            TierMutationRpcPhase::Commit,
21286            mutation_id,
21287            committed_etag.as_bytes(),
21288        )
21289        .await
21290        .expect("matching commit should converge the rejected late abort fixture");
21291        assert!(committed.applied);
21292        assert_eq!(committed.state, TierMutationPeerState::Committed);
21293
21294        let refresh_store = store.clone();
21295        let refresh_manager = store.tier_config_mgr();
21296        let refresh_worker = tokio::spawn(async move {
21297            TierConfigMgr::refresh_tier_config_handle_with(refresh_manager, refresh_store).await;
21298        });
21299        tokio::time::timeout(Duration::from_secs(5), async {
21300            loop {
21301                if let Ok(lease) = TierConfigMgr::acquire_operation_lease(&store.tier_config_mgr(), "COLD-B").await {
21302                    drop(lease);
21303                    break;
21304                }
21305                tokio::task::yield_now().await;
21306            }
21307        })
21308        .await
21309        .expect("commit recovery should publish before clearing the late-abort fence");
21310        refresh_worker.abort();
21311        let _ = refresh_worker.await;
21312    }
21313
21314    #[cfg(feature = "test-util")]
21315    #[tokio::test]
21316    #[serial_test::serial(storage_class_env)]
21317    async fn missing_record_commit_promotes_prepared_fence_until_worker_publish() {
21318        let temp_dir = tempfile::tempdir().expect("create temp store dir");
21319        let (_ctx, store, _shutdown) =
21320            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "tier-mutation-missing-record-commit", &[4]))
21321                .await;
21322        register_mock_tier(&store.tier_config_mgr(), "COLD-A").await;
21323        let tier_config_mgr = store.tier_config_mgr();
21324        let candidate_digest = {
21325            let manager = tier_config_mgr.read().await;
21326            let digest = tier_config_candidate_digest(&manager).expect("candidate digest should build");
21327            manager
21328                .save_tiering_config(store.clone())
21329                .await
21330                .expect("candidate config should persist");
21331            digest
21332        };
21333        let config_info = store
21334            .get_object_info(
21335                RUSTFS_META_BUCKET,
21336                &format!("{}/{}", com::CONFIG_PREFIX, TIER_CONFIG_FILE),
21337                &ObjectOptions::default(),
21338            )
21339            .await
21340            .expect("candidate config metadata should load");
21341        let config_etag = config_info.etag.expect("candidate config should carry an ETag");
21342        let mutation_id = uuid::Uuid::new_v4();
21343        let intent = tier_mutation_peer_test_intent(mutation_id, "COLD-A", candidate_digest);
21344        handle_tier_mutation_peer_request(
21345            store.clone(),
21346            TIER_MUTATION_RPC_PROTOCOL_VERSION,
21347            TierMutationRpcPhase::Prepare,
21348            mutation_id,
21349            &intent.encode().expect("prepare intent should encode"),
21350        )
21351        .await
21352        .expect("prepare should install the runtime fence");
21353        delete_tier_mutation_intent_record(store.clone(), mutation_id)
21354            .await
21355            .expect("simulate shared intent cleanup before the local commit arrives");
21356
21357        let committed = handle_tier_mutation_peer_request(
21358            store.clone(),
21359            TIER_MUTATION_RPC_PROTOCOL_VERSION,
21360            TierMutationRpcPhase::Commit,
21361            mutation_id,
21362            config_etag.as_bytes(),
21363        )
21364        .await
21365        .expect("matching commit after shared cleanup should be terminal");
21366        assert!(!committed.applied);
21367        assert_eq!(committed.state, TierMutationPeerState::Committed);
21368        let blocked = match TierConfigMgr::acquire_operation_lease(&tier_config_mgr, "COLD-A").await {
21369            Ok(_) => panic!("the promoted committed fence must block old-generation leases"),
21370            Err(err) => err,
21371        };
21372        assert!(blocked.message.contains("being replaced"), "{blocked}");
21373
21374        let refresh_store = store.clone();
21375        let refresh_manager = tier_config_mgr.clone();
21376        let refresh_worker = tokio::spawn(async move {
21377            TierConfigMgr::refresh_tier_config_handle_with(refresh_manager, refresh_store).await;
21378        });
21379        tokio::time::timeout(Duration::from_secs(5), async {
21380            loop {
21381                if let Ok(lease) = TierConfigMgr::acquire_operation_lease(&tier_config_mgr, "COLD-A").await {
21382                    drop(lease);
21383                    break;
21384                }
21385                tokio::task::yield_now().await;
21386            }
21387        })
21388        .await
21389        .expect("the commit notification should drive publish and clear the promoted fence");
21390        refresh_worker.abort();
21391        let _ = refresh_worker.await;
21392    }
21393
21394    #[cfg(feature = "test-util")]
21395    #[tokio::test]
21396    #[serial_test::serial(storage_class_env)]
21397    async fn tier_mutation_peer_handler_rejects_conflicting_prepare_without_overwrite() {
21398        let temp_dir = tempfile::tempdir().expect("create temp store dir");
21399        let (_ctx, store, _shutdown) =
21400            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "tier-mutation-peer-conflict", &[4])).await;
21401        let mutation_id = uuid::Uuid::new_v4();
21402        let intent = tier_mutation_peer_test_intent(mutation_id, "COLD-A", [3; 32]);
21403        let prepare_payload = intent.encode().expect("prepare intent should encode");
21404        handle_tier_mutation_peer_request(
21405            store.clone(),
21406            TIER_MUTATION_RPC_PROTOCOL_VERSION,
21407            TierMutationRpcPhase::Prepare,
21408            mutation_id,
21409            &prepare_payload,
21410        )
21411        .await
21412        .expect("first prepare should create the peer intent");
21413
21414        let conflicting = tier_mutation_peer_test_intent(mutation_id, "COLD-A", [4; 32]);
21415        let conflicting_payload = conflicting.encode().expect("conflicting intent should encode");
21416        let conflict = handle_tier_mutation_peer_request(
21417            store.clone(),
21418            TIER_MUTATION_RPC_PROTOCOL_VERSION,
21419            TierMutationRpcPhase::Prepare,
21420            mutation_id,
21421            &conflicting_payload,
21422        )
21423        .await
21424        .expect_err("conflicting prepare must fail closed");
21425        assert!(matches!(conflict, TierMutationPeerError::ConflictingIntent));
21426
21427        let loaded = load_tier_mutation_intent_record(store, mutation_id)
21428            .await
21429            .expect("conflicting prepare must not overwrite the first record");
21430        assert_eq!(loaded.candidate_digest, [3; 32]);
21431        assert_eq!(loaded.state, TierMutationIntentState::Prepared);
21432    }
21433
21434    #[cfg(feature = "test-util")]
21435    #[tokio::test]
21436    #[serial_test::serial(storage_class_env)]
21437    async fn tier_mutation_intent_record_scan_paginates_exact_limit() {
21438        let temp_dir = tempfile::tempdir().expect("create temp store dir");
21439        let (_ctx, store, _shutdown) =
21440            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "tier-mutation-intent-scan-page", &[4])).await;
21441        let build_intent = |mutation_id: uuid::Uuid, tier_name: &str| TierMutationIntent {
21442            mutation_id,
21443            revision: 1,
21444            kind: TierMutationIntentKind::Edit,
21445            state: TierMutationIntentState::Prepared,
21446            old_config_etag: Some("old-etag".to_string()),
21447            committed_config_etag: None,
21448            candidate_digest: [3; 32],
21449            affected_targets: vec![TierMutationIntentTarget {
21450                tier_name: tier_name.to_string(),
21451                old_backend_identity: Some([1; 32]),
21452                new_backend_identity: Some([2; 32]),
21453            }],
21454            expires_at_unix_nanos: 1_780_000_000_000_000_000,
21455        };
21456        let intent_ids = vec![
21457            uuid::Uuid::parse_str("11345678-1234-5678-9abc-def012345678").expect("first uuid should parse"),
21458            uuid::Uuid::parse_str("22345678-1234-5678-9abc-def012345678").expect("second uuid should parse"),
21459            uuid::Uuid::parse_str("33345678-1234-5678-9abc-def012345678").expect("third uuid should parse"),
21460        ];
21461        for (index, mutation_id) in intent_ids.iter().copied().enumerate() {
21462            let intent = build_intent(mutation_id, &format!("COLD-{index}"));
21463            save_tier_mutation_intent_record(store.clone(), &intent)
21464                .await
21465                .expect("tier mutation intent record should persist");
21466        }
21467
21468        let exact_page = list_tier_mutation_intent_records(store.clone(), 3, None)
21469            .await
21470            .expect("exact full page should scan");
21471        assert_eq!(exact_page.scanned, 3);
21472        assert_eq!(exact_page.intents.len(), 3);
21473        assert_eq!(exact_page.failed, 0);
21474        assert!(!exact_page.truncated);
21475        assert_eq!(exact_page.next_marker, None);
21476
21477        let first_page = list_tier_mutation_intent_records(store.clone(), 2, None)
21478            .await
21479            .expect("first page should scan");
21480        assert_eq!(first_page.scanned, 2);
21481        assert_eq!(first_page.intents.len(), 2);
21482        assert_eq!(first_page.failed, 0);
21483        assert!(first_page.truncated);
21484        assert!(first_page.next_marker.is_some());
21485
21486        let second_page = list_tier_mutation_intent_records(store, 2, first_page.next_marker)
21487            .await
21488            .expect("second page should scan");
21489        let mut loaded_ids: Vec<_> = first_page
21490            .intents
21491            .into_iter()
21492            .chain(second_page.intents.into_iter())
21493            .map(|intent| intent.mutation_id)
21494            .collect();
21495        loaded_ids.sort();
21496
21497        assert_eq!(second_page.scanned, 1);
21498        assert_eq!(second_page.failed, 0);
21499        assert!(!second_page.truncated);
21500        assert_eq!(second_page.next_marker, None);
21501        assert_eq!(loaded_ids, intent_ids);
21502    }
21503
21504    #[cfg(feature = "test-util")]
21505    fn transition_mutation_measurement(observations: &[TransitionTransactionMutationObservation]) -> (usize, usize, u128) {
21506        (
21507            observations.len(),
21508            observations.iter().map(|observation| observation.encoded_bytes).sum(),
21509            observations.iter().map(|observation| observation.elapsed.as_micros()).sum(),
21510        )
21511    }
21512
21513    #[cfg(feature = "test-util")]
21514    #[tokio::test]
21515    #[serial_test::serial(storage_class_env)]
21516    async fn compact_transition_transactions_halve_success_path_quorum_mutations() {
21517        let temp_dir = tempfile::tempdir().expect("create transition mutation measurement dir");
21518        let (ctx, store, shutdown) =
21519            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "transition-mutation-measurement", &[4])).await;
21520        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
21521        let tier_name = "MUTATION-MEASURE";
21522        register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
21523        let bucket = "transition-mutation-measurement";
21524        store
21525            .make_bucket(bucket, &MakeBucketOptions::default())
21526            .await
21527            .expect("measurement bucket should be created");
21528
21529        let run_case = |profile: &'static str, size: usize| {
21530            let store = store.clone();
21531            async move {
21532                let object = format!("{profile}-{size}.bin");
21533                let mut reader = PutObjReader::from_vec(vec![b'm'; size]);
21534                let original = store
21535                    .put_object(bucket, &object, &mut reader, &ObjectOptions::default())
21536                    .await
21537                    .expect("measurement source should be written");
21538                let probe = TransitionTransactionMutationProbe::install(bucket, &object);
21539                store
21540                    .transition_object(
21541                        bucket,
21542                        &object,
21543                        &ObjectOptions {
21544                            transition: TransitionOptions {
21545                                status: TRANSITION_PENDING.to_string(),
21546                                tier: tier_name.to_string(),
21547                                etag: original.etag.clone().expect("measurement source should have an ETag"),
21548                                ..Default::default()
21549                            },
21550                            version_id: original.version_id.map(|version| version.to_string()),
21551                            mod_time: original.mod_time,
21552                            ..Default::default()
21553                        },
21554                    )
21555                    .await
21556                    .expect("measurement transition should commit");
21557                probe.observations()
21558            }
21559        };
21560
21561        let sizes = [4 * 1024, 1024 * 1024];
21562        let mut legacy = Vec::with_capacity(sizes.len());
21563        for size in sizes {
21564            legacy.push((size, run_case("legacy", size).await));
21565        }
21566
21567        let compaction_proof = install_transition_transaction_compaction_fleet_proof_for_test("object-transaction-fencing-test");
21568        for ((size, legacy_observations), compact_size) in legacy.into_iter().zip(sizes) {
21569            assert_eq!(size, compact_size);
21570            let compact_observations = run_case("compact", size).await;
21571            let legacy_measurement = transition_mutation_measurement(&legacy_observations);
21572            let compact_measurement = transition_mutation_measurement(&compact_observations);
21573            assert_eq!(legacy_measurement.0, 6, "legacy success should use five saves and one delete");
21574            assert_eq!(compact_measurement.0, 3, "compact success should use two saves and one delete");
21575            assert!(
21576                compact_measurement.1 < legacy_measurement.1,
21577                "compact transaction bodies should write fewer aggregate bytes"
21578            );
21579            assert_eq!(
21580                legacy_observations
21581                    .iter()
21582                    .map(|observation| (observation.kind, observation.previous_state, observation.state))
21583                    .collect::<Vec<_>>(),
21584                vec![
21585                    (TransitionTransactionMutationKind::Create, None, TransitionTransactionState::UploadStarted),
21586                    (
21587                        TransitionTransactionMutationKind::CompareAndSave,
21588                        Some(TransitionTransactionState::UploadStarted),
21589                        TransitionTransactionState::UploadOutcomeUnknown,
21590                    ),
21591                    (
21592                        TransitionTransactionMutationKind::CompareAndSave,
21593                        Some(TransitionTransactionState::UploadOutcomeUnknown),
21594                        TransitionTransactionState::Uploaded,
21595                    ),
21596                    (
21597                        TransitionTransactionMutationKind::CompareAndSave,
21598                        Some(TransitionTransactionState::Uploaded),
21599                        TransitionTransactionState::LocalCommitStarted,
21600                    ),
21601                    (
21602                        TransitionTransactionMutationKind::CompareAndSave,
21603                        Some(TransitionTransactionState::LocalCommitStarted),
21604                        TransitionTransactionState::Committed,
21605                    ),
21606                    (
21607                        TransitionTransactionMutationKind::Delete,
21608                        Some(TransitionTransactionState::Committed),
21609                        TransitionTransactionState::Committed,
21610                    ),
21611                ]
21612            );
21613            assert_eq!(
21614                compact_observations
21615                    .iter()
21616                    .map(|observation| (observation.kind, observation.previous_state, observation.state))
21617                    .collect::<Vec<_>>(),
21618                vec![
21619                    (
21620                        TransitionTransactionMutationKind::Create,
21621                        None,
21622                        TransitionTransactionState::UploadOutcomeUnknown,
21623                    ),
21624                    (
21625                        TransitionTransactionMutationKind::CompareAndSave,
21626                        Some(TransitionTransactionState::UploadOutcomeUnknown),
21627                        TransitionTransactionState::LocalCommitStarted,
21628                    ),
21629                    (
21630                        TransitionTransactionMutationKind::Delete,
21631                        Some(TransitionTransactionState::LocalCommitStarted),
21632                        TransitionTransactionState::LocalCommitStarted,
21633                    ),
21634                ]
21635            );
21636            assert!(legacy_observations.iter().all(|observation| observation.succeeded));
21637            assert!(compact_observations.iter().all(|observation| observation.succeeded));
21638            println!(
21639                "transition_mutation_measurement,profile=legacy,size={size},mutations={},encoded_bytes={},latency_us={},quorum_operations={}",
21640                legacy_measurement.0, legacy_measurement.1, legacy_measurement.2, legacy_measurement.0
21641            );
21642            println!(
21643                "transition_mutation_measurement,profile=compact,size={size},mutations={},encoded_bytes={},latency_us={},quorum_operations={}",
21644                compact_measurement.0, compact_measurement.1, compact_measurement.2, compact_measurement.0
21645            );
21646        }
21647        assert_eq!(transition_transaction_record_count(store.clone()).await, 0);
21648        let admitted = acquire_transition_transaction_compaction_fleet_proof()
21649            .expect("published homogeneous proof should admit one compact writer");
21650        assert!(transition_transaction_compaction_fleet_proof_matches(&admitted));
21651        drop(compaction_proof);
21652        assert!(
21653            !transition_transaction_compaction_fleet_proof_matches(&admitted),
21654            "revocation must fence a writer admitted by the previous process-epoch snapshot"
21655        );
21656        drop(admitted);
21657        assert!(
21658            acquire_transition_transaction_compaction_fleet_proof().is_none(),
21659            "revoking the homogeneous proof must restore the legacy writer profile"
21660        );
21661        shutdown.cancel();
21662    }
21663
21664    #[cfg(feature = "test-util")]
21665    #[tokio::test]
21666    #[serial_test::serial(storage_class_env)]
21667    async fn compact_transition_kill_points_preserve_the_only_remote_owner() {
21668        let temp_dir = tempfile::tempdir().expect("create compact transition kill-point dir");
21669        let (ctx, store, shutdown) =
21670            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "compact-transition-kill-points", &[4])).await;
21671        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
21672        let tier_name = "COMPACT-KILL";
21673        let backend = register_transition_reconcile_test_tier(&ctx.tier_config_mgr(), tier_name).await;
21674        let _tier_lease = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
21675            .await
21676            .expect("mock tier lease should remain available during recovery");
21677        let bucket = "compact-transition-kill-points";
21678        store
21679            .make_bucket(bucket, &MakeBucketOptions::default())
21680            .await
21681            .expect("kill-point bucket should be created");
21682        let _compaction_proof = install_transition_transaction_compaction_fleet_proof_for_test("object-transaction-fencing-test");
21683
21684        for (index, point, expected_state, expected_revision, expected_committed, expected_recovered) in [
21685            (
21686                0,
21687                TransitionTransactionKillPoint::PrePutFence,
21688                TransitionTransactionState::UploadOutcomeUnknown,
21689                1,
21690                false,
21691                true,
21692            ),
21693            (
21694                1,
21695                TransitionTransactionKillPoint::UploadBeforeCommitFence,
21696                TransitionTransactionState::UploadOutcomeUnknown,
21697                1,
21698                false,
21699                true,
21700            ),
21701            (
21702                2,
21703                TransitionTransactionKillPoint::LocalCommitBeforeDelete,
21704                TransitionTransactionState::LocalCommitStarted,
21705                2,
21706                true,
21707                true,
21708            ),
21709            (
21710                3,
21711                TransitionTransactionKillPoint::CommitFenceBeforeLocalCommit,
21712                TransitionTransactionState::LocalCommitStarted,
21713                2,
21714                false,
21715                false,
21716            ),
21717        ] {
21718            let object = format!("kill-point-{index}.bin");
21719            let payload = vec![b'k' + u8::try_from(index).expect("small case index should fit u8"); 64 * 1024];
21720            let mut reader = PutObjReader::from_vec(payload.clone());
21721            let source = store
21722                .put_object(bucket, &object, &mut reader, &ObjectOptions::default())
21723                .await
21724                .expect("kill-point source should be written");
21725            let remote_before = backend.object_count().await;
21726            let barrier = TransitionTransactionKillPointBarrier::install(bucket, &object, point);
21727            let transition_store = store.clone();
21728            let transition_object = object.clone();
21729            let transition = tokio::spawn(async move {
21730                transition_store
21731                    .transition_object(
21732                        bucket,
21733                        &transition_object,
21734                        &ObjectOptions {
21735                            transition: TransitionOptions {
21736                                status: TRANSITION_PENDING.to_string(),
21737                                tier: tier_name.to_string(),
21738                                etag: source.etag.clone().expect("kill-point source should have an ETag"),
21739                                ..Default::default()
21740                            },
21741                            version_id: source.version_id.map(|version| version.to_string()),
21742                            mod_time: source.mod_time,
21743                            ..Default::default()
21744                        },
21745                    )
21746                    .await
21747            });
21748            barrier.wait_until_paused().await;
21749            transition.abort();
21750            assert!(
21751                transition
21752                    .await
21753                    .expect_err("kill-point transition should be cancelled")
21754                    .is_cancelled(),
21755                "kill-point transition should stop without unwinding"
21756            );
21757            drop(barrier);
21758
21759            let transaction = only_transition_transaction(store.clone()).await;
21760            assert_eq!((transaction.state, transaction.revision), (expected_state, expected_revision));
21761            let paused_source = store
21762                .get_object_info(
21763                    bucket,
21764                    &object,
21765                    &ObjectOptions {
21766                        metadata_cache_safe: false,
21767                        ..Default::default()
21768                    },
21769                )
21770                .await
21771                .expect("kill-point source metadata should remain readable");
21772            assert_eq!(
21773                paused_source.transitioned_object.status == rustfs_filemeta::TRANSITION_COMPLETE,
21774                expected_committed
21775            );
21776            let expected_remote_at_pause = remote_before + usize::from(point != TransitionTransactionKillPoint::PrePutFence);
21777            assert_eq!(backend.object_count().await, expected_remote_at_pause);
21778
21779            let stats = recover_transition_transaction_records_at(
21780                store.clone(),
21781                100,
21782                None,
21783                i128::from(transaction.not_after_unix_nanos) + 1,
21784            )
21785            .await
21786            .expect("kill-point transaction recovery should complete");
21787            if expected_recovered {
21788                assert_eq!((stats.scanned, stats.recovered, stats.retained, stats.failed), (1, 1, 0, 0));
21789                assert_eq!(transition_transaction_record_count(store.clone()).await, 0);
21790            } else {
21791                assert_eq!((stats.scanned, stats.recovered, stats.retained, stats.failed), (1, 0, 1, 0));
21792                assert_eq!(
21793                    transition_transaction_record_count(store.clone()).await,
21794                    1,
21795                    "an uncommitted local-commit fence must retain its exact remote owner"
21796                );
21797            }
21798            let expected_remote_after_recovery = if matches!(
21799                point,
21800                TransitionTransactionKillPoint::LocalCommitBeforeDelete
21801                    | TransitionTransactionKillPoint::CommitFenceBeforeLocalCommit
21802            ) {
21803                remote_before + 1
21804            } else {
21805                remote_before
21806            };
21807            assert_eq!(backend.object_count().await, expected_remote_after_recovery);
21808            assert_eq!(backend.remove_count().await, usize::from(index >= 1));
21809
21810            let mut restored = Vec::new();
21811            store
21812                .get_object_reader(bucket, &object, None, HeaderMap::new(), &ObjectOptions::default())
21813                .await
21814                .expect("kill-point source should remain readable through its authoritative location")
21815                .stream
21816                .read_to_end(&mut restored)
21817                .await
21818                .expect("kill-point source body should drain");
21819            assert_eq!(restored, payload);
21820
21821            if !expected_recovered {
21822                break;
21823            }
21824        }
21825        shutdown.cancel();
21826    }
21827
21828    #[cfg(feature = "test-util")]
21829    #[tokio::test]
21830    #[serial_test::serial(storage_class_env)]
21831    async fn cancelled_transition_cleanup_recovers_from_its_own_instance_transaction() {
21832        struct ResolverReset(Arc<std::sync::Mutex<Option<std::sync::Weak<crate::store::ECStore>>>>);
21833
21834        impl Drop for ResolverReset {
21835            fn drop(&mut self) {
21836                *self.0.lock().unwrap_or_else(std::sync::PoisonError::into_inner) = None;
21837            }
21838        }
21839
21840        let temp_a = tempfile::tempdir().expect("create transition store dir a");
21841        let temp_b = tempfile::tempdir().expect("create transition store dir b");
21842        let shutdown_a = CancellationToken::new();
21843        let shutdown_b = CancellationToken::new();
21844        shutdown_a.cancel();
21845        shutdown_b.cancel();
21846        let (ctx_a, store_a, shutdown_a) = without_storage_class_env(build_isolated_test_store_with_shutdown(
21847            temp_a.path(),
21848            "transition-cleanup-context-a",
21849            &[4],
21850            shutdown_a,
21851        ))
21852        .await;
21853        let (ctx_b, store_b, shutdown_b) = without_storage_class_env(build_isolated_test_store_with_shutdown(
21854            temp_b.path(),
21855            "transition-cleanup-context-b",
21856            &[4],
21857            shutdown_b,
21858        ))
21859        .await;
21860        assert!(shutdown_a.is_cancelled());
21861        assert!(shutdown_b.is_cancelled());
21862        crate::bucket::metadata_sys::init_bucket_metadata_sys(store_a.clone(), Vec::new()).await;
21863
21864        let resolver_target = Arc::new(std::sync::Mutex::new(Some(Arc::downgrade(&store_b))));
21865        let resolver_store = resolver_target.clone();
21866        assert!(
21867            set_object_store_resolver(Arc::new(move || {
21868                resolver_store
21869                    .lock()
21870                    .unwrap_or_else(std::sync::PoisonError::into_inner)
21871                    .as_ref()
21872                    .and_then(std::sync::Weak::upgrade)
21873            })),
21874            "the cross-context regression test must install the only process object-store resolver"
21875        );
21876        let _resolver_reset = ResolverReset(resolver_target);
21877        assert!(
21878            runtime_sources::object_store_handle().is_some_and(|store| Arc::ptr_eq(&store, &store_b)),
21879            "the process resolver must deliberately point at store B"
21880        );
21881
21882        let tier_name = "CROSSCTXA";
21883        let backend = register_transition_reconcile_test_tier(&ctx_a.tier_config_mgr(), tier_name).await;
21884        backend.set_put_remote_version(Some(uuid::Uuid::new_v4().to_string())).await;
21885        backend.reject_next_non_empty_remote_version_validation();
21886        let remove_barrier = backend.arm_failing_remove_barrier().await;
21887
21888        let bucket = "transition-cleanup-context-a";
21889        let object = "rejected-candidate.bin";
21890        store_a
21891            .make_bucket(bucket, &MakeBucketOptions::default())
21892            .await
21893            .expect("store A bucket should be created");
21894        let mut reader = PutObjReader::from_vec(b"cross-context rejected transition cleanup".repeat(1024));
21895        let original = store_a
21896            .put_object(bucket, object, &mut reader, &ObjectOptions::default())
21897            .await
21898            .expect("store A source object should be written");
21899        let opts = ObjectOptions {
21900            no_lock: true,
21901            transition: TransitionOptions {
21902                status: TRANSITION_PENDING.to_string(),
21903                tier: tier_name.to_string(),
21904                etag: original.etag.clone().expect("the source object should have an ETag"),
21905                ..Default::default()
21906            },
21907            version_id: original.version_id.map(|version| version.to_string()),
21908            mod_time: original.mod_time,
21909            ..Default::default()
21910        };
21911
21912        let cleanup_store_barrier = TransitionCleanupStoreBarrier::install();
21913        let transition_store = store_a.clone();
21914        let transition = tokio::spawn(async move { transition_store.transition_object(bucket, object, &opts).await });
21915        cleanup_store_barrier.wait_until_paused().await;
21916        transition.abort();
21917        assert!(
21918            transition
21919                .await
21920                .expect_err("the transition task should observe cancellation")
21921                .is_cancelled()
21922        );
21923
21924        remove_barrier.wait_until_paused().await;
21925        let transaction_counts = (
21926            transition_transaction_record_count(store_a.clone()).await,
21927            transition_transaction_record_count(store_b.clone()).await,
21928        );
21929        assert_eq!(
21930            transaction_counts,
21931            (1, 0),
21932            "the transition transaction must remain only on store A even while the process resolver points at store B"
21933        );
21934        let journal_counts = (
21935            tier_delete_journal_count(store_a.clone()).await,
21936            tier_delete_journal_count(store_b.clone()).await,
21937        );
21938        assert_eq!(
21939            journal_counts,
21940            (0, 0),
21941            "rejected transition uploads must not create a tier-delete journal"
21942        );
21943        assert_eq!(backend.object_count().await, 1, "failed cleanup should retain the remote candidate");
21944        remove_barrier.release();
21945        remove_barrier.wait_until_operation_dropped().await;
21946
21947        let recovered = recover_transition_transaction_records(store_a.clone(), 100, None)
21948            .await
21949            .expect("store A should recover its own cancelled-transition transaction");
21950        assert_eq!(
21951            (recovered.scanned, recovered.recovered, recovered.retained, recovered.failed),
21952            (1, 1, 0, 0)
21953        );
21954        assert_eq!(transition_transaction_record_count(store_a.clone()).await, 0);
21955        assert_eq!(transition_transaction_record_count(store_b.clone()).await, 0);
21956        assert_eq!(tier_delete_journal_count(store_a.clone()).await, 0);
21957        assert_eq!(tier_delete_journal_count(store_b.clone()).await, 0);
21958        assert_eq!(
21959            backend.object_count().await,
21960            0,
21961            "store A recovery should delete the exact remote candidate"
21962        );
21963        assert!(!Arc::ptr_eq(&ctx_a, &ctx_b), "the regression requires two distinct instance contexts");
21964    }
21965
21966    #[cfg(feature = "test-util")]
21967    #[tokio::test]
21968    #[serial_test::serial(storage_class_env)]
21969    async fn transition_transaction_recovery_expires_abandoned_attempt_at_budget_bound() {
21970        let temp_dir = tempfile::tempdir().expect("create temp store dir");
21971        let (ctx, store, _shutdown) = without_storage_class_env(build_isolated_test_store(
21972            temp_dir.path(),
21973            "transition-transaction-expired-attempt-budget",
21974            &[4],
21975        ))
21976        .await;
21977        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
21978
21979        let transaction = TransitionTransaction::new(TransitionTransactionInit {
21980            deployment_id: ctx.deployment_id().expect("test store should initialize deployment id"),
21981            transaction_id: uuid::Uuid::new_v4(),
21982            owner_epoch: uuid::Uuid::new_v4(),
21983            write_id: uuid::Uuid::new_v4(),
21984            source: TransitionSourceIdentity {
21985                bucket: "source-bucket".to_string(),
21986                object: "source-object".to_string(),
21987                version_id: Some(uuid::Uuid::new_v4()),
21988                data_dir: uuid::Uuid::new_v4(),
21989                mod_time_unix_nanos: 1_770_000_000_000_000_000,
21990                size: 42,
21991                etag: "source-etag".to_string(),
21992                version_mode: TransitionSourceVersionMode::Versioned,
21993            },
21994            tier_name: "UNUSEDABANDONEDTIER".to_string(),
21995            backend_fingerprint: [7; 32],
21996            not_after_unix_nanos: 1,
21997        })
21998        .expect("transaction should build");
21999        save_transition_transaction_record(store.clone(), &transaction)
22000            .await
22001            .expect("transaction record should persist");
22002
22003        let record_name =
22004            transition_transaction_record_object_name(transaction.transaction_id).expect("transaction record name should derive");
22005        let source = observe_recovery_source(
22006            store.clone(),
22007            &record_name,
22008            crate::bucket::lifecycle::transition_transaction::TRANSITION_TRANSACTION_SCHEMA,
22009        )
22010        .await
22011        .expect("transaction source generation should be observable");
22012        let mut control = IlmRecoveryControl::new(
22013            IlmRecoveryControlIdentity {
22014                protocol: IlmRecoveryProtocol::TransitionTransaction,
22015                canonical_source_path: record_name,
22016                stable_operation_identity: transaction.transaction_id.to_string(),
22017                record_class: "transition_transaction_v1".to_string(),
22018            },
22019            source.generation,
22020            IlmRecoveryClassification::Retrying,
22021            2_000_000_000,
22022            IlmRecoveryErrorCode::None,
22023        )
22024        .expect("recovery control should build");
22025        let mut now = 3_000_000_000;
22026        for _ in 1..MAX_RECOVERY_ATTEMPTS {
22027            now = now.max(control.next_attempt_at_unix_nanos.unwrap_or(now));
22028            control
22029                .claim("cancelled-or-timed-out-owner", uuid::Uuid::new_v4(), now, 1)
22030                .expect("abandoned attempt should claim");
22031            control
22032                .record_expired_attempt(now + 1)
22033                .expect("expired attempt should consume retry budget");
22034            now += 2;
22035        }
22036        now = now.max(control.next_attempt_at_unix_nanos.expect("last retry should have a backoff"));
22037        control
22038            .claim("cancelled-or-timed-out-owner", uuid::Uuid::new_v4(), now, 1)
22039            .expect("final abandoned attempt should claim");
22040        save_recovery_control_if_absent(store.clone(), &control)
22041            .await
22042            .expect("claimed recovery control should persist");
22043
22044        let stats = recover_transition_transaction_records_at(store.clone(), 100, None, i128::from(now + 1))
22045            .await
22046            .expect("recovery should account for the expired attempt");
22047        assert_eq!((stats.scanned, stats.recovered, stats.retained, stats.failed), (1, 0, 1, 0));
22048
22049        let control_id = transition_recovery_control_id(&transaction).expect("control id should derive");
22050        let persisted = load_recovery_control(store.clone(), IlmRecoveryProtocol::TransitionTransaction, &control_id)
22051            .await
22052            .expect("expired recovery control should remain inspectable");
22053        assert_eq!(persisted.control.classification, IlmRecoveryClassification::OperatorRequired);
22054        assert_eq!(persisted.control.attempt_count, u64::from(MAX_RECOVERY_ATTEMPTS));
22055        assert_eq!(persisted.control.consecutive_failure_count, MAX_RECOVERY_ATTEMPTS);
22056        assert_eq!(persisted.control.last_error_code, IlmRecoveryErrorCode::AttemptLeaseExpired);
22057        assert!(persisted.control.owner.is_none());
22058        assert_eq!(
22059            transition_transaction_record_count(store).await,
22060            1,
22061            "budget exhaustion must retain the source record"
22062        );
22063    }
22064
22065    #[cfg(feature = "test-util")]
22066    #[tokio::test]
22067    #[serial_test::serial(storage_class_env)]
22068    async fn transition_transaction_recovery_retains_active_uploaded_candidate_until_commit() {
22069        let temp_dir = tempfile::tempdir().expect("create temp store dir");
22070        let (ctx, store, _shutdown) =
22071            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "transition-transaction-active-uploaded", &[4]))
22072                .await;
22073        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
22074
22075        let tier_name = "TXACTIVEUPLOADED";
22076        let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
22077        let bucket = "transition-transaction-active-uploaded-bucket";
22078        let object = "object.bin";
22079        let payload = b"active Uploaded ownership must survive recovery until local commit".repeat(1024);
22080        store
22081            .make_bucket(bucket, &MakeBucketOptions::default())
22082            .await
22083            .expect("source bucket should be created");
22084        let mut reader = PutObjReader::from_vec(payload.clone());
22085        let source = store
22086            .put_object(bucket, object, &mut reader, &ObjectOptions::default())
22087            .await
22088            .expect("source object should be written");
22089        let barrier = TransitionUploadedCommitBarrier::install(bucket, object);
22090        let transition_store = store.clone();
22091        let transition = tokio::spawn(async move {
22092            transition_store
22093                .transition_object(
22094                    bucket,
22095                    object,
22096                    &ObjectOptions {
22097                        transition: TransitionOptions {
22098                            status: TRANSITION_PENDING.to_string(),
22099                            tier: tier_name.to_string(),
22100                            etag: source.etag.clone().expect("source object should have an ETag"),
22101                            ..Default::default()
22102                        },
22103                        version_id: source.version_id.map(|version_id| version_id.to_string()),
22104                        mod_time: source.mod_time,
22105                        ..Default::default()
22106                    },
22107                )
22108                .await
22109        });
22110        barrier.wait_until_paused().await;
22111
22112        let stats = recover_transition_transaction_records(store.clone(), 100, None)
22113            .await
22114            .expect("recovery should inspect the active Uploaded transaction");
22115        assert_eq!((stats.scanned, stats.recovered, stats.retained, stats.failed), (1, 0, 1, 0));
22116        assert_eq!(backend.object_count().await, 1, "active ownership must retain the remote candidate");
22117        assert_eq!(backend.remove_count().await, 0, "active ownership must fence remote DELETE");
22118        assert_eq!(transition_transaction_record_count(store.clone()).await, 1);
22119
22120        barrier.release();
22121        transition
22122            .await
22123            .expect("transition task should join")
22124            .expect("transition should commit after the barrier is released");
22125        assert_eq!(transition_transaction_record_count(store.clone()).await, 0);
22126        assert_eq!(backend.remove_count().await, 0);
22127
22128        let mut body = Vec::new();
22129        store
22130            .get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
22131            .await
22132            .expect("committed tier object should remain readable")
22133            .stream
22134            .read_to_end(&mut body)
22135            .await
22136            .expect("tier object body should drain");
22137        assert_eq!(body, payload);
22138    }
22139
22140    #[cfg(feature = "test-util")]
22141    #[tokio::test]
22142    #[serial_test::serial(storage_class_env)]
22143    async fn transition_transaction_recovery_deletes_expired_uploaded_remote_candidate() {
22144        let temp_dir = tempfile::tempdir().expect("create temp store dir");
22145        let (ctx, store, _shutdown) =
22146            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "transition-transaction-recovery", &[4])).await;
22147        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
22148
22149        let tier_name = "TXRECOVERY";
22150        let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
22151        let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
22152            .await
22153            .expect("tier lease should resolve")
22154            .backend_identity();
22155        let exact_version = uuid::Uuid::new_v4().to_string();
22156        let cases = [
22157            (
22158                "exact",
22159                exact_version.clone(),
22160                TransitionRemoteVersion::versioned(exact_version),
22161                TransitionSourceVersionMode::Versioned,
22162            ),
22163            (
22164                "suspended-null",
22165                "null".to_string(),
22166                TransitionRemoteVersion::versioned("null"),
22167                TransitionSourceVersionMode::VersionSuspended,
22168            ),
22169            (
22170                "known-unversioned",
22171                String::new(),
22172                TransitionRemoteVersion::unversioned(),
22173                TransitionSourceVersionMode::Unversioned,
22174            ),
22175        ];
22176        let mut expected_removes = Vec::new();
22177        let mut recovery_control_ids = Vec::new();
22178        for (case, put_version, remote_version, source_mode) in cases {
22179            let mut transaction = TransitionTransaction::new(TransitionTransactionInit {
22180                deployment_id: ctx.deployment_id().expect("test store should initialize deployment id"),
22181                transaction_id: uuid::Uuid::new_v4(),
22182                owner_epoch: uuid::Uuid::new_v4(),
22183                write_id: uuid::Uuid::new_v4(),
22184                source: TransitionSourceIdentity {
22185                    bucket: "source-bucket".to_string(),
22186                    object: format!("source-{case}"),
22187                    version_id: (source_mode == TransitionSourceVersionMode::Versioned).then(uuid::Uuid::new_v4),
22188                    data_dir: uuid::Uuid::new_v4(),
22189                    mod_time_unix_nanos: 1_770_000_000_000_000_000,
22190                    size: 42,
22191                    etag: "source-etag".to_string(),
22192                    version_mode: source_mode,
22193                },
22194                tier_name: tier_name.to_string(),
22195                backend_fingerprint: backend_identity,
22196                not_after_unix_nanos: 1,
22197            })
22198            .expect("transaction should build");
22199            transaction
22200                .advance(transaction.fence(), TransitionTransactionState::Uploaded, Some(remote_version))
22201                .expect("transaction should enter uploaded state");
22202            backend.set_put_remote_version(Some(put_version.clone())).await;
22203            let candidate = bytes::Bytes::from_static(b"orphan candidate");
22204            backend
22205                .put(
22206                    &transaction.remote_object,
22207                    ReaderImpl::Body(candidate.clone()),
22208                    i64::try_from(candidate.len()).expect("test candidate length should fit i64"),
22209                )
22210                .await
22211                .expect("mock backend should accept candidate");
22212            save_transition_transaction_record(store.clone(), &transaction)
22213                .await
22214                .expect("transaction record should persist");
22215            recovery_control_ids
22216                .push(transition_recovery_control_id(&transaction).expect("transition recovery control id should derive"));
22217            expected_removes.push((transaction.remote_object, put_version));
22218        }
22219
22220        let stats = {
22221            let _proof = crate::services::notification_sys::install_current_remote_version_state_fleet_proof_for_test();
22222            temp_env::async_with_vars(
22223                [
22224                    (rustfs_config::ENV_TIER_REMOTE_VERSION_STATE_WRITE, Some("true")),
22225                    (rustfs_config::ENV_TIER_REMOTE_VERSION_STATE_FLEET_CONFIRMED, Some("true")),
22226                ],
22227                recover_transition_transaction_records(store.clone(), 100, None),
22228            )
22229            .await
22230            .expect("transition transaction recovery should run")
22231        };
22232
22233        assert_eq!((stats.scanned, stats.recovered, stats.retained, stats.failed), (3, 3, 0, 0));
22234        assert_eq!(transition_transaction_record_count(store.clone()).await, 0);
22235        let mut actual_removes = backend.remove_versions().await;
22236        actual_removes.sort();
22237        expected_removes.sort();
22238        assert_eq!(actual_removes, expected_removes, "recovery must preserve each remote version shape");
22239        assert_eq!(backend.exact_remove_count(), 2);
22240        assert_eq!(backend.object_count().await, 0);
22241        for control_id in recovery_control_ids {
22242            let control = load_recovery_control(store.clone(), IlmRecoveryProtocol::TransitionTransaction, &control_id)
22243                .await
22244                .expect("completed recovery control should remain inspectable");
22245            assert_eq!(control.control.classification, IlmRecoveryClassification::Terminal);
22246            assert_eq!(control.control.attempt_count, 1);
22247            assert!(control.control.owner.is_none());
22248        }
22249
22250        let replay = recover_transition_transaction_records(store, 100, None)
22251            .await
22252            .expect("replayed recovery should remain idempotent");
22253        assert_eq!((replay.scanned, replay.recovered, replay.retained, replay.failed), (0, 0, 0, 0));
22254        assert_eq!(
22255            backend.remove_versions().await.len(),
22256            3,
22257            "each expired candidate must be deleted only once"
22258        );
22259    }
22260
22261    #[cfg(feature = "test-util")]
22262    #[tokio::test]
22263    #[serial_test::serial(storage_class_env)]
22264    async fn transition_transaction_recovery_resumes_source_cleanup_after_terminal_crash() {
22265        let temp_dir = tempfile::tempdir().expect("create temp store dir");
22266        let (ctx, store, _shutdown) =
22267            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "transition-transaction-terminal-crash", &[4]))
22268                .await;
22269        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
22270
22271        let tier_name = "TXTERMINALCRASH";
22272        let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
22273        let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
22274            .await
22275            .expect("tier lease should resolve")
22276            .backend_identity();
22277        let remote_version = uuid::Uuid::new_v4().to_string();
22278        let mut transaction = TransitionTransaction::new(TransitionTransactionInit {
22279            deployment_id: ctx.deployment_id().expect("test store should initialize deployment id"),
22280            transaction_id: uuid::Uuid::new_v4(),
22281            owner_epoch: uuid::Uuid::new_v4(),
22282            write_id: uuid::Uuid::new_v4(),
22283            source: TransitionSourceIdentity {
22284                bucket: "source-bucket".to_string(),
22285                object: "source-object".to_string(),
22286                version_id: Some(uuid::Uuid::new_v4()),
22287                data_dir: uuid::Uuid::new_v4(),
22288                mod_time_unix_nanos: 1_770_000_000_000_000_000,
22289                size: 42,
22290                etag: "source-etag".to_string(),
22291                version_mode: TransitionSourceVersionMode::Versioned,
22292            },
22293            tier_name: tier_name.to_string(),
22294            backend_fingerprint: backend_identity,
22295            not_after_unix_nanos: 1,
22296        })
22297        .expect("transaction should build");
22298        transaction
22299            .advance(
22300                transaction.fence(),
22301                TransitionTransactionState::Uploaded,
22302                Some(TransitionRemoteVersion::versioned(remote_version.clone())),
22303            )
22304            .expect("transaction should enter uploaded state");
22305        backend.set_put_remote_version(Some(remote_version)).await;
22306        let candidate = bytes::Bytes::from_static(b"terminal crash candidate");
22307        backend
22308            .put(
22309                &transaction.remote_object,
22310                ReaderImpl::Body(candidate.clone()),
22311                i64::try_from(candidate.len()).expect("test candidate length should fit i64"),
22312            )
22313            .await
22314            .expect("mock backend should accept candidate");
22315        save_transition_transaction_record(store.clone(), &transaction)
22316            .await
22317            .expect("transaction record should persist");
22318        let control_id = transition_recovery_control_id(&transaction).expect("control id should derive");
22319
22320        let barrier = TransitionRecoveryTerminalBarrier::install(transaction.transaction_id);
22321        let recovery_store = store.clone();
22322        let recovery = tokio::spawn(async move { recover_transition_transaction_records(recovery_store, 100, None).await });
22323        barrier.wait_until_paused().await;
22324        let terminal = load_recovery_control(store.clone(), IlmRecoveryProtocol::TransitionTransaction, &control_id)
22325            .await
22326            .expect("terminal control should persist before source cleanup");
22327        assert_eq!(terminal.control.classification, IlmRecoveryClassification::Terminal);
22328        assert_eq!(transition_transaction_record_count(store.clone()).await, 1);
22329        assert_eq!(backend.object_count().await, 0);
22330        assert_eq!(backend.exact_remove_count(), 1);
22331
22332        recovery.abort();
22333        assert!(
22334            recovery
22335                .await
22336                .expect_err("recovery should be cancelled at the crash boundary")
22337                .is_cancelled()
22338        );
22339        drop(barrier);
22340
22341        let replay = recover_transition_transaction_records(store.clone(), 100, None)
22342            .await
22343            .expect("terminal control should resume source cleanup without another remote delete");
22344        assert_eq!((replay.scanned, replay.recovered, replay.retained, replay.failed), (1, 1, 0, 0));
22345        assert_eq!(transition_transaction_record_count(store).await, 0);
22346        assert_eq!(backend.exact_remove_count(), 1, "terminal replay must not repeat the remote delete");
22347    }
22348
22349    #[cfg(feature = "test-util")]
22350    #[tokio::test]
22351    #[serial_test::serial(storage_class_env)]
22352    async fn transition_transaction_recovery_cas_loses_to_active_state_advance() {
22353        let temp_dir = tempfile::tempdir().expect("create temp store dir");
22354        let (ctx, store, _shutdown) =
22355            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "transition-transaction-recovery-cas", &[4]))
22356                .await;
22357        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
22358
22359        let tier_name = "TXRECOVERYCAS";
22360        let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
22361        let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
22362            .await
22363            .expect("tier lease should resolve")
22364            .backend_identity();
22365        let remote_version = uuid::Uuid::new_v4().to_string();
22366        let mut uploaded = TransitionTransaction::new(TransitionTransactionInit {
22367            deployment_id: ctx.deployment_id().expect("test store should initialize deployment id"),
22368            transaction_id: uuid::Uuid::new_v4(),
22369            owner_epoch: uuid::Uuid::new_v4(),
22370            write_id: uuid::Uuid::new_v4(),
22371            source: TransitionSourceIdentity {
22372                bucket: "source-bucket".to_string(),
22373                object: "source-object".to_string(),
22374                version_id: Some(uuid::Uuid::new_v4()),
22375                data_dir: uuid::Uuid::new_v4(),
22376                mod_time_unix_nanos: 1_770_000_000_000_000_000,
22377                size: 42,
22378                etag: "source-etag".to_string(),
22379                version_mode: TransitionSourceVersionMode::Versioned,
22380            },
22381            tier_name: tier_name.to_string(),
22382            backend_fingerprint: backend_identity,
22383            not_after_unix_nanos: 1,
22384        })
22385        .expect("transaction should build");
22386        uploaded
22387            .advance(
22388                uploaded.fence(),
22389                TransitionTransactionState::Uploaded,
22390                Some(TransitionRemoteVersion::versioned(remote_version.clone())),
22391            )
22392            .expect("transaction should enter uploaded state");
22393        backend.set_put_remote_version(Some(remote_version)).await;
22394        let candidate = bytes::Bytes::from_static(b"CAS-owned transition remote candidate");
22395        backend
22396            .put(
22397                &uploaded.remote_object,
22398                ReaderImpl::Body(candidate.clone()),
22399                i64::try_from(candidate.len()).expect("test candidate length should fit i64"),
22400            )
22401            .await
22402            .expect("mock backend should accept candidate");
22403        save_transition_transaction_record(store.clone(), &uploaded)
22404            .await
22405            .expect("transaction record should persist");
22406        let recovery_control_id =
22407            transition_recovery_control_id(&uploaded).expect("transition recovery control id should derive");
22408
22409        let barrier = TransitionRecoveryClaimBarrier::install(uploaded.transaction_id);
22410        let recovery_store = store.clone();
22411        let recovery = tokio::spawn(async move { recover_transition_transaction_records(recovery_store, 100, None).await });
22412        barrier.wait_until_paused().await;
22413
22414        let mut active = uploaded.clone();
22415        active
22416            .advance(active.fence(), TransitionTransactionState::LocalCommitStarted, None)
22417            .expect("active owner should advance to local commit");
22418        save_transition_transaction_record_if_current(store.clone(), &uploaded, &active)
22419            .await
22420            .expect("active owner should win the persisted CAS");
22421        barrier.release();
22422
22423        let stats = recovery
22424            .await
22425            .expect("recovery task should join")
22426            .expect("recovery should treat the lost CAS as a retained transaction");
22427        assert_eq!((stats.scanned, stats.recovered, stats.retained, stats.failed), (1, 0, 1, 0));
22428        assert_eq!(
22429            load_transition_transaction_record(store.clone(), uploaded.transaction_id)
22430                .await
22431                .expect("newer transaction revision must remain"),
22432            active
22433        );
22434        let control = load_recovery_control(store, IlmRecoveryProtocol::TransitionTransaction, &recovery_control_id)
22435            .await
22436            .expect("lost source CAS should retain a retryable recovery control");
22437        assert_eq!(control.control.classification, IlmRecoveryClassification::Retrying);
22438        assert_eq!(control.control.consecutive_failure_count, 1);
22439        assert_eq!(control.control.last_error_code, IlmRecoveryErrorCode::SourceGenerationChanged);
22440        assert_eq!(backend.object_count().await, 1, "a stale recovery must not delete the candidate");
22441        assert_eq!(backend.remove_count().await, 0);
22442    }
22443
22444    #[cfg(feature = "test-util")]
22445    #[tokio::test]
22446    #[serial_test::serial(storage_class_env)]
22447    async fn transition_transaction_recovery_quarantines_missing_required_fields() {
22448        let temp_dir = tempfile::tempdir().expect("create temp store dir");
22449        let (ctx, store, _shutdown) = without_storage_class_env(build_isolated_test_store(
22450            temp_dir.path(),
22451            "transition-transaction-recovery-corrupt",
22452            &[4],
22453        ))
22454        .await;
22455        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
22456
22457        let tier_name = "TXRECOVERYCORRUPT";
22458        let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
22459        let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
22460            .await
22461            .expect("tier lease should resolve")
22462            .backend_identity();
22463        let remote_version = uuid::Uuid::new_v4().to_string();
22464        let mut transaction = TransitionTransaction::new(TransitionTransactionInit {
22465            deployment_id: ctx.deployment_id().expect("test store should initialize deployment id"),
22466            transaction_id: uuid::Uuid::new_v4(),
22467            owner_epoch: uuid::Uuid::new_v4(),
22468            write_id: uuid::Uuid::new_v4(),
22469            source: TransitionSourceIdentity {
22470                bucket: "source-bucket".to_string(),
22471                object: "source-object".to_string(),
22472                version_id: None,
22473                data_dir: uuid::Uuid::new_v4(),
22474                mod_time_unix_nanos: 1_770_000_000_000_000_000,
22475                size: 42,
22476                etag: "source-etag".to_string(),
22477                version_mode: TransitionSourceVersionMode::Unversioned,
22478            },
22479            tier_name: tier_name.to_string(),
22480            backend_fingerprint: backend_identity,
22481            not_after_unix_nanos: 1,
22482        })
22483        .expect("transaction should build");
22484        transaction
22485            .advance(
22486                transaction.fence(),
22487                TransitionTransactionState::Uploaded,
22488                Some(TransitionRemoteVersion::versioned(remote_version.clone())),
22489            )
22490            .expect("transaction should enter uploaded state");
22491        backend.set_put_remote_version(Some(remote_version)).await;
22492        let candidate = bytes::Bytes::from_static(b"corrupt journal candidate");
22493        backend
22494            .put(
22495                &transaction.remote_object,
22496                ReaderImpl::Body(candidate.clone()),
22497                i64::try_from(candidate.len()).expect("test candidate length should fit i64"),
22498            )
22499            .await
22500            .expect("mock backend should accept candidate");
22501
22502        let mut persisted: serde_json::Value = serde_json::from_slice(&transaction.encode().expect("transaction should encode"))
22503            .expect("transaction should be JSON");
22504        persisted["transaction"]
22505            .as_object_mut()
22506            .expect("transaction payload should be an object")
22507            .remove("not_after_unix_nanos");
22508        let path =
22509            transition_transaction_record_object_name(transaction.transaction_id).expect("transaction path should be canonical");
22510        com::save_config(
22511            store.clone(),
22512            &path,
22513            serde_json::to_vec(&persisted).expect("corrupt fixture should encode"),
22514        )
22515        .await
22516        .expect("corrupt transaction fixture should persist");
22517
22518        let stats = recover_transition_transaction_records(store.clone(), 100, None)
22519            .await
22520            .expect("recovery scan should isolate a corrupt record");
22521        assert_eq!((stats.scanned, stats.recovered, stats.retained, stats.failed), (1, 0, 0, 1));
22522        assert_eq!(
22523            transition_transaction_record_count(store).await,
22524            1,
22525            "corrupt evidence must remain quarantined"
22526        );
22527        assert_eq!(backend.object_count().await, 1, "corrupt evidence must never authorize remote DELETE");
22528        assert_eq!(backend.remove_count().await, 0);
22529    }
22530
22531    #[cfg(feature = "test-util")]
22532    #[tokio::test]
22533    #[serial_test::serial(storage_class_env)]
22534    async fn transition_transaction_recovery_retries_cleanup_pending_candidate() {
22535        let temp_dir = tempfile::tempdir().expect("create temp store dir");
22536        let (ctx, store, _shutdown) =
22537            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "transition-transaction-cleanup", &[4])).await;
22538        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
22539
22540        let tier_name = "TXCLEANUP";
22541        let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
22542        let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
22543            .await
22544            .expect("tier lease should resolve")
22545            .backend_identity();
22546        let remote_version = uuid::Uuid::new_v4().to_string();
22547        let mut transaction = TransitionTransaction::new(TransitionTransactionInit {
22548            deployment_id: ctx.deployment_id().expect("test store should initialize deployment id"),
22549            transaction_id: uuid::Uuid::new_v4(),
22550            owner_epoch: uuid::Uuid::new_v4(),
22551            write_id: uuid::Uuid::new_v4(),
22552            source: TransitionSourceIdentity {
22553                bucket: "source-bucket".to_string(),
22554                object: "source-object".to_string(),
22555                version_id: Some(uuid::Uuid::new_v4()),
22556                data_dir: uuid::Uuid::new_v4(),
22557                mod_time_unix_nanos: 1_770_000_000_000_000_000,
22558                size: 42,
22559                etag: "source-etag".to_string(),
22560                version_mode: TransitionSourceVersionMode::Versioned,
22561            },
22562            tier_name: tier_name.to_string(),
22563            backend_fingerprint: backend_identity,
22564            not_after_unix_nanos: 1_780_000_000_000_000_000,
22565        })
22566        .expect("transaction should build");
22567        let uploaded_fence = transaction
22568            .advance(
22569                transaction.fence(),
22570                TransitionTransactionState::Uploaded,
22571                Some(TransitionRemoteVersion::versioned(remote_version.clone())),
22572            )
22573            .expect("transaction should enter uploaded state");
22574        transaction
22575            .mark_cleanup_pending(
22576                uploaded_fence,
22577                TransitionCleanupProof {
22578                    transaction_id: transaction.transaction_id,
22579                    write_id: transaction.write_id,
22580                    remote_object: transaction.remote_object.clone(),
22581                    remote_version: transaction.remote_version.clone(),
22582                    backend_fingerprint: transaction.backend_fingerprint,
22583                    decision: TransitionCleanupDecision::UploadAbortedBeforeLocalCommit,
22584                },
22585            )
22586            .expect("transaction should enter cleanup pending state");
22587        backend.set_put_remote_version(Some(remote_version.clone())).await;
22588        let candidate = bytes::Bytes::from_static(b"cleanup pending candidate");
22589        backend
22590            .put(
22591                &transaction.remote_object,
22592                ReaderImpl::Body(candidate.clone()),
22593                i64::try_from(candidate.len()).expect("test candidate length should fit i64"),
22594            )
22595            .await
22596            .expect("mock backend should accept candidate");
22597        save_transition_transaction_record(store.clone(), &transaction)
22598            .await
22599            .expect("transaction record should persist");
22600
22601        let stats = recover_transition_transaction_records(store.clone(), 100, None)
22602            .await
22603            .expect("transition transaction recovery should run");
22604
22605        assert_eq!((stats.scanned, stats.recovered, stats.retained, stats.failed), (1, 1, 0, 0));
22606        assert_eq!(transition_transaction_record_count(store.clone()).await, 0);
22607        assert_eq!(
22608            backend.remove_versions().await,
22609            vec![(transaction.remote_object.clone(), remote_version)],
22610            "cleanup pending recovery must retry the exact remote candidate delete"
22611        );
22612        assert_eq!(backend.exact_remove_count(), 1);
22613        assert_eq!(backend.object_count().await, 0);
22614    }
22615
22616    #[cfg(feature = "test-util")]
22617    #[tokio::test]
22618    #[serial_test::serial(storage_class_env)]
22619    async fn transition_transaction_recovery_retries_cleanup_pending_unversioned_candidate() {
22620        let temp_dir = tempfile::tempdir().expect("create temp store dir");
22621        let (ctx, store, _shutdown) = without_storage_class_env(build_isolated_test_store(
22622            temp_dir.path(),
22623            "transition-transaction-cleanup-unversioned",
22624            &[4],
22625        ))
22626        .await;
22627        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
22628
22629        let tier_name = "TXCLEANUPUNVERSIONED";
22630        let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
22631        let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
22632            .await
22633            .expect("tier lease should resolve")
22634            .backend_identity();
22635        let mut transaction = TransitionTransaction::new(TransitionTransactionInit {
22636            deployment_id: ctx.deployment_id().expect("test store should initialize deployment id"),
22637            transaction_id: uuid::Uuid::new_v4(),
22638            owner_epoch: uuid::Uuid::new_v4(),
22639            write_id: uuid::Uuid::new_v4(),
22640            source: TransitionSourceIdentity {
22641                bucket: "source-bucket".to_string(),
22642                object: "source-object".to_string(),
22643                version_id: None,
22644                data_dir: uuid::Uuid::new_v4(),
22645                mod_time_unix_nanos: 1_770_000_000_000_000_000,
22646                size: 42,
22647                etag: "source-etag".to_string(),
22648                version_mode: TransitionSourceVersionMode::Unversioned,
22649            },
22650            tier_name: tier_name.to_string(),
22651            backend_fingerprint: backend_identity,
22652            not_after_unix_nanos: 1_780_000_000_000_000_000,
22653        })
22654        .expect("transaction should build");
22655        let uploaded_fence = transaction
22656            .advance(
22657                transaction.fence(),
22658                TransitionTransactionState::Uploaded,
22659                Some(TransitionRemoteVersion::unversioned()),
22660            )
22661            .expect("transaction should enter uploaded state");
22662        transaction
22663            .mark_cleanup_pending(
22664                uploaded_fence,
22665                TransitionCleanupProof {
22666                    transaction_id: transaction.transaction_id,
22667                    write_id: transaction.write_id,
22668                    remote_object: transaction.remote_object.clone(),
22669                    remote_version: transaction.remote_version.clone(),
22670                    backend_fingerprint: transaction.backend_fingerprint,
22671                    decision: TransitionCleanupDecision::UploadAbortedBeforeLocalCommit,
22672                },
22673            )
22674            .expect("transaction should enter cleanup pending state");
22675        backend.set_put_remote_version(Some(String::new())).await;
22676        let candidate = bytes::Bytes::from_static(b"cleanup pending unversioned candidate");
22677        backend
22678            .put(
22679                &transaction.remote_object,
22680                ReaderImpl::Body(candidate.clone()),
22681                i64::try_from(candidate.len()).expect("test candidate length should fit i64"),
22682            )
22683            .await
22684            .expect("mock backend should accept candidate");
22685        save_transition_transaction_record(store.clone(), &transaction)
22686            .await
22687            .expect("transaction record should persist");
22688
22689        let stats = recover_transition_transaction_records(store.clone(), 100, None)
22690            .await
22691            .expect("transition transaction recovery should run");
22692
22693        assert_eq!((stats.scanned, stats.recovered, stats.retained, stats.failed), (1, 1, 0, 0));
22694        assert_eq!(transition_transaction_record_count(store.clone()).await, 0);
22695        assert_eq!(
22696            backend.remove_versions().await,
22697            vec![(transaction.remote_object.clone(), String::new())],
22698            "unversioned cleanup must not send a synthetic remote version"
22699        );
22700        assert_eq!(backend.exact_remove_count(), 0);
22701        assert_eq!(backend.object_count().await, 0);
22702    }
22703
22704    #[cfg(feature = "test-util")]
22705    #[tokio::test]
22706    #[serial_test::serial(storage_class_env)]
22707    async fn transition_transaction_recovery_keeps_cleanup_pending_record_when_delete_fails() {
22708        let temp_dir = tempfile::tempdir().expect("create temp store dir");
22709        let (ctx, store, _shutdown) =
22710            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "transition-transaction-cleanup-fail", &[4]))
22711                .await;
22712        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
22713
22714        let tier_name = "TXCLEANUPFAIL";
22715        let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
22716        let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
22717            .await
22718            .expect("tier lease should resolve")
22719            .backend_identity();
22720        let remote_version = uuid::Uuid::new_v4().to_string();
22721        let mut transaction = TransitionTransaction::new(TransitionTransactionInit {
22722            deployment_id: ctx.deployment_id().expect("test store should initialize deployment id"),
22723            transaction_id: uuid::Uuid::new_v4(),
22724            owner_epoch: uuid::Uuid::new_v4(),
22725            write_id: uuid::Uuid::new_v4(),
22726            source: TransitionSourceIdentity {
22727                bucket: "source-bucket".to_string(),
22728                object: "source-object".to_string(),
22729                version_id: Some(uuid::Uuid::new_v4()),
22730                data_dir: uuid::Uuid::new_v4(),
22731                mod_time_unix_nanos: 1_770_000_000_000_000_000,
22732                size: 42,
22733                etag: "source-etag".to_string(),
22734                version_mode: TransitionSourceVersionMode::Versioned,
22735            },
22736            tier_name: tier_name.to_string(),
22737            backend_fingerprint: backend_identity,
22738            not_after_unix_nanos: 1_780_000_000_000_000_000,
22739        })
22740        .expect("transaction should build");
22741        transaction
22742            .advance(
22743                transaction.fence(),
22744                TransitionTransactionState::Uploaded,
22745                Some(TransitionRemoteVersion::versioned(remote_version.clone())),
22746            )
22747            .expect("transaction should enter uploaded state");
22748        let candidate = bytes::Bytes::from_static(b"cleanup pending candidate retained after failure");
22749        backend.set_put_remote_version(Some(remote_version)).await;
22750        backend
22751            .put(
22752                &transaction.remote_object,
22753                ReaderImpl::Body(candidate.clone()),
22754                i64::try_from(candidate.len()).expect("test candidate length should fit i64"),
22755            )
22756            .await
22757            .expect("mock backend should accept candidate");
22758        save_transition_transaction_record(store.clone(), &transaction)
22759            .await
22760            .expect("transaction record should persist");
22761        let recovery_control_id =
22762            transition_recovery_control_id(&transaction).expect("transition recovery control id should derive");
22763
22764        backend.set_remove_failure(true);
22765        let stats = recover_transition_transaction_records(store.clone(), 100, None)
22766            .await
22767            .expect("transition transaction recovery should keep scanning after cleanup failure");
22768
22769        assert_eq!((stats.scanned, stats.recovered, stats.retained, stats.failed), (1, 0, 0, 1));
22770        assert_eq!(
22771            transition_transaction_record_count(store.clone()).await,
22772            1,
22773            "failed cleanup must keep the cleanup-pending transaction for retry"
22774        );
22775        assert_eq!(backend.remove_versions().await, Vec::<(String, String)>::new());
22776        assert_eq!(backend.exact_remove_count(), 1);
22777        assert_eq!(backend.object_count().await, 1);
22778        let control = load_recovery_control(store.clone(), IlmRecoveryProtocol::TransitionTransaction, &recovery_control_id)
22779            .await
22780            .expect("failed recovery control should persist");
22781        assert_eq!(control.control.classification, IlmRecoveryClassification::Retrying);
22782        assert_eq!(control.control.attempt_count, 1);
22783        assert_eq!(control.control.consecutive_failure_count, 1);
22784        assert!(
22785            control
22786                .control
22787                .next_attempt_at_unix_nanos
22788                .is_some_and(|next| next > OffsetDateTime::now_utc().unix_timestamp_nanos() as i64)
22789        );
22790
22791        backend.set_remove_failure(false);
22792        let replay = recover_transition_transaction_records(store.clone(), 100, None)
22793            .await
22794            .expect("recovery before the persisted deadline should be skipped");
22795        assert_eq!((replay.scanned, replay.recovered, replay.retained, replay.failed), (1, 0, 1, 0));
22796        assert_eq!(backend.exact_remove_count(), 1, "persisted backoff must prevent an immediate retry");
22797
22798        let retry_at = control
22799            .control
22800            .next_attempt_at_unix_nanos
22801            .expect("retry deadline should persist");
22802        let retried = recover_transition_transaction_records_at(store.clone(), 100, None, i128::from(retry_at) + 1)
22803            .await
22804            .expect("recovery at the persisted deadline should retry the advanced source generation");
22805        assert_eq!((retried.scanned, retried.recovered, retried.retained, retried.failed), (1, 1, 0, 0));
22806        assert_eq!(backend.exact_remove_count(), 2);
22807        assert_eq!(backend.object_count().await, 0);
22808        assert_eq!(transition_transaction_record_count(store.clone()).await, 0);
22809        let terminal = load_recovery_control(store, IlmRecoveryProtocol::TransitionTransaction, &recovery_control_id)
22810            .await
22811            .expect("completed retry control should remain inspectable");
22812        assert_eq!(terminal.control.classification, IlmRecoveryClassification::Terminal);
22813        assert_eq!(terminal.control.attempt_count, 2);
22814    }
22815
22816    #[cfg(feature = "test-util")]
22817    #[tokio::test]
22818    #[serial_test::serial(storage_class_env)]
22819    async fn transition_transaction_recovery_keeps_cleanup_pending_local_commit() {
22820        let temp_dir = tempfile::tempdir().expect("create temp store dir");
22821        let (ctx, store, _shutdown) = without_storage_class_env(build_isolated_test_store(
22822            temp_dir.path(),
22823            "transition-transaction-cleanup-committed",
22824            &[4],
22825        ))
22826        .await;
22827        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
22828
22829        let tier_name = "TXCLEANUPCOMMITTED";
22830        let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
22831        let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
22832            .await
22833            .expect("tier lease should resolve")
22834            .backend_identity();
22835        let bucket = "transition-transaction-cleanup-committed-bucket";
22836        let object = "object.bin";
22837        store
22838            .make_bucket(bucket, &MakeBucketOptions::default())
22839            .await
22840            .expect("bucket should be created");
22841        let mut reader = PutObjReader::from_vec(b"cleanup pending local commit record cleanup".repeat(1024));
22842        let original = store
22843            .put_object(bucket, object, &mut reader, &ObjectOptions::default())
22844            .await
22845            .expect("source object should be written");
22846        let opts = ObjectOptions {
22847            no_lock: true,
22848            transition: TransitionOptions {
22849                status: TRANSITION_PENDING.to_string(),
22850                tier: tier_name.to_string(),
22851                etag: original.etag.clone().expect("source object should have etag"),
22852                ..Default::default()
22853            },
22854            mod_time: original.mod_time,
22855            ..Default::default()
22856        };
22857        store
22858            .transition_object(bucket, object, &opts)
22859            .await
22860            .expect("transition should commit");
22861        let committed = store
22862            .get_object_info(
22863                bucket,
22864                object,
22865                &ObjectOptions {
22866                    no_lock: true,
22867                    metadata_cache_safe: false,
22868                    ..Default::default()
22869                },
22870            )
22871            .await
22872            .expect("committed object info should be readable");
22873        let mut remote_parts = committed.transitioned_object.name.rsplit('/');
22874        let write_id = uuid::Uuid::parse_str(remote_parts.next().expect("remote object should contain write id"))
22875            .expect("write id should parse");
22876        let transaction_id = uuid::Uuid::parse_str(remote_parts.next().expect("remote object should contain transaction id"))
22877            .expect("transaction id should parse");
22878        let source = TransitionSourceIdentity {
22879            bucket: bucket.to_string(),
22880            object: object.to_string(),
22881            version_id: None,
22882            data_dir: original.data_dir.expect("source object should have data_dir"),
22883            mod_time_unix_nanos: original
22884                .mod_time
22885                .expect("source object should have mod_time")
22886                .unix_timestamp_nanos()
22887                .try_into()
22888                .expect("test timestamp should fit i64"),
22889            size: original.size,
22890            etag: original.etag.expect("source object should have etag"),
22891            version_mode: TransitionSourceVersionMode::Unversioned,
22892        };
22893        let mut transaction = TransitionTransaction::new(TransitionTransactionInit {
22894            deployment_id: ctx.deployment_id().expect("test store should initialize deployment id"),
22895            transaction_id,
22896            owner_epoch: uuid::Uuid::new_v4(),
22897            write_id,
22898            source,
22899            tier_name: tier_name.to_string(),
22900            backend_fingerprint: backend_identity,
22901            not_after_unix_nanos: 1_780_000_000_000_000_000,
22902        })
22903        .expect("transaction should build");
22904        transaction
22905            .advance(
22906                transaction.fence(),
22907                TransitionTransactionState::Uploaded,
22908                Some(TransitionRemoteVersion::known_from_put_response(
22909                    committed.transitioned_object.version_id.clone(),
22910                )),
22911            )
22912            .expect("transaction should enter uploaded state");
22913        let local_commit_fence = transaction
22914            .advance(transaction.fence(), TransitionTransactionState::LocalCommitStarted, None)
22915            .expect("transaction should enter local commit state");
22916        transaction
22917            .mark_cleanup_pending(
22918                local_commit_fence,
22919                TransitionCleanupProof {
22920                    transaction_id: transaction.transaction_id,
22921                    write_id: transaction.write_id,
22922                    remote_object: transaction.remote_object.clone(),
22923                    remote_version: transaction.remote_version.clone(),
22924                    backend_fingerprint: transaction.backend_fingerprint,
22925                    decision: TransitionCleanupDecision::SourceReconciledUnchanged {
22926                        observed_source: transaction.source.clone(),
22927                    },
22928                },
22929            )
22930            .expect("transaction should enter cleanup pending state");
22931        save_transition_transaction_record(store.clone(), &transaction)
22932            .await
22933            .expect("transaction record should persist");
22934
22935        let stats = recover_transition_transaction_records(store.clone(), 100, None)
22936            .await
22937            .expect("transition transaction recovery should run");
22938
22939        assert_eq!((stats.scanned, stats.recovered, stats.retained, stats.failed), (1, 1, 0, 0));
22940        assert_eq!(transition_transaction_record_count(store.clone()).await, 0);
22941        assert_eq!(
22942            backend.object_count().await,
22943            1,
22944            "cleanup pending recovery must keep the remote body once local metadata references it"
22945        );
22946        assert_eq!(backend.remove_count().await, 0);
22947    }
22948
22949    #[cfg(feature = "test-util")]
22950    #[tokio::test]
22951    #[serial_test::serial(storage_class_env)]
22952    async fn transition_transaction_recovery_drops_record_after_confirmed_local_commit() {
22953        let temp_dir = tempfile::tempdir().expect("create temp store dir");
22954        let (ctx, store, _shutdown) =
22955            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "transition-transaction-committed", &[4])).await;
22956        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
22957
22958        let tier_name = "TXCOMMITTED";
22959        let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
22960        let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
22961            .await
22962            .expect("tier lease should resolve")
22963            .backend_identity();
22964        let bucket = "transition-transaction-committed-bucket";
22965        let object = "object.bin";
22966        store
22967            .make_bucket(bucket, &MakeBucketOptions::default())
22968            .await
22969            .expect("bucket should be created");
22970        let mut reader = PutObjReader::from_vec(b"confirmed local commit record cleanup".repeat(1024));
22971        let original = store
22972            .put_object(bucket, object, &mut reader, &ObjectOptions::default())
22973            .await
22974            .expect("source object should be written");
22975        let opts = ObjectOptions {
22976            no_lock: true,
22977            transition: TransitionOptions {
22978                status: TRANSITION_PENDING.to_string(),
22979                tier: tier_name.to_string(),
22980                etag: original.etag.clone().expect("source object should have etag"),
22981                ..Default::default()
22982            },
22983            mod_time: original.mod_time,
22984            ..Default::default()
22985        };
22986        store
22987            .transition_object(bucket, object, &opts)
22988            .await
22989            .expect("transition should commit");
22990        let committed = store
22991            .get_object_info(
22992                bucket,
22993                object,
22994                &ObjectOptions {
22995                    no_lock: true,
22996                    metadata_cache_safe: false,
22997                    ..Default::default()
22998                },
22999            )
23000            .await
23001            .expect("committed object info should be readable");
23002        let mut remote_parts = committed.transitioned_object.name.rsplit('/');
23003        let write_id = uuid::Uuid::parse_str(remote_parts.next().expect("remote object should contain write id"))
23004            .expect("write id should parse");
23005        let transaction_id = uuid::Uuid::parse_str(remote_parts.next().expect("remote object should contain transaction id"))
23006            .expect("transaction id should parse");
23007        let mut transaction = TransitionTransaction::new(TransitionTransactionInit {
23008            deployment_id: ctx.deployment_id().expect("test store should initialize deployment id"),
23009            transaction_id,
23010            owner_epoch: uuid::Uuid::new_v4(),
23011            write_id,
23012            source: TransitionSourceIdentity {
23013                bucket: bucket.to_string(),
23014                object: object.to_string(),
23015                version_id: None,
23016                data_dir: original.data_dir.expect("source object should have data_dir"),
23017                mod_time_unix_nanos: original
23018                    .mod_time
23019                    .expect("source object should have mod_time")
23020                    .unix_timestamp_nanos()
23021                    .try_into()
23022                    .expect("test timestamp should fit i64"),
23023                size: original.size,
23024                etag: original.etag.expect("source object should have etag"),
23025                version_mode: TransitionSourceVersionMode::Unversioned,
23026            },
23027            tier_name: tier_name.to_string(),
23028            backend_fingerprint: backend_identity,
23029            not_after_unix_nanos: 1_780_000_000_000_000_000,
23030        })
23031        .expect("transaction should build");
23032        transaction
23033            .advance(
23034                transaction.fence(),
23035                TransitionTransactionState::Uploaded,
23036                Some(TransitionRemoteVersion::known_from_put_response(
23037                    committed.transitioned_object.version_id.clone(),
23038                )),
23039            )
23040            .expect("transaction should enter uploaded state");
23041        transaction
23042            .advance(transaction.fence(), TransitionTransactionState::LocalCommitStarted, None)
23043            .expect("transaction should enter local commit state");
23044        save_transition_transaction_record(store.clone(), &transaction)
23045            .await
23046            .expect("transaction record should persist");
23047        assert_eq!(transition_transaction_record_count(store.clone()).await, 1);
23048
23049        let stats = recover_transition_transaction_records(store.clone(), 100, None)
23050            .await
23051            .expect("transition transaction recovery should run");
23052
23053        assert_eq!((stats.scanned, stats.recovered, stats.retained, stats.failed), (1, 1, 0, 0));
23054        assert_eq!(transition_transaction_record_count(store.clone()).await, 0);
23055        assert_eq!(
23056            backend.object_count().await,
23057            1,
23058            "confirmed local commit recovery must not delete the committed remote body"
23059        );
23060        assert_eq!(backend.remove_count().await, 0);
23061    }
23062
23063    #[cfg(feature = "test-util")]
23064    #[tokio::test]
23065    #[serial_test::serial(storage_class_env)]
23066    async fn transition_transaction_recovery_retains_unproven_remote_candidates() {
23067        let temp_dir = tempfile::tempdir().expect("create temp store dir");
23068        let (ctx, store, _shutdown) =
23069            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "transition-transaction-unproven", &[4])).await;
23070        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
23071
23072        let tier_name = "TXUNPROVEN";
23073        let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
23074        let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
23075            .await
23076            .expect("tier lease should resolve")
23077            .backend_identity();
23078        let remote_version = uuid::Uuid::new_v4().to_string();
23079        let new_transaction = || {
23080            TransitionTransaction::new(TransitionTransactionInit {
23081                deployment_id: ctx.deployment_id().expect("test store should initialize deployment id"),
23082                transaction_id: uuid::Uuid::new_v4(),
23083                owner_epoch: uuid::Uuid::new_v4(),
23084                write_id: uuid::Uuid::new_v4(),
23085                source: TransitionSourceIdentity {
23086                    bucket: "absent-source-bucket".to_string(),
23087                    object: "source-object".to_string(),
23088                    version_id: None,
23089                    data_dir: uuid::Uuid::new_v4(),
23090                    mod_time_unix_nanos: 1_770_000_000_000_000_000,
23091                    size: 42,
23092                    etag: "source-etag".to_string(),
23093                    version_mode: TransitionSourceVersionMode::Unversioned,
23094                },
23095                tier_name: tier_name.to_string(),
23096                backend_fingerprint: backend_identity,
23097                not_after_unix_nanos: 1_780_000_000_000_000_000,
23098            })
23099            .expect("transaction should build")
23100        };
23101
23102        let upload_started = new_transaction();
23103        let mut local_commit_started = new_transaction();
23104        local_commit_started
23105            .advance(
23106                local_commit_started.fence(),
23107                TransitionTransactionState::Uploaded,
23108                Some(TransitionRemoteVersion::versioned(remote_version.clone())),
23109            )
23110            .expect("transaction should enter uploaded state");
23111        local_commit_started
23112            .advance(local_commit_started.fence(), TransitionTransactionState::LocalCommitStarted, None)
23113            .expect("transaction should enter local commit state");
23114        let upload_started_control_id =
23115            transition_recovery_control_id(&upload_started).expect("upload-started control id should derive");
23116        let local_commit_control_id =
23117            transition_recovery_control_id(&local_commit_started).expect("local-commit control id should derive");
23118
23119        backend.set_put_remote_version(Some(remote_version)).await;
23120        for transaction in [&upload_started, &local_commit_started] {
23121            let candidate = bytes::Bytes::from_static(b"unproven transition remote candidate");
23122            backend
23123                .put(
23124                    &transaction.remote_object,
23125                    ReaderImpl::Body(candidate.clone()),
23126                    i64::try_from(candidate.len()).expect("test candidate length should fit i64"),
23127                )
23128                .await
23129                .expect("mock backend should accept candidate");
23130            save_transition_transaction_record(store.clone(), transaction)
23131                .await
23132                .expect("transaction record should persist");
23133        }
23134
23135        let stats = recover_transition_transaction_records(store.clone(), 100, None)
23136            .await
23137            .expect("transition transaction recovery should run");
23138
23139        assert_eq!((stats.scanned, stats.recovered, stats.retained, stats.failed), (2, 0, 2, 0));
23140        assert_eq!(
23141            transition_transaction_record_count(store.clone()).await,
23142            2,
23143            "an upload without completion proof or unproven local commit must remain for authoritative reconcile"
23144        );
23145        assert_eq!(backend.object_count().await, 2, "recovery must not delete an unproven remote candidate");
23146        assert_eq!(backend.remove_count().await, 0);
23147        assert_eq!(backend.exact_remove_count(), 0);
23148        let upload_started_control =
23149            load_recovery_control(store.clone(), IlmRecoveryProtocol::TransitionTransaction, &upload_started_control_id)
23150                .await
23151                .expect("upload-started control should persist");
23152        assert_eq!(
23153            upload_started_control.control.classification,
23154            IlmRecoveryClassification::RetainedAmbiguous
23155        );
23156        let local_commit_control =
23157            load_recovery_control(store.clone(), IlmRecoveryProtocol::TransitionTransaction, &local_commit_control_id)
23158                .await
23159                .expect("local-commit control should persist");
23160        assert_eq!(local_commit_control.control.classification, IlmRecoveryClassification::OperatorRequired);
23161
23162        let upload_status = inspect_transition_recovery_retry_for_operator(store.clone(), &upload_started_control_id)
23163            .await
23164            .expect("retained upload should be inspectable for a bounded retry");
23165        let local_status = inspect_transition_recovery_retry_for_operator(store.clone(), &local_commit_control_id)
23166            .await
23167            .expect("operator-required local commit should be inspectable for a bounded retry");
23168        assert!(upload_status.retry_ready);
23169        assert!(local_status.retry_ready);
23170        assert!(matches!(
23171            retry_transition_recovery_for_operator(
23172                store.clone(),
23173                &upload_started_control_id,
23174                upload_status.control_revision + 1,
23175                &upload_status.source_generation_sha256,
23176            )
23177            .await,
23178            Err(TransitionOperatorError::StaleRecoveryControl)
23179        ));
23180
23181        let put_count_before_retry = backend.put_count().await;
23182        let get_count_before_retry = backend.get_count().await;
23183        let remove_count_before_retry = backend.remove_count().await;
23184        let upload_retry = retry_transition_recovery_for_operator(
23185            store.clone(),
23186            &upload_started_control_id,
23187            upload_status.control_revision,
23188            &upload_status.source_generation_sha256,
23189        )
23190        .await
23191        .expect("exact retained upload generation should be rearmed");
23192        let local_retry = retry_transition_recovery_for_operator(
23193            store.clone(),
23194            &local_commit_control_id,
23195            local_status.control_revision,
23196            &local_status.source_generation_sha256,
23197        )
23198        .await
23199        .expect("exact operator-required local commit generation should be rearmed");
23200        assert_eq!(upload_retry.classification, IlmRecoveryClassification::Retrying);
23201        assert_eq!(local_retry.classification, IlmRecoveryClassification::Retrying);
23202        assert_eq!(upload_retry.attempt_count, upload_status.attempt_count);
23203        assert_eq!(local_retry.attempt_count, local_status.attempt_count);
23204        assert_eq!(backend.put_count().await, put_count_before_retry);
23205        assert_eq!(backend.get_count().await, get_count_before_retry);
23206        assert_eq!(backend.remove_count().await, remove_count_before_retry);
23207        assert_eq!(backend.exact_remove_count(), 0, "operator retry must not directly issue remote DELETE");
23208
23209        let retried = recover_transition_transaction_records(store.clone(), 100, None)
23210            .await
23211            .expect("rearmed records should be re-evaluated through normal recovery");
23212        assert_eq!((retried.scanned, retried.recovered, retried.retained, retried.failed), (2, 0, 2, 0));
23213        let upload_retained =
23214            load_recovery_control(store.clone(), IlmRecoveryProtocol::TransitionTransaction, &upload_started_control_id)
23215                .await
23216                .expect("upload retry result should persist");
23217        let local_retained = load_recovery_control(store, IlmRecoveryProtocol::TransitionTransaction, &local_commit_control_id)
23218            .await
23219            .expect("local commit retry result should persist");
23220        assert_eq!(upload_retained.control.classification, IlmRecoveryClassification::RetainedAmbiguous);
23221        assert_eq!(local_retained.control.classification, IlmRecoveryClassification::OperatorRequired);
23222        assert_eq!(upload_retained.control.attempt_count, upload_status.attempt_count + 1);
23223        assert_eq!(local_retained.control.attempt_count, local_status.attempt_count + 1);
23224        assert_eq!(backend.put_count().await, put_count_before_retry);
23225        assert_eq!(backend.get_count().await, get_count_before_retry);
23226        assert_eq!(backend.remove_count().await, remove_count_before_retry);
23227        assert_eq!(backend.exact_remove_count(), 0);
23228    }
23229
23230    #[cfg(feature = "test-util")]
23231    #[tokio::test]
23232    #[serial_test::serial(storage_class_env)]
23233    async fn transition_transaction_recovery_handles_provider_recovered_unknown_upload() {
23234        let versioned_remote = uuid::Uuid::new_v4().to_string();
23235        let nil_remote = uuid::Uuid::nil().to_string();
23236        for (case, tier_name, remote_version, should_recover) in [
23237            ("missing", "TXPROBEMISSING", None, true),
23238            ("unversioned", "TXPROBEUNVERSIONED", Some(String::new()), true),
23239            ("versioned", "TXPROBEVERSIONED", Some(versioned_remote), true),
23240            ("nil-version", "TXPROBENILVERSION", Some(nil_remote), false),
23241        ] {
23242            let temp_dir = tempfile::tempdir().expect("create temp store dir");
23243            let (ctx, store, _shutdown) = without_storage_class_env(build_isolated_test_store(
23244                temp_dir.path(),
23245                &format!("transition-transaction-probe-{case}"),
23246                &[4],
23247            ))
23248            .await;
23249            crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
23250
23251            let backend = register_transition_reconcile_test_tier(&ctx.tier_config_mgr(), tier_name).await;
23252            let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
23253                .await
23254                .expect("tier lease should resolve")
23255                .backend_identity();
23256            let mut transaction = TransitionTransaction::new(TransitionTransactionInit {
23257                deployment_id: ctx.deployment_id().expect("test store should initialize deployment id"),
23258                transaction_id: uuid::Uuid::new_v4(),
23259                owner_epoch: uuid::Uuid::new_v4(),
23260                write_id: uuid::Uuid::new_v4(),
23261                source: TransitionSourceIdentity {
23262                    bucket: "source-bucket".to_string(),
23263                    object: "source-object".to_string(),
23264                    version_id: None,
23265                    data_dir: uuid::Uuid::new_v4(),
23266                    mod_time_unix_nanos: 1_770_000_000_000_000_000,
23267                    size: 42,
23268                    etag: "source-etag".to_string(),
23269                    version_mode: TransitionSourceVersionMode::Unversioned,
23270                },
23271                tier_name: tier_name.to_string(),
23272                backend_fingerprint: backend_identity,
23273                not_after_unix_nanos: 1,
23274            })
23275            .expect("transaction should build");
23276            transaction
23277                .advance(transaction.fence(), TransitionTransactionState::UploadOutcomeUnknown, None)
23278                .expect("transaction should enter unknown upload outcome state");
23279
23280            if let Some(version) = &remote_version {
23281                backend.set_put_remote_version(Some(version.clone())).await;
23282                let candidate = bytes::Bytes::from_static(b"provider-recovered transition remote candidate");
23283                backend
23284                    .put(
23285                        &transaction.remote_object,
23286                        ReaderImpl::Body(candidate.clone()),
23287                        i64::try_from(candidate.len()).expect("test candidate length should fit i64"),
23288                    )
23289                    .await
23290                    .expect("mock backend should accept candidate");
23291            }
23292            save_transition_transaction_record(store.clone(), &transaction)
23293                .await
23294                .expect("transaction record should persist");
23295
23296            let stats = recover_transition_transaction_records(store.clone(), 100, None)
23297                .await
23298                .expect("transition transaction recovery should run");
23299
23300            assert_eq!(
23301                (stats.scanned, stats.recovered, stats.retained, stats.failed),
23302                if should_recover { (1, 1, 0, 0) } else { (1, 0, 1, 0) }
23303            );
23304            assert_eq!(transition_transaction_record_count(store.clone()).await, usize::from(!should_recover));
23305            assert_eq!(
23306                backend.object_count().await,
23307                usize::from(!should_recover),
23308                "case {case}: only a valid provider version state may be recovered destructively"
23309            );
23310            let removed = remote_version
23311                .filter(|_| should_recover)
23312                .map(|version| vec![(transaction.remote_object.clone(), version)])
23313                .unwrap_or_default();
23314            assert_eq!(
23315                backend.remove_versions().await,
23316                removed,
23317                "case {case}: recovery must delete only provider-recovered candidates"
23318            );
23319            assert_eq!(
23320                backend.exact_remove_count(),
23321                usize::from(removed.first().is_some_and(|(_, version)| !version.is_empty()))
23322            );
23323        }
23324    }
23325
23326    #[cfg(feature = "test-util")]
23327    #[tokio::test]
23328    #[serial_test::serial(storage_class_env)]
23329    async fn operator_reconcile_deletes_exact_candidate_before_finalizing_record() {
23330        let temp_dir = tempfile::tempdir().expect("create temp store dir");
23331        let (ctx, store, _shutdown) =
23332            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "transition-operator-reconcile", &[4])).await;
23333        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
23334
23335        let tier_name = "TXOPERATOR";
23336        let backend = register_transition_reconcile_test_tier(&ctx.tier_config_mgr(), tier_name).await;
23337        let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
23338            .await
23339            .expect("tier lease should resolve")
23340            .backend_identity();
23341        let mut transaction = TransitionTransaction::new(TransitionTransactionInit {
23342            deployment_id: ctx.deployment_id().expect("test store should initialize deployment id"),
23343            transaction_id: uuid::Uuid::new_v4(),
23344            owner_epoch: uuid::Uuid::new_v4(),
23345            write_id: uuid::Uuid::new_v4(),
23346            source: TransitionSourceIdentity {
23347                bucket: "source-bucket".to_string(),
23348                object: "source-object".to_string(),
23349                version_id: None,
23350                data_dir: uuid::Uuid::new_v4(),
23351                mod_time_unix_nanos: 1_770_000_000_000_000_000,
23352                size: 42,
23353                etag: "source-etag".to_string(),
23354                version_mode: TransitionSourceVersionMode::Unversioned,
23355            },
23356            tier_name: tier_name.to_string(),
23357            backend_fingerprint: backend_identity,
23358            not_after_unix_nanos: 1,
23359        })
23360        .expect("transaction should build");
23361        transaction
23362            .advance(transaction.fence(), TransitionTransactionState::UploadOutcomeUnknown, None)
23363            .expect("transaction should enter unknown upload outcome state");
23364
23365        let remote_version = uuid::Uuid::new_v4().to_string();
23366        backend.set_put_remote_version(Some(remote_version.clone())).await;
23367        let candidate = bytes::Bytes::from_static(b"operator-confirmed transition candidate");
23368        backend
23369            .put(
23370                &transaction.remote_object,
23371                ReaderImpl::Body(candidate.clone()),
23372                i64::try_from(candidate.len()).expect("test candidate length should fit i64"),
23373            )
23374            .await
23375            .expect("mock backend should accept candidate");
23376        save_transition_transaction_record(store.clone(), &transaction)
23377            .await
23378            .expect("transaction record should persist");
23379
23380        let status = inspect_transition_transaction_for_operator(store.clone(), transaction.transaction_id)
23381            .await
23382            .expect("operator inspection should probe the candidate");
23383        assert_eq!(status.probe, TransitionOperatorProbe::VersionedPresent(remote_version.clone()));
23384
23385        let wrong_version = uuid::Uuid::new_v4().to_string();
23386        let err = delete_transition_candidate_for_operator(store.clone(), transaction.transaction_id, &wrong_version)
23387            .await
23388            .expect_err("a mismatched exact version must fail before deleting a candidate");
23389        assert!(matches!(
23390            err,
23391            TransitionOperatorError::CandidateVersionMismatch {
23392                expected,
23393                actual: TransitionOperatorProbe::VersionedPresent(ref observed),
23394            } if expected == wrong_version && observed == &remote_version
23395        ));
23396        assert!(backend.contains(&transaction.remote_object).await);
23397        assert_eq!(backend.exact_remove_count(), 0);
23398        load_transition_transaction_record(store.clone(), transaction.transaction_id)
23399            .await
23400            .expect("an incorrect exact version must retain the transaction journal");
23401
23402        let result = delete_transition_candidate_for_operator(store.clone(), transaction.transaction_id, &remote_version)
23403            .await
23404            .expect("operator-confirmed exact candidate should be deleted");
23405        assert_eq!(result.status.probe, TransitionOperatorProbe::Missing);
23406        assert!(result.journal_observed_after_delete);
23407        assert_eq!(backend.exact_remove_count(), 1);
23408        assert_eq!(backend.remove_versions().await, vec![(transaction.remote_object.clone(), remote_version)]);
23409        load_transition_transaction_record(store.clone(), transaction.transaction_id)
23410            .await
23411            .expect("candidate deletion must retain the transaction journal");
23412
23413        finalize_missing_transition_transaction_for_operator(store.clone(), transaction.transaction_id)
23414            .await
23415            .expect("a separately confirmed missing candidate should permit finalization");
23416        assert!(matches!(
23417            load_transition_transaction_record(store, transaction.transaction_id).await,
23418            Err(Error::ConfigNotFound)
23419        ));
23420    }
23421
23422    #[cfg(feature = "test-util")]
23423    #[tokio::test]
23424    #[serial_test::serial(storage_class_env)]
23425    async fn operator_finalize_retains_record_without_missing_proof() {
23426        let temp_dir = tempfile::tempdir().expect("create temp store dir");
23427        let (ctx, store, _shutdown) =
23428            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "transition-operator-fail-closed", &[4])).await;
23429        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
23430
23431        let tier_name = "TXOPERATORFAIL";
23432        let backend = register_transition_reconcile_test_tier(&ctx.tier_config_mgr(), tier_name).await;
23433        let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
23434            .await
23435            .expect("tier lease should resolve")
23436            .backend_identity();
23437        let mut transaction = TransitionTransaction::new(TransitionTransactionInit {
23438            deployment_id: ctx.deployment_id().expect("test store should initialize deployment id"),
23439            transaction_id: uuid::Uuid::new_v4(),
23440            owner_epoch: uuid::Uuid::new_v4(),
23441            write_id: uuid::Uuid::new_v4(),
23442            source: TransitionSourceIdentity {
23443                bucket: "source-bucket".to_string(),
23444                object: "source-object".to_string(),
23445                version_id: None,
23446                data_dir: uuid::Uuid::new_v4(),
23447                mod_time_unix_nanos: 1_770_000_000_000_000_000,
23448                size: 42,
23449                etag: "source-etag".to_string(),
23450                version_mode: TransitionSourceVersionMode::Unversioned,
23451            },
23452            tier_name: tier_name.to_string(),
23453            backend_fingerprint: backend_identity,
23454            not_after_unix_nanos: 1,
23455        })
23456        .expect("transaction should build");
23457        transaction
23458            .advance(transaction.fence(), TransitionTransactionState::UploadOutcomeUnknown, None)
23459            .expect("transaction should enter unknown upload outcome state");
23460        save_transition_transaction_record(store.clone(), &transaction)
23461            .await
23462            .expect("transaction record should persist");
23463        backend
23464            .set_transition_candidate_probe_override(Some(TransitionCandidateProbe::Unsupported))
23465            .await;
23466
23467        assert!(matches!(
23468            finalize_missing_transition_transaction_for_operator(store.clone(), transaction.transaction_id).await,
23469            Err(TransitionOperatorError::CandidateNotMissing(TransitionOperatorProbe::Unsupported))
23470        ));
23471        load_transition_transaction_record(store, transaction.transaction_id)
23472            .await
23473            .expect("an unsupported probe must retain the transaction journal");
23474        assert_eq!(backend.remove_count().await, 0);
23475    }
23476
23477    #[cfg(feature = "test-util")]
23478    #[tokio::test]
23479    #[serial_test::serial(storage_class_env)]
23480    async fn transition_response_loss_persists_unknown_outcome_for_provider_recovery() {
23481        let temp_dir = tempfile::tempdir().expect("create temp store dir");
23482        let (ctx, store, _shutdown) =
23483            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "transition-response-loss", &[4])).await;
23484        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
23485
23486        let tier_name = "TXRESPONSELOSS";
23487        let backend = register_transition_reconcile_test_tier(&ctx.tier_config_mgr(), tier_name).await;
23488        let _compaction_proof = install_transition_transaction_compaction_fleet_proof_for_test("object-transaction-fencing-test");
23489        let bucket = "transition-response-loss-bucket";
23490        let object = "source.bin";
23491        store
23492            .make_bucket(bucket, &MakeBucketOptions::default())
23493            .await
23494            .expect("source bucket should be created");
23495        let payload = b"a response-lost tier PUT must remain recoverable".repeat(1024);
23496        let mut reader = PutObjReader::from_vec(payload.clone());
23497        let source = store
23498            .put_object(bucket, object, &mut reader, &ObjectOptions::default())
23499            .await
23500            .expect("source object should be written");
23501        backend.lose_next_put_response();
23502
23503        let error = store
23504            .transition_object(
23505                bucket,
23506                object,
23507                &ObjectOptions {
23508                    no_lock: true,
23509                    transition: TransitionOptions {
23510                        status: TRANSITION_PENDING.to_string(),
23511                        tier: tier_name.to_string(),
23512                        etag: source.etag.clone().expect("source object should have an ETag"),
23513                        ..Default::default()
23514                    },
23515                    version_id: source.version_id.map(|version| version.to_string()),
23516                    mod_time: source.mod_time,
23517                    ..Default::default()
23518                },
23519            )
23520            .await
23521            .expect_err("a lost tier PUT response must fail the transition request");
23522        assert!(
23523            matches!(error, StorageError::Io(ref err) if err.kind() == std::io::ErrorKind::ConnectionReset),
23524            "the response-loss error must remain visible to the caller: {error:?}"
23525        );
23526
23527        let records = store
23528            .clone()
23529            .list_objects_v2(
23530                RUSTFS_META_BUCKET,
23531                TRANSITION_TRANSACTION_RECORD_PREFIX,
23532                None,
23533                None,
23534                10,
23535                false,
23536                None,
23537                false,
23538            )
23539            .await
23540            .expect("transition transaction records should be listable");
23541        assert_eq!(records.objects.len(), 1, "response loss must leave one durable transaction record");
23542        let transaction_id = records.objects[0]
23543            .name
23544            .rsplit('/')
23545            .next()
23546            .and_then(|name| name.strip_suffix(".json"))
23547            .and_then(|name| uuid::Uuid::parse_str(name).ok())
23548            .expect("transaction record name should contain a UUID");
23549        let transaction = load_transition_transaction_record(store.clone(), transaction_id)
23550            .await
23551            .expect("response loss transaction record should load");
23552        assert_eq!(
23553            transaction.state,
23554            TransitionTransactionState::UploadOutcomeUnknown,
23555            "a response-lost PUT must not remain in UploadStarted"
23556        );
23557        assert_eq!(transaction.revision, 1, "compact response loss must retain the pre-PUT fence generation");
23558        assert!(
23559            backend.contains(&transaction.remote_object).await,
23560            "the test backend must retain the remote candidate"
23561        );
23562
23563        backend
23564            .set_transition_candidate_probe_override(Some(TransitionCandidateProbe::Unsupported))
23565            .await;
23566        let unsupported_stats = recover_transition_transaction_records(store.clone(), 100, None)
23567            .await
23568            .expect("active unknown ownership should remain fenced before provider recovery");
23569        assert_eq!(
23570            (
23571                unsupported_stats.scanned,
23572                unsupported_stats.recovered,
23573                unsupported_stats.retained,
23574                unsupported_stats.failed
23575            ),
23576            (1, 0, 1, 0),
23577            "active unknown ownership must retain the upload before the recovery deadline"
23578        );
23579        assert_eq!(transition_transaction_record_count(store.clone()).await, 1);
23580        let recovery_control_id =
23581            transition_recovery_control_id(&transaction).expect("transition recovery control id should derive");
23582        assert!(matches!(
23583            load_recovery_control(store.clone(), IlmRecoveryProtocol::TransitionTransaction, &recovery_control_id).await,
23584            Err(Error::ConfigNotFound)
23585        ));
23586        assert!(
23587            backend.contains(&transaction.remote_object).await,
23588            "active ownership must not delete the candidate"
23589        );
23590        assert_eq!(backend.remove_count().await, 0, "active ownership must not attempt cleanup");
23591        assert!(
23592            !backend
23593                .op_log()
23594                .await
23595                .iter()
23596                .any(|operation| matches!(operation, MockWarmOp::Probe { .. })),
23597            "active ownership must not probe the provider"
23598        );
23599
23600        backend.set_transition_candidate_probe_override(None).await;
23601        let stats =
23602            recover_transition_transaction_records_at(store.clone(), 100, None, i128::from(transaction.not_after_unix_nanos) + 1)
23603                .await
23604                .expect("provider-authoritative recovery should run");
23605        assert_eq!((stats.scanned, stats.recovered, stats.retained, stats.failed), (1, 1, 0, 0));
23606        assert_eq!(transition_transaction_record_count(store.clone()).await, 0);
23607        assert_eq!(backend.object_count().await, 0, "recovery must delete the provider-confirmed candidate");
23608        let op_log = backend.op_log().await;
23609        assert!(
23610            op_log.iter().any(|operation| matches!(operation, MockWarmOp::Probe { .. })),
23611            "response-loss recovery must enter the provider probe branch"
23612        );
23613        assert!(
23614            op_log.iter().any(|operation| matches!(operation, MockWarmOp::Put { .. })),
23615            "response-loss fixture must record that the remote PUT reached the backend"
23616        );
23617        let source_after = store
23618            .get_object_info(bucket, object, &ObjectOptions::default())
23619            .await
23620            .expect("recovery must preserve the local source object");
23621        assert_eq!(source_after.size, i64::try_from(payload.len()).expect("payload length should fit i64"));
23622    }
23623}