Skip to main content

rustfs_ecstore/store/
init.rs

1// Copyright 2024 RustFS Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use super::*;
16use crate::core::pools::{
17    PoolMetaReplicaState, PoolMetaWriteState, local_decommission_queue_prefix, persist_pool_meta_identity_for_attested_pools,
18    persist_pool_meta_identity_for_startup, pool_meta_has_active_decommission,
19};
20use crate::runtime::instance::InstanceContext;
21use crate::runtime::sources as runtime_sources;
22use crate::storage_api_contracts::object::EcstoreObjectIO;
23use rustfs_config::server_config::KVS;
24use rustfs_credentials::{RPC_SECRET_REQUIRED_OPERATOR_MESSAGE, try_get_rpc_token};
25use std::future::Future;
26use tracing::{debug, error, info, warn};
27
28const LOG_COMPONENT_ECSTORE: &str = "ecstore";
29const LOG_SUBSYSTEM_STORE_INIT: &str = "store_init";
30const EVENT_DECOMMISSION_RESUME_RETRY: &str = "decommission_resume_retry";
31const EVENT_STORE_FORMAT_RETRY: &str = "store_format_retry";
32const EVENT_ECSTORE_INIT_STATUS: &str = "ecstore_init_status";
33const EVENT_STORE_RPC_SECRET_PREFLIGHT_FAILED: &str = "store_rpc_secret_preflight_failed";
34
35fn pool_first_endpoint_is_local(pool: &crate::layout::endpoints::PoolEndpoints) -> bool {
36    pool.endpoints.as_ref().first().is_some_and(|endpoint| endpoint.is_local)
37}
38
39fn startup_pool_drive_counts(endpoint_pools: &EndpointServerPools) -> Vec<usize> {
40    endpoint_pools.as_ref().iter().map(|pool| pool.drives_per_set).collect()
41}
42
43fn resolve_startup_pool_defaults(endpoint_pools: &EndpointServerPools) -> Result<Vec<usize>> {
44    resolve_startup_pool_defaults_with(endpoint_pools, ECStore::validate_startup_storage_class)
45}
46
47fn resolve_startup_pool_defaults_with(
48    endpoint_pools: &EndpointServerPools,
49    validate: impl FnOnce(&EndpointServerPools) -> Result<()>,
50) -> Result<Vec<usize>> {
51    validate(endpoint_pools)?;
52    let drive_counts = startup_pool_drive_counts(endpoint_pools);
53    drive_counts.into_iter().map(ec_drives_no_config).collect()
54}
55
56/// Fail fast when the topology spans remote nodes but no internode RPC secret
57/// resolves. Every remote format read would otherwise fail client-side with
58/// "No valid auth token" and startup would retry for minutes before dying with
59/// a misleading "erasure read quorum" error (issues #4939, #5153).
60fn preflight_startup_rpc_secret(endpoint_pools: &EndpointServerPools) -> Result<()> {
61    preflight_startup_rpc_secret_with(endpoint_pools, try_get_rpc_token)
62}
63
64fn preflight_startup_rpc_secret_with(
65    endpoint_pools: &EndpointServerPools,
66    resolve_rpc_token: impl FnOnce() -> std::io::Result<String>,
67) -> Result<()> {
68    let has_remote_endpoint = endpoint_pools
69        .as_ref()
70        .iter()
71        .flat_map(|pool| pool.endpoints.as_ref().iter())
72        .any(|endpoint| !endpoint.is_local);
73
74    if !has_remote_endpoint {
75        return Ok(());
76    }
77
78    match resolve_rpc_token() {
79        Ok(_) => Ok(()),
80        Err(err) => {
81            // The message prefix must stay aligned with the runtime log in
82            // cluster/rpc/http_auth.rs: the log-analyzer `rpc-secret-resolution`
83            // rule anchors on it, and this preflight aborts before that log
84            // would ever be emitted.
85            error!(
86                event = EVENT_STORE_RPC_SECRET_PREFLIGHT_FAILED,
87                component = LOG_COMPONENT_ECSTORE,
88                subsystem = LOG_SUBSYSTEM_STORE_INIT,
89                "RPC auth secret resolution failed: {err}; {RPC_SECRET_REQUIRED_OPERATOR_MESSAGE}"
90            );
91            Err(Error::other(format!(
92                "store init aborted: endpoints include remote nodes but {err}; {RPC_SECRET_REQUIRED_OPERATOR_MESSAGE}"
93            )))
94        }
95    }
96}
97
98const LOCAL_DECOMMISSION_RESUME_RETRY_DELAY: Duration = Duration::from_secs(30);
99const LOCAL_DECOMMISSION_WATCHDOG_INTERVAL: Duration = Duration::from_secs(30);
100const LOCAL_DECOMMISSION_WATCHDOG_MAX_RETRY_DELAY: Duration = Duration::from_secs(60 * 5);
101const REBALANCE_INITIAL_RESUME_DELAY: Duration = Duration::from_secs(10);
102const REBALANCE_RESUME_RETRY_DELAY: Duration = Duration::from_secs(10);
103
104fn should_retry_format_load(err: &Error) -> bool {
105    !matches!(
106        err,
107        Error::CorruptedFormat | Error::UnsupportedSnsdExpansion { .. } | Error::PoolTopologyMismatch { .. }
108    )
109}
110
111fn should_auto_start_rebalance_after_init(decommission_running: bool, rebalance_resume_required: bool) -> bool {
112    rebalance_resume_required && !decommission_running
113}
114
115fn should_defer_rebalance_auto_start(distributed: bool, fleet_proof_available: bool) -> bool {
116    distributed && !fleet_proof_available
117}
118
119async fn wait_for_local_decommission_resume_delay(rx: &CancellationToken, delay: Duration) -> bool {
120    tokio::select! {
121        _ = rx.cancelled() => false,
122        _ = tokio::time::sleep(delay) => true,
123    }
124}
125
126fn local_decommission_watchdog_retry_delay(consecutive_failures: u32) -> Duration {
127    let exponent = consecutive_failures.saturating_sub(1).min(4);
128    LOCAL_DECOMMISSION_RESUME_RETRY_DELAY
129        .saturating_mul(1_u32 << exponent)
130        .min(LOCAL_DECOMMISSION_WATCHDOG_MAX_RETRY_DELAY)
131}
132
133async fn wait_for_rebalance_resume_delay(rx: &CancellationToken, delay: Duration) -> bool {
134    tokio::select! {
135        _ = rx.cancelled() => false,
136        _ = tokio::time::sleep(delay) => true,
137    }
138}
139
140async fn wait_for_rebalance_resume_retry(rx: &CancellationToken) -> bool {
141    wait_for_rebalance_resume_delay(rx, REBALANCE_RESUME_RETRY_DELAY).await
142}
143
144fn resolve_store_init_stage_result(result: Result<()>, stage: &str) -> Result<()> {
145    result.map_err(|err| Error::other(format!("store init failed during {stage}: {err}")))
146}
147
148async fn load_pool_meta_for_startup<S>(
149    pools: Vec<Arc<S>>,
150    write_state: &mut PoolMetaWriteState,
151) -> Result<(PoolMeta, PoolMetaReplicaState)>
152where
153    S: EcstoreObjectIO,
154{
155    let mut meta = PoolMeta::default();
156    let replica_state = meta
157        .load_for_startup_observing(pools, write_state)
158        .await
159        .map_err(|err| Error::other_with_context("store init failed during load_pool_meta", err))?;
160    write_state.observe_replicas(replica_state);
161    write_state
162        .ensure_missing_metadata_can_initialize()
163        .map_err(|err| Error::other(format!("store init failed during classify_pool_meta_absence: {err}")))?;
164    Ok((meta, replica_state))
165}
166
167async fn establish_pool_meta_bootstrap_identity_if_proven<S>(
168    pools: Vec<Arc<S>>,
169    write_state: &mut PoolMetaWriteState,
170    elected_writer: bool,
171) -> Result<()>
172where
173    S: EcstoreObjectIO,
174{
175    if elected_writer && write_state.bootstrap_identity_proven() {
176        return persist_pool_meta_identity_for_startup(pools, write_state, false).await;
177    }
178    if write_state.bootstrap_identity_proven() {
179        return Ok(());
180    }
181    // Multi-pool bootstrap whose pools were formatted by different nodes: no
182    // single process can prove the whole deployment fresh in memory, so each
183    // creator attests the pools it formatted first-hand with the shared nonce
184    // and the elected writer waits for a complete, agreeing pending set.
185    let attested = write_state.attested_pool_indices();
186    if attested.is_empty() {
187        return Ok(());
188    }
189    persist_pool_meta_identity_for_attested_pools(pools, write_state, &attested).await
190}
191
192async fn save_validated_pool_meta_for_startup<S>(
193    meta: &PoolMeta,
194    pools: Vec<Arc<S>>,
195    write_state: &mut PoolMetaWriteState,
196) -> Result<PoolMeta>
197where
198    S: EcstoreObjectIO,
199{
200    meta.save_for_startup_observing(pools, write_state)
201        .await
202        .map_err(|err| Error::other(format!("store init failed during save_validated_pool_meta: {err}")))
203}
204
205async fn persist_pool_meta_for_startup_if_safe<S>(
206    meta: &PoolMeta,
207    pools: Vec<Arc<S>>,
208    replica_state: PoolMetaReplicaState,
209    write_state: &mut PoolMetaWriteState,
210    topology_update: bool,
211    elected_writer: bool,
212) -> Result<PoolMeta>
213where
214    S: EcstoreObjectIO,
215{
216    if !elected_writer {
217        return Ok(meta.clone());
218    }
219    let should_write = topology_update || (replica_state.needs_repair && replica_state.repair_write_safe);
220    if topology_update {
221        replica_state.ensure_write_safe("store init failed during save_validated_pool_meta")?;
222    }
223    if should_write || write_state.identity_requires_repair() {
224        write_state.ensure_write_safe("store init failed during save_validated_pool_meta")?;
225    }
226    let mut committed = meta.clone();
227    if should_write {
228        if write_state.bootstrap_identity_proven() || write_state.identity_is_pending() {
229            persist_pool_meta_identity_for_startup(pools.clone(), write_state, false)
230                .await
231                .map_err(|err| Error::other(format!("store init failed during prepare_pool_meta_identity: {err}")))?;
232        }
233        committed = save_validated_pool_meta_for_startup(meta, pools.clone(), write_state).await?;
234    }
235    if should_write || write_state.identity_requires_repair() {
236        persist_pool_meta_identity_for_startup(pools, write_state, true)
237            .await
238            .map_err(|err| Error::other(format!("store init failed during commit_pool_meta_identity: {err}")))?;
239    }
240    Ok(committed)
241}
242
243async fn run_local_decommission_watchdog<F, Fut>(rx: CancellationToken, mut reconcile: F)
244where
245    F: FnMut() -> Fut,
246    Fut: Future<Output = Result<()>>,
247{
248    let mut consecutive_failures = 0_u32;
249
250    loop {
251        if rx.is_cancelled() {
252            return;
253        }
254
255        let delay = match reconcile().await {
256            Ok(()) => {
257                consecutive_failures = 0;
258                LOCAL_DECOMMISSION_WATCHDOG_INTERVAL
259            }
260            Err(err) => {
261                consecutive_failures = consecutive_failures.saturating_add(1);
262                let retry_delay = local_decommission_watchdog_retry_delay(consecutive_failures);
263                warn!(
264                    event = EVENT_DECOMMISSION_RESUME_RETRY,
265                    component = LOG_COMPONENT_ECSTORE,
266                    subsystem = LOG_SUBSYSTEM_STORE_INIT,
267                    consecutive_failures,
268                    retry_delay_secs = retry_delay.as_secs(),
269                    error = %err,
270                    "Retrying decommission worker recovery"
271                );
272                retry_delay
273            }
274        };
275
276        if !wait_for_local_decommission_resume_delay(&rx, delay).await {
277            return;
278        }
279    }
280}
281
282async fn reconcile_local_decommission_after_init(store: &Arc<ECStore>, rx: CancellationToken) -> Result<()> {
283    store
284        .ensure_pool_meta_side_effects_safe("decommission worker recovery blocked while pool metadata requires recovery")
285        .await?;
286    if store.has_active_local_decommission_worker().await {
287        return Ok(());
288    }
289    store.refresh_pool_status_meta().await?;
290    let resume_required = pool_meta_has_active_decommission(&*store.pool_meta.read().await);
291    if resume_required {
292        crate::core::pools::acquire_pool_activation_fleet_proof(&store.ctx).await?;
293    }
294    store.spawn_missing_local_decommission_routines_with_token(rx).await
295}
296
297async fn supervise_local_decommission_after_init(store: Arc<ECStore>, rx: CancellationToken) {
298    run_local_decommission_watchdog(rx.clone(), || {
299        let store = store.clone();
300        let worker_rx = rx.clone();
301        async move { reconcile_local_decommission_after_init(&store, worker_rx).await }
302    })
303    .await;
304}
305
306async fn resume_rebalance_after_init(store: Arc<ECStore>, rx: CancellationToken) {
307    if !wait_for_rebalance_resume_delay(&rx, REBALANCE_INITIAL_RESUME_DELAY).await {
308        return;
309    }
310
311    loop {
312        if rx.is_cancelled() {
313            return;
314        }
315
316        let resume_required = store
317            .rebalance_meta
318            .read()
319            .await
320            .as_ref()
321            .is_some_and(crate::services::rebalance::rebalance_requires_worker_activation);
322        if !resume_required {
323            return;
324        }
325
326        if should_defer_rebalance_auto_start(
327            store.ctx.is_dist_erasure().await,
328            crate::services::notification_sys::acquire_cross_pool_fence_fleet_proof().is_some(),
329        ) {
330            if !wait_for_rebalance_resume_retry(&rx).await {
331                return;
332            }
333            continue;
334        }
335
336        match store.start_rebalance().await {
337            Ok(()) => return,
338            Err(err) if crate::core::pools::is_pool_activation_fleet_proof_error(&err) => {
339                warn!(
340                    event = EVENT_ECSTORE_INIT_STATUS,
341                    component = LOG_COMPONENT_ECSTORE,
342                    subsystem = LOG_SUBSYSTEM_STORE_INIT,
343                    stage = "start_rebalance",
344                    state = "retrying",
345                    reason = "fleet_capability_proof_unavailable",
346                    error = %err,
347                    retry_delay_secs = REBALANCE_RESUME_RETRY_DELAY.as_secs(),
348                    "Retrying deferred rebalance auto-start"
349                );
350                if !wait_for_rebalance_resume_retry(&rx).await {
351                    return;
352                }
353            }
354            Err(err) => {
355                error!(
356                    event = EVENT_ECSTORE_INIT_STATUS,
357                    component = LOG_COMPONENT_ECSTORE,
358                    subsystem = LOG_SUBSYSTEM_STORE_INIT,
359                    stage = "start_rebalance",
360                    state = "failed",
361                    reason = "deferred_resume_failed",
362                    error = %err,
363                    "Failed to resume rebalance after store initialization"
364                );
365                return;
366            }
367        }
368    }
369}
370
371impl ECStore {
372    /// Shutdown token owned by this store instance.
373    pub fn background_cancel_token(&self) -> Option<CancellationToken> {
374        self.ctx.background_cancel_token()
375    }
376
377    /// Validate topology and process storage-class overrides before any disk is opened.
378    pub fn validate_startup_storage_class(endpoint_pools: &EndpointServerPools) -> Result<()> {
379        let drive_counts = startup_pool_drive_counts(endpoint_pools);
380        storageclass::lookup_config_for_pools(&KVS::new(), &drive_counts).map(|_| ())
381    }
382
383    #[allow(clippy::new_ret_no_self)]
384    #[instrument(level = "debug", skip(endpoint_pools))]
385    pub async fn new(address: SocketAddr, endpoint_pools: EndpointServerPools, ctx: CancellationToken) -> Result<Arc<Self>> {
386        Self::new_with_instance_ctx(address, endpoint_pools, ctx, crate::runtime::instance::bootstrap_ctx()).await
387    }
388
389    /// Build a store around an explicit instance context (Phase 5 follow-up,
390    /// backlog#1052). The legacy [`ECStore::new`] entry adopts the process
391    /// bootstrap context, keeping single-instance startup byte-for-byte
392    /// unchanged; a caller that owns its own context (a future second embedded
393    /// server) passes it here so every construction-time write — pool sets,
394    /// local-disk registry, deployment id — lands on that context instead of
395    /// the shared bootstrap one.
396    #[allow(clippy::new_ret_no_self)]
397    #[instrument(level = "debug", skip(endpoint_pools, instance_ctx))]
398    pub async fn new_with_instance_ctx(
399        address: SocketAddr,
400        endpoint_pools: EndpointServerPools,
401        ctx: CancellationToken,
402        instance_ctx: Arc<InstanceContext>,
403    ) -> Result<Arc<Self>> {
404        Box::pin(Self::new_with_instance_ctx_inner(address, endpoint_pools, ctx, instance_ctx)).await
405    }
406
407    async fn new_with_instance_ctx_inner(
408        address: SocketAddr,
409        endpoint_pools: EndpointServerPools,
410        ctx: CancellationToken,
411        instance_ctx: Arc<InstanceContext>,
412    ) -> Result<Arc<Self>> {
413        instance_ctx.bind_background_cancel_token(ctx.clone());
414
415        // let layouts = DisksLayout::from_volumes(endpoints.as_slice())?;
416
417        // Validate topology and environment overrides before opening any disk.
418        // The values stored on SetDisks remain pure per-pool topology defaults;
419        // payload writes use the runtime storage-class snapshot published later
420        // from config before the store is marked ready.
421        let default_pool_parities = resolve_startup_pool_defaults(&endpoint_pools)?;
422
423        preflight_startup_rpc_secret(&endpoint_pools)?;
424
425        let mut deployment_id = None;
426        let mut pool_meta_bootstrap_authorities = Vec::new();
427
428        // let (endpoint_pools, _) = EndpointServerPools::create_server_endpoints(address.as_str(), &layouts)?;
429
430        let mut pools = Vec::with_capacity(endpoint_pools.as_ref().len());
431        let mut disk_map = HashMap::with_capacity(endpoint_pools.as_ref().len());
432
433        let mut local_disks = Vec::new();
434
435        debug!(
436            event = EVENT_ECSTORE_INIT_STATUS,
437            component = LOG_COMPONENT_ECSTORE,
438            subsystem = LOG_SUBSYSTEM_STORE_INIT,
439            address = %address,
440            "Initializing ECStore address"
441        );
442        let mut host = address.ip().to_string();
443        if host.is_empty() {
444            host = runtime_sources::rustfs_host().await
445        }
446        let mut port = address.port().to_string();
447        if port.is_empty() {
448            port = runtime_sources::rustfs_port().to_string()
449        }
450        debug!(
451            event = EVENT_ECSTORE_INIT_STATUS,
452            component = LOG_COMPONENT_ECSTORE,
453            subsystem = LOG_SUBSYSTEM_STORE_INIT,
454            host = %host,
455            port = %port,
456            "Initializing ECStore host"
457        );
458        init_local_peer(&endpoint_pools, &host, &port).await;
459
460        // debug!("endpoint_pools: {:?}", endpoint_pools);
461
462        for (i, pool_eps) in endpoint_pools.as_ref().iter().enumerate() {
463            let pool_first_is_local = pool_first_endpoint_is_local(pool_eps);
464            let parity_drives = default_pool_parities
465                .get(i)
466                .copied()
467                .ok_or_else(|| Error::other(format!("store init failed to resolve default parity for pool {i}")))?;
468
469            // validate_parity(parity_count, pool_eps.drives_per_set)?;
470
471            // Build disks with health monitoring available, but do not start
472            // periodic monitoring until format loading succeeds. Startup RPC
473            // failures can still spawn recovery probes for peers that come up
474            // after this node.
475            let (mut disks, errs) = init_format::init_disks(
476                &pool_eps.endpoints,
477                &DiskOption {
478                    cleanup: true,
479                    health_check: true,
480                },
481            )
482            .await;
483
484            check_disk_fatal_errs(&errs)?;
485
486            let loaded_format = {
487                let mut times = 0;
488                let mut interval = 1;
489                loop {
490                    match init_format::connect_load_init_formats_with_instance_ctx(
491                        &instance_ctx,
492                        pool_first_is_local,
493                        &mut disks,
494                        pool_eps.set_count,
495                        pool_eps.drives_per_set,
496                        deployment_id,
497                    )
498                    .await
499                    {
500                        Ok(fm) => break Ok(fm),
501                        Err(e) if !should_retry_format_load(&e) => break Err(e),
502                        // Wrap the final error if we are giving up
503                        Err(e) if times >= 10 => {
504                            break Err(Error::other(format!("store init failed to load formats after {times} retries: {e}")));
505                        }
506                        // Retrying so just drop the error
507                        Err(_) => {}
508                    }
509                    times += 1;
510                    if interval < 16 {
511                        interval *= 2;
512                    }
513                    debug!(
514                        event = EVENT_STORE_FORMAT_RETRY,
515                        component = LOG_COMPONENT_ECSTORE,
516                        subsystem = LOG_SUBSYSTEM_STORE_INIT,
517                        retry_count = times,
518                        retry_delay_secs = interval,
519                        "Retrying storage format load"
520                    );
521                    select! {
522                        _ = tokio::signal::ctrl_c() => {
523                            info!(
524                                event = EVENT_STORE_FORMAT_RETRY,
525                                component = LOG_COMPONENT_ECSTORE,
526                                subsystem = LOG_SUBSYSTEM_STORE_INIT,
527                                reason = "ctrl_c",
528                                "Interrupted storage format retry loop"
529                            );
530                            exit(0);
531                        }
532                        _ = sleep(Duration::from_secs(interval)) => {
533                        }
534                    }
535                    // After waiting for peers, clear transient faulty marks so the next attempt can open RPCs again
536                    // (these `DiskStore` handles are reused; `is_faulty()` would otherwise short-circuit).
537                    for disk in disks.iter().flatten() {
538                        disk.reset_health_for_store_init_retry();
539                    }
540                }
541            }?;
542            // First-hand authority for this pool only: `Fresh` when this process
543            // formatted it, `LegacyAdoption` when it verified the migration, and
544            // `None` when it merely read a format another node created.
545            pool_meta_bootstrap_authorities.push(loaded_format.pool_meta_bootstrap_authority);
546            let fm = loaded_format.format;
547
548            // Format loading succeeded, enable health monitoring on all disks
549            for disk in disks.iter().flatten() {
550                disk.enable_health_check();
551            }
552
553            if deployment_id.is_none() {
554                deployment_id = Some(fm.id);
555            }
556
557            if deployment_id != Some(fm.id) {
558                return Err(Error::other("store init failed: deployment IDs do not match across pools"));
559            }
560
561            if deployment_id.is_some_and(|id| id.is_nil()) {
562                deployment_id = Some(Uuid::new_v4());
563            }
564
565            for disk in disks.iter() {
566                if disk.is_some() && disk.as_ref().expect("operation should succeed").is_local() {
567                    local_disks.push(disk.as_ref().expect("operation should succeed").clone());
568                }
569            }
570
571            let sets = Sets::new_with_instance_ctx(disks.clone(), pool_eps, &fm, i, parity_drives, instance_ctx.clone()).await?;
572            pools.push(sets);
573
574            disk_map.insert(i, disks);
575        }
576
577        // Replace the local disk
578        if !instance_ctx.is_dist_erasure().await {
579            runtime_sources::record_local_disks(&instance_ctx, local_disks).await;
580        }
581
582        let deployment_id = deployment_id.ok_or_else(|| Error::other("store init failed: deployment id is not initialized"))?;
583        let peer_sys = S3PeerSys::new_with_instance_ctx(&endpoint_pools, instance_ctx.clone());
584        let mut pool_meta = PoolMeta::new(&pools, &PoolMeta::default());
585        pool_meta.dont_save = true;
586        let elected_bootstrap_writer = pools
587            .first()
588            .is_some_and(|pool| pool_first_endpoint_is_local(&pool.endpoints));
589        let pool_meta_write_state = PoolMetaWriteState::for_startup_with_pool_bootstrap_authorities(
590            deployment_id,
591            pool_meta_bootstrap_authorities,
592            elected_bootstrap_writer,
593        );
594
595        let decommission_cancelers = RwLock::new(vec![None; pools.len()]);
596        let ec = Arc::new(ECStore {
597            id: deployment_id,
598            disk_map,
599            pools,
600            peer_sys,
601            pool_meta: RwLock::new(pool_meta),
602            rebalance_meta: RwLock::new(None),
603            decommission_cancelers,
604            start_gate: Mutex::new(()),
605            pool_meta_save_gate: Mutex::new(pool_meta_write_state),
606            // Adopt the caller's context (the process bootstrap one on the
607            // legacy path) so startup writes (erasure type recorded before
608            // this point) and later reads share one cell.
609            ctx: instance_ctx.clone(),
610            bucket_fence_registry: std::sync::Arc::default(),
611        });
612
613        // Only set it when this instance's deployment ID is not yet configured
614        if instance_ctx.deployment_id().is_none() {
615            instance_ctx.set_deployment_id(deployment_id);
616        }
617
618        let wait_sec = 5;
619        let mut exit_count = 0;
620        loop {
621            if let Err(err) = ec.init(ctx.clone()).await {
622                error!("init err: {}", err);
623                error!("retry after  {} second", wait_sec);
624                sleep(Duration::from_secs(wait_sec)).await;
625
626                if exit_count > 10 {
627                    return Err(Error::other("store init failed: init retry budget exhausted"));
628                }
629
630                exit_count += 1;
631
632                continue;
633            }
634
635            break;
636        }
637
638        runtime_sources::publish_object_store(ec.clone()).await;
639
640        Ok(ec)
641    }
642
643    #[instrument(level = "debug", skip(self, rx))]
644    pub async fn init(self: &Arc<Self>, rx: CancellationToken) -> Result<()> {
645        runtime_sources::ensure_boot_time().await;
646
647        let should_persist_pool_meta = self
648            .pools
649            .first()
650            .is_some_and(|pool| pool_first_endpoint_is_local(&pool.endpoints));
651        #[cfg(feature = "e2e-test-hooks")]
652        let startup_attempt = uuid::Uuid::new_v4();
653        let (meta, pool_meta_replica_state) = {
654            let mut write_state = self.pool_meta_save_gate.lock().await;
655            establish_pool_meta_bootstrap_identity_if_proven(self.pools.clone(), &mut write_state, should_persist_pool_meta)
656                .await
657                .map_err(|err| Error::other(format!("store init failed during establish_pool_meta_bootstrap_identity: {err}")))?;
658            let load = load_pool_meta_for_startup(self.pools.clone(), &mut write_state);
659            #[cfg(feature = "e2e-test-hooks")]
660            let load = crate::core::pools::startup_cas_test_scope(startup_attempt, "load", &self.pools, load);
661            load.await?
662        };
663        let update = meta.validate(self.pools.clone())?;
664        #[cfg(feature = "e2e-test-hooks")]
665        crate::core::pools::startup_cas_test_observe(serde_json::json!({
666            "kind": "startup-classifier", "attempt": startup_attempt,
667            "elected_writer": should_persist_pool_meta,
668            "needs_repair": pool_meta_replica_state.needs_repair,
669            "repair_write_safe": pool_meta_replica_state.repair_write_safe,
670            "topology_update": update,
671        }));
672        let endpoints = runtime_sources::endpoint_pools_or_default();
673
674        let mut installed_pool_meta = if update {
675            PoolMeta::new(&self.pools, &meta)
676        } else {
677            meta.clone()
678        };
679        // Only one local node should persist validated pool metadata here; otherwise
680        // distributed startup can race on the same lock and replay the prior init bug.
681        {
682            let mut write_state = self.pool_meta_save_gate.lock().await;
683            let persist = persist_pool_meta_for_startup_if_safe(
684                &installed_pool_meta,
685                self.pools.clone(),
686                pool_meta_replica_state,
687                &mut write_state,
688                update,
689                should_persist_pool_meta,
690            );
691            #[cfg(feature = "e2e-test-hooks")]
692            let persist = crate::core::pools::startup_cas_test_scope(startup_attempt, "persist", &self.pools, persist);
693            installed_pool_meta = persist.await?;
694        }
695
696        {
697            let mut pool_meta = self.pool_meta.write().await;
698            *pool_meta = installed_pool_meta.clone();
699        }
700
701        resolve_store_init_stage_result(self.load_rebalance_meta().await, "load_rebalance_meta")?;
702        let rebalance_resume_required = {
703            let rebalance_meta = self.rebalance_meta.read().await;
704            rebalance_meta
705                .as_ref()
706                .is_some_and(crate::services::rebalance::rebalance_requires_worker_activation)
707        };
708        let decommission_running =
709            pool_meta_has_active_decommission(&installed_pool_meta) || self.is_decommission_running().await;
710        let distributed = self.ctx.is_dist_erasure().await;
711        let fleet_proof_available = crate::services::notification_sys::acquire_cross_pool_fence_fleet_proof().is_some();
712        let mut rebalance_auto_start_deferred = false;
713        if should_auto_start_rebalance_after_init(decommission_running, rebalance_resume_required) {
714            if should_defer_rebalance_auto_start(distributed, fleet_proof_available) {
715                rebalance_auto_start_deferred = true;
716                warn!(
717                    event = EVENT_ECSTORE_INIT_STATUS,
718                    component = LOG_COMPONENT_ECSTORE,
719                    subsystem = LOG_SUBSYSTEM_STORE_INIT,
720                    stage = "start_rebalance",
721                    state = "deferred",
722                    reason = "fleet_capability_proof_unavailable",
723                    retry_delay_secs = REBALANCE_RESUME_RETRY_DELAY.as_secs(),
724                    "Deferred rebalance auto-start until a live fleet capability proof is available"
725                );
726            } else if let Err(err) = self.start_rebalance().await {
727                if crate::core::pools::is_pool_activation_fleet_proof_error(&err) {
728                    rebalance_auto_start_deferred = true;
729                    warn!(
730                        event = EVENT_ECSTORE_INIT_STATUS,
731                        component = LOG_COMPONENT_ECSTORE,
732                        subsystem = LOG_SUBSYSTEM_STORE_INIT,
733                        stage = "start_rebalance",
734                        state = "deferred",
735                        reason = "fleet_capability_proof_changed",
736                        error = %err,
737                        retry_delay_secs = REBALANCE_RESUME_RETRY_DELAY.as_secs(),
738                        "Deferred rebalance auto-start after the fleet capability proof changed"
739                    );
740                } else {
741                    return resolve_store_init_stage_result(Err(err), "start_rebalance");
742                }
743            }
744        } else if decommission_running && rebalance_resume_required {
745            warn!(
746                event = EVENT_ECSTORE_INIT_STATUS,
747                component = LOG_COMPONENT_ECSTORE,
748                subsystem = LOG_SUBSYSTEM_STORE_INIT,
749                stage = "start_rebalance",
750                reason = "active_decommission",
751                "Deferred rebalance auto-start during store init because decommission is active"
752            );
753        }
754
755        // Initialize the storage-owned scanner publication state only after
756        // both movement metadata sources have been loaded. SetDisks cache
757        // writers remain fail-closed until this snapshot is available.
758        let _ = self.scanner_data_usage_publication_blocked().await;
759
760        let pools = installed_pool_meta.return_resumable_pools();
761        let mut pool_indices = Vec::with_capacity(pools.len());
762
763        for p in pools.iter() {
764            if let Some(idx) = endpoints.get_pool_idx(&p.cmd_line) {
765                pool_indices.push(idx);
766            } else {
767                return Err(Error::other(format!(
768                    "store init failed to resolve resumable decommission pool `{}` from current endpoints",
769                    p.cmd_line
770                )));
771            }
772        }
773
774        let local_pool_indices = local_decommission_queue_prefix(&endpoints, &pool_indices)?;
775        let has_local_decommission_leadership = endpoints.as_ref().iter().any(pool_first_endpoint_is_local);
776        let pool_meta_write_safe = self
777            .ensure_pool_meta_side_effects_safe("decommission resume blocked while pool metadata requires recovery")
778            .await
779            .is_ok();
780        if !pool_meta_replica_state.repair_write_safe || !pool_meta_write_safe {
781            warn!(
782                event = EVENT_DECOMMISSION_RESUME_RETRY,
783                component = LOG_COMPONENT_ECSTORE,
784                subsystem = LOG_SUBSYSTEM_STORE_INIT,
785                state = "blocked",
786                pool_indices = ?local_pool_indices,
787                reason = "pool_meta_write_blocked",
788                "Decommission watchdog waiting for pool metadata recovery"
789            );
790        }
791        if has_local_decommission_leadership {
792            // The watchdog checks recovery safety and retries transient failures.
793            // Resume persisted work without an unconditional cold-start delay.
794            tokio::spawn(supervise_local_decommission_after_init(self.clone(), rx.clone()));
795        }
796
797        let recovery_store = self.clone();
798        let recovery_rx = rx.clone();
799        tokio::spawn(async move {
800            let mut delay = std::time::Duration::from_secs(5);
801            loop {
802                tokio::select! {
803                    _ = recovery_rx.cancelled() => return,
804                    _ = tokio::time::sleep(delay) => {}
805                }
806                let result = tokio::select! {
807                    _ = recovery_rx.cancelled() => return,
808                    result = tokio::time::timeout(std::time::Duration::from_secs(30), recovery_store.recover_pool_meta_transaction()) => result,
809                };
810                delay = match result {
811                    Ok(Ok(_)) => std::time::Duration::from_secs(5),
812                    failure => {
813                        let error = match failure {
814                            Ok(Err(error)) => error,
815                            _ => Error::Timeout,
816                        };
817                        recovery_store.record_pool_meta_recovery_failure(error);
818                        (delay * 2).min(std::time::Duration::from_secs(60))
819                    }
820                };
821            }
822        });
823
824        runtime_sources::init_bucket_monitor_for_current_endpoints();
825        crate::bucket::bucket_target_sys::BucketTargetSys::get().start_heartbeat();
826
827        init_background_expiry(self.clone()).await;
828        crate::bucket::lifecycle::bucket_lifecycle_ops::init_background_stale_multipart_upload_cleanup(self.clone());
829
830        TransitionState::init(self.clone()).await;
831        crate::services::tier::tier::try_migrate_tiering_config(self.clone()).await;
832
833        if let Err(err) = runtime_sources::init_tier_config_mgr(self.clone()).await {
834            info!("TierConfigMgr init error: {}", err);
835        }
836
837        if rebalance_auto_start_deferred {
838            let store = self.clone();
839            tokio::spawn(resume_rebalance_after_init(store, rx));
840        }
841
842        Ok(())
843    }
844
845    pub fn init_local_disks() {}
846
847    pub fn single_pool(&self) -> bool {
848        self.pools.len() == 1
849    }
850
851    /// The set-local create-only check is atomic only when every object
852    /// mutation uses that same, enabled namespace lock domain.
853    pub fn supports_atomic_create_only_write_back(&self) -> bool {
854        !self.ctx.lock_manager().is_disabled() && self.pools.len() == 1 && self.pools[0].disk_set.len() == 1
855    }
856}
857
858#[cfg(test)]
859mod tests {
860    use super::{
861        LOCAL_DECOMMISSION_RESUME_RETRY_DELAY, LOCAL_DECOMMISSION_WATCHDOG_MAX_RETRY_DELAY, PoolMetaWriteState,
862        establish_pool_meta_bootstrap_identity_if_proven, load_pool_meta_for_startup, local_decommission_watchdog_retry_delay,
863        persist_pool_meta_for_startup_if_safe, pool_first_endpoint_is_local, pool_meta_has_active_decommission,
864        preflight_startup_rpc_secret_with, resolve_startup_pool_defaults_with, resolve_store_init_stage_result,
865        run_local_decommission_watchdog, save_validated_pool_meta_for_startup, should_auto_start_rebalance_after_init,
866        should_defer_rebalance_auto_start, should_retry_format_load, wait_for_local_decommission_resume_delay,
867    };
868    use crate::core::pools::PoolMetaBootstrapAuthority;
869    #[cfg(feature = "test-util")]
870    use crate::disk::DiskAPI;
871    #[cfg(feature = "test-util")]
872    use crate::{
873        bucket::lifecycle::{
874            DurableIlmRecordCheckpoint, ILM_META_PREFIX, ValidatedDurableIlmRecord,
875            bucket_lifecycle_ops::{ExpiryState, ManualTransitionRunOptions, recover_manual_transition_jobs_once},
876            lifecycle::{TRANSITION_PENDING, TransitionOptions},
877            manual_transition_job::{
878                ManualTransitionJobRecord, ManualTransitionScopeAdmission, ManualTransitionTaskRecord,
879                ManualTransitionWorkerResult, ManualTransitionWorkerResultRecord, manual_transition_job_record_object_name,
880                manual_transition_scope_record_object_name, manual_transition_task_object_name,
881                manual_transition_worker_result_object_name, manual_transition_worker_result_task_key,
882            },
883            recovery_control::{
884                IlmRecoveryClassification, IlmRecoveryControl, IlmRecoveryControlIdentity, IlmRecoveryErrorCode,
885                IlmRecoveryProtocol, MAX_RECOVERY_ATTEMPTS, list_recovery_controls, load_recovery_control,
886                observe_recovery_source, recovery_control_record_object_name, save_recovery_control_if_absent,
887            },
888            recovery_disposition::{
889                IlmRecoveryDispositionExecutionOutcome, IlmRecoveryDispositionState, RecoveryDispositionCrashStage,
890                dry_run_recovery_disposition, execute_recovery_disposition, inject_recovery_disposition_crash_once,
891                load_recovery_disposition,
892            },
893            recovery_disposition_runtime::garbage_collect_completed_recovery_disposition,
894            recovery_export::{
895                create_recovery_export, inspect_recovery_export_observation, load_recovery_export,
896                recovery_export_record_object_name,
897            },
898            tier_delete_journal::{
899                DecommissionCheckpointTargetFailureHook, TIER_DELETE_DISPATCH_MANIFEST_PREFIX, TIER_DELETE_JOURNAL_PREFIX,
900                TierDeleteChunkTestBarrier, TierDeleteChunkTestStage, TierDeleteDispatchBatchLimitGuard,
901                TierDeleteDispatchManifestState, TierDeleteDispatchMemberReadTestHook, TierDeleteDispatchMemberReadTestStage,
902                TierDeleteDispatchRollbackTestHook, complete_tier_delete_dispatch, encode_tier_delete_journal_entry,
903                install_test_tier_delete_dispatch_fixture, persist_tier_delete_journal_entry, prepare_tier_delete_dispatch,
904                recover_test_tier_delete_dispatch_manifest, recover_test_tier_delete_dispatch_manifest_with_page_budget,
905                recover_test_tier_delete_journal_pass_with_budget, recover_tier_delete_dispatch_manifests,
906                recover_tier_delete_journal_entries, test_tier_delete_dispatch_manifest_checkpoint,
907                test_tier_delete_dispatch_manifest_state, tier_delete_dispatch_manifest_operation_lock_held_for_test,
908                tier_delete_dispatch_manifest_recovery_count_for_test, tier_delete_dispatch_manifest_recovery_inflight_for_test,
909                tier_delete_journal_object_name,
910            },
911            tier_free_version_recovery::recover_tier_free_versions,
912            tier_sweeper::{
913                Jentry, TierDeleteJournalState, TierDeleteSourceIdentity, transitioned_delete_journal_entry_for_source,
914            },
915            transition_transaction::{
916                TRANSITION_TRANSACTION_RECORD_PREFIX, TransitionCleanupDecision, TransitionCleanupProof, TransitionOperatorError,
917                TransitionOperatorProbe, TransitionRecoveryClaimBarrier, TransitionRecoveryTerminalBarrier,
918                TransitionRemoteVersion, TransitionSourceIdentity, TransitionSourceVersionMode, TransitionTransaction,
919                TransitionTransactionInit, TransitionTransactionState, delete_transition_candidate_for_operator,
920                finalize_missing_transition_transaction_for_operator, inspect_transition_recovery_retry_for_operator,
921                inspect_transition_transaction_for_operator, load_transition_transaction_record,
922                recover_transition_transaction_records, recover_transition_transaction_records_at,
923                retry_transition_recovery_for_operator, save_transition_transaction_record,
924                save_transition_transaction_record_if_current, transition_recovery_control_id,
925                transition_transaction_record_object_name,
926            },
927            validate_durable_ilm_record,
928        },
929        bucket::metadata::{BUCKET_LIFECYCLE_CONFIG, BUCKET_VERSIONING_CONFIG},
930        config::com,
931        core::pools::DecomBucketInfo,
932        data_movement::SourceCleanupDeleteBarrier,
933        disk::{BUCKET_META_PREFIX, RUSTFS_META_BUCKET, STORAGE_FORMAT_FILE},
934        runtime::{global::set_object_store_resolver, sources as runtime_sources},
935        services::notification_sys::{
936            acquire_tier_delete_journal_fleet_proof, acquire_transition_transaction_compaction_fleet_proof,
937            install_transition_transaction_compaction_fleet_proof_for_test,
938            transition_transaction_compaction_fleet_proof_matches,
939        },
940        services::tier::{
941            test_util::{MockWarmBackend, MockWarmOp, TransitionCleanupStoreBarrier, register_mock_tier},
942            tier::{
943                ERR_TIER_BACKEND_IN_USE, ERR_TIER_INVALID_CONFIG, TIER_CONFIG_FILE, TierConfigMgr,
944                ensure_no_authoritative_tier_references_for_test, tier_config_candidate_digest,
945            },
946            tier_config::{TierConfig, TierType, TierWasabi},
947            tier_mutation_intent::{
948                TIER_MUTATION_INTENT_RECORD_PREFIX, TierMutationIntent, TierMutationIntentKind, TierMutationIntentState,
949                TierMutationIntentTarget, advance_tier_mutation_intent_record_idempotent, delete_tier_mutation_intent_record,
950                list_tier_mutation_intent_records, load_tier_mutation_intent_record, load_tier_mutation_intent_record_with_etag,
951                save_tier_mutation_intent_record, save_tier_mutation_intent_record_if_current,
952            },
953            tier_mutation_peer::{TierMutationPeerError, TierMutationPeerState, handle_tier_mutation_peer_request},
954            tier_probe_intent::{
955                TierProbeIntent, TierProbeIntentState, TierProbeOperationIdentity, TierProbeOwnerFence, TierProbeRemoteVersion,
956                delete_tier_probe_intent_record_if_current, load_tier_probe_intent_record,
957                save_tier_probe_intent_record_if_absent, save_tier_probe_intent_record_if_current,
958            },
959            warm_backend::{TransitionCandidateProbe, WarmBackend},
960        },
961        set_disk::{
962            SetDiskTransitionTransactionKillPoint as TransitionTransactionKillPoint,
963            SetDiskTransitionTransactionKillPointBarrier as TransitionTransactionKillPointBarrier,
964            SetDiskTransitionTransactionMutationKind as TransitionTransactionMutationKind,
965            SetDiskTransitionTransactionMutationObservation as TransitionTransactionMutationObservation,
966            SetDiskTransitionTransactionMutationProbe as TransitionTransactionMutationProbe,
967            SetDiskTransitionUploadedCommitBarrier as TransitionUploadedCommitBarrier,
968        },
969        storage_api_contracts::list::ListOperations as _,
970    };
971    #[cfg(feature = "test-util")]
972    use crate::{bucket::replication::ReplicationObjectBridge, storage_api_contracts::bucket::DeleteBucketOptions};
973    use crate::{
974        bucket::replication::{ReplicationState, ReplicationStatusType, replication_statuses_map},
975        core::pools::{
976            DecommissionErasureLayout, DecommissionPoolCapacityInfo, POOL_META_IDENTITY_NAME, POOL_META_NAME, POOL_META_VERSION,
977            PoolDecommissionInfo, PoolMeta, PoolStatus, pending_pool_meta_identity_for_test,
978            pool_meta_identity_initialized_for_test, pool_meta_v3_commit_state_for_test,
979            set_decommission_capacity_info_overrides_for_test,
980        },
981        disk::endpoint::Endpoint,
982        error::{Error, Result, StorageError},
983        io_support::rio::{WritePlan, compression_metadata_value},
984        layout::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints},
985        object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader},
986        services::rebalance::{RebalStatus, RebalanceInfo, RebalanceMeta, RebalanceStats},
987        storage_api_contracts::{
988            bucket::{BucketOperations as _, MakeBucketOptions},
989            multipart::MultipartOperations as _,
990            object::{ObjectIO, ObjectOperations as _, ObjectToDelete},
991            range::HTTPRangeSpec,
992        },
993    };
994    #[cfg(feature = "test-util")]
995    use futures::{StreamExt as _, TryStreamExt as _};
996    use http::HeaderMap;
997    use rustfs_config::server_config::KVS;
998    #[cfg(feature = "test-util")]
999    use rustfs_filemeta::{FileInfo, FileMeta};
1000    use rustfs_filemeta::{FileInfoVersions, MetaCacheEntry};
1001    #[cfg(feature = "test-util")]
1002    use rustfs_protos::{TIER_MUTATION_RPC_PROTOCOL_VERSION, TierMutationRpcPhase};
1003    use rustfs_rio::{Checksum, ChecksumType};
1004    #[cfg(feature = "test-util")]
1005    use rustfs_s3_client::transition_api::ReaderImpl;
1006    use rustfs_utils::{
1007        CompressionAlgorithm,
1008        http::{SUFFIX_COMPRESSION, insert_str},
1009    };
1010    use std::{
1011        collections::HashMap,
1012        future::Future,
1013        io::Cursor,
1014        sync::{
1015            Arc, Mutex,
1016            atomic::{AtomicBool, AtomicUsize, Ordering},
1017        },
1018        time::Duration,
1019    };
1020    use time::OffsetDateTime;
1021    use tokio::io::AsyncReadExt;
1022    use tokio_util::sync::CancellationToken;
1023
1024    fn run_large_stack_async_test<C, F>(name: &str, case: C)
1025    where
1026        C: FnOnce() -> F + Send + 'static,
1027        F: Future<Output = ()> + 'static,
1028    {
1029        const STACK_SIZE: usize = if cfg!(debug_assertions) {
1030            8 * rustfs_config::DEFAULT_THREAD_STACK_SIZE
1031        } else if cfg!(target_os = "macos") {
1032            2 * rustfs_config::DEFAULT_THREAD_STACK_SIZE
1033        } else {
1034            rustfs_config::DEFAULT_THREAD_STACK_SIZE
1035        };
1036        std::thread::Builder::new()
1037            .name(name.to_string())
1038            .stack_size(STACK_SIZE)
1039            .spawn(move || {
1040                let runtime = tokio::runtime::Builder::new_multi_thread()
1041                    .enable_all()
1042                    .worker_threads(2)
1043                    .thread_stack_size(STACK_SIZE)
1044                    .build()
1045                    .expect("large-stack store test runtime should build");
1046                runtime.block_on(case());
1047            })
1048            .expect("large-stack store test thread should spawn")
1049            .join()
1050            .expect("large-stack store test thread should complete");
1051    }
1052    use uuid::Uuid;
1053
1054    #[test]
1055    fn new_with_instance_ctx_future_remains_stack_bounded() {
1056        let future = crate::store::ECStore::new_with_instance_ctx(
1057            std::net::SocketAddr::from(([127, 0, 0, 1], 0)),
1058            EndpointServerPools(Vec::new()),
1059            CancellationToken::new(),
1060            Arc::new(crate::runtime::instance::InstanceContext::new()),
1061        );
1062        let future_size = std::mem::size_of_val(&future);
1063
1064        assert!(
1065            future_size <= 4 * 1024,
1066            "ECStore constructor future must remain stack-bounded; measured {future_size} bytes"
1067        );
1068    }
1069
1070    fn startup_pool_meta_payload(meta: &PoolMeta) -> Vec<u8> {
1071        meta.encode_config_data_for_test().expect("pool metadata should encode")
1072    }
1073
1074    #[derive(Debug)]
1075    struct StartupPoolMetaStorage {
1076        read_payload: Vec<u8>,
1077        read_error: bool,
1078        read_without_lock: AtomicBool,
1079        wrote_without_lock: AtomicBool,
1080        wrote_with_max_parity: AtomicBool,
1081        written_payload: Mutex<Option<Vec<u8>>>,
1082        revision: AtomicUsize,
1083        pool_meta_write_attempts: AtomicUsize,
1084        fail_pool_meta_write_at: AtomicUsize,
1085        pool_meta_written_versions: Mutex<Vec<u16>>,
1086        objects: Mutex<HashMap<String, (Vec<u8>, String)>>,
1087    }
1088
1089    impl StartupPoolMetaStorage {
1090        fn new(read_payload: Vec<u8>) -> Self {
1091            let mut objects = HashMap::new();
1092            if !read_payload.is_empty() {
1093                objects.insert(POOL_META_NAME.to_string(), (read_payload.clone(), "startup-pool-meta-0".to_string()));
1094            }
1095            Self {
1096                read_payload,
1097                read_error: false,
1098                read_without_lock: AtomicBool::new(false),
1099                wrote_without_lock: AtomicBool::new(false),
1100                wrote_with_max_parity: AtomicBool::new(false),
1101                written_payload: Mutex::new(None),
1102                revision: AtomicUsize::new(0),
1103                pool_meta_write_attempts: AtomicUsize::new(0),
1104                fail_pool_meta_write_at: AtomicUsize::new(0),
1105                pool_meta_written_versions: Mutex::new(Vec::new()),
1106                objects: Mutex::new(objects),
1107            }
1108        }
1109
1110        fn unreadable() -> Self {
1111            Self {
1112                read_payload: Vec::new(),
1113                read_error: true,
1114                read_without_lock: AtomicBool::new(false),
1115                wrote_without_lock: AtomicBool::new(false),
1116                wrote_with_max_parity: AtomicBool::new(false),
1117                written_payload: Mutex::new(None),
1118                revision: AtomicUsize::new(0),
1119                pool_meta_write_attempts: AtomicUsize::new(0),
1120                fail_pool_meta_write_at: AtomicUsize::new(0),
1121                pool_meta_written_versions: Mutex::new(Vec::new()),
1122                objects: Mutex::new(HashMap::new()),
1123            }
1124        }
1125
1126        fn object_info(&self, bucket: &str, object: &str, size: usize, etag: String) -> ObjectInfo {
1127            ObjectInfo {
1128                bucket: bucket.to_string(),
1129                name: object.to_string(),
1130                size: size as i64,
1131                actual_size: size as i64,
1132                etag: Some(etag),
1133                ..Default::default()
1134            }
1135        }
1136    }
1137
1138    #[async_trait::async_trait]
1139    impl ObjectIO for StartupPoolMetaStorage {
1140        type Error = Error;
1141        type RangeSpec = HTTPRangeSpec;
1142        type HeaderMap = HeaderMap;
1143        type ObjectOptions = ObjectOptions;
1144        type ObjectInfo = ObjectInfo;
1145        type GetObjectReader = GetObjectReader;
1146        type PutObjectReader = PutObjReader;
1147
1148        async fn get_object_reader(
1149            &self,
1150            bucket: &str,
1151            object: &str,
1152            _range: Option<HTTPRangeSpec>,
1153            _h: HeaderMap,
1154            opts: &ObjectOptions,
1155        ) -> Result<GetObjectReader> {
1156            assert!(opts.no_lock, "store init pool metadata load must not require namespace locks");
1157            self.read_without_lock.store(true, Ordering::SeqCst);
1158            if self.read_error {
1159                return Err(Error::other("pool metadata read quorum unavailable"));
1160            }
1161            let Some((payload, etag)) = self
1162                .objects
1163                .lock()
1164                .unwrap_or_else(std::sync::PoisonError::into_inner)
1165                .get(object)
1166                .cloned()
1167            else {
1168                return Err(Error::FileNotFound);
1169            };
1170
1171            Ok(GetObjectReader {
1172                stream: Box::new(Cursor::new(payload.clone())),
1173                object_info: self.object_info(bucket, object, payload.len(), etag),
1174                buffered_body: None,
1175                body_source: Default::default(),
1176            })
1177        }
1178
1179        async fn put_object(
1180            &self,
1181            bucket: &str,
1182            object: &str,
1183            data: &mut PutObjReader,
1184            opts: &ObjectOptions,
1185        ) -> Result<ObjectInfo> {
1186            assert!(opts.no_lock, "store init pool metadata save must not require namespace locks");
1187            self.wrote_without_lock.store(true, Ordering::SeqCst);
1188            self.wrote_with_max_parity.store(opts.max_parity, Ordering::SeqCst);
1189            if object == POOL_META_NAME {
1190                let attempt = self.pool_meta_write_attempts.fetch_add(1, Ordering::SeqCst) + 1;
1191                if self.fail_pool_meta_write_at.load(Ordering::SeqCst) == attempt {
1192                    return Err(Error::Timeout);
1193                }
1194            }
1195            let current_etag = self
1196                .objects
1197                .lock()
1198                .unwrap_or_else(std::sync::PoisonError::into_inner)
1199                .get(object)
1200                .map(|(_, etag)| etag.clone());
1201            if opts
1202                .http_preconditions
1203                .as_ref()
1204                .and_then(|preconditions| preconditions.if_none_match_value())
1205                == Some("*")
1206                && current_etag.is_some()
1207            {
1208                return Err(Error::PreconditionFailed);
1209            }
1210            if let Some(expected) = opts
1211                .http_preconditions
1212                .as_ref()
1213                .and_then(|preconditions| preconditions.if_match_value())
1214                && current_etag.as_deref() != Some(expected)
1215            {
1216                return Err(Error::PreconditionFailed);
1217            }
1218            let mut payload = Vec::new();
1219            data.stream.read_to_end(&mut payload).await?;
1220            let size = payload.len();
1221            if object == POOL_META_NAME {
1222                *self.written_payload.lock().unwrap_or_else(std::sync::PoisonError::into_inner) = Some(payload.clone());
1223                if payload.len() >= 4 {
1224                    self.pool_meta_written_versions
1225                        .lock()
1226                        .unwrap_or_else(std::sync::PoisonError::into_inner)
1227                        .push(u16::from_le_bytes([payload[2], payload[3]]));
1228                }
1229            }
1230            let etag = format!("startup-pool-meta-{}", self.revision.fetch_add(1, Ordering::SeqCst) + 1);
1231            self.objects
1232                .lock()
1233                .unwrap_or_else(std::sync::PoisonError::into_inner)
1234                .insert(object.to_string(), (payload, etag.clone()));
1235            Ok(self.object_info(bucket, object, size, etag))
1236        }
1237    }
1238
1239    fn init_test_pool_meta(decommission: Option<PoolDecommissionInfo>) -> PoolMeta {
1240        PoolMeta {
1241            version: POOL_META_VERSION,
1242            pools: vec![PoolStatus {
1243                id: 0,
1244                cmd_line: "pool-0".to_string(),
1245                last_update: OffsetDateTime::UNIX_EPOCH,
1246                decommission,
1247            }],
1248            dont_save: false,
1249        }
1250    }
1251
1252    #[tokio::test]
1253    async fn test_fresh_bootstrap_pool_meta_save_reaches_v3_cas_without_blocking_itself() {
1254        let storage = Arc::new(StartupPoolMetaStorage::new(Vec::new()));
1255        let mut write_state = PoolMetaWriteState::for_startup(Uuid::new_v4(), true);
1256        establish_pool_meta_bootstrap_identity_if_proven(vec![storage.clone()], &mut write_state, true)
1257            .await
1258            .expect("the elected fresh bootstrap should persist its identity before loading pool metadata");
1259
1260        let (loaded, replica_state) = load_pool_meta_for_startup(vec![storage.clone()], &mut write_state)
1261            .await
1262            .expect("startup pool metadata load should tolerate missing metadata without locks");
1263        assert!(loaded.pools.is_empty());
1264        assert!(!replica_state.needs_repair);
1265        assert!(replica_state.repair_write_safe);
1266        assert!(storage.read_without_lock.load(Ordering::SeqCst));
1267
1268        let meta = PoolMeta {
1269            version: POOL_META_VERSION,
1270            pools: Vec::new(),
1271            dont_save: false,
1272        };
1273        save_validated_pool_meta_for_startup(&meta, vec![storage.clone()], &mut write_state)
1274            .await
1275            .expect("startup pool metadata save should bypass locks");
1276        assert!(storage.wrote_without_lock.load(Ordering::SeqCst));
1277        assert!(storage.wrote_with_max_parity.load(Ordering::SeqCst));
1278        assert_eq!(
1279            storage.pool_meta_write_attempts.load(Ordering::SeqCst),
1280            2,
1281            "fresh bootstrap should reach both V3 prepare and commit CAS writes"
1282        );
1283        assert_eq!(
1284            *storage
1285                .pool_meta_written_versions
1286                .lock()
1287                .unwrap_or_else(std::sync::PoisonError::into_inner),
1288            vec![3, 3]
1289        );
1290        write_state
1291            .ensure_write_safe("fresh bootstrap publication")
1292            .expect("successful startup publication must disarm the transaction guard");
1293    }
1294
1295    #[tokio::test]
1296    async fn test_legacy_adoption_pool_meta_bootstrap_reaches_v3_cas() {
1297        let deployment_id = Uuid::new_v4();
1298        let storage = Arc::new(StartupPoolMetaStorage::new(Vec::new()));
1299        let mut write_state =
1300            PoolMetaWriteState::for_startup_with_bootstrap_authority(deployment_id, PoolMetaBootstrapAuthority::LegacyAdoption);
1301
1302        establish_pool_meta_bootstrap_identity_if_proven(vec![storage.clone()], &mut write_state, true)
1303            .await
1304            .expect("verified legacy adoption should persist a nonce-bound identity before loading pool metadata");
1305        let (loaded, replica_state) = load_pool_meta_for_startup(vec![storage.clone()], &mut write_state)
1306            .await
1307            .expect("verified legacy adoption should authorize initially missing pool metadata");
1308        assert!(loaded.pools.is_empty());
1309
1310        let committed = persist_pool_meta_for_startup_if_safe(
1311            &init_test_pool_meta(None),
1312            vec![storage.clone()],
1313            replica_state,
1314            &mut write_state,
1315            true,
1316            true,
1317        )
1318        .await
1319        .expect("verified legacy adoption should publish initial pool metadata");
1320        assert_eq!(committed.pools[0].cmd_line, "pool-0");
1321
1322        let objects = storage.objects.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
1323        assert!(objects.contains_key(POOL_META_NAME));
1324        let identity = objects
1325            .get(POOL_META_IDENTITY_NAME)
1326            .map(|(payload, _)| payload.clone())
1327            .expect("legacy adoption should commit the bootstrap identity");
1328        assert!(pool_meta_identity_initialized_for_test(&identity).expect("decode committed identity"));
1329    }
1330
1331    #[tokio::test]
1332    async fn test_non_elected_legacy_adoption_cannot_initialize_pool_meta() {
1333        let storage = Arc::new(StartupPoolMetaStorage::new(Vec::new()));
1334        let mut write_state =
1335            PoolMetaWriteState::for_startup_with_bootstrap_authority(Uuid::new_v4(), PoolMetaBootstrapAuthority::LegacyAdoption);
1336
1337        establish_pool_meta_bootstrap_identity_if_proven(vec![storage.clone()], &mut write_state, false)
1338            .await
1339            .expect("a non-elected distributed node must not create legacy adoption authority");
1340        let err = load_pool_meta_for_startup(vec![storage.clone()], &mut write_state)
1341            .await
1342            .expect_err("legacy adoption still requires the elected writer to create a durable identity");
1343        assert!(err.to_string().contains("no durable bootstrap identity"));
1344        assert!(
1345            !storage
1346                .objects
1347                .lock()
1348                .unwrap_or_else(std::sync::PoisonError::into_inner)
1349                .contains_key(POOL_META_NAME),
1350            "classification must not create pool metadata on non-elected nodes"
1351        );
1352    }
1353
1354    #[tokio::test]
1355    async fn test_unproven_pending_identity_cannot_authorize_all_missing_pool_meta() {
1356        let deployment_id = Uuid::new_v4();
1357        let storage = Arc::new(StartupPoolMetaStorage::new(Vec::new()));
1358        let mut proven_bootstrap = PoolMetaWriteState::for_startup(deployment_id, true);
1359        establish_pool_meta_bootstrap_identity_if_proven(vec![storage.clone()], &mut proven_bootstrap, true)
1360            .await
1361            .expect("fresh topology proof should persist a nonce-bound pending identity");
1362
1363        let identity = storage
1364            .objects
1365            .lock()
1366            .unwrap_or_else(std::sync::PoisonError::into_inner)
1367            .get(POOL_META_IDENTITY_NAME)
1368            .map(|(payload, _)| payload.clone())
1369            .expect("pending identity should be durable");
1370        assert!(!pool_meta_identity_initialized_for_test(&identity).expect("decode pending identity"));
1371
1372        let mut unproven_restart = PoolMetaWriteState::for_startup(deployment_id, false);
1373        let err = load_pool_meta_for_startup(vec![storage.clone()], &mut unproven_restart)
1374            .await
1375            .expect_err("a pending identity alone must not authorize an all-missing restart");
1376        assert!(err.to_string().contains("no verified fresh-bootstrap proof"));
1377        unproven_restart
1378            .ensure_write_safe("unproven pending identity restart")
1379            .expect_err("the rejected restart must latch the write gate");
1380        assert!(
1381            !storage
1382                .objects
1383                .lock()
1384                .unwrap_or_else(std::sync::PoisonError::into_inner)
1385                .contains_key(POOL_META_NAME),
1386            "classification must not create pool metadata"
1387        );
1388    }
1389
1390    #[tokio::test]
1391    async fn test_nonfresh_identity_repair_crash_never_persists_pending_bootstrap_authority() {
1392        let deployment_id = Uuid::new_v4();
1393        let initial = init_test_pool_meta(None);
1394        let storage = Arc::new(StartupPoolMetaStorage::new(startup_pool_meta_payload(&initial)));
1395        let mut write_state = PoolMetaWriteState::for_startup(deployment_id, false);
1396        let (loaded, replica_state) = load_pool_meta_for_startup(vec![storage.clone()], &mut write_state)
1397            .await
1398            .expect("an existing pool metadata snapshot may repair its missing identity");
1399
1400        storage.fail_pool_meta_write_at.store(1, Ordering::SeqCst);
1401        persist_pool_meta_for_startup_if_safe(&loaded, vec![storage.clone()], replica_state, &mut write_state, true, true)
1402            .await
1403            .expect_err("the injected crash boundary should stop before the pool metadata replacement");
1404
1405        let identity = storage
1406            .objects
1407            .lock()
1408            .unwrap_or_else(std::sync::PoisonError::into_inner)
1409            .get(POOL_META_IDENTITY_NAME)
1410            .map(|(payload, _)| payload.clone())
1411            .expect("identity repair should be durable before the failed topology write");
1412        assert!(
1413            pool_meta_identity_initialized_for_test(&identity).expect("decode repaired identity"),
1414            "an existing cluster must never leave pending bootstrap authority at this crash boundary"
1415        );
1416
1417        storage
1418            .objects
1419            .lock()
1420            .unwrap_or_else(std::sync::PoisonError::into_inner)
1421            .remove(POOL_META_NAME);
1422        let mut restarted = PoolMetaWriteState::for_startup(deployment_id, false);
1423        let err = load_pool_meta_for_startup(vec![storage], &mut restarted)
1424            .await
1425            .expect_err("wiping pool metadata after the crash must require recovery");
1426        assert!(err.to_string().contains("initialized cluster identity exists"));
1427    }
1428
1429    #[tokio::test]
1430    #[serial_test::serial(pool_meta_version_env)]
1431    async fn test_existing_legacy_identity_and_replica_repair_respects_disabled_v3_gates() {
1432        temp_env::async_with_vars(
1433            [
1434                (rustfs_config::ENV_POOL_META_V3_WRITE, None::<&str>),
1435                (rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, None::<&str>),
1436            ],
1437            async {
1438                let deployment_id = Uuid::new_v4();
1439                let corrupt = Arc::new(StartupPoolMetaStorage::new(vec![0, 1, 2]));
1440                let legacy = init_test_pool_meta(None);
1441                let valid = Arc::new(StartupPoolMetaStorage::new(startup_pool_meta_payload(&legacy)));
1442                let mut write_state = PoolMetaWriteState::for_startup(deployment_id, false);
1443                let (loaded, replica_state) = load_pool_meta_for_startup(vec![corrupt.clone(), valid.clone()], &mut write_state)
1444                    .await
1445                    .expect("existing V2 metadata should remain readable while its identity is missing");
1446                assert!(replica_state.needs_repair);
1447
1448                let committed = persist_pool_meta_for_startup_if_safe(
1449                    &loaded,
1450                    vec![corrupt.clone(), valid.clone()],
1451                    replica_state,
1452                    &mut write_state,
1453                    true,
1454                    true,
1455                )
1456                .await
1457                .expect("legacy replica and identity repair should succeed without crossing the V3 gate");
1458                assert_eq!(committed.version, POOL_META_VERSION);
1459                for storage in [corrupt, valid] {
1460                    assert_eq!(
1461                        *storage
1462                            .pool_meta_written_versions
1463                            .lock()
1464                            .unwrap_or_else(std::sync::PoisonError::into_inner),
1465                        vec![POOL_META_VERSION],
1466                        "legacy repair must write exactly one V2 snapshot"
1467                    );
1468                    let identity = storage
1469                        .objects
1470                        .lock()
1471                        .unwrap_or_else(std::sync::PoisonError::into_inner)
1472                        .get(POOL_META_IDENTITY_NAME)
1473                        .map(|(payload, _)| payload.clone())
1474                        .expect("identity repair should persist on every pool");
1475                    assert!(pool_meta_identity_initialized_for_test(&identity).expect("decode repaired identity"));
1476                }
1477            },
1478        )
1479        .await;
1480    }
1481
1482    fn startup_object(storage: &StartupPoolMetaStorage, object: &str) -> Option<Vec<u8>> {
1483        storage
1484            .objects
1485            .lock()
1486            .unwrap_or_else(std::sync::PoisonError::into_inner)
1487            .get(object)
1488            .map(|(payload, _)| payload.clone())
1489    }
1490
1491    /// Startup errors wrap their cause in context whose `Display` hides the
1492    /// source, so assertions walk the chain the same way
1493    /// `Error::pool_metadata_failure` does.
1494    fn error_chain_text(err: &Error) -> String {
1495        let mut parts = vec![err.to_string()];
1496        let mut current: Option<&(dyn std::error::Error + 'static)> = Some(err);
1497        while let Some(error) = current {
1498            current = if let Some(io) = error.downcast_ref::<std::io::Error>() {
1499                io.get_ref().map(|inner| inner as &(dyn std::error::Error + 'static))
1500            } else {
1501                error.source()
1502            };
1503            if let Some(next) = current {
1504                parts.push(next.to_string());
1505            }
1506        }
1507        parts.join(" <- ")
1508    }
1509
1510    fn inject_startup_object(storage: &StartupPoolMetaStorage, object: &str, payload: Vec<u8>) {
1511        storage
1512            .objects
1513            .lock()
1514            .unwrap_or_else(std::sync::PoisonError::into_inner)
1515            .insert(object.to_string(), (payload, format!("injected-{object}")));
1516    }
1517
1518    fn init_test_pool_meta_with_pools(pool_count: usize) -> PoolMeta {
1519        PoolMeta {
1520            version: POOL_META_VERSION,
1521            pools: (0..pool_count)
1522                .map(|id| PoolStatus {
1523                    id,
1524                    cmd_line: format!("pool-{id}"),
1525                    last_update: OffsetDateTime::UNIX_EPOCH,
1526                    decommission: None,
1527                })
1528                .collect(),
1529            dont_save: false,
1530        }
1531    }
1532
1533    /// Two single-node pools whose formats were created by different nodes:
1534    /// node0 formatted pool0 and only read pool1's format, node1 the reverse.
1535    fn two_pool_creator_states(deployment_id: Uuid) -> (PoolMetaWriteState, PoolMetaWriteState) {
1536        let node0 = PoolMetaWriteState::for_startup_with_pool_bootstrap_authorities(
1537            deployment_id,
1538            vec![PoolMetaBootstrapAuthority::Fresh, PoolMetaBootstrapAuthority::None],
1539            true,
1540        );
1541        let node1 = PoolMetaWriteState::for_startup_with_pool_bootstrap_authorities(
1542            deployment_id,
1543            vec![PoolMetaBootstrapAuthority::None, PoolMetaBootstrapAuthority::Fresh],
1544            false,
1545        );
1546        (node0, node1)
1547    }
1548
1549    #[tokio::test]
1550    async fn test_two_pool_bootstrap_with_distinct_format_creators_converges_through_creator_attestation() {
1551        let deployment_id = Uuid::new_v4();
1552        let pool0 = Arc::new(StartupPoolMetaStorage::new(Vec::new()));
1553        let pool1 = Arc::new(StartupPoolMetaStorage::new(Vec::new()));
1554        let pools = vec![pool0.clone(), pool1.clone()];
1555        let (mut node0, mut node1) = two_pool_creator_states(deployment_id);
1556        assert!(!node0.bootstrap_identity_proven(), "reading pool1's format is not deployment-wide proof");
1557        assert!(!node1.bootstrap_identity_proven());
1558
1559        // node1 (pool1 creator, non-elected) starts first: no durable nonce exists
1560        // yet, so it must neither mint one nor latch its write gate while waiting.
1561        establish_pool_meta_bootstrap_identity_if_proven(pools.clone(), &mut node1, false)
1562            .await
1563            .expect("a non-first creator without a durable nonce writes nothing");
1564        assert!(startup_object(&pool0, POOL_META_IDENTITY_NAME).is_none());
1565        assert!(startup_object(&pool1, POOL_META_IDENTITY_NAME).is_none());
1566        let err = load_pool_meta_for_startup(pools.clone(), &mut node1)
1567            .await
1568            .expect_err("nothing durable authorizes a non-elected node");
1569        assert!(err.to_string().contains("bootstrap pending"), "{err}");
1570        node1
1571            .ensure_write_safe("waiting non-elected creator")
1572            .expect("waiting for the elected writer must not latch the write gate");
1573
1574        // node0 (pool0 creator, elected) mints the nonce on the pool it created;
1575        // pool1 is still unattested, so it cannot publish pool.bin and must not latch.
1576        establish_pool_meta_bootstrap_identity_if_proven(pools.clone(), &mut node0, true)
1577            .await
1578            .expect("the first pool's creator mints the pending identity");
1579        let minted = startup_object(&pool0, POOL_META_IDENTITY_NAME).expect("pool0 pending identity");
1580        assert!(!pool_meta_identity_initialized_for_test(&minted).expect("decode pending identity"));
1581        assert!(
1582            startup_object(&pool1, POOL_META_IDENTITY_NAME).is_none(),
1583            "node0 holds no first-hand proof for pool1 and must not attest it"
1584        );
1585        let err = load_pool_meta_for_startup(pools.clone(), &mut node0)
1586            .await
1587            .expect_err("an unattested pool keeps the elected writer from publishing");
1588        assert!(err.to_string().contains("waiting for every pool creator"), "{err}");
1589        node0
1590            .ensure_write_safe("waiting elected writer")
1591            .expect("waiting for creators must not latch the write gate");
1592        assert!(startup_object(&pool0, POOL_META_NAME).is_none());
1593
1594        // node1 retries: it copies pool0's pending identity (same nonce) onto the
1595        // pool it created, then keeps waiting for the elected writer's pool.bin.
1596        establish_pool_meta_bootstrap_identity_if_proven(pools.clone(), &mut node1, false)
1597            .await
1598            .expect("the pool1 creator attests with the durable nonce");
1599        assert_eq!(startup_object(&pool1, POOL_META_IDENTITY_NAME).as_deref(), Some(minted.as_slice()));
1600        let err = load_pool_meta_for_startup(pools.clone(), &mut node1)
1601            .await
1602            .expect_err("a complete pending set never unlocks a non-elected node");
1603        assert!(err.to_string().contains("waiting for the elected writer to publish"), "{err}");
1604        node1
1605            .ensure_write_safe("attested non-elected creator")
1606            .expect("waiting for pool.bin must not latch the write gate");
1607        assert!(startup_object(&pool0, POOL_META_NAME).is_none());
1608
1609        // node0 retries: every pool is attested under one nonce, so it publishes
1610        // pool.bin and commits the identity on both pools.
1611        establish_pool_meta_bootstrap_identity_if_proven(pools.clone(), &mut node0, true)
1612            .await
1613            .expect("re-establishing an already minted identity is idempotent");
1614        let (_, replica_state) = load_pool_meta_for_startup(pools.clone(), &mut node0)
1615            .await
1616            .expect("complete creator attestation authorizes the initial pool metadata write");
1617        persist_pool_meta_for_startup_if_safe(
1618            &init_test_pool_meta_with_pools(2),
1619            pools.clone(),
1620            replica_state,
1621            &mut node0,
1622            true,
1623            true,
1624        )
1625        .await
1626        .expect("the elected writer publishes pool.bin and commits the identity");
1627        for pool in [&pool0, &pool1] {
1628            assert!(startup_object(pool, POOL_META_NAME).is_some());
1629            let identity = startup_object(pool, POOL_META_IDENTITY_NAME).expect("committed identity");
1630            assert!(pool_meta_identity_initialized_for_test(&identity).expect("decode committed identity"));
1631        }
1632
1633        // node1 retries once more: pool.bin exists and nothing is rewritten.
1634        let before = startup_object(&pool1, POOL_META_IDENTITY_NAME);
1635        establish_pool_meta_bootstrap_identity_if_proven(pools.clone(), &mut node1, false)
1636            .await
1637            .expect("an initialized deployment never reopens bootstrap");
1638        assert_eq!(startup_object(&pool1, POOL_META_IDENTITY_NAME), before);
1639        load_pool_meta_for_startup(pools, &mut node1)
1640            .await
1641            .expect("published pool metadata admits the non-elected node");
1642        node1
1643            .ensure_write_safe("converged non-elected creator")
1644            .expect("no latch remains after convergence");
1645    }
1646
1647    #[tokio::test]
1648    async fn test_two_pool_bootstrap_rejects_pending_replicas_from_different_bootstraps() {
1649        let deployment_id = Uuid::new_v4();
1650        let pool0 = Arc::new(StartupPoolMetaStorage::new(Vec::new()));
1651        let pool1 = Arc::new(StartupPoolMetaStorage::new(Vec::new()));
1652        let pools = vec![pool0.clone(), pool1.clone()];
1653        let (mut node0, _) = two_pool_creator_states(deployment_id);
1654        establish_pool_meta_bootstrap_identity_if_proven(pools.clone(), &mut node0, true)
1655            .await
1656            .expect("the first pool's creator mints the pending identity");
1657        inject_startup_object(
1658            &pool1,
1659            POOL_META_IDENTITY_NAME,
1660            pending_pool_meta_identity_for_test(deployment_id, 1, Uuid::new_v4()).expect("encode foreign pending identity"),
1661        );
1662
1663        let err = load_pool_meta_for_startup(pools.clone(), &mut node0)
1664            .await
1665            .expect_err("a pending replica bound to another bootstrap nonce must fail closed");
1666        let chain = error_chain_text(&err);
1667        assert!(chain.contains("disagree on fresh-bootstrap proof"), "{chain}");
1668        node0
1669            .ensure_write_safe("split bootstrap")
1670            .expect_err("a split bootstrap latches the write gate");
1671        assert!(startup_object(&pool0, POOL_META_NAME).is_none());
1672    }
1673
1674    #[tokio::test]
1675    async fn test_two_pool_bootstrap_treats_corrupt_creator_replica_as_recovery_not_waiting() {
1676        let deployment_id = Uuid::new_v4();
1677        let pool0 = Arc::new(StartupPoolMetaStorage::new(Vec::new()));
1678        let pool1 = Arc::new(StartupPoolMetaStorage::new(Vec::new()));
1679        let pools = vec![pool0.clone(), pool1.clone()];
1680        let (mut node0, _) = two_pool_creator_states(deployment_id);
1681        establish_pool_meta_bootstrap_identity_if_proven(pools.clone(), &mut node0, true)
1682            .await
1683            .expect("the first pool's creator mints the pending identity");
1684        // Keep the on-disk format/version header so the replica classifies as
1685        // corrupt (undecodable payload) rather than as an incompatible format.
1686        let mut corrupt = pending_pool_meta_identity_for_test(deployment_id, 1, Uuid::new_v4()).expect("encode identity");
1687        corrupt.truncate(4);
1688        corrupt.extend_from_slice(b"not a cluster identity");
1689        inject_startup_object(&pool1, POOL_META_IDENTITY_NAME, corrupt);
1690
1691        let err = load_pool_meta_for_startup(pools.clone(), &mut node0)
1692            .await
1693            .expect_err("a corrupt replica is not a creator that is still catching up");
1694        let chain = error_chain_text(&err);
1695        assert!(chain.contains("no verified fresh-bootstrap proof"), "{chain}");
1696        node0
1697            .ensure_write_safe("corrupt attestation")
1698            .expect_err("a corrupt attestation latches the write gate");
1699        assert!(startup_object(&pool0, POOL_META_NAME).is_none());
1700    }
1701
1702    #[tokio::test]
1703    async fn test_elected_restart_without_first_hand_proof_cannot_reuse_a_complete_pending_set() {
1704        let deployment_id = Uuid::new_v4();
1705        let pool0 = Arc::new(StartupPoolMetaStorage::new(Vec::new()));
1706        let pool1 = Arc::new(StartupPoolMetaStorage::new(Vec::new()));
1707        let pools = vec![pool0.clone(), pool1.clone()];
1708        let (mut node0, mut node1) = two_pool_creator_states(deployment_id);
1709        establish_pool_meta_bootstrap_identity_if_proven(pools.clone(), &mut node0, true)
1710            .await
1711            .expect("mint");
1712        establish_pool_meta_bootstrap_identity_if_proven(pools.clone(), &mut node1, false)
1713            .await
1714            .expect("attest");
1715        assert_eq!(
1716            startup_object(&pool0, POOL_META_IDENTITY_NAME),
1717            startup_object(&pool1, POOL_META_IDENTITY_NAME),
1718            "both creators attested the same pending identity"
1719        );
1720
1721        // The elected node restarts before publishing: it now merely reads both
1722        // formats, so the complete pending set alone must not reopen bootstrap.
1723        let mut restarted = PoolMetaWriteState::for_startup_with_pool_bootstrap_authorities(
1724            deployment_id,
1725            vec![PoolMetaBootstrapAuthority::None, PoolMetaBootstrapAuthority::None],
1726            true,
1727        );
1728        establish_pool_meta_bootstrap_identity_if_proven(pools.clone(), &mut restarted, true)
1729            .await
1730            .expect("a restart without first-hand proof writes nothing");
1731        let err = load_pool_meta_for_startup(pools.clone(), &mut restarted)
1732            .await
1733            .expect_err("a pending set alone never authorizes a writer without first-hand proof");
1734        assert!(err.to_string().contains("no verified fresh-bootstrap proof"), "{err}");
1735        restarted
1736            .ensure_write_safe("unproven restart")
1737            .expect_err("the rejected restart latches the write gate");
1738        assert!(startup_object(&pool0, POOL_META_NAME).is_none());
1739    }
1740
1741    #[tokio::test]
1742    async fn test_fresh_pool_joining_an_initialized_deployment_never_reopens_bootstrap() {
1743        let deployment_id = Uuid::new_v4();
1744        let pool0 = Arc::new(StartupPoolMetaStorage::new(Vec::new()));
1745        let mut founder = PoolMetaWriteState::for_startup(deployment_id, true);
1746        establish_pool_meta_bootstrap_identity_if_proven(vec![pool0.clone()], &mut founder, true)
1747            .await
1748            .expect("the founder mints");
1749        let (_, replica_state) = load_pool_meta_for_startup(vec![pool0.clone()], &mut founder)
1750            .await
1751            .expect("the founder may initialize");
1752        persist_pool_meta_for_startup_if_safe(
1753            &init_test_pool_meta(None),
1754            vec![pool0.clone()],
1755            replica_state,
1756            &mut founder,
1757            true,
1758            true,
1759        )
1760        .await
1761        .expect("the founder commits");
1762        let founded = startup_object(&pool0, POOL_META_IDENTITY_NAME).expect("committed identity");
1763        assert!(pool_meta_identity_initialized_for_test(&founded).expect("decode committed identity"));
1764
1765        // Expansion: pool1 is fresh and was formatted first-hand by the node
1766        // hosting its first endpoint, whether or not that node is elected.
1767        let pool1 = Arc::new(StartupPoolMetaStorage::new(Vec::new()));
1768        let pools = vec![pool0.clone(), pool1.clone()];
1769        for elected in [false, true] {
1770            let mut joiner = PoolMetaWriteState::for_startup_with_pool_bootstrap_authorities(
1771                deployment_id,
1772                vec![PoolMetaBootstrapAuthority::None, PoolMetaBootstrapAuthority::Fresh],
1773                elected,
1774            );
1775            establish_pool_meta_bootstrap_identity_if_proven(pools.clone(), &mut joiner, elected)
1776                .await
1777                .expect("an initialized deployment ignores first-hand proof for a new pool");
1778            assert!(
1779                startup_object(&pool1, POOL_META_IDENTITY_NAME).is_none(),
1780                "no pending identity may be written to an expansion pool"
1781            );
1782            assert_eq!(startup_object(&pool0, POOL_META_IDENTITY_NAME).as_deref(), Some(founded.as_slice()));
1783            let (_, replica_state) = load_pool_meta_for_startup(pools.clone(), &mut joiner)
1784                .await
1785                .expect("published pool metadata admits the joiner");
1786            joiner
1787                .ensure_write_safe("expansion joiner")
1788                .expect("joining never latches the write gate");
1789            if elected {
1790                persist_pool_meta_for_startup_if_safe(
1791                    &init_test_pool_meta_with_pools(2),
1792                    pools.clone(),
1793                    replica_state,
1794                    &mut joiner,
1795                    true,
1796                    true,
1797                )
1798                .await
1799                .expect("the topology update repairs the new pool's replicas");
1800                let identity = startup_object(&pool1, POOL_META_IDENTITY_NAME).expect("expansion pool identity");
1801                assert!(pool_meta_identity_initialized_for_test(&identity).expect("decode repaired identity"));
1802                assert!(startup_object(&pool1, POOL_META_NAME).is_some());
1803            }
1804        }
1805    }
1806
1807    #[tokio::test]
1808    async fn test_store_init_distinguishes_fresh_deployment_from_wiped_lagging_node() {
1809        let deployment_id = Uuid::new_v4();
1810        let storage = Arc::new(StartupPoolMetaStorage::new(Vec::new()));
1811        let mut fresh_state = PoolMetaWriteState::for_startup(deployment_id, true);
1812        establish_pool_meta_bootstrap_identity_if_proven(vec![storage.clone()], &mut fresh_state, true)
1813            .await
1814            .expect("fresh topology proof should become a durable pending identity");
1815        let (_, replica_state) = load_pool_meta_for_startup(vec![storage.clone()], &mut fresh_state)
1816            .await
1817            .expect("new formats with no identity may initialize pool metadata exactly once");
1818        let committed = persist_pool_meta_for_startup_if_safe(
1819            &init_test_pool_meta(None),
1820            vec![storage.clone()],
1821            replica_state,
1822            &mut fresh_state,
1823            true,
1824            true,
1825        )
1826        .await
1827        .expect("fresh deployment should durably commit identity and pool metadata");
1828        assert_eq!(committed.pools[0].cmd_line, "pool-0");
1829        {
1830            let objects = storage.objects.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
1831            assert!(objects.contains_key(POOL_META_NAME));
1832            assert!(objects.contains_key(POOL_META_IDENTITY_NAME));
1833        }
1834
1835        storage
1836            .objects
1837            .lock()
1838            .unwrap_or_else(std::sync::PoisonError::into_inner)
1839            .remove(POOL_META_NAME);
1840        let mut wiped_pool_state = PoolMetaWriteState::for_startup(deployment_id, false);
1841        let err = load_pool_meta_for_startup(vec![storage.clone()], &mut wiped_pool_state)
1842            .await
1843            .expect_err("an initialized identity must prevent an empty pool metadata rebuild");
1844        assert!(err.to_string().contains("initialized cluster identity exists"));
1845
1846        storage
1847            .objects
1848            .lock()
1849            .unwrap_or_else(std::sync::PoisonError::into_inner)
1850            .remove(POOL_META_IDENTITY_NAME);
1851        let mut wiped_all_state = PoolMetaWriteState::for_startup(deployment_id, false);
1852        let err = load_pool_meta_for_startup(vec![storage], &mut wiped_all_state)
1853            .await
1854            .expect_err("existing formats without identity or pool metadata must require recovery");
1855        assert!(err.to_string().contains("no durable bootstrap identity"));
1856
1857        let distributed_node = Arc::new(StartupPoolMetaStorage::new(Vec::new()));
1858        let mut non_elected_state = PoolMetaWriteState::for_startup(Uuid::new_v4(), true);
1859        establish_pool_meta_bootstrap_identity_if_proven(vec![distributed_node.clone()], &mut non_elected_state, false)
1860            .await
1861            .expect("a non-elected distributed node must not create bootstrap authority");
1862        let err = load_pool_meta_for_startup(vec![distributed_node], &mut non_elected_state)
1863            .await
1864            .expect_err("a distributed node without durable identity or pool metadata must require recovery");
1865        assert!(err.to_string().contains("no durable bootstrap identity"));
1866    }
1867
1868    #[tokio::test]
1869    async fn test_store_init_resumes_pending_v3_bootstrap_without_legacy_overwrite() {
1870        let deployment_id = Uuid::new_v4();
1871        let storage = Arc::new(StartupPoolMetaStorage::new(Vec::new()));
1872        let mut first_bootstrap = PoolMetaWriteState::for_startup(deployment_id, true);
1873        establish_pool_meta_bootstrap_identity_if_proven(vec![storage.clone()], &mut first_bootstrap, true)
1874            .await
1875            .expect("fresh bootstrap should persist a pending identity");
1876        let (_, replica_state) = load_pool_meta_for_startup(vec![storage.clone()], &mut first_bootstrap)
1877            .await
1878            .expect("the durable pending identity should authorize the initial pool metadata write");
1879
1880        storage.fail_pool_meta_write_at.store(2, Ordering::SeqCst);
1881        persist_pool_meta_for_startup_if_safe(
1882            &init_test_pool_meta(None),
1883            vec![storage.clone()],
1884            replica_state,
1885            &mut first_bootstrap,
1886            true,
1887            true,
1888        )
1889        .await
1890        .expect_err("the injected crash boundary should leave only the V3 prepare record");
1891
1892        let (pending_pool, pending_identity) = {
1893            let objects = storage.objects.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
1894            (
1895                objects
1896                    .get(POOL_META_NAME)
1897                    .map(|(payload, _)| payload.clone())
1898                    .expect("the V3 prepare record should be durable"),
1899                objects
1900                    .get(POOL_META_IDENTITY_NAME)
1901                    .map(|(payload, _)| payload.clone())
1902                    .expect("the bootstrap identity should be durable"),
1903            )
1904        };
1905        assert_eq!(pool_meta_v3_commit_state_for_test(pending_pool).expect("decode pending V3"), (1, false));
1906        assert!(!pool_meta_identity_initialized_for_test(&pending_identity).expect("decode pending identity"));
1907
1908        storage.fail_pool_meta_write_at.store(0, Ordering::SeqCst);
1909        let mut restarted = PoolMetaWriteState::for_startup(deployment_id, false);
1910        let (loaded, replica_state) = load_pool_meta_for_startup(vec![storage.clone()], &mut restarted)
1911            .await
1912            .expect("restart should recover the predecessor embedded in the pending V3 record");
1913        assert!(loaded.pools.is_empty());
1914        assert!(replica_state.needs_repair);
1915        let committed = persist_pool_meta_for_startup_if_safe(
1916            &init_test_pool_meta(None),
1917            vec![storage.clone()],
1918            replica_state,
1919            &mut restarted,
1920            true,
1921            true,
1922        )
1923        .await
1924        .expect("restart should finish generation 1 before promoting the identity");
1925        assert_eq!(committed.version, 3);
1926
1927        let (committed_pool, committed_identity) = {
1928            let objects = storage.objects.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
1929            (
1930                objects
1931                    .get(POOL_META_NAME)
1932                    .map(|(payload, _)| payload.clone())
1933                    .expect("the committed V3 record should be durable"),
1934                objects
1935                    .get(POOL_META_IDENTITY_NAME)
1936                    .map(|(payload, _)| payload.clone())
1937                    .expect("the committed identity should be durable"),
1938            )
1939        };
1940        assert_eq!(
1941            pool_meta_v3_commit_state_for_test(committed_pool).expect("decode committed V3"),
1942            (1, true)
1943        );
1944        assert!(pool_meta_identity_initialized_for_test(&committed_identity).expect("decode committed identity"));
1945        assert!(
1946            storage
1947                .pool_meta_written_versions
1948                .lock()
1949                .unwrap_or_else(std::sync::PoisonError::into_inner)
1950                .iter()
1951                .all(|version| *version == 3),
1952            "bootstrap recovery must never overwrite a pending V3 record with legacy metadata"
1953        );
1954    }
1955
1956    #[tokio::test]
1957    async fn test_store_init_pool_meta_falls_back_from_corrupt_first_replica() {
1958        let corrupt = Arc::new(StartupPoolMetaStorage::new(vec![0, 1, 2]));
1959        let expected = init_test_pool_meta(None);
1960        let backup = Arc::new(StartupPoolMetaStorage::new(startup_pool_meta_payload(&expected)));
1961        let mut write_state = PoolMetaWriteState::default();
1962
1963        let (loaded, replica_state) = load_pool_meta_for_startup(vec![corrupt.clone(), backup.clone()], &mut write_state)
1964            .await
1965            .expect("startup should select the validated backup replica");
1966
1967        assert!(replica_state.needs_repair);
1968        assert!(replica_state.repair_write_safe);
1969        assert_eq!(loaded.pools.len(), 1);
1970        assert_eq!(loaded.pools[0].cmd_line, expected.pools[0].cmd_line);
1971        assert!(corrupt.read_without_lock.load(Ordering::SeqCst));
1972        assert!(backup.read_without_lock.load(Ordering::SeqCst));
1973
1974        persist_pool_meta_for_startup_if_safe(
1975            &loaded,
1976            vec![corrupt.clone(), backup.clone()],
1977            replica_state,
1978            &mut write_state,
1979            false,
1980            true,
1981        )
1982        .await
1983        .expect("the elected startup writer should repair validated corrupt replicas");
1984
1985        let corrupt_write = corrupt
1986            .written_payload
1987            .lock()
1988            .unwrap_or_else(std::sync::PoisonError::into_inner)
1989            .clone()
1990            .expect("corrupt replica should be repaired");
1991        let backup_write = backup
1992            .written_payload
1993            .lock()
1994            .unwrap_or_else(std::sync::PoisonError::into_inner)
1995            .clone()
1996            .expect("backup replica should receive the same canonical snapshot");
1997        assert_eq!(corrupt_write, backup_write);
1998        assert_ne!(corrupt_write, corrupt.read_payload);
1999    }
2000
2001    #[tokio::test]
2002    async fn test_store_init_pool_meta_does_not_repair_unreadable_replica() {
2003        let valid = Arc::new(StartupPoolMetaStorage::new(startup_pool_meta_payload(&init_test_pool_meta(None))));
2004        let unreadable = Arc::new(StartupPoolMetaStorage::unreadable());
2005        let mut write_state = PoolMetaWriteState::default();
2006
2007        let (loaded, replica_state) = load_pool_meta_for_startup(vec![valid.clone(), unreadable.clone()], &mut write_state)
2008            .await
2009            .expect("startup should use a validated replica without overwriting an unreadable copy");
2010        assert!(replica_state.needs_repair);
2011        assert!(!replica_state.repair_write_safe);
2012
2013        persist_pool_meta_for_startup_if_safe(
2014            &loaded,
2015            vec![valid.clone(), unreadable.clone()],
2016            replica_state,
2017            &mut write_state,
2018            false,
2019            true,
2020        )
2021        .await
2022        .expect("an unreadable copy should defer repair when no topology write is needed");
2023        assert!(!valid.wrote_without_lock.load(Ordering::SeqCst));
2024        assert!(!unreadable.wrote_without_lock.load(Ordering::SeqCst));
2025
2026        let err = persist_pool_meta_for_startup_if_safe(
2027            &loaded,
2028            vec![valid.clone(), unreadable.clone()],
2029            replica_state,
2030            &mut write_state,
2031            true,
2032            true,
2033        )
2034        .await
2035        .expect_err("a topology update must not overwrite an unreadable replica");
2036        assert!(err.to_string().contains("cannot overwrite an unreadable replica"));
2037        assert!(!valid.wrote_without_lock.load(Ordering::SeqCst));
2038        assert!(!unreadable.wrote_without_lock.load(Ordering::SeqCst));
2039    }
2040
2041    #[tokio::test]
2042    async fn test_store_init_pool_meta_stays_blocked_after_all_replicas_were_unreadable() {
2043        let unreadable_a = Arc::new(StartupPoolMetaStorage::unreadable());
2044        let unreadable_b = Arc::new(StartupPoolMetaStorage::unreadable());
2045        let mut write_state = PoolMetaWriteState::default();
2046
2047        load_pool_meta_for_startup(vec![unreadable_a, unreadable_b], &mut write_state)
2048            .await
2049            .expect_err("startup must fail when no readable pool metadata replica exists");
2050
2051        let repaired = Arc::new(StartupPoolMetaStorage::new(startup_pool_meta_payload(&init_test_pool_meta(None))));
2052        let (loaded, replica_state) = load_pool_meta_for_startup(vec![repaired.clone()], &mut write_state)
2053            .await
2054            .expect("a later startup retry may read the repaired replica");
2055        assert!(replica_state.repair_write_safe);
2056
2057        let err =
2058            persist_pool_meta_for_startup_if_safe(&loaded, vec![repaired.clone()], replica_state, &mut write_state, true, true)
2059                .await
2060                .expect_err("the same store instance must not write after observing unreadable replicas");
2061        assert!(
2062            err.to_string()
2063                .contains("restart after all replicas are readable and consistent")
2064        );
2065        assert!(!repaired.wrote_without_lock.load(Ordering::SeqCst));
2066    }
2067
2068    #[test]
2069    fn test_pool_first_endpoint_is_local_respects_local_flag() {
2070        let mut local_endpoint = Endpoint::try_from("http://127.0.0.1:9000/data").expect("endpoint should parse");
2071        local_endpoint.is_local = true;
2072        let pool = PoolEndpoints {
2073            legacy: false,
2074            set_count: 1,
2075            drives_per_set: 1,
2076            endpoints: Endpoints::from(vec![local_endpoint]),
2077            cmd_line: "pool-0".to_string(),
2078            platform: String::new(),
2079        };
2080
2081        assert!(pool_first_endpoint_is_local(&pool));
2082    }
2083
2084    #[test]
2085    fn test_pool_first_endpoint_is_local_rejects_missing_endpoint() {
2086        let pool = PoolEndpoints {
2087            legacy: false,
2088            set_count: 1,
2089            drives_per_set: 1,
2090            endpoints: Endpoints::from(Vec::<Endpoint>::new()),
2091            cmd_line: "pool-0".to_string(),
2092            platform: String::new(),
2093        };
2094
2095        assert!(!pool_first_endpoint_is_local(&pool));
2096    }
2097
2098    #[test]
2099    fn test_local_decommission_watchdog_retry_delay_is_bounded() {
2100        assert_eq!(local_decommission_watchdog_retry_delay(1), LOCAL_DECOMMISSION_RESUME_RETRY_DELAY);
2101        assert_eq!(
2102            local_decommission_watchdog_retry_delay(u32::MAX),
2103            LOCAL_DECOMMISSION_WATCHDOG_MAX_RETRY_DELAY
2104        );
2105    }
2106
2107    #[tokio::test(start_paused = true)]
2108    async fn test_local_decommission_watchdog_cancelled_start_does_not_reconcile() {
2109        let rx = CancellationToken::new();
2110        rx.cancel();
2111        run_local_decommission_watchdog(rx, || async {
2112            panic!("cancelled startup must not schedule persisted work");
2113        })
2114        .await;
2115    }
2116
2117    #[tokio::test]
2118    #[serial_test::serial]
2119    async fn test_local_decommission_recovery_waits_for_live_fleet_proof_before_reserving_worker() {
2120        let (_temp_dirs, store, _other_store) = crate::services::rebalance::test_two_pool_stores(None).await;
2121        mark_test_pool_decommissioning(&store, 0).await;
2122        assert!(store.ctx.is_dist_erasure().await);
2123        let worker_rx = CancellationToken::new();
2124
2125        {
2126            let _proof_guard = crate::services::notification_sys::without_cross_pool_fence_fleet_proof_for_test();
2127            let err = super::reconcile_local_decommission_after_init(&store, worker_rx.clone())
2128                .await
2129                .expect_err("cold distributed recovery must wait for live fleet proof");
2130            assert!(
2131                crate::core::pools::is_pool_activation_fleet_proof_error(&err),
2132                "recovery must reach the live fleet proof gate: {err:?}"
2133            );
2134            assert!(store.decommission_cancelers.read().await.iter().all(Option::is_none));
2135            assert!(pool_meta_has_active_decommission(&*store.pool_meta.read().await));
2136        }
2137
2138        super::reconcile_local_decommission_after_init(&store, worker_rx.clone())
2139            .await
2140            .expect("restored fleet proof should admit the persisted worker");
2141        assert!(store.has_active_local_decommission_worker().await);
2142        worker_rx.cancel();
2143    }
2144
2145    #[tokio::test(start_paused = true)]
2146    async fn test_local_decommission_watchdog_retries_general_failures_until_cancelled() {
2147        let rx = CancellationToken::new();
2148        let attempts = Arc::new(AtomicUsize::new(0));
2149        let task = tokio::spawn(run_local_decommission_watchdog(rx.clone(), {
2150            let attempts = attempts.clone();
2151            let rx = rx.clone();
2152            move || {
2153                let attempts = attempts.clone();
2154                let rx = rx.clone();
2155                async move {
2156                    match attempts.fetch_add(1, Ordering::SeqCst) {
2157                        0 => Err(StorageError::other("pool activation requires a live fleet capability proof")),
2158                        1 => Err(StorageError::SlowDown),
2159                        _ => {
2160                            rx.cancel();
2161                            Ok(())
2162                        }
2163                    }
2164                }
2165            }
2166        }));
2167
2168        tokio::task::yield_now().await;
2169        assert_eq!(attempts.load(Ordering::SeqCst), 1);
2170        tokio::time::advance(LOCAL_DECOMMISSION_RESUME_RETRY_DELAY).await;
2171        tokio::task::yield_now().await;
2172        assert_eq!(attempts.load(Ordering::SeqCst), 2);
2173        tokio::time::advance(local_decommission_watchdog_retry_delay(2)).await;
2174        task.await.expect("watchdog task should exit after cancellation");
2175        assert_eq!(attempts.load(Ordering::SeqCst), 3);
2176    }
2177
2178    #[tokio::test(start_paused = true)]
2179    async fn test_local_decommission_watchdog_rescans_after_success() {
2180        let rx = CancellationToken::new();
2181        let attempts = Arc::new(AtomicUsize::new(0));
2182        let task = tokio::spawn(run_local_decommission_watchdog(rx.clone(), {
2183            let attempts = attempts.clone();
2184            let rx = rx.clone();
2185            move || {
2186                let attempts = attempts.clone();
2187                let rx = rx.clone();
2188                async move {
2189                    if attempts.fetch_add(1, Ordering::SeqCst) == 1 {
2190                        rx.cancel();
2191                    }
2192                    Ok(())
2193                }
2194            }
2195        }));
2196
2197        tokio::task::yield_now().await;
2198        assert_eq!(attempts.load(Ordering::SeqCst), 1);
2199        tokio::time::advance(super::LOCAL_DECOMMISSION_WATCHDOG_INTERVAL).await;
2200        task.await.expect("watchdog task should exit after cancellation");
2201        assert_eq!(attempts.load(Ordering::SeqCst), 2);
2202    }
2203
2204    #[test]
2205    fn test_should_retry_format_load_rejects_permanent_corruption() {
2206        assert!(!should_retry_format_load(&StorageError::CorruptedFormat));
2207        assert!(should_retry_format_load(&StorageError::ErasureReadQuorum));
2208        assert!(should_retry_format_load(&StorageError::FirstDiskWait));
2209    }
2210
2211    #[test]
2212    fn test_should_retry_format_load_rejects_permanent_topology_errors() {
2213        for error in [
2214            StorageError::UnsupportedSnsdExpansion { configured_drives: 4 },
2215            StorageError::PoolTopologyMismatch {
2216                stored_drives: 4,
2217                stored_set_drive_count: 4,
2218                configured_drives: 8,
2219                configured_set_drive_count: 8,
2220            },
2221        ] {
2222            assert!(!should_retry_format_load(&error), "topology errors require operator action: {error}");
2223        }
2224        for error in [
2225            StorageError::DiskNotFound,
2226            StorageError::Timeout,
2227            StorageError::RemoteNotInitialized,
2228            StorageError::NotFirstDisk,
2229            StorageError::other(std::io::Error::from(std::io::ErrorKind::ConnectionRefused)),
2230        ] {
2231            assert!(
2232                should_retry_format_load(&error),
2233                "transient failures retain their existing retry path: {error}"
2234            );
2235        }
2236    }
2237
2238    #[test]
2239    fn test_should_auto_start_rebalance_after_init_allows_active_rebalance_without_decommission() {
2240        assert!(should_auto_start_rebalance_after_init(false, true));
2241    }
2242
2243    #[test]
2244    fn test_should_auto_start_rebalance_after_init_rejects_active_decommission() {
2245        assert!(!should_auto_start_rebalance_after_init(true, true));
2246    }
2247
2248    #[test]
2249    fn test_should_auto_start_rebalance_after_init_rejects_terminal_or_missing_rebalance() {
2250        assert!(!should_auto_start_rebalance_after_init(false, false));
2251    }
2252
2253    #[test]
2254    fn test_should_defer_rebalance_auto_start_only_without_distributed_fleet_proof() {
2255        assert!(should_defer_rebalance_auto_start(true, false));
2256        assert!(!should_defer_rebalance_auto_start(true, true));
2257        assert!(!should_defer_rebalance_auto_start(false, false));
2258    }
2259
2260    #[test]
2261    fn test_store_init_recovery_skips_rebalance_when_decommission_metadata_is_active() {
2262        let pool_meta = init_test_pool_meta(Some(PoolDecommissionInfo {
2263            start_time: Some(OffsetDateTime::UNIX_EPOCH),
2264            complete: false,
2265            failed: false,
2266            canceled: false,
2267            ..Default::default()
2268        }));
2269        let rebalance_meta = Some(RebalanceMeta {
2270            pool_stats: vec![RebalanceStats {
2271                participating: true,
2272                info: RebalanceInfo {
2273                    status: RebalStatus::Started,
2274                    ..Default::default()
2275                },
2276                ..Default::default()
2277            }],
2278            ..Default::default()
2279        });
2280
2281        assert!(!should_auto_start_rebalance_after_init(
2282            pool_meta_has_active_decommission(&pool_meta),
2283            rebalance_meta
2284                .as_ref()
2285                .is_some_and(crate::services::rebalance::rebalance_requires_worker_activation)
2286        ));
2287    }
2288
2289    #[test]
2290    fn test_store_init_recovery_allows_active_rebalance_without_decommission() {
2291        let pool_meta = init_test_pool_meta(None);
2292        let rebalance_meta = Some(RebalanceMeta {
2293            pool_stats: vec![RebalanceStats {
2294                participating: true,
2295                info: RebalanceInfo {
2296                    status: RebalStatus::Started,
2297                    ..Default::default()
2298                },
2299                ..Default::default()
2300            }],
2301            ..Default::default()
2302        });
2303
2304        assert!(should_auto_start_rebalance_after_init(
2305            pool_meta_has_active_decommission(&pool_meta),
2306            rebalance_meta
2307                .as_ref()
2308                .is_some_and(crate::services::rebalance::rebalance_requires_worker_activation)
2309        ));
2310    }
2311
2312    #[test]
2313    fn test_store_init_recovery_skips_completed_rebalance_metadata() {
2314        let pool_meta = init_test_pool_meta(None);
2315        let rebalance_meta = Some(RebalanceMeta {
2316            pool_stats: vec![RebalanceStats {
2317                participating: true,
2318                info: RebalanceInfo {
2319                    status: RebalStatus::Completed,
2320                    ..Default::default()
2321                },
2322                ..Default::default()
2323            }],
2324            ..Default::default()
2325        });
2326
2327        assert!(!should_auto_start_rebalance_after_init(
2328            pool_meta_has_active_decommission(&pool_meta),
2329            rebalance_meta
2330                .as_ref()
2331                .is_some_and(crate::services::rebalance::rebalance_requires_worker_activation)
2332        ));
2333    }
2334
2335    #[test]
2336    fn test_resolve_store_init_stage_result_passthrough_ok() {
2337        resolve_store_init_stage_result(Ok(()), "load_rebalance_meta").expect("successful stage should pass through");
2338    }
2339
2340    #[test]
2341    fn test_resolve_store_init_stage_result_wraps_error_context() {
2342        let err = resolve_store_init_stage_result(Err(StorageError::SlowDown), "start_rebalance")
2343            .expect_err("failed stage should be wrapped");
2344        let err_message = err.to_string();
2345        assert!(err_message.contains("store init failed during start_rebalance"));
2346        assert!(err_message.contains(&StorageError::SlowDown.to_string()));
2347    }
2348
2349    #[tokio::test]
2350    async fn test_wait_for_local_decommission_resume_delay_returns_true_after_delay() {
2351        let rx = CancellationToken::new();
2352        assert!(wait_for_local_decommission_resume_delay(&rx, Duration::from_millis(1)).await);
2353    }
2354
2355    #[tokio::test]
2356    async fn test_wait_for_local_decommission_resume_delay_returns_false_when_cancelled() {
2357        let rx = CancellationToken::new();
2358        rx.cancel();
2359        assert!(!wait_for_local_decommission_resume_delay(&rx, Duration::from_secs(1)).await);
2360    }
2361
2362    #[test]
2363    fn test_pool_first_endpoint_is_local_uses_pool_scope_for_expansion() {
2364        let mut remote_endpoint = Endpoint::try_from("http://127.0.0.2:9000/data1").expect("remote endpoint should parse");
2365        remote_endpoint.is_local = false;
2366
2367        let mut local_endpoint = Endpoint::try_from("http://127.0.0.1:9000/data1").expect("local endpoint should parse");
2368        local_endpoint.is_local = true;
2369
2370        let endpoints = EndpointServerPools::from(vec![
2371            PoolEndpoints {
2372                legacy: false,
2373                set_count: 1,
2374                drives_per_set: 1,
2375                endpoints: Endpoints::from(vec![remote_endpoint]),
2376                cmd_line: "pool-0".to_string(),
2377                platform: String::new(),
2378            },
2379            PoolEndpoints {
2380                legacy: false,
2381                set_count: 1,
2382                drives_per_set: 1,
2383                endpoints: Endpoints::from(vec![local_endpoint]),
2384                cmd_line: "pool-1".to_string(),
2385                platform: String::new(),
2386            },
2387        ]);
2388
2389        assert!(!endpoints.first_local(), "cluster first endpoint is intentionally remote");
2390        assert!(
2391            pool_first_endpoint_is_local(endpoints.as_ref().get(1).expect("second pool should exist")),
2392            "the expanded pool should be initialized by its own first local endpoint"
2393        );
2394    }
2395
2396    fn rpc_preflight_pools(pools: Vec<Vec<Endpoint>>) -> EndpointServerPools {
2397        EndpointServerPools::from(
2398            pools
2399                .into_iter()
2400                .enumerate()
2401                .map(|(pool_index, endpoints)| PoolEndpoints {
2402                    legacy: false,
2403                    set_count: 1,
2404                    drives_per_set: endpoints.len(),
2405                    endpoints: Endpoints::from(endpoints),
2406                    cmd_line: format!("pool-{pool_index}"),
2407                    platform: String::new(),
2408                })
2409                .collect::<Vec<_>>(),
2410        )
2411    }
2412
2413    fn parsed_local_endpoint() -> Endpoint {
2414        let mut endpoint = Endpoint::try_from("http://127.0.0.1:9000/data").expect("local endpoint should parse");
2415        endpoint.is_local = true;
2416        endpoint
2417    }
2418
2419    fn parsed_remote_endpoint() -> Endpoint {
2420        let mut endpoint = Endpoint::try_from("http://10.0.0.2:9000/data").expect("remote endpoint should parse");
2421        endpoint.is_local = false;
2422        endpoint
2423    }
2424
2425    #[test]
2426    fn test_rpc_secret_preflight_aborts_for_remote_endpoints_without_secret() {
2427        // The remote endpoint intentionally sits behind a local one: the
2428        // preflight must scan every endpoint, not just the first.
2429        let pools = rpc_preflight_pools(vec![vec![parsed_local_endpoint(), parsed_remote_endpoint()]]);
2430
2431        let err = preflight_startup_rpc_secret_with(&pools, || {
2432            Err(std::io::Error::other(rustfs_credentials::RPC_SECRET_REQUIRED_MESSAGE))
2433        })
2434        .expect_err("remote endpoints without an RPC secret must abort startup before the format retry loop");
2435
2436        let message = err.to_string();
2437        assert!(message.contains("store init aborted"), "unexpected error: {message}");
2438        assert!(
2439            message.contains(rustfs_credentials::RPC_SECRET_REQUIRED_MESSAGE),
2440            "error must state the resolution failure: {message}"
2441        );
2442        assert!(
2443            message.contains(rustfs_credentials::RPC_SECRET_REQUIRED_OPERATOR_MESSAGE),
2444            "error must carry the operator remediation guidance: {message}"
2445        );
2446    }
2447
2448    #[test]
2449    fn test_rpc_secret_preflight_aborts_for_remote_endpoint_in_later_pool() {
2450        // The remote endpoint hides in a later, otherwise-local pool: a
2451        // first-pool shortcut (the `first_local` shape) must not pass.
2452        let pools = rpc_preflight_pools(vec![vec![parsed_local_endpoint()], vec![parsed_remote_endpoint()]]);
2453
2454        let err = preflight_startup_rpc_secret_with(&pools, || {
2455            Err(std::io::Error::other(rustfs_credentials::RPC_SECRET_REQUIRED_MESSAGE))
2456        })
2457        .expect_err("a remote endpoint in a later pool must abort startup without an RPC secret");
2458
2459        let message = err.to_string();
2460        assert!(message.contains("store init aborted"), "unexpected error: {message}");
2461    }
2462
2463    #[test]
2464    fn test_rpc_secret_preflight_skips_resolution_for_local_only_endpoints() {
2465        preflight_startup_rpc_secret_with(&rpc_preflight_pools(vec![vec![parsed_local_endpoint()]]), || {
2466            panic!("single-node topology must not resolve an RPC secret")
2467        })
2468        .expect("local-only topology must start without an RPC secret");
2469    }
2470
2471    #[test]
2472    fn test_rpc_secret_preflight_accepts_remote_endpoints_with_resolved_secret() {
2473        preflight_startup_rpc_secret_with(&rpc_preflight_pools(vec![vec![parsed_remote_endpoint()]]), || {
2474            Ok("resolved-secret".to_string())
2475        })
2476        .expect("a resolvable RPC secret must not block startup");
2477    }
2478
2479    #[tokio::test]
2480    #[serial_test::serial(storage_class_env)]
2481    async fn test_new_with_instance_ctx_aborts_before_format_retry_loop_without_rpc_secret() {
2482        // Guard: under a shared-process `cargo test` run another test may have
2483        // already pinned the process-global RPC secret (ensure_test_rpc_secret),
2484        // which would let init proceed past the preflight into remote disk
2485        // init. A failing probe pins nothing, so it cannot poison later tests;
2486        // under nextest's process-per-test model (CI) the guard never fires.
2487        if rustfs_credentials::try_get_rpc_token().is_ok() {
2488            return;
2489        }
2490
2491        let mut remote_endpoint = parsed_remote_endpoint();
2492        remote_endpoint.set_pool_index(0);
2493        remote_endpoint.set_set_index(0);
2494        remote_endpoint.set_disk_index(0);
2495
2496        let err = crate::store::ECStore::new_with_instance_ctx(
2497            "127.0.0.1:0".parse().expect("test address"),
2498            rpc_preflight_pools(vec![vec![remote_endpoint]]),
2499            CancellationToken::new(),
2500            Arc::new(crate::runtime::instance::InstanceContext::new()),
2501        )
2502        .await
2503        .expect_err("distributed init without an RPC secret must abort before the format retry loop");
2504
2505        let message = err.to_string();
2506        assert!(message.contains("store init aborted"), "unexpected error: {message}");
2507        assert!(
2508            message.contains(rustfs_credentials::RPC_SECRET_REQUIRED_OPERATOR_MESSAGE),
2509            "abort must carry the operator remediation guidance: {message}"
2510        );
2511    }
2512
2513    fn endpoint_pools_with_drive_counts(counts: &[usize]) -> EndpointServerPools {
2514        EndpointServerPools::from(
2515            counts
2516                .iter()
2517                .enumerate()
2518                .map(|(pool_index, &drives_per_set)| PoolEndpoints {
2519                    legacy: false,
2520                    set_count: 1,
2521                    drives_per_set,
2522                    endpoints: Endpoints::from(Vec::new()),
2523                    cmd_line: format!("pool-{pool_index}"),
2524                    platform: String::new(),
2525                })
2526                .collect::<Vec<_>>(),
2527        )
2528    }
2529
2530    #[test]
2531    fn startup_pool_defaults_are_resolved_per_pool() {
2532        let validate = |pools: &EndpointServerPools| {
2533            let drive_counts: Vec<_> = pools.as_ref().iter().map(|pool| pool.drives_per_set).collect();
2534            crate::config::storageclass::lookup_config_for_pools_without_env(&KVS::new(), &drive_counts).map(|_| ())
2535        };
2536        let defaults = resolve_startup_pool_defaults_with(&endpoint_pools_with_drive_counts(&[4, 2]), validate)
2537            .expect("heterogeneous topology should resolve");
2538        assert_eq!(defaults, vec![2, 1]);
2539
2540        let defaults = resolve_startup_pool_defaults_with(&endpoint_pools_with_drive_counts(&[4, 6]), validate)
2541            .expect("heterogeneous topology should resolve");
2542        assert_eq!(defaults, vec![2, 3]);
2543    }
2544
2545    #[test]
2546    fn startup_pool_defaults_validate_explicit_environment_for_every_pool() {
2547        let validate = |pools: &EndpointServerPools| {
2548            let drive_counts: Vec<_> = pools.as_ref().iter().map(|pool| pool.drives_per_set).collect();
2549            let mut kvs = KVS::new();
2550            kvs.insert(crate::config::storageclass::CLASS_STANDARD.to_string(), "EC:2".to_string());
2551            crate::config::storageclass::lookup_config_for_pools_without_env(&kvs, &drive_counts).map(|_| ())
2552        };
2553        let err = resolve_startup_pool_defaults_with(&endpoint_pools_with_drive_counts(&[4, 2]), validate)
2554            .expect_err("explicit EC:2 must fail before any two-drive pool I/O");
2555        assert!(err.to_string().contains("pool 1") && err.to_string().contains("2 drives"));
2556    }
2557
2558    #[test]
2559    #[serial_test::serial(storage_class_env)]
2560    fn startup_pool_defaults_validate_environment_without_changing_metadata_fallback() {
2561        temp_env::with_vars(
2562            [
2563                (crate::config::storageclass::STANDARD_ENV, Some("EC:1")),
2564                (crate::config::storageclass::RRS_ENV, None),
2565                (crate::config::storageclass::OPTIMIZE_ENV, None),
2566                (crate::config::storageclass::INLINE_BLOCK_ENV, None),
2567            ],
2568            || {
2569                let runtime = crate::config::storageclass::lookup_config_for_pools(&KVS::new(), &[6, 4])
2570                    .expect("explicit standard parity must resolve the runtime candidate");
2571                assert_eq!(runtime.parities_for_sc(crate::config::storageclass::STANDARD), Some(vec![1, 1]));
2572
2573                let defaults = super::resolve_startup_pool_defaults(&endpoint_pools_with_drive_counts(&[6, 4]))
2574                    .expect("explicit standard parity must validate for every pool");
2575                assert_eq!(defaults, vec![3, 2]);
2576            },
2577        );
2578    }
2579
2580    async fn without_storage_class_env<F: Future>(future: F) -> F::Output {
2581        temp_env::async_with_vars(
2582            [
2583                (crate::config::storageclass::STANDARD_ENV, None::<&str>),
2584                (crate::config::storageclass::RRS_ENV, None::<&str>),
2585                (crate::config::storageclass::OPTIMIZE_ENV, None::<&str>),
2586                (crate::config::storageclass::INLINE_BLOCK_ENV, None::<&str>),
2587            ],
2588            future,
2589        )
2590        .await
2591    }
2592
2593    // Build a real local store over a temp dir around a fresh instance context.
2594    async fn build_isolated_test_store(
2595        temp_dir: &std::path::Path,
2596        cmd_line: &str,
2597        pool_drive_counts: &[usize],
2598    ) -> (
2599        Arc<crate::runtime::instance::InstanceContext>,
2600        Arc<crate::store::ECStore>,
2601        CancellationToken,
2602    ) {
2603        build_isolated_test_store_with_shutdown(temp_dir, cmd_line, pool_drive_counts, CancellationToken::new()).await
2604    }
2605
2606    async fn build_isolated_test_store_with_shutdown(
2607        temp_dir: &std::path::Path,
2608        cmd_line: &str,
2609        pool_drive_counts: &[usize],
2610        shutdown: CancellationToken,
2611    ) -> (
2612        Arc<crate::runtime::instance::InstanceContext>,
2613        Arc<crate::store::ECStore>,
2614        CancellationToken,
2615    ) {
2616        let pool_layouts = pool_drive_counts
2617            .iter()
2618            .map(|&drives_per_set| (1, drives_per_set))
2619            .collect::<Vec<_>>();
2620        build_isolated_test_store_with_layout(temp_dir, cmd_line, &pool_layouts, shutdown, None).await
2621    }
2622
2623    async fn build_isolated_test_store_with_layout(
2624        temp_dir: &std::path::Path,
2625        cmd_line: &str,
2626        pool_layouts: &[(usize, usize)],
2627        shutdown: CancellationToken,
2628        instance_ctx: Option<Arc<crate::runtime::instance::InstanceContext>>,
2629    ) -> (
2630        Arc<crate::runtime::instance::InstanceContext>,
2631        Arc<crate::store::ECStore>,
2632        CancellationToken,
2633    ) {
2634        let mut pools = Vec::with_capacity(pool_layouts.len());
2635        for (pool_index, &(set_count, drives_per_set)) in pool_layouts.iter().enumerate() {
2636            let mut endpoints = Vec::with_capacity(set_count * drives_per_set);
2637            for set_index in 0..set_count {
2638                for disk_index in 0..drives_per_set {
2639                    let path = temp_dir.join(format!("pool{pool_index}/set{set_index}/disk{disk_index}"));
2640                    tokio::fs::create_dir_all(&path).await.expect("create disk dir");
2641                    let mut endpoint =
2642                        Endpoint::try_from(path.to_str().expect("disk path should be utf-8")).expect("local endpoint");
2643                    endpoint.set_pool_index(pool_index);
2644                    endpoint.set_set_index(set_index);
2645                    endpoint.set_disk_index(disk_index);
2646                    endpoints.push(endpoint);
2647                }
2648            }
2649            pools.push(PoolEndpoints {
2650                legacy: false,
2651                set_count,
2652                drives_per_set,
2653                endpoints: Endpoints::from(endpoints),
2654                cmd_line: format!("{cmd_line}-pool-{pool_index}"),
2655                platform: "test".to_string(),
2656            });
2657        }
2658        let endpoint_pools = EndpointServerPools(pools);
2659        crate::services::notification_sys::install_cross_pool_fence_fleet_proof_for_test();
2660
2661        let instance_ctx = instance_ctx.unwrap_or_else(|| Arc::new(crate::runtime::instance::InstanceContext::new()));
2662        crate::store::init_local_disks_with_instance_ctx(&instance_ctx, endpoint_pools.clone())
2663            .await
2664            .expect("register local disks into the fresh context");
2665
2666        let store = crate::store::ECStore::new_with_instance_ctx(
2667            "127.0.0.1:0".parse().expect("test address"),
2668            endpoint_pools,
2669            shutdown.clone(),
2670            instance_ctx.clone(),
2671        )
2672        .await
2673        .expect("store should build around the fresh context");
2674
2675        // Capacity admission in these local fixtures must not depend on the
2676        // host volume's statvfs values. Keep enough identical snapshots for
2677        // startup, recovery, and the mutation probes exercised by each test.
2678        let layout = DecommissionErasureLayout { data: 2, parity: 2 };
2679        let snapshot: Vec<DecommissionPoolCapacityInfo> = store
2680            .pools
2681            .iter()
2682            .enumerate()
2683            .map(|(pool_index, _)| DecommissionPoolCapacityInfo::for_test(pool_index, layout, 1 << 40, 1 << 40, 1 << 30))
2684            .collect();
2685        set_decommission_capacity_info_overrides_for_test(store.id, (0..128).map(|_| snapshot.clone()).collect());
2686
2687        (instance_ctx, store, shutdown)
2688    }
2689
2690    #[cfg(feature = "test-util")]
2691    #[tokio::test]
2692    #[serial_test::serial(storage_class_env)]
2693    async fn cross_key_materialized_copy_does_not_publish_remote_tuple_ownership() {
2694        let temp_dir = tempfile::tempdir().expect("create materialized-copy store dir");
2695        let (ctx, store, shutdown) =
2696            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "copy-materialized", &[4])).await;
2697        crate::bucket::metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
2698
2699        let bucket = format!("copy-materialized-{}", Uuid::new_v4());
2700        let source_object = "source-restored.bin";
2701        let target_object = "target-local.bin";
2702        let payload = b"materialized copy must own only its new local bytes".to_vec();
2703        store
2704            .make_bucket(&bucket, &MakeBucketOptions::default())
2705            .await
2706            .expect("copy test bucket should be created");
2707
2708        let tier_name = "COPY-MATERIALIZED-TIER";
2709        register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
2710        let lease = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
2711            .await
2712            .expect("mock tier generation should be available");
2713        let backend_identity = lease.backend_identity();
2714        drop(lease);
2715
2716        let mut source_metadata = HashMap::from([
2717            ("content-type".to_string(), "application/octet-stream".to_string()),
2718            (
2719                "x-amz-restore".to_string(),
2720                "ongoing-request=\"false\", expiry-date=\"2099-01-01T00:00:00Z\"".to_string(),
2721            ),
2722        ]);
2723        for (suffix, value) in [
2724            (
2725                rustfs_utils::http::SUFFIX_TRANSITION_STATUS,
2726                crate::bucket::lifecycle::core::TRANSITION_COMPLETE.to_string(),
2727            ),
2728            (
2729                rustfs_utils::http::SUFFIX_TRANSITIONED_OBJECTNAME,
2730                "remote/source-restored.bin".to_string(),
2731            ),
2732            (rustfs_utils::http::SUFFIX_TRANSITION_TIER, tier_name.to_string()),
2733            (rustfs_utils::http::SUFFIX_TRANSITIONED_VERSION_ID, "remote-source-version".to_string()),
2734            (
2735                rustfs_utils::http::SUFFIX_TRANSITIONED_VERSION_STATE,
2736                rustfs_filemeta::TransitionVersionState::Exact.as_str().to_string(),
2737            ),
2738            (
2739                rustfs_utils::http::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
2740                rustfs_utils::crypto::hex(backend_identity),
2741            ),
2742            (rustfs_utils::http::SUFFIX_TRANSITION_TRANSACTION_ID, Uuid::new_v4().to_string()),
2743        ] {
2744            rustfs_utils::http::metadata_compat::insert_str(&mut source_metadata, suffix, value);
2745        }
2746
2747        let mut source_body = PutObjReader::from_vec(payload.clone());
2748        store
2749            .put_object(
2750                &bucket,
2751                source_object,
2752                &mut source_body,
2753                &ObjectOptions {
2754                    user_defined: source_metadata,
2755                    ..Default::default()
2756                },
2757            )
2758            .await
2759            .expect("restored transitioned source should be written");
2760
2761        let source_opts = ObjectOptions::default();
2762        let mut source_info = store
2763            .get_object_info(&bucket, source_object, &source_opts)
2764            .await
2765            .expect("source metadata should be readable");
2766        assert_eq!(
2767            source_info.transitioned_object.status,
2768            crate::bucket::lifecycle::core::TRANSITION_COMPLETE
2769        );
2770        assert_eq!(source_info.transitioned_object.tier, tier_name);
2771        assert!(source_info.data_dir.is_some(), "restored source must retain local data");
2772        source_info.put_object_reader = Some(PutObjReader::from_vec(payload.clone()));
2773
2774        let copied = store
2775            .copy_object(
2776                &bucket,
2777                source_object,
2778                &bucket,
2779                target_object,
2780                &mut source_info,
2781                &source_opts,
2782                &ObjectOptions::default(),
2783            )
2784            .await
2785            .expect("cross-key materialized copy should succeed");
2786        assert!(copied.transitioned_object.status.is_empty());
2787        assert!(copied.transitioned_object.name.is_empty());
2788        assert!(copied.transitioned_object.version_id.is_empty());
2789        assert!(copied.transitioned_object.tier.is_empty());
2790        assert!(!copied.transitioned_object.free_version);
2791        for suffix in [
2792            rustfs_utils::http::SUFFIX_TRANSITION_STATUS,
2793            rustfs_utils::http::SUFFIX_TRANSITIONED_OBJECTNAME,
2794            rustfs_utils::http::SUFFIX_TRANSITIONED_VERSION_ID,
2795            rustfs_utils::http::SUFFIX_TRANSITIONED_VERSION_STATE,
2796            rustfs_utils::http::SUFFIX_TRANSITION_TIER,
2797            rustfs_utils::http::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
2798            rustfs_utils::http::SUFFIX_TRANSITION_TRANSACTION_ID,
2799        ] {
2800            assert!(
2801                !rustfs_utils::http::metadata_compat::contains_key_str(copied.user_defined.as_ref(), suffix),
2802                "target retained protected source suffix {suffix}"
2803            );
2804        }
2805        assert_eq!(
2806            copied.user_defined.get("content-type").map(String::as_str),
2807            Some("application/octet-stream")
2808        );
2809
2810        let mut target_reader = store
2811            .get_object_reader(&bucket, target_object, None, HeaderMap::new(), &ObjectOptions::default())
2812            .await
2813            .expect("target object should be readable");
2814        let mut target_body = Vec::new();
2815        target_reader
2816            .stream
2817            .read_to_end(&mut target_body)
2818            .await
2819            .expect("target body should stream");
2820        assert_eq!(target_body, payload);
2821
2822        let source_after = store
2823            .get_object_info(&bucket, source_object, &source_opts)
2824            .await
2825            .expect("source metadata should remain readable after copy");
2826        assert_eq!(
2827            source_after.transitioned_object.status,
2828            crate::bucket::lifecycle::core::TRANSITION_COMPLETE
2829        );
2830        assert_eq!(source_after.transitioned_object.tier, tier_name);
2831        assert_eq!(source_after.transitioned_object.name, "remote/source-restored.bin");
2832        shutdown.cancel();
2833    }
2834
2835    #[cfg(feature = "test-util")]
2836    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2837    #[serial_test::serial(storage_class_env)]
2838    async fn tier_overwrite_put_and_self_copy_recover_persisted_cleanup_owners() {
2839        use crate::bucket::lifecycle::bucket_lifecycle_ops::ExpiryState;
2840        use crate::bucket::lifecycle::tier_free_version_recovery::recover_tier_free_versions;
2841        use rustfs_filemeta::TransitionVersionState::{Exact, KnownDisabled, SuspendedNull};
2842        use rustfs_s3_client::transition_api::ReaderImpl;
2843        use rustfs_utils::http::{
2844            SUFFIX_TRANSITION_STATUS, SUFFIX_TRANSITION_TIER, SUFFIX_TRANSITION_TIER_DESTINATION_ID,
2845            SUFFIX_TRANSITIONED_OBJECTNAME, SUFFIX_TRANSITIONED_VERSION_ID, SUFFIX_TRANSITIONED_VERSION_STATE, insert_str,
2846        };
2847
2848        let temp_dir = tempfile::tempdir().expect("create tier overwrite store");
2849        let (mut ctx, mut store, mut shutdown) =
2850            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "tier-overwrite", &[4])).await;
2851        crate::bucket::metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
2852        let tier = "OVERWRITE-TIER";
2853        let backend = register_mock_tier(&ctx.tier_config_mgr(), tier).await;
2854        let lease = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier)
2855            .await
2856            .expect("tier identity");
2857        let identity = rustfs_utils::crypto::hex(lease.backend_identity());
2858        drop(lease);
2859
2860        for state in [Exact, KnownDisabled, SuspendedNull] {
2861            for suspended in [false, true] {
2862                for self_copy in [false, true] {
2863                    let bucket = format!("tier-overwrite-{}", Uuid::new_v4());
2864                    let object = "object";
2865                    let remote = format!("remote/{bucket}");
2866                    let version = match state {
2867                        Exact => "opaque-overwrite-version",
2868                        SuspendedNull => "null",
2869                        _ => "",
2870                    };
2871                    let payload = vec![0x5b; if suspended { 512 * 1024 } else { 257 }];
2872                    store
2873                        .make_bucket(&bucket, &MakeBucketOptions::default())
2874                        .await
2875                        .expect("create bucket");
2876                    backend.set_put_remote_version(Some(version.to_string())).await;
2877                    let lease = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier)
2878                        .await
2879                        .expect("seed tier lease");
2880                    lease
2881                        .put(
2882                            &remote,
2883                            ReaderImpl::Body(bytes::Bytes::from(payload.clone())),
2884                            payload.len().try_into().expect("payload size"),
2885                        )
2886                        .await
2887                        .expect("seed remote bytes");
2888                    drop(lease);
2889                    let mut metadata = HashMap::from([
2890                        ("content-type".to_string(), "application/octet-stream".to_string()),
2891                        (
2892                            "x-amz-restore".to_string(),
2893                            "ongoing-request=\"false\", expiry-date=\"2099-01-01T00:00:00Z\"".to_string(),
2894                        ),
2895                    ]);
2896                    for (suffix, value) in [
2897                        (SUFFIX_TRANSITION_STATUS, "complete"),
2898                        (SUFFIX_TRANSITION_TIER, tier),
2899                        (SUFFIX_TRANSITION_TIER_DESTINATION_ID, identity.as_str()),
2900                        (SUFFIX_TRANSITIONED_OBJECTNAME, remote.as_str()),
2901                        (SUFFIX_TRANSITIONED_VERSION_STATE, state.as_str()),
2902                    ] {
2903                        insert_str(&mut metadata, suffix, value.to_string());
2904                    }
2905                    if !version.is_empty() {
2906                        insert_str(&mut metadata, SUFFIX_TRANSITIONED_VERSION_ID, version.to_string());
2907                    }
2908                    let options = ObjectOptions {
2909                        version_suspended: suspended,
2910                        ..Default::default()
2911                    };
2912                    store
2913                        .put_object(
2914                            &bucket,
2915                            object,
2916                            &mut PutObjReader::from_vec(payload.clone()),
2917                            &ObjectOptions {
2918                                user_defined: metadata,
2919                                ..options.clone()
2920                            },
2921                        )
2922                        .await
2923                        .expect("seed transitioned source with locally restored bytes");
2924                    // The overwrite queues its committed cleanup owner right
2925                    // away, and from the second iteration on the restarted
2926                    // store already runs expiry workers. Fail that first remote
2927                    // DELETE so the owner stays durable and the restart below
2928                    // still has to rediscover it from xl.meta.
2929                    backend.set_remove_failure(true);
2930                    let expected = if self_copy {
2931                        payload.clone()
2932                    } else {
2933                        vec![0x73; payload.len()]
2934                    };
2935                    let new_metadata = HashMap::from([
2936                        ("content-type".to_string(), "text/plain".to_string()),
2937                        ("x-amz-meta-replacement".to_string(), "kept".to_string()),
2938                    ]);
2939                    if self_copy {
2940                        let mut source = store
2941                            .get_object_info(&bucket, object, &options)
2942                            .await
2943                            .expect("self-copy source");
2944                        source.metadata_only = false;
2945                        source.user_defined = Arc::new(new_metadata);
2946                        source.put_object_reader = Some(PutObjReader::from_vec(expected.clone()));
2947                        store
2948                            .copy_object(&bucket, object, &bucket, object, &mut source, &options, &options)
2949                            .await
2950                            .expect("materialized self-copy");
2951                    } else {
2952                        store
2953                            .put_object(
2954                                &bucket,
2955                                object,
2956                                &mut PutObjReader::from_vec(expected.clone()),
2957                                &ObjectOptions {
2958                                    user_defined: new_metadata,
2959                                    ..options.clone()
2960                                },
2961                            )
2962                            .await
2963                            .expect("overwrite transitioned null version");
2964                    }
2965
2966                    let set = store.pools[0].get_disks_by_key(object);
2967                    let versions = set
2968                        .load_file_info_versions_exact(&bucket, object)
2969                        .await
2970                        .expect("read committed disk metadata")
2971                        .expect("replacement metadata exists");
2972                    let free: Vec<_> = versions
2973                        .versions
2974                        .iter()
2975                        .chain(versions.free_versions.iter())
2976                        .filter(|fi| fi.tier_free_version())
2977                        .collect();
2978                    assert_eq!(free.len(), 1, "{state:?}, suspended={suspended}, copy={self_copy}");
2979                    assert_eq!(free[0].transitioned_objname, remote);
2980                    assert_eq!(free[0].transition_version_state, state);
2981                    assert!(backend.contains(&remote).await, "commit must not delete remote bytes before cleanup");
2982                    let removed_before = backend.remove_count().await;
2983
2984                    // Later matrix cases reuse this restarted store. If its
2985                    // expiry worker has already dequeued this failed cleanup,
2986                    // drain it before cancelling the context so the next
2987                    // store cannot inherit object locks that block recovery.
2988                    wait_for_expiry_workers_idle(&store).await;
2989
2990                    // Restart after the failed cleanup attempt. The new
2991                    // runtime must reconstruct ownership from committed
2992                    // xl.meta regardless of whether the old queue delivered it.
2993                    let tier_config = ctx
2994                        .tier_config_mgr()
2995                        .read()
2996                        .await
2997                        .tiers
2998                        .get(tier)
2999                        .expect("tier configuration survives restart")
3000                        .clone_with_credentials();
3001                    drop(set);
3002                    shutdown.cancel();
3003                    drop(store);
3004                    drop(ctx);
3005                    (ctx, store, shutdown) =
3006                        without_storage_class_env(build_isolated_test_store(temp_dir.path(), "tier-overwrite-restart", &[4]))
3007                            .await;
3008                    crate::bucket::metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
3009                    {
3010                        let manager = ctx.tier_config_mgr();
3011                        let mut manager = manager.write().await;
3012                        manager.tiers.insert(tier.to_string(), tier_config);
3013                        manager
3014                            .install_test_driver(tier, Box::new(backend.clone()))
3015                            .expect("rebind the same remote destination after restart");
3016                    }
3017                    let set = store.pools[0].get_disks_by_key(object);
3018                    // A deferred first cleanup must retain its durable owner
3019                    // until a later recovery scan can retry the operation.
3020                    backend.set_remove_failure(true);
3021                    ExpiryState::resize_workers(1, Arc::clone(&store)).await;
3022                    let recovered = recover_tier_free_versions(Arc::clone(&store), 100, None, None)
3023                        .await
3024                        .expect("recover persisted cleanup owner");
3025                    assert!(recovered.enqueued >= 1);
3026                    wait_for_expiry_workers_idle(&store).await;
3027                    assert!(backend.contains(&remote).await, "failed cleanup must retain remote bytes");
3028                    assert_eq!(backend.remove_count().await, removed_before);
3029                    backend.set_remove_failure(false);
3030                    // This fixture starts expiry workers without the runtime's
3031                    // recovery loop, so drive its durable rescan explicitly.
3032                    eprintln!("Recovering overwrite cleanup: state={state:?}, suspended={suspended}, copy={self_copy}");
3033                    wait_for_tier_free_version_recovery(Arc::clone(&store), &backend, removed_before + 1).await;
3034                    let versions = set
3035                        .load_file_info_versions_exact(&bucket, object)
3036                        .await
3037                        .expect("read cleanup progress")
3038                        .expect("new object must survive cleanup");
3039                    assert!(
3040                        versions
3041                            .versions
3042                            .iter()
3043                            .chain(versions.free_versions.iter())
3044                            .all(|fi| !fi.tier_free_version()),
3045                        "cleanup must remove its owner: {state:?}, suspended={suspended}, copy={self_copy}"
3046                    );
3047                    assert!(!backend.contains(&remote).await);
3048                    assert_eq!(backend.remove_count().await, removed_before + 1, "one successful remote DELETE per owner");
3049                    assert_eq!(backend.remove_versions().await.last(), Some(&(remote.clone(), version.to_string())));
3050                    let mut reader = store
3051                        .get_object_reader(&bucket, object, None, HeaderMap::new(), &options)
3052                        .await
3053                        .expect("replacement remains readable");
3054                    let mut actual = Vec::new();
3055                    reader.stream.read_to_end(&mut actual).await.expect("read replacement bytes");
3056                    assert_eq!(actual, expected);
3057                    let current = store
3058                        .get_object_info(&bucket, object, &options)
3059                        .await
3060                        .expect("replacement metadata");
3061                    assert_eq!(current.user_defined.get("content-type").map(String::as_str), Some("text/plain"));
3062                    assert_eq!(current.user_defined.get("x-amz-meta-replacement").map(String::as_str), Some("kept"));
3063                    assert!(current.transitioned_object.status.is_empty());
3064                }
3065            }
3066        }
3067        shutdown.cancel();
3068    }
3069
3070    #[cfg(feature = "test-util")]
3071    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
3072    #[serial_test::serial(storage_class_env)]
3073    async fn copy_object_immediately_reads_small_completed_multipart_source() {
3074        let temp_dir = tempfile::tempdir().expect("create small multipart copy store dir");
3075        let (_ctx, store, shutdown) =
3076            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "small-multipart-copy", &[1])).await;
3077        crate::bucket::metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
3078
3079        let bucket = format!("small-multipart-copy-{}", Uuid::new_v4());
3080        let source_object = "docker/registry/v2/repositories/example/_uploads/upload-id/data";
3081        let target_object = "docker/registry/v2/blobs/sha256/c0/digest/data";
3082        let payload = vec![0xAB; 273];
3083
3084        store
3085            .make_bucket(&bucket, &MakeBucketOptions::default())
3086            .await
3087            .expect("create bucket for small multipart copy");
3088        let upload = store
3089            .new_multipart_upload(&bucket, source_object, &ObjectOptions::default())
3090            .await
3091            .expect("create source multipart upload");
3092        let mut part_reader = PutObjReader::from_vec(payload.clone());
3093        let part = store
3094            .put_object_part(&bucket, source_object, &upload.upload_id, 1, &mut part_reader, &ObjectOptions::default())
3095            .await
3096            .expect("stage small multipart source part");
3097        let completed = store
3098            .clone()
3099            .complete_multipart_upload(
3100                &bucket,
3101                source_object,
3102                &upload.upload_id,
3103                vec![crate::storage_api_contracts::multipart::CompletePart {
3104                    part_num: part.part_num,
3105                    etag: part.etag,
3106                    ..Default::default()
3107                }],
3108                &ObjectOptions::default(),
3109            )
3110            .await
3111            .expect("complete the small multipart source");
3112        assert_eq!(completed.get_actual_size().expect("completed object logical size"), payload.len() as i64);
3113
3114        let source_reader = store
3115            .get_object_reader(&bucket, source_object, None, HeaderMap::new(), &ObjectOptions::default())
3116            .await
3117            .expect("completed multipart source should be immediately readable");
3118        let mut copy_info = source_reader.object_info.clone();
3119        let actual_size = copy_info.get_actual_size().expect("copy source logical size should resolve");
3120        assert_eq!(actual_size, payload.len() as i64);
3121        let copy_reader = rustfs_rio::HashReader::from_stream(source_reader.stream, actual_size, actual_size, None, None, false)
3122            .expect("copy source hash reader should build");
3123        copy_info.put_object_reader = Some(PutObjReader::new(copy_reader));
3124
3125        store
3126            .copy_object(
3127                &bucket,
3128                source_object,
3129                &bucket,
3130                target_object,
3131                &mut copy_info,
3132                &ObjectOptions::default(),
3133                &ObjectOptions::default(),
3134            )
3135            .await
3136            .expect("CopyObject should accept a freshly completed multipart source");
3137
3138        let mut target_reader = store
3139            .get_object_reader(&bucket, target_object, None, HeaderMap::new(), &ObjectOptions::default())
3140            .await
3141            .expect("copied target should be readable");
3142        let mut target_body = Vec::new();
3143        target_reader
3144            .stream
3145            .read_to_end(&mut target_body)
3146            .await
3147            .expect("target body should stream");
3148        assert_eq!(target_body, payload);
3149        shutdown.cancel();
3150    }
3151
3152    #[cfg(feature = "test-util")]
3153    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
3154    #[serial_test::serial(storage_class_env)]
3155    async fn unfinished_multipart_upload_is_not_copy_source_readable() {
3156        let temp_dir = tempfile::tempdir().expect("create unfinished multipart copy store dir");
3157        let (_ctx, store, shutdown) =
3158            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "unfinished-multipart-copy", &[1])).await;
3159        crate::bucket::metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
3160
3161        let bucket = format!("unfinished-multipart-copy-{}", Uuid::new_v4());
3162        let source_object = "docker/registry/v2/repositories/example/_uploads/upload-id/data";
3163        let payload = vec![0xCD; 273];
3164
3165        store
3166            .make_bucket(&bucket, &MakeBucketOptions::default())
3167            .await
3168            .expect("create bucket for unfinished multipart copy source");
3169        let upload = store
3170            .new_multipart_upload(&bucket, source_object, &ObjectOptions::default())
3171            .await
3172            .expect("create source multipart upload");
3173        let mut part_reader = PutObjReader::from_vec(payload);
3174        store
3175            .put_object_part(&bucket, source_object, &upload.upload_id, 1, &mut part_reader, &ObjectOptions::default())
3176            .await
3177            .expect("stage unfinished multipart source part");
3178
3179        let source_err = match store
3180            .get_object_reader(&bucket, source_object, None, HeaderMap::new(), &ObjectOptions::default())
3181            .await
3182        {
3183            Ok(_) => panic!("an uncompleted multipart upload must not be readable as a CopyObject source"),
3184            Err(err) => err,
3185        };
3186        assert!(
3187            matches!(
3188                source_err,
3189                StorageError::ObjectNotFound(_, _) | StorageError::FileNotFound | StorageError::VersionNotFound(_, _, _)
3190            ),
3191            "unexpected unfinished multipart source error: {source_err:?}"
3192        );
3193
3194        shutdown.cancel();
3195    }
3196
3197    #[tokio::test]
3198    #[serial_test::serial(storage_class_env)]
3199    async fn pool_metadata_preflight_recovery_preserves_single_and_multi_pool_public_mutations() {
3200        for layout in [vec![4], vec![4, 4]] {
3201            let temp_dir = tempfile::tempdir().unwrap();
3202            let (_ctx, store, shutdown) =
3203                without_storage_class_env(build_isolated_test_store(temp_dir.path(), "pool-meta-retry", &layout)).await;
3204            crate::bucket::metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
3205            let bucket = format!("pool-meta-retry-{}", Uuid::new_v4());
3206            store.make_bucket(&bucket, &MakeBucketOptions::default()).await.unwrap();
3207            let mut saved_disks = Vec::new();
3208            for set in &store.pools[0].disk_set {
3209                let mut disks = set.disks.write().await;
3210                let count = disks.len();
3211                saved_disks.push((set.clone(), std::mem::replace(&mut *disks, vec![None; count])));
3212            }
3213            let indices = (0..layout.len()).collect::<Vec<_>>();
3214            let err = store.save_current_pool_meta_for_test(&indices).await.unwrap_err();
3215            assert_eq!(
3216                err.pool_metadata_failure().unwrap().kind,
3217                crate::error::PoolMetadataFailure::ReadUnavailable
3218            );
3219            for (set, disks) in saved_disks {
3220                *set.disks.write().await = disks;
3221            }
3222            store.save_current_pool_meta_for_test(&indices).await.unwrap();
3223            assert!(store.pool_meta_writes_ready().await);
3224
3225            let payload = b"pool metadata recovery payload".to_vec();
3226            store
3227                .put_object(&bucket, "put", &mut PutObjReader::from_vec(payload.clone()), &ObjectOptions::default())
3228                .await
3229                .unwrap();
3230            let mut reader = store
3231                .get_object_reader(&bucket, "put", None, HeaderMap::new(), &ObjectOptions::default())
3232                .await
3233                .unwrap();
3234            let mut actual = Vec::new();
3235            reader.stream.read_to_end(&mut actual).await.unwrap();
3236            assert_eq!(actual, payload);
3237            drop(reader);
3238            store.delete_object(&bucket, "put", ObjectOptions::default()).await.unwrap();
3239            assert!(crate::error::is_err_object_not_found(
3240                &store
3241                    .get_object_info(&bucket, "put", &ObjectOptions::default())
3242                    .await
3243                    .unwrap_err()
3244            ));
3245
3246            let upload = store
3247                .new_multipart_upload(&bucket, "multipart", &ObjectOptions::default())
3248                .await
3249                .unwrap();
3250            let part = store
3251                .put_object_part(
3252                    &bucket,
3253                    "multipart",
3254                    &upload.upload_id,
3255                    1,
3256                    &mut PutObjReader::from_vec(payload.clone()),
3257                    &ObjectOptions::default(),
3258                )
3259                .await
3260                .unwrap();
3261            store
3262                .clone()
3263                .complete_multipart_upload(
3264                    &bucket,
3265                    "multipart",
3266                    &upload.upload_id,
3267                    vec![crate::storage_api_contracts::multipart::CompletePart {
3268                        part_num: part.part_num,
3269                        etag: part.etag,
3270                        ..Default::default()
3271                    }],
3272                    &ObjectOptions::default(),
3273                )
3274                .await
3275                .unwrap();
3276            let mut reader = store
3277                .get_object_reader(&bucket, "multipart", None, HeaderMap::new(), &ObjectOptions::default())
3278                .await
3279                .unwrap();
3280            actual.clear();
3281            reader.stream.read_to_end(&mut actual).await.unwrap();
3282            assert_eq!(actual, payload);
3283            drop(reader);
3284            let upload = store
3285                .new_multipart_upload(&bucket, "abort", &ObjectOptions::default())
3286                .await
3287                .unwrap();
3288            store
3289                .abort_multipart_upload(&bucket, "abort", &upload.upload_id, &ObjectOptions::default())
3290                .await
3291                .unwrap();
3292            assert!(store.pool_meta_writes_ready().await);
3293            shutdown.cancel();
3294        }
3295    }
3296
3297    #[cfg(feature = "test-util")]
3298    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
3299    #[serial_test::serial(storage_class_env)]
3300    async fn complete_multipart_waits_for_tail_rename_before_copy_source_visibility() {
3301        let temp_dir = tempfile::tempdir().expect("create early-ack multipart copy store dir");
3302        let (_ctx, store, shutdown) =
3303            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "early-ack-multipart-copy", &[4])).await;
3304        crate::bucket::metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
3305
3306        let bucket = format!("early-ack-multipart-copy-{}", Uuid::new_v4());
3307        let source_object = "docker/registry/v2/repositories/example/_uploads/upload-id/data";
3308        let target_object = "docker/registry/v2/blobs/sha256/c0/digest/data";
3309        let payload = vec![0xCD; 273];
3310
3311        store
3312            .make_bucket(&bucket, &MakeBucketOptions::default())
3313            .await
3314            .expect("create bucket for early-ack multipart copy");
3315        let upload = store
3316            .new_multipart_upload(&bucket, source_object, &ObjectOptions::default())
3317            .await
3318            .expect("create source multipart upload");
3319        let mut part_reader = PutObjReader::from_vec(payload.clone());
3320        let part = store
3321            .put_object_part(&bucket, source_object, &upload.upload_id, 1, &mut part_reader, &ObjectOptions::default())
3322            .await
3323            .expect("stage small multipart source part");
3324        let completed_parts = vec![crate::storage_api_contracts::multipart::CompletePart {
3325            part_num: part.part_num,
3326            etag: part.etag,
3327            ..Default::default()
3328        }];
3329
3330        temp_env::async_with_vars([(crate::set_disk::ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async {
3331            let rename_tasks = crate::set_disk::rename_fanout_barrier::observe_tasks(source_object);
3332            let rename_barrier = crate::set_disk::rename_fanout_barrier::arm(
3333                source_object,
3334                0,
3335                crate::set_disk::rename_fanout_barrier::PHASE_RENAME,
3336            );
3337            let complete_store = Arc::clone(&store);
3338            let complete_bucket = bucket.clone();
3339            let complete_upload_id = upload.upload_id.clone();
3340            let mut complete = tokio::spawn(async move {
3341                complete_store
3342                    .complete_multipart_upload(
3343                        &complete_bucket,
3344                        source_object,
3345                        &complete_upload_id,
3346                        completed_parts,
3347                        &ObjectOptions::default(),
3348                    )
3349                    .await
3350            });
3351            tokio::time::timeout(Duration::from_secs(30), rename_barrier.wait_until_paused())
3352                .await
3353                .expect("multipart completion should pause one tail disk during rename");
3354            assert!(
3355                rename_tasks.running() >= 1,
3356                "the paused multipart tail disk must remain in flight before completion returns"
3357            );
3358            assert!(
3359                tokio::time::timeout(Duration::from_millis(100), &mut complete).await.is_err(),
3360                "CompleteMultipartUpload must not return while a copy source rename tail is still pending"
3361            );
3362            rename_barrier.release();
3363            complete
3364                .await
3365                .expect("multipart completion task should join")
3366                .expect("multipart completion should return after every rename tail finishes");
3367
3368            let source_reader = store
3369                .get_object_reader(&bucket, source_object, None, HeaderMap::new(), &ObjectOptions::default())
3370                .await
3371                .expect("completed multipart source should be immediately readable after success");
3372            let mut copy_info = source_reader.object_info.clone();
3373            let actual_size = copy_info.get_actual_size().expect("copy source logical size should resolve");
3374            assert_eq!(actual_size, payload.len() as i64);
3375            let copy_reader =
3376                rustfs_rio::HashReader::from_stream(source_reader.stream, actual_size, actual_size, None, None, false)
3377                    .expect("copy source hash reader should build");
3378            copy_info.put_object_reader = Some(PutObjReader::new(copy_reader));
3379
3380            store
3381                .copy_object(
3382                    &bucket,
3383                    source_object,
3384                    &bucket,
3385                    target_object,
3386                    &mut copy_info,
3387                    &ObjectOptions::default(),
3388                    &ObjectOptions::default(),
3389                )
3390                .await
3391                .expect("CopyObject should accept a freshly completed multipart source");
3392        })
3393        .await;
3394
3395        let mut target_reader = store
3396            .get_object_reader(&bucket, target_object, None, HeaderMap::new(), &ObjectOptions::default())
3397            .await
3398            .expect("copied target should be readable after tail release");
3399        let mut target_body = Vec::new();
3400        target_reader
3401            .stream
3402            .read_to_end(&mut target_body)
3403            .await
3404            .expect("target body should stream");
3405        assert_eq!(target_body, payload);
3406        shutdown.cancel();
3407    }
3408
3409    #[cfg(feature = "test-util")]
3410    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
3411    #[serial_test::serial(storage_class_env)]
3412    async fn early_ack_put_tails_block_scanner_publication_until_all_renames_finish() {
3413        use crate::storage_api_contracts::namespace::NamespaceLocking as _;
3414
3415        let temp_dir = tempfile::tempdir().expect("create scanner PUT tail store dir");
3416        let (ctx, store, shutdown) =
3417            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "scanner-put-tails", &[4])).await;
3418        crate::bucket::metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
3419        let bucket = format!("scanner-put-tails-{}", Uuid::new_v4());
3420        store
3421            .make_bucket(&bucket, &MakeBucketOptions::default())
3422            .await
3423            .expect("create scanner PUT tail bucket");
3424        let set = &store.pools[0].disk_set[0];
3425        let objects = [("scanner-tail-a", vec![0xA1; 273]), ("scanner-tail-b", vec![0xB2; 379])];
3426
3427        temp_env::async_with_vars([(crate::set_disk::ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async {
3428            let (active, blocked, movement_generation) = store.scanner_data_movement_activity().await;
3429            assert!(!active && !blocked);
3430            assert!(ctx.scanner_publication_state_allowed(), "the set admission cache should start allowed");
3431            let (old_lease, _) = store
3432                .acquire_scanner_publication_lease(movement_generation, crate::runtime::instance::SCANNER_PUBLICATION_LEASE_TTL)
3433                .await
3434                .expect("publication lease should be admitted before either PUT starts");
3435
3436            let barriers: Vec<_> = objects
3437                .iter()
3438                .map(|(object, _)| {
3439                    crate::set_disk::rename_fanout_barrier::arm(object, 0, crate::set_disk::rename_fanout_barrier::PHASE_RENAME)
3440                })
3441                .collect();
3442            let trackers: Vec<_> = objects
3443                .iter()
3444                .map(|(object, _)| crate::set_disk::rename_fanout_barrier::observe_tasks(object))
3445                .collect();
3446            let puts: Vec<_> = objects
3447                .iter()
3448                .map(|(object, body)| {
3449                    let put_store = Arc::clone(&store);
3450                    let put_bucket = bucket.clone();
3451                    let object = *object;
3452                    let body = body.clone();
3453                    tokio::spawn(async move {
3454                        let mut reader = PutObjReader::from_vec(body);
3455                        put_store
3456                            .put_object(&put_bucket, object, &mut reader, &ObjectOptions::default())
3457                            .await
3458                    })
3459                })
3460                .collect();
3461            let committed = tokio::time::timeout(Duration::from_secs(30), async {
3462                for barrier in &barriers {
3463                    barrier.wait_until_paused().await;
3464                }
3465                let mut committed = Vec::with_capacity(puts.len());
3466                for put in puts {
3467                    committed.push(
3468                        put.await
3469                            .expect("early-ACK PUT task should join while its tail is paused")
3470                            .expect("root PUT should return after quorum without waiting for its tail"),
3471                    );
3472                }
3473                committed
3474            })
3475            .await
3476            .expect("both root PUTs must quorum-ACK while their tail disks remain paused");
3477
3478            assert!(trackers.iter().all(|tracker| tracker.running() >= 1));
3479            assert!(ctx.namespace_commits_pending());
3480            assert!(
3481                ctx.scanner_publication_state_allowed(),
3482                "pending PUT tails must not disable scanner namespace walks"
3483            );
3484            let (active, blocked, observed_movement_generation) = store.scanner_data_movement_activity().await;
3485            assert!(!active, "ordinary PUT tails are not decommission or rebalance work");
3486            assert!(!blocked, "ordinary PUT tails must not block the movement-only scan baseline");
3487            assert_eq!(observed_movement_generation, movement_generation);
3488            assert!(store.scanner_data_usage_publication_blocked().await);
3489            assert!(store.scanner_data_usage_publication_admission_guard().await.is_some());
3490            assert!(set.scanner_data_usage_publication_admission_guard().await.is_some());
3491            for error in [
3492                store
3493                    .acquire_scanner_publication_lease(
3494                        movement_generation,
3495                        crate::runtime::instance::SCANNER_PUBLICATION_LEASE_TTL,
3496                    )
3497                    .await
3498                    .expect_err("a new remote publication lease must reject pending PUT tails"),
3499                store
3500                    .validate_scanner_publication_lease(old_lease, movement_generation)
3501                    .await
3502                    .expect_err("an existing remote lease must not bypass pending PUT tails"),
3503                store
3504                    .acquire_scanner_publication_lease_guard(old_lease)
3505                    .await
3506                    .expect_err("target-side publication admission must reject pending PUT tails"),
3507            ] {
3508                assert!(
3509                    error.to_string().contains("blocked"),
3510                    "publication must fail because of active tails: {error}"
3511                );
3512            }
3513            store.release_scanner_publication_lease(old_lease).await;
3514
3515            for (index, barrier) in barriers.iter().enumerate() {
3516                let commit_generation = ctx.namespace_commit_generation();
3517                let namespace_generation = store.scanner_namespace_mutation_generation();
3518                barrier.release();
3519                tokio::time::timeout(Duration::from_secs(30), async {
3520                    while trackers[index].running() != 0 || ctx.namespace_commit_generation() <= commit_generation {
3521                        tokio::task::yield_now().await;
3522                    }
3523                    if index + 1 == barriers.len() {
3524                        while ctx.namespace_commits_pending() {
3525                            tokio::task::yield_now().await;
3526                        }
3527                    }
3528                })
3529                .await
3530                .expect("released tail must drain and publish its terminal namespace generation");
3531                assert!(store.scanner_namespace_mutation_generation() > namespace_generation);
3532                let pending = index + 1 < barriers.len();
3533                assert_eq!(ctx.namespace_commits_pending(), pending);
3534                assert_eq!(store.scanner_data_usage_publication_blocked().await, pending);
3535                assert!(!store.scanner_data_movement_activity().await.1);
3536                assert!(store.scanner_data_usage_publication_admission_guard().await.is_some());
3537                assert!(set.scanner_data_usage_publication_admission_guard().await.is_some());
3538            }
3539
3540            let (lease, generation) = store
3541                .acquire_scanner_publication_lease(movement_generation, crate::runtime::instance::SCANNER_PUBLICATION_LEASE_TTL)
3542                .await
3543                .expect("remote publication lease should resume after both tails drain");
3544            store
3545                .validate_scanner_publication_lease(lease, generation)
3546                .await
3547                .expect("a resumed remote publication lease should validate");
3548            drop(
3549                store
3550                    .acquire_scanner_publication_lease_guard(lease)
3551                    .await
3552                    .expect("target-side publication admission should resume after both tails drain"),
3553            );
3554            assert!(store.release_scanner_publication_lease(lease).await);
3555
3556            let disks = set.disk_inventory().await;
3557            assert_eq!(disks.len(), 4);
3558            for ((object, body), committed) in objects.iter().zip(&committed) {
3559                let logical_size = i64::try_from(body.len()).expect("fixture payload size should fit i64");
3560                let etag = committed.etag.as_ref().expect("root PUT should return a committed ETag");
3561                for (disk_index, disk) in disks.iter().enumerate() {
3562                    let file_info = disk
3563                        .as_ref()
3564                        .expect("every fixture disk should remain online")
3565                        .read_version(
3566                            "",
3567                            &bucket,
3568                            object,
3569                            "",
3570                            &crate::disk::ReadOptions {
3571                                read_data: true,
3572                                ..Default::default()
3573                            },
3574                        )
3575                        .await
3576                        .unwrap_or_else(|err| panic!("disk {disk_index} should publish {object} after its tail finishes: {err}"));
3577                    assert_eq!(file_info.size, logical_size);
3578                    assert_eq!(file_info.metadata.get(http::header::ETAG.as_str()), Some(etag));
3579                    assert!(
3580                        file_info.inline_data(),
3581                        "small fixture payloads should have an inline shard on every disk"
3582                    );
3583                    let inline_data = file_info.data.as_ref().expect("every disk should retain its inline shard");
3584                    let erasure = crate::erasure::coding::Erasure::try_new_with_options(
3585                        file_info.erasure.data_blocks,
3586                        file_info.erasure.parity_blocks,
3587                        file_info.erasure.block_size,
3588                        file_info.uses_legacy_checksum,
3589                    )
3590                    .expect("persisted erasure geometry should be valid");
3591                    let shard_size =
3592                        usize::try_from(erasure.shard_file_size(logical_size)).expect("fixture shard size should fit usize");
3593                    crate::erasure::coding::bitrot_verify(
3594                        Cursor::new(inline_data.clone()),
3595                        inline_data.len(),
3596                        shard_size,
3597                        rustfs_utils::HashAlgorithm::HighwayHash256S,
3598                        erasure.shard_size(),
3599                    )
3600                    .await
3601                    .unwrap_or_else(|err| panic!("disk {disk_index} should retain a complete valid shard for {object}: {err}"));
3602                }
3603                let mut reader = store
3604                    .get_object_reader(&bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
3605                    .await
3606                    .expect("fully drained PUT should be readable");
3607                let mut actual = Vec::new();
3608                reader.stream.read_to_end(&mut actual).await.expect("PUT body should drain");
3609                assert_eq!(&actual, body);
3610            }
3611
3612            let generation_before_internal_put = ctx.namespace_commit_generation();
3613            let internal_object = "scanner-tail-regression/internal-metadata";
3614            let internal_body = b"scanner metadata must not invalidate its own publication";
3615            let mut internal_reader = PutObjReader::from_vec(internal_body.to_vec());
3616            store
3617                .put_object(RUSTFS_META_BUCKET, internal_object, &mut internal_reader, &ObjectOptions::default())
3618                .await
3619                .expect("internal metadata PUT should commit without scanner self-invalidation");
3620            let internal_lock = set
3621                .new_ns_lock(RUSTFS_META_BUCKET, internal_object)
3622                .await
3623                .expect("internal metadata tail lock should be available");
3624            drop(
3625                internal_lock
3626                    .get_write_lock(Duration::from_secs(30))
3627                    .await
3628                    .expect("internal metadata tail should drain"),
3629            );
3630            assert_eq!(ctx.namespace_commit_generation(), generation_before_internal_put);
3631            assert!(!ctx.namespace_commits_pending());
3632            assert!(store.scanner_data_usage_publication_admission_guard().await.is_some());
3633            assert!(set.scanner_data_usage_publication_admission_guard().await.is_some());
3634            let mut internal_reader = store
3635                .get_object_reader(RUSTFS_META_BUCKET, internal_object, None, HeaderMap::new(), &ObjectOptions::default())
3636                .await
3637                .expect("internal metadata should remain readable");
3638            let mut actual = Vec::new();
3639            internal_reader
3640                .stream
3641                .read_to_end(&mut actual)
3642                .await
3643                .expect("internal metadata body should drain");
3644            assert_eq!(actual, internal_body);
3645        })
3646        .await;
3647        shutdown.cancel();
3648    }
3649
3650    #[cfg(feature = "test-util")]
3651    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
3652    #[serial_test::serial(storage_class_env)]
3653    async fn cancelled_early_ack_put_keeps_scanner_publication_blocked_until_tail_finishes() {
3654        let temp_dir = tempfile::tempdir().expect("create cancelled scanner PUT tail store dir");
3655        let (ctx, store, shutdown) =
3656            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "scanner-cancelled-put-tail", &[4])).await;
3657        crate::bucket::metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
3658        let bucket = format!("scanner-cancelled-put-tail-{}", Uuid::new_v4());
3659        let object = "scanner-cancelled-tail";
3660        let body = vec![0xC3; 273];
3661        store
3662            .make_bucket(&bucket, &MakeBucketOptions::default())
3663            .await
3664            .expect("create cancelled scanner PUT tail bucket");
3665
3666        temp_env::async_with_vars([(crate::set_disk::ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async {
3667            let tracker = crate::set_disk::rename_fanout_barrier::observe_tasks(object);
3668            let tail =
3669                crate::set_disk::rename_fanout_barrier::arm(object, 0, crate::set_disk::rename_fanout_barrier::PHASE_RENAME);
3670            let quorum = crate::set_disk::PutObjectCommitBarrier::install(
3671                &bucket,
3672                object,
3673                crate::set_disk::PutObjectCommitPause::AfterRenameQuorum,
3674            );
3675            let handoff = crate::set_disk::PutObjectCommitBarrier::install(
3676                &bucket,
3677                object,
3678                crate::set_disk::PutObjectCommitPause::AfterRenameHandoff,
3679            );
3680            let put_store = Arc::clone(&store);
3681            let put_bucket = bucket.clone();
3682            let put_body = body.clone();
3683            let put = tokio::spawn(async move {
3684                let mut reader = PutObjReader::from_vec(put_body);
3685                put_store
3686                    .put_object(&put_bucket, object, &mut reader, &ObjectOptions::default())
3687                    .await
3688            });
3689            tokio::time::timeout(Duration::from_secs(30), tail.wait_until_paused())
3690                .await
3691                .expect("cancelled PUT should pause one disk before rename");
3692            quorum.wait_until_paused().await;
3693            put.abort();
3694            assert!(
3695                put.await
3696                    .expect_err("caller should be cancelled after rename quorum")
3697                    .is_cancelled()
3698            );
3699            quorum.release();
3700            handoff.wait_until_paused().await;
3701            assert!(tracker.running() >= 1);
3702            assert!(ctx.namespace_commits_pending());
3703            assert!(!store.scanner_data_movement_activity().await.1);
3704            assert!(store.scanner_data_usage_publication_blocked().await);
3705            assert!(store.scanner_data_usage_publication_admission_guard().await.is_some());
3706            assert!(
3707                store.pools[0].disk_set[0]
3708                    .scanner_data_usage_publication_admission_guard()
3709                    .await
3710                    .is_some()
3711            );
3712            let generation = store.scanner_namespace_mutation_generation();
3713
3714            handoff.release();
3715            tail.release();
3716            tokio::time::timeout(Duration::from_secs(30), async {
3717                while tracker.running() != 0 || ctx.namespace_commits_pending() {
3718                    tokio::task::yield_now().await;
3719                }
3720            })
3721            .await
3722            .expect("cancelled request's detached fanout must release scanner admission after finishing");
3723            assert!(store.scanner_namespace_mutation_generation() > generation);
3724            assert!(!store.scanner_data_usage_publication_blocked().await);
3725            assert!(store.scanner_data_usage_publication_admission_guard().await.is_some());
3726            for (disk_index, disk) in store.pools[0].disk_set[0].disk_inventory().await.iter().enumerate() {
3727                let file_info = disk
3728                    .as_ref()
3729                    .expect("cancelled PUT fixture disk should remain online")
3730                    .read_version("", &bucket, object, "", &crate::disk::ReadOptions::default())
3731                    .await
3732                    .unwrap_or_else(|err| panic!("cancelled PUT must still publish on disk {disk_index}: {err}"));
3733                assert_eq!(file_info.size, i64::try_from(body.len()).expect("fixture body size should fit i64"));
3734            }
3735            let mut reader = store
3736                .get_object_reader(&bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
3737                .await
3738                .expect("a cancelled caller must not discard its quorum-committed object");
3739            let mut actual = Vec::new();
3740            reader
3741                .stream
3742                .read_to_end(&mut actual)
3743                .await
3744                .expect("cancelled PUT body should drain");
3745            assert_eq!(actual, body);
3746        })
3747        .await;
3748        shutdown.cancel();
3749    }
3750
3751    #[cfg(feature = "test-util")]
3752    #[test]
3753    #[serial_test::serial(storage_class_env)]
3754    fn data_movement_multipart_part_staging_holds_no_publication_lock_or_tier_lease() {
3755        run_large_stack_async_test(
3756            "multipart-part-staging-publication-fence",
3757            data_movement_multipart_part_staging_holds_no_publication_lock_or_tier_lease_case,
3758        );
3759    }
3760
3761    #[cfg(feature = "test-util")]
3762    async fn data_movement_multipart_part_staging_holds_no_publication_lock_or_tier_lease_case() {
3763        let temp_dir = tempfile::tempdir().expect("create multipart staging-fence store dir");
3764        let (ctx, store, shutdown) =
3765            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "mpu-staging-publication", &[4, 4])).await;
3766        crate::bucket::metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
3767
3768        let bucket = format!("mpu-staging-publication-{}", Uuid::new_v4());
3769        let object = "remote-owned.bin";
3770        let payload = b"multipart part staging must not publish an object".to_vec();
3771        store
3772            .make_bucket(&bucket, &MakeBucketOptions::default())
3773            .await
3774            .expect("multipart staging test bucket should be created");
3775
3776        let tier_name = "MPU-STAGING-TIER";
3777        register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
3778        let lease = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
3779            .await
3780            .expect("mock tier generation should be available");
3781        let backend_identity = lease.backend_identity();
3782        drop(lease);
3783
3784        let mut source_metadata = HashMap::from([(
3785            "x-amz-restore".to_string(),
3786            "ongoing-request=\"false\", expiry-date=\"2099-01-01T00:00:00Z\"".to_string(),
3787        )]);
3788        for (suffix, value) in [
3789            (
3790                rustfs_utils::http::SUFFIX_TRANSITION_STATUS,
3791                crate::bucket::lifecycle::core::TRANSITION_COMPLETE.to_string(),
3792            ),
3793            (rustfs_utils::http::SUFFIX_TRANSITIONED_OBJECTNAME, "remote/mpu.bin".to_string()),
3794            (rustfs_utils::http::SUFFIX_TRANSITION_TIER, tier_name.to_string()),
3795            (rustfs_utils::http::SUFFIX_TRANSITIONED_VERSION_ID, "remote-mpu-version".to_string()),
3796            (
3797                rustfs_utils::http::SUFFIX_TRANSITIONED_VERSION_STATE,
3798                rustfs_filemeta::TransitionVersionState::Exact.as_str().to_string(),
3799            ),
3800            (
3801                rustfs_utils::http::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
3802                rustfs_utils::crypto::hex(backend_identity),
3803            ),
3804        ] {
3805            rustfs_utils::http::metadata_compat::insert_str(&mut source_metadata, suffix, value);
3806        }
3807        let mut source_body = PutObjReader::from_vec(payload.clone());
3808        store.pools[0]
3809            .put_object(
3810                &bucket,
3811                object,
3812                &mut source_body,
3813                &ObjectOptions {
3814                    user_defined: source_metadata,
3815                    ..Default::default()
3816                },
3817            )
3818            .await
3819            .expect("remote-owned source should be written to the source pool");
3820        let source = store.pools[0]
3821            .get_object_info(&bucket, object, &ObjectOptions::default())
3822            .await
3823            .expect("remote-owned source metadata should be readable");
3824        let publication = store
3825            .acquire_remote_tuple_publication_fence(&bucket, 0, &source, true)
3826            .await
3827            .expect("multipart migration should capture its publication capability");
3828        assert_eq!(
3829            TierConfigMgr::active_operation_lease_count(&ctx.tier_config_mgr(), tier_name).await,
3830            0,
3831            "capturing the commit-late capability must not pin the tier generation"
3832        );
3833
3834        let bucket_incarnation_id = store
3835            .bucket_incarnation_id(&bucket)
3836            .await
3837            .expect("bucket incarnation should resolve");
3838        mark_test_pool_decommissioning(&store, 0).await;
3839        let capacity_owner = test_decommission_capacity_owner(store.as_ref(), 0)
3840            .await
3841            .with_mutation_id(Uuid::new_v4());
3842        let upload_metadata = source.user_defined.as_ref().clone();
3843        let mut staging_opts = ObjectOptions {
3844            data_movement: true,
3845            src_pool_idx: 0,
3846            versioned: source.version_id.is_some(),
3847            version_id: source.version_id.map(|version_id| version_id.to_string()),
3848            mod_time: source.mod_time,
3849            user_defined: upload_metadata,
3850            expected_bucket_incarnation_id: Some(bucket_incarnation_id),
3851            ..Default::default()
3852        };
3853        let upload_identity = crate::data_movement::data_movement_upload_identity_from_options(&staging_opts);
3854        rustfs_utils::http::insert_str(
3855            &mut staging_opts.user_defined,
3856            rustfs_utils::http::SUFFIX_DATA_MOVEMENT_UPLOAD,
3857            upload_identity,
3858        );
3859        capacity_owner.apply_to(&mut staging_opts);
3860        let (upload, target_pool_idx, staged_incarnation_id) = store
3861            .handle_new_multipart_upload_with_pool_idx(&bucket, object, &staging_opts, None)
3862            .await
3863            .expect("target multipart upload should be staged");
3864        assert_eq!(target_pool_idx, 1, "the active pool must receive the staged upload");
3865        assert_eq!(staged_incarnation_id, Some(bucket_incarnation_id));
3866
3867        let barrier = crate::set_disk::MultipartCommitBarrier::install(
3868            &bucket,
3869            object,
3870            crate::set_disk::MultipartCommitPause::PutPartBeforeLockLost,
3871        );
3872        let part_store = Arc::clone(&store);
3873        let part_bucket = bucket.clone();
3874        let upload_id = upload.upload_id.clone();
3875        let part_payload = payload.clone();
3876        let part = tokio::spawn(async move {
3877            let mut reader = PutObjReader::from_vec(part_payload);
3878            let mut part_opts = ObjectOptions {
3879                data_movement: true,
3880                src_pool_idx: 0,
3881                part_number: Some(1),
3882                expected_bucket_incarnation_id: Some(bucket_incarnation_id),
3883                ..Default::default()
3884            };
3885            capacity_owner.apply_to(&mut part_opts);
3886            part_store
3887                .put_object_part_for_data_movement(1, &part_bucket, object, &upload_id, &mut reader, &part_opts)
3888                .await
3889        });
3890        barrier.wait_until_paused().await;
3891
3892        assert_eq!(
3893            TierConfigMgr::active_operation_lease_count(&ctx.tier_config_mgr(), tier_name).await,
3894            0,
3895            "UploadPart staging must not pin the tier generation"
3896        );
3897        let encoded = rustfs_utils::path::encode_dir_object(object);
3898        let mut proof_opts = ObjectOptions::default();
3899        let proof = tokio::time::timeout(
3900            Duration::from_millis(250),
3901            store.acquire_all_physical_object_read_locks("tier_delete_journal_recovery", &bucket, &encoded, &mut proof_opts),
3902        )
3903        .await
3904        .expect("UploadPart staging must not hold any object publication write lock")
3905        .expect("the all-physical recovery proof should acquire every object namespace");
3906        let source_head = store.pools[0]
3907            .get_object_info(&bucket, object, &ObjectOptions::default())
3908            .await
3909            .expect("source HEAD should remain available while the part commit is paused");
3910        assert_eq!(source_head.etag, source.etag);
3911        assert!(
3912            store.pools[1]
3913                .get_object_info(&bucket, object, &ObjectOptions::default())
3914                .await
3915                .is_err(),
3916            "UploadPart staging must not publish target object metadata"
3917        );
3918        drop(proof);
3919
3920        barrier.release();
3921        drop(barrier);
3922        part.await
3923            .expect("UploadPart staging task should join")
3924            .expect("UploadPart staging should commit after the observation releases");
3925        let mut abort_opts = ObjectOptions {
3926            data_movement: true,
3927            src_pool_idx: 0,
3928            versioned: source.version_id.is_some(),
3929            version_id: source.version_id.map(|version_id| version_id.to_string()),
3930            mod_time: source.mod_time,
3931            expected_bucket_incarnation_id: Some(bucket_incarnation_id),
3932            ..Default::default()
3933        };
3934        capacity_owner.apply_to(&mut abort_opts);
3935        store
3936            .abort_multipart_upload_for_data_movement(1, &bucket, object, &upload.upload_id, &abort_opts)
3937            .await
3938            .expect("staged target upload should be removable without publishing");
3939        drop(publication);
3940        assert_eq!(TierConfigMgr::active_operation_lease_count(&ctx.tier_config_mgr(), tier_name).await, 0);
3941        shutdown.cancel();
3942    }
3943
3944    #[cfg(feature = "test-util")]
3945    struct OfflineTestDisks {
3946        disks: Vec<crate::disk::DiskStore>,
3947    }
3948
3949    #[cfg(feature = "test-util")]
3950    impl Drop for OfflineTestDisks {
3951        fn drop(&mut self) {
3952            for disk in &self.disks {
3953                disk.reset_health_for_store_init_retry();
3954            }
3955        }
3956    }
3957
3958    #[cfg(feature = "test-util")]
3959    async fn force_set_disks_offline_for_test(set: &Arc<crate::set_disk::SetDisks>) -> OfflineTestDisks {
3960        let disks = set
3961            .disks
3962            .read()
3963            .await
3964            .iter()
3965            .map(|disk| disk.clone().expect("fault-injection set should start fully online"))
3966            .collect::<Vec<_>>();
3967        for disk in &disks {
3968            disk.close().await.expect("fault injection should stop per-disk monitoring");
3969            disk.force_runtime_state_for_test(crate::disk::health_state::RuntimeDriveHealthState::Offline);
3970        }
3971
3972        // Sets has an independent endpoint monitor that renews missing slots.
3973        // Keep each slot populated by the original, deliberately faulty handle
3974        // and prove an explicit reconnect pass cannot heal this test fault.
3975        set.connect_disks().await;
3976        let current = set.disks.read().await.clone();
3977        assert_eq!(current.len(), disks.len());
3978        for (slot, expected) in current.iter().zip(&disks) {
3979            let disk = slot.as_ref().expect("offline test slot must remain populated");
3980            assert!(Arc::ptr_eq(disk, expected), "endpoint monitor must not replace an offline test handle");
3981            assert_eq!(disk.runtime_state(), crate::disk::health_state::RuntimeDriveHealthState::Offline);
3982            assert!(
3983                disk.is_online().await,
3984                "the valid disk identity must prevent endpoint renewal while the IO health gate remains offline"
3985            );
3986        }
3987        OfflineTestDisks { disks }
3988    }
3989
3990    #[cfg(feature = "test-util")]
3991    async fn force_set_disk_range_offline_for_test(
3992        set: &Arc<crate::set_disk::SetDisks>,
3993        range: std::ops::Range<usize>,
3994    ) -> OfflineTestDisks {
3995        let disks = set
3996            .disks
3997            .read()
3998            .await
3999            .get(range)
4000            .expect("offline test range must fit the set")
4001            .iter()
4002            .map(|disk| disk.clone().expect("fault-injection disk should start online"))
4003            .collect::<Vec<_>>();
4004        for disk in &disks {
4005            disk.close().await.expect("fault injection should stop per-disk monitoring");
4006            disk.force_runtime_state_for_test(crate::disk::health_state::RuntimeDriveHealthState::Offline);
4007        }
4008        set.connect_disks().await;
4009        for disk in &disks {
4010            assert_eq!(disk.runtime_state(), crate::disk::health_state::RuntimeDriveHealthState::Offline);
4011        }
4012        OfflineTestDisks { disks }
4013    }
4014
4015    fn active_rebalance_meta_for_pool(pool_count: usize, active_pool_idx: usize) -> RebalanceMeta {
4016        let now = OffsetDateTime::now_utc();
4017        let mut pool_stats = vec![RebalanceStats::default(); pool_count];
4018        pool_stats[active_pool_idx] = RebalanceStats {
4019            participating: true,
4020            info: RebalanceInfo {
4021                start_time: Some(now),
4022                status: RebalStatus::Started,
4023                ..Default::default()
4024            },
4025            ..Default::default()
4026        };
4027
4028        RebalanceMeta {
4029            id: uuid::Uuid::new_v4().to_string(),
4030            pool_stats,
4031            ..Default::default()
4032        }
4033    }
4034
4035    #[tokio::test]
4036    #[serial_test::serial(storage_class_env)]
4037    async fn quota_object_fence_ignores_an_unrelated_offline_pool() {
4038        let temp_dir = tempfile::tempdir().expect("create quota fence store dir");
4039        let (_ctx, store, shutdown) =
4040            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "quota-object-fence", &[4, 4])).await;
4041        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
4042        let bucket = format!("quota-object-fence-{}", uuid::Uuid::new_v4());
4043        let object = "object.bin";
4044        store
4045            .make_bucket(&bucket, &MakeBucketOptions::default())
4046            .await
4047            .expect("create quota fence bucket");
4048        store.pools[1].disk_set[0].disks.write().await.fill(None);
4049
4050        crate::bucket::quota::reservation::fence_namespace_mutations_for_test(&store, &bucket, object, Some((0, 0)))
4051            .await
4052            .expect("the selected pool fence should ignore an unrelated offline pool");
4053        let err = crate::bucket::quota::reservation::fence_namespace_mutations_for_test(&store, &bucket, object, None)
4054            .await
4055            .expect_err("legacy reservations must conservatively fence every pool");
4056        assert!(matches!(err, StorageError::ErasureWriteQuorum));
4057
4058        shutdown.cancel();
4059    }
4060
4061    async fn migrate_versioned_decommission_test_object(
4062        store: &Arc<crate::store::ECStore>,
4063        bucket: &str,
4064        object: &str,
4065        payload: &[u8],
4066        op_label: &'static str,
4067    ) -> (uuid::Uuid, FileInfoVersions) {
4068        let mut source = PutObjReader::from_vec(payload.to_vec());
4069        let source_info = store.pools[0]
4070            .put_object(
4071                bucket,
4072                object,
4073                &mut source,
4074                &ObjectOptions {
4075                    versioned: true,
4076                    ..Default::default()
4077                },
4078            )
4079            .await
4080            .expect("write versioned source to the pool being decommissioned");
4081        let source_version = source_info.version_id.expect("versioned source must have a version ID");
4082        let expected_source_versions = store.pools[0]
4083            .get_disks_by_key(object)
4084            .load_file_info_versions_exact(bucket, object)
4085            .await
4086            .expect("source versions should be readable before migration")
4087            .expect("source versions should exist before migration");
4088        {
4089            let mut pool_meta = store.pool_meta.write().await;
4090            pool_meta.pools[0].decommission = Some(PoolDecommissionInfo {
4091                start_time: Some(OffsetDateTime::now_utc()),
4092                ..Default::default()
4093            });
4094        }
4095
4096        let barrier = crate::set_disk::PutObjectCommitBarrier::install(
4097            bucket,
4098            object,
4099            crate::set_disk::PutObjectCommitPause::AfterNamespace,
4100        );
4101        let migration_store = Arc::clone(store);
4102        let migration_bucket = bucket.to_string();
4103        let migration_object = object.to_string();
4104        let migration = tokio::spawn(async move {
4105            let source_reader = migration_store.pools[0]
4106                .get_object_reader(
4107                    &migration_bucket,
4108                    &migration_object,
4109                    None,
4110                    HeaderMap::new(),
4111                    &ObjectOptions {
4112                        versioned: true,
4113                        version_id: Some(source_version.to_string()),
4114                        no_lock: true,
4115                        data_movement: true,
4116                        raw_data_movement_read: true,
4117                        ..Default::default()
4118                    },
4119                )
4120                .await?;
4121            crate::data_movement::migrate_decommission_object(
4122                migration_store,
4123                0,
4124                migration_bucket,
4125                source_reader,
4126                None,
4127                op_label,
4128                None,
4129            )
4130            .await
4131        });
4132        barrier.wait_until_paused().await;
4133        barrier.release();
4134        migration
4135            .await
4136            .expect("versioned decommission migration task should join")
4137            .expect("versioned decommission migration should commit");
4138
4139        (source_version, expected_source_versions)
4140    }
4141
4142    fn set_test_decommission_capacity_override(store: &Arc<crate::store::ECStore>, pool_idx: usize) {
4143        let layout = DecommissionErasureLayout { data: 2, parity: 2 };
4144        let source_physical_bytes = 1024 * 1024 * 1024;
4145        let target_physical_bytes = source_physical_bytes * 8;
4146        let capacity: Vec<DecommissionPoolCapacityInfo> = store
4147            .pools
4148            .iter()
4149            .enumerate()
4150            .map(|(index, _)| {
4151                if index == pool_idx {
4152                    DecommissionPoolCapacityInfo::for_test(index, layout, 0, source_physical_bytes, source_physical_bytes)
4153                } else {
4154                    DecommissionPoolCapacityInfo::for_test(index, layout, target_physical_bytes, target_physical_bytes, 0)
4155                }
4156            })
4157            .collect();
4158        set_decommission_capacity_info_overrides_for_test(store.id, (0..128).map(|_| capacity.clone()).collect());
4159    }
4160
4161    async fn mark_test_pool_decommissioning(store: &Arc<crate::store::ECStore>, pool_idx: usize) {
4162        set_test_decommission_capacity_override(store, pool_idx);
4163        store
4164            .save_current_pool_meta_for_decommission_start(&[pool_idx], Vec::new())
4165            .await
4166            .expect("test decommission capacity reservation should activate");
4167    }
4168
4169    #[cfg(feature = "test-util")]
4170    async fn mark_test_pool_decommissioning_with_split_targets(store: &Arc<crate::store::ECStore>, pool_idx: usize) {
4171        let layout = DecommissionErasureLayout { data: 2, parity: 2 };
4172        let source_physical_bytes = 1024 * 1024 * 1024;
4173        let capacity = store
4174            .pools
4175            .iter()
4176            .enumerate()
4177            .map(|(index, _)| {
4178                if index == pool_idx {
4179                    DecommissionPoolCapacityInfo::for_test(index, layout, 0, source_physical_bytes, source_physical_bytes)
4180                } else {
4181                    // The reservation peak is twice the predicted target size.
4182                    // Give each target only half so the real allocator must
4183                    // authorize both receipt-bearing pools.
4184                    DecommissionPoolCapacityInfo::for_test(index, layout, source_physical_bytes, source_physical_bytes, 0)
4185                }
4186            })
4187            .collect::<Vec<_>>();
4188        set_decommission_capacity_info_overrides_for_test(store.id, (0..128).map(|_| capacity.clone()).collect());
4189        store
4190            .save_current_pool_meta_for_decommission_start(&[pool_idx], Vec::new())
4191            .await
4192            .expect("split-target decommission capacity reservation should activate");
4193    }
4194
4195    #[cfg(feature = "test-util")]
4196    async fn test_decommission_capacity_owner(
4197        store: &crate::store::ECStore,
4198        source_pool_index: usize,
4199    ) -> crate::core::pools::DecommissionCapacityOwner {
4200        let pool_meta = store.pool_meta.read().await;
4201        let reservation = pool_meta.pools[source_pool_index]
4202            .decommission
4203            .as_ref()
4204            .and_then(|info| info.capacity_reservation.as_ref())
4205            .expect("test decommission capacity reservation should exist");
4206        crate::core::pools::DecommissionCapacityOwner {
4207            source_pool_index,
4208            operation_id: reservation.operation_id,
4209            generation: reservation.generation,
4210            owner_nonce: reservation.owner_nonce,
4211            mutation_id: None,
4212        }
4213    }
4214
4215    const DECOMMISSION_TEST_FAULT_STAGE_DELETE_MARKER: &str = "delete_marker_copy";
4216    const DECOMMISSION_TEST_FAULT_STAGE_MIGRATE_OBJECT: &str = "migrate_object";
4217    #[cfg(feature = "test-util")]
4218    const DECOMMISSION_TEST_FAULT_STAGE_TIERED: &str = "decommission_tiered_object";
4219
4220    fn decommission_retry_fault_hook(
4221        bucket: &str,
4222        object: &str,
4223        faults: Arc<AtomicUsize>,
4224    ) -> crate::core::pools::DecommissionTestFaultDecision {
4225        let target_bucket = bucket.to_string();
4226        let target_object = object.to_string();
4227        Arc::new(move |stage, bucket, object, attempt, succeeded| {
4228            if !succeeded
4229                || attempt >= crate::core::pools::DECOMMISSION_VERSION_COPY_ATTEMPTS
4230                || stage != DECOMMISSION_TEST_FAULT_STAGE_MIGRATE_OBJECT
4231                || bucket != target_bucket
4232                || object != target_object
4233            {
4234                return false;
4235            }
4236
4237            // Entry retries reset the local attempt; real copy errors can skip
4238            // successful attempts. Only injected faults spend this global budget.
4239            // A real failure may consume an attempt, so preserve the final chance.
4240            faults
4241                .fetch_update(Ordering::SeqCst, Ordering::SeqCst, |faults| {
4242                    (faults < crate::core::pools::DECOMMISSION_VERSION_COPY_ATTEMPTS.saturating_sub(1))
4243                        .then_some(faults.saturating_add(1))
4244                })
4245                .is_ok()
4246        })
4247    }
4248
4249    async fn seed_decommission_source(
4250        store: &Arc<crate::store::ECStore>,
4251        bucket: &str,
4252        object: &str,
4253        body: Vec<u8>,
4254        opts: &ObjectOptions,
4255    ) {
4256        let mut reader = PutObjReader::from_vec(body);
4257        store.pools[0]
4258            .put_object(bucket, object, &mut reader, opts)
4259            .await
4260            .expect("seed decommission source object");
4261    }
4262
4263    async fn run_decommission_entry_retry_test(
4264        store: &Arc<crate::store::ECStore>,
4265        rx: CancellationToken,
4266        bucket: &str,
4267        object: &str,
4268        expected_bucket_incarnation_id: Option<uuid::Uuid>,
4269        source_changed_exhaustions: Arc<AtomicUsize>,
4270    ) -> crate::error::Result<()> {
4271        let source_set = store.pools[0].get_disks_by_key(object);
4272        store
4273            .decommission_entry_with_retry_state_for_test(
4274                rx,
4275                0,
4276                MetaCacheEntry {
4277                    name: object.to_string(),
4278                    ..Default::default()
4279                },
4280                bucket.to_string(),
4281                source_set,
4282                expected_bucket_incarnation_id,
4283                source_changed_exhaustions,
4284            )
4285            .await
4286    }
4287
4288    async fn read_decommission_target_body(
4289        store: &Arc<crate::store::ECStore>,
4290        bucket: &str,
4291        object: &str,
4292        opts: &ObjectOptions,
4293    ) -> Vec<u8> {
4294        let mut reader = store.pools[1]
4295            .get_object_reader(bucket, object, None, HeaderMap::new(), opts)
4296            .await
4297            .expect("read decommission target object");
4298        let mut body = Vec::new();
4299        reader
4300            .stream
4301            .read_to_end(&mut body)
4302            .await
4303            .expect("drain decommission target body");
4304        body
4305    }
4306
4307    async fn assert_decommission_source_absent(
4308        store: &Arc<crate::store::ECStore>,
4309        bucket: &str,
4310        object: &str,
4311        opts: &ObjectOptions,
4312    ) {
4313        let err = store.pools[0]
4314            .get_object_info(bucket, object, opts)
4315            .await
4316            .expect_err("decommission source must be retained until target commit, then removed");
4317        assert!(
4318            matches!(err, StorageError::ObjectNotFound(_, _) | StorageError::VersionNotFound(_, _, _)),
4319            "unexpected decommission source result: {err:?}"
4320        );
4321    }
4322
4323    #[cfg(feature = "test-util")]
4324    async fn seed_transitioned_free_version(
4325        ctx: &Arc<crate::runtime::instance::InstanceContext>,
4326        store: &Arc<crate::store::ECStore>,
4327        bucket: &str,
4328        object: &str,
4329    ) -> (uuid::Uuid, uuid::Uuid) {
4330        let tier_name = format!("DECOMFREE{}", uuid::Uuid::new_v4().simple());
4331        register_mock_tier(&ctx.tier_config_mgr(), &tier_name).await;
4332
4333        let mut reader = PutObjReader::from_vec(b"transitioned source bytes".to_vec());
4334        let source = store.pools[0]
4335            .put_object(
4336                bucket,
4337                object,
4338                &mut reader,
4339                &ObjectOptions {
4340                    versioned: true,
4341                    ..Default::default()
4342                },
4343            )
4344            .await
4345            .expect("write transitioned decommission source");
4346        let source_version = source.version_id.expect("transitioned source must be versioned");
4347        store.pools[0]
4348            .transition_object(
4349                bucket,
4350                object,
4351                &ObjectOptions {
4352                    versioned: true,
4353                    version_id: Some(source_version.to_string()),
4354                    transition: TransitionOptions {
4355                        status: TRANSITION_PENDING.to_string(),
4356                        tier: tier_name,
4357                        etag: source.etag.clone().expect("transitioned source must have an ETag"),
4358                        ..Default::default()
4359                    },
4360                    mod_time: source.mod_time,
4361                    ..Default::default()
4362                },
4363            )
4364            .await
4365            .expect("transition source before decommission");
4366        store.pools[0]
4367            .delete_object(
4368                bucket,
4369                object,
4370                ObjectOptions {
4371                    versioned: true,
4372                    version_id: Some(source_version.to_string()),
4373                    ..Default::default()
4374                },
4375            )
4376            .await
4377            .expect("delete transitioned source version");
4378
4379        let versions = store.pools[0]
4380            .get_disks_by_key(object)
4381            .load_file_info_versions_exact(bucket, object)
4382            .await
4383            .expect("source versions should decode after transition delete")
4384            .expect("source free version should remain after transition delete");
4385        let free_version = versions
4386            .versions
4387            .iter()
4388            .find(|version| version.tier_free_version())
4389            .and_then(|version| version.version_id)
4390            .expect("transition delete should create a free version");
4391        (source_version, free_version)
4392    }
4393
4394    async fn write_decommission_test_multipart_source(
4395        store: &Arc<crate::store::ECStore>,
4396        pool_idx: usize,
4397        bucket: &str,
4398        object: &str,
4399    ) {
4400        let pool = &store.pools[pool_idx];
4401        let upload = pool
4402            .new_multipart_upload(bucket, object, &ObjectOptions::default())
4403            .await
4404            .expect("create decommission multipart source upload");
4405        let first_part = vec![b'm'; 5 * 1024 * 1024];
4406        let second_part = b"decommission multipart tail".to_vec();
4407        let mut completed_parts = Vec::with_capacity(2);
4408        for (part_number, body) in [(1, first_part), (2, second_part)] {
4409            let mut reader = PutObjReader::from_vec(body);
4410            let part = pool
4411                .put_object_part(bucket, object, &upload.upload_id, part_number, &mut reader, &ObjectOptions::default())
4412                .await
4413                .expect("write decommission multipart source part");
4414            completed_parts.push(crate::storage_api_contracts::multipart::CompletePart {
4415                part_num: part.part_num,
4416                etag: part.etag,
4417                ..Default::default()
4418            });
4419        }
4420        pool.clone()
4421            .complete_multipart_upload(bucket, object, &upload.upload_id, completed_parts, &ObjectOptions::default())
4422            .await
4423            .expect("complete decommission multipart source object");
4424    }
4425
4426    struct DataMovementSourceSeed {
4427        pool_idx: usize,
4428        version_id: Option<Uuid>,
4429        body: Vec<u8>,
4430        multipart_split: Option<usize>,
4431        mod_time: OffsetDateTime,
4432        user_defined: HashMap<String, String>,
4433    }
4434
4435    async fn seed_data_movement_source_reader(
4436        store: &Arc<crate::store::ECStore>,
4437        bucket: &str,
4438        object: &str,
4439        seed: DataMovementSourceSeed,
4440    ) -> GetObjectReader {
4441        let DataMovementSourceSeed {
4442            pool_idx,
4443            version_id,
4444            body,
4445            multipart_split,
4446            mod_time,
4447            user_defined,
4448        } = seed;
4449        let pool = &store.pools[pool_idx];
4450        let write_opts = ObjectOptions {
4451            versioned: version_id.is_some(),
4452            version_id: version_id.map(|version_id| version_id.to_string()),
4453            mod_time: Some(mod_time),
4454            user_defined,
4455            ..Default::default()
4456        };
4457        if let Some(split) = multipart_split {
4458            assert!(split > 0 && split < body.len(), "multipart source split must create two non-empty parts");
4459            let upload = pool
4460                .new_multipart_upload(bucket, object, &write_opts)
4461                .await
4462                .expect("create physical data-movement source upload");
4463            let mut completed_parts = Vec::with_capacity(2);
4464            for (part_number, bytes) in [(1, &body[..split]), (2, &body[split..])] {
4465                let mut reader = PutObjReader::from_vec(bytes.to_vec());
4466                let part = pool
4467                    .put_object_part(bucket, object, &upload.upload_id, part_number, &mut reader, &ObjectOptions::default())
4468                    .await
4469                    .expect("write physical data-movement source part");
4470                completed_parts.push(crate::storage_api_contracts::multipart::CompletePart {
4471                    part_num: part.part_num,
4472                    etag: part.etag,
4473                    ..Default::default()
4474                });
4475            }
4476            pool.clone()
4477                .complete_multipart_upload(bucket, object, &upload.upload_id, completed_parts, &write_opts)
4478                .await
4479                .expect("complete physical data-movement source object");
4480        } else {
4481            let mut reader = PutObjReader::from_vec(body);
4482            pool.put_object(bucket, object, &mut reader, &write_opts)
4483                .await
4484                .expect("write physical data-movement source object");
4485        }
4486
4487        pool.get_object_reader(
4488            bucket,
4489            object,
4490            None,
4491            HeaderMap::new(),
4492            &ObjectOptions {
4493                versioned: version_id.is_some(),
4494                version_id: version_id.map(|version_id| version_id.to_string()),
4495                raw_data_movement_read: true,
4496                include_part_checksums: true,
4497                ..Default::default()
4498            },
4499        )
4500        .await
4501        .expect("read the physical data-movement source snapshot")
4502    }
4503
4504    async fn assert_pool_object_present(pool: &Arc<crate::core::sets::Sets>, bucket: &str, object: &str) {
4505        pool.get_object_info(bucket, object, &ObjectOptions::default())
4506            .await
4507            .expect("expected object generation must remain present");
4508    }
4509
4510    async fn assert_pool_object_absent(pool: &Arc<crate::core::sets::Sets>, bucket: &str, object: &str) {
4511        let err = pool
4512            .get_object_info(bucket, object, &ObjectOptions::default())
4513            .await
4514            .expect_err("fenced decommission target must remain absent");
4515        assert!(
4516            matches!(err, StorageError::ObjectNotFound(_, _) | StorageError::VersionNotFound(_, _, _)),
4517            "unexpected fenced target result: {err:?}"
4518        );
4519    }
4520
4521    async fn write_suspended_decommission_source(store: &Arc<crate::store::ECStore>, bucket: &str, object: &str) {
4522        let mut reader = PutObjReader::from_vec(b"suspended source generation".to_vec());
4523        let source = store.pools[0]
4524            .put_object(
4525                bucket,
4526                object,
4527                &mut reader,
4528                &ObjectOptions {
4529                    version_suspended: true,
4530                    mod_time: Some(OffsetDateTime::UNIX_EPOCH + time::Duration::SECOND),
4531                    ..Default::default()
4532                },
4533            )
4534            .await
4535            .expect("write suspended null source version");
4536        assert!(
4537            source.version_id.is_none_or(|version_id| version_id.is_nil()),
4538            "suspended source must use the null version identity"
4539        );
4540    }
4541
4542    async fn assert_suspended_null_source_present(store: &Arc<crate::store::ECStore>, bucket: &str, object: &str) {
4543        let versions = store.pools[0]
4544            .get_disks_by_key(object)
4545            .load_file_info_versions_exact(bucket, object)
4546            .await
4547            .expect("suspended source versions should be readable")
4548            .expect("suspended source must exist before worker convergence");
4549        assert_eq!(versions.versions.len(), 1, "DELETE must not add a marker to the retiring source");
4550        let source = &versions.versions[0];
4551        assert!(
4552            !source.deleted && source.version_id.is_none_or(|version_id| version_id.is_nil()),
4553            "the source pool must retain its null data version until worker convergence"
4554        );
4555        assert_eq!(source.mod_time, Some(OffsetDateTime::UNIX_EPOCH + time::Duration::SECOND));
4556
4557        let mut reader = store.pools[0]
4558            .get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
4559            .await
4560            .expect("the retiring source must remain directly readable before worker convergence");
4561        let mut body = Vec::new();
4562        reader
4563            .stream
4564            .read_to_end(&mut body)
4565            .await
4566            .expect("read retained source bytes");
4567        assert_eq!(body, b"suspended source generation");
4568    }
4569
4570    async fn assert_suspended_null_delete_marker_visible(
4571        store: &Arc<crate::store::ECStore>,
4572        bucket: &str,
4573        object: &str,
4574        marker_mod_time: OffsetDateTime,
4575    ) {
4576        let versions = store.pools[1]
4577            .get_disks_by_key(object)
4578            .load_file_info_versions_exact(bucket, object)
4579            .await
4580            .expect("healthy target versions should be readable")
4581            .expect("the healthy target must retain the DELETE marker");
4582        assert_eq!(versions.versions.len(), 1, "the target must contain only the null delete marker");
4583        let marker = &versions.versions[0];
4584        assert!(marker.deleted, "migration must not replace the DELETE marker with source data");
4585        assert!(marker.version_id.is_none_or(|version_id| version_id.is_nil()));
4586        assert_eq!(marker.size, 0);
4587        assert_eq!(marker.mod_time, Some(marker_mod_time), "migration must preserve the marker generation");
4588        assert!(marker_mod_time > OffsetDateTime::UNIX_EPOCH + time::Duration::SECOND);
4589
4590        let head_err = store
4591            .get_object_info(bucket, object, &ObjectOptions::default())
4592            .await
4593            .expect_err("HEAD must observe the DELETE marker instead of the old null source");
4594        assert!(matches!(head_err, Error::ObjectNotFound(_, _)), "unexpected HEAD result: {head_err:?}");
4595        let get_err = match store
4596            .get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
4597            .await
4598        {
4599            Ok(_) => panic!("GET must not resurrect the deleted null source"),
4600            Err(err) => err,
4601        };
4602        assert!(matches!(get_err, Error::ObjectNotFound(_, _)), "unexpected GET result: {get_err:?}");
4603    }
4604
4605    #[tokio::test]
4606    #[serial_test::serial(storage_class_env)]
4607    async fn tag_updates_skip_active_rebalance_source_pool() {
4608        let temp_dir = tempfile::tempdir().expect("create writer-fencing store dir");
4609        let (_ctx, store, shutdown) =
4610            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "writer-fencing-tags", &[4, 4])).await;
4611        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
4612
4613        let bucket = format!("writer-fencing-tags-{}", uuid::Uuid::new_v4());
4614        let object = "tagged-object.bin";
4615        store
4616            .make_bucket(&bucket, &MakeBucketOptions::default())
4617            .await
4618            .expect("create writer fencing bucket");
4619
4620        let old_time = OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("fixed timestamp should be valid");
4621        let newer_time = old_time + time::Duration::seconds(10);
4622        let mut source_reader = PutObjReader::from_vec(b"source-body".to_vec());
4623        store.pools[0]
4624            .put_object(
4625                &bucket,
4626                object,
4627                &mut source_reader,
4628                &ObjectOptions {
4629                    mod_time: Some(newer_time),
4630                    ..Default::default()
4631                },
4632            )
4633            .await
4634            .expect("write newer source object");
4635        let mut target_reader = PutObjReader::from_vec(b"target-body".to_vec());
4636        store.pools[1]
4637            .put_object(
4638                &bucket,
4639                object,
4640                &mut target_reader,
4641                &ObjectOptions {
4642                    mod_time: Some(old_time),
4643                    ..Default::default()
4644                },
4645            )
4646            .await
4647            .expect("write older target object");
4648
4649        *store.rebalance_meta.write().await = Some(active_rebalance_meta_for_pool(store.pools.len(), 0));
4650        assert!(store.is_pool_rebalancing(0).await, "pool 0 must be marked as an active rebalance source");
4651
4652        let tags = "rebalance=target";
4653        assert_ne!(
4654            store.pools[0]
4655                .get_object_tags(&bucket, object, &ObjectOptions::default())
4656                .await
4657                .expect("source object tags should be readable before update"),
4658            tags,
4659            "source object must start without the target tag"
4660        );
4661        assert_ne!(
4662            store.pools[1]
4663                .get_object_tags(&bucket, object, &ObjectOptions::default())
4664                .await
4665                .expect("target object tags should be readable before update"),
4666            tags,
4667            "target object must start without the target tag"
4668        );
4669        let selected_pool = store
4670            .get_pool_idx_existing_with_opts(
4671                &bucket,
4672                object,
4673                &ObjectOptions {
4674                    no_lock: true,
4675                    metadata_chg: true,
4676                    skip_decommissioned: true,
4677                    skip_rebalancing: true,
4678                    ..Default::default()
4679                },
4680            )
4681            .await
4682            .expect("writer lookup should select an existing non-rebalancing pool");
4683        assert_eq!(selected_pool, 1, "writer lookup must skip active rebalance pool 0");
4684
4685        let updated = store
4686            .put_object_tags(&bucket, object, tags, &ObjectOptions::default())
4687            .await
4688            .expect("tag update should use the non-rebalancing target pool");
4689        assert_eq!(
4690            updated.mod_time,
4691            Some(old_time),
4692            "tag update must return the non-rebalancing pool object rather than the newer active source"
4693        );
4694
4695        let target_tags = store.pools[1]
4696            .get_object_tags(&bucket, object, &ObjectOptions::default())
4697            .await
4698            .expect("target object tags should be readable");
4699        assert_eq!(target_tags, tags, "non-rebalancing pool must receive writer tag updates");
4700
4701        shutdown.cancel();
4702    }
4703
4704    #[tokio::test]
4705    #[serial_test::serial(storage_class_env)]
4706    async fn multipart_listing_skips_active_rebalance_source_pool() {
4707        let temp_dir = tempfile::tempdir().expect("create multipart writer-fencing store dir");
4708        let (_ctx, store, shutdown) =
4709            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "writer-fencing-multipart", &[4, 4])).await;
4710        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
4711
4712        let bucket = format!("writer-fencing-multipart-{}", uuid::Uuid::new_v4());
4713        store
4714            .make_bucket(&bucket, &MakeBucketOptions::default())
4715            .await
4716            .expect("create multipart writer fencing bucket");
4717
4718        let incarnation = store.bucket_incarnation_id(&bucket).await.expect("read bucket incarnation");
4719        let lifecycle_guard = store
4720            .acquire_bucket_lifecycle_read_lock(&bucket)
4721            .await
4722            .expect("acquire multipart test lifecycle fence");
4723        let mut upload_opts = ObjectOptions {
4724            expected_bucket_incarnation_id: Some(incarnation),
4725            ..Default::default()
4726        };
4727        upload_opts.add_bucket_lifecycle_lock_guard(&lifecycle_guard);
4728        let source_upload = store.pools[0]
4729            .new_multipart_upload(&bucket, "source-only.bin", &upload_opts)
4730            .await
4731            .expect("create source upload");
4732        let target_upload = store.pools[1]
4733            .new_multipart_upload(&bucket, "target-visible.bin", &upload_opts)
4734            .await
4735            .expect("create target upload");
4736
4737        *store.rebalance_meta.write().await = Some(active_rebalance_meta_for_pool(store.pools.len(), 0));
4738        assert!(store.is_pool_rebalancing(0).await, "pool 0 must be marked as an active rebalance source");
4739
4740        let listed = store
4741            .list_multipart_uploads(&bucket, "", None, None, None, 100)
4742            .await
4743            .expect("list multipart uploads");
4744        let listed_uploads: Vec<(&str, &str)> = listed
4745            .uploads
4746            .iter()
4747            .map(|upload| (upload.object.as_str(), upload.upload_id.as_str()))
4748            .collect();
4749
4750        assert!(
4751            !listed_uploads.contains(&("source-only.bin", source_upload.upload_id.as_str())),
4752            "active source pool upload must be hidden from multipart listing"
4753        );
4754        assert!(
4755            listed_uploads.contains(&("target-visible.bin", target_upload.upload_id.as_str())),
4756            "non-rebalancing pool upload must remain visible"
4757        );
4758
4759        shutdown.cancel();
4760    }
4761
4762    #[tokio::test]
4763    #[serial_test::serial(storage_class_env)]
4764    async fn suspended_decommission_source_multipart_remains_operable_until_drained() {
4765        let temp_dir = tempfile::tempdir().expect("create decommission multipart drain store dir");
4766        let (_ctx, store, shutdown) =
4767            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "decommission-multipart-drain", &[4, 4])).await;
4768        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
4769
4770        let bucket = format!("decommission-mp-drain-{}", uuid::Uuid::new_v4());
4771        let complete_object = "complete.bin";
4772        let abort_object = "abort.bin";
4773        store
4774            .make_bucket(&bucket, &MakeBucketOptions::default())
4775            .await
4776            .expect("create decommission multipart drain bucket");
4777
4778        let incarnation = store.bucket_incarnation_id(&bucket).await.expect("read bucket incarnation");
4779        let lifecycle_guard = store
4780            .acquire_bucket_lifecycle_read_lock(&bucket)
4781            .await
4782            .expect("acquire multipart creation lifecycle fence");
4783        let mut upload_opts = ObjectOptions {
4784            expected_bucket_incarnation_id: Some(incarnation),
4785            ..Default::default()
4786        };
4787        upload_opts.add_bucket_lifecycle_lock_guard(&lifecycle_guard);
4788        let complete_upload = store.pools[0]
4789            .new_multipart_upload(&bucket, complete_object, &upload_opts)
4790            .await
4791            .expect("create source upload to complete");
4792        let abort_upload = store.pools[0]
4793            .new_multipart_upload(&bucket, abort_object, &upload_opts)
4794            .await
4795            .expect("create source upload to abort");
4796        drop(lifecycle_guard);
4797
4798        mark_test_pool_decommissioning(&store, 0).await;
4799        let err = store
4800            .ensure_decommission_multipart_uploads_drained_for_test(0)
4801            .await
4802            .expect_err("an unresolved source multipart upload must block final decommission");
4803        let drain_error = err.to_string();
4804        assert!(
4805            drain_error.contains("still contains multipart upload") && drain_error.contains(&bucket),
4806            "the drain error must identify both the upload path and user bucket: {drain_error}"
4807        );
4808
4809        let listed = store
4810            .list_multipart_uploads(&bucket, "", None, None, None, 100)
4811            .await
4812            .expect("list uploads from suspended decommission source");
4813        assert!(
4814            listed
4815                .uploads
4816                .iter()
4817                .any(|upload| upload.upload_id.as_str() == complete_upload.upload_id.as_str()),
4818            "the upload selected before suspension must remain visible"
4819        );
4820        store
4821            .get_multipart_info(&bucket, complete_object, &complete_upload.upload_id, &ObjectOptions::default())
4822            .await
4823            .expect("read upload metadata from suspended decommission source");
4824
4825        let mut part_reader = PutObjReader::from_vec(b"multipart body".to_vec());
4826        let part = store
4827            .put_object_part(
4828                &bucket,
4829                complete_object,
4830                &complete_upload.upload_id,
4831                1,
4832                &mut part_reader,
4833                &ObjectOptions::default(),
4834            )
4835            .await
4836            .expect("write part to suspended decommission source");
4837        let parts = store
4838            .list_object_parts(&bucket, complete_object, &complete_upload.upload_id, None, 100, &ObjectOptions::default())
4839            .await
4840            .expect("list parts from suspended decommission source");
4841        assert_eq!(parts.parts.len(), 1);
4842        assert_eq!(parts.parts[0].etag.as_deref(), part.etag.as_deref());
4843
4844        store
4845            .clone()
4846            .complete_multipart_upload(
4847                &bucket,
4848                complete_object,
4849                &complete_upload.upload_id,
4850                vec![crate::storage_api_contracts::multipart::CompletePart {
4851                    part_num: part.part_num,
4852                    etag: part.etag,
4853                    ..Default::default()
4854                }],
4855                &ObjectOptions::default(),
4856            )
4857            .await
4858            .expect("complete upload on suspended decommission source");
4859        store
4860            .abort_multipart_upload(&bucket, abort_object, &abort_upload.upload_id, &ObjectOptions::default())
4861            .await
4862            .expect("abort upload on suspended decommission source");
4863
4864        tokio::time::timeout(Duration::from_secs(5), async {
4865            loop {
4866                match store.ensure_decommission_multipart_uploads_drained_for_test(0).await {
4867                    Ok(()) => break,
4868                    Err(err) if err.to_string().contains("still contains multipart upload") => {
4869                        tokio::time::sleep(Duration::from_millis(10)).await;
4870                    }
4871                    Err(err) => panic!("unexpected final decommission drain error: {err:?}"),
4872                }
4873            }
4874        })
4875        .await
4876        .expect("final decommission gate should open after upload cleanup converges");
4877        assert_pool_object_present(&store.pools[0], &bucket, complete_object).await;
4878
4879        shutdown.cancel();
4880    }
4881
4882    #[tokio::test]
4883    #[serial_test::serial(storage_class_env)]
4884    async fn active_multipart_upload_routes_before_faulted_suspended_source() {
4885        use sha2::Digest;
4886
4887        let temp_dir = tempfile::tempdir().expect("create active-first multipart routing store dir");
4888        let (_ctx, store, shutdown) =
4889            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "active-first-multipart-routing", &[4, 4]))
4890                .await;
4891        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
4892
4893        let bucket = format!("active-first-mp-route-{}", uuid::Uuid::new_v4());
4894        let object = "target-upload.bin";
4895        store
4896            .make_bucket(&bucket, &MakeBucketOptions::default())
4897            .await
4898            .expect("create active-first multipart routing bucket");
4899
4900        let incarnation = store.bucket_incarnation_id(&bucket).await.expect("read bucket incarnation");
4901        let lifecycle_guard = store
4902            .acquire_bucket_lifecycle_read_lock(&bucket)
4903            .await
4904            .expect("acquire multipart creation lifecycle fence");
4905        let mut upload_opts = ObjectOptions {
4906            expected_bucket_incarnation_id: Some(incarnation),
4907            ..Default::default()
4908        };
4909        upload_opts.add_bucket_lifecycle_lock_guard(&lifecycle_guard);
4910        let upload = store.pools[1]
4911            .new_multipart_upload(&bucket, object, &upload_opts)
4912            .await
4913            .expect("create upload in active target pool");
4914        drop(lifecycle_guard);
4915
4916        mark_test_pool_decommissioning(&store, 0).await;
4917        // Corrupt only the source pool's entry for this UploadID. Taking the
4918        // whole source set offline would also make the bucket-incarnation
4919        // sidecar unreadable before multipart routing is reached.
4920        let upload_sha =
4921            hex_simd::encode_to_string(sha2::Sha256::digest(format!("{bucket}/{object}").as_bytes()), hex_simd::AsciiCase::Lower);
4922        let upload_uuid = crate::runtime::sources::upload_uuid_suffix(&upload.upload_id);
4923        for disk_index in 0..4 {
4924            let metadata_path = temp_dir
4925                .path()
4926                .join(format!("pool0/set0/disk{disk_index}"))
4927                .join(crate::disk::RUSTFS_META_MULTIPART_BUCKET)
4928                .join(&upload_sha)
4929                .join(&upload_uuid)
4930                .join(crate::disk::STORAGE_FORMAT_FILE);
4931            tokio::fs::create_dir_all(metadata_path.parent().expect("multipart metadata path should have a parent"))
4932                .await
4933                .expect("create corrupt source upload directory");
4934            tokio::fs::write(metadata_path, b"not-xl-meta")
4935                .await
4936                .expect("inject source upload metadata read failure");
4937        }
4938
4939        let source_result = store.pools[0]
4940            .get_multipart_info(&bucket, object, &upload.upload_id, &ObjectOptions::default())
4941            .await;
4942        let routed_result = store
4943            .get_multipart_info(&bucket, object, &upload.upload_id, &ObjectOptions::default())
4944            .await;
4945
4946        assert!(
4947            source_result
4948                .as_ref()
4949                .is_err_and(|err| !crate::error::is_err_invalid_upload_id(err)),
4950            "the suspended source must expose the injected hard read failure: {source_result:?}"
4951        );
4952        let routed = routed_result.expect("the active target UploadID must be resolved before the faulted suspended source");
4953        assert_eq!(routed.upload_id, upload.upload_id);
4954
4955        shutdown.cancel();
4956    }
4957
4958    #[tokio::test]
4959    #[serial_test::serial(storage_class_env)]
4960    async fn delete_objects_skips_active_rebalance_source_pool() {
4961        let temp_dir = tempfile::tempdir().expect("create batch-delete writer-fencing store dir");
4962        let (_ctx, store, shutdown) =
4963            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "batch-delete-rebalance", &[4, 4])).await;
4964        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
4965
4966        let bucket = format!("batch-delete-rebalance-{}", uuid::Uuid::new_v4());
4967        let object = "delete-me.bin";
4968        store
4969            .make_bucket(&bucket, &MakeBucketOptions::default())
4970            .await
4971            .expect("create batch delete rebalance bucket");
4972
4973        let mut source_reader = PutObjReader::from_vec(b"source-body".to_vec());
4974        store.pools[0]
4975            .put_object(&bucket, object, &mut source_reader, &ObjectOptions::default())
4976            .await
4977            .expect("write object on active source pool");
4978        let mut target_reader = PutObjReader::from_vec(b"target-body".to_vec());
4979        store.pools[1]
4980            .put_object(&bucket, object, &mut target_reader, &ObjectOptions::default())
4981            .await
4982            .expect("write object on non-rebalancing target pool");
4983
4984        let mut pool_stats = vec![RebalanceStats::default(); store.pools.len()];
4985        pool_stats[0] = RebalanceStats {
4986            participating: true,
4987            info: RebalanceInfo {
4988                start_time: Some(OffsetDateTime::now_utc()),
4989                status: RebalStatus::Started,
4990                ..Default::default()
4991            },
4992            ..Default::default()
4993        };
4994        *store.rebalance_meta.write().await = Some(RebalanceMeta {
4995            id: uuid::Uuid::new_v4().to_string(),
4996            pool_stats,
4997            ..Default::default()
4998        });
4999        assert!(store.is_pool_rebalancing(0).await, "pool 0 must be marked as an active rebalance source");
5000
5001        let (deleted, errs) = store
5002            .delete_objects(
5003                &bucket,
5004                vec![crate::storage_api_contracts::object::ObjectToDelete {
5005                    object_name: object.to_string(),
5006                    ..Default::default()
5007                }],
5008                ObjectOptions::default(),
5009            )
5010            .await;
5011        assert!(matches!(errs.as_slice(), [None]), "batch delete must not fail: {errs:?}");
5012        assert!(
5013            matches!(deleted.as_slice(), [deleted] if deleted.found && deleted.object_name == object),
5014            "batch delete must report the non-rebalancing pool deletion"
5015        );
5016
5017        store.pools[0]
5018            .get_object_info(&bucket, object, &ObjectOptions::default())
5019            .await
5020            .expect("active source pool object must not be deleted by DeleteObjects");
5021        let target_err = store.pools[1]
5022            .get_object_info(&bucket, object, &ObjectOptions::default())
5023            .await
5024            .expect_err("non-rebalancing target pool object must be deleted");
5025        assert!(
5026            matches!(target_err, StorageError::ObjectNotFound(_, _)),
5027            "target pool should report object not found after DeleteObjects, got {target_err:?}"
5028        );
5029
5030        shutdown.cancel();
5031    }
5032
5033    #[test]
5034    #[serial_test::serial(storage_class_env)]
5035    fn data_movement_conflicts_and_multipart_retries_converge_safely() {
5036        // Run this large async scenario on a larger dedicated stack so debug
5037        // test threads do not overflow before the regression assertions.
5038        std::thread::Builder::new()
5039            .stack_size(16 * 1024 * 1024)
5040            .spawn(|| {
5041                let runtime = tokio::runtime::Builder::new_multi_thread()
5042                    .enable_all()
5043                    .worker_threads(2)
5044                    .build()
5045                    .expect("build data movement test runtime");
5046                runtime.block_on(async move {
5047                    let temp_dir = tempfile::tempdir().expect("create data movement store dir");
5048                    let (_ctx, store, _shutdown) = without_storage_class_env(build_isolated_test_store(
5049                        temp_dir.path(),
5050                        "data-movement-conflict-convergence",
5051                        &[4, 4],
5052                    ))
5053                    .await;
5054                    crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
5055
5056                    let bucket = format!("data-movement-conflict-{}", uuid::Uuid::new_v4());
5057                    store
5058                        .make_bucket(&bucket, &MakeBucketOptions::default())
5059                        .await
5060                        .expect("create data movement bucket");
5061                    let source_mod_time = OffsetDateTime::UNIX_EPOCH + time::Duration::SECOND;
5062                    let target_mod_time = source_mod_time + time::Duration::SECOND;
5063
5064                    let object = "single-object";
5065                    let target_body = b"newer client body".to_vec();
5066                    let mut target_reader = PutObjReader::from_vec(target_body.clone());
5067                    store.pools[1]
5068                        .put_object(
5069                            &bucket,
5070                            object,
5071                            &mut target_reader,
5072                            &ObjectOptions {
5073                                mod_time: Some(target_mod_time),
5074                                ..Default::default()
5075                            },
5076                        )
5077                        .await
5078                        .expect("write newer single-part target");
5079
5080                    let source_body = b"stale migration body".to_vec();
5081                    let source_reader = seed_data_movement_source_reader(
5082                        &store,
5083                        &bucket,
5084                        object,
5085                        DataMovementSourceSeed {
5086                            pool_idx: 0,
5087                            version_id: None,
5088                            body: source_body,
5089                            multipart_split: None,
5090                            mod_time: source_mod_time,
5091                            user_defined: HashMap::new(),
5092                        },
5093                    )
5094                    .await;
5095                    crate::data_movement::migrate_object(
5096                        store.clone(),
5097                        0,
5098                        bucket.clone(),
5099                        source_reader,
5100                        None,
5101                        "test_data_movement",
5102                    )
5103                    .await
5104                    .expect("newer single-part target should converge migration");
5105
5106                    let mut reader = store.pools[1]
5107                        .get_object_reader(&bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
5108                        .await
5109                        .expect("read converged single-part target");
5110                    let mut body = Vec::new();
5111                    reader.stream.read_to_end(&mut body).await.expect("drain single-part target");
5112                    assert_eq!(body, target_body);
5113
5114                    let multipart_object = "multipart-object";
5115                    let multipart_target_body = b"newer multipart client body".to_vec();
5116                    let mut multipart_target_reader = PutObjReader::from_vec(multipart_target_body.clone());
5117                    store.pools[1]
5118                        .put_object(
5119                            &bucket,
5120                            multipart_object,
5121                            &mut multipart_target_reader,
5122                            &ObjectOptions {
5123                                mod_time: Some(target_mod_time),
5124                                ..Default::default()
5125                            },
5126                        )
5127                        .await
5128                        .expect("write newer multipart target");
5129
5130                    let first_part_size = 5 * 1024 * 1024;
5131                    let mut multipart_source_body = vec![b'a'; first_part_size];
5132                    multipart_source_body.push(b'b');
5133                    let multipart_source_reader = seed_data_movement_source_reader(
5134                        &store,
5135                        &bucket,
5136                        multipart_object,
5137                        DataMovementSourceSeed {
5138                            pool_idx: 0,
5139                            version_id: None,
5140                            body: multipart_source_body,
5141                            multipart_split: Some(first_part_size),
5142                            mod_time: source_mod_time,
5143                            user_defined: HashMap::new(),
5144                        },
5145                    )
5146                    .await;
5147                    crate::data_movement::migrate_object(
5148                        store.clone(),
5149                        0,
5150                        bucket.clone(),
5151                        multipart_source_reader,
5152                        None,
5153                        "test_data_movement",
5154                    )
5155                    .await
5156                    .expect("newer multipart target should converge migration");
5157
5158                    let uploads = store.pools[1]
5159                        .list_multipart_uploads(&bucket, multipart_object, None, None, None, 100)
5160                        .await
5161                        .expect("list target pool multipart uploads");
5162                    assert!(uploads.uploads.is_empty(), "superseded migration staging must be aborted");
5163
5164                    let mut reader = store.pools[1]
5165                        .get_object_reader(&bucket, multipart_object, None, HeaderMap::new(), &ObjectOptions::default())
5166                        .await
5167                        .expect("read converged multipart target");
5168                    let mut body = Vec::new();
5169                    reader.stream.read_to_end(&mut body).await.expect("drain multipart target");
5170                    assert_eq!(body, multipart_target_body);
5171
5172                    let retry_object = "multipart-retry-object";
5173                    let retry_object_etag = "0123456789abcdef0123456789abcdef".to_string();
5174                    let retry_first_part_size = 5 * 1024 * 1024;
5175                    let retry_object_mod_time =
5176                        OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("fixed retry timestamp should be valid");
5177                    let mut retry_source_body = vec![b'c'; retry_first_part_size];
5178                    retry_source_body.push(b'd');
5179                    let retry_first_checksum =
5180                        Checksum::new_from_data(ChecksumType::CRC32C, &retry_source_body[..retry_first_part_size])
5181                            .expect("first source part checksum should be created");
5182                    let retry_second_checksum =
5183                        Checksum::new_from_data(ChecksumType::CRC32C, &retry_source_body[retry_first_part_size..])
5184                            .expect("second source part checksum should be created");
5185                    let mut retry_checksum_parts = retry_first_checksum.raw.clone();
5186                    retry_checksum_parts.extend_from_slice(&retry_second_checksum.raw);
5187                    let mut retry_checksum_type = ChecksumType::CRC32C;
5188                    retry_checksum_type
5189                        .merge(ChecksumType::MULTIPART)
5190                        .merge(ChecksumType::INCLUDES_MULTIPART);
5191                    let retry_object_checksum = Checksum::new_from_data(retry_checksum_type, &retry_checksum_parts)
5192                        .expect("source multipart checksum should be created");
5193                    let retry_object_checksum_bytes = retry_object_checksum.to_bytes(&retry_checksum_parts);
5194                    let mut retry_metadata = HashMap::from([("x-amz-meta-retry".to_string(), "stable".to_string())]);
5195                    insert_str(
5196                        &mut retry_metadata,
5197                        SUFFIX_COMPRESSION,
5198                        compression_metadata_value(CompressionAlgorithm::default()),
5199                    );
5200                    let retry_upload = store.pools[0]
5201                        .new_multipart_upload(
5202                            &bucket,
5203                            retry_object,
5204                            &ObjectOptions {
5205                                user_defined: retry_metadata,
5206                                want_checksum: Some(retry_object_checksum.clone()),
5207                                ..Default::default()
5208                            },
5209                        )
5210                        .await
5211                        .expect("create source multipart upload");
5212                    let mut retry_parts = Vec::with_capacity(2);
5213                    for (part_number, plaintext, checksum) in [
5214                        (1, &retry_source_body[..retry_first_part_size], &retry_first_checksum),
5215                        (2, &retry_source_body[retry_first_part_size..], &retry_second_checksum),
5216                    ] {
5217                        let part_size = i64::try_from(plaintext.len()).expect("source part size should fit i64");
5218                        let mut plaintext_reader = rustfs_rio::HashReader::from_stream(
5219                            Cursor::new(plaintext.to_vec()),
5220                            part_size,
5221                            part_size,
5222                            None,
5223                            None,
5224                            false,
5225                        )
5226                        .expect("create source part plaintext reader");
5227                        plaintext_reader
5228                            .add_non_trailing_checksum(Some(checksum.clone()), false)
5229                            .expect("set source part checksum");
5230                        let compressed = WritePlan::new()
5231                            .with_compression(CompressionAlgorithm::default())
5232                            .apply(plaintext_reader, part_size)
5233                            .expect("compress source part");
5234                        let mut reader = PutObjReader::new(compressed);
5235                        let part = store.pools[0]
5236                            .put_object_part(
5237                                &bucket,
5238                                retry_object,
5239                                &retry_upload.upload_id,
5240                                part_number,
5241                                &mut reader,
5242                                &ObjectOptions::default(),
5243                            )
5244                            .await
5245                            .expect("write source multipart part");
5246                        retry_parts.push(crate::storage_api_contracts::multipart::CompletePart {
5247                            part_num: part.part_num,
5248                            etag: part.etag,
5249                            checksum_crc32c: Some(checksum.encoded.clone()),
5250                            ..Default::default()
5251                        });
5252                    }
5253                    store.pools[0]
5254                        .clone()
5255                        .complete_multipart_upload(
5256                            &bucket,
5257                            retry_object,
5258                            &retry_upload.upload_id,
5259                            retry_parts,
5260                            &ObjectOptions {
5261                                mod_time: Some(retry_object_mod_time),
5262                                want_checksum: Some(retry_object_checksum),
5263                                preserve_etag: Some(retry_object_etag.clone()),
5264                                ..Default::default()
5265                            },
5266                        )
5267                        .await
5268                        .expect("complete source multipart object");
5269
5270                    *store.rebalance_meta.write().await = Some(active_rebalance_meta_for_pool(store.pools.len(), 0));
5271                    assert!(store.is_pool_rebalancing(0).await, "pool 0 must be marked as an active rebalance source");
5272
5273                    let mut retry_reader = store.pools[0]
5274                        .get_object_reader(
5275                            &bucket,
5276                            retry_object,
5277                            None,
5278                            HeaderMap::new(),
5279                            &ObjectOptions {
5280                                raw_data_movement_read: true,
5281                                ..Default::default()
5282                            },
5283                        )
5284                        .await
5285                        .expect("read source multipart object for first migration");
5286                    let mut retry_source_info = retry_reader.object_info.clone();
5287                    let mut retry_source_parts = retry_source_info.parts.as_ref().clone();
5288                    for (part, checksum) in retry_source_parts
5289                        .iter_mut()
5290                        .zip([&retry_first_checksum, &retry_second_checksum])
5291                    {
5292                        part.mod_time = Some(OffsetDateTime::UNIX_EPOCH);
5293                        part.checksums = Some(HashMap::from([(ChecksumType::CRC32C.to_string(), checksum.encoded.clone())]));
5294                    }
5295                    retry_source_info.parts = Arc::new(retry_source_parts);
5296                    assert_eq!(retry_source_info.etag.as_deref(), Some(retry_object_etag.as_str()));
5297                    assert!(retry_source_info.is_multipart());
5298                    assert!(retry_source_info.parts.iter().all(|part| part.checksums.is_some()));
5299                    assert_eq!(retry_source_info.checksum.as_deref(), Some(retry_object_checksum_bytes.as_ref()));
5300                    assert!(
5301                        !retry_source_info
5302                            .user_defined
5303                            .contains_key(rustfs_rio::RUSTFS_MULTIPART_CHECKSUM)
5304                    );
5305                    assert!(
5306                        !retry_source_info
5307                            .user_defined
5308                            .contains_key(rustfs_rio::RUSTFS_MULTIPART_CHECKSUM_TYPE)
5309                    );
5310                    retry_reader.object_info = retry_source_info.clone();
5311                    temp_env::async_with_vars(
5312                        [
5313                            (rustfs_config::ENV_DATA_MOVEMENT_PART_CHECKSUMS_WRITE, None::<&str>),
5314                            (rustfs_config::ENV_DATA_MOVEMENT_PART_CHECKSUMS_FLEET_CONFIRMED, None::<&str>),
5315                        ],
5316                        crate::data_movement::migrate_object(
5317                            store.clone(),
5318                            0,
5319                            bucket.clone(),
5320                            retry_reader,
5321                            None,
5322                            "test_data_movement_compatible_retry",
5323                        ),
5324                    )
5325                    .await
5326                    .expect("default multipart migration should not require the checksum sidecar capability");
5327
5328                    let compatible_target = store.pools[1]
5329                        .get_object_info(
5330                            &bucket,
5331                            retry_object,
5332                            &ObjectOptions {
5333                                include_part_checksums: true,
5334                                ..Default::default()
5335                            },
5336                        )
5337                        .await
5338                        .expect("read compatible multipart target metadata");
5339                    assert_eq!(compatible_target.checksum, retry_source_info.checksum);
5340                    assert!(compatible_target.parts.iter().all(|part| part.checksums.is_none()));
5341                    assert_eq!(
5342                        rustfs_utils::http::get_consistent_str(
5343                            &compatible_target.user_defined,
5344                            rustfs_utils::http::SUFFIX_DATA_MOVED
5345                        ),
5346                        Some("true")
5347                    );
5348                    assert!(!rustfs_utils::http::contains_key_str(
5349                        &compatible_target.user_defined,
5350                        rustfs_utils::http::SUFFIX_PART_CHECKSUMS
5351                    ));
5352                    let mut compatible_reader = store.pools[1]
5353                        .get_object_reader(&bucket, retry_object, None, HeaderMap::new(), &ObjectOptions::default())
5354                        .await
5355                        .expect("read compatible multipart target body");
5356                    let mut compatible_body = Vec::new();
5357                    compatible_reader
5358                        .stream
5359                        .read_to_end(&mut compatible_body)
5360                        .await
5361                        .expect("drain compatible multipart target body");
5362                    assert_eq!(compatible_body, retry_source_body);
5363
5364                    let mut compatible_retry_reader = store.pools[0]
5365                        .get_object_reader(
5366                            &bucket,
5367                            retry_object,
5368                            None,
5369                            HeaderMap::new(),
5370                            &ObjectOptions {
5371                                raw_data_movement_read: true,
5372                                ..Default::default()
5373                            },
5374                        )
5375                        .await
5376                        .expect("read source multipart object for compatible retry");
5377                    compatible_retry_reader.object_info = retry_source_info.clone();
5378                    temp_env::async_with_vars(
5379                        [
5380                            (rustfs_config::ENV_DATA_MOVEMENT_PART_CHECKSUMS_WRITE, Some("true")),
5381                            (rustfs_config::ENV_DATA_MOVEMENT_PART_CHECKSUMS_FLEET_CONFIRMED, Some("true")),
5382                        ],
5383                        crate::data_movement::migrate_object(
5384                            store.clone(),
5385                            0,
5386                            bucket.clone(),
5387                            compatible_retry_reader,
5388                            None,
5389                            "test_data_movement_compatible_retry",
5390                        ),
5391                    )
5392                    .await
5393                    .expect("pre-gate multipart target should converge after enabling checksum persistence");
5394                    let compatible_target_after_retry = store.pools[1]
5395                        .get_object_info(
5396                            &bucket,
5397                            retry_object,
5398                            &ObjectOptions {
5399                                include_part_checksums: true,
5400                                ..Default::default()
5401                            },
5402                        )
5403                        .await
5404                        .expect("read pre-gate target after fleet-confirmed retry");
5405                    assert_eq!(compatible_target_after_retry.data_dir, compatible_target.data_dir);
5406                    assert_eq!(compatible_target_after_retry.parts, compatible_target.parts);
5407                    let compatible_uploads = store.pools[1]
5408                        .list_multipart_uploads(&bucket, retry_object, None, None, None, 100)
5409                        .await
5410                        .expect("list compatible target multipart uploads");
5411                    assert!(compatible_uploads.uploads.is_empty(), "compatible retry staging must be aborted");
5412
5413                    store.pools[1]
5414                        .delete_object(&bucket, retry_object, ObjectOptions::default())
5415                        .await
5416                        .expect("remove compatible target before fleet-confirmed migration");
5417                    let mut retry_reader = store.pools[0]
5418                        .get_object_reader(
5419                            &bucket,
5420                            retry_object,
5421                            None,
5422                            HeaderMap::new(),
5423                            &ObjectOptions {
5424                                raw_data_movement_read: true,
5425                                ..Default::default()
5426                            },
5427                        )
5428                        .await
5429                        .expect("read source multipart object for fleet-confirmed migration");
5430                    retry_reader.object_info = retry_source_info.clone();
5431                    temp_env::async_with_vars(
5432                        [
5433                            (rustfs_config::ENV_DATA_MOVEMENT_PART_CHECKSUMS_WRITE, Some("true")),
5434                            (rustfs_config::ENV_DATA_MOVEMENT_PART_CHECKSUMS_FLEET_CONFIRMED, Some("true")),
5435                        ],
5436                        crate::data_movement::migrate_object(
5437                            store.clone(),
5438                            0,
5439                            bucket.clone(),
5440                            retry_reader,
5441                            None,
5442                            "test_data_movement_retry",
5443                        ),
5444                    )
5445                    .await
5446                    .expect("first multipart migration should succeed");
5447
5448                    let target_before_retry = store.pools[1]
5449                        .get_object_info(
5450                            &bucket,
5451                            retry_object,
5452                            &ObjectOptions {
5453                                include_part_checksums: true,
5454                                ..Default::default()
5455                            },
5456                        )
5457                        .await
5458                        .expect("read target metadata before retry");
5459                    assert_eq!(target_before_retry.checksum, retry_source_info.checksum);
5460                    assert_eq!(
5461                        target_before_retry
5462                            .parts
5463                            .iter()
5464                            .map(|part| (&part.number, &part.checksums))
5465                            .collect::<Vec<_>>(),
5466                        retry_source_info
5467                            .parts
5468                            .iter()
5469                            .map(|part| (&part.number, &part.checksums))
5470                            .collect::<Vec<_>>()
5471                    );
5472                    assert_eq!(retry_source_info.mod_time, target_before_retry.mod_time);
5473                    assert!(
5474                        retry_source_info.parts.iter().any(|source_part| {
5475                            target_before_retry
5476                                .parts
5477                                .iter()
5478                                .find(|target_part| target_part.number == source_part.number)
5479                                .is_some_and(|target_part| target_part.mod_time != source_part.mod_time)
5480                        }),
5481                        "multipart migration must rewrite at least one target part timestamp"
5482                    );
5483                    let mut retry_reader = store.pools[0]
5484                        .get_object_reader(
5485                            &bucket,
5486                            retry_object,
5487                            None,
5488                            HeaderMap::new(),
5489                            &ObjectOptions {
5490                                raw_data_movement_read: true,
5491                                ..Default::default()
5492                            },
5493                        )
5494                        .await
5495                        .expect("read source multipart object for retry");
5496                    retry_reader.object_info = retry_source_info;
5497                    temp_env::async_with_vars(
5498                        [
5499                            (rustfs_config::ENV_DATA_MOVEMENT_PART_CHECKSUMS_WRITE, Some("true")),
5500                            (rustfs_config::ENV_DATA_MOVEMENT_PART_CHECKSUMS_FLEET_CONFIRMED, Some("true")),
5501                        ],
5502                        crate::data_movement::migrate_object(
5503                            store.clone(),
5504                            0,
5505                            bucket.clone(),
5506                            retry_reader,
5507                            None,
5508                            "test_data_movement_retry",
5509                        ),
5510                    )
5511                    .await
5512                    .expect("equivalent multipart migration retry should converge");
5513
5514                    let target_after_retry = store.pools[1]
5515                        .get_object_info(&bucket, retry_object, &ObjectOptions::default())
5516                        .await
5517                        .expect("read target metadata after retry");
5518                    assert_eq!(target_after_retry.data_dir, target_before_retry.data_dir);
5519                    assert_eq!(target_after_retry.etag, target_before_retry.etag);
5520                    assert_eq!(target_after_retry.checksum, target_before_retry.checksum);
5521                    assert_eq!(target_after_retry.mod_time, target_before_retry.mod_time);
5522                    assert!(!rustfs_utils::http::contains_key_str(
5523                        &target_after_retry.user_defined,
5524                        rustfs_utils::http::SUFFIX_PART_CHECKSUMS
5525                    ));
5526                    assert!(target_after_retry.parts.iter().all(|part| part.checksums.is_none()));
5527                    let cached_target_after_retry = store.pools[1]
5528                        .get_object_info(&bucket, retry_object, &ObjectOptions::default())
5529                        .await
5530                        .expect("read target metadata from the ordinary cache path");
5531                    assert!(!rustfs_utils::http::contains_key_str(
5532                        &cached_target_after_retry.user_defined,
5533                        rustfs_utils::http::SUFFIX_PART_CHECKSUMS
5534                    ));
5535                    assert!(cached_target_after_retry.parts.iter().all(|part| part.checksums.is_none()));
5536                    let hydrated_target_after_retry = store.pools[1]
5537                        .get_object_info(
5538                            &bucket,
5539                            retry_object,
5540                            &ObjectOptions {
5541                                include_part_checksums: true,
5542                                ..Default::default()
5543                            },
5544                        )
5545                        .await
5546                        .expect("hydrate target metadata after ordinary cache reads");
5547                    assert_eq!(hydrated_target_after_retry.user_defined, target_before_retry.user_defined);
5548                    assert_eq!(hydrated_target_after_retry.parts, target_before_retry.parts);
5549
5550                    let mut target_reader = store.pools[1]
5551                        .get_object_reader(&bucket, retry_object, None, HeaderMap::new(), &ObjectOptions::default())
5552                        .await
5553                        .expect("read target multipart body after retry");
5554                    let mut target_body = Vec::new();
5555                    target_reader
5556                        .stream
5557                        .read_to_end(&mut target_body)
5558                        .await
5559                        .expect("drain target multipart body after retry");
5560                    assert_eq!(target_body, retry_source_body);
5561
5562                    let uploads = store.pools[1]
5563                        .list_multipart_uploads(&bucket, retry_object, None, None, None, 100)
5564                        .await
5565                        .expect("list retry target multipart uploads");
5566                    assert!(uploads.uploads.is_empty(), "retry migration staging must be aborted");
5567
5568                    Box::pin(async {
5569                        store.pools[0]
5570                            .delete_object(&bucket, retry_object, ObjectOptions::default())
5571                            .await
5572                            .expect("remove first-hop source before second migration");
5573                        *store.rebalance_meta.write().await = Some(active_rebalance_meta_for_pool(store.pools.len(), 1));
5574                        let second_hop_reader = store.pools[1]
5575                            .get_object_reader(
5576                                &bucket,
5577                                retry_object,
5578                                None,
5579                                HeaderMap::new(),
5580                                &ObjectOptions {
5581                                    data_movement: true,
5582                                    raw_data_movement_read: true,
5583                                    ..Default::default()
5584                                },
5585                            )
5586                            .await
5587                            .expect("read first target as the second migration source");
5588                        assert!(
5589                            second_hop_reader
5590                                .object_info
5591                                .parts
5592                                .iter()
5593                                .all(|part| part.checksums.is_some()),
5594                            "data-movement source reads must hydrate persisted part checksums"
5595                        );
5596                        temp_env::async_with_vars(
5597                            [
5598                                (rustfs_config::ENV_DATA_MOVEMENT_PART_CHECKSUMS_WRITE, Some("true")),
5599                                (rustfs_config::ENV_DATA_MOVEMENT_PART_CHECKSUMS_FLEET_CONFIRMED, Some("true")),
5600                            ],
5601                            crate::data_movement::migrate_object(
5602                                store.clone(),
5603                                1,
5604                                bucket.clone(),
5605                                second_hop_reader,
5606                                None,
5607                                "test_data_movement_second_hop",
5608                            ),
5609                        )
5610                        .await
5611                        .expect("second multipart migration should preserve part checksums");
5612
5613                        let second_hop_target = store.pools[0]
5614                            .get_object_info(
5615                                &bucket,
5616                                retry_object,
5617                                &ObjectOptions {
5618                                    include_part_checksums: true,
5619                                    ..Default::default()
5620                                },
5621                            )
5622                            .await
5623                            .expect("read second-hop target metadata");
5624                        assert_eq!(second_hop_target.checksum, target_before_retry.checksum);
5625                        assert_eq!(
5626                            second_hop_target
5627                                .parts
5628                                .iter()
5629                                .map(|part| (&part.number, &part.checksums))
5630                                .collect::<Vec<_>>(),
5631                            target_before_retry
5632                                .parts
5633                                .iter()
5634                                .map(|part| (&part.number, &part.checksums))
5635                                .collect::<Vec<_>>()
5636                        );
5637                    })
5638                    .await;
5639                });
5640            })
5641            .expect("spawn data movement test thread")
5642            .join()
5643            .expect("join data movement test thread");
5644    }
5645
5646    #[test]
5647    #[serial_test::serial(storage_class_env)]
5648    fn data_movement_multipart_replaces_only_unlocked_owned_generation() {
5649        std::thread::Builder::new()
5650            .stack_size(16 * 1024 * 1024)
5651            .spawn(|| {
5652                let runtime = tokio::runtime::Builder::new_multi_thread()
5653                    .enable_all()
5654                    .worker_threads(2)
5655                    .build()
5656                    .expect("build data movement replacement test runtime");
5657                runtime.block_on(async move {
5658                    let temp_dir = tempfile::tempdir().expect("create data movement replacement store dir");
5659                    let (_ctx, store, _shutdown) = without_storage_class_env(build_isolated_test_store(
5660                        temp_dir.path(),
5661                        "data-movement-stale-replacement",
5662                        &[4, 4],
5663                    ))
5664                    .await;
5665                    crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
5666
5667                    let bucket = format!("dm-stale-replacement-{}", uuid::Uuid::new_v4());
5668                    store
5669                        .make_bucket(
5670                            &bucket,
5671                            &MakeBucketOptions {
5672                                lock_enabled: true,
5673                                ..Default::default()
5674                            },
5675                        )
5676                        .await
5677                        .expect("create data movement replacement bucket");
5678                    *store.rebalance_meta.write().await = Some(active_rebalance_meta_for_pool(store.pools.len(), 0));
5679
5680                    let version_id = uuid::Uuid::new_v4();
5681                    let old_time = OffsetDateTime::UNIX_EPOCH + time::Duration::SECOND;
5682                    let new_time = old_time + time::Duration::SECOND;
5683                    let first_part_size = 5 * 1024 * 1024;
5684                    let mut old_body = vec![b'a'; first_part_size];
5685                    old_body.push(b'b');
5686                    let mut new_body = vec![b'c'; first_part_size];
5687                    new_body.push(b'd');
5688
5689                    let source_reader = |name: &'static str, body: Vec<u8>, mod_time, metadata: HashMap<String, String>| {
5690                        seed_data_movement_source_reader(
5691                            &store,
5692                            &bucket,
5693                            name,
5694                            DataMovementSourceSeed {
5695                                pool_idx: 0,
5696                                version_id: Some(version_id),
5697                                body,
5698                                multipart_split: Some(first_part_size),
5699                                mod_time,
5700                                user_defined: metadata,
5701                            },
5702                        )
5703                    };
5704
5705                    let replaceable = "replaceable.bin";
5706                    crate::data_movement::migrate_object(
5707                        store.clone(),
5708                        0,
5709                        bucket.clone(),
5710                        source_reader(
5711                            replaceable,
5712                            old_body.clone(),
5713                            old_time,
5714                            HashMap::from([("x-amz-meta-generation".to_string(), "old".to_string())]),
5715                        )
5716                        .await,
5717                        None,
5718                        "test_stale_target_seed",
5719                    )
5720                    .await
5721                    .expect("seed old migrated target");
5722                    let seeded_target = store.pools[1]
5723                        .get_object_info(
5724                            &bucket,
5725                            replaceable,
5726                            &ObjectOptions {
5727                                versioned: true,
5728                                version_id: Some(version_id.to_string()),
5729                                ..Default::default()
5730                            },
5731                        )
5732                        .await
5733                        .expect("read old migrated target");
5734                    let replacement_opts = ObjectOptions {
5735                        data_movement: true,
5736                        versioned: true,
5737                        version_id: Some(version_id.to_string()),
5738                        mod_time: Some(new_time),
5739                        http_preconditions: Some(crate::data_movement::data_movement_target_precondition()),
5740                        ..Default::default()
5741                    };
5742                    assert!(
5743                        crate::data_movement::can_replace_stale_data_movement_target(&seeded_target, &replacement_opts),
5744                        "seeded target should be replaceable: {seeded_target:?}"
5745                    );
5746                    crate::data_movement::migrate_object(
5747                        store.clone(),
5748                        0,
5749                        bucket.clone(),
5750                        source_reader(
5751                            replaceable,
5752                            new_body.clone(),
5753                            new_time,
5754                            HashMap::from([("x-amz-meta-generation".to_string(), "new".to_string())]),
5755                        )
5756                        .await,
5757                        None,
5758                        "test_stale_target_replace",
5759                    )
5760                    .await
5761                    .expect("newer source generation should replace the old migrated target");
5762
5763                    let replacement = store.pools[1]
5764                        .get_object_info(
5765                            &bucket,
5766                            replaceable,
5767                            &ObjectOptions {
5768                                versioned: true,
5769                                version_id: Some(version_id.to_string()),
5770                                ..Default::default()
5771                            },
5772                        )
5773                        .await
5774                        .expect("read replaced target metadata");
5775                    assert_eq!(replacement.mod_time, Some(new_time));
5776                    assert_eq!(replacement.user_defined.get("x-amz-meta-generation").map(String::as_str), Some("new"));
5777                    let mut replacement_reader = store.pools[1]
5778                        .get_object_reader(
5779                            &bucket,
5780                            replaceable,
5781                            None,
5782                            HeaderMap::new(),
5783                            &ObjectOptions {
5784                                versioned: true,
5785                                version_id: Some(version_id.to_string()),
5786                                ..Default::default()
5787                            },
5788                        )
5789                        .await
5790                        .expect("read replaced target body");
5791                    let mut replacement_body = Vec::new();
5792                    replacement_reader
5793                        .stream
5794                        .read_to_end(&mut replacement_body)
5795                        .await
5796                        .expect("drain replaced target body");
5797                    assert_eq!(replacement_body, new_body);
5798
5799                    let client_target = "client-target.bin";
5800                    let client_body = b"client-owned exact version".to_vec();
5801                    let mut client_reader = PutObjReader::from_vec(client_body.clone());
5802                    store.pools[1]
5803                        .put_object(
5804                            &bucket,
5805                            client_target,
5806                            &mut client_reader,
5807                            &ObjectOptions {
5808                                versioned: true,
5809                                version_id: Some(version_id.to_string()),
5810                                mod_time: Some(old_time),
5811                                user_defined: HashMap::from([("x-amz-meta-owner".to_string(), "client".to_string())]),
5812                                ..Default::default()
5813                            },
5814                        )
5815                        .await
5816                        .expect("seed client-owned exact version");
5817                    let err = crate::data_movement::migrate_object(
5818                        store.clone(),
5819                        0,
5820                        bucket.clone(),
5821                        source_reader(client_target, new_body.clone(), new_time, HashMap::new()).await,
5822                        None,
5823                        "test_client_target_reject",
5824                    )
5825                    .await
5826                    .expect_err("data movement must not replace a client-owned exact version");
5827                    assert!(err.to_string().contains("complete_multipart_upload"), "unexpected migration error: {err}");
5828                    let mut preserved_reader = store.pools[1]
5829                        .get_object_reader(
5830                            &bucket,
5831                            client_target,
5832                            None,
5833                            HeaderMap::new(),
5834                            &ObjectOptions {
5835                                versioned: true,
5836                                version_id: Some(version_id.to_string()),
5837                                ..Default::default()
5838                            },
5839                        )
5840                        .await
5841                        .expect("read preserved client target");
5842                    let mut preserved_body = Vec::new();
5843                    preserved_reader
5844                        .stream
5845                        .read_to_end(&mut preserved_body)
5846                        .await
5847                        .expect("drain preserved client target");
5848                    assert_eq!(preserved_body, client_body);
5849
5850                    let acknowledged_target = "acknowledged-target.bin";
5851                    crate::data_movement::migrate_object(
5852                        store.clone(),
5853                        0,
5854                        bucket.clone(),
5855                        source_reader(
5856                            acknowledged_target,
5857                            old_body.clone(),
5858                            old_time,
5859                            HashMap::from([("x-amz-meta-generation".to_string(), "old".to_string())]),
5860                        )
5861                        .await,
5862                        None,
5863                        "test_acknowledged_target_seed",
5864                    )
5865                    .await
5866                    .expect("seed migrated target before tag acknowledgement");
5867                    let target_version_opts = ObjectOptions {
5868                        versioned: true,
5869                        version_id: Some(version_id.to_string()),
5870                        ..Default::default()
5871                    };
5872                    store
5873                        .put_object_tags(&bucket, acknowledged_target, "acknowledged=true", &target_version_opts)
5874                        .await
5875                        .expect("tag update should acknowledge the migrated target");
5876                    let acknowledged = store.pools[1]
5877                        .get_object_info(&bucket, acknowledged_target, &target_version_opts)
5878                        .await
5879                        .expect("read acknowledged migrated target");
5880                    let rustfs_data_moved_key = rustfs_utils::http::internal_key_rustfs(rustfs_utils::http::SUFFIX_DATA_MOVED);
5881                    let minio_data_moved_key =
5882                        format!("{}{}", rustfs_utils::http::MINIO_INTERNAL_PREFIX, rustfs_utils::http::SUFFIX_DATA_MOVED);
5883                    assert_eq!(acknowledged.user_defined.get(&rustfs_data_moved_key).map(String::as_str), Some(""));
5884                    assert_eq!(acknowledged.user_defined.get(&minio_data_moved_key).map(String::as_str), Some(""));
5885                    assert_eq!(
5886                        rustfs_utils::http::get_consistent_str(&acknowledged.user_defined, rustfs_utils::http::SUFFIX_DATA_MOVED),
5887                        None
5888                    );
5889
5890                    let err = crate::data_movement::migrate_object(
5891                        store.clone(),
5892                        0,
5893                        bucket.clone(),
5894                        source_reader(
5895                            acknowledged_target,
5896                            new_body.clone(),
5897                            new_time,
5898                            HashMap::from([("x-amz-meta-generation".to_string(), "new".to_string())]),
5899                        )
5900                        .await,
5901                        None,
5902                        "test_acknowledged_target_reject",
5903                    )
5904                    .await
5905                    .expect_err("data movement must not replace a target acknowledged by a tag update");
5906                    assert!(err.to_string().contains("complete_multipart_upload"), "unexpected migration error: {err}");
5907
5908                    let acknowledged_tags = store.pools[1]
5909                        .get_object_tags(&bucket, acknowledged_target, &target_version_opts)
5910                        .await
5911                        .expect("read preserved acknowledged target tags");
5912                    assert_eq!(acknowledged_tags, "acknowledged=true");
5913                    let mut acknowledged_reader = store.pools[1]
5914                        .get_object_reader(&bucket, acknowledged_target, None, HeaderMap::new(), &target_version_opts)
5915                        .await
5916                        .expect("read preserved acknowledged target body");
5917                    let mut acknowledged_body = Vec::new();
5918                    acknowledged_reader
5919                        .stream
5920                        .read_to_end(&mut acknowledged_body)
5921                        .await
5922                        .expect("drain preserved acknowledged target body");
5923                    assert_eq!(acknowledged_body, old_body);
5924
5925                    let metadata_acknowledged_target = "metadata-acknowledged-target.bin";
5926                    crate::data_movement::migrate_object(
5927                        store.clone(),
5928                        0,
5929                        bucket.clone(),
5930                        source_reader(
5931                            metadata_acknowledged_target,
5932                            old_body.clone(),
5933                            old_time,
5934                            HashMap::from([("x-amz-meta-generation".to_string(), "old".to_string())]),
5935                        )
5936                        .await,
5937                        None,
5938                        "test_metadata_acknowledged_target_seed",
5939                    )
5940                    .await
5941                    .expect("seed migrated target before metadata acknowledgement");
5942                    store
5943                        .put_object_metadata(
5944                            &bucket,
5945                            metadata_acknowledged_target,
5946                            &ObjectOptions {
5947                                eval_metadata: Some(HashMap::from([(
5948                                    rustfs_filemeta::metadata_keys::OBJECT_LOCK_LEGAL_HOLD.to_string(),
5949                                    s3s::dto::ObjectLockLegalHoldStatus::OFF.to_string(),
5950                                )])),
5951                                ..target_version_opts.clone()
5952                            },
5953                        )
5954                        .await
5955                        .expect("legal hold metadata update should acknowledge the migrated target");
5956                    let metadata_acknowledged = store.pools[1]
5957                        .get_object_info(&bucket, metadata_acknowledged_target, &target_version_opts)
5958                        .await
5959                        .expect("read metadata-acknowledged migrated target");
5960                    assert_eq!(
5961                        metadata_acknowledged
5962                            .user_defined
5963                            .get(&rustfs_data_moved_key)
5964                            .map(String::as_str),
5965                        Some("")
5966                    );
5967                    assert_eq!(
5968                        metadata_acknowledged
5969                            .user_defined
5970                            .get(&minio_data_moved_key)
5971                            .map(String::as_str),
5972                        Some("")
5973                    );
5974                    assert_eq!(
5975                        metadata_acknowledged
5976                            .user_defined
5977                            .get(rustfs_filemeta::metadata_keys::OBJECT_LOCK_LEGAL_HOLD)
5978                            .map(String::as_str),
5979                        Some("OFF")
5980                    );
5981                    let err = crate::data_movement::migrate_object(
5982                        store.clone(),
5983                        0,
5984                        bucket.clone(),
5985                        source_reader(
5986                            metadata_acknowledged_target,
5987                            new_body.clone(),
5988                            new_time,
5989                            HashMap::from([("x-amz-meta-generation".to_string(), "new".to_string())]),
5990                        )
5991                        .await,
5992                        None,
5993                        "test_metadata_acknowledged_target_reject",
5994                    )
5995                    .await
5996                    .expect_err("data movement must not replace a target acknowledged by a metadata update");
5997                    assert!(err.to_string().contains("complete_multipart_upload"), "unexpected migration error: {err}");
5998
5999                    let retain_until = (OffsetDateTime::now_utc() + time::Duration::days(1))
6000                        .format(&time::format_description::well_known::Rfc3339)
6001                        .expect("retain-until date should format");
6002                    for (object, mode) in [
6003                        ("compliance-target.bin", s3s::dto::ObjectLockRetentionMode::COMPLIANCE),
6004                        ("governance-target.bin", s3s::dto::ObjectLockRetentionMode::GOVERNANCE),
6005                    ] {
6006                        let retained_metadata = HashMap::from([
6007                            (rustfs_filemeta::metadata_keys::OBJECT_LOCK_MODE.to_string(), mode.to_string()),
6008                            (
6009                                rustfs_filemeta::metadata_keys::OBJECT_LOCK_RETAIN_UNTIL_DATE.to_string(),
6010                                retain_until.clone(),
6011                            ),
6012                        ]);
6013                        crate::data_movement::migrate_object(
6014                            store.clone(),
6015                            0,
6016                            bucket.clone(),
6017                            source_reader(object, old_body.clone(), old_time, retained_metadata.clone()).await,
6018                            None,
6019                            "test_retained_target_seed",
6020                        )
6021                        .await
6022                        .expect("seed retained migrated target");
6023                        let err = crate::data_movement::migrate_object(
6024                            store.clone(),
6025                            0,
6026                            bucket.clone(),
6027                            source_reader(object, new_body.clone(), new_time, retained_metadata).await,
6028                            None,
6029                            "test_retained_target_replace",
6030                        )
6031                        .await
6032                        .expect_err("active retention must block stale target replacement");
6033                        assert!(err.to_string().contains("complete_multipart_upload"), "unexpected retention error: {err}");
6034
6035                        let mut retained_reader = store.pools[1]
6036                            .get_object_reader(
6037                                &bucket,
6038                                object,
6039                                None,
6040                                HeaderMap::new(),
6041                                &ObjectOptions {
6042                                    versioned: true,
6043                                    version_id: Some(version_id.to_string()),
6044                                    ..Default::default()
6045                                },
6046                            )
6047                            .await
6048                            .expect("read retained target");
6049                        let mut retained_body = Vec::new();
6050                        retained_reader
6051                            .stream
6052                            .read_to_end(&mut retained_body)
6053                            .await
6054                            .expect("drain retained target");
6055                        assert_eq!(retained_body, old_body);
6056                    }
6057
6058                    for object in [
6059                        replaceable,
6060                        client_target,
6061                        acknowledged_target,
6062                        metadata_acknowledged_target,
6063                        "compliance-target.bin",
6064                        "governance-target.bin",
6065                    ] {
6066                        let uploads = store.pools[1]
6067                            .list_multipart_uploads(&bucket, object, None, None, None, 100)
6068                            .await
6069                            .expect("list target multipart staging");
6070                        assert!(uploads.uploads.is_empty(), "data movement staging must be cleaned for {object}");
6071                    }
6072                });
6073            })
6074            .expect("spawn data movement replacement test thread")
6075            .join()
6076            .expect("join data movement replacement test thread");
6077    }
6078
6079    #[tokio::test]
6080    #[serial_test::serial(storage_class_env)]
6081    async fn data_movement_delete_marker_retries_converge_safely() {
6082        let temp_dir = tempfile::tempdir().expect("create delete-marker data movement store dir");
6083        let (_ctx, store, _shutdown) =
6084            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "data-movement-delete-marker", &[4, 4])).await;
6085        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
6086
6087        let bucket = format!("dm-delete-{}", uuid::Uuid::new_v4());
6088        store
6089            .make_bucket(&bucket, &MakeBucketOptions::default())
6090            .await
6091            .expect("create delete-marker data movement bucket");
6092        let object = "delete-marker-retry";
6093        let version = uuid::Uuid::new_v4();
6094        let mod_time = OffsetDateTime::UNIX_EPOCH + time::Duration::SECOND;
6095        let replica_timestamp = mod_time + time::Duration::SECOND;
6096        let replication_timestamp = mod_time;
6097        let replica_timestamp_string = replica_timestamp
6098            .format(&time::format_description::well_known::Rfc3339)
6099            .expect("replica timestamp should format as RFC3339");
6100        let replication_timestamp_string = replication_timestamp
6101            .format(&time::format_description::well_known::Rfc3339)
6102            .expect("replication timestamp should format as RFC3339");
6103        let replication_status = "arn:minio:replication::TenantA:bucket=COMPLETED;";
6104        store.pools[0]
6105            .delete_object(
6106                &bucket,
6107                object,
6108                ObjectOptions {
6109                    versioned: true,
6110                    version_id: Some(version.to_string()),
6111                    delete_marker: true,
6112                    mod_time: Some(mod_time),
6113                    delete_replication: Some(ReplicationState {
6114                        replica_status: ReplicationStatusType::Replica,
6115                        replica_timestamp: Some(replica_timestamp),
6116                        replication_status_internal: Some(replication_status.to_string()),
6117                        replication_timestamp: Some(replication_timestamp),
6118                        targets: replication_statuses_map(replication_status),
6119                        delete_marker: true,
6120                        ..Default::default()
6121                    }),
6122                    ..Default::default()
6123                },
6124            )
6125            .await
6126            .expect("write source delete marker");
6127        let source = store.pools[0]
6128            .get_object_info(
6129                &bucket,
6130                object,
6131                &ObjectOptions {
6132                    versioned: true,
6133                    version_id: Some(version.to_string()),
6134                    ..Default::default()
6135                },
6136            )
6137            .await
6138            .expect("read source delete marker replication metadata");
6139        assert_eq!(
6140            rustfs_utils::http::get_str(&source.user_defined, rustfs_utils::http::SUFFIX_REPLICA_STATUS).as_deref(),
6141            Some("REPLICA"),
6142            "source metadata: {:?}",
6143            source.user_defined
6144        );
6145
6146        *store.rebalance_meta.write().await = Some(active_rebalance_meta_for_pool(store.pools.len(), 0));
6147        let missing_object = "missing-delete-marker";
6148        let missing_version = uuid::Uuid::new_v4();
6149        let missing_opts = ObjectOptions {
6150            versioned: true,
6151            version_id: Some(missing_version.to_string()),
6152            delete_marker: true,
6153            mod_time: Some(mod_time),
6154            src_pool_idx: 0,
6155            data_movement: true,
6156            skip_decommissioned: true,
6157            ..Default::default()
6158        };
6159        let missing_source_err = store
6160            .delete_object(&bucket, missing_object, missing_opts.clone())
6161            .await
6162            .expect_err("data movement must not recreate a delete marker missing from the source");
6163        assert!(matches!(missing_source_err, StorageError::DataMovementOverwriteErr(_, _, _)));
6164        let missing_target_err = store.pools[1]
6165            .get_object_info(&bucket, missing_object, &missing_opts)
6166            .await
6167            .expect_err("source-missing preflight must not write a target delete marker");
6168        assert!(matches!(
6169            missing_target_err,
6170            StorageError::ObjectNotFound(_, _) | StorageError::VersionNotFound(_, _, _)
6171        ));
6172
6173        let movement_opts = ObjectOptions {
6174            versioned: true,
6175            version_id: Some(version.to_string()),
6176            delete_marker: true,
6177            mod_time: Some(mod_time),
6178            src_pool_idx: 0,
6179            data_movement: true,
6180            skip_decommissioned: true,
6181            ..Default::default()
6182        };
6183        for attempt in 1..=2 {
6184            let moved = store
6185                .delete_object(&bucket, object, movement_opts.clone())
6186                .await
6187                .unwrap_or_else(|err| panic!("delete marker migration attempt {attempt} should converge: {err}"));
6188            assert!(moved.delete_marker);
6189            assert_eq!(moved.version_id, Some(version));
6190        }
6191        let target = store.pools[1]
6192            .get_object_info(&bucket, object, &movement_opts)
6193            .await
6194            .expect("retry must retain the target delete marker");
6195        assert!(target.delete_marker);
6196        assert_eq!(target.mod_time, Some(mod_time));
6197        assert_eq!(
6198            rustfs_utils::http::get_str(&target.user_defined, rustfs_utils::http::SUFFIX_REPLICA_STATUS).as_deref(),
6199            Some("REPLICA")
6200        );
6201        assert_eq!(
6202            rustfs_utils::http::get_str(&target.user_defined, rustfs_utils::http::SUFFIX_REPLICA_TIMESTAMP).as_deref(),
6203            Some(replica_timestamp_string.as_str())
6204        );
6205        assert_eq!(
6206            rustfs_utils::http::get_str(&target.user_defined, rustfs_utils::http::SUFFIX_REPLICATION_STATUS).as_deref(),
6207            Some(replication_status)
6208        );
6209        assert_eq!(
6210            rustfs_utils::http::get_str(&target.user_defined, rustfs_utils::http::SUFFIX_REPLICATION_TIMESTAMP).as_deref(),
6211            Some(replication_timestamp_string.as_str())
6212        );
6213
6214        let null_object = "null-delete-marker-retry";
6215        let null_mod_time = mod_time + time::Duration::seconds(3);
6216        let null_marker = store.pools[0]
6217            .delete_object(
6218                &bucket,
6219                null_object,
6220                ObjectOptions {
6221                    version_suspended: true,
6222                    version_id: Some(uuid::Uuid::nil().to_string()),
6223                    data_movement: true,
6224                    delete_marker: true,
6225                    mod_time: Some(null_mod_time),
6226                    ..Default::default()
6227                },
6228            )
6229            .await
6230            .expect("write source null delete marker");
6231        assert_eq!(null_marker.version_id, Some(uuid::Uuid::nil()));
6232        let null_opts = ObjectOptions {
6233            version_suspended: true,
6234            version_id: Some(uuid::Uuid::nil().to_string()),
6235            delete_marker: true,
6236            mod_time: Some(null_mod_time),
6237            src_pool_idx: 0,
6238            data_movement: true,
6239            skip_decommissioned: true,
6240            ..Default::default()
6241        };
6242        for attempt in 1..=2 {
6243            let moved = store
6244                .delete_object(&bucket, null_object, null_opts.clone())
6245                .await
6246                .unwrap_or_else(|err| panic!("null delete marker migration attempt {attempt} should converge: {err}"));
6247            assert!(moved.delete_marker);
6248            assert_eq!(moved.version_id, Some(uuid::Uuid::nil()));
6249        }
6250        let target_null_marker = store.pools[1]
6251            .get_object_info(&bucket, null_object, &null_opts)
6252            .await
6253            .expect("target must retain the null delete marker identity");
6254        assert!(target_null_marker.delete_marker);
6255        assert_eq!(target_null_marker.version_id, Some(uuid::Uuid::nil()));
6256
6257        let explicit_object = "suspended-explicit-version-delete";
6258        let explicit_version = uuid::Uuid::new_v4();
6259        let mut explicit_reader = PutObjReader::from_vec(b"explicit version".to_vec());
6260        store.pools[0]
6261            .put_object(
6262                &bucket,
6263                explicit_object,
6264                &mut explicit_reader,
6265                &ObjectOptions {
6266                    versioned: true,
6267                    version_id: Some(explicit_version.to_string()),
6268                    ..Default::default()
6269                },
6270            )
6271            .await
6272            .expect("write explicit version before suspended delete");
6273        store.pools[0]
6274            .delete_object(
6275                &bucket,
6276                explicit_object,
6277                ObjectOptions {
6278                    version_suspended: true,
6279                    version_id: Some(explicit_version.to_string()),
6280                    ..Default::default()
6281                },
6282            )
6283            .await
6284            .expect("delete explicit version from suspended bucket");
6285        store.pools[0]
6286            .get_object_info(
6287                &bucket,
6288                explicit_object,
6289                &ObjectOptions {
6290                    versioned: true,
6291                    version_id: Some(explicit_version.to_string()),
6292                    ..Default::default()
6293                },
6294            )
6295            .await
6296            .expect_err("suspended delete must remove the requested UUID version");
6297    }
6298
6299    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6300    #[serial_test::serial(storage_class_env)]
6301    async fn decommission_entry_carries_migration_and_cleanup_mutation_fences() {
6302        let temp_dir = tempfile::tempdir().expect("create decommission delete-fence store dir");
6303        let (_ctx, store, shutdown) = without_storage_class_env(build_isolated_test_store_with_layout(
6304            temp_dir.path(),
6305            "decommission-delete-fence",
6306            &[(2, 4), (1, 4)],
6307            CancellationToken::new(),
6308            None,
6309        ))
6310        .await;
6311        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
6312
6313        let bucket = format!("decom-delete-fence-{}", uuid::Uuid::new_v4());
6314        let object = (0..128)
6315            .map(|index| format!("object-{index}.bin"))
6316            .find(|candidate| store.pools[0].get_disks_by_key(candidate).set_index == 1)
6317            .expect("the deterministic object search should select source set 1");
6318        store
6319            .make_bucket(&bucket, &MakeBucketOptions::default())
6320            .await
6321            .expect("create decommission delete-fence bucket");
6322        let mut source = PutObjReader::from_vec(b"source generation".to_vec());
6323        store.pools[0]
6324            .put_object(&bucket, &object, &mut source, &ObjectOptions::default())
6325            .await
6326            .expect("write source object to the pool being decommissioned");
6327        mark_test_pool_decommissioning(&store, 0).await;
6328        assert!(store.is_suspended(0).await, "pool 0 must be a suspended decommission source");
6329
6330        let barrier = crate::set_disk::PutObjectCommitBarrier::install(
6331            &bucket,
6332            &object,
6333            crate::set_disk::PutObjectCommitPause::AfterNamespace,
6334        );
6335        let cleanup_barrier = crate::data_movement::SourceCleanupDeleteBarrier::install(&bucket, &object);
6336        let source_set = store.pools[0].get_disks_by_key(&object);
6337        assert_eq!(source_set.set_index, 1, "the source entry must exercise the non-fixed set cleanup lock");
6338        let worker_store = Arc::clone(&store);
6339        let worker_bucket = bucket.clone();
6340        let worker_object = object.clone();
6341        let worker = tokio::spawn(async move {
6342            worker_store
6343                .decommission_entry_for_test(
6344                    0,
6345                    MetaCacheEntry {
6346                        name: worker_object,
6347                        ..Default::default()
6348                    },
6349                    worker_bucket,
6350                    source_set,
6351                )
6352                .await
6353        });
6354        barrier.wait_until_paused().await;
6355
6356        let delete_barrier = crate::store::object::DeleteAfterObjectLockSnapshotBarrier::install(&bucket);
6357        let delete_store = Arc::clone(&store);
6358        let delete_bucket = bucket.clone();
6359        let delete_object = object.clone();
6360        let delete = tokio::spawn(async move {
6361            delete_store
6362                .delete_object(&delete_bucket, &delete_object, ObjectOptions::default())
6363                .await
6364        });
6365        delete_barrier.wait_until_paused().await;
6366        delete_barrier.release_and_wait_until_namespace_pending().await;
6367        assert!(
6368            !delete_barrier.namespace_acquired() && !delete.is_finished(),
6369            "DELETE must remain before namespace acquisition behind the decommission worker's target-commit mutation fence"
6370        );
6371
6372        barrier.release();
6373        cleanup_barrier.wait_until_paused().await;
6374        drop(barrier);
6375
6376        let fixed_set = Arc::clone(&store.pools[0].disk_set[0]);
6377        let fixed_mutation_barrier = crate::set_disk::PutObjectCommitBarrier::install(
6378            &bucket,
6379            &object,
6380            crate::set_disk::PutObjectCommitPause::BeforeNamespace,
6381        );
6382        let mutation_bucket = bucket.clone();
6383        let mutation_object = object.clone();
6384        let fixed_mutation = tokio::spawn(async move {
6385            let mut reader = PutObjReader::from_vec(b"fixed-domain replacement".to_vec());
6386            fixed_set
6387                .put_object(&mutation_bucket, &mutation_object, &mut reader, &ObjectOptions::default())
6388                .await
6389        });
6390        fixed_mutation_barrier.wait_until_paused().await;
6391        fixed_mutation_barrier.release_and_wait_until_namespace_pending().await;
6392        assert!(
6393            !fixed_mutation_barrier.namespace_acquired() && !fixed_mutation.is_finished(),
6394            "the source cleanup must retain the fixed mutation fence before the set-0 mutation acquires its namespace"
6395        );
6396        fixed_mutation.abort();
6397        assert!(
6398            fixed_mutation
6399                .await
6400                .expect_err("the fixed-domain mutation should be canceled")
6401                .is_cancelled(),
6402            "the competing fixed-domain mutation must remain cancelable while blocked"
6403        );
6404        drop(fixed_mutation_barrier);
6405
6406        cleanup_barrier.release();
6407        worker
6408            .await
6409            .expect("decommission entry worker should join")
6410            .expect("decommission entry should migrate and clean its source");
6411        delete
6412            .await
6413            .expect("DELETE task should join")
6414            .expect("DELETE should remove the source and migrated target generations");
6415
6416        for pool in &store.pools {
6417            let err = pool
6418                .get_object_info(&bucket, &object, &ObjectOptions::default())
6419                .await
6420                .expect_err("DELETE must remove the source and migrated target copies");
6421            assert!(
6422                matches!(err, StorageError::ObjectNotFound(_, _)),
6423                "unexpected post-delete pool result: {err:?}"
6424            );
6425        }
6426        let err = store
6427            .get_object_info(&bucket, &object, &ObjectOptions::default())
6428            .await
6429            .expect_err("the deleted generation must not become visible again");
6430        assert!(
6431            matches!(err, StorageError::ObjectNotFound(_, _)),
6432            "unexpected post-delete store result: {err:?}"
6433        );
6434
6435        shutdown.cancel();
6436    }
6437
6438    #[test]
6439    fn decommission_retry_fault_budget_counts_successes_across_attempt_changes() {
6440        let cases: &[&[(usize, bool, bool)]] = &[
6441            &[(1, true, true), (2, true, true), (3, true, false)],
6442            &[(1, true, true), (1, true, true), (2, true, false)],
6443            &[(1, true, true), (3, true, false), (3, true, false)],
6444            &[(1, true, true), (2, false, false), (1, true, true), (2, true, false)],
6445            &[(1, true, true), (2, false, false), (3, true, false)],
6446            &[(3, true, false), (4, true, false)],
6447        ];
6448        for case in cases {
6449            let faults = Arc::new(AtomicUsize::new(0));
6450            let hook = decommission_retry_fault_hook("bucket", "object", Arc::clone(&faults));
6451
6452            for (stage, bucket, object, succeeded) in [
6453                ("other-stage", "bucket", "object", true),
6454                (DECOMMISSION_TEST_FAULT_STAGE_MIGRATE_OBJECT, "other-bucket", "object", true),
6455                (DECOMMISSION_TEST_FAULT_STAGE_MIGRATE_OBJECT, "bucket", "other-object", true),
6456                (DECOMMISSION_TEST_FAULT_STAGE_MIGRATE_OBJECT, "bucket", "object", false),
6457            ] {
6458                assert!(!hook(stage, bucket, object, 1, succeeded));
6459            }
6460            assert_eq!(faults.load(Ordering::SeqCst), 0, "unrelated or failed copies must not consume faults");
6461
6462            let mut expected_faults = 0;
6463            for &(attempt, succeeded, expected) in *case {
6464                assert_eq!(
6465                    hook(DECOMMISSION_TEST_FAULT_STAGE_MIGRATE_OBJECT, "bucket", "object", attempt, succeeded),
6466                    expected,
6467                    "fault plan {case:?} at attempt {attempt}"
6468                );
6469                expected_faults += usize::from(expected);
6470                assert_eq!(faults.load(Ordering::SeqCst), expected_faults);
6471            }
6472        }
6473    }
6474
6475    #[test]
6476    #[serial_test::serial(storage_class_env)]
6477    fn decommission_entry_retries_source_changed_without_canceling_other_bucket() {
6478        let handle = std::thread::Builder::new()
6479            .name("decommission_entry_retries_source_changed_without_canceling_other_bucket".to_string())
6480            .stack_size(32 * 1024 * 1024)
6481            .spawn(|| {
6482                let runtime = tokio::runtime::Builder::new_multi_thread()
6483                    .enable_all()
6484                    .worker_threads(2)
6485                    .build()
6486                    .expect("test runtime should build");
6487                runtime.block_on(async {
6488                    let temp_dir = tempfile::tempdir().expect("create decommission retry store dir");
6489                    let (_ctx, store, shutdown) = without_storage_class_env(build_isolated_test_store(
6490                        temp_dir.path(),
6491                        "decommission-entry-retry",
6492                        &[4, 4],
6493                    ))
6494                    .await;
6495                    crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
6496
6497                    let changed_bucket = format!("decom-retry-a-{}", uuid::Uuid::new_v4());
6498                    let other_bucket = format!("decom-retry-b-{}", uuid::Uuid::new_v4());
6499                    for bucket in [&changed_bucket, &other_bucket] {
6500                        store
6501                            .make_bucket(bucket, &MakeBucketOptions::default())
6502                            .await
6503                            .expect("create decommission retry bucket");
6504                    }
6505
6506                    let changed_object = "changed.bin";
6507                    let first_version = uuid::Uuid::new_v4();
6508                    let second_version = uuid::Uuid::new_v4();
6509                    let base_time = OffsetDateTime::now_utc();
6510                    seed_decommission_source(
6511                        &store,
6512                        &changed_bucket,
6513                        changed_object,
6514                        b"first generation".to_vec(),
6515                        &ObjectOptions {
6516                            versioned: true,
6517                            version_id: Some(first_version.to_string()),
6518                            mod_time: Some(base_time),
6519                            ..Default::default()
6520                        },
6521                    )
6522                    .await;
6523                    let other_object = "other.bin";
6524                    seed_decommission_source(
6525                        &store,
6526                        &other_bucket,
6527                        other_object,
6528                        b"other bucket generation".to_vec(),
6529                        &ObjectOptions::default(),
6530                    )
6531                    .await;
6532                    mark_test_pool_decommissioning(&store, 0).await;
6533
6534                    let mutation_calls = Arc::new(AtomicUsize::new(0));
6535                    let mutation_calls_for_hook = Arc::clone(&mutation_calls);
6536                    let mutation_store = Arc::clone(&store);
6537                    let mutation_bucket = changed_bucket.clone();
6538                    let _mutation_guard = crate::core::pools::DecommissionCleanupMutationGuard::install(Arc::new(
6539                        move |bucket, object, attempt| {
6540                            let is_target = bucket == mutation_bucket.as_str() && object == changed_object;
6541                            let calls = Arc::clone(&mutation_calls_for_hook);
6542                            let store = Arc::clone(&mutation_store);
6543                            let bucket = mutation_bucket.clone();
6544                            Box::pin(async move {
6545                                if !is_target {
6546                                    return;
6547                                }
6548                                calls.fetch_add(1, Ordering::SeqCst);
6549                                if attempt == 1 {
6550                                    seed_decommission_source(
6551                                        &store,
6552                                        &bucket,
6553                                        changed_object,
6554                                        b"second generation".to_vec(),
6555                                        &ObjectOptions {
6556                                            versioned: true,
6557                                            version_id: Some(second_version.to_string()),
6558                                            mod_time: Some(base_time + time::Duration::seconds(1)),
6559                                            ..Default::default()
6560                                        },
6561                                    )
6562                                    .await;
6563                                }
6564                            })
6565                        },
6566                    ));
6567
6568                    let ordinary_faults = Arc::new(AtomicUsize::new(0));
6569                    let fault_hook = decommission_retry_fault_hook(&other_bucket, other_object, Arc::clone(&ordinary_faults));
6570                    let _fault_guard = crate::core::pools::DecommissionTestFaultGuard::install(fault_hook);
6571
6572                    let rx = CancellationToken::new();
6573                    let source_changed_exhaustions = Arc::new(AtomicUsize::new(0));
6574                    let changed_incarnation = Some(
6575                        store
6576                            .bucket_incarnation_id(&changed_bucket)
6577                            .await
6578                            .expect("changed bucket incarnation"),
6579                    );
6580                    let other_incarnation = Some(
6581                        store
6582                            .bucket_incarnation_id(&other_bucket)
6583                            .await
6584                            .expect("other bucket incarnation"),
6585                    );
6586                    let (changed_result, other_result) = tokio::join!(
6587                        run_decommission_entry_retry_test(
6588                            &store,
6589                            rx.clone(),
6590                            &changed_bucket,
6591                            changed_object,
6592                            changed_incarnation,
6593                            Arc::clone(&source_changed_exhaustions),
6594                        ),
6595                        run_decommission_entry_retry_test(
6596                            &store,
6597                            rx.clone(),
6598                            &other_bucket,
6599                            other_object,
6600                            other_incarnation,
6601                            Arc::clone(&source_changed_exhaustions),
6602                        )
6603                    );
6604                    changed_result.expect("SourceChanged entry retry must converge");
6605                    other_result.expect("other bucket entry must continue through ordinary copy retries");
6606
6607                    assert_eq!(
6608                        store.pool_meta.read().await.pools[0]
6609                            .decommission
6610                            .as_ref()
6611                            .expect("decommission progress should remain available")
6612                            .items_decommission_failed,
6613                        0,
6614                        "entry completion must not hide an exhausted copy failure"
6615                    );
6616                    assert!(!rx.is_cancelled(), "entry-level SourceChanged must not cancel the shared worker token");
6617                    assert_eq!(mutation_calls.load(Ordering::SeqCst), 2, "entry must be re-listed after SourceChanged");
6618                    assert_eq!(ordinary_faults.load(Ordering::SeqCst), 2, "ordinary copy must consume the retry budget");
6619                    assert_eq!(source_changed_exhaustions.load(Ordering::SeqCst), 0);
6620
6621                    for (version_id, expected_body) in [
6622                        (first_version, b"first generation".as_slice()),
6623                        (second_version, b"second generation".as_slice()),
6624                    ] {
6625                        let opts = ObjectOptions {
6626                            versioned: true,
6627                            version_id: Some(version_id.to_string()),
6628                            ..Default::default()
6629                        };
6630                        assert_decommission_source_absent(&store, &changed_bucket, changed_object, &opts).await;
6631                        assert_eq!(
6632                            read_decommission_target_body(&store, &changed_bucket, changed_object, &opts).await,
6633                            expected_body
6634                        );
6635                    }
6636                    assert_decommission_source_absent(&store, &other_bucket, other_object, &ObjectOptions::default()).await;
6637                    assert_eq!(
6638                        read_decommission_target_body(&store, &other_bucket, other_object, &ObjectOptions::default()).await,
6639                        b"other bucket generation"
6640                    );
6641
6642                    shutdown.cancel();
6643                });
6644            })
6645            .expect("spawn decommission retry test thread");
6646        if let Err(payload) = handle.join() {
6647            std::panic::resume_unwind(payload);
6648        }
6649    }
6650
6651    #[test]
6652    #[serial_test::serial(storage_class_env)]
6653    fn decommission_entry_exhausted_source_changed_retains_source_and_records_failure() {
6654        let handle = std::thread::Builder::new()
6655            .name("decommission_entry_exhausted_source_changed_retains_source_and_records_failure".to_string())
6656            .stack_size(32 * 1024 * 1024)
6657            .spawn(|| {
6658                let runtime = tokio::runtime::Builder::new_multi_thread()
6659                    .enable_all()
6660                    .worker_threads(2)
6661                    .build()
6662                    .expect("test runtime should build");
6663                runtime.block_on(async {
6664                    let temp_dir = tempfile::tempdir().expect("create decommission exhaustion store dir");
6665                    let (_ctx, store, shutdown) = without_storage_class_env(build_isolated_test_store(
6666                        temp_dir.path(),
6667                        "decommission-entry-exhaustion",
6668                        &[4, 4],
6669                    ))
6670                    .await;
6671                    crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
6672
6673                    let bucket = format!("decom-exhausted-{}", uuid::Uuid::new_v4());
6674                    let object = "exhausted.bin";
6675                    store
6676                        .make_bucket(&bucket, &MakeBucketOptions::default())
6677                        .await
6678                        .expect("create decommission exhaustion bucket");
6679                    let original_version = uuid::Uuid::new_v4();
6680                    let base_time = OffsetDateTime::now_utc();
6681                    seed_decommission_source(
6682                        &store,
6683                        &bucket,
6684                        object,
6685                        b"original generation".to_vec(),
6686                        &ObjectOptions {
6687                            versioned: true,
6688                            version_id: Some(original_version.to_string()),
6689                            mod_time: Some(base_time),
6690                            ..Default::default()
6691                        },
6692                    )
6693                    .await;
6694                    mark_test_pool_decommissioning(&store, 0).await;
6695
6696                    let mutation_calls = Arc::new(AtomicUsize::new(0));
6697                    let mutation_calls_for_hook = Arc::clone(&mutation_calls);
6698                    let mutation_store = Arc::clone(&store);
6699                    let mutation_bucket = bucket.clone();
6700                    let _mutation_guard = crate::core::pools::DecommissionCleanupMutationGuard::install(Arc::new(
6701                        move |called_bucket, called_object, _attempt| {
6702                            let is_target = called_bucket == mutation_bucket.as_str() && called_object == object;
6703                            let calls = Arc::clone(&mutation_calls_for_hook);
6704                            let store = Arc::clone(&mutation_store);
6705                            let bucket = mutation_bucket.clone();
6706                            Box::pin(async move {
6707                                if !is_target {
6708                                    return;
6709                                }
6710                                let call = calls.fetch_add(1, Ordering::SeqCst) + 1;
6711                                let offset = i64::try_from(call).expect("entry retry count should fit i64");
6712                                seed_decommission_source(
6713                                    &store,
6714                                    &bucket,
6715                                    object,
6716                                    format!("concurrent generation {call}").into_bytes(),
6717                                    &ObjectOptions {
6718                                        versioned: true,
6719                                        version_id: Some(uuid::Uuid::new_v4().to_string()),
6720                                        mod_time: Some(base_time + time::Duration::seconds(offset)),
6721                                        ..Default::default()
6722                                    },
6723                                )
6724                                .await;
6725                            })
6726                        },
6727                    ));
6728
6729                    let rx = CancellationToken::new();
6730                    let source_changed_exhaustions = Arc::new(AtomicUsize::new(0));
6731                    let incarnation = Some(store.bucket_incarnation_id(&bucket).await.expect("bucket incarnation"));
6732                    run_decommission_entry_retry_test(
6733                        &store,
6734                        rx.clone(),
6735                        &bucket,
6736                        object,
6737                        incarnation,
6738                        Arc::clone(&source_changed_exhaustions),
6739                    )
6740                    .await
6741                    .expect("entry-level exhaustion must stay local below the pool threshold");
6742
6743                    assert!(!rx.is_cancelled(), "one exhausted entry must not cancel other bucket workers");
6744                    assert_eq!(mutation_calls.load(Ordering::SeqCst), crate::core::pools::DECOMMISSION_ENTRY_MAX_ATTEMPTS);
6745                    assert_eq!(source_changed_exhaustions.load(Ordering::SeqCst), 1);
6746                    store.pools[0]
6747                        .get_object_info(
6748                            &bucket,
6749                            object,
6750                            &ObjectOptions {
6751                                versioned: true,
6752                                version_id: Some(original_version.to_string()),
6753                                ..Default::default()
6754                            },
6755                        )
6756                        .await
6757                        .expect("retry exhaustion must retain the original source version");
6758                    let pool_meta = store.pool_meta.read().await;
6759                    let info = pool_meta.pools[0]
6760                        .decommission
6761                        .as_ref()
6762                        .expect("decommission progress must be initialized");
6763                    assert_eq!(info.items_decommission_failed, 1, "exhausted entry must be visible as failed");
6764                    drop(pool_meta);
6765
6766                    shutdown.cancel();
6767                });
6768            })
6769            .expect("spawn decommission retry test thread");
6770        if let Err(payload) = handle.join() {
6771            std::panic::resume_unwind(payload);
6772        }
6773    }
6774
6775    #[test]
6776    #[serial_test::serial(storage_class_env)]
6777    fn decommission_entry_delete_marker_copy_retries_real_path() {
6778        let handle = std::thread::Builder::new()
6779            .name("decommission_entry_delete_marker_copy_retries_real_path".to_string())
6780            .stack_size(32 * 1024 * 1024)
6781            .spawn(|| {
6782                let runtime = tokio::runtime::Builder::new_multi_thread()
6783                    .enable_all()
6784                    .worker_threads(2)
6785                    .build()
6786                    .expect("test runtime should build");
6787                runtime.block_on(async {
6788                    let temp_dir = tempfile::tempdir().expect("create delete marker retry store dir");
6789                    let (_ctx, store, shutdown) = without_storage_class_env(build_isolated_test_store(
6790                        temp_dir.path(),
6791                        "decommission-delete-marker-retry",
6792                        &[4, 4],
6793                    ))
6794                    .await;
6795                    crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
6796
6797                    let bucket = format!("decom-marker-{}", uuid::Uuid::new_v4());
6798                    let object = "marker.bin";
6799                    store
6800                        .make_bucket(&bucket, &MakeBucketOptions::default())
6801                        .await
6802                        .expect("create delete marker retry bucket");
6803                    let data_version = uuid::Uuid::new_v4();
6804                    let marker_version = uuid::Uuid::new_v4();
6805                    let base_time = OffsetDateTime::now_utc();
6806                    seed_decommission_source(
6807                        &store,
6808                        &bucket,
6809                        object,
6810                        b"delete marker data".to_vec(),
6811                        &ObjectOptions {
6812                            versioned: true,
6813                            version_id: Some(data_version.to_string()),
6814                            mod_time: Some(base_time),
6815                            ..Default::default()
6816                        },
6817                    )
6818                    .await;
6819                    store.pools[0]
6820                        .delete_object(
6821                            &bucket,
6822                            object,
6823                            ObjectOptions {
6824                                versioned: true,
6825                                version_id: Some(marker_version.to_string()),
6826                                delete_marker: true,
6827                                mod_time: Some(base_time + time::Duration::seconds(1)),
6828                                ..Default::default()
6829                            },
6830                        )
6831                        .await
6832                        .expect("seed source delete marker");
6833                    mark_test_pool_decommissioning(&store, 0).await;
6834
6835                    let fault_calls = Arc::new(AtomicUsize::new(0));
6836                    let fault_calls_for_hook = Arc::clone(&fault_calls);
6837                    let fault_bucket = bucket.clone();
6838                    let _fault_guard = crate::core::pools::DecommissionTestFaultGuard::install(Arc::new(
6839                        move |stage, called_bucket, called_object, attempt, succeeded| {
6840                            let injected = succeeded
6841                                && stage == DECOMMISSION_TEST_FAULT_STAGE_DELETE_MARKER
6842                                && called_bucket == fault_bucket.as_str()
6843                                && called_object == object
6844                                && attempt < crate::core::pools::DECOMMISSION_VERSION_COPY_ATTEMPTS;
6845                            if injected {
6846                                fault_calls_for_hook.fetch_add(1, Ordering::SeqCst);
6847                            }
6848                            injected
6849                        },
6850                    ));
6851
6852                    let incarnation = Some(store.bucket_incarnation_id(&bucket).await.expect("bucket incarnation"));
6853                    run_decommission_entry_retry_test(
6854                        &store,
6855                        CancellationToken::new(),
6856                        &bucket,
6857                        object,
6858                        incarnation,
6859                        Arc::new(AtomicUsize::new(0)),
6860                    )
6861                    .await
6862                    .expect("delete marker copy retries must converge");
6863
6864                    assert_eq!(
6865                        fault_calls.load(Ordering::SeqCst),
6866                        crate::core::pools::DECOMMISSION_VERSION_COPY_ATTEMPTS - 1
6867                    );
6868                    let marker_opts = ObjectOptions {
6869                        versioned: true,
6870                        version_id: Some(marker_version.to_string()),
6871                        ..Default::default()
6872                    };
6873                    let target_marker = store.pools[1]
6874                        .get_object_info(&bucket, object, &marker_opts)
6875                        .await
6876                        .expect("target delete marker must exist");
6877                    assert!(target_marker.delete_marker);
6878                    assert_decommission_source_absent(&store, &bucket, object, &marker_opts).await;
6879
6880                    let data_opts = ObjectOptions {
6881                        versioned: true,
6882                        version_id: Some(data_version.to_string()),
6883                        ..Default::default()
6884                    };
6885                    assert_eq!(
6886                        read_decommission_target_body(&store, &bucket, object, &data_opts).await,
6887                        b"delete marker data"
6888                    );
6889                    assert_decommission_source_absent(&store, &bucket, object, &data_opts).await;
6890
6891                    shutdown.cancel();
6892                });
6893            })
6894            .expect("spawn decommission retry test thread");
6895        if let Err(payload) = handle.join() {
6896            std::panic::resume_unwind(payload);
6897        }
6898    }
6899
6900    #[cfg(feature = "test-util")]
6901    #[test]
6902    #[serial_test::serial(storage_class_env)]
6903    fn decommission_entry_tiered_copy_retries_real_path() {
6904        let handle = std::thread::Builder::new()
6905            .name("decommission_entry_tiered_copy_retries_real_path".to_string())
6906            .stack_size(32 * 1024 * 1024)
6907            .spawn(|| {
6908                let runtime = tokio::runtime::Builder::new_multi_thread()
6909                    .enable_all()
6910                    .worker_threads(2)
6911                    .build()
6912                    .expect("test runtime should build");
6913                runtime.block_on(async {
6914                    let temp_dir = tempfile::tempdir().expect("create tiered retry store dir");
6915                    let (ctx, store, shutdown) = without_storage_class_env(build_isolated_test_store(
6916                        temp_dir.path(),
6917                        "decommission-tiered-retry",
6918                        &[4, 4],
6919                    ))
6920                    .await;
6921                    crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
6922
6923                    let bucket = format!("decom-tiered-{}", uuid::Uuid::new_v4());
6924                    let object = "tiered.bin";
6925                    store
6926                        .make_bucket(&bucket, &MakeBucketOptions::default())
6927                        .await
6928                        .expect("create tiered retry bucket");
6929                    let mut reader = PutObjReader::from_vec(b"tiered generation".to_vec());
6930                    let original = store.pools[0]
6931                        .put_object(&bucket, object, &mut reader, &ObjectOptions::default())
6932                        .await
6933                        .expect("seed tiered source object");
6934                    let tier_name = format!("DECOM{}", &uuid::Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
6935                    register_mock_tier(&ctx.tier_config_mgr(), &tier_name).await;
6936                    store.pools[0]
6937                        .transition_object(
6938                            &bucket,
6939                            object,
6940                            &ObjectOptions {
6941                                transition: TransitionOptions {
6942                                    status: TRANSITION_PENDING.to_string(),
6943                                    tier: tier_name,
6944                                    etag: original.etag.clone().expect("tiered source ETag"),
6945                                    ..Default::default()
6946                                },
6947                                version_id: original.version_id.map(|version_id| version_id.to_string()),
6948                                mod_time: original.mod_time,
6949                                ..Default::default()
6950                            },
6951                        )
6952                        .await
6953                        .expect("transition source object to mock tier");
6954                    mark_test_pool_decommissioning(&store, 0).await;
6955
6956                    let fault_calls = Arc::new(AtomicUsize::new(0));
6957                    let fault_calls_for_hook = Arc::clone(&fault_calls);
6958                    let fault_bucket = bucket.clone();
6959                    let _fault_guard = crate::core::pools::DecommissionTestFaultGuard::install(Arc::new(
6960                        move |stage, called_bucket, called_object, attempt, succeeded| {
6961                            let injected = succeeded
6962                                && stage == DECOMMISSION_TEST_FAULT_STAGE_TIERED
6963                                && called_bucket == fault_bucket.as_str()
6964                                && called_object == object
6965                                && attempt < crate::core::pools::DECOMMISSION_VERSION_COPY_ATTEMPTS;
6966                            if injected {
6967                                fault_calls_for_hook.fetch_add(1, Ordering::SeqCst);
6968                            }
6969                            injected
6970                        },
6971                    ));
6972
6973                    let incarnation = Some(store.bucket_incarnation_id(&bucket).await.expect("bucket incarnation"));
6974                    run_decommission_entry_retry_test(
6975                        &store,
6976                        CancellationToken::new(),
6977                        &bucket,
6978                        object,
6979                        incarnation,
6980                        Arc::new(AtomicUsize::new(0)),
6981                    )
6982                    .await
6983                    .expect("tiered copy retries must converge");
6984
6985                    assert_eq!(
6986                        fault_calls.load(Ordering::SeqCst),
6987                        crate::core::pools::DECOMMISSION_VERSION_COPY_ATTEMPTS - 1
6988                    );
6989                    let target = store.pools[1]
6990                        .get_object_info(&bucket, object, &ObjectOptions::default())
6991                        .await
6992                        .expect("tiered target metadata must exist");
6993                    assert_eq!(target.transitioned_object.status, rustfs_filemeta::TRANSITION_COMPLETE);
6994                    assert_decommission_source_absent(&store, &bucket, object, &ObjectOptions::default()).await;
6995
6996                    shutdown.cancel();
6997                });
6998            })
6999            .expect("spawn decommission retry test thread");
7000        if let Err(payload) = handle.join() {
7001            std::panic::resume_unwind(payload);
7002        }
7003    }
7004
7005    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7006    #[serial_test::serial(storage_class_env)]
7007    async fn decommission_outer_fence_loss_blocks_target_put_commit() {
7008        let temp_dir = tempfile::tempdir().expect("create decommission PUT fence-loss store dir");
7009        let (_ctx, store, shutdown) =
7010            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "decommission-put-fence-loss", &[4, 4])).await;
7011        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
7012
7013        let bucket = format!("decom-put-fence-loss-{}", uuid::Uuid::new_v4());
7014        let object = "ordinary.bin";
7015        store
7016            .make_bucket(&bucket, &MakeBucketOptions::default())
7017            .await
7018            .expect("create decommission PUT fence-loss bucket");
7019        let mut source = PutObjReader::from_vec(b"source generation".to_vec());
7020        store.pools[0]
7021            .put_object(&bucket, object, &mut source, &ObjectOptions::default())
7022            .await
7023            .expect("write decommission PUT source");
7024        mark_test_pool_decommissioning(&store, 0).await;
7025
7026        let loss_hook = crate::store::object::DecommissionMutationFenceLossHook::install(
7027            &bucket,
7028            object,
7029            crate::store::object::DecommissionMutationFenceTestPhase::Migration,
7030        );
7031        let barrier = crate::set_disk::PutObjectCommitBarrier::install(
7032            &bucket,
7033            object,
7034            crate::set_disk::PutObjectCommitPause::BeforeQuotaRename,
7035        );
7036        let source_set = store.pools[0].get_disks_by_key(object);
7037        let worker_store = Arc::clone(&store);
7038        let worker_bucket = bucket.clone();
7039        let worker = tokio::spawn(async move {
7040            worker_store
7041                .decommission_entry_for_test(
7042                    0,
7043                    MetaCacheEntry {
7044                        name: object.to_string(),
7045                        ..Default::default()
7046                    },
7047                    worker_bucket,
7048                    source_set,
7049                )
7050                .await
7051        });
7052
7053        barrier.wait_until_paused().await;
7054        loss_hook.mark_lost();
7055        barrier.release();
7056        drop(barrier);
7057        worker
7058            .await
7059            .expect("decommission PUT fence-loss worker should join")
7060            .expect("a fenced migration failure should remain retryable at entry scope");
7061
7062        assert_pool_object_absent(&store.pools[1], &bucket, object).await;
7063        assert_pool_object_present(&store.pools[0], &bucket, object).await;
7064        shutdown.cancel();
7065    }
7066
7067    #[test]
7068    #[serial_test::serial(storage_class_env)]
7069    fn decommission_outer_fence_loss_blocks_multipart_commits() {
7070        run_large_stack_async_test(
7071            "decommission-multipart-outer-fence-loss",
7072            decommission_outer_fence_loss_blocks_multipart_commits_case,
7073        );
7074    }
7075
7076    async fn decommission_outer_fence_loss_blocks_multipart_commits_case() {
7077        for (object, pause) in [
7078            ("complete.bin", crate::set_disk::MultipartCommitPause::BeforeLockLost),
7079            ("new-upload.bin", crate::set_disk::MultipartCommitPause::NewUploadBeforeLockLost),
7080        ] {
7081            // Each injected commit failure intentionally leaves a capacity intent for
7082            // reconciliation.  Isolate the scenarios so one failure cannot turn the
7083            // next scenario into a capacity-recovery test before its barrier is hit.
7084            let temp_dir = tempfile::tempdir().expect("create decommission multipart fence-loss store dir");
7085            let (_ctx, store, shutdown) = without_storage_class_env(build_isolated_test_store(
7086                temp_dir.path(),
7087                "decommission-multipart-fence-loss",
7088                &[4, 4],
7089            ))
7090            .await;
7091            crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
7092
7093            let bucket = format!("decom-mpu-fence-loss-{}", uuid::Uuid::new_v4());
7094            store
7095                .make_bucket(&bucket, &MakeBucketOptions::default())
7096                .await
7097                .expect("create decommission multipart fence-loss bucket");
7098            write_decommission_test_multipart_source(&store, 0, &bucket, object).await;
7099            mark_test_pool_decommissioning(&store, 0).await;
7100
7101            let loss_hook = crate::store::object::DecommissionMutationFenceLossHook::install(
7102                &bucket,
7103                object,
7104                crate::store::object::DecommissionMutationFenceTestPhase::Migration,
7105            );
7106            let commit_observation = (pause == crate::set_disk::MultipartCommitPause::NewUploadBeforeLockLost)
7107                .then(|| crate::set_disk::NewMultipartUploadCommitObservation::install(&bucket, object));
7108            let barrier = crate::set_disk::MultipartCommitBarrier::install(&bucket, object, pause);
7109            let source_set = store.pools[0].get_disks_by_key(object);
7110            let recovery_source_set = Arc::clone(&source_set);
7111            let worker_store = Arc::clone(&store);
7112            let worker_bucket = bucket.clone();
7113            let mut worker = tokio::spawn(async move {
7114                worker_store
7115                    .decommission_entry_for_test(
7116                        0,
7117                        MetaCacheEntry {
7118                            name: object.to_string(),
7119                            ..Default::default()
7120                        },
7121                        worker_bucket,
7122                        source_set,
7123                    )
7124                    .await
7125            });
7126
7127            tokio::select! {
7128                _ = barrier.wait_until_paused() => {}
7129                result = &mut worker => {
7130                    panic!("decommission multipart worker finished before the requested {pause:?} commit barrier: {result:?}");
7131                }
7132            }
7133            loss_hook.mark_lost();
7134            barrier.release();
7135            drop(barrier);
7136            worker
7137                .await
7138                .expect("decommission multipart fence-loss worker should join")
7139                .expect("a fenced multipart migration failure should remain retryable at entry scope");
7140
7141            if let Some(commit_observation) = commit_observation {
7142                assert!(
7143                    !commit_observation.committed(),
7144                    "NewMultipart must reject a lost outer decommission/capacity fence even though it does not acquire the final remote-tuple publication fence"
7145                );
7146            }
7147            assert_pool_object_absent(&store.pools[1], &bucket, object).await;
7148            assert_pool_object_present(&store.pools[0], &bucket, object).await;
7149            let uploads = store.pools[1]
7150                .list_multipart_uploads(&bucket, object, None, None, None, 100)
7151                .await
7152                .expect("list target multipart uploads after fenced migration");
7153            assert!(uploads.uploads.is_empty(), "fenced multipart migration must not retain target staging");
7154            drop(loss_hook);
7155            store
7156                .decommission_entry_for_test(
7157                    0,
7158                    MetaCacheEntry {
7159                        name: object.to_string(),
7160                        ..Default::default()
7161                    },
7162                    bucket.clone(),
7163                    recovery_source_set,
7164                )
7165                .await
7166                .expect("same-mutation retry should recover the durable capacity intent");
7167
7168            shutdown.cancel();
7169        }
7170    }
7171
7172    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7173    #[serial_test::serial(storage_class_env)]
7174    async fn decommission_outer_fence_loss_blocks_source_cleanup_delete_commit() {
7175        let temp_dir = tempfile::tempdir().expect("create decommission cleanup fence-loss store dir");
7176        let (_ctx, store, shutdown) =
7177            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "decommission-cleanup-fence-loss", &[4, 4]))
7178                .await;
7179        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
7180
7181        let bucket = format!("decom-cleanup-fence-loss-{}", uuid::Uuid::new_v4());
7182        let object = "cleanup.bin";
7183        store
7184            .make_bucket(&bucket, &MakeBucketOptions::default())
7185            .await
7186            .expect("create decommission cleanup fence-loss bucket");
7187        let mut source = PutObjReader::from_vec(b"source generation".to_vec());
7188        store.pools[0]
7189            .put_object(&bucket, object, &mut source, &ObjectOptions::default())
7190            .await
7191            .expect("write decommission cleanup source");
7192        mark_test_pool_decommissioning(&store, 0).await;
7193
7194        let loss_hook = crate::store::object::DecommissionMutationFenceLossHook::install(
7195            &bucket,
7196            object,
7197            crate::store::object::DecommissionMutationFenceTestPhase::SourceCleanup,
7198        );
7199        let barrier = crate::data_movement::SourceCleanupDeleteBarrier::install(&bucket, object);
7200        let source_set = store.pools[0].get_disks_by_key(object);
7201        let worker_store = Arc::clone(&store);
7202        let worker_bucket = bucket.clone();
7203        let worker = tokio::spawn(async move {
7204            worker_store
7205                .decommission_entry_for_test(
7206                    0,
7207                    MetaCacheEntry {
7208                        name: object.to_string(),
7209                        ..Default::default()
7210                    },
7211                    worker_bucket,
7212                    source_set,
7213                )
7214                .await
7215        });
7216
7217        barrier.wait_until_paused().await;
7218        loss_hook.mark_lost();
7219        barrier.release();
7220        drop(barrier);
7221        let err = worker
7222            .await
7223            .expect("decommission cleanup fence-loss worker should join")
7224            .expect_err("source cleanup must fail after its outer fence is lost");
7225        assert!(
7226            err.to_string().contains("delete_object_commit"),
7227            "cleanup failure must come from the delete commit fence: {err:?}"
7228        );
7229
7230        assert_pool_object_present(&store.pools[0], &bucket, object).await;
7231        assert_pool_object_present(&store.pools[1], &bucket, object).await;
7232        shutdown.cancel();
7233    }
7234
7235    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7236    #[serial_test::serial(storage_class_env)]
7237    async fn reverse_decommission_reuses_fixed_target_fence_for_put_and_multipart() {
7238        let temp_dir = tempfile::tempdir().expect("create reverse decommission store dir");
7239        let (_ctx, store, shutdown) = without_storage_class_env(build_isolated_test_store_with_layout(
7240            temp_dir.path(),
7241            "reverse-decommission-fixed-target",
7242            &[(1, 4), (1, 4)],
7243            CancellationToken::new(),
7244            None,
7245        ))
7246        .await;
7247        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
7248
7249        let bucket = format!("reverse-decom-fixed-target-{}", uuid::Uuid::new_v4());
7250        let object = "ordinary.bin";
7251        let object_body = b"reverse ordinary generation".to_vec();
7252        let self_copy_object = "source-only-self-copy.bin";
7253        let self_copy_body = b"source only copy generation".to_vec();
7254        let multipart_object = "multipart.bin";
7255        let first_part = vec![b'm'; 5 * 1024 * 1024];
7256        let second_part = b"reverse multipart tail".to_vec();
7257        let mut multipart_body = first_part.clone();
7258        multipart_body.extend_from_slice(&second_part);
7259
7260        store
7261            .make_bucket(&bucket, &MakeBucketOptions::default())
7262            .await
7263            .expect("create reverse decommission bucket");
7264        let mut source = PutObjReader::from_vec(object_body.clone());
7265        store.pools[1]
7266            .put_object(&bucket, object, &mut source, &ObjectOptions::default())
7267            .await
7268            .expect("write ordinary source object to pool 1");
7269        let mut self_copy_source = PutObjReader::from_vec(self_copy_body.clone());
7270        store.pools[1]
7271            .put_object(&bucket, self_copy_object, &mut self_copy_source, &ObjectOptions::default())
7272            .await
7273            .expect("write self-copy source object to pool 1");
7274
7275        let upload = store.pools[1]
7276            .new_multipart_upload(&bucket, multipart_object, &ObjectOptions::default())
7277            .await
7278            .expect("create source multipart upload in pool 1");
7279        let mut completed_parts = Vec::with_capacity(2);
7280        for (part_number, bytes) in [(1, first_part.as_slice()), (2, second_part.as_slice())] {
7281            let mut reader = PutObjReader::from_vec(bytes.to_vec());
7282            let part = store.pools[1]
7283                .put_object_part(
7284                    &bucket,
7285                    multipart_object,
7286                    &upload.upload_id,
7287                    part_number,
7288                    &mut reader,
7289                    &ObjectOptions::default(),
7290                )
7291                .await
7292                .expect("write source multipart part");
7293            completed_parts.push(crate::storage_api_contracts::multipart::CompletePart {
7294                part_num: part.part_num,
7295                etag: part.etag,
7296                ..Default::default()
7297            });
7298        }
7299        store.pools[1]
7300            .clone()
7301            .complete_multipart_upload(&bucket, multipart_object, &upload.upload_id, completed_parts, &ObjectOptions::default())
7302            .await
7303            .expect("complete source multipart object in pool 1");
7304
7305        {
7306            let mut pool_meta = store.pool_meta.write().await;
7307            pool_meta.pools[1].decommission = Some(PoolDecommissionInfo {
7308                start_time: Some(OffsetDateTime::now_utc()),
7309                ..Default::default()
7310            });
7311        }
7312        assert!(store.is_suspended(1).await, "pool 1 must be the reverse decommission source");
7313
7314        let self_copy_opts = ObjectOptions {
7315            no_lock: true,
7316            ..Default::default()
7317        };
7318        let mut self_copy_reader = store
7319            .get_object_reader(&bucket, self_copy_object, None, HeaderMap::new(), &self_copy_opts)
7320            .await
7321            .expect("read the suspended source before its target is committed");
7322        let mut self_copy_info = self_copy_reader.object_info.clone();
7323        let mut self_copy_source_body = Vec::new();
7324        self_copy_reader
7325            .stream
7326            .read_to_end(&mut self_copy_source_body)
7327            .await
7328            .expect("drain the suspended self-copy source");
7329        assert_eq!(self_copy_source_body, self_copy_body);
7330        self_copy_info.metadata_only = true;
7331        self_copy_info.put_object_reader = Some(PutObjReader::from_vec(self_copy_source_body));
7332        store
7333            .copy_object(
7334                &bucket,
7335                self_copy_object,
7336                &bucket,
7337                self_copy_object,
7338                &mut self_copy_info,
7339                &self_copy_opts,
7340                &self_copy_opts,
7341            )
7342            .await
7343            .expect("self-copy should read the suspended source and commit to an active pool");
7344        assert_pool_object_present(&store.pools[0], &bucket, self_copy_object).await;
7345        assert_pool_object_present(&store.pools[1], &bucket, self_copy_object).await;
7346
7347        let mut active_copy_reader = store
7348            .get_object_reader(&bucket, self_copy_object, None, HeaderMap::new(), &self_copy_opts)
7349            .await
7350            .expect("read the active self-copy target while the source remains");
7351        let active_copy_data_dir = active_copy_reader.object_info.data_dir;
7352        let mut active_copy_info = active_copy_reader.object_info.clone();
7353        let mut active_copy_body = Vec::new();
7354        active_copy_reader
7355            .stream
7356            .read_to_end(&mut active_copy_body)
7357            .await
7358            .expect("drain the active self-copy target");
7359        assert_eq!(active_copy_body, self_copy_body);
7360        active_copy_info.metadata_only = true;
7361        active_copy_info.put_object_reader = Some(PutObjReader::from_vec(active_copy_body));
7362        let active_copy_result = store
7363            .copy_object(
7364                &bucket,
7365                self_copy_object,
7366                &bucket,
7367                self_copy_object,
7368                &mut active_copy_info,
7369                &self_copy_opts,
7370                &self_copy_opts,
7371            )
7372            .await
7373            .expect("self-copy should keep using the committed active target");
7374        assert_eq!(active_copy_result.data_dir, active_copy_data_dir);
7375
7376        {
7377            let mut pool_meta = store.pool_meta.write().await;
7378            pool_meta.pools[1].decommission = None;
7379        }
7380        mark_test_pool_decommissioning(&store, 1).await;
7381
7382        let cleanup_barrier = crate::data_movement::SourceCleanupDeleteBarrier::install(&bucket, object);
7383        let commit_barrier = crate::set_disk::PutObjectCommitBarrier::install(
7384            &bucket,
7385            object,
7386            crate::set_disk::PutObjectCommitPause::AfterNamespace,
7387        );
7388        let source_set = store.pools[1].get_disks_by_key(object);
7389        let worker_store = Arc::clone(&store);
7390        let worker_bucket = bucket.clone();
7391        let worker = tokio::spawn(async move {
7392            worker_store
7393                .decommission_entry_for_test(
7394                    1,
7395                    MetaCacheEntry {
7396                        name: object.to_string(),
7397                        ..Default::default()
7398                    },
7399                    worker_bucket,
7400                    source_set,
7401                )
7402                .await
7403        });
7404        commit_barrier.wait_until_paused().await;
7405
7406        let delete_barrier = crate::store::object::DeleteAfterObjectLockSnapshotBarrier::install(&bucket);
7407        let delete_store = Arc::clone(&store);
7408        let delete_bucket = bucket.clone();
7409        let delete = tokio::spawn(async move {
7410            delete_store
7411                .delete_object(&delete_bucket, object, ObjectOptions::default())
7412                .await
7413        });
7414        delete_barrier.wait_until_paused().await;
7415        delete_barrier.release_and_wait_until_namespace_pending().await;
7416        assert!(
7417            !delete_barrier.namespace_acquired() && !delete.is_finished(),
7418            "the reverse target commit must keep DELETE behind the fixed read fence"
7419        );
7420        delete.abort();
7421        assert!(
7422            delete
7423                .await
7424                .expect_err("the blocked DELETE should be canceled")
7425                .is_cancelled(),
7426            "canceling the blocked DELETE must not mutate either pool"
7427        );
7428        drop(delete_barrier);
7429
7430        commit_barrier.release();
7431        cleanup_barrier.wait_until_paused().await;
7432        drop(commit_barrier);
7433
7434        let read_opts = ObjectOptions {
7435            no_lock: true,
7436            ..Default::default()
7437        };
7438        let target_info = store.pools[0]
7439            .get_object_info(&bucket, object, &read_opts)
7440            .await
7441            .expect("read the committed active target before source cleanup");
7442        store.pools[1]
7443            .get_object_info(&bucket, object, &read_opts)
7444            .await
7445            .expect("the source must remain present until cleanup is released");
7446        let routed_info = store
7447            .get_object_info(&bucket, object, &read_opts)
7448            .await
7449            .expect("HEAD routing should prefer the active target before source cleanup");
7450        assert_eq!(routed_info.data_dir, target_info.data_dir);
7451
7452        let mut ranged_reader = store
7453            .get_object_reader(
7454                &bucket,
7455                object,
7456                Some(HTTPRangeSpec {
7457                    is_suffix_length: false,
7458                    start: 8,
7459                    end: 15,
7460                }),
7461                HeaderMap::new(),
7462                &read_opts,
7463            )
7464            .await
7465            .expect("ranged GET routing should prefer the active target before source cleanup");
7466        assert_eq!(ranged_reader.object_info.data_dir, target_info.data_dir);
7467        let mut ranged_body = Vec::new();
7468        ranged_reader
7469            .stream
7470            .read_to_end(&mut ranged_body)
7471            .await
7472            .expect("drain the routed target range");
7473        assert_eq!(ranged_body, object_body[8..=15]);
7474
7475        cleanup_barrier.release();
7476        drop(cleanup_barrier);
7477        tokio::time::timeout(Duration::from_secs(60), worker)
7478            .await
7479            .expect("reverse ordinary decommission must not self-deadlock on the fixed target set")
7480            .expect("reverse ordinary decommission worker should join")
7481            .expect("reverse ordinary decommission should complete");
7482
7483        let mut ordinary_reader = store.pools[0]
7484            .get_object_reader(&bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
7485            .await
7486            .expect("read the ordinary object from the fixed target set");
7487        let mut ordinary_target_body = Vec::new();
7488        ordinary_reader
7489            .stream
7490            .read_to_end(&mut ordinary_target_body)
7491            .await
7492            .expect("drain the ordinary target body");
7493        assert_eq!(ordinary_target_body, object_body, "ordinary migration must preserve the full body");
7494        let ordinary_source_err = store.pools[1]
7495            .get_object_info(&bucket, object, &ObjectOptions::default())
7496            .await
7497            .expect_err("ordinary source generation must be cleaned after migration");
7498        assert!(matches!(ordinary_source_err, StorageError::ObjectNotFound(_, _)));
7499
7500        let multipart_source_set = store.pools[1].get_disks_by_key(multipart_object);
7501        let multipart_store = Arc::clone(&store);
7502        let multipart_bucket = bucket.clone();
7503        let multipart_worker = tokio::spawn(async move {
7504            multipart_store
7505                .decommission_entry_for_test(
7506                    1,
7507                    MetaCacheEntry {
7508                        name: multipart_object.to_string(),
7509                        ..Default::default()
7510                    },
7511                    multipart_bucket,
7512                    multipart_source_set,
7513                )
7514                .await
7515        });
7516        tokio::time::timeout(Duration::from_secs(60), multipart_worker)
7517            .await
7518            .expect("reverse multipart decommission must not self-deadlock on new or complete")
7519            .expect("reverse multipart decommission worker should join")
7520            .expect("reverse multipart decommission should complete");
7521
7522        let target_info = store.pools[0]
7523            .get_object_info(&bucket, multipart_object, &ObjectOptions::default())
7524            .await
7525            .expect("read migrated multipart metadata from the fixed target set");
7526        assert!(target_info.is_multipart(), "migration must retain multipart identity");
7527        let mut multipart_reader = store.pools[0]
7528            .get_object_reader(&bucket, multipart_object, None, HeaderMap::new(), &ObjectOptions::default())
7529            .await
7530            .expect("read migrated multipart object from the fixed target set");
7531        let mut multipart_target_body = Vec::new();
7532        multipart_reader
7533            .stream
7534            .read_to_end(&mut multipart_target_body)
7535            .await
7536            .expect("drain the multipart target body");
7537        assert_eq!(multipart_target_body, multipart_body, "multipart migration must preserve the full body");
7538        let multipart_source_err = store.pools[1]
7539            .get_object_info(&bucket, multipart_object, &ObjectOptions::default())
7540            .await
7541            .expect_err("multipart source generation must be cleaned after migration");
7542        assert!(matches!(multipart_source_err, StorageError::ObjectNotFound(_, _)));
7543
7544        shutdown.cancel();
7545    }
7546
7547    #[tokio::test]
7548    #[serial_test::serial(storage_class_env)]
7549    async fn batch_delete_real_path_preserves_source_pool_errors_in_any_pool_order() {
7550        let temp_dir = tempfile::tempdir().expect("create batch delete pool-error store dir");
7551        let (_ctx, store, shutdown) =
7552            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "batch-delete-pool-errors", &[4, 4])).await;
7553        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
7554
7555        for source_pool_idx in [0, 1] {
7556            {
7557                let mut pool_meta = store.pool_meta.write().await;
7558                for pool in &mut pool_meta.pools {
7559                    pool.decommission = None;
7560                }
7561            }
7562
7563            let bucket = format!("batch-del-pool-error-{source_pool_idx}-{}", uuid::Uuid::new_v4());
7564            let object_names = vec![
7565                format!("third-{source_pool_idx}.bin"),
7566                format!("first-{source_pool_idx}.bin"),
7567                format!("second-{source_pool_idx}.bin"),
7568            ];
7569            store
7570                .make_bucket(&bucket, &MakeBucketOptions::default())
7571                .await
7572                .expect("create batch delete pool-error bucket");
7573            for pool in &store.pools {
7574                for object_name in &object_names {
7575                    let mut reader = PutObjReader::from_vec(format!("pool {} {object_name}", pool.pool_idx).into_bytes());
7576                    pool.put_object(&bucket, object_name, &mut reader, &ObjectOptions::default())
7577                        .await
7578                        .expect("seed each object in both the source and active pools");
7579                }
7580            }
7581            {
7582                let mut pool_meta = store.pool_meta.write().await;
7583                pool_meta.pools[source_pool_idx].decommission = Some(PoolDecommissionInfo {
7584                    start_time: Some(OffsetDateTime::now_utc()),
7585                    ..Default::default()
7586                });
7587            }
7588            assert!(
7589                store.is_suspended(source_pool_idx).await,
7590                "the injected error pool must be the decommission source"
7591            );
7592
7593            let expected_errors = [
7594                StorageError::ErasureWriteQuorum,
7595                StorageError::NamespaceLockQuorumUnavailable {
7596                    mode: "delete_objects_commit",
7597                    bucket: bucket.clone(),
7598                    object: object_names[1].clone(),
7599                    required: 3,
7600                    achieved: 2,
7601                },
7602                StorageError::ErasureWriteQuorum,
7603            ];
7604            let injection = crate::store::object::BatchDeletePoolErrorInjection::install(
7605                &bucket,
7606                source_pool_idx,
7607                object_names.iter().cloned().zip(expected_errors.iter().cloned()).collect(),
7608            );
7609            let requests = object_names
7610                .iter()
7611                .map(|object_name| ObjectToDelete {
7612                    object_name: object_name.clone(),
7613                    ..Default::default()
7614                })
7615                .collect();
7616
7617            let (deleted, errors) = store.delete_objects(&bucket, requests, ObjectOptions::default()).await;
7618
7619            assert_eq!(
7620                injection.observed(),
7621                object_names.len(),
7622                "the source pool must first complete every real delete"
7623            );
7624            assert_eq!(
7625                errors,
7626                expected_errors.iter().cloned().map(Some).collect::<Vec<_>>(),
7627                "a successful pool must not clear a source pool failure at any request index"
7628            );
7629            assert_eq!(
7630                deleted.iter().map(|object| object.object_name.as_str()).collect::<Vec<_>>(),
7631                object_names.iter().map(String::as_str).collect::<Vec<_>>(),
7632                "DeleteObjects must preserve request index mapping while aggregating pool failures"
7633            );
7634            assert!(
7635                deleted.iter().all(|object| object.found),
7636                "the injected source results must retain real delete success data"
7637            );
7638
7639            for pool in &store.pools {
7640                for object_name in &object_names {
7641                    let error = pool
7642                        .get_object_info(&bucket, object_name, &ObjectOptions::default())
7643                        .await
7644                        .expect_err("both the active and source pool delete calls must execute");
7645                    assert!(
7646                        matches!(error, StorageError::ObjectNotFound(_, _)),
7647                        "unexpected residual object: {error:?}"
7648                    );
7649                }
7650            }
7651            drop(injection);
7652        }
7653
7654        shutdown.cancel();
7655    }
7656
7657    #[tokio::test]
7658    #[serial_test::serial(storage_class_env)]
7659    async fn decommission_source_cleanup_holds_hashed_set_lock_across_preflight() {
7660        let temp_dir = tempfile::tempdir().expect("create multi-set decommission cleanup store dir");
7661        let (_ctx, store, shutdown) = without_storage_class_env(build_isolated_test_store_with_layout(
7662            temp_dir.path(),
7663            "multi-set-decommission-source-cleanup",
7664            &[(2, 4)],
7665            CancellationToken::new(),
7666            None,
7667        ))
7668        .await;
7669        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
7670
7671        let bucket = format!("decom-source-cleanup-lock-{}", uuid::Uuid::new_v4());
7672        let object = (0..128)
7673            .map(|index| format!("object-{index}.bin"))
7674            .find(|candidate| store.pools[0].get_disks_by_key(candidate).set_index == 1)
7675            .expect("the deterministic object search should select source set 1");
7676        store
7677            .make_bucket(&bucket, &MakeBucketOptions::default())
7678            .await
7679            .expect("create multi-set decommission cleanup bucket");
7680        let mut source = PutObjReader::from_vec(b"source generation".to_vec());
7681        store.pools[0]
7682            .put_object(&bucket, &object, &mut source, &ObjectOptions::default())
7683            .await
7684            .expect("write the source generation to set 1");
7685        let source_set = store.pools[0].get_disks_by_key(&object);
7686        assert_eq!(source_set.set_index, 1, "the source must not share the fixed set-0 namespace");
7687        let expected_source_versions = source_set
7688            .load_file_info_versions_exact(&bucket, &object)
7689            .await
7690            .expect("source versions should be readable")
7691            .expect("the source generation should exist");
7692
7693        let cleanup_barrier = crate::data_movement::SourceCleanupDeleteBarrier::install(&bucket, &object);
7694        let cleanup_store = Arc::clone(&store);
7695        let cleanup_bucket = bucket.clone();
7696        let cleanup_object = object.clone();
7697        let cleanup = tokio::spawn(async move {
7698            let mutation_fence = cleanup_store
7699                .acquire_decommission_source_cleanup_fence(&cleanup_bucket, &cleanup_object, source_set.as_ref())
7700                .await?;
7701            crate::data_movement::cleanup_source_entry_if_unchanged(
7702                source_set,
7703                &cleanup_bucket,
7704                &cleanup_object,
7705                &expected_source_versions,
7706                &[],
7707                crate::data_movement::SourceCleanupBucketFence {
7708                    object_mutation_fence: Some(&mutation_fence),
7709                    ..Default::default()
7710                },
7711                "test_multi_set_decommission_source_cleanup",
7712            )
7713            .await
7714        });
7715        cleanup_barrier.wait_until_paused().await;
7716
7717        let put_barrier = crate::set_disk::PutObjectCommitBarrier::install(
7718            &bucket,
7719            &object,
7720            crate::set_disk::PutObjectCommitPause::BeforeNamespace,
7721        );
7722        let mutation_pool = Arc::clone(&store.pools[0]);
7723        let mutation_bucket = bucket.clone();
7724        let mutation_object = object.clone();
7725        let replacement = b"replacement generation".to_vec();
7726        let expected_replacement = replacement.clone();
7727        let mutation = tokio::spawn(async move {
7728            let mut reader = PutObjReader::from_vec(replacement);
7729            mutation_pool
7730                .put_object(&mutation_bucket, &mutation_object, &mut reader, &ObjectOptions::default())
7731                .await
7732        });
7733        put_barrier.wait_until_paused().await;
7734        put_barrier.release_and_wait_until_namespace_pending().await;
7735        assert!(!mutation.is_finished(), "a source mutation must wait behind cleanup's set-1 write lock");
7736
7737        cleanup_barrier.release();
7738        cleanup
7739            .await
7740            .expect("source cleanup task should join")
7741            .expect("source cleanup should remove only the preflight generation");
7742        mutation
7743            .await
7744            .expect("source mutation task should join")
7745            .expect("source mutation should commit after cleanup releases the set lock");
7746
7747        let mut reader = store.pools[0]
7748            .get_object_reader(&bucket, &object, None, HeaderMap::new(), &ObjectOptions::default())
7749            .await
7750            .expect("the replacement generation must remain readable");
7751        let mut actual = Vec::new();
7752        reader
7753            .stream
7754            .read_to_end(&mut actual)
7755            .await
7756            .expect("read the replacement generation");
7757        assert_eq!(actual, expected_replacement, "cleanup must not delete the replacement generation");
7758
7759        shutdown.cancel();
7760    }
7761
7762    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7763    #[serial_test::serial(storage_class_env)]
7764    async fn versioned_delete_marker_survives_decommission_source_cleanup() {
7765        let temp_dir = tempfile::tempdir().expect("create versioned decommission delete-fence store dir");
7766        let (_ctx, store, shutdown) =
7767            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "versioned-decommission-delete-fence", &[4, 4]))
7768                .await;
7769        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
7770
7771        let bucket = format!("versioned-decom-delete-{}", uuid::Uuid::new_v4());
7772        let object = "object.bin";
7773        store
7774            .make_bucket(&bucket, &MakeBucketOptions::default())
7775            .await
7776            .expect("create versioned decommission delete-fence bucket");
7777        let (source_version, expected_source_versions) = migrate_versioned_decommission_test_object(
7778            &store,
7779            &bucket,
7780            object,
7781            b"source generation",
7782            "test_versioned_decommission_delete_fence",
7783        )
7784        .await;
7785
7786        let delete_barrier = crate::store::object::VersionedDeleteMarkerCommitBarrier::install(&bucket, object);
7787        let delete_store = Arc::clone(&store);
7788        let delete_bucket = bucket.clone();
7789        let delete = tokio::spawn(async move {
7790            delete_store
7791                .delete_object(
7792                    &delete_bucket,
7793                    object,
7794                    ObjectOptions {
7795                        versioned: true,
7796                        ..Default::default()
7797                    },
7798                )
7799                .await
7800        });
7801        delete_barrier.wait_until_paused().await;
7802        let cleanup_set = store.pools[0].get_disks_by_key(object);
7803        crate::data_movement::ensure_source_cleanup_versions_unchanged(
7804            Arc::clone(&cleanup_set),
7805            &bucket,
7806            object,
7807            &expected_source_versions,
7808            &[],
7809            "test_versioned_decommission_delete_fence",
7810        )
7811        .await
7812        .expect("the committed delete marker must not be published to the suspended source pool");
7813
7814        let cleanup_delete_barrier = crate::data_movement::SourceCleanupDeleteBarrier::install(&bucket, object);
7815        let cleanup_store = Arc::clone(&store);
7816        let cleanup_bucket = bucket.clone();
7817        let cleanup = tokio::spawn(async move {
7818            let mutation_fence = cleanup_store
7819                .acquire_decommission_source_cleanup_fence(&cleanup_bucket, object, cleanup_set.as_ref())
7820                .await?;
7821            crate::data_movement::cleanup_source_entry_if_unchanged(
7822                cleanup_set,
7823                &cleanup_bucket,
7824                object,
7825                &expected_source_versions,
7826                &[],
7827                crate::data_movement::SourceCleanupBucketFence {
7828                    object_mutation_fence: Some(&mutation_fence),
7829                    ..Default::default()
7830                },
7831                "test_versioned_decommission_delete_fence",
7832            )
7833            .await
7834        });
7835        cleanup_delete_barrier.wait_until_fence_pending().await;
7836        assert!(
7837            !cleanup_delete_barrier.is_paused(),
7838            "source cleanup must wait for the versioned DELETE mutation fence"
7839        );
7840
7841        delete_barrier.release();
7842        let marker = delete
7843            .await
7844            .expect("versioned DELETE task should join")
7845            .expect("versioned DELETE should publish a delete marker after migration");
7846        assert!(marker.delete_marker, "versioned DELETE must publish a delete marker");
7847        assert!(
7848            marker.version_id.is_some_and(|version_id| !version_id.is_nil()),
7849            "the delete marker must have a non-nil version ID"
7850        );
7851
7852        cleanup_delete_barrier.wait_until_paused().await;
7853        cleanup_delete_barrier.release();
7854        cleanup
7855            .await
7856            .expect("source cleanup task should join")
7857            .expect("source cleanup should preserve the active-pool delete marker");
7858
7859        let err = store
7860            .get_object_info(
7861                &bucket,
7862                object,
7863                &ObjectOptions {
7864                    versioned: true,
7865                    ..Default::default()
7866                },
7867            )
7868            .await
7869            .expect_err("the post-migration delete marker must hide the migrated version");
7870        assert!(
7871            matches!(err, StorageError::ObjectNotFound(_, _)),
7872            "unexpected latest-version result: {err:?}"
7873        );
7874        store
7875            .get_object_info(
7876                &bucket,
7877                object,
7878                &ObjectOptions {
7879                    versioned: true,
7880                    version_id: Some(source_version.to_string()),
7881                    ..Default::default()
7882                },
7883            )
7884            .await
7885            .expect("the migrated source version must remain addressable below the delete marker");
7886        store.pools[0]
7887            .get_object_info(
7888                &bucket,
7889                object,
7890                &ObjectOptions {
7891                    versioned: true,
7892                    version_id: Some(source_version.to_string()),
7893                    ..Default::default()
7894                },
7895            )
7896            .await
7897            .expect_err("source cleanup must remove the decommissioned source versions");
7898
7899        shutdown.cancel();
7900    }
7901
7902    #[cfg(feature = "test-util")]
7903    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7904    #[serial_test::serial(storage_class_env)]
7905    async fn decommission_entry_skips_cleanup_only_marker_when_free_version_is_present() {
7906        let temp_dir = tempfile::tempdir().expect("create free-version decommission store dir");
7907        let (ctx, store, shutdown) =
7908            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "decommission-free-marker", &[4, 4])).await;
7909        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
7910        let bucket = format!("decom-free-marker-{}", uuid::Uuid::new_v4());
7911        let object = "free-marker-object";
7912        store
7913            .make_bucket(&bucket, &MakeBucketOptions::default())
7914            .await
7915            .expect("create free-version decommission bucket");
7916        let (_, free_version) = seed_transitioned_free_version(&ctx, &store, &bucket, object).await;
7917        let source_free = store.pools[0]
7918            .get_disks_by_key(object)
7919            .load_file_info_versions_exact(&bucket, object)
7920            .await
7921            .expect("source free-version metadata should decode")
7922            .and_then(|versions| {
7923                versions
7924                    .versions
7925                    .into_iter()
7926                    .find(|version| version.version_id == Some(free_version) && version.tier_free_version())
7927            })
7928            .expect("source free-version identity should be present before decommission");
7929        let mut target_reader = PutObjReader::from_vec(b"target ordinary bytes".to_vec());
7930        let target_w = store.pools[1]
7931            .put_object(
7932                &bucket,
7933                object,
7934                &mut target_reader,
7935                &ObjectOptions {
7936                    versioned: true,
7937                    ..Default::default()
7938                },
7939            )
7940            .await
7941            .expect("write unrelated target version");
7942        let target_w_version = target_w.version_id.expect("target version should have an id");
7943        let marker = store.pools[0]
7944            .delete_object(
7945                &bucket,
7946                object,
7947                ObjectOptions {
7948                    versioned: true,
7949                    ..Default::default()
7950                },
7951            )
7952            .await
7953            .expect("write cleanup-only delete marker");
7954        assert!(marker.delete_marker);
7955
7956        mark_test_pool_decommissioning(&store, 0).await;
7957        let source_set = store.pools[0].get_disks_by_key(object);
7958        store
7959            .decommission_entry_for_test_with_bucket_incarnation(
7960                0,
7961                MetaCacheEntry {
7962                    name: object.to_string(),
7963                    ..Default::default()
7964                },
7965                bucket.clone(),
7966                source_set.clone(),
7967            )
7968            .await
7969            .expect("real decommission entry should migrate the free version");
7970
7971        let target_versions = store.pools[1]
7972            .get_disks_by_key(object)
7973            .load_file_info_versions_exact(&bucket, object)
7974            .await
7975            .expect("target versions should decode")
7976            .expect("target free version should be present");
7977        assert!(
7978            target_versions
7979                .versions
7980                .iter()
7981                .any(|version| { version.version_id == Some(free_version) && version.tier_free_version() })
7982        );
7983        let migrated_free = target_versions
7984            .versions
7985            .iter()
7986            .find(|version| version.version_id == Some(free_version) && version.tier_free_version())
7987            .expect("migrated free-version identity should remain readable from target disks");
7988        assert!(
7989            crate::store::tiered_data_movement_source_matches(&source_free, migrated_free)
7990                .expect("migrated free-version identity should decode")
7991        );
7992        let retained_w = target_versions
7993            .versions
7994            .iter()
7995            .find(|version| version.version_id == Some(target_w_version))
7996            .expect("unrelated target version should remain");
7997        assert!(!retained_w.deleted && !retained_w.tier_free_version());
7998        assert_eq!(retained_w.size, target_w.size);
7999        assert_eq!(retained_w.get_etag(), target_w.etag);
8000        assert!(
8001            target_versions
8002                .versions
8003                .iter()
8004                .all(|version| { version.tier_free_version() || !version.deleted })
8005        );
8006        assert!(
8007            source_set
8008                .load_file_info_versions_exact(&bucket, object)
8009                .await
8010                .expect("source versions should be readable after cleanup")
8011                .is_none(),
8012            "successful free-version migration should permit source cleanup"
8013        );
8014        let (heal_versions, _, _) = store
8015            .heal_walk_versions_page(1, 0, &bucket, "", None, 2, 16, true)
8016            .await
8017            .expect("heal walk should decode the migrated free version");
8018        let free_version_string = free_version.to_string();
8019        let healed_free = heal_versions
8020            .iter()
8021            .find(|version| version.version_id.as_deref() == Some(free_version_string.as_str()))
8022            .expect("heal walk should surface the migrated free version");
8023        let healed_info = healed_free
8024            .lifecycle_object_info
8025            .as_ref()
8026            .expect("heal walk should retain lifecycle identity for the migrated free version");
8027        assert!(healed_info.transitioned_object.free_version);
8028        assert_eq!(healed_info.transitioned_object.tier, source_free.transition_tier);
8029        assert_eq!(healed_info.transitioned_object.name, source_free.transitioned_objname);
8030        shutdown.cancel();
8031    }
8032
8033    #[cfg(feature = "test-util")]
8034    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8035    #[serial_test::serial(storage_class_env)]
8036    async fn decommission_entry_allows_free_version_consumed_before_source_lock() {
8037        let temp_dir = tempfile::tempdir().expect("create consumed free-version store dir");
8038        let (ctx, store, shutdown) =
8039            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "decommission-free-consumed", &[4, 4])).await;
8040        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
8041        let bucket = format!("decom-free-consumed-{}", uuid::Uuid::new_v4());
8042        let object = "free-consumed-object";
8043        store
8044            .make_bucket(&bucket, &MakeBucketOptions::default())
8045            .await
8046            .expect("create consumed free-version bucket");
8047        let (_, free_version) = seed_transitioned_free_version(&ctx, &store, &bucket, object).await;
8048
8049        mark_test_pool_decommissioning(&store, 0).await;
8050        let source_set = store.pools[0].get_disks_by_key(object);
8051        let barrier = crate::store::object::DecommissionFreeVersionSourceRaceBarrier::install(&bucket, object);
8052        let decommission = tokio::spawn({
8053            let store = store.clone();
8054            let bucket = bucket.clone();
8055            let source_set = source_set.clone();
8056            async move {
8057                store
8058                    .decommission_entry_for_test_with_bucket_incarnation(
8059                        0,
8060                        MetaCacheEntry {
8061                            name: object.to_string(),
8062                            ..Default::default()
8063                        },
8064                        bucket,
8065                        source_set,
8066                    )
8067                    .await
8068            }
8069        });
8070
8071        barrier.wait_until_paused().await;
8072        store.pools[0]
8073            .delete_object(
8074                &bucket,
8075                object,
8076                ObjectOptions {
8077                    versioned: true,
8078                    version_id: Some(free_version.to_string()),
8079                    incl_free_versions: true,
8080                    ..Default::default()
8081                },
8082            )
8083            .await
8084            .expect("lifecycle should consume the source free version before decommission locks it");
8085        assert!(
8086            source_set
8087                .load_file_info_versions_exact(&bucket, object)
8088                .await
8089                .expect("consumed source metadata should remain readable")
8090                .is_none(),
8091            "the lifecycle delete should remove the source free version"
8092        );
8093
8094        barrier.release();
8095        decommission
8096            .await
8097            .expect("decommission task should join")
8098            .expect("a concurrently consumed free version should not fail source cleanup");
8099        assert!(
8100            store.pools[1]
8101                .get_disks_by_key(object)
8102                .load_file_info_versions_exact(&bucket, object)
8103                .await
8104                .expect("target metadata should remain readable")
8105                .is_none(),
8106            "an already consumed free version should not be recreated on the target"
8107        );
8108        shutdown.cancel();
8109    }
8110
8111    #[cfg(feature = "test-util")]
8112    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8113    #[serial_test::serial(storage_class_env)]
8114    async fn decommission_entry_rejects_subquorum_free_version_conflict_and_retains_source() {
8115        let temp_dir = tempfile::tempdir().expect("create sub-quorum free-version store dir");
8116        let (ctx, store, shutdown) =
8117            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "decommission-free-conflict", &[4, 4])).await;
8118        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
8119        let bucket = format!("decom-free-conflict-{}", uuid::Uuid::new_v4());
8120        let object = "free-conflict-object";
8121        store
8122            .make_bucket(&bucket, &MakeBucketOptions::default())
8123            .await
8124            .expect("create sub-quorum conflict bucket");
8125        let (_, free_version) = seed_transitioned_free_version(&ctx, &store, &bucket, object).await;
8126        let source_free = store.pools[0]
8127            .get_disks_by_key(object)
8128            .load_file_info_versions_exact(&bucket, object)
8129            .await
8130            .expect("source free version should decode before crash replay setup")
8131            .and_then(|versions| {
8132                versions
8133                    .versions
8134                    .into_iter()
8135                    .find(|version| version.version_id == Some(free_version))
8136            })
8137            .expect("source free version should be available for crash replay setup");
8138
8139        let mut target_reader = PutObjReader::from_vec(b"target ordinary bytes".to_vec());
8140        let target = store.pools[1]
8141            .put_object(
8142                &bucket,
8143                object,
8144                &mut target_reader,
8145                &ObjectOptions {
8146                    versioned: true,
8147                    ..Default::default()
8148                },
8149            )
8150            .await
8151            .expect("seed ordinary target version");
8152        let target_version = target.version_id.expect("target version must have an ID");
8153        // A regular PUT may acknowledge write quorum while its rename tail still owns the
8154        // target namespace lock. Drain that tail through a locked read before bypassing
8155        // the namespace API with the per-disk crash-replay fixture writes below.
8156        store.pools[1]
8157            .get_object_info(
8158                &bucket,
8159                object,
8160                &ObjectOptions {
8161                    versioned: true,
8162                    version_id: Some(target_version.to_string()),
8163                    ..Default::default()
8164                },
8165            )
8166            .await
8167            .expect("target PUT tail should drain before per-disk fixture mutation");
8168        let target_disks = store.pools[1].get_disks_by_key(object).disks.read().await.clone();
8169        for disk in target_disks.iter().skip(1) {
8170            disk.as_ref()
8171                .expect("target crash replay quorum disk should be online")
8172                .write_metadata("", &bucket, object, source_free.clone())
8173                .await
8174                .expect("seed an equivalent free version on the target quorum");
8175        }
8176        let conflict_path = temp_dir
8177            .path()
8178            .join(format!("pool1/set0/disk0/{bucket}/{object}/{STORAGE_FORMAT_FILE}"));
8179        let encoded = tokio::fs::read(&conflict_path)
8180            .await
8181            .expect("target metadata should be readable");
8182        let mut metadata = FileMeta::load(&encoded).expect("target metadata should decode");
8183        let target_index = metadata
8184            .versions
8185            .iter()
8186            .position(|version| version.header.version_id == Some(target_version))
8187            .expect("target version should be present on the conflict disk");
8188        let mut target_meta = metadata.versions[target_index]
8189            .parse_version_meta()
8190            .expect("target version metadata should decode");
8191        target_meta
8192            .object
8193            .as_mut()
8194            .expect("target conflict must remain an ordinary object")
8195            .version_id = Some(free_version);
8196        metadata.versions[target_index] = target_meta.try_into().expect("conflict metadata should encode");
8197        let expected_conflict_meta = metadata.versions[target_index].meta.clone();
8198        let expected_conflict = metadata.versions[target_index]
8199            .into_fileinfo(&bucket, object, true)
8200            .expect("conflict metadata should decode as an ordinary object");
8201        let duplicate_free: rustfs_filemeta::FileMetaShallowVersion = rustfs_filemeta::FileMetaVersion::from(source_free.clone())
8202            .try_into()
8203            .expect("duplicate free metadata should encode");
8204        metadata.versions.insert(target_index, duplicate_free);
8205        tokio::fs::write(&conflict_path, metadata.marshal_msg().expect("conflict metadata should encode"))
8206            .await
8207            .expect("write sub-quorum conflict metadata");
8208
8209        let mut expected_target_metadata = Vec::with_capacity(4);
8210        for disk_index in 0..4 {
8211            let target_path = temp_dir
8212                .path()
8213                .join(format!("pool1/set0/disk{disk_index}/{bucket}/{object}/{STORAGE_FORMAT_FILE}"));
8214            let target_encoded = tokio::fs::read(&target_path)
8215                .await
8216                .expect("target fixture metadata should be readable before decommission");
8217            let target_meta = FileMeta::load(&target_encoded).expect("target fixture metadata should decode");
8218            let same_id = target_meta
8219                .versions
8220                .iter()
8221                .filter(|version| version.header.version_id == Some(free_version))
8222                .collect::<Vec<_>>();
8223            if disk_index == 0 {
8224                assert_eq!(same_id.len(), 2, "conflict fixture should contain both same-ID records");
8225                assert!(same_id[0].header.free_version());
8226                assert!(!same_id[1].header.free_version());
8227            } else {
8228                assert_eq!(same_id.len(), 1, "target fixture disk {disk_index} should contain one free version");
8229                assert!(same_id[0].header.free_version());
8230            }
8231            expected_target_metadata.push(target_encoded);
8232        }
8233
8234        mark_test_pool_decommissioning(&store, 0).await;
8235        let source_set = store.pools[0].get_disks_by_key(object);
8236        store
8237            .decommission_entry_for_test_with_bucket_incarnation(
8238                0,
8239                MetaCacheEntry {
8240                    name: object.to_string(),
8241                    ..Default::default()
8242                },
8243                bucket.clone(),
8244                source_set.clone(),
8245            )
8246            .await
8247            .expect("conflicted decommission entry should retain the source and retry later");
8248
8249        let source_versions = source_set
8250            .load_file_info_versions_exact(&bucket, object)
8251            .await
8252            .expect("retained source versions should decode")
8253            .expect("source free version should be retained after conflict");
8254        assert!(
8255            source_versions
8256                .versions
8257                .iter()
8258                .any(|version| { version.version_id == Some(free_version) && version.tier_free_version() })
8259        );
8260        let post_encoded = tokio::fs::read(&conflict_path)
8261            .await
8262            .expect("conflict metadata should remain readable");
8263        let post_metadata = FileMeta::load(&post_encoded).expect("post-conflict metadata should decode");
8264        let same_id = post_metadata
8265            .versions
8266            .iter()
8267            .filter(|version| version.header.version_id == Some(free_version))
8268            .collect::<Vec<_>>();
8269        assert_eq!(same_id.len(), 2, "conflict metadata should retain both same-ID records");
8270        assert_eq!(same_id.iter().filter(|version| version.header.free_version()).count(), 1);
8271        assert_eq!(same_id.iter().filter(|version| !version.header.free_version()).count(), 1);
8272        let post_conflict = same_id
8273            .into_iter()
8274            .find(|version| !version.header.free_version())
8275            .expect("ordinary conflict version must remain addressable by the source ID");
8276        let post_conflict_info = post_conflict
8277            .into_fileinfo(&bucket, object, true)
8278            .expect("post-conflict ordinary metadata should decode");
8279        assert!(!post_conflict_info.deleted && !post_conflict_info.tier_free_version());
8280        assert_eq!(post_conflict.meta, expected_conflict_meta);
8281        assert_eq!(post_conflict_info.size, expected_conflict.size);
8282        assert_eq!(post_conflict_info.data_dir, expected_conflict.data_dir);
8283        assert_eq!(post_conflict_info.metadata, expected_conflict.metadata);
8284        assert_eq!(post_conflict_info.get_etag(), expected_conflict.get_etag());
8285
8286        for (disk_index, expected_target_encoded) in expected_target_metadata.iter().enumerate() {
8287            let target_path = temp_dir
8288                .path()
8289                .join(format!("pool1/set0/disk{disk_index}/{bucket}/{object}/{STORAGE_FORMAT_FILE}"));
8290            let target_encoded = tokio::fs::read(&target_path)
8291                .await
8292                .expect("target metadata should remain readable");
8293            assert_eq!(
8294                &target_encoded, expected_target_encoded,
8295                "conflicted decommission must not mutate target disk {disk_index}"
8296            );
8297            let target_meta = FileMeta::load(&target_encoded).expect("target metadata should decode");
8298            let same_id = target_meta
8299                .versions
8300                .iter()
8301                .filter(|version| version.header.version_id == Some(free_version))
8302                .collect::<Vec<_>>();
8303            if disk_index == 0 {
8304                assert_eq!(same_id.len(), 2);
8305                assert!(same_id[0].header.free_version());
8306                assert!(!same_id[1].header.free_version());
8307            } else {
8308                assert_eq!(same_id.len(), 1);
8309                assert!(same_id[0].header.free_version());
8310            }
8311        }
8312        let sweep_err = store
8313            .check_after_decommission_for_test(0)
8314            .await
8315            .expect_err("final sweep must report the retained free version");
8316        assert!(
8317            sweep_err.to_string().contains("version(s) were found"),
8318            "unexpected final sweep error: {sweep_err}"
8319        );
8320        shutdown.cancel();
8321    }
8322
8323    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8324    #[serial_test::serial(storage_class_env)]
8325    async fn versioned_batch_delete_marker_skips_decommission_source() {
8326        let temp_dir = tempfile::tempdir().expect("create versioned batch decommission store dir");
8327        let (_ctx, store, shutdown) = without_storage_class_env(build_isolated_test_store(
8328            temp_dir.path(),
8329            "versioned-batch-decommission-delete-fence",
8330            &[4, 4, 4],
8331        ))
8332        .await;
8333        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
8334
8335        let bucket = format!("vbatch-decom-delete-{}", uuid::Uuid::new_v4());
8336        let object = "batch-object.bin";
8337        store
8338            .make_bucket(&bucket, &MakeBucketOptions::default())
8339            .await
8340            .expect("create versioned batch decommission bucket");
8341        let (_source_version, expected_source_versions) = migrate_versioned_decommission_test_object(
8342            &store,
8343            &bucket,
8344            object,
8345            b"batch source generation",
8346            "test_versioned_batch_decommission_delete_fence",
8347        )
8348        .await;
8349
8350        let delete_config_snapshot =
8351            Arc::new(crate::bucket::replication::DeleteReplicationConfigSnapshot::from_configs_for_test(
8352                s3s::dto::VersioningConfiguration {
8353                    status: Some(s3s::dto::BucketVersioningStatus::from_static(s3s::dto::BucketVersioningStatus::ENABLED)),
8354                    ..Default::default()
8355                },
8356                None,
8357            ));
8358        let delete_barrier = crate::store::object::VersionedDeleteMarkerCommitBarrier::install(&bucket, object);
8359        let delete_store = Arc::clone(&store);
8360        let delete_bucket = bucket.clone();
8361        let delete = tokio::spawn(async move {
8362            delete_store
8363                .delete_objects(
8364                    &delete_bucket,
8365                    vec![ObjectToDelete {
8366                        object_name: object.to_string(),
8367                        ..Default::default()
8368                    }],
8369                    ObjectOptions {
8370                        delete_replication_config_snapshot: Some(delete_config_snapshot),
8371                        ..Default::default()
8372                    },
8373                )
8374                .await
8375        });
8376        delete_barrier.wait_until_paused().await;
8377
8378        let source_set = store.pools[0].get_disks_by_key(object);
8379        crate::data_movement::ensure_source_cleanup_versions_unchanged(
8380            source_set,
8381            &bucket,
8382            object,
8383            &expected_source_versions,
8384            &[],
8385            "test_versioned_batch_decommission_delete_fence",
8386        )
8387        .await
8388        .expect("batch DELETE must not publish a marker to the suspended source");
8389
8390        delete_barrier.release();
8391        let (deleted, errors) = delete.await.expect("versioned batch DELETE task should join");
8392        assert!(errors.iter().all(Option::is_none), "versioned batch DELETE should succeed: {errors:?}");
8393        assert_eq!(deleted.len(), 1);
8394        assert!(deleted[0].delete_marker, "versioned batch DELETE must return a marker");
8395        assert!(
8396            deleted[0]
8397                .delete_marker_version_id
8398                .is_some_and(|version_id| !version_id.is_nil()),
8399            "versioned batch DELETE marker must have a non-nil version ID"
8400        );
8401
8402        let mut active_marker_count = 0;
8403        for pool in store.pools.iter().skip(1) {
8404            let Some(versions) = pool
8405                .get_disks_by_key(object)
8406                .load_file_info_versions_exact(&bucket, object)
8407                .await
8408                .expect("active-pool versions should be readable")
8409            else {
8410                continue;
8411            };
8412            active_marker_count += versions.versions.iter().filter(|version| version.deleted).count();
8413        }
8414        assert_eq!(active_marker_count, 1, "batch DELETE must publish exactly one active-pool marker");
8415
8416        shutdown.cancel();
8417    }
8418
8419    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8420    #[serial_test::serial(storage_class_env)]
8421    async fn suspended_delete_marker_then_decommission_worker_converges_null_source() {
8422        let temp_dir = tempfile::tempdir().expect("create suspended decommission DELETE store dir");
8423        let (_ctx, store, shutdown) = without_storage_class_env(build_isolated_test_store(
8424            temp_dir.path(),
8425            "suspended-decommission-delete-convergence",
8426            &[4, 4],
8427        ))
8428        .await;
8429        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
8430
8431        let bucket = format!("suspended-decom-delete-{}", uuid::Uuid::new_v4());
8432        let object = "single.bin";
8433        store
8434            .make_bucket(&bucket, &MakeBucketOptions::default())
8435            .await
8436            .expect("create suspended decommission DELETE bucket");
8437        write_suspended_decommission_source(&store, &bucket, object).await;
8438        mark_test_pool_decommissioning(&store, 0).await;
8439
8440        let deleted = store
8441            .delete_object(
8442                &bucket,
8443                object,
8444                ObjectOptions {
8445                    version_suspended: true,
8446                    ..Default::default()
8447                },
8448            )
8449            .await
8450            .expect("a healthy reserved target must accept suspended DELETE");
8451        assert!(deleted.delete_marker);
8452        assert_eq!(deleted.version_id, Some(uuid::Uuid::nil()));
8453        let marker_mod_time = deleted.mod_time.expect("DELETE must return the marker generation");
8454        assert_suspended_null_source_present(&store, &bucket, object).await;
8455        assert_suspended_null_delete_marker_visible(&store, &bucket, object, marker_mod_time).await;
8456
8457        let source_set = store.pools[0].get_disks_by_key(object);
8458        let worker_store = Arc::clone(&store);
8459        let worker_bucket = bucket.clone();
8460        tokio::spawn(async move {
8461            worker_store
8462                .decommission_entry_for_test(
8463                    0,
8464                    MetaCacheEntry {
8465                        name: object.to_string(),
8466                        ..Default::default()
8467                    },
8468                    worker_bucket,
8469                    source_set,
8470                )
8471                .await
8472        })
8473        .await
8474        .expect("suspended decommission worker should join")
8475        .expect("worker must converge the old null source behind the newer DELETE marker");
8476
8477        assert_decommission_source_absent(
8478            &store,
8479            &bucket,
8480            object,
8481            &ObjectOptions {
8482                version_suspended: true,
8483                ..Default::default()
8484            },
8485        )
8486        .await;
8487        assert_suspended_null_delete_marker_visible(&store, &bucket, object, marker_mod_time).await;
8488        shutdown.cancel();
8489    }
8490
8491    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8492    #[serial_test::serial(storage_class_env)]
8493    async fn suspended_batch_delete_marker_then_decommission_worker_converges_null_source() {
8494        let temp_dir = tempfile::tempdir().expect("create suspended batch decommission DELETE store dir");
8495        let (_ctx, store, shutdown) = without_storage_class_env(build_isolated_test_store(
8496            temp_dir.path(),
8497            "suspended-batch-decommission-delete-convergence",
8498            &[4, 4],
8499        ))
8500        .await;
8501        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
8502
8503        let bucket = format!("susp-batch-decom-delete-{}", uuid::Uuid::new_v4());
8504        let object = "batch.bin";
8505        store
8506            .make_bucket(&bucket, &MakeBucketOptions::default())
8507            .await
8508            .expect("create suspended batch decommission DELETE bucket");
8509        write_suspended_decommission_source(&store, &bucket, object).await;
8510        mark_test_pool_decommissioning(&store, 0).await;
8511
8512        let delete_config_snapshot =
8513            Arc::new(crate::bucket::replication::DeleteReplicationConfigSnapshot::from_configs_for_test(
8514                s3s::dto::VersioningConfiguration {
8515                    status: Some(s3s::dto::BucketVersioningStatus::from_static(s3s::dto::BucketVersioningStatus::SUSPENDED)),
8516                    ..Default::default()
8517                },
8518                None,
8519            ));
8520        let (deleted, errors) = store
8521            .delete_objects(
8522                &bucket,
8523                vec![ObjectToDelete {
8524                    object_name: object.to_string(),
8525                    ..Default::default()
8526                }],
8527                ObjectOptions {
8528                    delete_replication_config_snapshot: Some(delete_config_snapshot),
8529                    ..Default::default()
8530                },
8531            )
8532            .await;
8533        assert!(
8534            matches!(errors.as_slice(), [None]),
8535            "unexpected suspended batch DELETE result: {errors:?}"
8536        );
8537        assert_eq!(deleted.len(), 1);
8538        assert!(deleted[0].delete_marker);
8539        assert_eq!(deleted[0].object_name, object);
8540        assert!(
8541            deleted[0]
8542                .delete_marker_version_id
8543                .is_none_or(|version_id| version_id.is_nil()),
8544            "batch DELETE must retain the native null version identity"
8545        );
8546        let marker_mod_time = deleted[0]
8547            .delete_marker_mtime
8548            .expect("batch DELETE must return the marker generation");
8549        assert_suspended_null_source_present(&store, &bucket, object).await;
8550        assert_suspended_null_delete_marker_visible(&store, &bucket, object, marker_mod_time).await;
8551
8552        let source_set = store.pools[0].get_disks_by_key(object);
8553        let worker_store = Arc::clone(&store);
8554        let worker_bucket = bucket.clone();
8555        tokio::spawn(async move {
8556            worker_store
8557                .decommission_entry_for_test(
8558                    0,
8559                    MetaCacheEntry {
8560                        name: object.to_string(),
8561                        ..Default::default()
8562                    },
8563                    worker_bucket,
8564                    source_set,
8565                )
8566                .await
8567        })
8568        .await
8569        .expect("suspended batch decommission worker should join")
8570        .expect("worker must converge the old null source behind the newer batch DELETE marker");
8571
8572        assert_decommission_source_absent(
8573            &store,
8574            &bucket,
8575            object,
8576            &ObjectOptions {
8577                version_suspended: true,
8578                ..Default::default()
8579            },
8580        )
8581        .await;
8582        assert_suspended_null_delete_marker_visible(&store, &bucket, object, marker_mod_time).await;
8583        shutdown.cancel();
8584    }
8585
8586    #[cfg(feature = "test-util")]
8587    #[tokio::test]
8588    #[serial_test::serial(storage_class_env)]
8589    async fn tiered_data_movement_rejects_a_stale_source_snapshot_before_target_write() {
8590        let temp_dir = tempfile::tempdir().expect("create stale-source data movement store dir");
8591        let (ctx, store, _shutdown) =
8592            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "stale-tier-source", &[4, 4])).await;
8593        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
8594        let bucket = "stale-tier-source-bucket";
8595        let object = "stale-tier-source-object";
8596        store
8597            .make_bucket(bucket, &MakeBucketOptions::default())
8598            .await
8599            .expect("bucket should be created");
8600        register_mock_tier(&ctx.tier_config_mgr(), "STALE-TIER").await;
8601        let incarnation = store
8602            .bucket_incarnation_id(bucket)
8603            .await
8604            .expect("bucket incarnation should exist");
8605        let version_id = uuid::Uuid::new_v4();
8606        let stale = FileInfo {
8607            volume: bucket.to_string(),
8608            name: object.to_string(),
8609            version_id: Some(version_id),
8610            transition_status: rustfs_filemeta::TRANSITION_COMPLETE.to_string(),
8611            transitioned_objname: "remote/stale-source".to_string(),
8612            transition_tier: "STALE-TIER".to_string(),
8613            transition_version_id: Some(uuid::Uuid::new_v4()),
8614            transition_version_state: rustfs_filemeta::TransitionVersionState::Exact,
8615            data_dir: Some(uuid::Uuid::new_v4()),
8616            mod_time: Some(OffsetDateTime::UNIX_EPOCH),
8617            size: 1,
8618            metadata: HashMap::new(),
8619            ..Default::default()
8620        };
8621        let err = store
8622            .decommission_tiered_object(
8623                bucket,
8624                object,
8625                &stale,
8626                &ObjectOptions {
8627                    versioned: true,
8628                    version_id: Some(version_id.to_string()),
8629                    src_pool_idx: 0,
8630                    data_movement: true,
8631                    expected_bucket_incarnation_id: Some(incarnation),
8632                    ..Default::default()
8633                },
8634            )
8635            .await
8636            .expect_err("a source removed after queue capture must fail closed");
8637        assert!(matches!(err, Error::ObjectNotFound(_, _) | Error::FileNotFound));
8638        let target_err = store.pools[1]
8639            .get_object_info(bucket, object, &ObjectOptions::default())
8640            .await
8641            .expect_err("stale source rejection must not write target metadata");
8642        assert!(matches!(
8643            target_err,
8644            StorageError::ObjectNotFound(_, _) | StorageError::VersionNotFound(_, _, _)
8645        ));
8646    }
8647
8648    #[tokio::test]
8649    #[serial_test::serial(storage_class_env)]
8650    async fn data_movement_put_conflict_validates_only_selected_target_pool() {
8651        let temp_dir = tempfile::tempdir().expect("create three-pool data movement store dir");
8652        let (_ctx, store, _shutdown) =
8653            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "data-movement-selected-target", &[4, 4, 4]))
8654                .await;
8655        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
8656
8657        let bucket = format!("dm-selected-target-{}", uuid::Uuid::new_v4());
8658        let object = "object.bin";
8659        let source_body = b"source-and-equivalent".to_vec();
8660        let conflicting_body = b"newer-conflicting-target".to_vec();
8661        let source_mod_time = OffsetDateTime::UNIX_EPOCH + time::Duration::SECOND;
8662        let version = uuid::Uuid::new_v4();
8663        store
8664            .make_bucket(&bucket, &MakeBucketOptions::default())
8665            .await
8666            .expect("create selected-target bucket");
8667        for (pool_idx, body, mod_time) in [
8668            (0, source_body.clone(), source_mod_time),
8669            (1, conflicting_body.clone(), source_mod_time + time::Duration::SECOND),
8670            (2, source_body.clone(), source_mod_time),
8671        ] {
8672            let mut reader = PutObjReader::from_vec(body);
8673            store.pools[pool_idx]
8674                .put_object(
8675                    &bucket,
8676                    object,
8677                    &mut reader,
8678                    &ObjectOptions {
8679                        versioned: true,
8680                        version_id: Some(version.to_string()),
8681                        mod_time: Some(mod_time),
8682                        ..Default::default()
8683                    },
8684                )
8685                .await
8686                .expect("seed data movement pool");
8687        }
8688        *store.rebalance_meta.write().await = Some(active_rebalance_meta_for_pool(store.pools.len(), 0));
8689
8690        let source_reader = store.pools[0]
8691            .get_object_reader(
8692                &bucket,
8693                object,
8694                None,
8695                HeaderMap::new(),
8696                &ObjectOptions {
8697                    versioned: true,
8698                    version_id: Some(version.to_string()),
8699                    raw_data_movement_read: true,
8700                    ..Default::default()
8701                },
8702            )
8703            .await
8704            .expect("read source for selected-target migration");
8705        crate::data_movement::migrate_object(store.clone(), 0, bucket.clone(), source_reader, None, "test_selected_target")
8706            .await
8707            .expect_err("an equivalent object in another pool must not mask the selected target conflict");
8708
8709        let mut selected_target = store.pools[1]
8710            .get_object_reader(
8711                &bucket,
8712                object,
8713                None,
8714                HeaderMap::new(),
8715                &ObjectOptions {
8716                    versioned: true,
8717                    version_id: Some(version.to_string()),
8718                    ..Default::default()
8719                },
8720            )
8721            .await
8722            .expect("read selected conflicting target");
8723        let mut selected_body = Vec::new();
8724        selected_target
8725            .stream
8726            .read_to_end(&mut selected_body)
8727            .await
8728            .expect("drain selected conflicting target");
8729        assert_eq!(selected_body, conflicting_body);
8730    }
8731
8732    #[tokio::test]
8733    #[serial_test::serial(storage_class_env)]
8734    async fn data_movement_target_remains_readable_before_source_cleanup() {
8735        let temp_dir = tempfile::tempdir().expect("create data movement read-window store dir");
8736        let (_ctx, store, shutdown) =
8737            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "data-movement-read-window", &[4, 4])).await;
8738        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
8739
8740        let bucket = format!("dm-read-window-{}", uuid::Uuid::new_v4());
8741        let object = "object.bin";
8742        let version = uuid::Uuid::new_v4();
8743        let mod_time = OffsetDateTime::UNIX_EPOCH + time::Duration::SECOND;
8744        store
8745            .make_bucket(&bucket, &MakeBucketOptions::default())
8746            .await
8747            .expect("create data movement read-window bucket");
8748
8749        let mut reader = PutObjReader::from_vec(b"source body".to_vec());
8750        store.pools[0]
8751            .put_object(
8752                &bucket,
8753                object,
8754                &mut reader,
8755                &ObjectOptions {
8756                    versioned: true,
8757                    version_id: Some(version.to_string()),
8758                    mod_time: Some(mod_time),
8759                    ..Default::default()
8760                },
8761            )
8762            .await
8763            .expect("seed data movement source");
8764        *store.rebalance_meta.write().await = Some(active_rebalance_meta_for_pool(store.pools.len(), 0));
8765
8766        let source_reader = store.pools[0]
8767            .get_object_reader(
8768                &bucket,
8769                object,
8770                None,
8771                HeaderMap::new(),
8772                &ObjectOptions {
8773                    versioned: true,
8774                    version_id: Some(version.to_string()),
8775                    raw_data_movement_read: true,
8776                    ..Default::default()
8777                },
8778            )
8779            .await
8780            .expect("read data movement source");
8781        let source_data_dir = source_reader.object_info.data_dir;
8782        crate::data_movement::migrate_object(store.clone(), 0, bucket.clone(), source_reader, None, "test_read_window")
8783            .await
8784            .expect("commit data movement target");
8785
8786        let target = store.pools[1]
8787            .get_object_info(
8788                &bucket,
8789                object,
8790                &ObjectOptions {
8791                    versioned: true,
8792                    version_id: Some(version.to_string()),
8793                    ..Default::default()
8794                },
8795            )
8796            .await
8797            .expect("read committed data movement target");
8798        assert_ne!(source_data_dir, target.data_dir, "the target must use its own data directory");
8799        assert_eq!(
8800            rustfs_utils::http::get_consistent_str(&target.user_defined, rustfs_utils::http::SUFFIX_DATA_MOVED),
8801            Some("true")
8802        );
8803
8804        let resolved = store
8805            .get_object_info(
8806                &bucket,
8807                object,
8808                &ObjectOptions {
8809                    versioned: true,
8810                    version_id: Some(version.to_string()),
8811                    ..Default::default()
8812                },
8813            )
8814            .await
8815            .expect("the equal-time migration window must remain readable");
8816        assert_eq!(resolved.data_dir, target.data_dir);
8817
8818        shutdown.cancel();
8819    }
8820
8821    #[tokio::test]
8822    #[serial_test::serial(storage_class_env)]
8823    async fn data_movement_multipart_conflict_validates_exact_version_target_pool() {
8824        let temp_dir = tempfile::tempdir().expect("create three-pool multipart data movement store dir");
8825        let (_ctx, store, _shutdown) =
8826            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "data-movement-multipart-target", &[4, 4, 4]))
8827                .await;
8828        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
8829
8830        let bucket = format!("dm-multipart-target-{}", uuid::Uuid::new_v4());
8831        let object = "object.bin";
8832        let source_version = uuid::Uuid::new_v4();
8833        let other_version = uuid::Uuid::new_v4();
8834        let source_mod_time = OffsetDateTime::UNIX_EPOCH + time::Duration::SECOND;
8835        store
8836            .make_bucket(&bucket, &MakeBucketOptions::default())
8837            .await
8838            .expect("create multipart selected-target bucket");
8839        let first_part_size = 5 * 1024 * 1024;
8840        let mut source_body = vec![b'a'; first_part_size];
8841        source_body.push(b'b');
8842        for (pool_idx, version_id, body, mod_time) in [
8843            (
8844                1,
8845                other_version,
8846                b"newer different version".to_vec(),
8847                source_mod_time + time::Duration::seconds(2),
8848            ),
8849            (2, source_version, b"conflicting exact version".to_vec(), source_mod_time),
8850        ] {
8851            let mut reader = PutObjReader::from_vec(body);
8852            store.pools[pool_idx]
8853                .put_object(
8854                    &bucket,
8855                    object,
8856                    &mut reader,
8857                    &ObjectOptions {
8858                        versioned: true,
8859                        version_id: Some(version_id.to_string()),
8860                        mod_time: Some(mod_time),
8861                        ..Default::default()
8862                    },
8863                )
8864                .await
8865                .expect("seed multipart data movement pool");
8866        }
8867        let source_reader = seed_data_movement_source_reader(
8868            &store,
8869            &bucket,
8870            object,
8871            DataMovementSourceSeed {
8872                pool_idx: 0,
8873                version_id: Some(source_version),
8874                body: source_body,
8875                multipart_split: Some(first_part_size),
8876                mod_time: source_mod_time,
8877                user_defined: HashMap::new(),
8878            },
8879        )
8880        .await;
8881        *store.rebalance_meta.write().await = Some(active_rebalance_meta_for_pool(store.pools.len(), 0));
8882
8883        let completion_barrier = crate::store::multipart::DataMovementMultipartCompletionBarrier::install(&bucket);
8884        let migration_store = store.clone();
8885        let migration_bucket = bucket.clone();
8886        let migration = tokio::spawn(async move {
8887            crate::data_movement::migrate_object(
8888                migration_store,
8889                0,
8890                migration_bucket,
8891                source_reader,
8892                None,
8893                "test_multipart_selected_target",
8894            )
8895            .await
8896        });
8897        completion_barrier.wait_until_paused().await;
8898        let unrelated_set = store.pools[1].get_disks_by_key(object);
8899        let original_disks = {
8900            let mut disks = unrelated_set.disks.write().await;
8901            let original = disks.clone();
8902            for disk in disks.iter_mut().take(3) {
8903                *disk = None;
8904            }
8905            original
8906        };
8907        drop(completion_barrier);
8908        let err = migration
8909            .await
8910            .expect("multipart migration task should join")
8911            .expect_err("the exact-version target conflict must not be bypassed by a newer different version");
8912        *unrelated_set.disks.write().await = original_disks;
8913        let rendered = err.to_string();
8914        assert!(
8915            rendered.contains("complete_multipart_upload"),
8916            "the selected target should be reached despite an unrelated degraded pool: {rendered}"
8917        );
8918        assert!(
8919            !rendered.contains("put_object_part failed"),
8920            "part upload must not scan the unrelated degraded pool: {rendered}"
8921        );
8922
8923        let unexpected_version = store.pools[1]
8924            .get_object_info(
8925                &bucket,
8926                object,
8927                &ObjectOptions {
8928                    versioned: true,
8929                    version_id: Some(source_version.to_string()),
8930                    ..Default::default()
8931                },
8932            )
8933            .await
8934            .expect_err("multipart migration must not duplicate the source version into the latest-version pool");
8935        assert!(matches!(
8936            unexpected_version,
8937            StorageError::ObjectNotFound(_, _) | StorageError::VersionNotFound(_, _, _)
8938        ));
8939    }
8940
8941    #[cfg(feature = "test-util")]
8942    #[tokio::test]
8943    #[serial_test::serial(storage_class_env)]
8944    async fn decommission_durable_ilm_target_read_error_is_not_masked_by_peer_success() {
8945        let temp_dir = tempfile::tempdir().expect("create temp store dir");
8946        let (_ctx, store, _shutdown) =
8947            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "durable-ilm-target-read-error", &[4, 4, 4]))
8948                .await;
8949        let job_id = uuid::Uuid::new_v4();
8950        let job =
8951            ManualTransitionJobRecord::new(job_id, "manual-target-read-error", &ManualTransitionRunOptions::default(), "owner");
8952        let path = manual_transition_job_record_object_name(job_id).expect("manual job path should build");
8953        let data = job.encode().expect("manual job should encode");
8954        for pool in &store.pools {
8955            com::save_config(pool.clone(), &path, data.clone())
8956                .await
8957                .expect("manual job fixture should persist in every pool");
8958        }
8959        store.pool_meta.write().await.pools[0].decommission = Some(PoolDecommissionInfo {
8960            start_time: Some(OffsetDateTime::now_utc()),
8961            ..Default::default()
8962        });
8963
8964        let failing_target = store.pools[2].get_disks_by_key(&path);
8965        let original_disks = {
8966            let mut disks = failing_target.disks.write().await;
8967            let original = disks.clone();
8968            for disk in disks.iter_mut().take(3) {
8969                *disk = None;
8970            }
8971            original
8972        };
8973        let error = store
8974            .verify_and_cleanup_decommissioned_durable_ilm_record_for_test(0, store.pools[0].get_disks_by_key(&path), &path)
8975            .await
8976            .expect_err("one target read-quorum error must fail closed despite another target success")
8977            .to_string();
8978        *failing_target.disks.write().await = original_disks;
8979
8980        assert!(error.contains(&path));
8981        assert!(error.contains("pool 2"));
8982        assert_eq!(
8983            com::read_config(store.pools[0].clone(), &path)
8984                .await
8985                .expect("target read error must retain the source"),
8986            data
8987        );
8988        assert_eq!(
8989            store
8990                .decommission_durable_ilm_receipt_count_for_test(0)
8991                .await
8992                .expect("failed target verification should not create a receipt"),
8993            0
8994        );
8995    }
8996
8997    #[cfg(feature = "test-util")]
8998    #[tokio::test]
8999    #[serial_test::serial(storage_class_env)]
9000    async fn decommission_durable_ilm_terminal_receipt_recovers_failed_source_cleanup() {
9001        let temp_dir = tempfile::tempdir().expect("create temp store dir");
9002        let (ctx, store, _shutdown) =
9003            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "durable-ilm-terminal-receipt", &[4, 4])).await;
9004        let tier_name = "DECOMMISSION-RECEIPT";
9005        let backend = register_transition_reconcile_test_tier(&ctx.tier_config_mgr(), tier_name).await;
9006        let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
9007            .await
9008            .expect("tier lease should resolve")
9009            .backend_identity();
9010        let entry = Jentry {
9011            persisted_version: 0,
9012            obj_name: "receipt-recovery-object".to_string(),
9013            version_id: "receipt-recovery-version".to_string(),
9014            tier_name: tier_name.to_string(),
9015            backend_identity: Some(backend_identity),
9016            version_id_exact: true,
9017            version_state: rustfs_filemeta::TransitionVersionState::Exact,
9018            state: TierDeleteJournalState::Committed,
9019            source: None,
9020            dispatch: None,
9021        };
9022        let path = tier_delete_journal_object_name(&entry);
9023        let data = encode_tier_delete_journal_entry(&entry).expect("tier journal should encode");
9024        com::save_config(store.pools[0].clone(), &path, data.clone())
9025            .await
9026            .expect("source tier journal should persist");
9027        com::save_config(store.pools[1].clone(), &path, data.clone())
9028            .await
9029            .expect("target tier journal should persist");
9030        let active_pool_meta = {
9031            let mut pool_meta = store.pool_meta.write().await;
9032            pool_meta.pools[0].decommission = Some(PoolDecommissionInfo {
9033                start_time: Some(OffsetDateTime::now_utc()),
9034                ..Default::default()
9035            });
9036            pool_meta.clone()
9037        };
9038        active_pool_meta
9039            .save(store.pools.clone())
9040            .await
9041            .expect("active decommission run identity should persist");
9042
9043        let source_set = store.pools[0].get_disks_by_key(&path);
9044        let barrier = SourceCleanupDeleteBarrier::install(RUSTFS_META_BUCKET, &path);
9045        let cleanup_store = store.clone();
9046        let cleanup_set = source_set.clone();
9047        let cleanup_path = path.clone();
9048        let cleanup = tokio::spawn(async move {
9049            cleanup_store
9050                .verify_and_cleanup_decommissioned_durable_ilm_record_for_test(0, cleanup_set, &cleanup_path)
9051                .await
9052        });
9053        barrier.wait_until_paused().await;
9054        let original_source_disks = {
9055            let mut disks = source_set.disks.write().await;
9056            let original = disks.clone();
9057            for disk in disks.iter_mut().take(3) {
9058                *disk = None;
9059            }
9060            original
9061        };
9062        barrier.release();
9063        let cleanup_error = cleanup
9064            .await
9065            .expect("source cleanup task should not panic")
9066            .expect_err("injected source delete quorum failure must fail cleanup")
9067            .to_string();
9068        *source_set.disks.write().await = original_source_disks;
9069        drop(barrier);
9070
9071        assert!(cleanup_error.contains("source durable ILM cleanup failed"));
9072        assert_eq!(
9073            store
9074                .decommission_durable_ilm_receipt_count_for_test(0)
9075                .await
9076                .expect("receipt should persist before source cleanup"),
9077            1
9078        );
9079        assert_eq!(
9080            com::read_config(store.pools[0].clone(), &path)
9081                .await
9082                .expect("failed cleanup must retain the source"),
9083            data
9084        );
9085
9086        let mut restarted_pool_meta = PoolMeta::default();
9087        restarted_pool_meta
9088            .load(store.pools[0].clone(), store.pools.clone())
9089            .await
9090            .expect("decommission run identity should reload after restart");
9091        *store.pool_meta.write().await = restarted_pool_meta;
9092        let stats = recover_tier_delete_journal_entries(store.clone(), 100, None)
9093            .await
9094            .expect("target recovery should commit terminal proof and delete the target");
9095        assert_eq!((stats.scanned, stats.deleted, stats.failed), (1, 1, 0));
9096        assert!(matches!(
9097            com::read_config(store.pools[1].clone(), &path).await,
9098            Err(Error::ConfigNotFound)
9099        ));
9100        assert_eq!(
9101            com::read_config(store.pools[0].clone(), &path)
9102                .await
9103                .expect("target recovery must not delete the decommission source"),
9104            data
9105        );
9106
9107        store
9108            .verify_and_cleanup_decommissioned_durable_ilm_record_for_test(0, source_set, &path)
9109            .await
9110            .expect("terminal receipt should authorize cleanup after target deletion");
9111        assert!(matches!(
9112            com::read_config(store.pools[0].clone(), &path).await,
9113            Err(Error::ConfigNotFound)
9114        ));
9115        assert!(backend.remove_versions().await.contains(&(entry.obj_name, entry.version_id)));
9116    }
9117
9118    #[cfg(feature = "test-util")]
9119    #[tokio::test]
9120    #[serial_test::serial(storage_class_env)]
9121    async fn decommission_final_sweep_blocks_cancel_until_source_cleanup_finishes() {
9122        let temp_dir = tempfile::tempdir().expect("create final sweep gate store dir");
9123        let (_ctx, store, _shutdown) =
9124            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "durable-ilm-final-sweep-gate", &[4, 4])).await;
9125        let job_id = uuid::Uuid::new_v4();
9126        let job = ManualTransitionJobRecord::new(job_id, "final-sweep-gate", &ManualTransitionRunOptions::default(), "owner");
9127        let path = manual_transition_job_record_object_name(job_id).expect("manual job path should build");
9128        let data = job.encode().expect("manual job should encode");
9129        for pool in &store.pools {
9130            com::save_config(pool.clone(), &path, data.clone())
9131                .await
9132                .expect("manual job fixture should persist in both pools");
9133        }
9134        let active_pool_meta = {
9135            let mut pool_meta = store.pool_meta.write().await;
9136            pool_meta.pools[0].decommission = Some(PoolDecommissionInfo {
9137                start_time: Some(OffsetDateTime::now_utc()),
9138                ..Default::default()
9139            });
9140            pool_meta.clone()
9141        };
9142        active_pool_meta
9143            .save(store.pools.clone())
9144            .await
9145            .expect("active decommission run identity should persist");
9146
9147        let barrier = SourceCleanupDeleteBarrier::install(RUSTFS_META_BUCKET, &path);
9148        let final_sweep = tokio::spawn({
9149            let store = store.clone();
9150            async move { store.check_after_decommission_for_test(0).await }
9151        });
9152        barrier.wait_until_paused().await;
9153
9154        let mut cancel = tokio::spawn({
9155            let store = store.clone();
9156            async move { store.decommission_cancel(0).await }
9157        });
9158        assert!(
9159            tokio::time::timeout(Duration::from_millis(100), &mut cancel).await.is_err(),
9160            "cancel must wait for the final sweep source cleanup"
9161        );
9162        {
9163            let pool_meta = store.pool_meta.read().await;
9164            let decommission = pool_meta.pools[0]
9165                .decommission
9166                .as_ref()
9167                .expect("decommission state should remain present");
9168            assert!(
9169                decommission.canceled,
9170                "cancel must publish its durable terminal state before waiting for the final sweep"
9171            );
9172            assert!(
9173                decommission.start_time.is_none(),
9174                "the durable cancel state must clear the active run identity before quiescence"
9175            );
9176        }
9177
9178        barrier.release();
9179        final_sweep
9180            .await
9181            .expect("final sweep task should not panic")
9182            .expect("final sweep should finish after the barrier releases");
9183        cancel
9184            .await
9185            .expect("cancel task should not panic")
9186            .expect("cancel should complete after the final sweep releases the operation gate");
9187        let pool_meta = store.pool_meta.read().await;
9188        let decommission = pool_meta.pools[0]
9189            .decommission
9190            .as_ref()
9191            .expect("decommission state should remain present");
9192        assert!(decommission.canceled);
9193        assert!(decommission.start_time.is_none());
9194    }
9195
9196    #[cfg(feature = "test-util")]
9197    #[tokio::test]
9198    #[serial_test::serial(storage_class_env)]
9199    async fn decommission_durable_ilm_recovery_keeps_multiple_active_sources() {
9200        let temp_dir = tempfile::tempdir().expect("create multi-source recovery store dir");
9201        let (ctx, store, _shutdown) = without_storage_class_env(build_isolated_test_store(
9202            temp_dir.path(),
9203            "durable-ilm-multi-source-recovery",
9204            &[4, 4, 4],
9205        ))
9206        .await;
9207        let tier_name = "DECOMMISSION-MULTI-SOURCE";
9208        let backend = register_transition_reconcile_test_tier(&ctx.tier_config_mgr(), tier_name).await;
9209        let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
9210            .await
9211            .expect("tier lease should resolve")
9212            .backend_identity();
9213        let entry = Jentry {
9214            persisted_version: 0,
9215            obj_name: "multi-source-recovery-object".to_string(),
9216            version_id: "multi-source-recovery-version".to_string(),
9217            tier_name: tier_name.to_string(),
9218            backend_identity: Some(backend_identity),
9219            version_id_exact: true,
9220            version_state: rustfs_filemeta::TransitionVersionState::Exact,
9221            state: TierDeleteJournalState::Committed,
9222            source: None,
9223            dispatch: None,
9224        };
9225        let path = tier_delete_journal_object_name(&entry);
9226        let data = encode_tier_delete_journal_entry(&entry).expect("tier journal should encode");
9227        for pool in &store.pools {
9228            com::save_config(pool.clone(), &path, data.clone())
9229                .await
9230                .expect("source and target tier journals should persist");
9231        }
9232
9233        let active_pool_meta = {
9234            let mut pool_meta = store.pool_meta.write().await;
9235            let start_time = OffsetDateTime::now_utc();
9236            for pool_idx in [0, 1] {
9237                pool_meta.pools[pool_idx].decommission = Some(PoolDecommissionInfo {
9238                    start_time: Some(start_time),
9239                    ..Default::default()
9240                });
9241            }
9242            pool_meta.clone()
9243        };
9244        active_pool_meta
9245            .save(store.pools.clone())
9246            .await
9247            .expect("multiple active decommission runs should persist");
9248        let mut restarted_pool_meta = PoolMeta::default();
9249        restarted_pool_meta
9250            .load(store.pools[0].clone(), store.pools.clone())
9251            .await
9252            .expect("multiple active decommission runs should reload");
9253        *store.pool_meta.write().await = restarted_pool_meta;
9254
9255        let record = validate_durable_ilm_record(&path, &data).expect("tier journal should validate");
9256        let source_zero_receipt = store
9257            .persist_decommission_durable_ilm_receipt_for_test(0, 1, &path, &record, false)
9258            .await
9259            .expect("source pool zero receipt should persist on the other active source");
9260        let source_one_receipt = store
9261            .persist_decommission_durable_ilm_receipt_for_test(1, 0, &path, &record, false)
9262            .await
9263            .expect("source pool one receipt should persist on the other active source");
9264        assert_ne!(
9265            source_zero_receipt, source_one_receipt,
9266            "active source runs must have distinct receipt paths"
9267        );
9268
9269        let stats = recover_tier_delete_journal_entries(store.clone(), 100, None)
9270            .await
9271            .expect("cross-source receipts should not remove active source journals");
9272        assert_eq!((stats.scanned, stats.deleted, stats.failed), (1, 1, 0));
9273        for pool in &store.pools {
9274            assert_eq!(
9275                com::read_config(pool.clone(), &path)
9276                    .await
9277                    .expect("cross-source receipts alone must retain every journal copy"),
9278                data
9279            );
9280        }
9281
9282        store
9283            .persist_decommission_durable_ilm_receipt_for_test(0, 2, &path, &record, false)
9284            .await
9285            .expect("source pool zero receipt should persist on the target");
9286        store
9287            .persist_decommission_durable_ilm_receipt_for_test(1, 2, &path, &record, false)
9288            .await
9289            .expect("source pool one receipt should persist on the target");
9290
9291        let stats = recover_tier_delete_journal_entries(store.clone(), 100, None)
9292            .await
9293            .expect("multi-source tier journal recovery should complete");
9294        assert_eq!((stats.scanned, stats.deleted, stats.failed), (1, 1, 0));
9295        assert_eq!(
9296            com::read_config(store.pools[0].clone(), &path)
9297                .await
9298                .expect("first active source must remain after target recovery"),
9299            data
9300        );
9301        assert_eq!(
9302            com::read_config(store.pools[1].clone(), &path)
9303                .await
9304                .expect("second active source must remain after target recovery"),
9305            data
9306        );
9307        assert!(matches!(
9308            com::read_config(store.pools[2].clone(), &path).await,
9309            Err(Error::ConfigNotFound)
9310        ));
9311        assert!(backend.remove_versions().await.contains(&(entry.obj_name, entry.version_id)));
9312    }
9313
9314    #[cfg(feature = "test-util")]
9315    #[tokio::test]
9316    #[serial_test::serial(storage_class_env)]
9317    async fn decommission_durable_ilm_receipt_pagination_fails_closed_on_second_page() {
9318        const RECEIPT_COUNT: usize = 1001;
9319
9320        let temp_dir = tempfile::tempdir().expect("create paginated receipt store dir");
9321        let (_ctx, store, _shutdown) =
9322            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "durable-ilm-receipt-pages", &[4, 4])).await;
9323        store.pool_meta.write().await.pools[0].decommission = Some(PoolDecommissionInfo {
9324            start_time: Some(OffsetDateTime::now_utc()),
9325            ..Default::default()
9326        });
9327
9328        futures::stream::iter(0..RECEIPT_COUNT)
9329            .map(|index| {
9330                let store = store.clone();
9331                async move {
9332                    let id = format!("{index:064x}");
9333                    let source_path = format!("ilm/tier-delete-journal/{id}.json");
9334                    let record = ValidatedDurableIlmRecord {
9335                        namespace: "tier-delete-journal",
9336                        id_kind: "operation_id",
9337                        id,
9338                        checkpoint: DurableIlmRecordCheckpoint::TierDeleteJournal {
9339                            content_sha256: format!("{:064x}", index + RECEIPT_COUNT),
9340                            identity_sha256: "f".repeat(64),
9341                            committed: false,
9342                            dispatch_identity_sha256: None,
9343                            state: None,
9344                        },
9345                    };
9346                    store
9347                        .persist_decommission_durable_ilm_receipt_for_test(0, 0, &source_path, &record, true)
9348                        .await?;
9349                    store
9350                        .persist_decommission_durable_ilm_receipt_for_test(0, 1, &source_path, &record, true)
9351                        .await?;
9352                    Ok::<(), Error>(())
9353                }
9354            })
9355            .buffer_unordered(32)
9356            .try_collect::<Vec<_>>()
9357            .await
9358            .expect("more than one receipt page should persist");
9359        store
9360            .persist_decommission_durable_ilm_manifest_for_test(0)
9361            .await
9362            .expect("paginated source receipts should produce a manifest");
9363
9364        let target_receipts = store
9365            .decommission_durable_ilm_receipt_paths_for_test(0)
9366            .await
9367            .expect("paginated target receipts should list");
9368        assert_eq!(target_receipts.len(), RECEIPT_COUNT);
9369        let (target_pool_idx, second_page_path) = target_receipts
9370            .get(1000)
9371            .cloned()
9372            .expect("the real 1000-item page boundary should expose a second page receipt");
9373        let receipt_bytes = com::read_config(store.pools[target_pool_idx].clone(), &second_page_path)
9374            .await
9375            .expect("second page receipt should be readable");
9376
9377        com::delete_config(store.pools[target_pool_idx].clone(), &second_page_path)
9378            .await
9379            .expect("second page receipt should delete");
9380        let missing = store
9381            .complete_decommission(0)
9382            .await
9383            .expect_err("a missing second page receipt must block completion")
9384            .to_string();
9385        assert!(missing.contains(&second_page_path));
9386        assert!(
9387            !store.pool_meta.read().await.pools[0]
9388                .decommission
9389                .as_ref()
9390                .expect("source pool should remain in decommission")
9391                .complete
9392        );
9393        assert!(com::read_config(store.pools[0].clone(), &second_page_path).await.is_ok());
9394
9395        let full_tail = ObjectOptions {
9396            max_parity: true,
9397            write_completion: crate::object_api::WriteCompletion::TailDrained,
9398            ..Default::default()
9399        };
9400        com::save_config_with_opts(store.pools[target_pool_idx].clone(), &second_page_path, receipt_bytes.clone(), &full_tail)
9401            .await
9402            .expect("second page receipt should restore");
9403        com::save_config_with_opts(store.pools[target_pool_idx].clone(), &second_page_path, b"{corrupt".to_vec(), &full_tail)
9404            .await
9405            .expect("second page receipt should corrupt deterministically");
9406        let corrupt = store
9407            .complete_decommission(0)
9408            .await
9409            .expect_err("a corrupt second page receipt must block completion")
9410            .to_string();
9411        assert!(corrupt.contains(&second_page_path));
9412        assert!(corrupt.contains("invalid"));
9413        assert!(
9414            !store.pool_meta.read().await.pools[0]
9415                .decommission
9416                .as_ref()
9417                .expect("source pool should remain in decommission")
9418                .complete
9419        );
9420        assert!(com::read_config(store.pools[0].clone(), &second_page_path).await.is_ok());
9421    }
9422
9423    #[cfg(feature = "test-util")]
9424    #[test]
9425    #[serial_test::serial(storage_class_env)]
9426    fn decommission_migrates_and_verifies_registered_durable_ilm_records() {
9427        std::thread::Builder::new()
9428            .name("durable-ilm-decommission-test".to_string())
9429            .stack_size(16 * 1024 * 1024)
9430            .spawn(|| {
9431                let runtime = tokio::runtime::Builder::new_multi_thread()
9432                    .enable_all()
9433                    .worker_threads(2)
9434                    .build()
9435                    .expect("durable ILM decommission runtime should build");
9436                runtime.block_on(decommission_migrates_and_verifies_registered_durable_ilm_records_scenario());
9437            })
9438            .expect("durable ILM decommission scenario thread should spawn")
9439            .join()
9440            .expect("durable ILM decommission scenario should not panic");
9441    }
9442
9443    #[cfg(feature = "test-util")]
9444    async fn decommission_migrates_and_verifies_registered_durable_ilm_records_scenario() {
9445        let temp_dir = tempfile::tempdir().expect("create temp store dir");
9446        let (ctx, store, _shutdown) =
9447            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "durable-ilm-decommission", &[4, 4])).await;
9448
9449        let tier_name = "DECOMMISSION-ILM";
9450        let backend = register_transition_reconcile_test_tier(&ctx.tier_config_mgr(), tier_name).await;
9451        let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
9452            .await
9453            .expect("tier lease should resolve")
9454            .backend_identity();
9455        let tier_entry = Jentry {
9456            persisted_version: 0,
9457            obj_name: "decommissioned-remote-object".to_string(),
9458            version_id: "decommissioned-remote-version".to_string(),
9459            tier_name: tier_name.to_string(),
9460            backend_identity: Some(backend_identity),
9461            version_id_exact: true,
9462            version_state: rustfs_filemeta::TransitionVersionState::Exact,
9463            state: TierDeleteJournalState::Committed,
9464            source: None,
9465            dispatch: None,
9466        };
9467        let tier_path = tier_delete_journal_object_name(&tier_entry);
9468        let tier_bytes = encode_tier_delete_journal_entry(&tier_entry).expect("tier journal should encode");
9469
9470        let mut transaction = TransitionTransaction::new(TransitionTransactionInit {
9471            deployment_id: ctx.deployment_id().expect("test store should initialize deployment id"),
9472            transaction_id: uuid::Uuid::new_v4(),
9473            owner_epoch: uuid::Uuid::new_v4(),
9474            write_id: uuid::Uuid::new_v4(),
9475            source: TransitionSourceIdentity {
9476                bucket: "source-bucket".to_string(),
9477                object: "source-object".to_string(),
9478                version_id: Some(uuid::Uuid::new_v4()),
9479                data_dir: uuid::Uuid::new_v4(),
9480                mod_time_unix_nanos: 1_770_000_000_000_000_000,
9481                size: 42,
9482                etag: "source-etag".to_string(),
9483                version_mode: TransitionSourceVersionMode::Versioned,
9484            },
9485            tier_name: tier_name.to_string(),
9486            backend_fingerprint: backend_identity,
9487            not_after_unix_nanos: 1_780_000_000_000_000_000,
9488        })
9489        .expect("transition transaction should build");
9490        let regressed_transaction = transaction.clone();
9491        transaction
9492            .advance(transaction.fence(), TransitionTransactionState::UploadOutcomeUnknown, None)
9493            .expect("transition transaction should advance before migration");
9494        let transaction_path = transition_transaction_record_object_name(transaction.transaction_id)
9495            .expect("transition transaction path should build");
9496        let transaction_bytes = transaction.encode().expect("transition transaction should encode");
9497
9498        let manual_job_id = uuid::Uuid::new_v4();
9499        let manual_bucket = format!("manual-decommission-{}", manual_job_id.simple());
9500        let manual_options = ManualTransitionRunOptions {
9501            prefix: "logs/".to_string(),
9502            tier: Some(tier_name.to_string()),
9503            ..Default::default()
9504        };
9505        let mut manual_job = ManualTransitionJobRecord::new(manual_job_id, &manual_bucket, &manual_options, "old-owner");
9506        manual_job.scan_completed = true;
9507        manual_job.report.enqueued = 1;
9508        manual_job.lease_expires_at_unix_nanos = 0;
9509        let manual_scope = ManualTransitionScopeAdmission::from_job(&manual_job);
9510        let task_key = manual_transition_worker_result_task_key(&manual_bucket, "logs/a", None);
9511        let manual_task = ManualTransitionTaskRecord::new(manual_job_id, &task_key, &manual_bucket, "logs/a", None, tier_name);
9512        let manual_result =
9513            ManualTransitionWorkerResultRecord::new(manual_job_id, &task_key, ManualTransitionWorkerResult::Completed);
9514
9515        let manual_job_path = manual_transition_job_record_object_name(manual_job_id).expect("manual job path should build");
9516        let manual_scope_path =
9517            manual_transition_scope_record_object_name(&manual_scope.scope_key).expect("manual scope path should build");
9518        let manual_task_path =
9519            manual_transition_task_object_name(manual_job_id, &task_key).expect("manual task path should build");
9520        let manual_result_path = manual_transition_worker_result_object_name(manual_job_id, &task_key)
9521            .expect("manual worker result path should build");
9522        let manual_job_bytes = manual_job.encode().expect("manual job should encode");
9523        let manual_scope_bytes = serde_json::to_vec(&manual_scope).expect("manual scope should encode");
9524        let manual_task_bytes = manual_task.encode().expect("manual task should encode");
9525        let manual_result_bytes = manual_result.encode().expect("manual result should encode");
9526
9527        let records = vec![
9528            (tier_path.clone(), tier_bytes.clone()),
9529            (transaction_path.clone(), transaction_bytes.clone()),
9530            (manual_job_path.clone(), manual_job_bytes.clone()),
9531            (manual_scope_path.clone(), manual_scope_bytes.clone()),
9532            (manual_task_path.clone(), manual_task_bytes.clone()),
9533            (manual_result_path.clone(), manual_result_bytes.clone()),
9534        ];
9535        for (path, data) in &records {
9536            com::save_config(store.pools[0].clone(), path, data.clone())
9537                .await
9538                .expect("durable ILM source record should persist");
9539        }
9540
9541        let legacy_queue = [com::CONFIG_PREFIX, BUCKET_META_PREFIX]
9542            .into_iter()
9543            .map(|prefix| {
9544                DecomBucketInfo {
9545                    name: RUSTFS_META_BUCKET.to_string(),
9546                    prefix: prefix.to_string(),
9547                }
9548                .to_string()
9549            })
9550            .collect();
9551        let legacy_pool_meta = {
9552            let mut pool_meta = store.pool_meta.write().await;
9553            pool_meta.pools[0].decommission = Some(PoolDecommissionInfo {
9554                queued: true,
9555                queued_buckets: legacy_queue,
9556                ..Default::default()
9557            });
9558            pool_meta.clone()
9559        };
9560        legacy_pool_meta
9561            .save(store.pools.clone())
9562            .await
9563            .expect("legacy decommission queue should persist before restart");
9564        let mut restarted_pool_meta = PoolMeta::default();
9565        restarted_pool_meta
9566            .load(store.pools[0].clone(), store.pools.clone())
9567            .await
9568            .expect("legacy decommission queue should reload after restart");
9569        *store.pool_meta.write().await = restarted_pool_meta;
9570        set_test_decommission_capacity_override(&store, 0);
9571        store
9572            .promote_queued_decommission_for_test(0)
9573            .await
9574            .expect("legacy queued decommission should resume");
9575        let expected_ilm_queue = DecomBucketInfo {
9576            name: RUSTFS_META_BUCKET.to_string(),
9577            prefix: ILM_META_PREFIX.to_string(),
9578        }
9579        .to_string();
9580        {
9581            let pool_meta = store.pool_meta.read().await;
9582            let decommission = pool_meta.pools[0]
9583                .decommission
9584                .as_ref()
9585                .expect("decommission state should remain present");
9586            assert!(!decommission.queued);
9587            assert!(decommission.queued_buckets.contains(&expected_ilm_queue));
9588        }
9589
9590        let ilm_bucket = DecomBucketInfo {
9591            name: RUSTFS_META_BUCKET.to_string(),
9592            prefix: ILM_META_PREFIX.to_string(),
9593        };
9594        for _ in 0..2 {
9595            store
9596                .decommission_pool_for_test(CancellationToken::new(), 0, store.pools[0].clone(), ilm_bucket.clone())
9597                .await
9598                .expect("durable ILM decommission should be idempotent");
9599        }
9600        for (path, expected) in &records {
9601            assert_eq!(
9602                com::read_config(store.pools[0].clone(), path)
9603                    .await
9604                    .expect("source should remain until the final sweep"),
9605                *expected
9606            );
9607            assert_eq!(
9608                com::read_config(store.pools[1].clone(), path)
9609                    .await
9610                    .expect("target should contain the migrated record"),
9611                *expected
9612            );
9613        }
9614        assert_eq!(
9615            store
9616                .decommission_durable_ilm_receipt_count_for_test(0)
9617                .await
9618                .expect("no receipt should exist before the final sweep"),
9619            0
9620        );
9621        let isolated_tier_stats = recover_tier_delete_journal_entries(store.clone(), 100, None)
9622            .await
9623            .expect("tier recovery should retain a terminal record until its receipt is committed");
9624        assert!(isolated_tier_stats.scanned >= 1);
9625        assert_eq!(isolated_tier_stats.deleted, 0);
9626        assert!(isolated_tier_stats.failed >= 1);
9627        assert_eq!(
9628            com::read_config(store.pools[1].clone(), &tier_path)
9629                .await
9630                .expect("receipt isolation must retain the target tier journal"),
9631            tier_bytes
9632        );
9633
9634        com::delete_config(store.pools[1].clone(), &manual_job_path)
9635            .await
9636            .expect("target manual job should delete");
9637        let missing = store
9638            .verify_and_cleanup_decommissioned_durable_ilm_record_for_test(
9639                0,
9640                store.pools[0].get_disks_by_key(&manual_job_path),
9641                &manual_job_path,
9642            )
9643            .await
9644            .expect_err("missing target must block source cleanup");
9645        let missing = missing.to_string();
9646        assert!(missing.contains(&manual_job_path) && missing.contains(&manual_job_id.to_string()));
9647        assert_eq!(
9648            com::read_config(store.pools[0].clone(), &manual_job_path)
9649                .await
9650                .expect("missing target must retain source"),
9651            manual_job_bytes
9652        );
9653        com::save_config(store.pools[1].clone(), &manual_job_path, manual_job_bytes.clone())
9654            .await
9655            .expect("target manual job should restore");
9656
9657        com::save_config(store.pools[1].clone(), &transaction_path, b"{corrupt".to_vec())
9658            .await
9659            .expect("target transaction should corrupt deterministically");
9660        let corrupt = store
9661            .verify_and_cleanup_decommissioned_durable_ilm_record_for_test(
9662                0,
9663                store.pools[0].get_disks_by_key(&transaction_path),
9664                &transaction_path,
9665            )
9666            .await
9667            .expect_err("corrupt target must block source cleanup");
9668        let corrupt = corrupt.to_string();
9669        assert!(corrupt.contains(&transaction_path) && corrupt.contains(&transaction.transaction_id.to_string()));
9670        assert_eq!(
9671            com::read_config(store.pools[0].clone(), &transaction_path)
9672                .await
9673                .expect("corrupt target must retain source"),
9674            transaction_bytes
9675        );
9676        com::save_config(store.pools[1].clone(), &transaction_path, transaction_bytes.clone())
9677            .await
9678            .expect("target transaction should restore");
9679
9680        com::save_config(store.pools[1].clone(), &manual_scope_path, manual_scope_bytes.clone())
9681            .await
9682            .expect("target scope rewrite should invalidate cached metadata before the quorum check");
9683        let target_scope_set = store.pools[1].get_disks_by_key(&manual_scope_path);
9684        let original_target_scope_disks = {
9685            let mut disks = target_scope_set.disks.write().await;
9686            let original = disks.clone();
9687            for disk in disks.iter_mut().take(3) {
9688                *disk = None;
9689            }
9690            original
9691        };
9692        let quorum_error = store
9693            .verify_and_cleanup_decommissioned_durable_ilm_record_for_test(
9694                0,
9695                store.pools[0].get_disks_by_key(&manual_scope_path),
9696                &manual_scope_path,
9697            )
9698            .await
9699            .expect_err("target below read quorum must block source cleanup");
9700        *target_scope_set.disks.write().await = original_target_scope_disks;
9701        let quorum_error = quorum_error.to_string();
9702        assert!(quorum_error.contains(&manual_scope_path) && quorum_error.contains(&manual_job_id.to_string()));
9703        assert!(com::read_config(store.pools[0].clone(), &manual_scope_path).await.is_ok());
9704
9705        com::save_config(store.pools[1].clone(), &manual_task_path, manual_task_bytes.clone())
9706            .await
9707            .expect("target task rewrite should invalidate cached metadata before the quorum check");
9708        let manual_task_record =
9709            validate_durable_ilm_record(&manual_task_path, &manual_task_bytes).expect("manual task should validate");
9710        let manual_task_receipt_path = store
9711            .decommission_durable_ilm_receipt_path_for_test(0, &manual_task_path, &manual_task_record)
9712            .await
9713            .expect("manual task receipt path should resolve");
9714        let target_task_set = store.pools[1].get_disks_by_key(&manual_task_receipt_path);
9715        let original_target_task_disks = {
9716            let mut disks = target_task_set.disks.write().await;
9717            let original = disks.clone();
9718            for disk in disks.iter_mut().take(2) {
9719                *disk = None;
9720            }
9721            original
9722        };
9723        let receipt_quorum_error = store
9724            .verify_and_cleanup_decommissioned_durable_ilm_record_for_test(
9725                0,
9726                store.pools[0].get_disks_by_key(&manual_task_path),
9727                &manual_task_path,
9728            )
9729            .await
9730            .expect_err("target read quorum without receipt write quorum must retain the source");
9731        *target_task_set.disks.write().await = original_target_task_disks;
9732        let receipt_quorum_error = receipt_quorum_error.to_string();
9733        assert!(receipt_quorum_error.contains("receipt"));
9734        assert!(receipt_quorum_error.contains(&manual_task_path));
9735        assert!(receipt_quorum_error.contains(&manual_job_id.to_string()));
9736        assert!(com::read_config(store.pools[0].clone(), &manual_task_path).await.is_ok());
9737        store
9738            .verify_and_cleanup_decommissioned_durable_ilm_record_for_test(
9739                0,
9740                store.pools[0].get_disks_by_key(&manual_task_path),
9741                &manual_task_path,
9742            )
9743            .await
9744            .expect("healthy target should persist the receipt before source cleanup");
9745
9746        let unknown_path = "ilm/future-durable/jobs/one.json";
9747        com::save_config(store.pools[0].clone(), unknown_path, b"{}".to_vec())
9748            .await
9749            .expect("unknown durable ILM record should persist for the guard test");
9750        let unknown_migration = store
9751            .decommission_pool_for_test(CancellationToken::new(), 0, store.pools[0].clone(), ilm_bucket)
9752            .await
9753            .expect_err("unregistered durable ILM namespace must block migration");
9754        assert!(unknown_migration.to_string().contains(unknown_path));
9755        let unknown_final_sweep = store
9756            .check_after_decommission_for_test(0)
9757            .await
9758            .expect_err("unregistered durable ILM namespace must block completion");
9759        assert!(unknown_final_sweep.to_string().contains(unknown_path));
9760        com::delete_config(store.pools[0].clone(), unknown_path)
9761            .await
9762            .expect("unknown guard fixture should be removed before the successful final sweep");
9763
9764        store
9765            .check_after_decommission_for_test(0)
9766            .await
9767            .expect("production final sweep should validate every target before cleanup");
9768        assert_eq!(
9769            store
9770                .decommission_durable_ilm_receipt_count_for_test(0)
9771                .await
9772                .expect("durable ILM receipts should be listable"),
9773            records.len(),
9774            "every cleaned source record must have a durable validation receipt"
9775        );
9776        for (path, expected) in &records {
9777            assert!(
9778                matches!(com::read_config(store.pools[0].clone(), path).await, Err(Error::ConfigNotFound)),
9779                "final sweep should remove the validated source `{path}`"
9780            );
9781            assert_eq!(
9782                com::read_config(store.pools[1].clone(), path)
9783                    .await
9784                    .expect("final sweep must preserve the target"),
9785                *expected
9786            );
9787        }
9788
9789        let mut crash_restarted_pool_meta = PoolMeta::default();
9790        crash_restarted_pool_meta
9791            .load(store.pools[0].clone(), store.pools.clone())
9792            .await
9793            .expect("pool metadata should reload after the simulated pre-complete crash");
9794        *store.pool_meta.write().await = crash_restarted_pool_meta;
9795
9796        let (manual_job_receipt_pool, manual_job_receipt_path) = store
9797            .decommission_durable_ilm_receipt_paths_for_test(0)
9798            .await
9799            .expect("durable ILM receipt paths should be listable")
9800            .into_iter()
9801            .find(|(_, path)| path.contains(&manual_job_path))
9802            .expect("manual job should have one target receipt");
9803        let manual_job_receipt_bytes = com::read_config(store.pools[manual_job_receipt_pool].clone(), &manual_job_receipt_path)
9804            .await
9805            .expect("manual job receipt should be readable before deletion");
9806        com::delete_config(store.pools[manual_job_receipt_pool].clone(), &manual_job_receipt_path)
9807            .await
9808            .expect("manual job receipt should delete after source cleanup");
9809        com::delete_config(store.pools[1].clone(), &manual_job_path)
9810            .await
9811            .expect("post-crash target manual job should delete");
9812        let missing_after_crash = store
9813            .complete_decommission(0)
9814            .await
9815            .expect_err("completion must reject a missing target after source cleanup and restart")
9816            .to_string();
9817        assert!(missing_after_crash.contains("receipt"));
9818        assert!(missing_after_crash.contains(&manual_job_path));
9819        assert!(missing_after_crash.contains(&manual_job_id.to_string()));
9820        assert!(
9821            !store.pool_meta.read().await.pools[0]
9822                .decommission
9823                .as_ref()
9824                .expect("decommission state should survive restart")
9825                .complete
9826        );
9827        com::save_config(store.pools[1].clone(), &manual_job_path, manual_job_bytes.clone())
9828            .await
9829            .expect("post-crash target manual job should restore");
9830        com::save_config(
9831            store.pools[manual_job_receipt_pool].clone(),
9832            &manual_job_receipt_path,
9833            manual_job_receipt_bytes,
9834        )
9835        .await
9836        .expect("manual job receipt should restore after the missing-receipt check");
9837
9838        com::save_config(store.pools[1].clone(), &transaction_path, b"{corrupt".to_vec())
9839            .await
9840            .expect("post-crash target transaction should corrupt deterministically");
9841        let corrupt_after_crash = store
9842            .complete_decommission(0)
9843            .await
9844            .expect_err("completion must reject a corrupt target after source cleanup and restart")
9845            .to_string();
9846        assert!(corrupt_after_crash.contains(&transaction_path));
9847        assert!(corrupt_after_crash.contains(&transaction.transaction_id.to_string()));
9848        com::save_config(store.pools[1].clone(), &transaction_path, transaction_bytes.clone())
9849            .await
9850            .expect("post-crash target transaction should restore");
9851
9852        let mut wrong_manual_job = manual_job.clone();
9853        wrong_manual_job.job_id = uuid::Uuid::new_v4();
9854        com::save_config(
9855            store.pools[1].clone(),
9856            &manual_job_path,
9857            wrong_manual_job.encode().expect("wrong-id job should encode"),
9858        )
9859        .await
9860        .expect("post-crash target manual job should accept the wrong-id fixture");
9861        let wrong_id_after_crash = store
9862            .complete_decommission(0)
9863            .await
9864            .expect_err("completion must reject a target record with the wrong id")
9865            .to_string();
9866        assert!(wrong_id_after_crash.contains(&manual_job_path));
9867        assert!(wrong_id_after_crash.contains(&manual_job_id.to_string()));
9868        com::save_config(store.pools[1].clone(), &manual_job_path, manual_job_bytes.clone())
9869            .await
9870            .expect("post-crash target manual job should restore after the wrong-id check");
9871
9872        com::save_config(
9873            store.pools[1].clone(),
9874            &transaction_path,
9875            regressed_transaction
9876                .encode()
9877                .expect("regressed transition transaction should encode"),
9878        )
9879        .await
9880        .expect("post-crash target transaction should accept the regression fixture");
9881        let regression_after_crash = store
9882            .complete_decommission(0)
9883            .await
9884            .expect_err("completion must reject a lower transition transaction revision")
9885            .to_string();
9886        assert!(regression_after_crash.contains("generation mismatch"));
9887        assert!(regression_after_crash.contains(&transaction_path));
9888        assert!(regression_after_crash.contains(&transaction.transaction_id.to_string()));
9889        com::save_config(store.pools[1].clone(), &transaction_path, transaction_bytes.clone())
9890            .await
9891            .expect("post-crash target transaction should restore after the regression check");
9892
9893        let (manual_task_receipt_pool, manual_task_receipt_path) = store
9894            .decommission_durable_ilm_receipt_paths_for_test(0)
9895            .await
9896            .expect("durable ILM receipt paths should be listable")
9897            .into_iter()
9898            .find(|(_, path)| path.contains(&manual_task_path))
9899            .expect("manual task receipt should retain its reversible source path");
9900        let manual_task_receipt_bytes =
9901            com::read_config(store.pools[manual_task_receipt_pool].clone(), &manual_task_receipt_path)
9902                .await
9903                .expect("manual task receipt should be readable before corruption");
9904        com::save_config(
9905            store.pools[manual_task_receipt_pool].clone(),
9906            &manual_task_receipt_path,
9907            b"{corrupt".to_vec(),
9908        )
9909        .await
9910        .expect("manual task receipt should corrupt deterministically");
9911        let corrupt_receipt = store
9912            .complete_decommission(0)
9913            .await
9914            .expect_err("completion must fail closed on a corrupt receipt")
9915            .to_string();
9916        assert!(corrupt_receipt.contains(&manual_task_path));
9917        assert!(corrupt_receipt.contains(&manual_job_id.to_string()));
9918        com::save_config(
9919            store.pools[manual_task_receipt_pool].clone(),
9920            &manual_task_receipt_path,
9921            manual_task_receipt_bytes,
9922        )
9923        .await
9924        .expect("manual task receipt should restore after the corruption check");
9925
9926        let tier_stats = recover_tier_delete_journal_entries(store.clone(), 100, None)
9927            .await
9928            .expect("tier journal recovery should consume the migrated record before completion");
9929        assert_eq!((tier_stats.scanned, tier_stats.deleted, tier_stats.failed), (1, 1, 0));
9930        assert!(matches!(com::read_config(store.clone(), &tier_path).await, Err(Error::ConfigNotFound)));
9931
9932        let recovered_transition_version = "recovered-transition-version".to_string();
9933        backend
9934            .set_transition_candidate_probe_override(Some(TransitionCandidateProbe::VersionedPresent(
9935                recovered_transition_version.clone(),
9936            )))
9937            .await;
9938        let transaction_stats = recover_transition_transaction_records(store.clone(), 100, None)
9939            .await
9940            .expect("transition recovery should advance and consume the migrated transaction before completion");
9941        backend.set_transition_candidate_probe_override(None).await;
9942        assert_eq!(
9943            (
9944                transaction_stats.scanned,
9945                transaction_stats.recovered,
9946                transaction_stats.retained,
9947                transaction_stats.failed,
9948            ),
9949            (1, 1, 0, 0)
9950        );
9951        assert!(matches!(
9952            com::read_config(store.clone(), &transaction_path).await,
9953            Err(Error::ConfigNotFound)
9954        ));
9955        com::save_config(
9956            store.pools[1].clone(),
9957            &transaction_path,
9958            regressed_transaction
9959                .encode()
9960                .expect("post-terminal transition rollback should encode"),
9961        )
9962        .await
9963        .expect("target should accept the post-terminal rollback fixture");
9964        let post_terminal_regression = store
9965            .complete_decommission(0)
9966            .await
9967            .expect_err("terminal proof must not mask a lower transition revision")
9968            .to_string();
9969        assert!(post_terminal_regression.contains("generation mismatch"));
9970        assert!(post_terminal_regression.contains(&transaction_path));
9971        assert!(post_terminal_regression.contains(&transaction.transaction_id.to_string()));
9972        com::delete_config(store.pools[1].clone(), &transaction_path)
9973            .await
9974            .expect("post-terminal rollback fixture should be removed");
9975
9976        let manual_stats = recover_manual_transition_jobs_once(store.clone(), 100, None)
9977            .await
9978            .expect("manual recovery should advance the migrated job and consume its scope before completion");
9979        assert_eq!(
9980            (manual_stats.scanned, manual_stats.resumed, manual_stats.skipped, manual_stats.failed,),
9981            (1, 1, 0, 0)
9982        );
9983        assert!(matches!(
9984            com::read_config(store.clone(), &manual_scope_path).await,
9985            Err(Error::ConfigNotFound)
9986        ));
9987        let recovered_manual_job_bytes = com::read_config(store.pools[1].clone(), &manual_job_path)
9988            .await
9989            .expect("manual recovery should retain the advanced job record");
9990        assert_ne!(recovered_manual_job_bytes, manual_job_bytes);
9991
9992        com::save_config(store.pools[1].clone(), &manual_job_path, manual_job_bytes.clone())
9993            .await
9994            .expect("target manual job should accept the rollback fixture");
9995        let manual_regression = store
9996            .complete_decommission(0)
9997            .await
9998            .expect_err("completion must reject a manual job generation rollback")
9999            .to_string();
10000        assert!(manual_regression.contains("generation mismatch"));
10001        assert!(manual_regression.contains(&manual_job_path));
10002        assert!(manual_regression.contains(&manual_job_id.to_string()));
10003        com::save_config(store.pools[1].clone(), &manual_job_path, recovered_manual_job_bytes)
10004            .await
10005            .expect("target manual job should restore its recovered generation");
10006
10007        store
10008            .complete_decommission(0)
10009            .await
10010            .expect("completion should persist before receipt cleanup");
10011        assert!(
10012            store.pool_meta.read().await.pools[0]
10013                .decommission
10014                .as_ref()
10015                .expect("completed decommission state should remain present")
10016                .complete
10017        );
10018        assert_eq!(
10019            store
10020                .decommission_durable_ilm_receipt_count_for_test(0)
10021                .await
10022                .expect("receipt cleanup should be observable"),
10023            0
10024        );
10025        store
10026            .cleanup_decommission_durable_ilm_receipts_for_test(0)
10027            .await
10028            .expect("receipt cleanup should be idempotent");
10029        let removed_versions = backend.remove_versions().await;
10030        assert!(removed_versions.contains(&(tier_entry.obj_name.clone(), tier_entry.version_id.clone())));
10031        assert!(removed_versions.contains(&(transaction.remote_object.clone(), recovered_transition_version)));
10032    }
10033
10034    #[cfg(feature = "test-util")]
10035    async fn tier_delete_journal_count(store: Arc<crate::store::ECStore>) -> usize {
10036        store
10037            .list_objects_v2(RUSTFS_META_BUCKET, TIER_DELETE_JOURNAL_PREFIX, None, None, 100, false, None, false)
10038            .await
10039            .expect("tier delete journal should be listable")
10040            .objects
10041            .len()
10042    }
10043
10044    #[cfg(feature = "test-util")]
10045    async fn tier_delete_dispatch_manifest_count(store: Arc<crate::store::ECStore>) -> usize {
10046        store
10047            .list_objects_v2(
10048                RUSTFS_META_BUCKET,
10049                TIER_DELETE_DISPATCH_MANIFEST_PREFIX,
10050                None,
10051                None,
10052                100,
10053                false,
10054                None,
10055                false,
10056            )
10057            .await
10058            .expect("tier delete dispatch manifests should be listable")
10059            .objects
10060            .len()
10061    }
10062
10063    #[cfg(feature = "test-util")]
10064    async fn install_committed_tier_delete_journals(
10065        store: Arc<crate::store::ECStore>,
10066        tier_name: &str,
10067        backend_identity: [u8; 32],
10068        count: usize,
10069    ) -> Vec<Jentry> {
10070        let mut entries = Vec::with_capacity(count);
10071        for index in 0..count {
10072            let entry = Jentry {
10073                persisted_version: 0,
10074                obj_name: format!("remote/pass-drain-{index:06}.bin"),
10075                version_id: uuid::Uuid::new_v4().to_string(),
10076                tier_name: tier_name.to_string(),
10077                backend_identity: Some(backend_identity),
10078                version_id_exact: true,
10079                version_state: rustfs_filemeta::TransitionVersionState::Exact,
10080                state: TierDeleteJournalState::Committed,
10081                source: None,
10082                dispatch: None,
10083            };
10084            persist_tier_delete_journal_entry(store.clone(), &entry)
10085                .await
10086                .expect("committed tier-delete journal fixture should persist");
10087            entries.push(entry);
10088        }
10089        entries
10090    }
10091
10092    #[cfg(feature = "test-util")]
10093    fn synthetic_v6_dispatch_entry(
10094        bucket: &str,
10095        object: &str,
10096        tier_name: &str,
10097        backend_identity: [u8; 32],
10098        remote_version: &str,
10099    ) -> Jentry {
10100        Jentry {
10101            persisted_version: 0,
10102            obj_name: format!("remote/{object}"),
10103            version_id: remote_version.to_string(),
10104            tier_name: tier_name.to_string(),
10105            backend_identity: Some(backend_identity),
10106            version_id_exact: true,
10107            version_state: rustfs_filemeta::TransitionVersionState::Exact,
10108            state: TierDeleteJournalState::Prepared,
10109            source: Some(TierDeleteSourceIdentity {
10110                bucket: bucket.to_string(),
10111                object: object.to_string(),
10112                version_id: Some(uuid::Uuid::new_v4().to_string()),
10113                versioned: true,
10114                version_suspended: false,
10115                data_dir: Some(uuid::Uuid::new_v4().to_string()),
10116                etag: Some(format!("etag-{object}")),
10117                mod_time: Some(OffsetDateTime::UNIX_EPOCH.to_string()),
10118            }),
10119            dispatch: None,
10120        }
10121    }
10122
10123    #[cfg(feature = "test-util")]
10124    async fn install_aborting_dispatch_fixture(
10125        store: Arc<crate::store::ECStore>,
10126        bucket: &str,
10127        incarnation: uuid::Uuid,
10128        prefix: &str,
10129        tier_name: &str,
10130        backend_identity: [u8; 32],
10131        journal_count: usize,
10132    ) -> (String, Vec<Jentry>) {
10133        let entries = (0..journal_count)
10134            .map(|index| {
10135                (
10136                    synthetic_v6_dispatch_entry(
10137                        bucket,
10138                        &format!("{prefix}{index:06}.bin"),
10139                        tier_name,
10140                        backend_identity,
10141                        &uuid::Uuid::new_v4().to_string(),
10142                    ),
10143                    Some(TierDeleteJournalState::Prepared),
10144                )
10145            })
10146            .collect();
10147        install_test_tier_delete_dispatch_fixture(
10148            store,
10149            bucket,
10150            incarnation,
10151            prefix,
10152            entries,
10153            TierDeleteDispatchManifestState::Aborting,
10154        )
10155        .await
10156        .expect("Aborting rollback fixture should persist")
10157    }
10158
10159    #[cfg(feature = "test-util")]
10160    async fn drive_tier_delete_dispatch_restart_to_convergence(store: Arc<crate::store::ECStore>) {
10161        tokio::time::timeout(Duration::from_secs(30), async {
10162            let mut stable_empty_observations = 0;
10163            loop {
10164                recover_tier_delete_dispatch_manifests(store.clone(), 100, None)
10165                    .await
10166                    .expect("restarted manifest recovery pass should finish");
10167                recover_tier_delete_journal_entries(store.clone(), 100, None)
10168                    .await
10169                    .expect("restarted journal recovery pass should finish");
10170                if tier_delete_journal_count(store.clone()).await == 0
10171                    && tier_delete_dispatch_manifest_count(store.clone()).await == 0
10172                    && tier_delete_dispatch_manifest_recovery_count_for_test(&store) == 0
10173                {
10174                    stable_empty_observations += 1;
10175                    if stable_empty_observations == 10 {
10176                        break;
10177                    }
10178                } else {
10179                    stable_empty_observations = 0;
10180                }
10181                tokio::time::sleep(Duration::from_millis(50)).await;
10182            }
10183        })
10184        .await
10185        .expect("same-disk restart recovery should converge without retained dispatch state");
10186    }
10187
10188    #[cfg(feature = "test-util")]
10189    #[tokio::test]
10190    #[serial_test::serial(storage_class_env)]
10191    async fn tier_delete_recovery_pass_drains_multiple_fast_manifest_pages() {
10192        const MANIFEST_COUNT: usize = 10;
10193
10194        let temp_dir = tempfile::tempdir().expect("create fast manifest pass recovery store dir");
10195        let mut instance_ctx = crate::runtime::instance::InstanceContext::new();
10196        instance_ctx.suppress_tier_delete_journal_recovery_for_test();
10197        let (ctx, store, shutdown) = without_storage_class_env(build_isolated_test_store_with_layout(
10198            temp_dir.path(),
10199            "tier-delete-fast-manifest-pass",
10200            &[(1, 4)],
10201            CancellationToken::new(),
10202            Some(Arc::new(instance_ctx)),
10203        ))
10204        .await;
10205        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
10206        let tier_name = "FAST-MANIFEST-PASS";
10207        let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
10208        let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
10209            .await
10210            .expect("fast manifest pass tier lease should resolve")
10211            .backend_identity();
10212        for index in 0..MANIFEST_COUNT {
10213            // Pagination must not depend on same-bucket lock wait deadlines.
10214            let bucket = format!("tier-delete-fast-manifest-pass-{index}");
10215            store
10216                .make_bucket(&bucket, &MakeBucketOptions::default())
10217                .await
10218                .expect("fast manifest pass bucket should be created");
10219            let incarnation = store
10220                .bucket_incarnation_id(&bucket)
10221                .await
10222                .expect("fast manifest pass bucket incarnation should resolve");
10223            install_aborting_dispatch_fixture(
10224                store.clone(),
10225                &bucket,
10226                incarnation,
10227                &format!("manifest-page-{index:06}/"),
10228                tier_name,
10229                backend_identity,
10230                1,
10231            )
10232            .await;
10233        }
10234        assert_eq!(tier_delete_dispatch_manifest_count(store.clone()).await, MANIFEST_COUNT);
10235        assert_eq!(tier_delete_journal_count(store.clone()).await, MANIFEST_COUNT);
10236
10237        let cancel = CancellationToken::new();
10238        let mut manifest_marker = None;
10239        let mut journal_marker = None;
10240        let stats = recover_test_tier_delete_journal_pass_with_budget(
10241            store.clone(),
10242            &cancel,
10243            &mut manifest_marker,
10244            &mut journal_marker,
10245            Duration::from_secs(30),
10246        )
10247        .await;
10248
10249        assert!(!stats.canceled);
10250        assert!(!stats.deadline_exhausted);
10251        assert!(
10252            stats.manifest_pages > 1,
10253            "one production pass must cross the default eight-manifest page limit"
10254        );
10255        assert_eq!(stats.manifests.scanned, MANIFEST_COUNT);
10256        assert_eq!(stats.manifests.deleted, MANIFEST_COUNT, "full recovery result: {stats:?}");
10257        assert_eq!(stats.manifests.failed, 0, "full recovery result: {stats:?}");
10258        assert_eq!(manifest_marker, None);
10259        assert_eq!(tier_delete_dispatch_manifest_count(store.clone()).await, 0);
10260        assert_eq!(tier_delete_journal_count(store).await, 0);
10261        assert_eq!(backend.remove_count().await, 0, "rollback recovery must not call the remote tier");
10262        shutdown.cancel();
10263    }
10264
10265    #[cfg(feature = "test-util")]
10266    #[tokio::test]
10267    #[serial_test::serial(storage_class_env)]
10268    async fn tier_delete_manual_pass_retains_manifest_owned_by_startup_recovery() {
10269        let temp_dir = tempfile::tempdir().expect("create automatic recovery ownership store dir");
10270        let (ctx, store, shutdown) =
10271            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "tier-delete-auto-owner", &[4])).await;
10272        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
10273        let bucket = "tier-delete-auto-owner-bucket";
10274        store
10275            .make_bucket(bucket, &MakeBucketOptions::default())
10276            .await
10277            .expect("automatic recovery bucket should be created");
10278        let incarnation = store.bucket_incarnation_id(bucket).await.expect("bucket incarnation");
10279        let tier_name = "AUTO-OWNER";
10280        let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
10281        let identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
10282            .await
10283            .expect("automatic recovery tier lease")
10284            .backend_identity();
10285
10286        // The automatic worker must not observe a partially installed fixture.
10287        let lifecycle_guard = store
10288            .acquire_bucket_lifecycle_write_lock(bucket)
10289            .await
10290            .expect("fixture lifecycle lock");
10291        let (manifest_name, entries) =
10292            install_aborting_dispatch_fixture(store.clone(), bucket, incarnation, "auto-owner/", tier_name, identity, 1).await;
10293        let journal_name = tier_delete_journal_object_name(&entries[0]);
10294        let hook = TierDeleteDispatchRollbackTestHook::install_slow_delete(&journal_name, &journal_name);
10295        drop(lifecycle_guard);
10296        ctx.wake_tier_delete_journal_recovery();
10297        tokio::time::timeout(Duration::from_secs(30), hook.wait_until_delete_paused())
10298            .await
10299            .expect("startup recovery should own the manifest before a manual pass");
10300        assert!(tier_delete_dispatch_manifest_recovery_inflight_for_test(&store, &manifest_name));
10301
10302        let stats = recover_tier_delete_dispatch_manifests(store.clone(), 8, None)
10303            .await
10304            .expect("manual recovery scan");
10305        assert_eq!(stats.scanned, 1, "{stats:?}");
10306        assert_eq!(stats.retained, 1, "{stats:?}");
10307        assert_eq!(stats.deleted, 0, "{stats:?}");
10308        assert_eq!(stats.failed, 0, "{stats:?}");
10309        assert_eq!(tier_delete_dispatch_manifest_count(store.clone()).await, 1);
10310        assert_eq!(tier_delete_journal_count(store.clone()).await, 1);
10311
10312        hook.release_delete();
10313        tokio::time::timeout(Duration::from_secs(30), async {
10314            loop {
10315                let manifest_gone = matches!(com::read_config(store.clone(), &manifest_name).await, Err(Error::ConfigNotFound));
10316                if manifest_gone && !tier_delete_dispatch_manifest_recovery_inflight_for_test(&store, &manifest_name) {
10317                    break;
10318                }
10319                tokio::time::sleep(Duration::from_millis(10)).await;
10320            }
10321        })
10322        .await
10323        .expect("automatic recovery should converge without a manual retry");
10324        assert_eq!(tier_delete_dispatch_manifest_count(store.clone()).await, 0);
10325        assert_eq!(tier_delete_journal_count(store).await, 0);
10326        assert_eq!(backend.remove_count().await, 0, "rollback must not delete from the remote tier");
10327        shutdown.cancel();
10328    }
10329
10330    #[cfg(feature = "test-util")]
10331    #[tokio::test]
10332    #[serial_test::serial(storage_class_env)]
10333    async fn tier_delete_recovery_pass_drains_multiple_fast_journal_pages() {
10334        const JOURNAL_COUNT: usize = 25;
10335
10336        let temp_dir = tempfile::tempdir().expect("create fast pass recovery store dir");
10337        let (ctx, store, _shutdown) =
10338            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "tier-delete-fast-pass", &[4])).await;
10339        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
10340        let tier_name = "FAST-PASS-RECOVERY";
10341        let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
10342        let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
10343            .await
10344            .expect("fast pass tier lease should resolve")
10345            .backend_identity();
10346        let entries = install_committed_tier_delete_journals(store.clone(), tier_name, backend_identity, JOURNAL_COUNT).await;
10347        assert_eq!(tier_delete_journal_count(store.clone()).await, JOURNAL_COUNT);
10348
10349        let cancel = CancellationToken::new();
10350        let mut manifest_marker = None;
10351        let mut journal_marker = None;
10352        let stats = recover_test_tier_delete_journal_pass_with_budget(
10353            store.clone(),
10354            &cancel,
10355            &mut manifest_marker,
10356            &mut journal_marker,
10357            Duration::from_secs(30),
10358        )
10359        .await;
10360
10361        assert!(!stats.canceled);
10362        assert!(!stats.deadline_exhausted);
10363        assert!(
10364            stats.journal_pages > 1,
10365            "one production pass must cross the default eight-record page limit"
10366        );
10367        assert_eq!(stats.journals.scanned, JOURNAL_COUNT);
10368        assert_eq!(stats.journals.deleted, JOURNAL_COUNT);
10369        assert_eq!(stats.journals.failed, 0);
10370        assert_eq!(journal_marker, None);
10371        assert_eq!(tier_delete_journal_count(store).await, 0);
10372        assert_eq!(backend.exact_remove_count(), JOURNAL_COUNT);
10373        let mut removed = backend.remove_versions().await;
10374        removed.sort();
10375        let mut expected = entries
10376            .into_iter()
10377            .map(|entry| (entry.obj_name, entry.version_id))
10378            .collect::<Vec<_>>();
10379        expected.sort();
10380        assert_eq!(removed, expected);
10381    }
10382
10383    #[cfg(feature = "test-util")]
10384    #[tokio::test]
10385    #[serial_test::serial(storage_class_env)]
10386    async fn tier_delete_recovery_pass_rotates_failed_journal_pages_without_busy_loop() {
10387        const JOURNAL_COUNT: usize = 9;
10388
10389        let temp_dir = tempfile::tempdir().expect("create failed pass recovery store dir");
10390        let (ctx, store, _shutdown) =
10391            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "tier-delete-failed-pass", &[4])).await;
10392        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
10393        let tier_name = "FAILED-PASS-RECOVERY";
10394        let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
10395        let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
10396            .await
10397            .expect("failed pass tier lease should resolve")
10398            .backend_identity();
10399        install_committed_tier_delete_journals(store.clone(), tier_name, backend_identity, JOURNAL_COUNT).await;
10400        backend.set_remove_failure(true);
10401
10402        let cancel = CancellationToken::new();
10403        let mut manifest_marker = None;
10404        let mut journal_marker = None;
10405        let first = recover_test_tier_delete_journal_pass_with_budget(
10406            store.clone(),
10407            &cancel,
10408            &mut manifest_marker,
10409            &mut journal_marker,
10410            Duration::from_secs(30),
10411        )
10412        .await;
10413        assert_eq!(first.journal_pages, 1, "a no-progress failed page must stop the pass");
10414        assert_eq!(first.journals.scanned, 8);
10415        assert_eq!(first.journals.deleted, 0);
10416        assert_eq!(first.journals.failed, 8);
10417        assert!(journal_marker.is_some(), "a truncated failed page must retain its continuation marker");
10418        assert_eq!(tier_delete_journal_count(store.clone()).await, JOURNAL_COUNT);
10419
10420        let second = recover_test_tier_delete_journal_pass_with_budget(
10421            store.clone(),
10422            &cancel,
10423            &mut manifest_marker,
10424            &mut journal_marker,
10425            Duration::from_secs(30),
10426        )
10427        .await;
10428        assert_eq!(second.journal_pages, 1);
10429        assert_eq!(second.journals.scanned, 1);
10430        assert_eq!(second.journals.deleted, 0);
10431        assert_eq!(second.journals.failed, 1);
10432        assert_eq!(
10433            journal_marker, None,
10434            "end-of-list rotation must revisit the failed prefix on a later pass"
10435        );
10436        assert_eq!(tier_delete_journal_count(store.clone()).await, JOURNAL_COUNT);
10437
10438        backend.set_remove_failure(false);
10439        let third = recover_test_tier_delete_journal_pass_with_budget(
10440            store.clone(),
10441            &cancel,
10442            &mut manifest_marker,
10443            &mut journal_marker,
10444            Duration::from_secs(30),
10445        )
10446        .await;
10447        assert_eq!(third.journals.scanned, JOURNAL_COUNT);
10448        assert_eq!(third.journals.deleted, JOURNAL_COUNT);
10449        assert_eq!(third.journals.failed, 0);
10450        assert_eq!(tier_delete_journal_count(store).await, 0);
10451    }
10452
10453    #[cfg(feature = "test-util")]
10454    #[tokio::test]
10455    #[serial_test::serial(storage_class_env)]
10456    async fn tier_delete_journal_rejects_unbound_new_prepare_without_persisting() {
10457        let temp_dir = tempfile::tempdir().expect("create journal admission store dir");
10458        let (_ctx, store, _shutdown) =
10459            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "tier-journal-unbound-prepare", &[4])).await;
10460        let entry = Jentry {
10461            persisted_version: 0,
10462            obj_name: "remote/object".to_string(),
10463            version_id: uuid::Uuid::new_v4().to_string(),
10464            tier_name: "COLD-A".to_string(),
10465            backend_identity: Some([7; 32]),
10466            version_id_exact: true,
10467            version_state: rustfs_filemeta::TransitionVersionState::Exact,
10468            state: TierDeleteJournalState::Prepared,
10469            source: Some(TierDeleteSourceIdentity {
10470                bucket: "source-bucket".to_string(),
10471                object: "source-object".to_string(),
10472                version_id: Some(uuid::Uuid::new_v4().to_string()),
10473                versioned: true,
10474                version_suspended: false,
10475                data_dir: Some(uuid::Uuid::new_v4().to_string()),
10476                etag: Some("source-etag".to_string()),
10477                mod_time: Some(OffsetDateTime::UNIX_EPOCH.to_string()),
10478            }),
10479            dispatch: None,
10480        };
10481        let error = persist_tier_delete_journal_entry(store.clone(), &entry)
10482            .await
10483            .expect_err("a new source-owning journal without a v6 manifest binding must fail closed");
10484        assert!(error.to_string().contains("dispatch manifest binding"), "unexpected error: {error}");
10485        assert_eq!(tier_delete_journal_count(store).await, 0, "rejected prepare must not write a journal");
10486    }
10487
10488    #[cfg(feature = "test-util")]
10489    #[tokio::test]
10490    #[serial_test::serial(storage_class_env)]
10491    async fn dispatch_manifest_recovery_rolls_back_unapproved_crash_states_without_remote_io() {
10492        let temp_dir = tempfile::tempdir().expect("create dispatch rollback store dir");
10493        let (ctx, store, _shutdown) =
10494            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "dispatch-manifest-rollback", &[4])).await;
10495        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
10496        let bucket = "dispatch-manifest-rollback-bucket";
10497        store
10498            .make_bucket(bucket, &MakeBucketOptions::default())
10499            .await
10500            .expect("dispatch rollback bucket should be created");
10501        let incarnation = store
10502            .bucket_incarnation_id(bucket)
10503            .await
10504            .expect("bucket incarnation should resolve");
10505        let tier_name = "DISPATCH-ROLLBACK";
10506        let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
10507        let identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
10508            .await
10509            .expect("tier lease should resolve")
10510            .backend_identity();
10511
10512        let preparing = [
10513            ("preparing/a.bin", TierDeleteJournalState::Prepared),
10514            ("preparing/b.bin", TierDeleteJournalState::Dispatched),
10515        ]
10516        .into_iter()
10517        .map(|(object, state)| {
10518            (
10519                synthetic_v6_dispatch_entry(bucket, object, tier_name, identity, &uuid::Uuid::new_v4().to_string()),
10520                Some(state),
10521            )
10522        })
10523        .collect();
10524        install_test_tier_delete_dispatch_fixture(
10525            store.clone(),
10526            bucket,
10527            incarnation,
10528            "preparing/",
10529            preparing,
10530            TierDeleteDispatchManifestState::Preparing,
10531        )
10532        .await
10533        .expect("Preparing crash fixture should persist");
10534
10535        let aborting = [
10536            ("aborting/missing.bin", None),
10537            ("aborting/prepared.bin", Some(TierDeleteJournalState::Prepared)),
10538            ("aborting/dispatched.bin", Some(TierDeleteJournalState::Dispatched)),
10539        ]
10540        .into_iter()
10541        .map(|(object, state)| {
10542            (
10543                synthetic_v6_dispatch_entry(bucket, object, tier_name, identity, &uuid::Uuid::new_v4().to_string()),
10544                state,
10545            )
10546        })
10547        .collect();
10548        install_test_tier_delete_dispatch_fixture(
10549            store.clone(),
10550            bucket,
10551            incarnation,
10552            "aborting/",
10553            aborting,
10554            TierDeleteDispatchManifestState::Aborting,
10555        )
10556        .await
10557        .expect("Aborting crash fixture should persist");
10558        install_test_tier_delete_dispatch_fixture(
10559            store.clone(),
10560            bucket,
10561            incarnation,
10562            "empty/",
10563            Vec::new(),
10564            TierDeleteDispatchManifestState::Completed,
10565        )
10566        .await
10567        .expect("empty Completed crash fixture should persist");
10568
10569        let deleted = tokio::time::timeout(Duration::from_secs(30), async {
10570            let mut deleted = 0;
10571            loop {
10572                let stats = recover_tier_delete_dispatch_manifests(store.clone(), 100, None)
10573                    .await
10574                    .expect("manifest recovery should reconcile negative-seal crash states");
10575                deleted += stats.deleted;
10576                if tier_delete_dispatch_manifest_count(store.clone()).await == 0 {
10577                    break deleted;
10578                }
10579                // A different process-global test can briefly revoke the
10580                // fleet proof. Recovery must fail closed for that pass and
10581                // converge after the proof generation is republished.
10582                tokio::time::sleep(Duration::from_millis(10)).await;
10583            }
10584        })
10585        .await
10586        .expect("valid negative-seal crash states should eventually converge");
10587        assert_eq!(deleted, 3);
10588        assert_eq!(tier_delete_journal_count(store.clone()).await, 0);
10589        assert_eq!(tier_delete_dispatch_manifest_count(store).await, 0);
10590        assert_eq!(
10591            backend.remove_count().await,
10592            0,
10593            "rollback recovery must perform zero remote delete operations"
10594        );
10595    }
10596
10597    #[cfg(feature = "test-util")]
10598    #[tokio::test]
10599    #[serial_test::serial(storage_class_env)]
10600    async fn dispatch_manifest_recovery_quarantines_impossible_committed_rollback_set() {
10601        let temp_dir = tempfile::tempdir().expect("create dispatch quarantine store dir");
10602        let (ctx, store, _shutdown) =
10603            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "dispatch-manifest-quarantine", &[4])).await;
10604        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
10605        let bucket = "dispatch-manifest-quarantine-bucket";
10606        store
10607            .make_bucket(bucket, &MakeBucketOptions::default())
10608            .await
10609            .expect("dispatch quarantine bucket should be created");
10610        let incarnation = store
10611            .bucket_incarnation_id(bucket)
10612            .await
10613            .expect("bucket incarnation should resolve");
10614        let tier_name = "DISPATCH-QUARANTINE";
10615        let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
10616        let identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
10617            .await
10618            .expect("tier lease should resolve")
10619            .backend_identity();
10620        let entries = [TierDeleteJournalState::Prepared, TierDeleteJournalState::Committed]
10621            .into_iter()
10622            .enumerate()
10623            .map(|(index, state)| {
10624                (
10625                    synthetic_v6_dispatch_entry(
10626                        bucket,
10627                        &format!("quarantine/{index}.bin"),
10628                        tier_name,
10629                        identity,
10630                        &uuid::Uuid::new_v4().to_string(),
10631                    ),
10632                    Some(state),
10633                )
10634            })
10635            .collect();
10636        let (manifest_name, _) = install_test_tier_delete_dispatch_fixture(
10637            store.clone(),
10638            bucket,
10639            incarnation,
10640            "quarantine/",
10641            entries,
10642            TierDeleteDispatchManifestState::Aborting,
10643        )
10644        .await
10645        .expect("impossible rollback fixture should persist");
10646
10647        let stats = recover_tier_delete_dispatch_manifests(store.clone(), 100, None)
10648            .await
10649            .expect("quarantined manifest scan should finish");
10650        assert_eq!((stats.scanned, stats.deleted, stats.failed), (1, 0, 1));
10651        assert_eq!(
10652            tier_delete_journal_count(store.clone()).await,
10653            2,
10654            "validation must precede every deletion"
10655        );
10656        assert_eq!(tier_delete_dispatch_manifest_count(store.clone()).await, 1);
10657        assert_eq!(
10658            test_tier_delete_dispatch_manifest_state(store, &manifest_name)
10659                .await
10660                .expect("quarantined manifest should remain readable"),
10661            Some(TierDeleteDispatchManifestState::Aborting)
10662        );
10663        assert_eq!(backend.remove_count().await, 0);
10664    }
10665
10666    #[cfg(feature = "test-util")]
10667    #[tokio::test]
10668    #[serial_test::serial(storage_class_env)]
10669    async fn dispatch_manifest_rollback_bounded_concurrency_reaches_tail_behind_slow_member() {
10670        const JOURNAL_COUNT: usize = 65;
10671
10672        let temp_dir = tempfile::tempdir().expect("create bounded rollback store dir");
10673        let (ctx, store, _shutdown) =
10674            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "bounded-dispatch-rollback", &[4])).await;
10675        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
10676        let bucket = "bounded-dispatch-rollback-bucket";
10677        store
10678            .make_bucket(bucket, &MakeBucketOptions::default())
10679            .await
10680            .expect("bounded rollback bucket should be created");
10681        let incarnation = store
10682            .bucket_incarnation_id(bucket)
10683            .await
10684            .expect("bounded rollback bucket incarnation should resolve");
10685        let tier_name = "BOUNDED-DISPATCH-ROLLBACK";
10686        let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
10687        let identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
10688            .await
10689            .expect("bounded rollback tier lease should resolve")
10690            .backend_identity();
10691        let (manifest_name, entries) =
10692            install_aborting_dispatch_fixture(store.clone(), bucket, incarnation, "slow/", tier_name, identity, JOURNAL_COUNT)
10693                .await;
10694        let first_name = tier_delete_journal_object_name(entries.first().expect("slow rollback fixture should not be empty"));
10695        let last_name = tier_delete_journal_object_name(entries.last().expect("slow rollback fixture should not be empty"));
10696        let hook = TierDeleteDispatchRollbackTestHook::install_slow_delete(&first_name, &last_name);
10697        let worker_store = store.clone();
10698        let worker_manifest = manifest_name.clone();
10699        let worker =
10700            tokio::spawn(async move { recover_test_tier_delete_dispatch_manifest(worker_store, &worker_manifest).await });
10701        tokio::time::timeout(Duration::from_secs(30), hook.wait_until_delete_paused())
10702            .await
10703            .expect("the first rollback member should reach the slow hook");
10704        tokio::time::timeout(Duration::from_secs(30), hook.wait_until_delete_observed())
10705            .await
10706            .expect("a member beyond the first two concurrency windows should remain reachable");
10707        assert!(
10708            (2..=32).contains(&hook.max_in_flight()),
10709            "rollback delete concurrency must make progress without exceeding its bound; observed {}",
10710            hook.max_in_flight()
10711        );
10712        hook.release_delete();
10713        worker
10714            .await
10715            .expect("bounded rollback recovery task should join")
10716            .expect("bounded rollback recovery should converge after the slow member releases");
10717        drop(hook);
10718
10719        assert_eq!(tier_delete_journal_count(store.clone()).await, 0);
10720        assert_eq!(tier_delete_dispatch_manifest_count(store.clone()).await, 0);
10721        assert_eq!(backend.remove_count().await, 0, "rollback must not call the remote tier");
10722    }
10723
10724    #[cfg(feature = "test-util")]
10725    #[tokio::test]
10726    #[serial_test::serial(storage_class_env)]
10727    async fn aborted_dispatch_retry_defers_residual_scan_to_recovery() {
10728        const JOURNAL_COUNT: usize = 65;
10729
10730        let temp_dir = tempfile::tempdir().expect("create aborted retry store dir");
10731        let (ctx, store, shutdown) =
10732            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "aborted-dispatch-retry", &[4])).await;
10733        shutdown.cancel();
10734        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
10735        let bucket = "aborted-dispatch-retry-bucket";
10736        let prefix = "archive/";
10737        store
10738            .make_bucket(bucket, &MakeBucketOptions::default())
10739            .await
10740            .expect("aborted retry bucket should be created");
10741        let incarnation = store
10742            .bucket_incarnation_id(bucket)
10743            .await
10744            .expect("aborted retry bucket incarnation should resolve");
10745        let tier_name = "ABORTED-DISPATCH-RETRY";
10746        let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
10747        let identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
10748            .await
10749            .expect("aborted retry tier lease should resolve")
10750            .backend_identity();
10751        let aborted_entries = (0..JOURNAL_COUNT)
10752            .map(|index| {
10753                (
10754                    synthetic_v6_dispatch_entry(
10755                        bucket,
10756                        &format!("{prefix}{index:06}.bin"),
10757                        tier_name,
10758                        identity,
10759                        &uuid::Uuid::new_v4().to_string(),
10760                    ),
10761                    Some(TierDeleteJournalState::Prepared),
10762                )
10763            })
10764            .collect::<Vec<_>>();
10765        let entries = aborted_entries.iter().map(|(entry, _)| entry.clone()).collect::<Vec<_>>();
10766        let (manifest_name, _) = install_test_tier_delete_dispatch_fixture(
10767            store.clone(),
10768            bucket,
10769            incarnation,
10770            prefix,
10771            aborted_entries,
10772            TierDeleteDispatchManifestState::Aborted,
10773        )
10774        .await
10775        .expect("Aborted retry fixture should persist");
10776
10777        let lifecycle_guard = store
10778            .acquire_bucket_lifecycle_write_lock(bucket)
10779            .await
10780            .expect("aborted retry should acquire the bucket lifecycle fence");
10781        let mut fence_opts = ObjectOptions::default();
10782        fence_opts.add_bucket_lifecycle_lock_guard(&lifecycle_guard);
10783        let bucket_fence = fence_opts
10784            .bucket_lifecycle_lock_fence
10785            .clone()
10786            .expect("aborted retry should capture the bucket lifecycle fence");
10787        let fleet_proof = acquire_tier_delete_journal_fleet_proof().expect("aborted retry fixture should have a fleet proof");
10788        let hook = TierDeleteDispatchMemberReadTestHook::install_pause(TierDeleteDispatchMemberReadTestStage::Validation);
10789
10790        let error =
10791            match prepare_tier_delete_dispatch(store.clone(), bucket, incarnation, prefix, entries, fleet_proof, &bucket_fence)
10792                .await
10793            {
10794                Ok(_) => panic!("an Aborted manifest must be retained for bounded recovery cleanup"),
10795                Err(err) => err,
10796            };
10797
10798        assert!(
10799            error.to_string().contains("bounded recovery cleanup"),
10800            "unexpected aborted retry error: {error}"
10801        );
10802        assert_eq!(
10803            hook.entry_count(),
10804            0,
10805            "request retry must not scan the retained Aborted journal set under bucket write lock"
10806        );
10807        assert_eq!(
10808            test_tier_delete_dispatch_manifest_state(store.clone(), &manifest_name)
10809                .await
10810                .expect("retained Aborted manifest should remain readable"),
10811            Some(TierDeleteDispatchManifestState::Aborted)
10812        );
10813        assert_eq!(tier_delete_journal_count(store.clone()).await, JOURNAL_COUNT);
10814        assert_eq!(backend.remove_count().await, 0, "retry must not call the remote tier");
10815        drop(hook);
10816        drop(lifecycle_guard);
10817
10818        recover_test_tier_delete_dispatch_manifest(store.clone(), &manifest_name)
10819            .await
10820            .expect("bounded recovery should clean the retained Aborted manifest");
10821        assert_eq!(tier_delete_journal_count(store.clone()).await, 0);
10822        assert_eq!(tier_delete_dispatch_manifest_count(store).await, 0);
10823    }
10824
10825    #[cfg(feature = "test-util")]
10826    #[tokio::test]
10827    #[serial_test::serial(storage_class_env)]
10828    async fn dispatch_manifest_page_timeout_detaches_one_deduplicated_worker_until_convergence() {
10829        let temp_dir = tempfile::tempdir().expect("create detached rollback store dir");
10830        let (ctx, store, _shutdown) =
10831            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "detached-dispatch-rollback", &[4])).await;
10832        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
10833        let bucket = "detached-dispatch-rollback-bucket";
10834        store
10835            .make_bucket(bucket, &MakeBucketOptions::default())
10836            .await
10837            .expect("detached rollback bucket should be created");
10838        let incarnation = store
10839            .bucket_incarnation_id(bucket)
10840            .await
10841            .expect("detached rollback bucket incarnation should resolve");
10842        let tier_name = "DETACHED-DISPATCH-ROLLBACK";
10843        let _backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
10844        let identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
10845            .await
10846            .expect("detached rollback tier lease should resolve")
10847            .backend_identity();
10848        let (manifest_name, entries) =
10849            install_aborting_dispatch_fixture(store.clone(), bucket, incarnation, "detached/", tier_name, identity, 1).await;
10850        let journal_name = tier_delete_journal_object_name(&entries[0]);
10851        let hook = TierDeleteDispatchRollbackTestHook::install_slow_delete(&journal_name, &journal_name);
10852
10853        assert!(
10854            recover_test_tier_delete_dispatch_manifest_with_page_budget(
10855                store.clone(),
10856                &manifest_name,
10857                Duration::from_millis(10),
10858            )
10859            .await,
10860            "the synthetic page budget must elapse while the manifest worker continues"
10861        );
10862        tokio::time::timeout(Duration::from_secs(30), hook.wait_until_delete_paused())
10863            .await
10864            .expect("the detached manifest worker should continue into its delete phase");
10865        assert!(
10866            tier_delete_dispatch_manifest_recovery_inflight_for_test(&store, &manifest_name),
10867            "the detached worker must retain its per-store/object deduplication guard"
10868        );
10869        assert!(
10870            !recover_test_tier_delete_dispatch_manifest_with_page_budget(store.clone(), &manifest_name, Duration::from_secs(30),)
10871                .await,
10872            "a duplicate page observation should be retained without queuing another worker"
10873        );
10874
10875        hook.release_delete();
10876        tokio::time::timeout(Duration::from_secs(30), async {
10877            loop {
10878                let manifest_gone = matches!(com::read_config(store.clone(), &manifest_name).await, Err(Error::ConfigNotFound));
10879                if manifest_gone && !tier_delete_dispatch_manifest_recovery_inflight_for_test(&store, &manifest_name) {
10880                    break;
10881                }
10882                tokio::time::sleep(Duration::from_millis(10)).await;
10883            }
10884        })
10885        .await
10886        .expect("the detached manifest worker should release its guard after convergence");
10887        drop(hook);
10888        assert_eq!(tier_delete_journal_count(store.clone()).await, 0);
10889    }
10890
10891    #[cfg(feature = "test-util")]
10892    #[tokio::test]
10893    #[serial_test::serial(storage_class_env)]
10894    async fn dispatch_manifest_detached_workers_are_store_bounded_and_shutdown_cancellable() {
10895        const MANIFEST_COUNT: usize = 6;
10896        const WORKER_LIMIT: usize = 4;
10897
10898        let temp_dir = tempfile::tempdir().expect("create bounded detached rollback store dir");
10899        let (ctx, store, shutdown) =
10900            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "bounded-detached-dispatch-rollback", &[4]))
10901                .await;
10902        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
10903        let tier_name = "BOUNDED-DETACHED-DISPATCH-ROLLBACK";
10904        let _backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
10905        let identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
10906            .await
10907            .expect("bounded detached rollback tier lease should resolve")
10908            .backend_identity();
10909        let mut manifest_names = Vec::with_capacity(MANIFEST_COUNT);
10910        for index in 0..MANIFEST_COUNT {
10911            let bucket = format!("bounded-detached-dispatch-rollback-{index}");
10912            store
10913                .make_bucket(&bucket, &MakeBucketOptions::default())
10914                .await
10915                .expect("bounded detached rollback bucket should be created");
10916            let incarnation = store
10917                .bucket_incarnation_id(&bucket)
10918                .await
10919                .expect("bounded detached rollback bucket incarnation should resolve");
10920            let (manifest_name, _) = install_aborting_dispatch_fixture(
10921                store.clone(),
10922                &bucket,
10923                incarnation,
10924                &format!("detached-{index}/"),
10925                tier_name,
10926                identity,
10927                1,
10928            )
10929            .await;
10930            manifest_names.push(manifest_name);
10931        }
10932        let hook = TierDeleteDispatchRollbackTestHook::install_pause_all_deletes();
10933
10934        let first_pass = futures::stream::iter(manifest_names.iter().cloned().map(|manifest_name| {
10935            let store = store.clone();
10936            async move {
10937                recover_test_tier_delete_dispatch_manifest_with_page_budget(store, &manifest_name, Duration::from_millis(10))
10938                    .await
10939            }
10940        }))
10941        .buffer_unordered(MANIFEST_COUNT)
10942        .collect::<Vec<_>>()
10943        .await;
10944        tokio::time::timeout(Duration::from_secs(30), hook.wait_until_delete_pause_count(WORKER_LIMIT))
10945            .await
10946            .expect("every admitted detached worker should pause before its first mutation");
10947        assert_eq!(
10948            first_pass.into_iter().filter(|timed_out| *timed_out).count(),
10949            WORKER_LIMIT,
10950            "only the per-store worker budget may detach"
10951        );
10952        assert_eq!(
10953            tier_delete_dispatch_manifest_recovery_count_for_test(&store),
10954            WORKER_LIMIT,
10955            "detached workers must remain bounded per store"
10956        );
10957
10958        let repeated_pass = futures::stream::iter(manifest_names.iter().cloned().map(|manifest_name| {
10959            let store = store.clone();
10960            async move {
10961                recover_test_tier_delete_dispatch_manifest_with_page_budget(store, &manifest_name, Duration::from_secs(30)).await
10962            }
10963        }))
10964        .buffer_unordered(MANIFEST_COUNT)
10965        .collect::<Vec<_>>()
10966        .await;
10967        assert!(
10968            repeated_pass.into_iter().all(|timed_out| !timed_out),
10969            "duplicates and over-budget manifests must be retained without a waiter queue"
10970        );
10971        assert_eq!(tier_delete_dispatch_manifest_recovery_count_for_test(&store), WORKER_LIMIT);
10972
10973        shutdown.cancel();
10974        tokio::time::sleep(Duration::from_millis(50)).await;
10975        assert_eq!(
10976            tier_delete_dispatch_manifest_recovery_count_for_test(&store),
10977            WORKER_LIMIT,
10978            "shutdown must not drop a bounded batch before its admitted work drains"
10979        );
10980        assert_eq!(tier_delete_journal_count(store.clone()).await, MANIFEST_COUNT);
10981        assert_eq!(tier_delete_dispatch_manifest_count(store.clone()).await, MANIFEST_COUNT);
10982        hook.release_all_deletes();
10983        tokio::time::timeout(Duration::from_secs(30), async {
10984            while tier_delete_dispatch_manifest_recovery_count_for_test(&store) != 0 {
10985                tokio::time::sleep(Duration::from_millis(10)).await;
10986            }
10987        })
10988        .await
10989        .expect("shutdown should cancel every detached worker and release its registry permit");
10990        assert_eq!(tier_delete_journal_count(store.clone()).await, MANIFEST_COUNT);
10991        assert_eq!(tier_delete_dispatch_manifest_count(store.clone()).await, MANIFEST_COUNT);
10992        drop(hook);
10993        tokio::time::sleep(Duration::from_millis(50)).await;
10994        assert_eq!(
10995            tier_delete_journal_count(store.clone()).await,
10996            MANIFEST_COUNT,
10997            "releasing the old hook after shutdown must not resume a cancelled mutation"
10998        );
10999        assert_eq!(
11000            tier_delete_dispatch_manifest_recovery_count_for_test(&store),
11001            0,
11002            "the shutdown store must leave no registry permits behind"
11003        );
11004        drop(store);
11005        drop(ctx);
11006
11007        let (_restarted_ctx, restarted_store, restarted_shutdown) = without_storage_class_env(build_isolated_test_store(
11008            temp_dir.path(),
11009            "bounded-detached-dispatch-rollback-restart",
11010            &[4],
11011        ))
11012        .await;
11013        crate::bucket::metadata_sys::init_bucket_metadata_sys(restarted_store.clone(), Vec::new()).await;
11014        assert!(!restarted_shutdown.is_cancelled(), "the restarted store needs a fresh shutdown token");
11015        tokio::time::timeout(Duration::from_secs(30), async {
11016            loop {
11017                let active = tier_delete_dispatch_manifest_recovery_count_for_test(&restarted_store);
11018                let manifests = tier_delete_dispatch_manifest_count(restarted_store.clone()).await;
11019                if active > 0 || manifests < MANIFEST_COUNT {
11020                    break;
11021                }
11022                tokio::task::yield_now().await;
11023            }
11024        })
11025        .await
11026        .expect("the restarted store should start a fresh manifest recovery");
11027        tokio::time::timeout(Duration::from_secs(30), async {
11028            while tier_delete_dispatch_manifest_recovery_count_for_test(&restarted_store) != 0 {
11029                tokio::time::sleep(Duration::from_millis(10)).await;
11030            }
11031        })
11032        .await
11033        .expect("the restarted store recovery should release every worker permit");
11034        if tier_delete_dispatch_manifest_count(restarted_store.clone()).await != 0 {
11035            let stats = recover_tier_delete_dispatch_manifests(restarted_store.clone(), 100, None)
11036                .await
11037                .expect("the restarted store should retry any conservatively retained manifest");
11038            assert_eq!(stats.failed, 0, "the restarted recovery must not quarantine a valid manifest");
11039        }
11040        assert_eq!(tier_delete_journal_count(restarted_store.clone()).await, 0);
11041        assert_eq!(tier_delete_dispatch_manifest_count(restarted_store).await, 0);
11042        restarted_shutdown.cancel();
11043    }
11044
11045    #[cfg(feature = "test-util")]
11046    #[tokio::test]
11047    #[serial_test::serial(storage_class_env)]
11048    async fn dispatch_manifest_shutdown_stops_aborting_validation_tail_reads() {
11049        const JOURNAL_COUNT: usize = 65;
11050        const ADMITTED_READS: usize = 32;
11051
11052        let temp_dir = tempfile::tempdir().expect("create validation-read shutdown store dir");
11053        let (ctx, store, shutdown) =
11054            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "dispatch-validation-read-shutdown", &[4]))
11055                .await;
11056        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
11057        let bucket = "dispatch-validation-read-shutdown-bucket";
11058        store
11059            .make_bucket(bucket, &MakeBucketOptions::default())
11060            .await
11061            .expect("validation-read shutdown bucket should be created");
11062        let incarnation = store
11063            .bucket_incarnation_id(bucket)
11064            .await
11065            .expect("validation-read shutdown bucket incarnation should resolve");
11066        let tier_name = "VALIDATION-READ-SHUTDOWN";
11067        let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
11068        let identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
11069            .await
11070            .expect("validation-read shutdown tier lease should resolve")
11071            .backend_identity();
11072        let (manifest_name, _) =
11073            install_aborting_dispatch_fixture(store.clone(), bucket, incarnation, "archive/", tier_name, identity, JOURNAL_COUNT)
11074                .await;
11075        let hook = TierDeleteDispatchMemberReadTestHook::install_pause(TierDeleteDispatchMemberReadTestStage::Validation);
11076        let worker_store = store.clone();
11077        let worker_manifest = manifest_name.clone();
11078        let worker = tokio::spawn(async move {
11079            recover_test_tier_delete_dispatch_manifest_with_page_budget(worker_store, &worker_manifest, Duration::from_secs(30))
11080                .await
11081        });
11082        tokio::time::timeout(Duration::from_secs(30), hook.wait_until_read_pause_count(ADMITTED_READS))
11083            .await
11084            .expect("the first validation read window should be admitted");
11085        assert_eq!(hook.entry_count(), ADMITTED_READS);
11086        assert_eq!(hook.in_flight(), ADMITTED_READS);
11087
11088        shutdown.cancel();
11089        tokio::time::sleep(Duration::from_millis(50)).await;
11090        assert_eq!(hook.entry_count(), ADMITTED_READS, "shutdown must not admit validation tail reads");
11091        assert_eq!(tier_delete_dispatch_manifest_recovery_count_for_test(&store), 1);
11092        assert!(tier_delete_dispatch_manifest_operation_lock_held_for_test(store.clone(), &manifest_name).await);
11093        assert!(crate::services::notification_sys::tier_delete_journal_fleet_proof_has_inflight_for_test());
11094        assert_eq!(tier_delete_journal_count(store.clone()).await, JOURNAL_COUNT);
11095        assert_eq!(backend.remove_count().await, 0);
11096
11097        hook.release_all_reads();
11098        assert!(
11099            !worker.await.expect("validation-read shutdown worker should join"),
11100            "cooperative shutdown should retain an unvalidated rollback"
11101        );
11102        assert_eq!(hook.entry_count(), ADMITTED_READS);
11103        assert_eq!(hook.in_flight(), 0);
11104        assert_eq!(tier_delete_dispatch_manifest_recovery_count_for_test(&store), 0);
11105        assert_eq!(
11106            tier_delete_journal_count(store.clone()).await,
11107            JOURNAL_COUNT,
11108            "no rollback DELETE may start"
11109        );
11110        assert_eq!(
11111            test_tier_delete_dispatch_manifest_state(store.clone(), &manifest_name)
11112                .await
11113                .expect("retained Aborting manifest should remain readable"),
11114            Some(TierDeleteDispatchManifestState::Aborting)
11115        );
11116        drop(hook);
11117        drop(store);
11118        drop(ctx);
11119
11120        let (_restarted_ctx, restarted_store, restarted_shutdown) = without_storage_class_env(build_isolated_test_store(
11121            temp_dir.path(),
11122            "dispatch-validation-read-shutdown-restart",
11123            &[4],
11124        ))
11125        .await;
11126        crate::bucket::metadata_sys::init_bucket_metadata_sys(restarted_store.clone(), Vec::new()).await;
11127        drive_tier_delete_dispatch_restart_to_convergence(restarted_store.clone()).await;
11128        restarted_shutdown.cancel();
11129    }
11130
11131    #[cfg(feature = "test-util")]
11132    #[tokio::test]
11133    #[serial_test::serial(storage_class_env)]
11134    async fn dispatch_manifest_shutdown_stops_authorized_tail_reads_before_receipt_or_cas() {
11135        const JOURNAL_COUNT: usize = 65;
11136        const ADMITTED_READS: usize = 32;
11137
11138        let temp_dir = tempfile::tempdir().expect("create authorized-read shutdown store dir");
11139        let (ctx, store, shutdown) =
11140            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "dispatch-authorized-read-shutdown", &[4]))
11141                .await;
11142        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
11143        let bucket = "dispatch-authorized-read-shutdown-bucket";
11144        store
11145            .make_bucket(bucket, &MakeBucketOptions::default())
11146            .await
11147            .expect("authorized-read shutdown bucket should be created");
11148        let incarnation = store
11149            .bucket_incarnation_id(bucket)
11150            .await
11151            .expect("authorized-read shutdown bucket incarnation should resolve");
11152        let tier_name = "AUTHORIZED-READ-SHUTDOWN";
11153        let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
11154        let identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
11155            .await
11156            .expect("authorized-read shutdown tier lease should resolve")
11157            .backend_identity();
11158        let entries = (0..JOURNAL_COUNT)
11159            .map(|index| {
11160                (
11161                    synthetic_v6_dispatch_entry(
11162                        bucket,
11163                        &format!("archive/{index:06}.bin"),
11164                        tier_name,
11165                        identity,
11166                        &uuid::Uuid::new_v4().to_string(),
11167                    ),
11168                    Some(TierDeleteJournalState::Committed),
11169                )
11170            })
11171            .collect();
11172        let (manifest_name, _) = install_test_tier_delete_dispatch_fixture(
11173            store.clone(),
11174            bucket,
11175            incarnation,
11176            "archive/",
11177            entries,
11178            TierDeleteDispatchManifestState::DispatchAuthorized,
11179        )
11180        .await
11181        .expect("authorized-read shutdown fixture should persist");
11182        let tier_config = ctx
11183            .tier_config_mgr()
11184            .read()
11185            .await
11186            .tiers
11187            .get(tier_name)
11188            .cloned()
11189            .expect("authorized-read tier config should remain available for restart");
11190        let hook = TierDeleteDispatchMemberReadTestHook::install_pause(TierDeleteDispatchMemberReadTestStage::Authorized);
11191        let worker_store = store.clone();
11192        let worker_manifest = manifest_name.clone();
11193        let worker = tokio::spawn(async move {
11194            recover_test_tier_delete_dispatch_manifest_with_page_budget(worker_store, &worker_manifest, Duration::from_secs(30))
11195                .await
11196        });
11197        tokio::time::timeout(Duration::from_secs(30), hook.wait_until_read_pause_count(ADMITTED_READS))
11198            .await
11199            .expect("the first authorized read window should be admitted");
11200
11201        shutdown.cancel();
11202        tokio::time::sleep(Duration::from_millis(50)).await;
11203        assert_eq!(hook.entry_count(), ADMITTED_READS, "shutdown must not admit authorized tail reads");
11204        assert_eq!(hook.in_flight(), ADMITTED_READS);
11205        assert_eq!(tier_delete_dispatch_manifest_recovery_count_for_test(&store), 1);
11206        assert!(tier_delete_dispatch_manifest_operation_lock_held_for_test(store.clone(), &manifest_name).await);
11207        assert!(crate::services::notification_sys::tier_delete_journal_fleet_proof_has_inflight_for_test());
11208
11209        hook.release_all_reads();
11210        assert!(
11211            !worker.await.expect("authorized-read shutdown worker should join"),
11212            "cooperative shutdown should retain an unscanned authorized manifest"
11213        );
11214        assert_eq!(hook.entry_count(), ADMITTED_READS);
11215        assert_eq!(hook.in_flight(), 0);
11216        assert_eq!(tier_delete_dispatch_manifest_recovery_count_for_test(&store), 0);
11217        assert_eq!(
11218            test_tier_delete_dispatch_manifest_state(store.clone(), &manifest_name)
11219                .await
11220                .expect("retained Authorized manifest should remain readable"),
11221            Some(TierDeleteDispatchManifestState::DispatchAuthorized),
11222            "cancellation before member reads must prevent the Completed CAS"
11223        );
11224        assert_eq!(
11225            hook.authorized_progress_count(),
11226            0,
11227            "cancellation before member reads must not attempt a decommission progress receipt"
11228        );
11229        assert_eq!(backend.remove_count().await, 0);
11230        drop(hook);
11231        drop(store);
11232        drop(ctx);
11233
11234        let (restarted_ctx, restarted_store, restarted_shutdown) = without_storage_class_env(build_isolated_test_store(
11235            temp_dir.path(),
11236            "dispatch-authorized-read-shutdown-restart",
11237            &[4],
11238        ))
11239        .await;
11240        {
11241            let tier_config_mgr = restarted_ctx.tier_config_mgr();
11242            let mut manager = tier_config_mgr.write().await;
11243            manager.tiers.insert(tier_name.to_string(), tier_config);
11244            manager
11245                .install_test_driver(tier_name, Box::new(backend.clone()))
11246                .expect("the exact authorized-read tier driver should reinstall after restart");
11247        }
11248        crate::bucket::metadata_sys::init_bucket_metadata_sys(restarted_store.clone(), Vec::new()).await;
11249        drive_tier_delete_dispatch_restart_to_convergence(restarted_store.clone()).await;
11250        assert_eq!(backend.remove_count().await, JOURNAL_COUNT);
11251        restarted_shutdown.cancel();
11252    }
11253
11254    #[cfg(feature = "test-util")]
11255    #[tokio::test]
11256    #[serial_test::serial(storage_class_env)]
11257    async fn dispatch_completion_cas_is_bounded_and_reaches_the_tail() {
11258        const JOURNAL_COUNT: usize = 65;
11259        const ADMITTED_COMMITS: usize = 32;
11260
11261        let temp_dir = tempfile::tempdir().expect("create dispatch completion concurrency store dir");
11262        let (ctx, store, shutdown) =
11263            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "dispatch-completion-concurrency", &[4])).await;
11264        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
11265        let bucket = "dispatch-completion-concurrency-bucket";
11266        let prefix = "archive/";
11267        store
11268            .make_bucket(bucket, &MakeBucketOptions::default())
11269            .await
11270            .expect("dispatch completion concurrency bucket should be created");
11271        let incarnation = store
11272            .bucket_incarnation_id(bucket)
11273            .await
11274            .expect("dispatch completion concurrency bucket incarnation should resolve");
11275        let tier_name = "DISPATCH-COMPLETION-CONCURRENCY";
11276        let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
11277        backend.set_remove_failure(true);
11278        let identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
11279            .await
11280            .expect("dispatch completion concurrency tier lease should resolve")
11281            .backend_identity();
11282        let entries = (0..JOURNAL_COUNT)
11283            .map(|index| {
11284                synthetic_v6_dispatch_entry(
11285                    bucket,
11286                    &format!("{prefix}{index:06}.bin"),
11287                    tier_name,
11288                    identity,
11289                    &uuid::Uuid::new_v4().to_string(),
11290                )
11291            })
11292            .collect::<Vec<_>>();
11293        let (manifest_name, _) = install_test_tier_delete_dispatch_fixture(
11294            store.clone(),
11295            bucket,
11296            incarnation,
11297            prefix,
11298            entries
11299                .iter()
11300                .cloned()
11301                .map(|entry| (entry, Some(TierDeleteJournalState::Dispatched)))
11302                .collect(),
11303            TierDeleteDispatchManifestState::DispatchAuthorized,
11304        )
11305        .await
11306        .expect("dispatch completion concurrency fixture should persist");
11307
11308        let lifecycle_guard = store
11309            .acquire_bucket_lifecycle_write_lock(bucket)
11310            .await
11311            .expect("dispatch completion should acquire the bucket lifecycle fence");
11312        let mut fence_opts = ObjectOptions::default();
11313        fence_opts.add_bucket_lifecycle_lock_guard(&lifecycle_guard);
11314        let bucket_fence = fence_opts
11315            .bucket_lifecycle_lock_fence
11316            .clone()
11317            .expect("dispatch completion should capture the bucket lifecycle fence");
11318        let fleet_proof =
11319            acquire_tier_delete_journal_fleet_proof().expect("dispatch completion concurrency fixture should have a fleet proof");
11320        let prepared =
11321            prepare_tier_delete_dispatch(store.clone(), bucket, incarnation, prefix, entries, fleet_proof, &bucket_fence)
11322                .await
11323                .expect("the existing Authorized dispatch should reload");
11324        let active = prepared
11325            .consume(bucket, incarnation, prefix)
11326            .expect("the existing Authorized dispatch permit should be consumable");
11327        active
11328            .authorization()
11329            .mark_mutation_started(bucket, incarnation, prefix)
11330            .expect("the completion test should mark its synthetic local mutation");
11331
11332        let hook = TierDeleteDispatchMemberReadTestHook::install_pause(TierDeleteDispatchMemberReadTestStage::Completion);
11333        let worker_store = store.clone();
11334        let worker = tokio::spawn(async move { complete_tier_delete_dispatch(worker_store, &active, &bucket_fence).await });
11335        tokio::time::timeout(Duration::from_secs(30), hook.wait_until_read_pause_count(ADMITTED_COMMITS))
11336            .await
11337            .expect("the first completion CAS window should be admitted");
11338        tokio::time::sleep(Duration::from_millis(50)).await;
11339        assert_eq!(hook.entry_count(), ADMITTED_COMMITS, "completion must not admit an unbounded CAS tail");
11340        assert_eq!(hook.in_flight(), ADMITTED_COMMITS);
11341
11342        hook.release_all_reads();
11343        worker
11344            .await
11345            .expect("dispatch completion concurrency worker should join")
11346            .expect("every bounded completion CAS should succeed");
11347        assert_eq!(
11348            hook.entry_count(),
11349            JOURNAL_COUNT,
11350            "completion must eventually admit the entire journal set"
11351        );
11352        assert_eq!(hook.in_flight(), 0);
11353        assert_eq!(
11354            test_tier_delete_dispatch_manifest_state(store.clone(), &manifest_name)
11355                .await
11356                .expect("completed concurrency manifest should remain readable"),
11357            Some(TierDeleteDispatchManifestState::Completed)
11358        );
11359        drop(hook);
11360        shutdown.cancel();
11361    }
11362
11363    #[cfg(feature = "test-util")]
11364    #[tokio::test]
11365    #[serial_test::serial(storage_class_env)]
11366    async fn dispatch_manifest_shutdown_stops_completed_missing_journal_tail_reads() {
11367        const JOURNAL_COUNT: usize = 65;
11368        const ADMITTED_READS: usize = 32;
11369
11370        let temp_dir = tempfile::tempdir().expect("create completed-read shutdown store dir");
11371        let (ctx, store, shutdown) =
11372            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "dispatch-completed-read-shutdown", &[4])).await;
11373        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
11374        let bucket = "dispatch-completed-read-shutdown-bucket";
11375        store
11376            .make_bucket(bucket, &MakeBucketOptions::default())
11377            .await
11378            .expect("completed-read shutdown bucket should be created");
11379        let incarnation = store
11380            .bucket_incarnation_id(bucket)
11381            .await
11382            .expect("completed-read shutdown bucket incarnation should resolve");
11383        let tier_name = "COMPLETED-READ-SHUTDOWN";
11384        let _backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
11385        let identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
11386            .await
11387            .expect("completed-read shutdown tier lease should resolve")
11388            .backend_identity();
11389        let entries = (0..JOURNAL_COUNT)
11390            .map(|index| {
11391                (
11392                    synthetic_v6_dispatch_entry(
11393                        bucket,
11394                        &format!("archive/{index:06}.bin"),
11395                        tier_name,
11396                        identity,
11397                        &uuid::Uuid::new_v4().to_string(),
11398                    ),
11399                    None,
11400                )
11401            })
11402            .collect();
11403        let (manifest_name, _) = install_test_tier_delete_dispatch_fixture(
11404            store.clone(),
11405            bucket,
11406            incarnation,
11407            "archive/",
11408            entries,
11409            TierDeleteDispatchManifestState::Completed,
11410        )
11411        .await
11412        .expect("completed-read shutdown fixture should persist");
11413        let hook = TierDeleteDispatchMemberReadTestHook::install_pause(TierDeleteDispatchMemberReadTestStage::Completed);
11414        let worker_store = store.clone();
11415        let worker_manifest = manifest_name.clone();
11416        let worker = tokio::spawn(async move {
11417            recover_test_tier_delete_dispatch_manifest_with_page_budget(worker_store, &worker_manifest, Duration::from_secs(30))
11418                .await
11419        });
11420        tokio::time::timeout(Duration::from_secs(30), hook.wait_until_read_pause_count(ADMITTED_READS))
11421            .await
11422            .expect("the first Completed read window should be admitted");
11423
11424        shutdown.cancel();
11425        tokio::time::sleep(Duration::from_millis(50)).await;
11426        assert_eq!(hook.entry_count(), ADMITTED_READS, "shutdown must not admit later Completed chunks");
11427        assert_eq!(hook.in_flight(), ADMITTED_READS);
11428        assert_eq!(tier_delete_dispatch_manifest_recovery_count_for_test(&store), 1);
11429        assert!(tier_delete_dispatch_manifest_operation_lock_held_for_test(store.clone(), &manifest_name).await);
11430        assert!(crate::services::notification_sys::tier_delete_journal_fleet_proof_has_inflight_for_test());
11431
11432        hook.release_all_reads();
11433        assert!(
11434            !worker.await.expect("completed-read shutdown worker should join"),
11435            "cooperative shutdown should retain a partially scanned Completed manifest"
11436        );
11437        assert_eq!(hook.entry_count(), ADMITTED_READS);
11438        assert_eq!(hook.in_flight(), 0);
11439        assert_eq!(tier_delete_dispatch_manifest_recovery_count_for_test(&store), 0);
11440        assert_eq!(tier_delete_journal_count(store.clone()).await, 0);
11441        assert_eq!(
11442            tier_delete_dispatch_manifest_count(store.clone()).await,
11443            1,
11444            "manifest DELETE must not run"
11445        );
11446        assert_eq!(
11447            test_tier_delete_dispatch_manifest_state(store.clone(), &manifest_name)
11448                .await
11449                .expect("retained Completed manifest should remain readable"),
11450            Some(TierDeleteDispatchManifestState::Completed)
11451        );
11452        drop(hook);
11453        drop(store);
11454        drop(ctx);
11455
11456        let (_restarted_ctx, restarted_store, restarted_shutdown) = without_storage_class_env(build_isolated_test_store(
11457            temp_dir.path(),
11458            "dispatch-completed-read-shutdown-restart",
11459            &[4],
11460        ))
11461        .await;
11462        crate::bucket::metadata_sys::init_bucket_metadata_sys(restarted_store.clone(), Vec::new()).await;
11463        drive_tier_delete_dispatch_restart_to_convergence(restarted_store.clone()).await;
11464        restarted_shutdown.cancel();
11465    }
11466
11467    #[cfg(feature = "test-util")]
11468    #[tokio::test]
11469    #[serial_test::serial(storage_class_env)]
11470    async fn dispatch_manifest_shutdown_drains_preparing_cas_before_releasing_fences() {
11471        let temp_dir = tempfile::tempdir().expect("create preparing CAS shutdown store dir");
11472        let (ctx, store, shutdown) =
11473            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "dispatch-preparing-cas-shutdown", &[4])).await;
11474        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
11475        let bucket = "dispatch-preparing-cas-shutdown-bucket";
11476        store
11477            .make_bucket(bucket, &MakeBucketOptions::default())
11478            .await
11479            .expect("preparing CAS shutdown bucket should be created");
11480        let incarnation = store
11481            .bucket_incarnation_id(bucket)
11482            .await
11483            .expect("preparing CAS shutdown bucket incarnation should resolve");
11484        let entry = synthetic_v6_dispatch_entry(
11485            bucket,
11486            "archive/preparing-cas.bin",
11487            "PREPARING-CAS-SHUTDOWN",
11488            [31; 32],
11489            &uuid::Uuid::new_v4().to_string(),
11490        );
11491        let (manifest_name, _) = install_test_tier_delete_dispatch_fixture(
11492            store.clone(),
11493            bucket,
11494            incarnation,
11495            "archive/",
11496            vec![(entry, Some(TierDeleteJournalState::Prepared))],
11497            TierDeleteDispatchManifestState::Preparing,
11498        )
11499        .await
11500        .expect("preparing CAS shutdown fixture should persist");
11501        let barrier = crate::set_disk::PutObjectCommitBarrier::install(
11502            RUSTFS_META_BUCKET,
11503            &manifest_name,
11504            crate::set_disk::PutObjectCommitPause::BeforeQuotaRename,
11505        );
11506        let worker_store = store.clone();
11507        let worker_manifest = manifest_name.clone();
11508        let worker = tokio::spawn(async move {
11509            recover_test_tier_delete_dispatch_manifest_with_page_budget(worker_store, &worker_manifest, Duration::from_secs(30))
11510                .await
11511        });
11512        barrier.wait_until_paused().await;
11513
11514        shutdown.cancel();
11515        tokio::time::sleep(Duration::from_millis(50)).await;
11516        assert_eq!(
11517            tier_delete_dispatch_manifest_recovery_count_for_test(&store),
11518            1,
11519            "a paused CAS must retain its registry permit after shutdown"
11520        );
11521        assert!(
11522            tier_delete_dispatch_manifest_operation_lock_held_for_test(store.clone(), &manifest_name).await,
11523            "a paused CAS must retain its manifest operation lock after shutdown"
11524        );
11525        assert!(
11526            crate::services::notification_sys::tier_delete_journal_fleet_proof_has_inflight_for_test(),
11527            "a paused CAS must retain its fleet-proof generation permit"
11528        );
11529
11530        barrier.release();
11531        assert!(
11532            !worker.await.expect("preparing CAS shutdown worker should join"),
11533            "cooperative shutdown should retain the partially advanced manifest"
11534        );
11535        assert_eq!(tier_delete_dispatch_manifest_recovery_count_for_test(&store), 0);
11536        assert!(
11537            !tier_delete_dispatch_manifest_operation_lock_held_for_test(store.clone(), &manifest_name).await,
11538            "the operation lock may release only after the admitted CAS drains"
11539        );
11540        assert!(
11541            com::read_config(store.clone(), &manifest_name).await.is_ok(),
11542            "shutdown must retain the durable manifest for restart replay"
11543        );
11544        drop(barrier);
11545        drop(store);
11546        drop(ctx);
11547
11548        let (_restarted_ctx, restarted_store, restarted_shutdown) = without_storage_class_env(build_isolated_test_store(
11549            temp_dir.path(),
11550            "dispatch-preparing-cas-shutdown-restart",
11551            &[4],
11552        ))
11553        .await;
11554        crate::bucket::metadata_sys::init_bucket_metadata_sys(restarted_store.clone(), Vec::new()).await;
11555        tokio::time::timeout(Duration::from_secs(30), async {
11556            let mut stable_empty_observations = 0;
11557            loop {
11558                if tier_delete_journal_count(restarted_store.clone()).await == 0
11559                    && tier_delete_dispatch_manifest_count(restarted_store.clone()).await == 0
11560                    && tier_delete_dispatch_manifest_recovery_count_for_test(&restarted_store) == 0
11561                {
11562                    stable_empty_observations += 1;
11563                    if stable_empty_observations == 10 {
11564                        break;
11565                    }
11566                } else {
11567                    stable_empty_observations = 0;
11568                }
11569                tokio::time::sleep(Duration::from_millis(50)).await;
11570            }
11571        })
11572        .await
11573        .expect("the restarted background recovery should replay the drained Preparing CAS");
11574        assert_eq!(tier_delete_journal_count(restarted_store.clone()).await, 0);
11575        assert_eq!(tier_delete_dispatch_manifest_count(restarted_store).await, 0);
11576        restarted_shutdown.cancel();
11577    }
11578
11579    #[cfg(feature = "test-util")]
11580    #[tokio::test]
11581    #[serial_test::serial(storage_class_env)]
11582    async fn dispatch_manifest_shutdown_drains_admitted_delete_batch_without_tail_admission() {
11583        const JOURNAL_COUNT: usize = 65;
11584        const ADMITTED_BATCH: usize = 32;
11585
11586        let temp_dir = tempfile::tempdir().expect("create delete-batch shutdown store dir");
11587        let (ctx, store, shutdown) =
11588            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "dispatch-delete-batch-shutdown", &[4])).await;
11589        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
11590        let bucket = "dispatch-delete-batch-shutdown-bucket";
11591        store
11592            .make_bucket(bucket, &MakeBucketOptions::default())
11593            .await
11594            .expect("delete-batch shutdown bucket should be created");
11595        let incarnation = store
11596            .bucket_incarnation_id(bucket)
11597            .await
11598            .expect("delete-batch shutdown bucket incarnation should resolve");
11599        let (manifest_name, entries) = install_aborting_dispatch_fixture(
11600            store.clone(),
11601            bucket,
11602            incarnation,
11603            "archive/",
11604            "DELETE-BATCH-SHUTDOWN",
11605            [32; 32],
11606            JOURNAL_COUNT,
11607        )
11608        .await;
11609        let mut journal_names = entries.iter().map(tier_delete_journal_object_name).collect::<Vec<_>>();
11610        journal_names.sort();
11611        let published_delete_name = journal_names[0].clone();
11612        let hook = TierDeleteDispatchRollbackTestHook::install_pause_all_except_delete(&published_delete_name);
11613        let barrier =
11614            crate::set_disk::DeleteObjectCommitBarrier::install_after_publish(RUSTFS_META_BUCKET, &published_delete_name);
11615        let worker_store = store.clone();
11616        let worker_manifest = manifest_name.clone();
11617        let worker = tokio::spawn(async move {
11618            recover_test_tier_delete_dispatch_manifest_with_page_budget(worker_store, &worker_manifest, Duration::from_secs(30))
11619                .await
11620        });
11621        barrier.wait_until_paused().await;
11622        tokio::time::timeout(Duration::from_secs(30), hook.wait_until_delete_pause_count(ADMITTED_BATCH - 1))
11623            .await
11624            .expect("the rest of the bounded delete batch should pause before mutation");
11625        assert_eq!(hook.delete_entry_count(), ADMITTED_BATCH);
11626
11627        shutdown.cancel();
11628        tokio::time::sleep(Duration::from_millis(50)).await;
11629        assert_eq!(tier_delete_dispatch_manifest_recovery_count_for_test(&store), 1);
11630        assert!(
11631            tier_delete_dispatch_manifest_operation_lock_held_for_test(store.clone(), &manifest_name).await,
11632            "the partially published DELETE must retain its operation lock"
11633        );
11634        assert!(
11635            crate::services::notification_sys::tier_delete_journal_fleet_proof_has_inflight_for_test(),
11636            "the partially published DELETE must retain its fleet-proof permit"
11637        );
11638        assert_eq!(
11639            hook.delete_entry_count(),
11640            ADMITTED_BATCH,
11641            "shutdown must stop admitting the tail behind the bounded batch"
11642        );
11643
11644        barrier.release();
11645        tokio::time::sleep(Duration::from_millis(50)).await;
11646        assert_eq!(
11647            tier_delete_dispatch_manifest_recovery_count_for_test(&store),
11648            1,
11649            "the worker must retain its guards until every admitted sibling drains"
11650        );
11651        hook.release_all_deletes();
11652        assert!(
11653            !worker.await.expect("delete-batch shutdown worker should join"),
11654            "cooperative shutdown should retain the partial rollback for restart"
11655        );
11656        assert_eq!(hook.delete_entry_count(), ADMITTED_BATCH);
11657        assert_eq!(tier_delete_dispatch_manifest_recovery_count_for_test(&store), 0);
11658        assert!(
11659            tier_delete_journal_count(store.clone()).await >= JOURNAL_COUNT - 1,
11660            "only the already-published DELETE may commit before restart"
11661        );
11662        drop(barrier);
11663        drop(hook);
11664        drop(store);
11665        drop(ctx);
11666
11667        let (_restarted_ctx, restarted_store, restarted_shutdown) =
11668            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "dispatch-delete-batch-shutdown-restart", &[4]))
11669                .await;
11670        crate::bucket::metadata_sys::init_bucket_metadata_sys(restarted_store.clone(), Vec::new()).await;
11671        tokio::time::timeout(Duration::from_secs(30), async {
11672            let mut stable_empty_observations = 0;
11673            loop {
11674                if tier_delete_journal_count(restarted_store.clone()).await == 0
11675                    && tier_delete_dispatch_manifest_count(restarted_store.clone()).await == 0
11676                    && tier_delete_dispatch_manifest_recovery_count_for_test(&restarted_store) == 0
11677                {
11678                    stable_empty_observations += 1;
11679                    if stable_empty_observations == 10 {
11680                        break;
11681                    }
11682                } else {
11683                    stable_empty_observations = 0;
11684                }
11685                tokio::time::sleep(Duration::from_millis(50)).await;
11686            }
11687        })
11688        .await
11689        .expect("the restarted background recovery should replay the partially drained delete batch");
11690        assert_eq!(tier_delete_journal_count(restarted_store.clone()).await, 0);
11691        assert_eq!(tier_delete_dispatch_manifest_count(restarted_store).await, 0);
11692        restarted_shutdown.cancel();
11693    }
11694
11695    #[cfg(feature = "test-util")]
11696    #[tokio::test]
11697    #[serial_test::serial(storage_class_env)]
11698    async fn dispatch_manifest_rollback_retries_delete_and_confirmation_failures() {
11699        const JOURNAL_COUNT: usize = 40;
11700
11701        let temp_dir = tempfile::tempdir().expect("create rollback retry store dir");
11702        // Manual retries must own progress between fault removal and the next attempt.
11703        let mut instance_ctx = crate::runtime::instance::InstanceContext::new();
11704        instance_ctx.suppress_tier_delete_journal_recovery_for_test();
11705        let (ctx, store, shutdown) = without_storage_class_env(build_isolated_test_store_with_layout(
11706            temp_dir.path(),
11707            "dispatch-rollback-retry",
11708            &[(1, 4)],
11709            CancellationToken::new(),
11710            Some(Arc::new(instance_ctx)),
11711        ))
11712        .await;
11713        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
11714        let bucket = "dispatch-rollback-retry-bucket";
11715        store
11716            .make_bucket(bucket, &MakeBucketOptions::default())
11717            .await
11718            .expect("rollback retry bucket should be created");
11719        let incarnation = store
11720            .bucket_incarnation_id(bucket)
11721            .await
11722            .expect("rollback retry bucket incarnation should resolve");
11723        let tier_name = "DISPATCH-ROLLBACK-RETRY";
11724        let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
11725        let identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
11726            .await
11727            .expect("rollback retry tier lease should resolve")
11728            .backend_identity();
11729
11730        let (delete_manifest, delete_entries) = install_aborting_dispatch_fixture(
11731            store.clone(),
11732            bucket,
11733            incarnation,
11734            "delete-failure/",
11735            tier_name,
11736            identity,
11737            JOURNAL_COUNT,
11738        )
11739        .await;
11740        let delete_failure_name = tier_delete_journal_object_name(&delete_entries[10]);
11741        let delete_hook = TierDeleteDispatchRollbackTestHook::install_delete_failure(&delete_failure_name);
11742        let delete_error = recover_test_tier_delete_dispatch_manifest(store.clone(), &delete_manifest)
11743            .await
11744            .expect_err("an injected member delete failure must retain the manifest")
11745            .to_string();
11746        assert!(delete_error.contains("injected tier delete dispatch rollback delete failure"));
11747        assert!(
11748            tier_delete_journal_count(store.clone()).await > 0,
11749            "the failed member and all work not admitted after the first error must remain retryable"
11750        );
11751        drop(delete_hook);
11752        recover_test_tier_delete_dispatch_manifest(store.clone(), &delete_manifest)
11753            .await
11754            .expect("the delete-stage retry should converge");
11755
11756        let (confirmation_manifest, confirmation_entries) = install_aborting_dispatch_fixture(
11757            store.clone(),
11758            bucket,
11759            incarnation,
11760            "confirmation-failure/",
11761            tier_name,
11762            identity,
11763            JOURNAL_COUNT,
11764        )
11765        .await;
11766        let confirmation_failure_name = tier_delete_journal_object_name(&confirmation_entries[10]);
11767        let confirmation_hook = TierDeleteDispatchRollbackTestHook::install_confirmation_failure(&confirmation_failure_name);
11768        let confirmation_error = recover_test_tier_delete_dispatch_manifest(store.clone(), &confirmation_manifest)
11769            .await
11770            .expect_err("an injected confirmation failure must retain the manifest")
11771            .to_string();
11772        assert!(confirmation_error.contains("injected tier delete dispatch rollback confirmation failure"));
11773        assert_eq!(
11774            tier_delete_journal_count(store.clone()).await,
11775            0,
11776            "confirmation failure occurs only after the bounded delete phase drains"
11777        );
11778        assert_eq!(tier_delete_dispatch_manifest_count(store.clone()).await, 1);
11779        drop(confirmation_hook);
11780        recover_test_tier_delete_dispatch_manifest(store.clone(), &confirmation_manifest)
11781            .await
11782            .expect("the confirmation-stage retry should converge over missing members");
11783
11784        assert_eq!(tier_delete_dispatch_manifest_count(store.clone()).await, 0);
11785        assert_eq!(backend.remove_count().await, 0, "rollback retries must never call the remote tier");
11786        shutdown.cancel();
11787    }
11788
11789    #[cfg(feature = "test-util")]
11790    #[test]
11791    #[serial_test::serial(storage_class_env)]
11792    fn v6_decommission_checkpoint_no_lock_put_rejects_lost_publication_fence() {
11793        run_large_stack_async_test(
11794            "v6-checkpoint-fence-loss",
11795            v6_decommission_checkpoint_no_lock_put_rejects_lost_publication_fence_case,
11796        );
11797    }
11798
11799    #[cfg(feature = "test-util")]
11800    async fn v6_decommission_checkpoint_no_lock_put_rejects_lost_publication_fence_case() {
11801        let temp_dir = tempfile::tempdir().expect("create v6 checkpoint fence-loss store dir");
11802        let (_ctx, store, _shutdown) =
11803            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "v6-checkpoint-fence-loss", &[4, 4])).await;
11804        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
11805        let bucket = "v6-checkpoint-fence-loss-bucket";
11806        store
11807            .make_bucket(bucket, &MakeBucketOptions::default())
11808            .await
11809            .expect("checkpoint fence-loss bucket should be created");
11810        let incarnation = store
11811            .bucket_incarnation_id(bucket)
11812            .await
11813            .expect("checkpoint fence-loss bucket incarnation should resolve");
11814        let entry = synthetic_v6_dispatch_entry(
11815            bucket,
11816            "archive/fence-loss.bin",
11817            "FENCE-LOSS-TIER",
11818            [18; 32],
11819            &uuid::Uuid::new_v4().to_string(),
11820        );
11821        let (manifest_name, entries) = install_test_tier_delete_dispatch_fixture(
11822            store.clone(),
11823            bucket,
11824            incarnation,
11825            "archive/",
11826            vec![(entry, Some(TierDeleteJournalState::Prepared))],
11827            TierDeleteDispatchManifestState::Preparing,
11828        )
11829        .await
11830        .expect("checkpoint fence-loss fixture should persist");
11831        let journal_name = tier_delete_journal_object_name(&entries[0]);
11832        let manifest_data = com::read_config(store.clone(), &manifest_name)
11833            .await
11834            .expect("checkpoint fence-loss manifest should be readable");
11835        let journal_data = com::read_config(store.clone(), &journal_name)
11836            .await
11837            .expect("checkpoint fence-loss journal should be readable");
11838        for pool in &store.pools {
11839            com::save_config(pool.clone(), &manifest_name, manifest_data.clone())
11840                .await
11841                .expect("manifest copy should persist in each checkpoint fence-loss pool");
11842            com::save_config(pool.clone(), &journal_name, journal_data.clone())
11843                .await
11844                .expect("journal copy should persist in each checkpoint fence-loss pool");
11845        }
11846        mark_test_pool_decommissioning(&store, 0).await;
11847        for (path, data) in [
11848            (&manifest_name, manifest_data.as_slice()),
11849            (&journal_name, journal_data.as_slice()),
11850        ] {
11851            let record = validate_durable_ilm_record(path, data).expect("checkpoint fence-loss v6 record should validate");
11852            store
11853                .persist_decommission_durable_ilm_receipt_for_test(0, 1, path, &record, false)
11854                .await
11855                .expect("checkpoint fence-loss receipt should persist");
11856        }
11857
11858        let loss_hook = crate::store::object::DecommissionMutationFenceLossHook::install(
11859            RUSTFS_META_BUCKET,
11860            &manifest_name,
11861            crate::store::object::DecommissionMutationFenceTestPhase::Migration,
11862        );
11863        let barrier = crate::set_disk::PutObjectCommitBarrier::install(
11864            RUSTFS_META_BUCKET,
11865            &manifest_name,
11866            crate::set_disk::PutObjectCommitPause::BeforeQuotaRename,
11867        );
11868        let worker_store = store.clone();
11869        let worker_manifest = manifest_name.clone();
11870        let worker =
11871            tokio::spawn(async move { recover_test_tier_delete_dispatch_manifest(worker_store, &worker_manifest).await });
11872        barrier.wait_until_paused().await;
11873        loss_hook.mark_lost();
11874        barrier.release();
11875        drop(barrier);
11876        let error = worker
11877            .await
11878            .expect("checkpoint fence-loss recovery task should join")
11879            .expect_err("lost publication fence must reject the no-lock checkpoint PUT")
11880            .to_string();
11881        assert!(
11882            error.contains("lock") || error.contains("fence"),
11883            "unexpected checkpoint fence-loss error: {error}"
11884        );
11885        assert_eq!(
11886            com::read_config(store.pools[1].clone(), &manifest_name)
11887                .await
11888                .expect("fenced target manifest should remain readable"),
11889            manifest_data,
11890            "the target checkpoint must not commit after its publication fence is lost"
11891        );
11892
11893        drop(loss_hook);
11894        recover_test_tier_delete_dispatch_manifest(store.clone(), &manifest_name)
11895            .await
11896            .expect("checkpoint recovery should converge after a fresh fence is acquired");
11897    }
11898
11899    #[cfg(feature = "test-util")]
11900    #[tokio::test]
11901    #[serial_test::serial(storage_class_env)]
11902    async fn v6_decommission_checkpoint_partial_target_failure_retries_without_advancing_receipts() {
11903        let temp_dir = tempfile::tempdir().expect("create v6 partial checkpoint store dir");
11904        let (_ctx, store, _shutdown) =
11905            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "v6-partial-checkpoint", &[4, 4, 4])).await;
11906        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
11907        let bucket = "v6-partial-checkpoint-bucket";
11908        store
11909            .make_bucket(bucket, &MakeBucketOptions::default())
11910            .await
11911            .expect("partial checkpoint bucket should be created");
11912        let incarnation = store
11913            .bucket_incarnation_id(bucket)
11914            .await
11915            .expect("partial checkpoint bucket incarnation should resolve");
11916        let entry = synthetic_v6_dispatch_entry(
11917            bucket,
11918            "archive/partial.bin",
11919            "PARTIAL-CHECKPOINT-TIER",
11920            [19; 32],
11921            &uuid::Uuid::new_v4().to_string(),
11922        );
11923        let (manifest_name, entries) = install_test_tier_delete_dispatch_fixture(
11924            store.clone(),
11925            bucket,
11926            incarnation,
11927            "archive/",
11928            vec![(entry, Some(TierDeleteJournalState::Prepared))],
11929            TierDeleteDispatchManifestState::Preparing,
11930        )
11931        .await
11932        .expect("partial checkpoint fixture should persist");
11933        let journal_name = tier_delete_journal_object_name(&entries[0]);
11934        let manifest_data = com::read_config(store.clone(), &manifest_name)
11935            .await
11936            .expect("partial checkpoint manifest should be readable");
11937        let journal_data = com::read_config(store.clone(), &journal_name)
11938            .await
11939            .expect("partial checkpoint journal should be readable");
11940        for pool in &store.pools {
11941            com::save_config(pool.clone(), &manifest_name, manifest_data.clone())
11942                .await
11943                .expect("manifest copy should persist in each partial checkpoint pool");
11944            com::save_config(pool.clone(), &journal_name, journal_data.clone())
11945                .await
11946                .expect("journal copy should persist in each partial checkpoint pool");
11947        }
11948        mark_test_pool_decommissioning_with_split_targets(&store, 0).await;
11949        for target_pool_index in [1, 2] {
11950            for (path, data) in [
11951                (&manifest_name, manifest_data.as_slice()),
11952                (&journal_name, journal_data.as_slice()),
11953            ] {
11954                let record = validate_durable_ilm_record(path, data).expect("partial checkpoint v6 record should validate");
11955                store
11956                    .persist_decommission_durable_ilm_receipt_for_test(0, target_pool_index, path, &record, false)
11957                    .await
11958                    .expect("partial checkpoint receipt should persist on each target");
11959            }
11960        }
11961        let (aborting_data, preparing_etag) = test_tier_delete_dispatch_manifest_checkpoint(
11962            store.clone(),
11963            &manifest_name,
11964            TierDeleteDispatchManifestState::Aborting,
11965        )
11966        .await
11967        .expect("partial checkpoint Aborting generation should encode");
11968        let targets = store
11969            .decommission_durable_ilm_checkpoint_targets(&manifest_name, &aborting_data, &preparing_etag)
11970            .await
11971            .expect("partial checkpoint targets should resolve")
11972            .expect("the active decommission should own the partial checkpoint");
11973        assert_eq!(
11974            targets.iter().map(|target| target.target_pool_index).collect::<Vec<_>>(),
11975            vec![1, 2],
11976            "the capacity reservation must exercise two independently confirmed targets"
11977        );
11978
11979        let failure_hook = DecommissionCheckpointTargetFailureHook::install(2);
11980        let error = recover_test_tier_delete_dispatch_manifest(store.clone(), &manifest_name)
11981            .await
11982            .expect_err("the injected second-target failure must reject the checkpoint");
11983        assert!(
11984            error.to_string().contains("injected decommission checkpoint target failure"),
11985            "unexpected partial checkpoint error: {error}"
11986        );
11987        let mut diagnostic = &error as &(dyn std::error::Error + 'static);
11988        let mut target_context_found = false;
11989        while let Some(source) = diagnostic.source() {
11990            if source.to_string().contains("target pool 2") {
11991                target_context_found = true;
11992                break;
11993            }
11994            diagnostic = source;
11995        }
11996        assert!(
11997            target_context_found,
11998            "the stable checkpoint error must retain the failed target in its diagnostic source chain: {error:?}"
11999        );
12000        assert_eq!(
12001            com::read_config(store.pools[1].clone(), &manifest_name)
12002                .await
12003                .expect("first target checkpoint should be readable"),
12004            aborting_data,
12005            "the first target should model a committed checkpoint before the later failure"
12006        );
12007        assert_eq!(
12008            com::read_config(store.pools[2].clone(), &manifest_name)
12009                .await
12010                .expect("second target checkpoint should be readable"),
12011            manifest_data,
12012            "the failed second target must retain the prior generation"
12013        );
12014        let retry_targets = store
12015            .decommission_durable_ilm_checkpoint_targets(&manifest_name, &aborting_data, &preparing_etag)
12016            .await
12017            .expect("unadvanced receipts must still authorize the exact partial checkpoint retry")
12018            .expect("the partial checkpoint retry should remain decommission-owned");
12019        assert!(retry_targets[0].already_committed);
12020        assert!(!retry_targets[1].already_committed);
12021
12022        drop(failure_hook);
12023        recover_test_tier_delete_dispatch_manifest(store.clone(), &manifest_name)
12024            .await
12025            .expect("the partial checkpoint should converge after the target recovers");
12026    }
12027
12028    #[cfg(feature = "test-util")]
12029    #[tokio::test]
12030    #[serial_test::serial(storage_class_env)]
12031    async fn v6_decommission_checkpoint_shutdown_drains_target_put_and_replays_capacity() {
12032        let temp_dir = tempfile::tempdir().expect("create checkpoint shutdown store dir");
12033        let (ctx, store, shutdown) =
12034            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "v6-checkpoint-shutdown", &[4, 4])).await;
12035        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
12036        let bucket = "v6-checkpoint-shutdown-bucket";
12037        store
12038            .make_bucket(bucket, &MakeBucketOptions::default())
12039            .await
12040            .expect("checkpoint shutdown bucket should be created");
12041        let incarnation = store
12042            .bucket_incarnation_id(bucket)
12043            .await
12044            .expect("checkpoint shutdown bucket incarnation should resolve");
12045        let entry = synthetic_v6_dispatch_entry(
12046            bucket,
12047            "archive/checkpoint-shutdown.bin",
12048            "CHECKPOINT-SHUTDOWN",
12049            [33; 32],
12050            &uuid::Uuid::new_v4().to_string(),
12051        );
12052        let (manifest_name, entries) = install_test_tier_delete_dispatch_fixture(
12053            store.clone(),
12054            bucket,
12055            incarnation,
12056            "archive/",
12057            vec![(entry, Some(TierDeleteJournalState::Prepared))],
12058            TierDeleteDispatchManifestState::Preparing,
12059        )
12060        .await
12061        .expect("checkpoint shutdown fixture should persist");
12062        let journal_name = tier_delete_journal_object_name(&entries[0]);
12063        let manifest_data = com::read_config(store.clone(), &manifest_name)
12064            .await
12065            .expect("checkpoint shutdown manifest should be readable");
12066        let journal_data = com::read_config(store.clone(), &journal_name)
12067            .await
12068            .expect("checkpoint shutdown journal should be readable");
12069        for pool in &store.pools {
12070            com::save_config(pool.clone(), &manifest_name, manifest_data.clone())
12071                .await
12072                .expect("manifest copy should persist in each checkpoint shutdown pool");
12073            com::save_config(pool.clone(), &journal_name, journal_data.clone())
12074                .await
12075                .expect("journal copy should persist in each checkpoint shutdown pool");
12076        }
12077        mark_test_pool_decommissioning(&store, 0).await;
12078        for (path, data) in [
12079            (&manifest_name, manifest_data.as_slice()),
12080            (&journal_name, journal_data.as_slice()),
12081        ] {
12082            let record = validate_durable_ilm_record(path, data).expect("checkpoint shutdown v6 record should validate");
12083            store
12084                .persist_decommission_durable_ilm_receipt_for_test(0, 1, path, &record, false)
12085                .await
12086                .expect("checkpoint shutdown receipt should persist");
12087        }
12088        let (aborting_data, preparing_etag) = test_tier_delete_dispatch_manifest_checkpoint(
12089            store.clone(),
12090            &manifest_name,
12091            TierDeleteDispatchManifestState::Aborting,
12092        )
12093        .await
12094        .expect("checkpoint shutdown Aborting generation should encode");
12095        let targets = store
12096            .decommission_durable_ilm_checkpoint_targets(&manifest_name, &aborting_data, &preparing_etag)
12097            .await
12098            .expect("checkpoint shutdown target should resolve")
12099            .expect("the active decommission should own the checkpoint target");
12100        assert_eq!(targets.len(), 1);
12101        let target = targets[0].clone();
12102        let consumed_before_checkpoint = store.pool_meta.read().await.pools[0]
12103            .decommission
12104            .as_ref()
12105            .and_then(|info| info.capacity_reservation.as_ref())
12106            .expect("checkpoint shutdown capacity reservation should exist")
12107            .consumed_target_physical_bytes;
12108        let precommit_error = store
12109            .run_decommission_capacity_non_growing_replacement_with_capacity_lease(
12110                target.target_pool_index,
12111                Some(target.capacity_owner),
12112                Some(aborting_data.len()),
12113                |_| async { Err::<(), Error>(Error::other("injected checkpoint failure before commit")) },
12114            )
12115            .await
12116            .expect_err("the first checkpoint attempt should fail before writing its target")
12117            .to_string();
12118        assert!(precommit_error.contains("injected checkpoint failure before commit"));
12119        assert!(
12120            store
12121                .has_decommission_capacity_temporary_mutation_state(target.target_pool_index, target.capacity_owner)
12122                .await,
12123            "a failed checkpoint attempt must retain exact retry state"
12124        );
12125        let barrier = crate::set_disk::PutObjectCommitBarrier::install(
12126            RUSTFS_META_BUCKET,
12127            &manifest_name,
12128            crate::set_disk::PutObjectCommitPause::BeforeQuotaRename,
12129        );
12130        let worker_store = store.clone();
12131        let worker_manifest = manifest_name.clone();
12132        let worker = tokio::spawn(async move {
12133            recover_test_tier_delete_dispatch_manifest_with_page_budget(worker_store, &worker_manifest, Duration::from_secs(30))
12134                .await
12135        });
12136        barrier.wait_until_paused().await;
12137        assert!(
12138            store
12139                .has_decommission_capacity_temporary_mutation_state(target.target_pool_index, target.capacity_owner)
12140                .await,
12141            "the target PUT must persist its capacity intent before commit"
12142        );
12143
12144        shutdown.cancel();
12145        tokio::time::sleep(Duration::from_millis(50)).await;
12146        assert_eq!(tier_delete_dispatch_manifest_recovery_count_for_test(&store), 1);
12147        assert!(
12148            tier_delete_dispatch_manifest_operation_lock_held_for_test(store.clone(), &manifest_name).await,
12149            "the target checkpoint PUT must retain its operation lock after shutdown"
12150        );
12151        assert!(
12152            crate::services::notification_sys::tier_delete_journal_fleet_proof_has_inflight_for_test(),
12153            "the target checkpoint PUT must retain its fleet-proof permit after shutdown"
12154        );
12155
12156        barrier.release();
12157        assert!(
12158            !worker.await.expect("checkpoint shutdown worker should join"),
12159            "cooperative shutdown should retain the partially committed checkpoint"
12160        );
12161        assert_eq!(tier_delete_dispatch_manifest_recovery_count_for_test(&store), 0);
12162        assert_eq!(
12163            com::read_config(store.pools[target.target_pool_index].clone(), &manifest_name)
12164                .await
12165                .expect("the committed target checkpoint should be readable"),
12166            aborting_data,
12167            "the admitted target PUT must finish before its fences release"
12168        );
12169        assert!(
12170            !store
12171                .has_decommission_capacity_temporary_mutation_state(target.target_pool_index, target.capacity_owner)
12172                .await,
12173            "the admitted target PUT must drain its capacity transaction before releasing the recovery fences"
12174        );
12175        {
12176            let pool_meta = store.pool_meta.read().await;
12177            let reservation = pool_meta.pools[0]
12178                .decommission
12179                .as_ref()
12180                .and_then(|info| info.capacity_reservation.as_ref())
12181                .expect("checkpoint shutdown capacity reservation should remain active");
12182            let capacity_target = reservation
12183                .targets
12184                .iter()
12185                .find(|candidate| candidate.pool_index == target.target_pool_index)
12186                .expect("checkpoint shutdown capacity target should remain allocated");
12187            assert_eq!(
12188                reservation.consumed_target_physical_bytes, consumed_before_checkpoint,
12189                "a non-growing checkpoint replacement must not consume durable migration capacity"
12190            );
12191            assert_eq!(reservation.pending_target_physical_bytes, 0);
12192            assert_eq!(reservation.inflight_target_physical_bytes, 0);
12193            assert_eq!(capacity_target.pending_physical_bytes, 0);
12194            assert!(capacity_target.temporary_mutations.is_empty());
12195        }
12196        drop(barrier);
12197        let mut reloaded_pool_meta = PoolMeta::default();
12198        reloaded_pool_meta
12199            .load(store.pools[0].clone(), store.pools.clone())
12200            .await
12201            .expect("checkpoint shutdown pool metadata should reload from disk");
12202        *store.pool_meta.write().await = reloaded_pool_meta;
12203        recover_test_tier_delete_dispatch_manifest(store.clone(), &manifest_name)
12204            .await
12205            .expect("a fresh recovery attempt should replay the committed checkpoint and receipt");
12206        assert!(
12207            !store
12208                .has_decommission_capacity_temporary_mutation_state(target.target_pool_index, target.capacity_owner)
12209                .await,
12210            "durable replay must reconcile the exact capacity transaction"
12211        );
12212        assert_eq!(
12213            store
12214                .decommission_durable_ilm_receipt_count_for_test(0)
12215                .await
12216                .expect("checkpoint shutdown receipts should be listable"),
12217            2,
12218            "durable replay must advance both the manifest and journal receipts"
12219        );
12220        drop(ctx);
12221    }
12222
12223    #[cfg(feature = "test-util")]
12224    #[tokio::test]
12225    #[serial_test::serial(storage_class_env)]
12226    async fn v6_dispatch_recovery_advances_decommission_receipts_and_cleans_target_copies() {
12227        let temp_dir = tempfile::tempdir().expect("create v6 decommission receipt store dir");
12228        let (_ctx, store, _shutdown) =
12229            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "v6-dispatch-decommission-receipts", &[4, 4]))
12230                .await;
12231        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
12232        let bucket = "v6-dispatch-decommission-receipts-bucket";
12233        store
12234            .make_bucket(bucket, &MakeBucketOptions::default())
12235            .await
12236            .expect("receipt bucket should be created");
12237        let incarnation = store
12238            .bucket_incarnation_id(bucket)
12239            .await
12240            .expect("receipt bucket incarnation should resolve");
12241        let entry = synthetic_v6_dispatch_entry(
12242            bucket,
12243            "archive/receipt.bin",
12244            "RECEIPT-TIER",
12245            [17; 32],
12246            &uuid::Uuid::new_v4().to_string(),
12247        );
12248        let (manifest_name, entries) = install_test_tier_delete_dispatch_fixture(
12249            store.clone(),
12250            bucket,
12251            incarnation,
12252            "archive/",
12253            vec![(entry, Some(TierDeleteJournalState::Prepared))],
12254            TierDeleteDispatchManifestState::Preparing,
12255        )
12256        .await
12257        .expect("Preparing v6 fixture should persist");
12258        let journal_name = tier_delete_journal_object_name(&entries[0]);
12259        let manifest_data = com::read_config(store.clone(), &manifest_name)
12260            .await
12261            .expect("manifest fixture should be readable");
12262        let journal_data = com::read_config(store.clone(), &journal_name)
12263            .await
12264            .expect("journal fixture should be readable");
12265
12266        // Model an in-flight decommission after both records have been copied
12267        // to its target pool but before their state machines advance.
12268        for pool in &store.pools {
12269            com::save_config(pool.clone(), &manifest_name, manifest_data.clone())
12270                .await
12271                .expect("manifest copy should persist in each pool");
12272            com::save_config(pool.clone(), &journal_name, journal_data.clone())
12273                .await
12274                .expect("journal copy should persist in each pool");
12275        }
12276        // Exercise the same durable capacity reservation and identity that a
12277        // real decommission start installs. A hand-written active flag makes
12278        // every target mutation look external and masks whether recovery is
12279        // correctly charged to the decommission owner.
12280        mark_test_pool_decommissioning(&store, 0).await;
12281
12282        let missing_receipt_error = recover_test_tier_delete_dispatch_manifest(store.clone(), &manifest_name)
12283            .await
12284            .expect_err("an active decommission without receipt coverage must block v6 checkpoint mutation")
12285            .to_string();
12286        assert!(
12287            missing_receipt_error.contains("missing receipt coverage"),
12288            "unexpected missing-receipt error: {missing_receipt_error}"
12289        );
12290        assert_eq!(
12291            com::read_config(store.pools[1].clone(), &manifest_name)
12292                .await
12293                .expect("blocked checkpoint must retain the target manifest generation"),
12294            manifest_data,
12295            "missing receipt coverage must fail before mutating a target copy"
12296        );
12297
12298        for (path, data) in [
12299            (&manifest_name, manifest_data.as_slice()),
12300            (&journal_name, journal_data.as_slice()),
12301        ] {
12302            let record = validate_durable_ilm_record(path, data).expect("v6 durable record should validate");
12303            store
12304                .persist_decommission_durable_ilm_receipt_for_test(0, 1, path, &record, false)
12305                .await
12306                .expect("initial decommission receipt should persist");
12307        }
12308
12309        // Model a crash after the target checkpoint PUT committed but before
12310        // its temporary-capacity intent was resolved. Recovery must recognize
12311        // the exact target bytes and finish that same deterministic intent
12312        // instead of treating the checkpoint as an uncharged success.
12313        let (aborting_data, preparing_etag) = test_tier_delete_dispatch_manifest_checkpoint(
12314            store.clone(),
12315            &manifest_name,
12316            TierDeleteDispatchManifestState::Aborting,
12317        )
12318        .await
12319        .expect("the Aborting checkpoint should encode");
12320        let checkpoint_targets = store
12321            .decommission_durable_ilm_checkpoint_targets(&manifest_name, &aborting_data, &preparing_etag)
12322            .await
12323            .expect("receipt-bearing checkpoint targets should resolve")
12324            .expect("an active decommission should own the checkpoint");
12325        assert_eq!(checkpoint_targets.len(), 1);
12326        let checkpoint_target = checkpoint_targets
12327            .into_iter()
12328            .next()
12329            .expect("one checkpoint target should exist");
12330        let checkpoint_owner = checkpoint_target.capacity_owner;
12331        let target_pool_index = checkpoint_target.target_pool_index;
12332        let injected_write_error = store
12333            .run_decommission_capacity_temporary_mutation_with_capacity_lease(
12334                target_pool_index,
12335                Some(checkpoint_owner),
12336                Some(aborting_data.len()),
12337                |_| {
12338                    let target_pool = store.pools[target_pool_index].clone();
12339                    let manifest_name = manifest_name.clone();
12340                    let aborting_data = aborting_data.clone();
12341                    async move {
12342                        com::save_config(target_pool, &manifest_name, aborting_data)
12343                            .await
12344                            .expect("the injected target checkpoint should commit");
12345                        Err::<(), Error>(Error::other("injected crash after checkpoint target commit"))
12346                    }
12347                },
12348            )
12349            .await
12350            .expect_err("the injected post-commit crash should leave a pending capacity intent")
12351            .to_string();
12352        assert!(injected_write_error.contains("injected crash after checkpoint target commit"));
12353        assert!(
12354            store
12355                .has_decommission_capacity_temporary_mutation_state(target_pool_index, checkpoint_owner)
12356                .await,
12357            "the committed target must retain its exact unresolved capacity intent"
12358        );
12359
12360        recover_test_tier_delete_dispatch_manifest(store.clone(), &manifest_name)
12361            .await
12362            .expect("Preparing rollback should advance receipts and finish terminal cleanup");
12363        assert!(
12364            !store
12365                .has_decommission_capacity_temporary_mutation_state(target_pool_index, checkpoint_owner)
12366                .await,
12367            "already-committed checkpoint recovery must resolve the pending capacity intent"
12368        );
12369        let stats = recover_tier_delete_dispatch_manifests(store.clone(), 100, None)
12370            .await
12371            .expect("the terminal source checkpoint should remain non-actionable");
12372        assert_eq!((stats.scanned, stats.deleted, stats.retained, stats.failed), (1, 0, 1, 0));
12373        assert_eq!(
12374            store
12375                .decommission_durable_ilm_receipt_count_for_test(0)
12376                .await
12377                .expect("v6 terminal receipts should be listable"),
12378            2
12379        );
12380
12381        {
12382            let _proof_guard = crate::services::notification_sys::without_cross_pool_fence_fleet_proof_for_test();
12383            let proof_error = store
12384                .verify_and_cleanup_decommissioned_durable_ilm_record_for_test(
12385                    0,
12386                    store.pools[0].get_disks_by_key(&manifest_name),
12387                    &manifest_name,
12388                )
12389                .await
12390                .expect_err("revoked v6 fleet proof must block final source cleanup")
12391                .to_string();
12392            assert!(
12393                proof_error.contains("fleet capability is unavailable"),
12394                "unexpected revoked-proof cleanup error: {proof_error}"
12395            );
12396            assert!(
12397                com::read_config(store.pools[0].clone(), &manifest_name).await.is_ok(),
12398                "revoked fleet proof must retain the last source manifest copy"
12399            );
12400        }
12401
12402        for path in [&manifest_name, &journal_name] {
12403            store
12404                .verify_and_cleanup_decommissioned_durable_ilm_record_for_test(0, store.pools[0].get_disks_by_key(path), path)
12405                .await
12406                .expect("terminal receipt should authorize verified source cleanup");
12407        }
12408        let stats = recover_tier_delete_dispatch_manifests(store.clone(), 100, None)
12409            .await
12410            .expect("verified source cleanup should remove the last manifest copy");
12411        assert_eq!((stats.scanned, stats.deleted, stats.failed), (0, 0, 0));
12412        for path in [&manifest_name, &journal_name] {
12413            for pool in &store.pools {
12414                assert!(
12415                    matches!(com::read_config(pool.clone(), path).await, Err(Error::ConfigNotFound)),
12416                    "terminal receipt cleanup must remove `{path}` from every staged pool"
12417                );
12418            }
12419        }
12420    }
12421
12422    #[cfg(feature = "test-util")]
12423    #[tokio::test]
12424    #[serial_test::serial(storage_class_env)]
12425    async fn authorized_all_committed_manifest_recovers_after_last_commit_crash() {
12426        let temp_dir = tempfile::tempdir().expect("create authorized manifest recovery store dir");
12427        let (ctx, store, _shutdown) =
12428            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "authorized-manifest-last-commit", &[4])).await;
12429        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
12430        let bucket = "authorized-manifest-last-commit-bucket";
12431        store
12432            .make_bucket(bucket, &MakeBucketOptions::default())
12433            .await
12434            .expect("authorized recovery bucket should be created");
12435        let incarnation = store
12436            .bucket_incarnation_id(bucket)
12437            .await
12438            .expect("bucket incarnation should resolve");
12439        let tier_name = "AUTHORIZED-LAST-COMMIT";
12440        let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
12441        let lease = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
12442            .await
12443            .expect("tier lease should resolve");
12444        let mut entries = Vec::new();
12445        for index in 0..2 {
12446            let remote_version = uuid::Uuid::new_v4().to_string();
12447            let entry = synthetic_v6_dispatch_entry(
12448                bucket,
12449                &format!("archive/{index}.bin"),
12450                tier_name,
12451                lease.backend_identity(),
12452                &remote_version,
12453            );
12454            backend.set_put_remote_version(Some(remote_version)).await;
12455            lease
12456                .put(&entry.obj_name, ReaderImpl::Body(bytes::Bytes::from_static(b"remote candidate")), 16)
12457                .await
12458                .expect("remote candidate should be seeded");
12459            entries.push((entry, Some(TierDeleteJournalState::Committed)));
12460        }
12461        let (manifest_name, _) = install_test_tier_delete_dispatch_fixture(
12462            store.clone(),
12463            bucket,
12464            incarnation,
12465            "archive/",
12466            entries,
12467            TierDeleteDispatchManifestState::DispatchAuthorized,
12468        )
12469        .await
12470        .expect("last-commit crash fixture should persist");
12471
12472        let manifest_stats = recover_tier_delete_dispatch_manifests(store.clone(), 100, None)
12473            .await
12474            .expect("all-Committed Authorized manifest should advance");
12475        assert_eq!((manifest_stats.scanned, manifest_stats.advanced, manifest_stats.failed), (1, 1, 0));
12476        assert_eq!(
12477            test_tier_delete_dispatch_manifest_state(store.clone(), &manifest_name)
12478                .await
12479                .expect("advanced manifest should remain readable"),
12480            Some(TierDeleteDispatchManifestState::Completed)
12481        );
12482        let journal_stats = recover_tier_delete_journal_entries(store.clone(), 100, None)
12483            .await
12484            .expect("Completed journal set should recover remote candidates");
12485        assert_eq!((journal_stats.scanned, journal_stats.deleted, journal_stats.failed), (2, 2, 0));
12486        assert_eq!(tier_delete_journal_count(store.clone()).await, 0);
12487        assert_eq!(tier_delete_dispatch_manifest_count(store.clone()).await, 1);
12488        let gc_stats = recover_tier_delete_dispatch_manifests(store.clone(), 100, None)
12489            .await
12490            .expect("manifest coordinator should remove the empty Completed manifest");
12491        assert_eq!((gc_stats.scanned, gc_stats.deleted, gc_stats.failed), (1, 1, 0));
12492        assert_eq!(tier_delete_dispatch_manifest_count(store).await, 0);
12493        assert_eq!(backend.object_count().await, 0);
12494        assert_eq!(backend.remove_versions().await.len(), 2);
12495    }
12496
12497    #[cfg(feature = "test-util")]
12498    async fn transition_transaction_record_count(store: Arc<crate::store::ECStore>) -> usize {
12499        store
12500            .list_objects_v2(
12501                RUSTFS_META_BUCKET,
12502                TRANSITION_TRANSACTION_RECORD_PREFIX,
12503                None,
12504                None,
12505                100,
12506                false,
12507                None,
12508                false,
12509            )
12510            .await
12511            .expect("transition transaction records should be listable")
12512            .objects
12513            .len()
12514    }
12515
12516    #[cfg(feature = "test-util")]
12517    async fn only_transition_transaction(store: Arc<crate::store::ECStore>) -> TransitionTransaction {
12518        let records = store
12519            .clone()
12520            .list_objects_v2(
12521                RUSTFS_META_BUCKET,
12522                TRANSITION_TRANSACTION_RECORD_PREFIX,
12523                None,
12524                None,
12525                100,
12526                false,
12527                None,
12528                false,
12529            )
12530            .await
12531            .expect("transition transaction records should be listable")
12532            .objects;
12533        assert_eq!(records.len(), 1, "test fixture should have exactly one transition transaction");
12534        let transaction_id = records[0]
12535            .name
12536            .rsplit('/')
12537            .next()
12538            .and_then(|name| name.strip_suffix(".json"))
12539            .and_then(|name| uuid::Uuid::parse_str(name).ok())
12540            .expect("transition transaction path should end in its UUID");
12541        load_transition_transaction_record(store, transaction_id)
12542            .await
12543            .expect("transition transaction should load")
12544    }
12545
12546    #[cfg(feature = "test-util")]
12547    async fn register_transition_reconcile_test_tier(
12548        handle: &Arc<tokio::sync::RwLock<TierConfigMgr>>,
12549        tier_name: &str,
12550    ) -> MockWarmBackend {
12551        let backend = MockWarmBackend::new();
12552        let mut manager = handle.write().await;
12553        manager.tiers.insert(
12554            tier_name.to_string(),
12555            TierConfig {
12556                version: "v1".to_string(),
12557                tier_type: TierType::Wasabi,
12558                name: tier_name.to_string(),
12559                wasabi: Some(TierWasabi {
12560                    name: tier_name.to_string(),
12561                    endpoint: "https://s3.wasabisys.com".to_string(),
12562                    access_key: "test-access-key".to_string(),
12563                    secret_key: "test-secret-key".to_string(),
12564                    bucket: "mock-tier".to_string(),
12565                    prefix: format!("mock/{}/", uuid::Uuid::new_v4()),
12566                    region: "us-east-1".to_string(),
12567                }),
12568                ..Default::default()
12569            },
12570        );
12571        manager
12572            .install_test_driver(tier_name, Box::new(backend.clone()))
12573            .expect("mock fallback tier driver should install");
12574        backend
12575    }
12576
12577    #[cfg(feature = "test-util")]
12578    async fn wait_for_tier_delete_journal_recovery(
12579        store: Arc<crate::store::ECStore>,
12580        backend: &MockWarmBackend,
12581        expected_removes: usize,
12582    ) {
12583        tokio::time::timeout(Duration::from_secs(30), async {
12584            loop {
12585                if backend.remove_versions().await.len() >= expected_removes
12586                    && tier_delete_journal_count(store.clone()).await == 0
12587                {
12588                    return;
12589                }
12590                tokio::time::sleep(Duration::from_millis(10)).await;
12591            }
12592        })
12593        .await
12594        .expect("tier delete journal recovery should complete");
12595    }
12596
12597    #[cfg(feature = "test-util")]
12598    async fn wait_for_tier_free_version_recovery(
12599        store: Arc<crate::store::ECStore>,
12600        backend: &MockWarmBackend,
12601        expected_removes: usize,
12602    ) {
12603        ExpiryState::resize_workers(1, store.clone()).await;
12604        let mut last_progress = None;
12605        let result = tokio::time::timeout(Duration::from_secs(30), async {
12606            loop {
12607                let stats = recover_tier_free_versions(store.clone(), 100, None, None)
12608                    .await
12609                    .expect("tier free-version recovery scan should succeed");
12610                let removes = backend.remove_versions().await.len();
12611                let recovered = removes >= expected_removes && stats.enqueued == 0 && stats.failed == 0;
12612                last_progress = Some((removes, stats));
12613                if recovered {
12614                    return;
12615                }
12616                tokio::time::sleep(Duration::from_millis(50)).await;
12617            }
12618        })
12619        .await;
12620        if result.is_err() {
12621            let expiry_state = store.ctx.expiry_state();
12622            let workers = expiry_state
12623                .try_read()
12624                .ok()
12625                .map(|state| (state.pending_tasks(), state.active_tasks()));
12626            panic!(
12627                "tier free-version recovery should complete: expected_removes={expected_removes}, \
12628                 last_progress={last_progress:?}, workers(pending, active)={workers:?}"
12629            );
12630        }
12631        wait_for_expiry_workers_idle(&store).await;
12632    }
12633
12634    /// Unlocked poll for exact metadata absence while asynchronous free-version
12635    /// cleanup removes the per-disk copies. Mid-cleanup, fewer than a read
12636    /// quorum of disks may still hold the record, so that transient result
12637    /// means "not converged yet"; every other error still fails the test.
12638    #[cfg(feature = "test-util")]
12639    async fn exact_metadata_absent_during_cleanup(store: &crate::store::ECStore, bucket: &str, object: &str) -> bool {
12640        match store.pools[0]
12641            .get_disks_by_key(object)
12642            .load_file_info_versions_exact(bucket, object)
12643            .await
12644        {
12645            Ok(metadata) => metadata.is_none(),
12646            Err(StorageError::InsufficientReadQuorum(_, _)) => false,
12647            Err(error) => panic!("{object} cleanup metadata should remain readable: {error:?}"),
12648        }
12649    }
12650
12651    #[cfg(feature = "test-util")]
12652    async fn wait_for_expiry_workers_idle(store: &crate::store::ECStore) {
12653        let expiry_state = store.ctx.expiry_state();
12654        tokio::time::timeout(Duration::from_secs(30), async {
12655            loop {
12656                let idle = {
12657                    let state = expiry_state.read().await;
12658                    state.pending_tasks() == 0 && state.active_tasks() == 0
12659                };
12660                if idle {
12661                    // Recheck after a scheduler turn so the tiny hand-off
12662                    // between dequeue and active-task accounting cannot make
12663                    // the test observe a false idle state.
12664                    tokio::time::sleep(Duration::from_millis(10)).await;
12665                    let state = expiry_state.read().await;
12666                    if state.pending_tasks() == 0 && state.active_tasks() == 0 {
12667                        return;
12668                    }
12669                } else {
12670                    tokio::time::sleep(Duration::from_millis(10)).await;
12671                }
12672            }
12673        })
12674        .await
12675        .expect("lifecycle expiry workers should become idle");
12676    }
12677
12678    #[cfg(feature = "test-util")]
12679    async fn run_transitioned_delete_free_version_owner_case(
12680        store_name: &str,
12681        tier_name: &str,
12682        bucket: &str,
12683        object: &str,
12684        restore_before_delete: bool,
12685        causal_enqueue: bool,
12686        delete_with_journal: bool,
12687    ) {
12688        let temp_dir = tempfile::tempdir().expect("create transitioned delete store dir");
12689        let (ctx, store, _shutdown) =
12690            without_storage_class_env(build_isolated_test_store(temp_dir.path(), store_name, &[4])).await;
12691        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
12692
12693        let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
12694        store
12695            .make_bucket(bucket, &MakeBucketOptions::default())
12696            .await
12697            .expect("source bucket should be created");
12698        let mut reader = PutObjReader::from_vec(vec![b'd'; 1024 * 1024]);
12699        let original = store
12700            .put_object(bucket, object, &mut reader, &ObjectOptions::default())
12701            .await
12702            .expect("source object should be written");
12703        store
12704            .transition_object(
12705                bucket,
12706                object,
12707                &ObjectOptions {
12708                    transition: TransitionOptions {
12709                        status: TRANSITION_PENDING.to_string(),
12710                        tier: tier_name.to_string(),
12711                        etag: original.etag.clone().expect("source should have an etag"),
12712                        ..Default::default()
12713                    },
12714                    mod_time: original.mod_time,
12715                    ..Default::default()
12716                },
12717            )
12718            .await
12719            .expect("source transition should commit");
12720        assert_eq!(backend.object_count().await, 1, "transition should create one remote object");
12721        if restore_before_delete {
12722            store
12723                .clone()
12724                .restore_transitioned_object(
12725                    bucket,
12726                    object,
12727                    &ObjectOptions {
12728                        transition: TransitionOptions {
12729                            restore_request: s3s::dto::RestoreRequest {
12730                                days: Some(1),
12731                                ..Default::default()
12732                            },
12733                            ..Default::default()
12734                        },
12735                        ..Default::default()
12736                    },
12737                )
12738                .await
12739                .expect("restore should complete before transitioned delete");
12740            let restored = store
12741                .get_object_info(bucket, object, &ObjectOptions::default())
12742                .await
12743                .expect("restored transitioned source should remain readable");
12744            assert!(
12745                restored.user_defined.contains_key(rustfs_filemeta::metadata_keys::RESTORE),
12746                "restore completion metadata must be present before the delete regression"
12747            );
12748        }
12749
12750        if causal_enqueue {
12751            ExpiryState::resize_workers(1, store.clone()).await;
12752        }
12753        backend.set_remove_failure(!causal_enqueue);
12754        if delete_with_journal {
12755            store
12756                .delete_object_with_tier_delete_journal(bucket, object, ObjectOptions::default())
12757                .await
12758                .expect("transitioned source journal-wrapper delete should commit");
12759        } else {
12760            store
12761                .delete_object(bucket, object, ObjectOptions::default())
12762                .await
12763                .expect("transitioned source plain object-layer delete should commit");
12764        }
12765
12766        if causal_enqueue {
12767            tokio::time::timeout(Duration::from_secs(30), async {
12768                loop {
12769                    let metadata_absent = {
12770                        // Observe one cleanup state, without spanning per-disk marker removal.
12771                        let mut read_opts = ObjectOptions::default();
12772                        let _guards = store
12773                            .acquire_all_physical_object_read_locks(
12774                                "transitioned_delete_cleanup_test",
12775                                bucket,
12776                                object,
12777                                &mut read_opts,
12778                            )
12779                            .await
12780                            .expect("causal free-version cleanup observation should acquire object read locks");
12781                        store.pools[0]
12782                            .get_disks_by_key(object)
12783                            .load_file_info_versions_exact(bucket, object)
12784                            .await
12785                            .expect("causal free-version cleanup metadata should remain readable")
12786                            .is_none()
12787                    };
12788                    if metadata_absent && backend.remove_versions().await.len() == 1 {
12789                        return;
12790                    }
12791                    tokio::time::sleep(Duration::from_millis(10)).await;
12792                }
12793            })
12794            .await
12795            .expect("committed free-version should be cleaned without a recovery scan");
12796            assert_eq!(backend.object_count().await, 0, "causal cleanup should remove the remote object");
12797            assert_eq!(
12798                tier_delete_journal_count(store.clone()).await,
12799                0,
12800                "ordinary causal cleanup must not create a journal"
12801            );
12802            store
12803                .delete_bucket(bucket, &DeleteBucketOptions::default())
12804                .await
12805                .expect("bucket delete should succeed after causal free-version cleanup");
12806            return;
12807        }
12808
12809        let local_versions = store.pools[0]
12810            .get_disks_by_key(object)
12811            .load_file_info_versions_exact(bucket, object)
12812            .await
12813            .expect("post-delete metadata should remain decodable");
12814        let local_versions = local_versions.expect("ordinary delete must leave a durable free-version cleanup owner");
12815        assert_eq!(
12816            local_versions
12817                .versions
12818                .iter()
12819                .chain(local_versions.free_versions.iter())
12820                .filter(|version| version.tier_free_version())
12821                .count(),
12822            1,
12823            "ordinary delete must leave exactly one hidden free-version owner"
12824        );
12825        assert_eq!(
12826            local_versions
12827                .versions
12828                .iter()
12829                .filter(|version| !version.tier_free_version())
12830                .count(),
12831            0,
12832            "ordinary delete must remove the visible transitioned source"
12833        );
12834        let listed = store
12835            .clone()
12836            .list_object_versions(bucket, "", None, None, None, 100)
12837            .await
12838            .expect("source versions should be listable");
12839        assert!(listed.objects.is_empty(), "source must have no visible versions after delete");
12840
12841        assert_eq!(
12842            tier_delete_journal_count(store.clone()).await,
12843            0,
12844            "ordinary delete must not create a journal"
12845        );
12846
12847        backend.set_remove_failure(false);
12848        ExpiryState::resize_workers(1, store.clone()).await;
12849        let recovered = recover_tier_free_versions(store.clone(), 100, None, None)
12850            .await
12851            .expect("free-version recovery should scan the hidden owner");
12852        assert_eq!((recovered.enqueued, recovered.failed), (1, 0));
12853        tokio::time::timeout(Duration::from_secs(30), async {
12854            loop {
12855                let metadata_absent = {
12856                    // Observe one cleanup state, without spanning per-disk marker removal.
12857                    let mut read_opts = ObjectOptions::default();
12858                    let _guards = store
12859                        .acquire_all_physical_object_read_locks(
12860                            "transitioned_delete_cleanup_test",
12861                            bucket,
12862                            object,
12863                            &mut read_opts,
12864                        )
12865                        .await
12866                        .expect("free-version cleanup observation should acquire object read locks");
12867                    store.pools[0]
12868                        .get_disks_by_key(object)
12869                        .load_file_info_versions_exact(bucket, object)
12870                        .await
12871                        .expect("free-version cleanup metadata should remain readable")
12872                        .is_none()
12873                };
12874                if metadata_absent && backend.remove_versions().await.len() == 1 {
12875                    return;
12876                }
12877                tokio::time::sleep(Duration::from_millis(10)).await;
12878            }
12879        })
12880        .await
12881        .expect("free-version recovery should delete the remote and exact local owner");
12882        assert_eq!(backend.object_count().await, 0, "free-version recovery should remove the remote object");
12883        store
12884            .delete_bucket(bucket, &DeleteBucketOptions::default())
12885            .await
12886            .expect("ordinary bucket delete should succeed after the last transitioned object");
12887    }
12888
12889    #[cfg(feature = "test-util")]
12890    #[tokio::test]
12891    #[serial_test::serial(storage_class_env)]
12892    async fn transitioned_delete_uses_free_version_as_cleanup_owner() {
12893        run_transitioned_delete_free_version_owner_case(
12894            "transitioned-delete-journal-owner",
12895            "DELETEOWNER",
12896            "transitioned-delete-journal-owner-bucket",
12897            "transition/archive.bin",
12898            false,
12899            false,
12900            true,
12901        )
12902        .await;
12903    }
12904
12905    #[cfg(feature = "test-util")]
12906    #[tokio::test]
12907    #[serial_test::serial(storage_class_env)]
12908    async fn restored_transitioned_delete_uses_free_version_as_cleanup_owner() {
12909        run_transitioned_delete_free_version_owner_case(
12910            "restored-transitioned-delete-journal-owner",
12911            "RESTOREDELETEOWNER",
12912            "restored-transitioned-delete-journal-owner-bucket",
12913            "transition/archive.bin",
12914            true,
12915            false,
12916            true,
12917        )
12918        .await;
12919    }
12920
12921    #[cfg(feature = "test-util")]
12922    #[tokio::test]
12923    #[serial_test::serial(storage_class_env)]
12924    async fn transitioned_delete_causally_enqueues_free_version() {
12925        run_transitioned_delete_free_version_owner_case(
12926            "transitioned-delete-causal-enqueue",
12927            "DELETE-CAUSAL",
12928            "transitioned-delete-causal-enqueue-bucket",
12929            "transition/archive.bin",
12930            false,
12931            true,
12932            false,
12933        )
12934        .await;
12935    }
12936
12937    #[cfg(feature = "test-util")]
12938    #[tokio::test]
12939    #[serial_test::serial(storage_class_env)]
12940    async fn restored_transitioned_delete_causally_enqueues_free_version() {
12941        run_transitioned_delete_free_version_owner_case(
12942            "restored-transitioned-delete-causal-enqueue",
12943            "RESTORE-DELETE-CAUSAL",
12944            "restored-transitioned-delete-causal-enqueue-bucket",
12945            "transition/archive.bin",
12946            true,
12947            true,
12948            true,
12949        )
12950        .await;
12951    }
12952
12953    #[cfg(feature = "test-util")]
12954    #[tokio::test]
12955    #[serial_test::serial(storage_class_env)]
12956    async fn force_tier_remove_blocks_on_physical_free_version_hidden_by_other_pool() {
12957        let temp_dir = tempfile::tempdir().expect("create tier reference store dir");
12958        let (ctx, store, _shutdown) =
12959            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "tier-free-reference-proof", &[4, 4])).await;
12960        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
12961
12962        let tier_name = "FREE-PROOF";
12963        let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
12964        let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
12965            .await
12966            .expect("test tier identity should resolve")
12967            .backend_identity();
12968        let bucket = "tier-free-reference-proof-bucket";
12969        let object = "archive.bin";
12970        store
12971            .make_bucket(bucket, &MakeBucketOptions::default())
12972            .await
12973            .expect("source bucket should be created");
12974        let mut reader = PutObjReader::from_vec(vec![b'f'; 1024 * 1024]);
12975        let original = store.pools[0]
12976            .put_object(bucket, object, &mut reader, &ObjectOptions::default())
12977            .await
12978            .expect("source object should be written");
12979        store.pools[0]
12980            .transition_object(
12981                bucket,
12982                object,
12983                &ObjectOptions {
12984                    transition: TransitionOptions {
12985                        status: TRANSITION_PENDING.to_string(),
12986                        tier: tier_name.to_string(),
12987                        etag: original.etag.clone().expect("source should have an etag"),
12988                        ..Default::default()
12989                    },
12990                    mod_time: original.mod_time,
12991                    ..Default::default()
12992                },
12993            )
12994            .await
12995            .expect("source transition should commit");
12996        backend.set_remove_failure(true);
12997        store.pools[0]
12998            .delete_object(
12999                bucket,
13000                object,
13001                ObjectOptions {
13002                    tier_delete_journal_api: Some(store.clone()),
13003                    ..Default::default()
13004                },
13005            )
13006            .await
13007            .expect("journal-less compatibility delete should retain a free-version");
13008
13009        let exact = store.pools[0]
13010            .get_disks_by_key(object)
13011            .load_file_info_versions_exact(bucket, object)
13012            .await
13013            .expect("free-version metadata should remain decodable")
13014            .expect("free-version metadata should remain on disk");
13015        assert_eq!(
13016            exact
13017                .versions
13018                .iter()
13019                .chain(exact.free_versions.iter())
13020                .filter(|version| version.tier_free_version())
13021                .count(),
13022            1,
13023            "fixture must contain one real serialized hidden free-version"
13024        );
13025
13026        let mut hot_reader = PutObjReader::from_vec(vec![b'h'; 1024 * 1024]);
13027        store.pools[1]
13028            .put_object(
13029                bucket,
13030                object,
13031                &mut hot_reader,
13032                &ObjectOptions {
13033                    mod_time: Some(OffsetDateTime::now_utc() + time::Duration::hours(1)),
13034                    ..Default::default()
13035                },
13036            )
13037            .await
13038            .expect("second pool hot source should be written");
13039        let merged = store
13040            .get_object_info(bucket, object, &ObjectOptions::default())
13041            .await
13042            .expect("merged lookup should return the second-pool hot source");
13043        assert!(
13044            merged.transitioned_object.status.is_empty() && !merged.transitioned_object.free_version,
13045            "the ordinary second-pool source must hide the first-pool free-version from merged lookup"
13046        );
13047
13048        let err = ensure_no_authoritative_tier_references_for_test(store, tier_name, backend_identity, true)
13049            .await
13050            .expect_err("force tier removal must not bypass a physical free-version cleanup obligation");
13051        assert_eq!(err.code, ERR_TIER_BACKEND_IN_USE.code);
13052    }
13053
13054    #[cfg(feature = "test-util")]
13055    #[tokio::test]
13056    #[serial_test::serial(storage_class_env)]
13057    async fn tier_reference_proof_fails_closed_when_physical_bucket_topology_is_incomplete() {
13058        let temp_dir = tempfile::tempdir().expect("create incomplete tier reference store dir");
13059        let (ctx, store, _shutdown) =
13060            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "tier-reference-incomplete-topology", &[4]))
13061                .await;
13062        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
13063
13064        let tier_name = "INCOMPLETE-PROOF";
13065        register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
13066        let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
13067            .await
13068            .expect("test tier identity should resolve")
13069            .backend_identity();
13070        store.pools[0].disk_set[0].disks.write().await[0] = None;
13071
13072        let err = ensure_no_authoritative_tier_references_for_test(store, tier_name, backend_identity, true)
13073            .await
13074            .expect_err("an incomplete physical bucket enumeration must block tier mutation");
13075        assert_eq!(err.code, ERR_TIER_INVALID_CONFIG.code);
13076        assert!(
13077            err.message.contains("complete set quorum"),
13078            "unexpected reference proof error: {}",
13079            err.message
13080        );
13081    }
13082
13083    #[cfg(feature = "test-util")]
13084    async fn copy_test_xlmeta_between_pools(
13085        root: &std::path::Path,
13086        source_pool: usize,
13087        target_pool: usize,
13088        bucket: &str,
13089        object: &str,
13090    ) {
13091        for disk_index in 0..4 {
13092            let source = root.join(format!("pool{source_pool}/set0/disk{disk_index}/{bucket}/{object}/{STORAGE_FORMAT_FILE}"));
13093            let target = root.join(format!("pool{target_pool}/set0/disk{disk_index}/{bucket}/{object}/{STORAGE_FORMAT_FILE}"));
13094            tokio::fs::create_dir_all(target.parent().expect("target xl.meta should have a parent"))
13095                .await
13096                .expect("target object directory should be created");
13097            tokio::fs::copy(&source, &target)
13098                .await
13099                .expect("xl.meta should copy exactly between pools");
13100        }
13101    }
13102
13103    #[cfg(feature = "test-util")]
13104    async fn rewrite_transitioned_xlmeta_as_legacy_unknown(
13105        root: &std::path::Path,
13106        pool_index: usize,
13107        bucket: &str,
13108        object: &str,
13109        minio_unversioned: bool,
13110    ) {
13111        for disk_index in 0..4 {
13112            let metadata_path =
13113                root.join(format!("pool{pool_index}/set0/disk{disk_index}/{bucket}/{object}/{STORAGE_FORMAT_FILE}"));
13114            let encoded = tokio::fs::read(&metadata_path)
13115                .await
13116                .expect("transitioned xl.meta should be readable");
13117            let mut metadata = FileMeta::load(&encoded).expect("transitioned xl.meta should decode");
13118            let mut found = false;
13119            for shallow in &mut metadata.versions {
13120                let mut version = shallow
13121                    .parse_version_meta()
13122                    .expect("transitioned version record should decode");
13123                let Some(object_meta) = version.object.as_mut() else {
13124                    continue;
13125                };
13126                if rustfs_utils::http::metadata_compat::get_bytes(
13127                    &object_meta.meta_sys,
13128                    rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_STATUS,
13129                )
13130                .as_deref()
13131                    != Some(rustfs_filemeta::TRANSITION_COMPLETE.as_bytes())
13132                {
13133                    continue;
13134                }
13135                found = true;
13136                for suffix in [
13137                    rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
13138                    rustfs_utils::http::metadata_compat::SUFFIX_TRANSITIONED_VERSION_STATE,
13139                    rustfs_utils::http::metadata_compat::SUFFIX_TRANSITIONED_VERSION_ID,
13140                ] {
13141                    rustfs_utils::http::metadata_compat::remove_bytes(&mut object_meta.meta_sys, suffix);
13142                }
13143                if minio_unversioned {
13144                    object_meta
13145                        .meta_sys
13146                        .insert("x-minio-internal-transitioned-versionID".to_string(), Vec::new());
13147                }
13148                *shallow = rustfs_filemeta::FileMetaShallowVersion::try_from(version)
13149                    .expect("legacy transitioned version should re-encode");
13150            }
13151            assert!(found, "transitioned source should exist in serialized xl.meta");
13152            tokio::fs::write(&metadata_path, metadata.marshal_msg().expect("legacy transitioned xl.meta should encode"))
13153                .await
13154                .expect("legacy transitioned xl.meta should persist");
13155        }
13156    }
13157
13158    #[cfg(feature = "test-util")]
13159    async fn read_store_body(
13160        store: &Arc<crate::store::ECStore>,
13161        bucket: &str,
13162        object: &str,
13163        range: Option<HTTPRangeSpec>,
13164        opts: &ObjectOptions,
13165    ) -> Vec<u8> {
13166        let mut reader = store
13167            .get_object_reader(bucket, object, range, HeaderMap::new(), opts)
13168            .await
13169            .expect("object reader should open");
13170        let mut body = Vec::new();
13171        reader.stream.read_to_end(&mut body).await.expect("object body should drain");
13172        body
13173    }
13174
13175    #[cfg(feature = "test-util")]
13176    #[test]
13177    #[serial_test::serial(storage_class_env)]
13178    fn legacy_transition_state_inspection_and_apply_keep_all_disk_copies_unchanged() {
13179        run_large_stack_async_test("legacy-state-reconcile-inspection", || {
13180            legacy_transition_state_inspection_and_apply_case(false)
13181        });
13182    }
13183
13184    #[cfg(feature = "test-util")]
13185    #[cfg(not(windows))]
13186    #[test]
13187    #[serial_test::serial(storage_class_env)]
13188    fn legacy_transition_state_backfill_retries_partial_commits_and_preserves_other_bytes() {
13189        run_large_stack_async_test("legacy-state-reconcile-backfill", || {
13190            legacy_transition_state_inspection_and_apply_case(true)
13191        });
13192    }
13193
13194    #[cfg(feature = "test-util")]
13195    async fn legacy_transition_state_inspection_and_apply_case(write_enabled: bool) {
13196        #[cfg(windows)]
13197        assert!(!write_enabled, "Windows supports inspection but cannot prove repair directory durability");
13198        use crate::bucket::lifecycle::legacy_transition_state_reconcile::{
13199            LegacyTransitionStateReconcileOutcome as Outcome, LegacyTransitionStateReconcileRequest,
13200            LegacyTransitionStateReconcileSelector,
13201        };
13202        for (remote_version, expected_state) in [
13203            ("", rustfs_filemeta::TransitionVersionState::KnownDisabled),
13204            ("null", rustfs_filemeta::TransitionVersionState::SuspendedNull),
13205            ("opaque-version", rustfs_filemeta::TransitionVersionState::Exact),
13206        ] {
13207            let temp_dir = tempfile::tempdir().expect("legacy reconcile store directory");
13208            let (ctx, store, _shutdown) =
13209                without_storage_class_env(build_isolated_test_store(temp_dir.path(), "legacy-state-reconcile-inspect", &[4]))
13210                    .await;
13211            crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
13212            let tier_name = "LEGACY-RECONCILE";
13213            let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
13214            backend.set_put_remote_version(Some(remote_version.to_string())).await;
13215            let bucket = "legacy-state-reconcile-bucket";
13216            let object = "archive.bin";
13217            store
13218                .make_bucket(bucket, &MakeBucketOptions::default())
13219                .await
13220                .expect("create legacy fixture bucket");
13221            let mut reader = PutObjReader::from_vec(b"legacy reconcile body".repeat(1024));
13222            let source = store
13223                .put_object(bucket, object, &mut reader, &ObjectOptions::default())
13224                .await
13225                .expect("write source");
13226            {
13227                // Create the fixture under the existing remote-version writer
13228                // gate. This does not authorize legacy metadata reconciliation.
13229                let _proof = crate::services::notification_sys::install_current_remote_version_state_fleet_proof_for_test();
13230                temp_env::async_with_vars(
13231                    [
13232                        (rustfs_config::ENV_TIER_REMOTE_VERSION_STATE_WRITE, Some("true")),
13233                        (rustfs_config::ENV_TIER_REMOTE_VERSION_STATE_FLEET_CONFIRMED, Some("true")),
13234                    ],
13235                    store.transition_object(
13236                        bucket,
13237                        object,
13238                        &ObjectOptions {
13239                            transition: TransitionOptions {
13240                                status: TRANSITION_PENDING.to_string(),
13241                                tier: tier_name.to_string(),
13242                                etag: source.etag.clone().expect("source ETag"),
13243                                ..Default::default()
13244                            },
13245                            mod_time: source.mod_time,
13246                            ..Default::default()
13247                        },
13248                    ),
13249                )
13250                .await
13251                .expect("transition source");
13252            }
13253            assert!(
13254                crate::services::notification_sys::acquire_legacy_transition_state_reconcile_fleet_proof()
13255                    .await
13256                    .is_none(),
13257                "fixture setup must not grant the missing reconciliation write capability"
13258            );
13259            let selector = LegacyTransitionStateReconcileSelector {
13260                bucket: bucket.to_string(),
13261                object: object.to_string(),
13262                version_id: "null".to_string(),
13263            };
13264            if expected_state == rustfs_filemeta::TransitionVersionState::Exact {
13265                backend
13266                    .set_transition_candidate_probe_override(Some(
13267                        crate::services::tier::warm_backend::TransitionCandidateProbe::Ambiguous,
13268                    ))
13269                    .await;
13270            }
13271            let converged = store
13272                .inspect_legacy_transition_state(selector.clone())
13273                .await
13274                .expect("inspect an already explicit transition");
13275            assert_eq!(converged.outcome, Outcome::Migrated, "{converged:?}");
13276            assert!(!converged.changed);
13277            backend.set_transition_candidate_probe_override(None).await;
13278            rewrite_transitioned_xlmeta_as_legacy_unknown(temp_dir.path(), 0, bucket, object, remote_version.is_empty()).await;
13279            let paths = (0..4)
13280                .map(|disk| {
13281                    temp_dir
13282                        .path()
13283                        .join(format!("pool0/set0/disk{disk}/{bucket}/{object}/{STORAGE_FORMAT_FILE}"))
13284                })
13285                .collect::<Vec<_>>();
13286            let mut original = Vec::new();
13287            for path in &paths {
13288                original.push(tokio::fs::read(path).await.expect("original xl.meta"));
13289            }
13290            backend.clear_op_log().await;
13291            let inspection = store.inspect_legacy_transition_state(selector.clone());
13292            assert!(
13293                std::mem::size_of_val(&inspection) <= 4 * 1024,
13294                "admin inspection future must remain stack-bounded"
13295            );
13296            let inspected = inspection.await.expect("inspect legacy state");
13297            assert_eq!(inspected.outcome, Outcome::ReadyToMigrate, "{inspected:?}");
13298            assert!(!inspected.readiness.post_ready, "current fleet cannot authorize conditional writes");
13299            let target = inspected.target.expect("live probe should establish one model");
13300            assert_eq!(target.state, expected_state);
13301            let request = LegacyTransitionStateReconcileRequest {
13302                confirm: true,
13303                selector,
13304                source: inspected.source.expect("immutable source"),
13305                original_sets: inspected.original_sets,
13306                target,
13307                reconciliation_digest: inspected.reconciliation_digest.expect("expected tuple digest"),
13308            };
13309            #[cfg(not(windows))]
13310            if write_enabled {
13311                crate::services::notification_sys::with_legacy_transition_state_fleet_proof_for_test(async {
13312                    crate::disk::local::bucket_durability::set(bucket, Some(crate::disk::local::DurabilityMode::None));
13313                    let unsynced = store.reconcile_legacy_transition_state(request.clone()).await;
13314                    crate::disk::local::bucket_durability::set(bucket, None);
13315                    let unsynced = unsynced.expect("repair without metadata durability");
13316                    assert_eq!(unsynced.outcome, Outcome::BackendUnavailable);
13317                    assert!(!unsynced.changed);
13318                    let rollback = paths[0].parent().expect("object directory").join(Uuid::new_v4().to_string());
13319                    tokio::fs::create_dir(&rollback).await.expect("pending rollback directory");
13320                    tokio::fs::write(rollback.join(crate::disk::STORAGE_FORMAT_FILE_BACKUP), &original[0])
13321                        .await
13322                        .expect("pending old metadata backup");
13323                    let unsettled = store.reconcile_legacy_transition_state(request.clone()).await;
13324                    tokio::fs::remove_dir_all(&rollback).await.expect("settle fixture rollback");
13325                    let unsettled = unsettled.expect("repair must wait for rollback");
13326                    assert_eq!(unsettled.outcome, Outcome::BackendUnavailable);
13327                    assert!(!unsettled.changed);
13328                    for (path, bytes) in paths.iter().zip(&original) {
13329                        assert_eq!(tokio::fs::read(path).await.expect("blocked repair leaves original bytes"), *bytes);
13330                    }
13331                    // The first disk commits; the second stops after staging.
13332                    // This models an interrupted cross-disk effect without rollback.
13333                    let disks = store.all_set_disks()[0].disk_inventory().await;
13334                    let first_disk = disks[0].as_ref().expect("first physical disk");
13335                    let publication_path = first_disk
13336                        .get_object_path_for_io_if_local(bucket, &format!("{object}/{STORAGE_FORMAT_FILE}"))
13337                        .expect("local disk")
13338                        .expect("publication path");
13339                    let crash_key = format!("{object}/{STORAGE_FORMAT_FILE}");
13340                    let _hook = crate::disk::os::prepared_publication_test_hooks::install_at(
13341                        crate::disk::os::prepared_publication_test_hooks::Stage::Rename,
13342                        &publication_path,
13343                        move || {
13344                            crate::crash_inject::arm(crate::crash_inject::CrashPoint::MetaWriteAfterTmpBeforeRename, &crash_key);
13345                        },
13346                    );
13347                    let partial = store
13348                        .reconcile_legacy_transition_state(request.clone())
13349                        .await
13350                        .expect("partial repair response");
13351                    assert_eq!(partial.outcome, Outcome::BackendUnavailable, "{partial:?}");
13352                    assert!(partial.changed, "first copy was committed: {partial:?}");
13353                    assert!(partial.changes_indeterminate);
13354                    assert_ne!(tokio::fs::read(&paths[0]).await.expect("first committed copy"), original[0]);
13355                    for (path, bytes) in paths[1..].iter().zip(&original[1..]) {
13356                        assert_eq!(tokio::fs::read(path).await.expect("uncommitted copy"), *bytes);
13357                    }
13358                    assert!(
13359                        store.all_set_disks()[0]
13360                            .load_file_info_versions_exact(bucket, object)
13361                            .await
13362                            .is_err(),
13363                        "cleanup cannot select a partial repair subset"
13364                    );
13365                    let repaired = store
13366                        .reconcile_legacy_transition_state(request.clone())
13367                        .await
13368                        .expect("retry original snapshot");
13369                    assert_eq!(repaired.outcome, Outcome::Migrated, "{repaired:?}");
13370                    assert!(repaired.changed);
13371                    assert!(!repaired.changes_indeterminate);
13372                    let mut committed = Vec::new();
13373                    for (path, original) in paths.iter().zip(&original) {
13374                        let raw = tokio::fs::read(path).await.expect("repaired copy");
13375                        let metadata = FileMeta::load(&raw).expect("decode repaired copy");
13376                        let previous = FileMeta::load(original).expect("decode original copy");
13377                        assert_eq!(
13378                            metadata.transition_reconcile_generation(None).unwrap(),
13379                            previous.transition_reconcile_generation(None).unwrap()
13380                        );
13381                        let (_, version) = metadata.find_version(None).expect("selected version");
13382                        let info = version.into_fileinfo(bucket, object, true).expect("repaired FileInfo");
13383                        assert_eq!(info.transition_version_state, expected_state);
13384                        assert_eq!(info.transition_version, request.target.remote_version);
13385                        committed.push(raw);
13386                    }
13387                    assert!(
13388                        store.all_set_disks()[0]
13389                            .load_file_info_versions_exact(bucket, object)
13390                            .await
13391                            .expect("converged cleanup snapshot")
13392                            .is_some()
13393                    );
13394                    let replay = store
13395                        .reconcile_legacy_transition_state(request.clone())
13396                        .await
13397                        .expect("idempotent original request replay");
13398                    assert_eq!(replay.outcome, Outcome::Migrated, "{replay:?}");
13399                    assert!(!replay.changed);
13400                    for (path, expected) in paths.iter().zip(&committed) {
13401                        assert_eq!(
13402                            tokio::fs::read(path).await.expect("replayed copy"),
13403                            *expected,
13404                            "idempotence preserves raw encoding"
13405                        );
13406                    }
13407                    for (path, bytes) in paths.iter().zip(&original) {
13408                        tokio::fs::write(path, bytes)
13409                            .await
13410                            .expect("reset independent cancellation fixture");
13411                    }
13412                    let (entered_tx, entered) = tokio::sync::oneshot::channel();
13413                    let (release, released) = std::sync::mpsc::channel::<()>();
13414                    let _pause = crate::disk::os::prepared_publication_test_hooks::install_at(
13415                        crate::disk::os::prepared_publication_test_hooks::Stage::Rename,
13416                        &publication_path,
13417                        move || {
13418                            let _ = entered_tx.send(());
13419                            let _ = released.recv();
13420                        },
13421                    );
13422                    let mut repair = Box::pin(store.reconcile_legacy_transition_state(request.clone()));
13423                    tokio::select! {
13424                        result = &mut repair => panic!("repair completed before publication pause: {result:?}"),
13425                        result = entered => result.expect("publication executor entered"),
13426                    }
13427                    let update_options = crate::disk::UpdateMetadataOpts::default();
13428                    let mut update = Box::pin(first_disk.update_metadata(
13429                        bucket,
13430                        object,
13431                        FileInfo {
13432                            metadata: HashMap::from([("x-amz-meta-concurrent".to_string(), "kept".to_string())]),
13433                            ..Default::default()
13434                        },
13435                        &update_options,
13436                    ));
13437                    assert!(
13438                        tokio::time::timeout(std::time::Duration::from_millis(25), update.as_mut())
13439                            .await
13440                            .is_err(),
13441                        "another metadata RMW must wait for publication"
13442                    );
13443                    drop(repair);
13444                    assert!(
13445                        tokio::time::timeout(std::time::Duration::from_millis(25), update.as_mut())
13446                            .await
13447                            .is_err(),
13448                        "cancelling the coordinator must not release an in-flight disk mutation"
13449                    );
13450                    release.send(()).expect("resume owned publication");
13451                    update.await.expect("serialized metadata update");
13452                    let raw = tokio::fs::read(&paths[0])
13453                        .await
13454                        .expect("cancelled repair and later metadata update");
13455                    let (_, version) = FileMeta::load(&raw)
13456                        .expect("metadata after cancellation")
13457                        .find_version(None)
13458                        .expect("selected version");
13459                    let info = version
13460                        .into_fileinfo(bucket, object, true)
13461                        .expect("metadata after serialized update");
13462                    assert_eq!(info.transition_version_state, expected_state);
13463                    assert_eq!(info.metadata.get("x-amz-meta-concurrent").map(String::as_str), Some("kept"));
13464                    let stale = store
13465                        .reconcile_legacy_transition_state(request.clone())
13466                        .await
13467                        .expect("stale original request");
13468                    assert_eq!(
13469                        stale.outcome,
13470                        Outcome::Corrupt,
13471                        "unrelated metadata change invalidates the original generation: {stale:?}"
13472                    );
13473                    assert!(!stale.changed);
13474                    assert_eq!(tokio::fs::read(&paths[0]).await.expect("stale write leaves bytes unchanged"), raw);
13475                    assert_eq!(backend.remove_count().await, 0);
13476                    // A pinned version probe uses GET to verify that exact
13477                    // candidate; every backend operation still targets it.
13478                    let operations = backend.op_log().await;
13479                    assert!(
13480                        operations.iter().all(|operation| match operation {
13481                            MockWarmOp::Probe { object } | MockWarmOp::Get { object } => object == &request.source.remote_object,
13482                            _ => false,
13483                        }),
13484                        "unexpected backend effects: {operations:?}"
13485                    );
13486                })
13487                .await;
13488                continue;
13489            }
13490            let mut tampered = request.clone();
13491            tampered.source.remote_object.push_str("-other");
13492            let probes_before = backend.op_log().await.len();
13493            let rejected = store
13494                .reconcile_legacy_transition_state(tampered)
13495                .await
13496                .expect("reject tampered tuple");
13497            assert_eq!(rejected.outcome, Outcome::Corrupt);
13498            assert_eq!(backend.op_log().await.len(), probes_before, "invalid digest must not probe the backend");
13499            let applied = store
13500                .reconcile_legacy_transition_state(request)
13501                .await
13502                .expect("apply must report unavailable write authority");
13503            assert_eq!(applied.outcome, Outcome::BackendUnavailable, "{applied:?}");
13504            assert_eq!(applied.reason_code, "write_fence_unavailable");
13505            assert!(!applied.changed);
13506            for (path, expected) in paths.iter().zip(&original) {
13507                assert_eq!(tokio::fs::read(path).await.expect("xl.meta after inspection"), *expected);
13508            }
13509            assert_eq!(backend.remove_count().await, 0);
13510            assert!(
13511                backend
13512                    .op_log()
13513                    .await
13514                    .iter()
13515                    .all(|operation| matches!(operation, MockWarmOp::Probe { .. }))
13516            );
13517
13518            backend.set_unreachable(true).await;
13519            let unavailable = store
13520                .inspect_legacy_transition_state(LegacyTransitionStateReconcileSelector {
13521                    bucket: bucket.to_string(),
13522                    object: object.to_string(),
13523                    version_id: "null".to_string(),
13524                })
13525                .await
13526                .expect("unreachable tier is a diagnostic outcome");
13527            assert_eq!(unavailable.outcome, Outcome::BackendUnavailable);
13528            assert!(!unavailable.changed);
13529            backend.set_unreachable(false).await;
13530            for candidate in ["", "00000000-0000-0000-0000-000000000000", "bad\nversion"] {
13531                backend
13532                    .set_transition_candidate_probe_override(Some(
13533                        crate::services::tier::warm_backend::TransitionCandidateProbe::VersionedPresent(candidate.to_string()),
13534                    ))
13535                    .await;
13536                let invalid_proof = store
13537                    .inspect_legacy_transition_state(LegacyTransitionStateReconcileSelector {
13538                        bucket: bucket.to_string(),
13539                        object: object.to_string(),
13540                        version_id: "null".to_string(),
13541                    })
13542                    .await
13543                    .expect("invalid backend proof is a diagnostic outcome");
13544                assert_eq!(invalid_proof.outcome, Outcome::BackendUnavailable, "{invalid_proof:?}");
13545                assert!(invalid_proof.target.is_none());
13546            }
13547            backend.set_transition_candidate_probe_override(None).await;
13548
13549            backend.clear_op_log().await;
13550            for path in &paths[1..] {
13551                tokio::fs::remove_file(path)
13552                    .await
13553                    .expect("hide majority metadata copies in fixture");
13554            }
13555            let minority = store
13556                .inspect_legacy_transition_state(LegacyTransitionStateReconcileSelector {
13557                    bucket: bucket.to_string(),
13558                    object: object.to_string(),
13559                    version_id: "null".to_string(),
13560                })
13561                .await
13562                .expect("inspect minority legacy record");
13563            assert_eq!(
13564                minority.outcome,
13565                Outcome::BackendUnavailable,
13566                "a minority owner must remain visible: {minority:?}"
13567            );
13568            assert!(
13569                backend.op_log().await.is_empty(),
13570                "unproven metadata quorum cannot initiate a remote probe"
13571            );
13572            for (path, bytes) in paths.iter().zip(&original) {
13573                tokio::fs::write(path, bytes).await.expect("restore fixture copies");
13574            }
13575            tokio::fs::write(&paths[0], b"corrupt-xl-meta")
13576                .await
13577                .expect("inject corrupt copy");
13578            let corrupt = store
13579                .inspect_legacy_transition_state(LegacyTransitionStateReconcileSelector {
13580                    bucket: bucket.to_string(),
13581                    object: object.to_string(),
13582                    version_id: "null".to_string(),
13583                })
13584                .await
13585                .expect("inspect corrupt legacy record");
13586            assert_eq!(corrupt.outcome, Outcome::Corrupt, "{corrupt:?}");
13587            assert!(backend.op_log().await.is_empty(), "corruption must fail before backend I/O");
13588        }
13589    }
13590
13591    #[cfg(feature = "test-util")]
13592    #[tokio::test]
13593    #[serial_test::serial(storage_class_env)]
13594    async fn legacy_unknown_unversioned_transition_supports_head_get_and_range_without_backfill() {
13595        let temp_dir = tempfile::tempdir().expect("create legacy unknown unversioned store dir");
13596        let (ctx, store, _shutdown) =
13597            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "legacy-unknown-unversioned-read", &[4])).await;
13598        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
13599        let tier_name = "LEGACY-UNKNOWN-UNVERSIONED-READ";
13600        let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
13601        backend.set_put_remote_version(Some(String::new())).await;
13602        let bucket = "legacy-unknown-unversioned-read-bucket";
13603        let object = "object.bin";
13604        let payload = b"legacy unversioned remote tier object remains readable".repeat(1024);
13605        store
13606            .make_bucket(bucket, &MakeBucketOptions::default())
13607            .await
13608            .expect("legacy source bucket should be created");
13609        let mut reader = PutObjReader::from_vec(payload.clone());
13610        let source = store
13611            .put_object(bucket, object, &mut reader, &ObjectOptions::default())
13612            .await
13613            .expect("legacy source should be written");
13614        store
13615            .transition_object(
13616                bucket,
13617                object,
13618                &ObjectOptions {
13619                    transition: TransitionOptions {
13620                        status: TRANSITION_PENDING.to_string(),
13621                        tier: tier_name.to_string(),
13622                        etag: source.etag.clone().expect("legacy source should have an etag"),
13623                        ..Default::default()
13624                    },
13625                    mod_time: source.mod_time,
13626                    ..Default::default()
13627                },
13628            )
13629            .await
13630            .expect("legacy source should transition");
13631        rewrite_transitioned_xlmeta_as_legacy_unknown(temp_dir.path(), 0, bucket, object, true).await;
13632        backend.clear_op_log().await;
13633
13634        let opts = ObjectOptions {
13635            metadata_cache_safe: false,
13636            ..Default::default()
13637        };
13638        let head = store
13639            .get_object_info(bucket, object, &opts)
13640            .await
13641            .expect("legacy transitioned HEAD should use local metadata");
13642        assert_eq!(head.transition_version_state, rustfs_filemeta::TransitionVersionState::Unknown);
13643        assert!(head.transitioned_object.version_id.is_empty());
13644        assert_eq!(
13645            head.user_defined
13646                .get("x-minio-internal-transitioned-versionID")
13647                .map(String::as_str),
13648            Some(""),
13649            "the MinIO empty version-key provenance must survive xl.meta decoding"
13650        );
13651        assert!(
13652            !rustfs_utils::http::metadata_compat::contains_key_str(
13653                &head.user_defined,
13654                rustfs_utils::http::metadata_compat::SUFFIX_TRANSITIONED_VERSION_STATE,
13655            ),
13656            "the compatibility read must not synthesize version-state metadata"
13657        );
13658
13659        let full_body = read_store_body(&store, bucket, object, None, &opts).await;
13660        assert_eq!(full_body, payload);
13661
13662        let range = HTTPRangeSpec {
13663            is_suffix_length: false,
13664            start: 7,
13665            end: 38,
13666        };
13667        let ranged_body = read_store_body(&store, bucket, object, Some(range), &opts).await;
13668        assert_eq!(ranged_body, &payload[7..=38]);
13669
13670        let after_read = store.pools[0]
13671            .get_disks_by_key(object)
13672            .load_file_info_versions_exact(bucket, object)
13673            .await
13674            .expect("legacy metadata should remain readable after GET")
13675            .expect("legacy object metadata should remain on disk")
13676            .versions
13677            .into_iter()
13678            .find(|version| version.transition_status == rustfs_filemeta::TRANSITION_COMPLETE)
13679            .expect("legacy transitioned source should remain visible after GET");
13680        assert_eq!(after_read.transition_version_state, rustfs_filemeta::TransitionVersionState::Unknown);
13681        assert!(after_read.transition_version.is_none());
13682        assert!(after_read.transition_version_id.is_none());
13683        assert_eq!(
13684            after_read
13685                .metadata
13686                .get("x-minio-internal-transitioned-versionID")
13687                .map(String::as_str),
13688            Some(""),
13689            "the MinIO empty version-key provenance must remain after GET and Range GET"
13690        );
13691        assert!(
13692            !rustfs_utils::http::metadata_compat::contains_key_str(
13693                &after_read.metadata,
13694                rustfs_utils::http::metadata_compat::SUFFIX_TRANSITIONED_VERSION_STATE,
13695            ),
13696            "the compatibility read must remain side-effect free"
13697        );
13698
13699        assert_eq!(
13700            backend.op_log().await,
13701            vec![
13702                MockWarmOp::Probe {
13703                    object: after_read.transitioned_objname.clone(),
13704                },
13705                MockWarmOp::Get {
13706                    object: after_read.transitioned_objname.clone(),
13707                },
13708                MockWarmOp::Probe {
13709                    object: after_read.transitioned_objname.clone(),
13710                },
13711                MockWarmOp::Get {
13712                    object: after_read.transitioned_objname,
13713                },
13714            ],
13715            "legacy reads should probe before each unversioned GET and never mutate local metadata"
13716        );
13717        assert_eq!(backend.remove_count().await, 0);
13718    }
13719
13720    #[cfg(feature = "test-util")]
13721    #[tokio::test]
13722    #[serial_test::serial(storage_class_env)]
13723    async fn legacy_unknown_transition_delete_falls_back_for_single_batch_and_blocks_prefix() {
13724        let temp_dir = tempfile::tempdir().expect("create legacy transitioned delete store dir");
13725        let (ctx, store, _shutdown) =
13726            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "legacy-unknown-transitioned-delete", &[4]))
13727                .await;
13728        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
13729        let tier_name = "LEGACY-UNKNOWN-DELETE";
13730        let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
13731        let bucket = "legacy-unknown-transitioned-delete-bucket";
13732        store
13733            .make_bucket(bucket, &MakeBucketOptions::default())
13734            .await
13735            .expect("legacy source bucket should be created");
13736
13737        for (index, object) in ["single.bin", "batch.bin", "prefix/legacy.bin"].into_iter().enumerate() {
13738            let mut reader = PutObjReader::from_vec(vec![b'l' + index as u8; 1024 * 1024]);
13739            let source = store
13740                .put_object(bucket, object, &mut reader, &ObjectOptions::default())
13741                .await
13742                .expect("legacy source should be written");
13743            store
13744                .transition_object(
13745                    bucket,
13746                    object,
13747                    &ObjectOptions {
13748                        transition: TransitionOptions {
13749                            status: TRANSITION_PENDING.to_string(),
13750                            tier: tier_name.to_string(),
13751                            etag: source.etag.clone().expect("legacy source should have an etag"),
13752                            ..Default::default()
13753                        },
13754                        mod_time: source.mod_time,
13755                        ..Default::default()
13756                    },
13757                )
13758                .await
13759                .expect("legacy source should transition");
13760            rewrite_transitioned_xlmeta_as_legacy_unknown(temp_dir.path(), 0, bucket, object, false).await;
13761            let legacy = store.pools[0]
13762                .get_disks_by_key(object)
13763                .load_file_info_versions_exact(bucket, object)
13764                .await
13765                .expect("legacy source metadata should remain decodable")
13766                .expect("legacy source should remain on disk")
13767                .versions
13768                .into_iter()
13769                .find(|version| version.transition_status == rustfs_filemeta::TRANSITION_COMPLETE)
13770                .expect("legacy transitioned source should remain visible before delete");
13771            assert_eq!(legacy.transition_version_state, rustfs_filemeta::TransitionVersionState::Unknown);
13772            assert_eq!(
13773                crate::services::tier::tier::tier_destination_id_from_metadata(&legacy.metadata)
13774                    .expect("legacy metadata identity lookup should not fail"),
13775                None
13776            );
13777        }
13778
13779        backend.set_remove_failure(true);
13780        store
13781            .delete_object_with_tier_delete_journal(bucket, "single.bin", ObjectOptions::default())
13782            .await
13783            .expect("legacy single delete should retain its free-version fallback");
13784        let free_version_prefix_error = store
13785            .delete_object_with_tier_delete_journal(
13786                bucket,
13787                "single.bin",
13788                ObjectOptions {
13789                    delete_prefix: true,
13790                    ..Default::default()
13791                },
13792            )
13793            .await
13794            .expect_err("recursive prefix delete must not discard an existing hidden cleanup owner");
13795        assert!(
13796            free_version_prefix_error
13797                .to_string()
13798                .contains("tier free-version cleanup obligation"),
13799            "unexpected hidden-owner prefix failure: {free_version_prefix_error}"
13800        );
13801        let (_deleted, errors) = store
13802            .delete_objects_with_tier_delete_journal(
13803                bucket,
13804                vec![ObjectToDelete {
13805                    object_name: "batch.bin".to_string(),
13806                    ..Default::default()
13807                }],
13808                ObjectOptions::default(),
13809            )
13810            .await;
13811        assert!(errors.iter().all(Option::is_none), "legacy batch delete should succeed: {errors:?}");
13812        let prefix_error = store
13813            .delete_object_with_tier_delete_journal(
13814                bucket,
13815                "prefix/",
13816                ObjectOptions {
13817                    delete_prefix: true,
13818                    ..Default::default()
13819                },
13820            )
13821            .await
13822            .expect_err("legacy recursive prefix delete must fail closed before physical removal");
13823        assert!(
13824            prefix_error.to_string().contains("legacy Unknown"),
13825            "unexpected prefix failure: {prefix_error}"
13826        );
13827
13828        assert_eq!(
13829            tier_delete_journal_count(store.clone()).await,
13830            0,
13831            "legacy cleanup must not create an unsafe journal"
13832        );
13833        assert_eq!(
13834            backend.object_count().await,
13835            3,
13836            "failed free-version cleanup must retain all remote objects"
13837        );
13838        for object in ["single.bin", "batch.bin"] {
13839            let exact = store.pools[0]
13840                .get_disks_by_key(object)
13841                .load_file_info_versions_exact(bucket, object)
13842                .await
13843                .expect("legacy post-delete metadata should decode")
13844                .expect("legacy cleanup owner should remain on disk");
13845            assert_eq!(
13846                exact
13847                    .versions
13848                    .iter()
13849                    .chain(exact.free_versions.iter())
13850                    .filter(|version| version.tier_free_version())
13851                    .count(),
13852                1,
13853                "legacy {object} must retain exactly one hidden cleanup owner"
13854            );
13855            assert_eq!(
13856                exact.versions.iter().filter(|version| !version.tier_free_version()).count(),
13857                0,
13858                "legacy {object} must have no visible source after delete"
13859            );
13860        }
13861        let retained_prefix = store.pools[0]
13862            .get_disks_by_key("prefix/legacy.bin")
13863            .load_file_info_versions_exact(bucket, "prefix/legacy.bin")
13864            .await
13865            .expect("rejected prefix metadata should decode")
13866            .expect("rejected prefix delete must retain its source");
13867        assert_eq!(
13868            retained_prefix
13869                .versions
13870                .iter()
13871                .filter(|version| !version.tier_free_version())
13872                .count(),
13873            1
13874        );
13875    }
13876
13877    #[cfg(feature = "test-util")]
13878    #[tokio::test]
13879    #[serial_test::serial(storage_class_env)]
13880    async fn stable_transitioned_recursive_prefix_delete_uses_journal_owners() {
13881        let temp_dir = tempfile::tempdir().expect("create stable transitioned prefix-delete store dir");
13882        let (ctx, store, _shutdown) =
13883            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "stable-transitioned-prefix-delete", &[4]))
13884                .await;
13885        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
13886        let tier_name = "STABLE-PREFIX-DELETE";
13887        let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
13888        let bucket = "stable-transitioned-prefix-delete-bucket";
13889        store
13890            .make_bucket(bucket, &MakeBucketOptions::default())
13891            .await
13892            .expect("stable prefix source bucket should be created");
13893
13894        let objects = ["prefix/first.bin", "prefix/second.bin"];
13895        for (index, object) in objects.into_iter().enumerate() {
13896            let mut reader = PutObjReader::from_vec(vec![b'a' + index as u8; 1024 * 1024]);
13897            let source = store
13898                .put_object(bucket, object, &mut reader, &ObjectOptions::default())
13899                .await
13900                .expect("stable prefix source should be written");
13901            store
13902                .transition_object(
13903                    bucket,
13904                    object,
13905                    &ObjectOptions {
13906                        transition: TransitionOptions {
13907                            status: TRANSITION_PENDING.to_string(),
13908                            tier: tier_name.to_string(),
13909                            etag: source.etag.clone().expect("stable prefix source should have an etag"),
13910                            ..Default::default()
13911                        },
13912                        mod_time: source.mod_time,
13913                        ..Default::default()
13914                    },
13915                )
13916                .await
13917                .expect("stable prefix source should transition");
13918        }
13919        assert_eq!(backend.object_count().await, objects.len());
13920        backend.set_remove_failure(true);
13921
13922        store
13923            .delete_object_with_tier_delete_journal(
13924                bucket,
13925                "prefix/",
13926                ObjectOptions {
13927                    delete_prefix: true,
13928                    ..Default::default()
13929                },
13930            )
13931            .await
13932            .expect("stable recursive prefix delete should use journal ownership");
13933
13934        assert_eq!(tier_delete_journal_count(store.clone()).await, objects.len());
13935        assert_eq!(backend.object_count().await, objects.len());
13936        for object in objects {
13937            assert!(
13938                store.pools[0]
13939                    .get_disks_by_key(object)
13940                    .load_file_info_versions_exact(bucket, object)
13941                    .await
13942                    .expect("stable prefix metadata lookup should succeed")
13943                    .is_none(),
13944                "stable recursive delete must leave neither a visible source nor a free-version for {object}"
13945            );
13946        }
13947
13948        backend.set_remove_failure(false);
13949        let stats = recover_tier_delete_journal_entries(store.clone(), 100, None)
13950            .await
13951            .expect("stable prefix journals should recover");
13952        assert_eq!(stats.failed, 0);
13953        assert!(stats.scanned <= objects.len() && stats.deleted <= objects.len());
13954        assert_eq!(tier_delete_journal_count(store).await, 0);
13955        assert_eq!(backend.object_count().await, 0);
13956        assert_eq!(backend.remove_versions().await.len(), objects.len());
13957    }
13958
13959    #[cfg(feature = "test-util")]
13960    #[test]
13961    #[serial_test::serial(storage_class_env)]
13962    fn tier_delete_prefix_limit_and_multi_chunk_batches_converge() {
13963        run_large_stack_async_test(
13964            "tier-delete-prefix-limit-and-multi-chunk",
13965            tier_delete_prefix_limit_and_multi_chunk_batches_converge_case,
13966        );
13967    }
13968
13969    #[cfg(feature = "test-util")]
13970    async fn tier_delete_prefix_limit_and_multi_chunk_batches_converge_case() {
13971        let _batch_limit = TierDeleteDispatchBatchLimitGuard::install(2);
13972        let temp_dir = tempfile::tempdir().expect("create chunked prefix-delete store dir");
13973        let (ctx, store, shutdown) =
13974            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "chunked-prefix-delete", &[4])).await;
13975        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
13976        let tier_name = "CHUNKED-PREFIX-DELETE";
13977        let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
13978        let bucket = "chunked-prefix-delete-bucket";
13979        store
13980            .make_bucket(bucket, &MakeBucketOptions::default())
13981            .await
13982            .expect("chunked prefix source bucket should be created");
13983
13984        for (prefix, count) in [("at-limit", 2), ("limit-plus-one", 3), ("multi-chunk", 5)] {
13985            for index in 0..count {
13986                let object = format!("{prefix}/object-{index}.bin");
13987                let mut reader = PutObjReader::from_vec(vec![b'a' + index as u8; 1024 * 1024]);
13988                let source = store
13989                    .put_object(bucket, &object, &mut reader, &ObjectOptions::default())
13990                    .await
13991                    .expect("chunked prefix source should be written");
13992                store
13993                    .transition_object(
13994                        bucket,
13995                        &object,
13996                        &ObjectOptions {
13997                            transition: TransitionOptions {
13998                                status: TRANSITION_PENDING.to_string(),
13999                                tier: tier_name.to_string(),
14000                                etag: source.etag.clone().expect("chunked prefix source should have an etag"),
14001                                ..Default::default()
14002                            },
14003                            mod_time: source.mod_time,
14004                            ..Default::default()
14005                        },
14006                    )
14007                    .await
14008                    .expect("chunked prefix source should transition");
14009            }
14010        }
14011        backend.set_remove_failure(true);
14012
14013        store
14014            .delete_object_with_tier_delete_journal(
14015                bucket,
14016                "at-limit/",
14017                ObjectOptions {
14018                    delete_prefix: true,
14019                    ..Default::default()
14020                },
14021            )
14022            .await
14023            .expect("exactly one batch must retain the v1 one-shot path");
14024
14025        let limit_plus_one_first = store
14026            .delete_object_with_tier_delete_journal(
14027                bucket,
14028                "limit-plus-one/",
14029                ObjectOptions {
14030                    delete_prefix: true,
14031                    ..Default::default()
14032                },
14033            )
14034            .await
14035            .expect_err("limit plus one must start a bounded parent transaction");
14036        assert!(
14037            limit_plus_one_first.to_string().contains("retry the next durable batch"),
14038            "unexpected first limit-plus-one result: {limit_plus_one_first}"
14039        );
14040        let active_limit_plus_one_records = store
14041            .clone()
14042            .list_objects_v2(
14043                RUSTFS_META_BUCKET,
14044                TIER_DELETE_DISPATCH_MANIFEST_PREFIX,
14045                None,
14046                None,
14047                10,
14048                false,
14049                None,
14050                false,
14051            )
14052            .await
14053            .expect("active limit-plus-one records should be listable");
14054        let mut active_limit_plus_one_parent_seen = false;
14055        for record in active_limit_plus_one_records
14056            .objects
14057            .iter()
14058            .filter(|record| !record.name.contains("/chunks/"))
14059        {
14060            let data = com::read_config(store.clone(), &record.name)
14061                .await
14062                .expect("an active dispatch root should be readable");
14063            let value: serde_json::Value = serde_json::from_slice(&data).expect("an active dispatch root should contain JSON");
14064            if value["prefix"] == "limit-plus-one/" {
14065                assert_eq!(value["record_type"], "chunked_parent");
14066                active_limit_plus_one_parent_seen = true;
14067            }
14068        }
14069        assert!(
14070            active_limit_plus_one_parent_seen,
14071            "limit plus one must establish the fail-closed parent at the legacy root"
14072        );
14073
14074        let mut limit_plus_one_completed = false;
14075        let mut limit_plus_one_retries = 1;
14076        for _ in 0..3 {
14077            match store
14078                .delete_object_with_tier_delete_journal(
14079                    bucket,
14080                    "limit-plus-one/",
14081                    ObjectOptions {
14082                        delete_prefix: true,
14083                        ..Default::default()
14084                    },
14085                )
14086                .await
14087            {
14088                Ok(_) => {
14089                    limit_plus_one_completed = true;
14090                    break;
14091                }
14092                Err(err) if err.to_string().contains("retry the next durable batch") => {
14093                    limit_plus_one_retries += 1;
14094                }
14095                Err(err) => panic!("limit-plus-one delete returned an unexpected error: {err}"),
14096            }
14097        }
14098        assert!(limit_plus_one_completed, "limit plus one must converge through two bounded children");
14099        assert_eq!(limit_plus_one_retries, 2, "limit plus one must require exactly two child batches");
14100
14101        let first_batch = store
14102            .delete_object_with_tier_delete_journal(
14103                bucket,
14104                "multi-chunk/",
14105                ObjectOptions {
14106                    delete_prefix: true,
14107                    ..Default::default()
14108                },
14109            )
14110            .await
14111            .expect_err("the first bounded child must request a successor batch");
14112        assert!(
14113            first_batch.to_string().contains("retry the next durable batch"),
14114            "unexpected first child result: {first_batch}"
14115        );
14116
14117        let newcomer = "multi-chunk/zzz-newcomer.bin";
14118        let mut newcomer_reader = PutObjReader::from_vec(vec![b'n'; 1024 * 1024]);
14119        let newcomer_source = store
14120            .put_object(bucket, newcomer, &mut newcomer_reader, &ObjectOptions::default())
14121            .await
14122            .expect("a source created between chunks should be written");
14123        store
14124            .transition_object(
14125                bucket,
14126                newcomer,
14127                &ObjectOptions {
14128                    transition: TransitionOptions {
14129                        status: TRANSITION_PENDING.to_string(),
14130                        tier: tier_name.to_string(),
14131                        etag: newcomer_source
14132                            .etag
14133                            .clone()
14134                            .expect("the between-chunks source should have an etag"),
14135                        ..Default::default()
14136                    },
14137                    mod_time: newcomer_source.mod_time,
14138                    ..Default::default()
14139                },
14140            )
14141            .await
14142            .expect("a source created between chunks should transition");
14143
14144        let mut completed = false;
14145        let mut durable_batch_retries = 1;
14146        for _ in 0..7 {
14147            match store
14148                .delete_object_with_tier_delete_journal(
14149                    bucket,
14150                    "multi-chunk/",
14151                    ObjectOptions {
14152                        delete_prefix: true,
14153                        ..Default::default()
14154                    },
14155                )
14156                .await
14157            {
14158                Ok(_) => {
14159                    completed = true;
14160                    break;
14161                }
14162                Err(err)
14163                    if err.to_string().contains("retry the next durable batch")
14164                        || err.to_string().contains("retry the next batch") =>
14165                {
14166                    durable_batch_retries += 1;
14167                }
14168                Err(err) => panic!("chunked prefix delete returned an unexpected error: {err}"),
14169            }
14170        }
14171        assert!(completed, "six entries must converge through three bounded child batches");
14172        assert_eq!(durable_batch_retries, 3, "limit two should require exactly three child batches");
14173        assert_eq!(tier_delete_journal_count(store.clone()).await, 11);
14174        assert_eq!(
14175            backend.object_count().await,
14176            11,
14177            "remote cleanup must remain durable while the tier is unavailable"
14178        );
14179        let dispatch_records = store
14180            .clone()
14181            .list_objects_v2(
14182                RUSTFS_META_BUCKET,
14183                TIER_DELETE_DISPATCH_MANIFEST_PREFIX,
14184                None,
14185                None,
14186                20,
14187                false,
14188                None,
14189                false,
14190            )
14191            .await
14192            .expect("bounded dispatch records should be listable");
14193        let mut child_batches = 0;
14194        let mut legacy_at_limit_seen = false;
14195        for record in &dispatch_records.objects {
14196            let data = com::read_config(store.clone(), &record.name)
14197                .await
14198                .expect("a retained dispatch record should be readable");
14199            let value: serde_json::Value = serde_json::from_slice(&data).expect("a retained dispatch record should contain JSON");
14200            if record.name.contains("/chunks/") {
14201                let journal_count = value["journal_count"]
14202                    .as_u64()
14203                    .expect("a retained child should declare its journal count");
14204                assert!(journal_count <= 2, "a child batch exceeded the configured resource bound");
14205                child_batches += 1;
14206            } else if value["prefix"] == "at-limit/" {
14207                assert!(
14208                    value.get("record_type").is_none(),
14209                    "the exact-limit root must remain a legacy v1 manifest"
14210                );
14211                assert_eq!(value["journal_count"], 2);
14212                legacy_at_limit_seen = true;
14213            }
14214        }
14215        assert!(
14216            legacy_at_limit_seen,
14217            "the exact-limit dispatch must retain its byte-compatible root shape"
14218        );
14219        assert_eq!(child_batches, 5, "nine chunked sources should persist exactly five bounded children");
14220
14221        backend.set_remove_failure(false);
14222        drive_tier_delete_dispatch_restart_to_convergence(store.clone()).await;
14223        assert_eq!(tier_delete_journal_count(store.clone()).await, 0);
14224        assert_eq!(tier_delete_dispatch_manifest_count(store.clone()).await, 0);
14225        assert_eq!(backend.object_count().await, 0);
14226        assert_eq!(
14227            backend.remove_versions().await.len(),
14228            11,
14229            "each remote version must be removed exactly once"
14230        );
14231        shutdown.cancel();
14232    }
14233
14234    #[cfg(feature = "test-util")]
14235    #[test]
14236    #[serial_test::serial(storage_class_env)]
14237    fn tier_delete_chunk_crash_boundaries_resume_without_skipping_sources() {
14238        run_large_stack_async_test(
14239            "tier-delete-chunk-crash-boundaries",
14240            tier_delete_chunk_crash_boundaries_resume_without_skipping_sources_case,
14241        );
14242    }
14243
14244    #[cfg(feature = "test-util")]
14245    async fn tier_delete_chunk_crash_boundaries_resume_without_skipping_sources_case() {
14246        let _batch_limit = TierDeleteDispatchBatchLimitGuard::install(1);
14247        let stages = [
14248            TierDeleteChunkTestStage::ParentPersisted,
14249            TierDeleteChunkTestStage::ChildManifestPersisted,
14250            TierDeleteChunkTestStage::ParentBound,
14251            TierDeleteChunkTestStage::DispatchAuthorized,
14252            TierDeleteChunkTestStage::LocalReplayCompleted,
14253            TierDeleteChunkTestStage::ChildCompleted,
14254            TierDeleteChunkTestStage::ParentProgressed,
14255            TierDeleteChunkTestStage::FinalLocalDeletionCompleted,
14256            TierDeleteChunkTestStage::ParentCompleted,
14257        ];
14258        for (case, stage) in stages.into_iter().enumerate() {
14259            let temp_dir = tempfile::tempdir().expect("create chunk crash-boundary store dir");
14260            let initial_name = format!("chunk-crash-boundary-{case}");
14261            let (ctx, store, shutdown) =
14262                without_storage_class_env(build_isolated_test_store(temp_dir.path(), &initial_name, &[4])).await;
14263            crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
14264            let tier_name = format!("CHUNK-CRASH-BOUNDARY-{case}");
14265            let backend = register_mock_tier(&ctx.tier_config_mgr(), &tier_name).await;
14266            backend.set_remove_failure(true);
14267            let bucket = format!("chunk-crash-boundary-{case}-bucket");
14268            let prefix = "prefix/";
14269            store
14270                .make_bucket(&bucket, &MakeBucketOptions::default())
14271                .await
14272                .expect("chunk crash-boundary bucket should be created");
14273            for index in 0..2 {
14274                let object = format!("{prefix}object-{index}.bin");
14275                let mut reader = PutObjReader::from_vec(vec![b'a' + index as u8; 1024 * 1024]);
14276                let source = store
14277                    .put_object(&bucket, &object, &mut reader, &ObjectOptions::default())
14278                    .await
14279                    .expect("chunk crash-boundary source should be written");
14280                store
14281                    .transition_object(
14282                        &bucket,
14283                        &object,
14284                        &ObjectOptions {
14285                            transition: TransitionOptions {
14286                                status: TRANSITION_PENDING.to_string(),
14287                                tier: tier_name.clone(),
14288                                etag: source.etag.clone().expect("chunk crash-boundary source should have an etag"),
14289                                ..Default::default()
14290                            },
14291                            mod_time: source.mod_time,
14292                            ..Default::default()
14293                        },
14294                    )
14295                    .await
14296                    .expect("chunk crash-boundary source should transition");
14297            }
14298            let local_only = format!("{prefix}local-only.bin");
14299            let mut local_reader = PutObjReader::from_vec(vec![b'l'; 1024 * 1024]);
14300            store
14301                .put_object(&bucket, &local_only, &mut local_reader, &ObjectOptions::default())
14302                .await
14303                .expect("chunk crash-boundary local-only object should be written");
14304            let tier_config = ctx
14305                .tier_config_mgr()
14306                .read()
14307                .await
14308                .tiers
14309                .get(&tier_name)
14310                .expect("chunk crash-boundary tier config should remain available for restart")
14311                .clone_with_credentials();
14312
14313            if matches!(
14314                stage,
14315                TierDeleteChunkTestStage::FinalLocalDeletionCompleted | TierDeleteChunkTestStage::ParentCompleted
14316            ) {
14317                for _ in 0..2 {
14318                    let retry = store
14319                        .delete_object_with_tier_delete_journal(
14320                            &bucket,
14321                            prefix,
14322                            ObjectOptions {
14323                                delete_prefix: true,
14324                                ..Default::default()
14325                            },
14326                        )
14327                        .await
14328                        .expect_err("each bounded child must complete before the final crash boundary");
14329                    assert!(
14330                        retry.to_string().contains("retry the next durable batch"),
14331                        "unexpected bounded-child result before {stage:?}: {retry}"
14332                    );
14333                }
14334            }
14335
14336            let barrier = TierDeleteChunkTestBarrier::install(stage);
14337            let worker_store = store.clone();
14338            let worker_bucket = bucket.clone();
14339            let worker = tokio::spawn(async move {
14340                worker_store
14341                    .delete_object_with_tier_delete_journal(
14342                        &worker_bucket,
14343                        prefix,
14344                        ObjectOptions {
14345                            delete_prefix: true,
14346                            ..Default::default()
14347                        },
14348                    )
14349                    .await
14350            });
14351            tokio::time::timeout(Duration::from_secs(30), barrier.wait_until_paused())
14352                .await
14353                .unwrap_or_else(|_| panic!("chunk delete did not reach crash boundary {stage:?}"));
14354            worker.abort();
14355            let _ = worker.await;
14356            drop(barrier);
14357            let released_bucket_guard =
14358                tokio::time::timeout(Duration::from_secs(5), store.acquire_bucket_lifecycle_write_lock(&bucket))
14359                    .await
14360                    .unwrap_or_else(|_| panic!("canceling at {stage:?} did not release the bucket lifecycle lock"))
14361                    .unwrap_or_else(|err| {
14362                        panic!("bucket lifecycle lock reacquire failed after cancellation at {stage:?}: {err}")
14363                    });
14364            drop(released_bucket_guard);
14365            shutdown.cancel();
14366            drop(store);
14367            drop(ctx);
14368
14369            let restarted_name = format!("chunk-crash-boundary-{case}-restart");
14370            let (restarted_ctx, restarted_store, restarted_shutdown) =
14371                without_storage_class_env(build_isolated_test_store(temp_dir.path(), &restarted_name, &[4])).await;
14372            {
14373                let tier_config_mgr = restarted_ctx.tier_config_mgr();
14374                let mut manager = tier_config_mgr.write().await;
14375                manager.tiers.insert(tier_name.clone(), tier_config);
14376                manager
14377                    .install_test_driver(&tier_name, Box::new(backend.clone()))
14378                    .expect("the exact chunk crash-boundary tier driver should reinstall after restart");
14379            }
14380            crate::bucket::metadata_sys::init_bucket_metadata_sys(restarted_store.clone(), Vec::new()).await;
14381
14382            let mut completed = false;
14383            for _ in 0..20 {
14384                let recovery = recover_tier_delete_dispatch_manifests(restarted_store.clone(), 100, None)
14385                    .await
14386                    .expect("chunk crash-boundary manifest recovery should remain readable");
14387                assert_eq!(recovery.failed, 0, "recovery must not quarantine a valid chunk boundary");
14388                match restarted_store
14389                    .delete_object_with_tier_delete_journal(
14390                        &bucket,
14391                        prefix,
14392                        ObjectOptions {
14393                            delete_prefix: true,
14394                            ..Default::default()
14395                        },
14396                    )
14397                    .await
14398                {
14399                    Ok(_) => {
14400                        completed = true;
14401                        break;
14402                    }
14403                    Err(err)
14404                        if err.to_string().contains("retry")
14405                            || err.to_string().contains("rollback")
14406                            || err.to_string().contains("durable cleanup") => {}
14407                    Err(err) => panic!("chunk crash-boundary retry returned an unexpected error at {stage:?}: {err}"),
14408                }
14409            }
14410            assert!(completed, "chunk state must converge after cancellation at {stage:?}");
14411            for index in 0..2 {
14412                let object = format!("{prefix}object-{index}.bin");
14413                assert!(
14414                    restarted_store.pools[0]
14415                        .get_disks_by_key(&object)
14416                        .load_file_info_versions_exact(&bucket, &object)
14417                        .await
14418                        .expect("chunk crash-boundary source lookup should succeed")
14419                        .is_none(),
14420                    "source {object} must not be skipped after cancellation at {stage:?}"
14421                );
14422            }
14423            assert!(
14424                restarted_store.pools[0]
14425                    .get_disks_by_key(&local_only)
14426                    .load_file_info_versions_exact(&bucket, &local_only)
14427                    .await
14428                    .expect("chunk crash-boundary local-only source lookup should succeed")
14429                    .is_none(),
14430                "the final raw delete must remove the local-only source after cancellation at {stage:?}"
14431            );
14432            assert_eq!(backend.object_count().await, 2);
14433            backend.set_remove_failure(false);
14434            drive_tier_delete_dispatch_restart_to_convergence(restarted_store.clone()).await;
14435            assert_eq!(tier_delete_journal_count(restarted_store.clone()).await, 0);
14436            assert_eq!(tier_delete_dispatch_manifest_count(restarted_store.clone()).await, 0);
14437            assert_eq!(backend.object_count().await, 0);
14438            assert_eq!(
14439                backend.remove_versions().await.len(),
14440                2,
14441                "each crash case must retain exactly one cleanup owner per remote version"
14442            );
14443            restarted_shutdown.cancel();
14444        }
14445    }
14446
14447    #[cfg(feature = "test-util")]
14448    #[test]
14449    #[serial_test::serial(storage_class_env)]
14450    fn tier_delete_chunk_missing_child_with_journals_fails_closed() {
14451        run_large_stack_async_test(
14452            "tier-delete-chunk-missing-child",
14453            tier_delete_chunk_missing_child_with_journals_fails_closed_case,
14454        );
14455    }
14456
14457    #[cfg(feature = "test-util")]
14458    async fn tier_delete_chunk_missing_child_with_journals_fails_closed_case() {
14459        let _batch_limit = TierDeleteDispatchBatchLimitGuard::install(1);
14460        let temp_dir = tempfile::tempdir().expect("create missing-child store dir");
14461        let (ctx, store, shutdown) =
14462            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "chunk-missing-child", &[4])).await;
14463        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
14464        let tier_name = "CHUNK-MISSING-CHILD";
14465        let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
14466        backend.set_remove_failure(true);
14467        let bucket = "chunk-missing-child-bucket";
14468        let prefix = "prefix/";
14469        store
14470            .make_bucket(bucket, &MakeBucketOptions::default())
14471            .await
14472            .expect("missing-child bucket should be created");
14473        for index in 0..2 {
14474            let object = format!("{prefix}object-{index}.bin");
14475            let mut reader = PutObjReader::from_vec(vec![b'm' + index as u8; 1024 * 1024]);
14476            let source = store
14477                .put_object(bucket, &object, &mut reader, &ObjectOptions::default())
14478                .await
14479                .expect("missing-child source should be written");
14480            store
14481                .transition_object(
14482                    bucket,
14483                    &object,
14484                    &ObjectOptions {
14485                        transition: TransitionOptions {
14486                            status: TRANSITION_PENDING.to_string(),
14487                            tier: tier_name.to_string(),
14488                            etag: source.etag.clone().expect("missing-child source should have an etag"),
14489                            ..Default::default()
14490                        },
14491                        mod_time: source.mod_time,
14492                        ..Default::default()
14493                    },
14494                )
14495                .await
14496                .expect("missing-child source should transition");
14497        }
14498
14499        let barrier = TierDeleteChunkTestBarrier::install(TierDeleteChunkTestStage::DispatchAuthorized);
14500        let worker_store = store.clone();
14501        let worker = tokio::spawn(async move {
14502            worker_store
14503                .delete_object_with_tier_delete_journal(
14504                    bucket,
14505                    prefix,
14506                    ObjectOptions {
14507                        delete_prefix: true,
14508                        ..Default::default()
14509                    },
14510                )
14511                .await
14512        });
14513        tokio::time::timeout(Duration::from_secs(30), barrier.wait_until_paused())
14514            .await
14515            .expect("chunk delete should persist its authorization before corruption injection");
14516        worker.abort();
14517        let _ = worker.await;
14518        drop(barrier);
14519
14520        let records = store
14521            .clone()
14522            .list_objects_v2(
14523                RUSTFS_META_BUCKET,
14524                TIER_DELETE_DISPATCH_MANIFEST_PREFIX,
14525                None,
14526                None,
14527                10,
14528                false,
14529                None,
14530                false,
14531            )
14532            .await
14533            .expect("parent and child records should be listable");
14534        let child = records
14535            .objects
14536            .iter()
14537            .find(|object| object.name.contains("/chunks/"))
14538            .expect("the bound child manifest should exist")
14539            .name
14540            .clone();
14541        com::delete_config(store.clone(), &child)
14542            .await
14543            .expect("the test should remove only the child manifest");
14544        assert_eq!(tier_delete_journal_count(store.clone()).await, 1);
14545
14546        let error = store
14547            .delete_object_with_tier_delete_journal(
14548                bucket,
14549                prefix,
14550                ObjectOptions {
14551                    delete_prefix: true,
14552                    ..Default::default()
14553                },
14554            )
14555            .await
14556            .expect_err("a missing child with retained journals must quarantine the parent");
14557        assert!(
14558            error.to_string().contains("missing child with retained journals"),
14559            "unexpected missing-child result: {error}"
14560        );
14561        assert_eq!(backend.object_count().await, 2, "fail-closed inspection must not remove a remote version");
14562        shutdown.cancel();
14563    }
14564
14565    #[cfg(feature = "test-util")]
14566    #[test]
14567    #[serial_test::serial(storage_class_env)]
14568    fn authorized_prefix_retry_replays_predecessor_before_newcomer() {
14569        let handle = std::thread::Builder::new()
14570            .name("authorized-prefix-predecessor-replay-test".to_string())
14571            .stack_size(32 * 1024 * 1024)
14572            .spawn(|| {
14573                let runtime = tokio::runtime::Builder::new_multi_thread()
14574                    .enable_all()
14575                    .worker_threads(2)
14576                    .build()
14577                    .expect("authorized predecessor replay test runtime should build");
14578                runtime.block_on(authorized_prefix_retry_replays_predecessor_before_newcomer_case());
14579            })
14580            .expect("authorized predecessor replay test thread should spawn");
14581        if let Err(payload) = handle.join() {
14582            std::panic::resume_unwind(payload);
14583        }
14584    }
14585
14586    #[cfg(feature = "test-util")]
14587    async fn authorized_prefix_retry_replays_predecessor_before_newcomer_case() {
14588        let _batch_limit = TierDeleteDispatchBatchLimitGuard::install(1);
14589        let temp_dir = tempfile::tempdir().expect("create authorized predecessor replay store dir");
14590        let (ctx, store, _shutdown) =
14591            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "authorized-prefix-predecessor-replay", &[4]))
14592                .await;
14593        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
14594        let tier_name = "AUTHORIZED-PREFIX-REPLAY";
14595        let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
14596        let lease = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
14597            .await
14598            .expect("authorized replay tier lease should resolve");
14599        let bucket = "authorized-prefix-predecessor-replay-bucket";
14600        let prefix = "prefix/";
14601        let predecessor = "prefix/000-predecessor.bin";
14602        let newcomer = "prefix/zzz-newcomer.bin";
14603        store
14604            .make_bucket(bucket, &MakeBucketOptions::default())
14605            .await
14606            .expect("authorized replay bucket should be created");
14607
14608        let mut predecessor_reader = PutObjReader::from_vec(vec![b'p'; 1024 * 1024]);
14609        let predecessor_source = store
14610            .put_object(bucket, predecessor, &mut predecessor_reader, &ObjectOptions::default())
14611            .await
14612            .expect("predecessor source should be written");
14613        store
14614            .transition_object(
14615                bucket,
14616                predecessor,
14617                &ObjectOptions {
14618                    transition: TransitionOptions {
14619                        status: TRANSITION_PENDING.to_string(),
14620                        tier: tier_name.to_string(),
14621                        etag: predecessor_source.etag.clone().expect("predecessor should have an etag"),
14622                        ..Default::default()
14623                    },
14624                    mod_time: predecessor_source.mod_time,
14625                    ..Default::default()
14626                },
14627            )
14628            .await
14629            .expect("predecessor should transition");
14630        let predecessor_info = store
14631            .get_object_info(bucket, predecessor, &ObjectOptions::default())
14632            .await
14633            .expect("transitioned predecessor should be readable");
14634        let mut predecessor_entry =
14635            transitioned_delete_journal_entry_for_source(None, false, false, bucket, predecessor, &predecessor_info)
14636                .expect("transitioned predecessor should produce a dispatch journal");
14637        predecessor_entry.backend_identity = Some(lease.backend_identity());
14638        let incarnation = store
14639            .bucket_incarnation_id(bucket)
14640            .await
14641            .expect("authorized replay bucket incarnation should resolve");
14642        let (manifest_name, _) = install_test_tier_delete_dispatch_fixture(
14643            store.clone(),
14644            bucket,
14645            incarnation,
14646            prefix,
14647            vec![(predecessor_entry, Some(TierDeleteJournalState::Dispatched))],
14648            TierDeleteDispatchManifestState::DispatchAuthorized,
14649        )
14650        .await
14651        .expect("authorized predecessor fixture should persist");
14652
14653        let mut newcomer_reader = PutObjReader::from_vec(vec![b'n'; 1024 * 1024]);
14654        let newcomer_source = store
14655            .put_object(bucket, newcomer, &mut newcomer_reader, &ObjectOptions::default())
14656            .await
14657            .expect("newcomer source should be written after authorization");
14658        store
14659            .transition_object(
14660                bucket,
14661                newcomer,
14662                &ObjectOptions {
14663                    transition: TransitionOptions {
14664                        status: TRANSITION_PENDING.to_string(),
14665                        tier: tier_name.to_string(),
14666                        etag: newcomer_source.etag.clone().expect("newcomer should have an etag"),
14667                        ..Default::default()
14668                    },
14669                    mod_time: newcomer_source.mod_time,
14670                    ..Default::default()
14671                },
14672            )
14673            .await
14674            .expect("newcomer should transition");
14675        backend.set_remove_failure(true);
14676
14677        let publication_epoch = store
14678            .scanner_data_usage_publication_epoch()
14679            .await
14680            .expect("authorized replay should observe the current scanner publication epoch");
14681        let publication_scope = store
14682            .scanner_data_usage_publication_commit_scope(
14683                publication_epoch,
14684                tokio::time::Instant::now() + Duration::from_secs(30),
14685                Vec::new(),
14686            )
14687            .await
14688            .expect("authorized replay should acquire a scanner publication scope");
14689
14690        let retry = store
14691            .delete_object_with_tier_delete_journal(
14692                bucket,
14693                prefix,
14694                ObjectOptions {
14695                    delete_prefix: true,
14696                    scanner_publication_commit_scope: Some(publication_scope.clone()),
14697                    ..Default::default()
14698                },
14699            )
14700            .await
14701            .expect_err("the predecessor-only replay must request a successor retry");
14702        assert!(
14703            retry.to_string().contains("retry the successor dispatch"),
14704            "unexpected predecessor replay result: {retry}"
14705        );
14706        assert_eq!(
14707            publication_scope.state(),
14708            crate::object_api::ScannerPublicationCommitState::Committed,
14709            "an exact predecessor replay must report its local mutation as committed"
14710        );
14711        assert!(publication_scope.release_movement_permit().await);
14712        assert!(
14713            store.pools[0]
14714                .get_disks_by_key(predecessor)
14715                .load_file_info_versions_exact(bucket, predecessor)
14716                .await
14717                .expect("predecessor metadata should remain readable")
14718                .is_none(),
14719            "the authorized predecessor must be removed exactly"
14720        );
14721        assert!(
14722            store.pools[0]
14723                .get_disks_by_key(newcomer)
14724                .load_file_info_versions_exact(bucket, newcomer)
14725                .await
14726                .expect("newcomer metadata should remain readable")
14727                .is_some(),
14728            "the predecessor authorization must not delete the newcomer"
14729        );
14730        assert_eq!(
14731            test_tier_delete_dispatch_manifest_state(store.clone(), &manifest_name)
14732                .await
14733                .expect("completed predecessor manifest should remain readable"),
14734            Some(TierDeleteDispatchManifestState::Completed)
14735        );
14736
14737        backend.set_remove_failure(false);
14738        let predecessor_journals = recover_tier_delete_journal_entries(store.clone(), 100, None)
14739            .await
14740            .expect("completed predecessor journal should recover");
14741        assert_eq!(predecessor_journals.failed, 0);
14742        assert!(predecessor_journals.scanned <= 1 && predecessor_journals.deleted <= 1);
14743        assert_eq!(tier_delete_journal_count(store.clone()).await, 0);
14744        assert_eq!(backend.object_count().await, 1, "only the newcomer remote object should remain");
14745        assert_eq!(backend.remove_versions().await.len(), 1);
14746        let predecessor_manifest = recover_tier_delete_dispatch_manifests(store.clone(), 100, None)
14747            .await
14748            .expect("completed predecessor manifest should be collected");
14749        assert_eq!(predecessor_manifest.failed, 0);
14750        assert!(predecessor_manifest.deleted <= 1);
14751        assert_eq!(tier_delete_dispatch_manifest_count(store.clone()).await, 0);
14752
14753        store
14754            .delete_object_with_tier_delete_journal(
14755                bucket,
14756                prefix,
14757                ObjectOptions {
14758                    delete_prefix: true,
14759                    ..Default::default()
14760                },
14761            )
14762            .await
14763            .expect("a successor dispatch should delete the remaining newcomer");
14764        assert!(
14765            store.pools[0]
14766                .get_disks_by_key(newcomer)
14767                .load_file_info_versions_exact(bucket, newcomer)
14768                .await
14769                .expect("newcomer metadata lookup should succeed after successor")
14770                .is_none()
14771        );
14772        drive_tier_delete_dispatch_restart_to_convergence(store.clone()).await;
14773        assert_eq!(tier_delete_journal_count(store.clone()).await, 0);
14774        assert_eq!(tier_delete_dispatch_manifest_count(store.clone()).await, 0);
14775        assert_eq!(backend.object_count().await, 0);
14776        assert_eq!(backend.remove_versions().await.len(), 2);
14777    }
14778
14779    #[cfg(feature = "test-util")]
14780    #[tokio::test]
14781    #[serial_test::serial(storage_class_env)]
14782    async fn multi_pool_same_tuple_recursive_prefix_uses_one_journal_owner() {
14783        let temp_dir = tempfile::tempdir().expect("create shared-tuple prefix-delete store dir");
14784        let (ctx, store, _shutdown) = without_storage_class_env(build_isolated_test_store(
14785            temp_dir.path(),
14786            "multi-pool-shared-tuple-prefix-delete",
14787            &[4, 4],
14788        ))
14789        .await;
14790        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
14791        let tier_name = "SHARED-PREFIX-DELETE";
14792        let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
14793        let bucket = "multi-pool-shared-tuple-prefix-delete-bucket";
14794        let object = "prefix/shared.bin";
14795        store
14796            .make_bucket(bucket, &MakeBucketOptions::default())
14797            .await
14798            .expect("shared-tuple prefix bucket should be created");
14799        let mut reader = PutObjReader::from_vec(vec![b's'; 1024 * 1024]);
14800        let source = store.pools[0]
14801            .put_object(bucket, object, &mut reader, &ObjectOptions::default())
14802            .await
14803            .expect("shared-tuple prefix source should be written");
14804        store.pools[0]
14805            .transition_object(
14806                bucket,
14807                object,
14808                &ObjectOptions {
14809                    transition: TransitionOptions {
14810                        status: TRANSITION_PENDING.to_string(),
14811                        tier: tier_name.to_string(),
14812                        etag: source.etag.clone().expect("shared-tuple prefix source should have an etag"),
14813                        ..Default::default()
14814                    },
14815                    mod_time: source.mod_time,
14816                    ..Default::default()
14817                },
14818            )
14819            .await
14820            .expect("shared-tuple prefix source should transition");
14821        copy_test_xlmeta_between_pools(temp_dir.path(), 0, 1, bucket, object).await;
14822        assert_eq!(backend.object_count().await, 1);
14823        backend.set_remove_failure(true);
14824
14825        store
14826            .delete_object_with_tier_delete_journal(
14827                bucket,
14828                "prefix/",
14829                ObjectOptions {
14830                    delete_prefix: true,
14831                    ..Default::default()
14832                },
14833            )
14834            .await
14835            .expect("shared-tuple recursive prefix delete should commit");
14836
14837        assert_eq!(tier_delete_journal_count(store.clone()).await, 1);
14838        for pool_idx in 0..2 {
14839            assert!(
14840                store.pools[pool_idx]
14841                    .get_disks_by_key(object)
14842                    .load_file_info_versions_exact(bucket, object)
14843                    .await
14844                    .expect("shared-tuple post-delete metadata should decode")
14845                    .is_none(),
14846                "pool {pool_idx} must be physically empty"
14847            );
14848        }
14849        backend.set_remove_failure(false);
14850        ctx.wake_tier_delete_journal_recovery();
14851        wait_for_tier_delete_journal_recovery(store.clone(), &backend, 1).await;
14852        assert_eq!(backend.object_count().await, 0);
14853        assert_eq!(backend.remove_versions().await.len(), 1);
14854    }
14855
14856    #[cfg(feature = "test-util")]
14857    #[tokio::test]
14858    #[serial_test::serial(storage_class_env)]
14859    async fn multi_pool_recursive_prefix_rejects_legacy_or_hidden_merge_loser_before_delete() {
14860        let temp_dir = tempfile::tempdir().expect("create merge-loser prefix-delete store dir");
14861        let (ctx, store, _shutdown) = without_storage_class_env(build_isolated_test_store(
14862            temp_dir.path(),
14863            "multi-pool-merge-loser-prefix-delete",
14864            &[4, 4],
14865        ))
14866        .await;
14867        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
14868        let tier_name = "MERGE-LOSER-PREFIX-DELETE";
14869        let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
14870        let bucket = "multi-pool-merge-loser-prefix-delete-bucket";
14871        store
14872            .make_bucket(bucket, &MakeBucketOptions::default())
14873            .await
14874            .expect("merge-loser prefix bucket should be created");
14875
14876        for (object, byte) in [("legacy/item.bin", b'l'), ("hidden/item.bin", b'h')] {
14877            let mut reader = PutObjReader::from_vec(vec![byte; 1024 * 1024]);
14878            let source = store.pools[0]
14879                .put_object(bucket, object, &mut reader, &ObjectOptions::default())
14880                .await
14881                .expect("merge-loser source should be written");
14882            store.pools[0]
14883                .transition_object(
14884                    bucket,
14885                    object,
14886                    &ObjectOptions {
14887                        transition: TransitionOptions {
14888                            status: TRANSITION_PENDING.to_string(),
14889                            tier: tier_name.to_string(),
14890                            etag: source.etag.clone().expect("merge-loser source should have an etag"),
14891                            ..Default::default()
14892                        },
14893                        mod_time: source.mod_time,
14894                        ..Default::default()
14895                    },
14896                )
14897                .await
14898                .expect("merge-loser source should transition");
14899            copy_test_xlmeta_between_pools(temp_dir.path(), 0, 1, bucket, object).await;
14900        }
14901        rewrite_transitioned_xlmeta_as_legacy_unknown(temp_dir.path(), 1, bucket, "legacy/item.bin", false).await;
14902        backend.set_remove_failure(true);
14903        store.pools[1]
14904            .delete_object(bucket, "hidden/item.bin", ObjectOptions::default())
14905            .await
14906            .expect("second-pool source delete should retain a hidden free-version");
14907
14908        let legacy_error = store
14909            .delete_object_with_tier_delete_journal(
14910                bucket,
14911                "legacy/",
14912                ObjectOptions {
14913                    delete_prefix: true,
14914                    ..Default::default()
14915                },
14916            )
14917            .await
14918            .expect_err("a legacy merge loser must reject recursive deletion");
14919        assert!(legacy_error.to_string().contains("legacy Unknown"), "unexpected error: {legacy_error}");
14920        let hidden_error = store
14921            .delete_object_with_tier_delete_journal(
14922                bucket,
14923                "hidden/",
14924                ObjectOptions {
14925                    delete_prefix: true,
14926                    ..Default::default()
14927                },
14928            )
14929            .await
14930            .expect_err("a hidden free-version merge loser must reject recursive deletion");
14931        assert!(
14932            hidden_error.to_string().contains("free-version cleanup obligation"),
14933            "unexpected error: {hidden_error}"
14934        );
14935
14936        assert_eq!(tier_delete_journal_count(store.clone()).await, 0);
14937        assert_eq!(backend.object_count().await, 2);
14938        assert_eq!(backend.remove_count().await, 0);
14939        for object in ["legacy/item.bin", "hidden/item.bin"] {
14940            let pool0 = store.pools[0]
14941                .get_disks_by_key(object)
14942                .load_file_info_versions_exact(bucket, object)
14943                .await
14944                .expect("winner metadata should decode")
14945                .expect("winner source must remain before destructive deletion");
14946            assert_eq!(pool0.versions.iter().filter(|version| !version.tier_free_version()).count(), 1);
14947        }
14948        let hidden_pool1 = store.pools[1]
14949            .get_disks_by_key("hidden/item.bin")
14950            .load_file_info_versions_exact(bucket, "hidden/item.bin")
14951            .await
14952            .expect("hidden loser metadata should decode")
14953            .expect("hidden loser cleanup owner must remain");
14954        assert_eq!(
14955            hidden_pool1
14956                .versions
14957                .iter()
14958                .chain(hidden_pool1.free_versions.iter())
14959                .filter(|version| version.tier_free_version())
14960                .count(),
14961            1
14962        );
14963    }
14964
14965    #[cfg(feature = "test-util")]
14966    #[tokio::test]
14967    #[serial_test::serial(storage_class_env)]
14968    async fn recursive_prefix_partial_pool_failure_keeps_prepared_cleanup_owners() {
14969        let temp_dir = tempfile::tempdir().expect("create partial prefix-delete store dir");
14970        let (ctx, store, _shutdown) =
14971            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "partial-pool-prefix-delete", &[4, 4])).await;
14972        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
14973        let tier_name = "PARTIAL-PREFIX-DELETE";
14974        let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
14975        let bucket = "partial-pool-prefix-delete-bucket";
14976        let object = "prefix/item.bin";
14977        store
14978            .make_bucket(bucket, &MakeBucketOptions::default())
14979            .await
14980            .expect("partial prefix bucket should be created");
14981        for (pool_idx, byte) in [(0, b'a'), (1, b'b')] {
14982            let mut reader = PutObjReader::from_vec(vec![byte; 1024 * 1024]);
14983            let source = store.pools[pool_idx]
14984                .put_object(bucket, object, &mut reader, &ObjectOptions::default())
14985                .await
14986                .expect("pool-local prefix source should be written");
14987            store.pools[pool_idx]
14988                .transition_object(
14989                    bucket,
14990                    object,
14991                    &ObjectOptions {
14992                        transition: TransitionOptions {
14993                            status: TRANSITION_PENDING.to_string(),
14994                            tier: tier_name.to_string(),
14995                            etag: source.etag.clone().expect("pool-local source should have an etag"),
14996                            ..Default::default()
14997                        },
14998                        mod_time: source.mod_time,
14999                        ..Default::default()
15000                    },
15001                )
15002                .await
15003                .expect("pool-local prefix source should transition");
15004        }
15005        assert_eq!(backend.object_count().await, 2);
15006        backend.set_remove_failure(true);
15007
15008        let barrier = crate::store::object::PrefixDeleteAfterJournalPrepareBarrier::install(bucket, "prefix/");
15009        let delete_store = store.clone();
15010        let delete = tokio::spawn(async move {
15011            delete_store
15012                .delete_object_with_tier_delete_journal(
15013                    bucket,
15014                    "prefix/",
15015                    ObjectOptions {
15016                        delete_prefix: true,
15017                        ..Default::default()
15018                    },
15019                )
15020                .await
15021        });
15022        tokio::time::timeout(Duration::from_secs(10), barrier.wait_until_paused())
15023            .await
15024            .expect("prefix delete should pause after every journal is Prepared");
15025        assert_eq!(tier_delete_journal_count(store.clone()).await, 2);
15026
15027        let failed_set = store.pools[1].get_disks_by_key(object);
15028        let offline_disks = force_set_disks_offline_for_test(&failed_set).await;
15029        barrier.release();
15030        let error = delete
15031            .await
15032            .expect("partial prefix delete task should join")
15033            .expect_err("an offline second pool must fail the recursive prefix delete");
15034        assert!(
15035            matches!(
15036                error,
15037                StorageError::InsufficientWriteQuorum(_, _) | StorageError::InsufficientReadQuorum(_, _)
15038            ),
15039            "unexpected error: {error:?}"
15040        );
15041        drop(offline_disks);
15042        drop(barrier);
15043
15044        assert_eq!(
15045            tier_delete_journal_count(store.clone()).await,
15046            2,
15047            "an ambiguous partial mutation must retain every Dispatched owner"
15048        );
15049        let first_pool_state = store.pools[0]
15050            .get_disks_by_key(object)
15051            .load_file_info_versions_exact(bucket, object)
15052            .await
15053            .expect("first-pool metadata should remain decodable after an ambiguous failure");
15054        if let Some(first_pool_state) = first_pool_state {
15055            assert_eq!(
15056                first_pool_state
15057                    .versions
15058                    .iter()
15059                    .filter(|version| !version.tier_free_version())
15060                    .count(),
15061                1,
15062                "a retained first-pool source must remain a visible source, not an unjournaled fallback owner"
15063            );
15064        }
15065        assert!(
15066            store.pools[1]
15067                .get_disks_by_key(object)
15068                .load_file_info_versions_exact(bucket, object)
15069                .await
15070                .expect("failed-pool metadata should decode")
15071                .is_some(),
15072            "the failed pool should retain its live source"
15073        );
15074
15075        let _ = recover_tier_delete_journal_entries(store.clone(), 100, None)
15076            .await
15077            .expect("Dispatched owners should reconcile after the failed pool returns");
15078        assert_eq!(
15079            tier_delete_journal_count(store.clone()).await,
15080            2,
15081            "recovery must never abort or delete a Dispatched journal while any source is still live"
15082        );
15083        store
15084            .delete_object_with_tier_delete_journal(
15085                bucket,
15086                "prefix/",
15087                ObjectOptions {
15088                    delete_prefix: true,
15089                    ..Default::default()
15090                },
15091            )
15092            .await
15093            .expect("retry should delete the remaining pool source");
15094        backend.set_remove_failure(false);
15095        for _ in 0..3 {
15096            let _ = recover_tier_delete_journal_entries(store.clone(), 100, None)
15097                .await
15098                .expect("remaining pool journals should settle after backend recovery");
15099            if tier_delete_journal_count(store.clone()).await == 0 {
15100                break;
15101            }
15102        }
15103        assert_eq!(tier_delete_journal_count(store.clone()).await, 0);
15104        assert_eq!(backend.object_count().await, 0);
15105        assert_eq!(backend.remove_versions().await.len(), 2);
15106    }
15107
15108    #[cfg(feature = "test-util")]
15109    #[tokio::test]
15110    #[serial_test::serial(storage_class_env)]
15111    async fn recursive_prefix_partial_set_failure_keeps_prepared_cleanup_owners() {
15112        let temp_dir = tempfile::tempdir().expect("create partial-set prefix-delete store dir");
15113        let (ctx, store, _shutdown) = without_storage_class_env(build_isolated_test_store_with_layout(
15114            temp_dir.path(),
15115            "partial-set-prefix-delete",
15116            &[(2, 4)],
15117            CancellationToken::new(),
15118            None,
15119        ))
15120        .await;
15121        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
15122        let tier_name = "PARTIAL-SET-PREFIX-DELETE";
15123        let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
15124        let bucket = "partial-set-prefix-delete-bucket";
15125        store
15126            .make_bucket(bucket, &MakeBucketOptions::default())
15127            .await
15128            .expect("partial-set prefix bucket should be created");
15129
15130        let objects = (0..2)
15131            .map(|target_set| {
15132                (0..512)
15133                    .map(|index| format!("prefix/set-{target_set}-{index}.bin"))
15134                    .find(|candidate| store.pools[0].get_disks_by_key(candidate).set_index == target_set)
15135                    .expect("a deterministic key should map to each test set")
15136            })
15137            .collect::<Vec<_>>();
15138        for (index, object) in objects.iter().enumerate() {
15139            let mut reader = PutObjReader::from_vec(vec![b'0' + index as u8; 1024 * 1024]);
15140            let source = store.pools[0]
15141                .put_object(bucket, object, &mut reader, &ObjectOptions::default())
15142                .await
15143                .expect("set-local prefix source should be written");
15144            store.pools[0]
15145                .transition_object(
15146                    bucket,
15147                    object,
15148                    &ObjectOptions {
15149                        transition: TransitionOptions {
15150                            status: TRANSITION_PENDING.to_string(),
15151                            tier: tier_name.to_string(),
15152                            etag: source.etag.clone().expect("set-local source should have an etag"),
15153                            ..Default::default()
15154                        },
15155                        mod_time: source.mod_time,
15156                        ..Default::default()
15157                    },
15158                )
15159                .await
15160                .expect("set-local prefix source should transition");
15161        }
15162        assert_eq!(backend.object_count().await, 2);
15163        backend.set_remove_failure(true);
15164
15165        let barrier = crate::store::object::PrefixDeleteAfterJournalPrepareBarrier::install(bucket, "prefix/");
15166        let delete_store = store.clone();
15167        let delete = tokio::spawn(async move {
15168            delete_store
15169                .delete_object_with_tier_delete_journal(
15170                    bucket,
15171                    "prefix/",
15172                    ObjectOptions {
15173                        delete_prefix: true,
15174                        ..Default::default()
15175                    },
15176                )
15177                .await
15178        });
15179        tokio::time::timeout(Duration::from_secs(10), barrier.wait_until_paused())
15180            .await
15181            .expect("set-scoped prefix delete should pause after journal Prepare");
15182        assert_eq!(tier_delete_journal_count(store.clone()).await, 2);
15183
15184        let failed_set = store.pools[0].disk_set[1].clone();
15185        let offline_disks = force_set_disks_offline_for_test(&failed_set).await;
15186        barrier.release();
15187        let error = delete
15188            .await
15189            .expect("partial-set prefix delete task should join")
15190            .expect_err("an offline second set must fail recursive prefix deletion");
15191        assert!(
15192            matches!(
15193                error,
15194                StorageError::InsufficientWriteQuorum(_, _) | StorageError::InsufficientReadQuorum(_, _)
15195            ),
15196            "unexpected error: {error:?}"
15197        );
15198        drop(offline_disks);
15199        drop(barrier);
15200
15201        assert_eq!(tier_delete_journal_count(store.clone()).await, 2);
15202        let first_set_state = store.pools[0]
15203            .get_disks_by_key(&objects[0])
15204            .load_file_info_versions_exact(bucket, &objects[0])
15205            .await
15206            .expect("first-set metadata should remain decodable after an ambiguous failure");
15207        if let Some(first_set_state) = first_set_state {
15208            assert_eq!(
15209                first_set_state
15210                    .versions
15211                    .iter()
15212                    .filter(|version| !version.tier_free_version())
15213                    .count(),
15214                1,
15215                "a retained first-set source must remain visible under its Dispatched journal"
15216            );
15217        }
15218        assert!(
15219            store.pools[0]
15220                .get_disks_by_key(&objects[1])
15221                .load_file_info_versions_exact(bucket, &objects[1])
15222                .await
15223                .expect("failed-set metadata should decode")
15224                .is_some()
15225        );
15226
15227        let _ = recover_tier_delete_journal_entries(store.clone(), 100, None)
15228            .await
15229            .expect("set-scoped Dispatched owners should reconcile");
15230        assert_eq!(
15231            tier_delete_journal_count(store.clone()).await,
15232            2,
15233            "recovery must retain the complete Authorized dispatch until every source is absent"
15234        );
15235        store
15236            .delete_object_with_tier_delete_journal(
15237                bucket,
15238                "prefix/",
15239                ObjectOptions {
15240                    delete_prefix: true,
15241                    ..Default::default()
15242                },
15243            )
15244            .await
15245            .expect("set-scoped retry should delete the remaining source");
15246        backend.set_remove_failure(false);
15247        for _ in 0..3 {
15248            let _ = recover_tier_delete_journal_entries(store.clone(), 100, None)
15249                .await
15250                .expect("remaining set journals should settle after backend recovery");
15251            if tier_delete_journal_count(store.clone()).await == 0 {
15252                break;
15253            }
15254        }
15255        assert_eq!(tier_delete_journal_count(store.clone()).await, 0);
15256        assert_eq!(backend.object_count().await, 0);
15257        assert_eq!(backend.remove_versions().await.len(), 2);
15258    }
15259
15260    #[cfg(feature = "test-util")]
15261    #[tokio::test]
15262    #[serial_test::serial(storage_class_env)]
15263    async fn versioned_explicit_transition_delete_preserves_other_version_then_allows_bucket_delete() {
15264        let temp_dir = tempfile::tempdir().expect("create versioned transitioned delete store dir");
15265        let (ctx, store, _shutdown) =
15266            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "versioned-transitioned-delete-owner", &[4]))
15267                .await;
15268        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
15269        let tier_name = "VERSIONED-DELETE";
15270        let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
15271        let bucket = "versioned-transitioned-delete-owner-bucket";
15272        let object = "archive.bin";
15273        store
15274            .make_bucket(bucket, &MakeBucketOptions::default())
15275            .await
15276            .expect("versioned source bucket should be created");
15277        crate::bucket::metadata_sys::update_in(
15278            &ctx,
15279            bucket,
15280            BUCKET_VERSIONING_CONFIG,
15281            b"<VersioningConfiguration><Status>Enabled</Status></VersioningConfiguration>".to_vec(),
15282        )
15283        .await
15284        .expect("bucket versioning should be enabled");
15285
15286        let mut first_reader = PutObjReader::from_vec(vec![b'1'; 1024 * 1024]);
15287        let first = store
15288            .put_object(
15289                bucket,
15290                object,
15291                &mut first_reader,
15292                &ObjectOptions {
15293                    versioned: true,
15294                    ..Default::default()
15295                },
15296            )
15297            .await
15298            .expect("first version should be written");
15299        let first_version = first.version_id.expect("versioned PUT should return an identity");
15300        store
15301            .transition_object(
15302                bucket,
15303                object,
15304                &ObjectOptions {
15305                    version_id: Some(first_version.to_string()),
15306                    versioned: true,
15307                    transition: TransitionOptions {
15308                        status: TRANSITION_PENDING.to_string(),
15309                        tier: tier_name.to_string(),
15310                        etag: first.etag.clone().expect("first version should have an etag"),
15311                        ..Default::default()
15312                    },
15313                    mod_time: first.mod_time,
15314                    ..Default::default()
15315                },
15316            )
15317            .await
15318            .expect("first version should transition");
15319        let mut second_reader = PutObjReader::from_vec(vec![b'2'; 1024 * 1024]);
15320        let second = store
15321            .put_object(
15322                bucket,
15323                object,
15324                &mut second_reader,
15325                &ObjectOptions {
15326                    versioned: true,
15327                    ..Default::default()
15328                },
15329            )
15330            .await
15331            .expect("second version should be written");
15332        let second_version = second.version_id.expect("second PUT should return an identity");
15333
15334        backend.set_remove_failure(true);
15335        store
15336            .delete_object_with_tier_delete_journal(
15337                bucket,
15338                object,
15339                ObjectOptions {
15340                    version_id: Some(first_version.to_string()),
15341                    versioned: true,
15342                    ..Default::default()
15343                },
15344            )
15345            .await
15346            .expect("explicit transitioned version delete should commit");
15347        store
15348            .get_object_info(
15349                bucket,
15350                object,
15351                &ObjectOptions {
15352                    version_id: Some(second_version.to_string()),
15353                    versioned: true,
15354                    ..Default::default()
15355                },
15356            )
15357            .await
15358            .expect("the unrelated second version must remain readable");
15359        let exact = store.pools[0]
15360            .get_disks_by_key(object)
15361            .load_file_info_versions_exact(bucket, object)
15362            .await
15363            .expect("remaining version metadata should decode")
15364            .expect("the second version should remain");
15365        assert_eq!(
15366            exact
15367                .versions
15368                .iter()
15369                .chain(exact.free_versions.iter())
15370                .filter(|version| version.tier_free_version())
15371                .count(),
15372            1,
15373            "ordinary exact-version delete must leave one hidden free-version owner"
15374        );
15375        assert!(
15376            exact
15377                .versions
15378                .iter()
15379                .any(|version| version.version_id == Some(second_version)),
15380            "the explicit delete must not remove the other version"
15381        );
15382        assert_eq!(
15383            tier_delete_journal_count(store.clone()).await,
15384            0,
15385            "ordinary exact-version delete must not create a journal"
15386        );
15387        backend.set_remove_failure(false);
15388        wait_for_tier_free_version_recovery(store.clone(), &backend, 1).await;
15389        let exact = store.pools[0]
15390            .get_disks_by_key(object)
15391            .load_file_info_versions_exact(bucket, object)
15392            .await
15393            .expect("post-cleanup metadata should decode")
15394            .expect("the unrelated second version should remain after cleanup");
15395        assert_eq!(
15396            exact
15397                .versions
15398                .iter()
15399                .chain(exact.free_versions.iter())
15400                .filter(|version| version.tier_free_version())
15401                .count(),
15402            0,
15403            "free-version recovery must remove only its exact owner"
15404        );
15405        assert!(
15406            exact
15407                .versions
15408                .iter()
15409                .any(|version| version.version_id == Some(second_version)),
15410            "free-version recovery must preserve the unrelated version"
15411        );
15412
15413        store
15414            .delete_object_with_tier_delete_journal(
15415                bucket,
15416                object,
15417                ObjectOptions {
15418                    version_id: Some(second_version.to_string()),
15419                    versioned: true,
15420                    ..Default::default()
15421                },
15422            )
15423            .await
15424            .expect("last ordinary version should be removed");
15425        store
15426            .delete_bucket(bucket, &DeleteBucketOptions::default())
15427            .await
15428            .expect("bucket delete should succeed after the last version is removed");
15429    }
15430
15431    #[cfg(feature = "test-util")]
15432    #[tokio::test]
15433    #[serial_test::serial(storage_class_env)]
15434    async fn suspended_null_transition_delete_uses_free_version_as_sole_owner() {
15435        let temp_dir = tempfile::tempdir().expect("create suspended transitioned delete store dir");
15436        let (ctx, store, _shutdown) =
15437            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "suspended-transitioned-delete-owner", &[4]))
15438                .await;
15439        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
15440        let tier_name = "SUSPENDED-DELETE";
15441        let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
15442        let bucket = "suspended-transitioned-delete-owner-bucket";
15443        let object = "archive.bin";
15444        store
15445            .make_bucket(bucket, &MakeBucketOptions::default())
15446            .await
15447            .expect("suspended source bucket should be created");
15448        crate::bucket::metadata_sys::update_in(
15449            &ctx,
15450            bucket,
15451            BUCKET_VERSIONING_CONFIG,
15452            b"<VersioningConfiguration><Status>Suspended</Status></VersioningConfiguration>".to_vec(),
15453        )
15454        .await
15455        .expect("bucket versioning should be suspended");
15456
15457        let mut reader = PutObjReader::from_vec(vec![b's'; 1024 * 1024]);
15458        let source = store
15459            .put_object(
15460                bucket,
15461                object,
15462                &mut reader,
15463                &ObjectOptions {
15464                    version_suspended: true,
15465                    ..Default::default()
15466                },
15467            )
15468            .await
15469            .expect("suspended null source should be written");
15470        let null_version = source.version_id.expect("suspended source should expose the null identity");
15471        assert!(null_version.is_nil(), "suspended source identity must be null");
15472        store
15473            .transition_object(
15474                bucket,
15475                object,
15476                &ObjectOptions {
15477                    version_id: Some(null_version.to_string()),
15478                    version_suspended: true,
15479                    transition: TransitionOptions {
15480                        status: TRANSITION_PENDING.to_string(),
15481                        tier: tier_name.to_string(),
15482                        etag: source.etag.clone().expect("suspended source should have an etag"),
15483                        ..Default::default()
15484                    },
15485                    mod_time: source.mod_time,
15486                    ..Default::default()
15487                },
15488            )
15489            .await
15490            .expect("suspended null source should transition");
15491
15492        backend.set_remove_failure(true);
15493        store
15494            .delete_object_with_tier_delete_journal(
15495                bucket,
15496                object,
15497                ObjectOptions {
15498                    version_id: Some(null_version.to_string()),
15499                    version_suspended: true,
15500                    ..Default::default()
15501                },
15502            )
15503            .await
15504            .expect("explicit null transitioned version delete should commit");
15505        let exact = store.pools[0]
15506            .get_disks_by_key(object)
15507            .load_file_info_versions_exact(bucket, object)
15508            .await
15509            .expect("post-delete null metadata should decode")
15510            .expect("ordinary null-version delete must retain a cleanup owner");
15511        assert_eq!(
15512            exact
15513                .versions
15514                .iter()
15515                .chain(exact.free_versions.iter())
15516                .filter(|version| version.tier_free_version())
15517                .count(),
15518            1,
15519            "ordinary null-version delete must retain exactly one hidden free-version owner"
15520        );
15521        assert_eq!(
15522            exact.versions.iter().filter(|version| !version.tier_free_version()).count(),
15523            0,
15524            "ordinary null-version delete must remove the visible source"
15525        );
15526        assert_eq!(tier_delete_journal_count(store.clone()).await, 0);
15527        backend.set_remove_failure(false);
15528        wait_for_tier_free_version_recovery(store.clone(), &backend, 1).await;
15529        store
15530            .delete_bucket(bucket, &DeleteBucketOptions::default())
15531            .await
15532            .expect("suspended bucket should be empty after exact null-version cleanup");
15533    }
15534
15535    #[cfg(feature = "test-util")]
15536    #[tokio::test]
15537    #[serial_test::serial(storage_class_env)]
15538    async fn versioned_delete_marker_keeps_transitioned_source_and_remote_object() {
15539        let temp_dir = tempfile::tempdir().expect("create versioned delete-marker store dir");
15540        let (ctx, store, _shutdown) =
15541            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "transitioned-delete-marker", &[4])).await;
15542        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
15543        let tier_name = "DELETE-MARKER";
15544        let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
15545        let bucket = "transitioned-delete-marker-bucket";
15546        let object = "archive.bin";
15547        store
15548            .make_bucket(bucket, &MakeBucketOptions::default())
15549            .await
15550            .expect("versioned source bucket should be created");
15551        crate::bucket::metadata_sys::update_in(
15552            &ctx,
15553            bucket,
15554            BUCKET_VERSIONING_CONFIG,
15555            b"<VersioningConfiguration><Status>Enabled</Status></VersioningConfiguration>".to_vec(),
15556        )
15557        .await
15558        .expect("bucket versioning should be enabled");
15559
15560        let mut reader = PutObjReader::from_vec(vec![b'm'; 1024 * 1024]);
15561        let source = store
15562            .put_object(
15563                bucket,
15564                object,
15565                &mut reader,
15566                &ObjectOptions {
15567                    versioned: true,
15568                    ..Default::default()
15569                },
15570            )
15571            .await
15572            .expect("versioned source should be written");
15573        let source_version = source.version_id.expect("versioned source should have an identity");
15574        store
15575            .transition_object(
15576                bucket,
15577                object,
15578                &ObjectOptions {
15579                    version_id: Some(source_version.to_string()),
15580                    versioned: true,
15581                    transition: TransitionOptions {
15582                        status: TRANSITION_PENDING.to_string(),
15583                        tier: tier_name.to_string(),
15584                        etag: source.etag.clone().expect("versioned source should have an etag"),
15585                        ..Default::default()
15586                    },
15587                    mod_time: source.mod_time,
15588                    ..Default::default()
15589                },
15590            )
15591            .await
15592            .expect("versioned source should transition");
15593
15594        let marker = store
15595            .delete_object_with_tier_delete_journal(
15596                bucket,
15597                object,
15598                ObjectOptions {
15599                    versioned: true,
15600                    ..Default::default()
15601                },
15602            )
15603            .await
15604            .expect("versioned DELETE without versionId should create a marker");
15605        assert!(marker.delete_marker);
15606        let marker_version = marker.version_id.expect("delete marker should have an identity");
15607        assert_eq!(backend.object_count().await, 1, "delete-marker creation must retain the remote object");
15608        assert_eq!(
15609            tier_delete_journal_count(store.clone()).await,
15610            0,
15611            "delete-marker creation must not schedule cleanup"
15612        );
15613        let exact = store.pools[0]
15614            .get_disks_by_key(object)
15615            .load_file_info_versions_exact(bucket, object)
15616            .await
15617            .expect("versioned delete-marker metadata should decode")
15618            .expect("source and delete marker should remain on disk");
15619        assert!(
15620            exact.versions.iter().any(|version| {
15621                version.version_id == Some(source_version) && version.transition_status == rustfs_filemeta::TRANSITION_COMPLETE
15622            }),
15623            "the transitioned source must remain behind the delete marker"
15624        );
15625        assert_eq!(
15626            exact
15627                .versions
15628                .iter()
15629                .chain(exact.free_versions.iter())
15630                .filter(|version| version.tier_free_version())
15631                .count(),
15632            0,
15633            "delete-marker creation must not create a cleanup free-version"
15634        );
15635
15636        store
15637            .delete_object_with_tier_delete_journal(
15638                bucket,
15639                object,
15640                ObjectOptions {
15641                    version_id: Some(marker_version.to_string()),
15642                    versioned: true,
15643                    ..Default::default()
15644                },
15645            )
15646            .await
15647            .expect("delete marker should be removable by exact identity");
15648        store
15649            .delete_object_with_tier_delete_journal(
15650                bucket,
15651                object,
15652                ObjectOptions {
15653                    version_id: Some(source_version.to_string()),
15654                    versioned: true,
15655                    ..Default::default()
15656                },
15657            )
15658            .await
15659            .expect("transitioned source should be removable by exact identity");
15660        wait_for_tier_free_version_recovery(store.clone(), &backend, 1).await;
15661        if let Some(exact) = store.pools[0]
15662            .get_disks_by_key(object)
15663            .load_file_info_versions_exact(bucket, object)
15664            .await
15665            .expect("post-recovery version metadata should decode")
15666        {
15667            assert!(
15668                exact
15669                    .versions
15670                    .iter()
15671                    .chain(exact.free_versions.iter())
15672                    .all(|version| !version.tier_free_version()),
15673                "free-version recovery must remove the all-physical cleanup owner"
15674            );
15675        }
15676        store
15677            .delete_bucket(bucket, &DeleteBucketOptions::default())
15678            .await
15679            .expect("versioned source bucket should be empty after explicit cleanup");
15680    }
15681
15682    #[cfg(feature = "test-util")]
15683    #[tokio::test]
15684    #[serial_test::serial(storage_class_env)]
15685    async fn batch_transitioned_delete_uses_free_version_per_item() {
15686        let temp_dir = tempfile::tempdir().expect("create batch transitioned delete store dir");
15687        let (ctx, store, _shutdown) =
15688            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "batch-transitioned-delete-owner", &[4])).await;
15689        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
15690        let tier_name = "BATCH-DELETE";
15691        let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
15692        let bucket = "batch-transitioned-delete-owner-bucket";
15693        let transitioned = "transitioned.bin";
15694        let ordinary = "ordinary.bin";
15695        store
15696            .make_bucket(bucket, &MakeBucketOptions::default())
15697            .await
15698            .expect("batch source bucket should be created");
15699        let mut transitioned_reader = PutObjReader::from_vec(vec![b't'; 1024 * 1024]);
15700        let transitioned_source = store
15701            .put_object(bucket, transitioned, &mut transitioned_reader, &ObjectOptions::default())
15702            .await
15703            .expect("transitioned batch source should be written");
15704        store
15705            .transition_object(
15706                bucket,
15707                transitioned,
15708                &ObjectOptions {
15709                    transition: TransitionOptions {
15710                        status: TRANSITION_PENDING.to_string(),
15711                        tier: tier_name.to_string(),
15712                        etag: transitioned_source
15713                            .etag
15714                            .clone()
15715                            .expect("transitioned batch source should have an etag"),
15716                        ..Default::default()
15717                    },
15718                    mod_time: transitioned_source.mod_time,
15719                    ..Default::default()
15720                },
15721            )
15722            .await
15723            .expect("batch source transition should commit");
15724        let mut ordinary_reader = PutObjReader::from_vec(vec![b'o'; 1024 * 1024]);
15725        store
15726            .put_object(bucket, ordinary, &mut ordinary_reader, &ObjectOptions::default())
15727            .await
15728            .expect("ordinary batch source should be written");
15729
15730        backend.set_remove_failure(true);
15731        let (_deleted, errors) = store
15732            .delete_objects_with_tier_delete_journal(
15733                bucket,
15734                vec![
15735                    ObjectToDelete {
15736                        object_name: transitioned.to_string(),
15737                        ..Default::default()
15738                    },
15739                    ObjectToDelete {
15740                        object_name: ordinary.to_string(),
15741                        ..Default::default()
15742                    },
15743                    ObjectToDelete {
15744                        object_name: "missing.bin".to_string(),
15745                        ..Default::default()
15746                    },
15747                ],
15748                ObjectOptions::default(),
15749            )
15750            .await;
15751        assert!(errors.iter().all(Option::is_none), "S3 batch delete should be idempotent: {errors:?}");
15752        let transitioned_versions = store.pools[0]
15753            .get_disks_by_key(transitioned)
15754            .load_file_info_versions_exact(bucket, transitioned)
15755            .await
15756            .expect("transitioned batch metadata should remain decodable")
15757            .expect("transitioned batch item must retain a cleanup owner");
15758        assert_eq!(
15759            transitioned_versions
15760                .versions
15761                .iter()
15762                .chain(transitioned_versions.free_versions.iter())
15763                .filter(|version| version.tier_free_version())
15764                .count(),
15765            1,
15766            "transitioned batch item must retain exactly one hidden free-version owner"
15767        );
15768        assert_eq!(
15769            transitioned_versions
15770                .versions
15771                .iter()
15772                .filter(|version| !version.tier_free_version())
15773                .count(),
15774            0,
15775            "transitioned batch item must have no visible source"
15776        );
15777        assert!(
15778            store.pools[0]
15779                .get_disks_by_key(ordinary)
15780                .load_file_info_versions_exact(bucket, ordinary)
15781                .await
15782                .expect("ordinary batch metadata should remain decodable")
15783                .is_none(),
15784            "ordinary batch item should have no local metadata"
15785        );
15786        assert_eq!(
15787            tier_delete_journal_count(store.clone()).await,
15788            0,
15789            "ordinary batch delete must not create a tier-delete journal"
15790        );
15791        assert_eq!(backend.object_count().await, 1, "failed remote cleanup must retain its object");
15792
15793        backend.set_remove_failure(false);
15794        wait_for_tier_free_version_recovery(store.clone(), &backend, 1).await;
15795        assert!(
15796            store.pools[0]
15797                .get_disks_by_key(transitioned)
15798                .load_file_info_versions_exact(bucket, transitioned)
15799                .await
15800                .expect("cleaned batch metadata should remain decodable")
15801                .is_none(),
15802            "free-version recovery must remove the exact cleanup owner"
15803        );
15804
15805        let causal = "causal.bin";
15806        let mut causal_reader = PutObjReader::from_vec(vec![b'c'; 1024 * 1024]);
15807        let causal_source = store
15808            .put_object(bucket, causal, &mut causal_reader, &ObjectOptions::default())
15809            .await
15810            .expect("causal batch source should be written");
15811        store
15812            .transition_object(
15813                bucket,
15814                causal,
15815                &ObjectOptions {
15816                    transition: TransitionOptions {
15817                        status: TRANSITION_PENDING.to_string(),
15818                        tier: tier_name.to_string(),
15819                        etag: causal_source.etag.clone().expect("causal batch source should have an etag"),
15820                        ..Default::default()
15821                    },
15822                    mod_time: causal_source.mod_time,
15823                    ..Default::default()
15824                },
15825            )
15826            .await
15827            .expect("causal batch source transition should commit");
15828        let (_deleted, errors) = store
15829            .delete_objects(
15830                bucket,
15831                vec![
15832                    ObjectToDelete {
15833                        object_name: causal.to_string(),
15834                        ..Default::default()
15835                    },
15836                    ObjectToDelete {
15837                        object_name: causal.to_string(),
15838                        ..Default::default()
15839                    },
15840                ],
15841                ObjectOptions::default(),
15842            )
15843            .await;
15844        assert!(
15845            errors.iter().all(Option::is_none),
15846            "duplicate causal batch deletes should remain idempotent: {errors:?}"
15847        );
15848        tokio::time::timeout(Duration::from_secs(30), async {
15849            loop {
15850                let metadata_absent = exact_metadata_absent_during_cleanup(&store, bucket, causal).await;
15851                if metadata_absent && backend.remove_versions().await.len() >= 2 {
15852                    return;
15853                }
15854                tokio::time::sleep(Duration::from_millis(10)).await;
15855            }
15856        })
15857        .await
15858        .expect("batch free-version receipt should converge without another recovery scan");
15859        assert_eq!(
15860            backend.remove_versions().await.len(),
15861            2,
15862            "duplicate batch requests must cause only one remote delete for the causal object"
15863        );
15864
15865        crate::bucket::metadata_sys::update_in(
15866            &ctx,
15867            bucket,
15868            BUCKET_VERSIONING_CONFIG,
15869            b"<VersioningConfiguration><Status>Enabled</Status></VersioningConfiguration>".to_vec(),
15870        )
15871        .await
15872        .expect("causal batch bucket versioning should be enabled");
15873        let versioned_causal = "versioned-causal.bin";
15874        let mut versioned_reader = PutObjReader::from_vec(vec![b'v'; 1024 * 1024]);
15875        let versioned_source = store
15876            .put_object(
15877                bucket,
15878                versioned_causal,
15879                &mut versioned_reader,
15880                &ObjectOptions {
15881                    versioned: true,
15882                    ..Default::default()
15883                },
15884            )
15885            .await
15886            .expect("versioned causal batch source should be written");
15887        let versioned_source_id = versioned_source
15888            .version_id
15889            .expect("versioned causal batch source should have an identity");
15890        store
15891            .transition_object(
15892                bucket,
15893                versioned_causal,
15894                &ObjectOptions {
15895                    version_id: Some(versioned_source_id.to_string()),
15896                    versioned: true,
15897                    transition: TransitionOptions {
15898                        status: TRANSITION_PENDING.to_string(),
15899                        tier: tier_name.to_string(),
15900                        etag: versioned_source
15901                            .etag
15902                            .clone()
15903                            .expect("versioned causal batch source should have an etag"),
15904                        ..Default::default()
15905                    },
15906                    mod_time: versioned_source.mod_time,
15907                    ..Default::default()
15908                },
15909            )
15910            .await
15911            .expect("versioned causal batch source transition should commit");
15912        let (_deleted, errors) = store
15913            .delete_objects(
15914                bucket,
15915                vec![ObjectToDelete {
15916                    object_name: versioned_causal.to_string(),
15917                    version_id: Some(versioned_source_id),
15918                    ..Default::default()
15919                }],
15920                ObjectOptions::default(),
15921            )
15922            .await;
15923        assert!(
15924            errors.iter().all(Option::is_none),
15925            "explicit-version causal batch delete should commit: {errors:?}"
15926        );
15927        tokio::time::timeout(Duration::from_secs(30), async {
15928            loop {
15929                let metadata_absent = exact_metadata_absent_during_cleanup(&store, bucket, versioned_causal).await;
15930                if metadata_absent && backend.remove_versions().await.len() == 3 {
15931                    return;
15932                }
15933                tokio::time::sleep(Duration::from_millis(10)).await;
15934            }
15935        })
15936        .await
15937        .expect("explicit-version batch receipt should converge without a recovery scan");
15938        store
15939            .delete_bucket(bucket, &DeleteBucketOptions::default())
15940            .await
15941            .expect("batch source bucket should be physically empty");
15942    }
15943
15944    #[cfg(feature = "test-util")]
15945    #[tokio::test]
15946    #[serial_test::serial(storage_class_env)]
15947    async fn batch_transitioned_delete_aggregate_error_still_enqueues_committed_free_version() {
15948        let temp_dir = tempfile::tempdir().expect("create aggregate-error batch delete store dir");
15949        let (ctx, store, shutdown) =
15950            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "batch-transitioned-aggregate-error", &[4, 4]))
15951                .await;
15952        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
15953        let tier_name = "BATCH-AGGREGATE-ERROR";
15954        let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
15955        let bucket = "batch-transitioned-aggregate-error-bucket";
15956        let object = "archive.bin";
15957        store
15958            .make_bucket(bucket, &MakeBucketOptions::default())
15959            .await
15960            .expect("aggregate-error source bucket should be created");
15961        let mut reader = PutObjReader::from_vec(vec![b'a'; 1024 * 1024]);
15962        let source = store.pools[0]
15963            .put_object(bucket, object, &mut reader, &ObjectOptions::default())
15964            .await
15965            .expect("aggregate-error source should be written");
15966        store.pools[0]
15967            .transition_object(
15968                bucket,
15969                object,
15970                &ObjectOptions {
15971                    transition: TransitionOptions {
15972                        status: TRANSITION_PENDING.to_string(),
15973                        tier: tier_name.to_string(),
15974                        etag: source.etag.clone().expect("aggregate-error source should have an etag"),
15975                        ..Default::default()
15976                    },
15977                    mod_time: source.mod_time,
15978                    ..Default::default()
15979                },
15980            )
15981            .await
15982            .expect("aggregate-error source should transition");
15983
15984        // Model a data-movement copy: both pools own the same logical source
15985        // and exact remote tuple, but each batch delete creates its own local
15986        // free-version UUID in the shared request sink.
15987        for disk_index in 0..4 {
15988            let source_meta = temp_dir
15989                .path()
15990                .join(format!("pool0/set0/disk{disk_index}/{bucket}/{object}/{STORAGE_FORMAT_FILE}"));
15991            let target_meta = temp_dir
15992                .path()
15993                .join(format!("pool1/set0/disk{disk_index}/{bucket}/{object}/{STORAGE_FORMAT_FILE}"));
15994            tokio::fs::create_dir_all(target_meta.parent().expect("target xl.meta should have a parent"))
15995                .await
15996                .expect("second-pool object directory should be created");
15997            tokio::fs::copy(&source_meta, &target_meta)
15998                .await
15999                .expect("transitioned xl.meta should copy exactly to the second pool");
16000        }
16001
16002        ExpiryState::resize_workers(1, store.clone()).await;
16003        let injection = crate::store::object::BatchDeletePoolErrorInjection::install(
16004            bucket,
16005            1,
16006            vec![(object.to_string(), StorageError::ErasureWriteQuorum)],
16007        );
16008        let (deleted, errors) = store
16009            .delete_objects(
16010                bucket,
16011                vec![ObjectToDelete {
16012                    object_name: object.to_string(),
16013                    ..Default::default()
16014                }],
16015                ObjectOptions::default(),
16016            )
16017            .await;
16018        assert_eq!(injection.observed(), 1, "the second pool should inject one post-commit aggregate error");
16019        assert_eq!(errors, vec![Some(StorageError::ErasureWriteQuorum)]);
16020        assert!(deleted[0].found, "the aggregate error must retain the committed pool result");
16021        drop(injection);
16022
16023        // Exact reads can see subquorum metadata while workers remove each
16024        // disk's free version. Inspect the final state after cleanup drains.
16025        wait_for_expiry_workers_idle(&store).await;
16026        for pool in &store.pools {
16027            assert!(
16028                pool.get_disks_by_key(object)
16029                    .load_file_info_versions_exact(bucket, object)
16030                    .await
16031                    .expect("aggregate-error cleanup metadata should remain readable")
16032                    .is_none(),
16033                "aggregate failure must not suppress committed receipt cleanup"
16034            );
16035        }
16036        assert_eq!(
16037            backend.remove_count().await,
16038            1,
16039            "committed receipts must remove the shared remote object once"
16040        );
16041        assert_eq!(backend.object_count().await, 0, "the shared remote object should be removed exactly once");
16042        store
16043            .delete_bucket(bucket, &DeleteBucketOptions::default())
16044            .await
16045            .expect("aggregate-error bucket should be physically empty");
16046        shutdown.cancel();
16047    }
16048
16049    #[cfg(feature = "test-util")]
16050    #[tokio::test]
16051    #[serial_test::serial(storage_class_env)]
16052    async fn transitioned_delete_free_version_replays_after_store_restart() {
16053        let temp_dir = tempfile::tempdir().expect("create restarted free-version store dir");
16054        let (ctx, store, shutdown) =
16055            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "transitioned-delete-restart-before", &[4]))
16056                .await;
16057        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
16058        let tier_name = "DELETE-RESTART";
16059        let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
16060        let tier_config = ctx
16061            .tier_config_mgr()
16062            .read()
16063            .await
16064            .tiers
16065            .get(tier_name)
16066            .expect("mock tier config should exist")
16067            .clone_with_credentials();
16068        let bucket = "transitioned-delete-restart-bucket";
16069        let object = "archive.bin";
16070        store
16071            .make_bucket(bucket, &MakeBucketOptions::default())
16072            .await
16073            .expect("restart source bucket should be created");
16074        let mut reader = PutObjReader::from_vec(vec![b'r'; 1024 * 1024]);
16075        let original = store
16076            .put_object(bucket, object, &mut reader, &ObjectOptions::default())
16077            .await
16078            .expect("restart source object should be written");
16079        store
16080            .transition_object(
16081                bucket,
16082                object,
16083                &ObjectOptions {
16084                    transition: TransitionOptions {
16085                        status: TRANSITION_PENDING.to_string(),
16086                        tier: tier_name.to_string(),
16087                        etag: original.etag.clone().expect("restart source should have an etag"),
16088                        ..Default::default()
16089                    },
16090                    mod_time: original.mod_time,
16091                    ..Default::default()
16092                },
16093            )
16094            .await
16095            .expect("restart source transition should commit");
16096        backend.set_remove_failure(true);
16097        store
16098            .delete_object_with_tier_delete_journal(bucket, object, ObjectOptions::default())
16099            .await
16100            .expect("local source delete should commit while remote cleanup is retryable");
16101        let retained = store.pools[0]
16102            .get_disks_by_key(object)
16103            .load_file_info_versions_exact(bucket, object)
16104            .await
16105            .expect("post-delete local metadata should decode")
16106            .expect("ordinary delete must retain a free-version across restart");
16107        assert_eq!(
16108            retained
16109                .versions
16110                .iter()
16111                .chain(retained.free_versions.iter())
16112                .filter(|version| version.tier_free_version())
16113                .count(),
16114            1,
16115            "ordinary delete must retain exactly one free-version across restart"
16116        );
16117        assert_eq!(
16118            retained
16119                .versions
16120                .iter()
16121                .filter(|version| !version.tier_free_version())
16122                .count(),
16123            0,
16124            "ordinary delete must remove the visible source before restart"
16125        );
16126        assert_eq!(tier_delete_journal_count(store.clone()).await, 0);
16127        assert_eq!(backend.object_count().await, 1);
16128
16129        shutdown.cancel();
16130        drop(store);
16131        drop(ctx);
16132        let (restarted_ctx, restarted_store, restarted_shutdown) =
16133            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "transitioned-delete-restart-after", &[4]))
16134                .await;
16135        crate::bucket::metadata_sys::init_bucket_metadata_sys(restarted_store.clone(), Vec::new()).await;
16136        {
16137            let restarted_tier_config_mgr = restarted_ctx.tier_config_mgr();
16138            let mut manager = restarted_tier_config_mgr.write().await;
16139            manager.tiers.insert(tier_name.to_string(), tier_config);
16140            manager
16141                .install_test_driver(tier_name, Box::new(backend.clone()))
16142                .expect("restarted store should bind the original tier destination");
16143        }
16144        backend.set_remove_failure(false);
16145        wait_for_tier_free_version_recovery(restarted_store.clone(), &backend, 1).await;
16146        assert_eq!(tier_delete_journal_count(restarted_store.clone()).await, 0);
16147        assert_eq!(backend.object_count().await, 0);
16148        assert!(
16149            restarted_store.pools[0]
16150                .get_disks_by_key(object)
16151                .load_file_info_versions_exact(bucket, object)
16152                .await
16153                .expect("restarted cleanup metadata should decode")
16154                .is_none(),
16155            "restarted recovery must remove the exact free-version owner"
16156        );
16157        restarted_store
16158            .delete_bucket(bucket, &DeleteBucketOptions::default())
16159            .await
16160            .expect("source bucket should be empty after restarted free-version recovery");
16161        restarted_shutdown.cancel();
16162    }
16163
16164    #[cfg(feature = "test-util")]
16165    #[tokio::test]
16166    #[serial_test::serial(storage_class_env)]
16167    async fn transitioned_delete_local_quorum_failure_rolls_back_without_cleanup_owner() {
16168        let temp_dir = tempfile::tempdir().expect("create failed local delete store dir");
16169        let (ctx, store, _shutdown) =
16170            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "transitioned-delete-local-failure", &[4]))
16171                .await;
16172        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
16173        let tier_name = "DELETE-LOCAL-FAIL";
16174        let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
16175        let bucket = "transitioned-delete-local-failure-bucket";
16176        let object = "archive.bin";
16177        store
16178            .make_bucket(bucket, &MakeBucketOptions::default())
16179            .await
16180            .expect("failed-delete source bucket should be created");
16181        let mut reader = PutObjReader::from_vec(vec![b'q'; 1024 * 1024]);
16182        let source = store
16183            .put_object(bucket, object, &mut reader, &ObjectOptions::default())
16184            .await
16185            .expect("failed-delete source should be written");
16186        store
16187            .transition_object(
16188                bucket,
16189                object,
16190                &ObjectOptions {
16191                    transition: TransitionOptions {
16192                        status: TRANSITION_PENDING.to_string(),
16193                        tier: tier_name.to_string(),
16194                        etag: source.etag.clone().expect("failed-delete source should have an etag"),
16195                        ..Default::default()
16196                    },
16197                    mod_time: source.mod_time,
16198                    ..Default::default()
16199                },
16200            )
16201            .await
16202            .expect("failed-delete source should transition");
16203
16204        let set = store.pools[0].get_disks_by_key(object);
16205        let saved_disk = {
16206            let mut disks = set.disks.write().await;
16207            let first = disks[0].as_ref().expect("first disk should be online");
16208            crate::disk::local::set_delete_version_fail_after_commit(first.path().as_path(), object);
16209            disks[1].take().expect("second disk should be online before fault injection")
16210        };
16211        let err = store
16212            .delete_object_with_tier_delete_journal(bucket, object, ObjectOptions::default())
16213            .await
16214            .expect_err("one post-commit error plus one offline disk must miss delete quorum");
16215        assert!(matches!(err, StorageError::Io(_)), "unexpected local delete failure: {err:?}");
16216        assert!(
16217            err.to_string()
16218                .contains("Storage resources are insufficient for the write operation"),
16219            "the caller must observe the wrapped quorum failure: {err}"
16220        );
16221        set.disks.write().await[1] = Some(saved_disk);
16222
16223        let retained = store
16224            .get_object_info(bucket, object, &ObjectOptions::default())
16225            .await
16226            .expect("failed local delete must roll back the transitioned source");
16227        assert_eq!(retained.transitioned_object.status, rustfs_filemeta::TRANSITION_COMPLETE);
16228        let retained_versions = set
16229            .load_file_info_versions_exact(bucket, object)
16230            .await
16231            .expect("rolled-back metadata should decode")
16232            .expect("rolled-back source should remain on disk");
16233        assert_eq!(
16234            retained_versions
16235                .versions
16236                .iter()
16237                .chain(retained_versions.free_versions.iter())
16238                .filter(|version| version.tier_free_version())
16239                .count(),
16240            0,
16241            "failed local quorum must not create a cleanup owner"
16242        );
16243        assert_eq!(
16244            tier_delete_journal_count(store.clone()).await,
16245            0,
16246            "ordinary delete must not create a journal, including on rollback"
16247        );
16248        assert_eq!(backend.object_count().await, 1, "failed local commit must retain the remote object");
16249        assert_eq!(backend.remove_count().await, 0, "failed local commit must not dispatch remote cleanup");
16250
16251        backend.set_remove_failure(true);
16252        store
16253            .delete_object_with_tier_delete_journal(bucket, object, ObjectOptions::default())
16254            .await
16255            .expect("retry after disk recovery should commit");
16256        let cleanup_owner = set
16257            .load_file_info_versions_exact(bucket, object)
16258            .await
16259            .expect("retry metadata should decode")
16260            .expect("successful retry must leave a free-version owner");
16261        assert_eq!(
16262            cleanup_owner
16263                .versions
16264                .iter()
16265                .chain(cleanup_owner.free_versions.iter())
16266                .filter(|version| version.tier_free_version())
16267                .count(),
16268            1,
16269            "successful retry must leave exactly one free-version owner"
16270        );
16271        assert_eq!(tier_delete_journal_count(store.clone()).await, 0);
16272        backend.set_remove_failure(false);
16273        wait_for_tier_free_version_recovery(store.clone(), &backend, 1).await;
16274        store
16275            .delete_bucket(bucket, &DeleteBucketOptions::default())
16276            .await
16277            .expect("retry should leave the source bucket empty");
16278    }
16279
16280    #[cfg(feature = "test-util")]
16281    #[tokio::test]
16282    #[serial_test::serial(storage_class_env)]
16283    async fn batch_transitioned_delete_post_commit_failures_roll_back_without_free_version_receipt() {
16284        let temp_dir = tempfile::tempdir().expect("create failed batch delete store dir");
16285        let (ctx, store, shutdown) =
16286            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "batch-delete-local-failure", &[4])).await;
16287        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
16288        let tier_name = "BATCH-DELETE-LOCAL-FAIL";
16289        let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
16290        let bucket = "batch-delete-local-failure-bucket";
16291        let object = "archive.bin";
16292        store
16293            .make_bucket(bucket, &MakeBucketOptions::default())
16294            .await
16295            .expect("failed batch source bucket should be created");
16296        crate::bucket::metadata_sys::update_in(
16297            &ctx,
16298            bucket,
16299            BUCKET_VERSIONING_CONFIG,
16300            b"<VersioningConfiguration><Status>Enabled</Status></VersioningConfiguration>".to_vec(),
16301        )
16302        .await
16303        .expect("failed batch bucket versioning should be enabled");
16304        let mut transitioned_reader = PutObjReader::from_vec(vec![b't'; 1024 * 1024]);
16305        let transitioned_source = store
16306            .put_object(
16307                bucket,
16308                object,
16309                &mut transitioned_reader,
16310                &ObjectOptions {
16311                    versioned: true,
16312                    ..Default::default()
16313                },
16314            )
16315            .await
16316            .expect("failed batch transitioned version should be written");
16317        let transitioned_version_id = transitioned_source
16318            .version_id
16319            .expect("failed batch transitioned source should have a version identity");
16320        store
16321            .transition_object(
16322                bucket,
16323                object,
16324                &ObjectOptions {
16325                    version_id: Some(transitioned_version_id.to_string()),
16326                    versioned: true,
16327                    transition: TransitionOptions {
16328                        status: TRANSITION_PENDING.to_string(),
16329                        tier: tier_name.to_string(),
16330                        etag: transitioned_source
16331                            .etag
16332                            .clone()
16333                            .expect("failed batch transitioned source should have an etag"),
16334                        ..Default::default()
16335                    },
16336                    mod_time: transitioned_source.mod_time,
16337                    ..Default::default()
16338                },
16339            )
16340            .await
16341            .expect("failed batch source version should transition");
16342        let mut ordinary_reader = PutObjReader::from_vec(vec![b'o'; 1024 * 1024]);
16343        let ordinary_source = store
16344            .put_object(
16345                bucket,
16346                object,
16347                &mut ordinary_reader,
16348                &ObjectOptions {
16349                    versioned: true,
16350                    ..Default::default()
16351                },
16352            )
16353            .await
16354            .expect("failed batch ordinary sibling should be written");
16355        let ordinary_version_id = ordinary_source
16356            .version_id
16357            .expect("failed batch ordinary sibling should have a version identity");
16358        let delete_requests = || {
16359            vec![
16360                ObjectToDelete {
16361                    object_name: object.to_string(),
16362                    version_id: Some(transitioned_version_id),
16363                    ..Default::default()
16364                },
16365                ObjectToDelete {
16366                    object_name: object.to_string(),
16367                    version_id: Some(ordinary_version_id),
16368                    ..Default::default()
16369                },
16370            ]
16371        };
16372
16373        let set = store.pools[0].get_disks_by_key(object);
16374        let disks = set.disks.read().await;
16375        assert_eq!(disks.len(), 4, "the rollback fixture must use four disks");
16376        // Keep discovery fully online, then make a quorum of disks report an
16377        // error only after their batch metadata commit has completed.
16378        for disk in disks.iter().take(3) {
16379            let disk = disk.as_ref().expect("injected rollback disks should be online");
16380            crate::disk::local::set_delete_version_fail_after_commit(disk.path().as_path(), object);
16381        }
16382        drop(disks);
16383        let receipt_sink = crate::object_api::TierFreeVersionReceiptSink::new();
16384        let (_deleted, errors) = store
16385            .delete_objects(
16386                bucket,
16387                delete_requests(),
16388                ObjectOptions {
16389                    tier_free_version_receipt_sink: Some(receipt_sink.clone()),
16390                    ..Default::default()
16391                },
16392            )
16393            .await;
16394        assert_eq!(
16395            errors,
16396            vec![Some(StorageError::Unexpected), Some(StorageError::Unexpected),],
16397            "three post-commit disk errors must fail batch delete before receipts publish"
16398        );
16399
16400        assert!(
16401            receipt_sink
16402                .drain()
16403                .expect("the test-owned failed-batch sink should drain exactly once")
16404                .is_empty(),
16405            "a rolled-back physical group must publish no cleanup receipt"
16406        );
16407        let retained_transitioned = store
16408            .get_object_info(
16409                bucket,
16410                object,
16411                &ObjectOptions {
16412                    version_id: Some(transitioned_version_id.to_string()),
16413                    versioned: true,
16414                    ..Default::default()
16415                },
16416            )
16417            .await
16418            .expect("failed batch delete must restore the transitioned sibling");
16419        assert_eq!(retained_transitioned.transitioned_object.status, rustfs_filemeta::TRANSITION_COMPLETE);
16420        let retained_ordinary = store
16421            .get_object_info(
16422                bucket,
16423                object,
16424                &ObjectOptions {
16425                    version_id: Some(ordinary_version_id.to_string()),
16426                    versioned: true,
16427                    ..Default::default()
16428                },
16429            )
16430            .await
16431            .expect("failed batch delete must restore the ordinary sibling");
16432        assert_ne!(retained_ordinary.transitioned_object.status, rustfs_filemeta::TRANSITION_COMPLETE);
16433        let retained_versions = set
16434            .load_file_info_versions_exact(bucket, object)
16435            .await
16436            .expect("rolled-back batch metadata should decode")
16437            .expect("rolled-back batch source should remain on disk");
16438        assert_eq!(
16439            retained_versions
16440                .versions
16441                .iter()
16442                .chain(retained_versions.free_versions.iter())
16443                .filter(|version| version.tier_free_version())
16444                .count(),
16445            0,
16446            "failed batch quorum must not retain a free-version owner"
16447        );
16448        let retained_version_ids = retained_versions
16449            .versions
16450            .iter()
16451            .filter_map(|version| version.version_id)
16452            .collect::<std::collections::HashSet<_>>();
16453        assert_eq!(
16454            retained_version_ids,
16455            std::collections::HashSet::from([transitioned_version_id, ordinary_version_id]),
16456            "the physical-group rollback must restore both explicit siblings"
16457        );
16458        assert_eq!(backend.object_count().await, 1, "failed batch commit must retain the remote object");
16459        assert_eq!(backend.remove_count().await, 0, "failed batch commit must not dispatch remote cleanup");
16460
16461        ExpiryState::resize_workers(1, store.clone()).await;
16462        let (_deleted, retry_errors) = store
16463            .delete_objects(bucket, delete_requests(), ObjectOptions::default())
16464            .await;
16465        assert!(
16466            retry_errors.iter().all(Option::is_none),
16467            "retry after disk recovery should commit: {retry_errors:?}"
16468        );
16469        tokio::time::timeout(Duration::from_secs(30), async {
16470            loop {
16471                // Cleanup rewrites xl.meta disk by disk, so a read racing it
16472                // can briefly miss quorum; any other error is a real failure.
16473                let metadata_absent = match set.load_file_info_versions_exact(bucket, object).await {
16474                    Ok(versions) => versions.is_none(),
16475                    Err(StorageError::InsufficientReadQuorum(..)) => false,
16476                    Err(err) => panic!("retry cleanup metadata should remain readable: {err:?}"),
16477                };
16478                if metadata_absent && backend.remove_count().await == 1 {
16479                    return;
16480                }
16481                tokio::time::sleep(Duration::from_millis(10)).await;
16482            }
16483        })
16484        .await
16485        .expect("successful batch retry should converge without a recovery scan");
16486        store
16487            .delete_bucket(bucket, &DeleteBucketOptions::default())
16488            .await
16489            .expect("successful batch retry should leave the bucket empty");
16490        shutdown.cancel();
16491    }
16492
16493    #[cfg(feature = "test-util")]
16494    async fn run_multi_pool_same_remote_tuple_delete_case(batch: bool) {
16495        let temp_dir = tempfile::tempdir().expect("create shared-tuple multi-pool store dir");
16496        let case = if batch { "batch" } else { "single" };
16497        let (ctx, store, _shutdown) = without_storage_class_env(build_isolated_test_store(
16498            temp_dir.path(),
16499            &format!("multi-pool-same-tuple-{case}"),
16500            &[4, 4],
16501        ))
16502        .await;
16503        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
16504        let tier_name = if batch { "SAME-TUPLE-BATCH" } else { "SAME-TUPLE-SINGLE" };
16505        let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
16506        let bucket = format!("multi-pool-same-tuple-{case}-bucket");
16507        let object = "archive.bin";
16508        store
16509            .make_bucket(&bucket, &MakeBucketOptions::default())
16510            .await
16511            .expect("shared-tuple bucket should be created");
16512        let mut reader = PutObjReader::from_vec(vec![b's'; 1024 * 1024]);
16513        let source = store.pools[0]
16514            .put_object(&bucket, object, &mut reader, &ObjectOptions::default())
16515            .await
16516            .expect("first pool source should be written");
16517        store.pools[0]
16518            .transition_object(
16519                &bucket,
16520                object,
16521                &ObjectOptions {
16522                    transition: TransitionOptions {
16523                        status: TRANSITION_PENDING.to_string(),
16524                        tier: tier_name.to_string(),
16525                        etag: source.etag.clone().expect("shared source should have an etag"),
16526                        ..Default::default()
16527                    },
16528                    mod_time: source.mod_time,
16529                    ..Default::default()
16530                },
16531            )
16532            .await
16533            .expect("first pool source should transition");
16534        let first = store.pools[0]
16535            .get_object_info(&bucket, object, &ObjectOptions::default())
16536            .await
16537            .expect("first pool transitioned source should be readable");
16538
16539        // Model the exact metadata duplication produced by data movement:
16540        // both pools refer to the same remote object/version/backend and carry
16541        // the same stable source identity.
16542        for disk_index in 0..4 {
16543            let source_meta = temp_dir
16544                .path()
16545                .join(format!("pool0/set0/disk{disk_index}/{bucket}/{object}/{STORAGE_FORMAT_FILE}"));
16546            let target_meta = temp_dir
16547                .path()
16548                .join(format!("pool1/set0/disk{disk_index}/{bucket}/{object}/{STORAGE_FORMAT_FILE}"));
16549            tokio::fs::create_dir_all(target_meta.parent().expect("target xl.meta should have a parent"))
16550                .await
16551                .expect("second pool object directory should be created");
16552            tokio::fs::copy(&source_meta, &target_meta)
16553                .await
16554                .expect("transitioned xl.meta should copy exactly to the second pool");
16555        }
16556        let second = store.pools[1]
16557            .get_object_info(&bucket, object, &ObjectOptions::default())
16558            .await
16559            .expect("copied second-pool source should be readable");
16560        assert_eq!(
16561            (first.transitioned_object.name.clone(), first.transitioned_object.version_id.clone()),
16562            (second.transitioned_object.name.clone(), second.transitioned_object.version_id.clone())
16563        );
16564        assert_eq!(backend.object_count().await, 1);
16565
16566        let receipt_sink = crate::object_api::TierFreeVersionReceiptSink::new();
16567        let mut delete_opts = ObjectOptions {
16568            tier_delete_journal_api: Some(store.clone()),
16569            tier_free_version_receipt_sink: Some(receipt_sink.clone()),
16570            ..Default::default()
16571        };
16572        if batch {
16573            delete_opts.delete_replication_config_snapshot = Some(Arc::new(
16574                ReplicationObjectBridge::delete_request_config_in(&store.ctx, &bucket)
16575                    .await
16576                    .expect("batch delete replication snapshot should load"),
16577            ));
16578            let (_deleted, errors, _accounting) = store.pools[0]
16579                .delete_objects_with_accounting(
16580                    &bucket,
16581                    vec![ObjectToDelete {
16582                        object_name: object.to_string(),
16583                        ..Default::default()
16584                    }],
16585                    delete_opts.clone(),
16586                )
16587                .await;
16588            assert!(errors.iter().all(Option::is_none), "first pool batch delete should commit: {errors:?}");
16589        } else {
16590            store.pools[0]
16591                .delete_object(&bucket, object, delete_opts.clone())
16592                .await
16593                .expect("first pool single delete should commit");
16594        }
16595
16596        let first_pool_versions = store.pools[0]
16597            .get_disks_by_key(object)
16598            .load_file_info_versions_exact(&bucket, object)
16599            .await
16600            .expect("first-pool metadata should decode")
16601            .expect("first pool must retain a free-version owner");
16602        assert_eq!(
16603            first_pool_versions
16604                .versions
16605                .iter()
16606                .chain(first_pool_versions.free_versions.iter())
16607                .filter(|version| version.tier_free_version())
16608                .count(),
16609            1,
16610            "first pool must retain exactly one free-version owner"
16611        );
16612        assert_eq!(
16613            first_pool_versions
16614                .versions
16615                .iter()
16616                .filter(|version| !version.tier_free_version())
16617                .count(),
16618            0,
16619            "first pool must remove its visible source"
16620        );
16621        assert_eq!(
16622            tier_delete_journal_count(store.clone()).await,
16623            0,
16624            "ordinary pool-local delete must not create a journal"
16625        );
16626
16627        ExpiryState::resize_workers(1, store.clone()).await;
16628        let first_recovery = recover_tier_free_versions(store.clone(), 100, None, None)
16629            .await
16630            .expect("first-pool free-version should be recoverable");
16631        assert_eq!(
16632            (first_recovery.enqueued, first_recovery.failed),
16633            (0, 0),
16634            "the live second-pool source is the merged walk winner, so its hidden loser must not dispatch cleanup"
16635        );
16636        wait_for_expiry_workers_idle(&store).await;
16637        assert_eq!(backend.remove_count().await, 0, "live second-pool source must gate remote DELETE");
16638        assert_eq!(backend.object_count().await, 1, "live source must retain its remote body");
16639        assert!(
16640            store.pools[0]
16641                .get_disks_by_key(object)
16642                .load_file_info_versions_exact(&bucket, object)
16643                .await
16644                .expect("blocked cleanup metadata should decode")
16645                .is_some(),
16646            "blocked cleanup must retain the first-pool free-version"
16647        );
16648        let surviving = store.pools[1]
16649            .get_object_info(&bucket, object, &ObjectOptions::default())
16650            .await
16651            .expect("second-pool source must remain GET-addressable");
16652        assert_eq!(surviving.transitioned_object.name, first.transitioned_object.name);
16653
16654        backend.set_remove_failure(true);
16655        if batch {
16656            let (_deleted, errors, _accounting) = store.pools[1]
16657                .delete_objects_with_accounting(
16658                    &bucket,
16659                    vec![ObjectToDelete {
16660                        object_name: object.to_string(),
16661                        ..Default::default()
16662                    }],
16663                    delete_opts,
16664                )
16665                .await;
16666            assert!(errors.iter().all(Option::is_none), "second pool batch retry should commit: {errors:?}");
16667        } else {
16668            store.pools[1]
16669                .delete_object(&bucket, object, delete_opts)
16670                .await
16671                .expect("second pool single retry should commit");
16672        }
16673        wait_for_expiry_workers_idle(&store).await;
16674        let mut free_version_ids = Vec::new();
16675        for pool_idx in 0..2 {
16676            let retained = store.pools[pool_idx]
16677                .get_disks_by_key(object)
16678                .load_file_info_versions_exact(&bucket, object)
16679                .await
16680                .expect("shared-tuple free-version metadata should decode")
16681                .expect("remote failure must retain every shared-tuple cleanup owner");
16682            let owners = retained
16683                .versions
16684                .iter()
16685                .chain(retained.free_versions.iter())
16686                .filter(|version| version.tier_free_version())
16687                .collect::<Vec<_>>();
16688            assert_eq!(owners.len(), 1, "pool {pool_idx} must retain one free-version owner");
16689            free_version_ids.push(owners[0].version_id);
16690        }
16691        assert_ne!(
16692            free_version_ids[0], free_version_ids[1],
16693            "ordinary pool-local deletes must model distinct local free-version identities"
16694        );
16695
16696        backend.set_remove_failure(false);
16697        let receipts = receipt_sink
16698            .drain()
16699            .expect("the simulated outer multi-pool wrapper should drain exactly once");
16700        assert_eq!(
16701            receipts.len(),
16702            1,
16703            "the same physical key and remote tuple must collapse to one causal task"
16704        );
16705        assert_eq!(
16706            crate::bucket::lifecycle::bucket_lifecycle_ops::enqueue_committed_free_versions(&store, receipts).await,
16707            1,
16708            "the committed shared-tuple task should enter the running worker"
16709        );
16710        wait_for_expiry_workers_idle(&store).await;
16711        assert_eq!(backend.remove_count().await, 1, "shared remote tuple should be deleted exactly once");
16712        for pool_idx in 0..2 {
16713            assert!(
16714                store.pools[pool_idx]
16715                    .get_disks_by_key(object)
16716                    .load_file_info_versions_exact(&bucket, object)
16717                    .await
16718                    .expect("post-delete metadata should decode")
16719                    .is_none(),
16720                "pool {pool_idx} must not retain source or free-version metadata"
16721            );
16722        }
16723        store
16724            .delete_bucket(&bucket, &DeleteBucketOptions::default())
16725            .await
16726            .expect("shared-tuple bucket should be physically empty");
16727    }
16728
16729    #[cfg(feature = "test-util")]
16730    #[tokio::test]
16731    #[serial_test::serial(storage_class_env)]
16732    async fn multi_pool_same_remote_tuple_single_delete_waits_for_all_sources() {
16733        run_multi_pool_same_remote_tuple_delete_case(false).await;
16734    }
16735
16736    #[cfg(feature = "test-util")]
16737    #[tokio::test]
16738    #[serial_test::serial(storage_class_env)]
16739    async fn multi_pool_same_remote_tuple_batch_delete_waits_for_all_sources() {
16740        run_multi_pool_same_remote_tuple_delete_case(true).await;
16741    }
16742
16743    #[cfg(feature = "test-util")]
16744    #[tokio::test]
16745    #[serial_test::serial(storage_class_env)]
16746    async fn multi_pool_transitioned_delete_persists_one_free_version_per_remote_tuple() {
16747        let temp_dir = tempfile::tempdir().expect("create multi-pool transitioned delete store dir");
16748        let (ctx, store, _shutdown) = without_storage_class_env(build_isolated_test_store(
16749            temp_dir.path(),
16750            "multi-pool-transitioned-delete-owner",
16751            &[4, 4],
16752        ))
16753        .await;
16754        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
16755        let tier_name = "MULTIPOOL-DELETE";
16756        let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
16757        let bucket = "multi-pool-transitioned-delete-owner-bucket";
16758        let object = "archive.bin";
16759        store
16760            .make_bucket(bucket, &MakeBucketOptions::default())
16761            .await
16762            .expect("multi-pool source bucket should be created");
16763
16764        let mut remote_tuples = Vec::new();
16765        for pool_idx in 0..2 {
16766            let mut reader = PutObjReader::from_vec(vec![b'a' + pool_idx as u8; 1024 * 1024]);
16767            let source = store.pools[pool_idx]
16768                .put_object(
16769                    bucket,
16770                    object,
16771                    &mut reader,
16772                    &ObjectOptions {
16773                        mod_time: Some(OffsetDateTime::now_utc() + time::Duration::seconds(pool_idx as i64)),
16774                        ..Default::default()
16775                    },
16776                )
16777                .await
16778                .expect("pool-local source should be written");
16779            store.pools[pool_idx]
16780                .transition_object(
16781                    bucket,
16782                    object,
16783                    &ObjectOptions {
16784                        transition: TransitionOptions {
16785                            status: TRANSITION_PENDING.to_string(),
16786                            tier: tier_name.to_string(),
16787                            etag: source.etag.clone().expect("pool-local source should have an etag"),
16788                            ..Default::default()
16789                        },
16790                        mod_time: source.mod_time,
16791                        ..Default::default()
16792                    },
16793                )
16794                .await
16795                .expect("pool-local source should transition");
16796            let transitioned = store.pools[pool_idx]
16797                .get_object_info(bucket, object, &ObjectOptions::default())
16798                .await
16799                .expect("pool-local transitioned source should be readable");
16800            remote_tuples.push((
16801                transitioned.transitioned_object.name.clone(),
16802                transitioned.transitioned_object.version_id.clone(),
16803            ));
16804        }
16805        assert_ne!(remote_tuples[0], remote_tuples[1], "fixture must use distinct remote cleanup tuples");
16806        assert_eq!(backend.object_count().await, 2);
16807
16808        backend.set_remove_failure(true);
16809        store
16810            .delete_object_with_tier_delete_journal(bucket, object, ObjectOptions::default())
16811            .await
16812            .expect("store-level delete should remove both pool copies");
16813        for pool_idx in 0..2 {
16814            let retained = store.pools[pool_idx]
16815                .get_disks_by_key(object)
16816                .load_file_info_versions_exact(bucket, object)
16817                .await
16818                .expect("pool-local post-delete metadata should decode")
16819                .expect("each pool-specific remote tuple must retain a cleanup owner");
16820            assert_eq!(
16821                retained
16822                    .versions
16823                    .iter()
16824                    .chain(retained.free_versions.iter())
16825                    .filter(|version| version.tier_free_version())
16826                    .count(),
16827                1,
16828                "pool {pool_idx} must retain exactly one free-version owner"
16829            );
16830            assert_eq!(
16831                retained
16832                    .versions
16833                    .iter()
16834                    .filter(|version| !version.tier_free_version())
16835                    .count(),
16836                0,
16837                "pool {pool_idx} must not retain a visible source"
16838            );
16839        }
16840        assert_eq!(
16841            tier_delete_journal_count(store.clone()).await,
16842            0,
16843            "ordinary multi-pool delete must not create tier-delete journals"
16844        );
16845        assert_eq!(backend.object_count().await, 2, "failed remote cleanup must retain both remote objects");
16846
16847        backend.set_remove_failure(false);
16848        wait_for_tier_free_version_recovery(store.clone(), &backend, 2).await;
16849        let mut removed = backend.remove_versions().await;
16850        removed.sort();
16851        remote_tuples.sort();
16852        assert_eq!(removed, remote_tuples, "recovery must use each pool's exact remote tuple");
16853        for pool_idx in 0..2 {
16854            assert!(
16855                store.pools[pool_idx]
16856                    .get_disks_by_key(object)
16857                    .load_file_info_versions_exact(bucket, object)
16858                    .await
16859                    .expect("post-recovery pool metadata should decode")
16860                    .is_none(),
16861                "pool {pool_idx} must not retain source or free-version metadata"
16862            );
16863        }
16864        store
16865            .delete_bucket(bucket, &DeleteBucketOptions::default())
16866            .await
16867            .expect("multi-pool source bucket should be physically empty");
16868    }
16869
16870    // Phase 5 follow-up (backlog#1052): building a real store through the
16871    // ctx-explicit constructor lands every construction-time write — object
16872    // graph adoption, local-disk registry, deployment id — on the passed
16873    // context, not on the process bootstrap one. This is the storage-layer
16874    // seam a future second embedded server needs to stay isolated.
16875    #[tokio::test]
16876    #[serial_test::serial(storage_class_env)]
16877    async fn new_with_instance_ctx_threads_context_through_store_graph() {
16878        let temp_dir = tempfile::tempdir().expect("create temp store dir");
16879        let (instance_ctx, store, _shutdown) =
16880            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "instance-ctx-store-graph-test", &[4])).await;
16881
16882        assert!(
16883            Arc::ptr_eq(&store.ctx, &instance_ctx),
16884            "the store must adopt the explicitly passed instance context"
16885        );
16886        for sets in &store.pools {
16887            assert!(
16888                Arc::ptr_eq(sets.instance_ctx(), &instance_ctx),
16889                "every pool's Sets must carry the passed instance context"
16890            );
16891        }
16892        assert_eq!(
16893            instance_ctx.deployment_id(),
16894            Some(store.id),
16895            "the deployment id must land on the passed context and mirror the store id"
16896        );
16897
16898        let registered: Vec<String> = instance_ctx.local_disk_map().read().await.keys().cloned().collect();
16899        assert_eq!(registered.len(), 4, "the passed context must register all four local disks");
16900        let registered_disk_ids = instance_ctx.local_disk_id_map();
16901        let registered_disk_ids = registered_disk_ids.read().await;
16902        assert_eq!(registered_disk_ids.len(), 4, "the passed context must publish all four disk IDs");
16903        for endpoint in registered_disk_ids.values() {
16904            assert!(
16905                registered.contains(endpoint),
16906                "every disk ID in the passed context must resolve to one of its registered endpoints"
16907            );
16908        }
16909        drop(registered_disk_ids);
16910        let bootstrap = crate::runtime::instance::bootstrap_ctx();
16911        assert_ne!(
16912            bootstrap.deployment_id(),
16913            Some(store.id),
16914            "the bootstrap context must not absorb the fresh store's deployment id"
16915        );
16916        let bootstrap_map = bootstrap.local_disk_map();
16917        let bootstrap_map = bootstrap_map.read().await;
16918        for key in &registered {
16919            assert!(
16920                !bootstrap_map.contains_key(key),
16921                "the bootstrap context must not absorb the fresh store's disks"
16922            );
16923        }
16924        drop(bootstrap_map);
16925        let bootstrap_disk_ids = bootstrap.local_disk_id_map();
16926        let bootstrap_disk_ids = bootstrap_disk_ids.read().await;
16927        for endpoint in bootstrap_disk_ids.values() {
16928            assert!(
16929                !registered.contains(endpoint),
16930                "the bootstrap context must not absorb the fresh store's disk IDs"
16931            );
16932        }
16933    }
16934
16935    #[tokio::test]
16936    #[serial_test::serial(storage_class_env)]
16937    async fn new_with_instance_ctx_applies_default_parity_to_each_real_pool() {
16938        let temp_dir = tempfile::tempdir().expect("create multi-pool store dir");
16939        let (_, store, _shutdown) =
16940            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "pool-parity-regression", &[4, 2])).await;
16941
16942        assert_eq!(store.pools.len(), 2);
16943        assert_eq!(store.pools[0].default_parity_count, 2);
16944        assert_eq!(store.pools[0].disk_set[0].default_parity_count, 2);
16945        assert_eq!(store.pools[1].default_parity_count, 1);
16946        assert_eq!(store.pools[1].disk_set[0].default_parity_count, 1);
16947    }
16948
16949    // backlog#1052 S3: two stores in one process each initialize their own
16950    // bucket metadata system on their own instance context. Before this, the
16951    // second `init_bucket_metadata_sys` panicked on the process-global
16952    // OnceLock — the hard blocker for a second embedded server's services.
16953    #[tokio::test]
16954    #[serial_test::serial(storage_class_env)]
16955    async fn two_stores_initialize_their_own_bucket_metadata_sys() {
16956        let temp_a = tempfile::tempdir().expect("create temp store dir a");
16957        let temp_b = tempfile::tempdir().expect("create temp store dir b");
16958        let (ctx_a, store_a, _shutdown_a) =
16959            without_storage_class_env(build_isolated_test_store(temp_a.path(), "bucket-metadata-isolation-a", &[4])).await;
16960        let (ctx_b, store_b, _shutdown_b) =
16961            without_storage_class_env(build_isolated_test_store(temp_b.path(), "bucket-metadata-isolation-b", &[4])).await;
16962
16963        crate::bucket::metadata_sys::init_bucket_metadata_sys(store_a.clone(), Vec::new()).await;
16964        // The old process-global cell would panic right here.
16965        crate::bucket::metadata_sys::init_bucket_metadata_sys(store_b.clone(), Vec::new()).await;
16966
16967        let sys_a = ctx_a
16968            .bucket_metadata_sys()
16969            .expect("store A's context must hold its metadata system");
16970        let sys_b = ctx_b
16971            .bucket_metadata_sys()
16972            .expect("store B's context must hold its metadata system");
16973        assert!(!Arc::ptr_eq(&sys_a, &sys_b), "each store must own a distinct bucket metadata system");
16974    }
16975
16976    #[tokio::test]
16977    #[serial_test::serial(storage_class_env)]
16978    async fn same_name_bucket_recursive_delete_uses_owning_instance_object_lock_state() {
16979        let temp_a = tempfile::tempdir().expect("create temp store dir a");
16980        let temp_b = tempfile::tempdir().expect("create temp store dir b");
16981        let (_ctx_a, store_a, _shutdown_a) =
16982            without_storage_class_env(build_isolated_test_store(temp_a.path(), "delete-scope-a", &[4])).await;
16983        let (_ctx_b, store_b, _shutdown_b) =
16984            without_storage_class_env(build_isolated_test_store(temp_b.path(), "delete-scope-b", &[4])).await;
16985        crate::bucket::metadata_sys::init_bucket_metadata_sys(store_a.clone(), Vec::new()).await;
16986        crate::bucket::metadata_sys::init_bucket_metadata_sys(store_b.clone(), Vec::new()).await;
16987
16988        let bucket = format!("same-name-delete-scope-{}", uuid::Uuid::new_v4());
16989        store_a
16990            .make_bucket(&bucket, &MakeBucketOptions::default())
16991            .await
16992            .expect("create plain bucket in store A");
16993        store_b
16994            .make_bucket(
16995                &bucket,
16996                &MakeBucketOptions {
16997                    lock_enabled: true,
16998                    ..Default::default()
16999                },
17000            )
17001            .await
17002            .expect("create Object Lock bucket in store B");
17003
17004        for store in [&store_a, &store_b] {
17005            let mut reader = PutObjReader::from_vec(b"payload".to_vec());
17006            store
17007                .put_object(
17008                    &bucket,
17009                    "prefix/object.bin",
17010                    &mut reader,
17011                    &ObjectOptions {
17012                        versioned: Arc::ptr_eq(store, &store_b),
17013                        ..Default::default()
17014                    },
17015                )
17016                .await
17017                .expect("put same-name object");
17018        }
17019
17020        store_a
17021            .delete_object(
17022                &bucket,
17023                "prefix",
17024                ObjectOptions {
17025                    delete_prefix: true,
17026                    ..Default::default()
17027                },
17028            )
17029            .await
17030            .expect("plain store prefix delete should remain allowed");
17031
17032        let err = store_b
17033            .delete_object(
17034                &bucket,
17035                "prefix",
17036                ObjectOptions {
17037                    delete_prefix: true,
17038                    ..Default::default()
17039                },
17040            )
17041            .await
17042            .expect_err("store B must not consult store A's plain bucket metadata");
17043        assert!(matches!(err, StorageError::InvalidArgument(_, _, _)));
17044        store_b
17045            .get_object_info(&bucket, "prefix/object.bin", &ObjectOptions::default())
17046            .await
17047            .expect("locked store object must survive the rejected prefix delete");
17048    }
17049
17050    #[tokio::test]
17051    #[serial_test::serial(storage_class_env)]
17052    async fn recursive_delete_checks_object_lock_on_every_version_page() {
17053        let temp = tempfile::tempdir().expect("create temp store dir");
17054        let (_ctx, store, _shutdown) =
17055            without_storage_class_env(build_isolated_test_store(temp.path(), "recursive-delete-pagination", &[4])).await;
17056        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
17057
17058        let bucket = format!("recursive-page-{}", uuid::Uuid::new_v4());
17059        store
17060            .make_bucket(&bucket, &MakeBucketOptions::default())
17061            .await
17062            .expect("create plain bucket");
17063        for object in ["prefix/a.bin", "prefix/b.bin"] {
17064            let mut reader = PutObjReader::from_vec(b"plain".to_vec());
17065            store
17066                .put_object(&bucket, object, &mut reader, &ObjectOptions::default())
17067                .await
17068                .expect("write first-page object");
17069        }
17070        let locked_object = "prefix/z-locked.bin";
17071        let mut reader = PutObjReader::from_vec(b"locked".to_vec());
17072        store
17073            .put_object(
17074                &bucket,
17075                locked_object,
17076                &mut reader,
17077                &ObjectOptions {
17078                    user_defined: HashMap::from([
17079                        (
17080                            rustfs_filemeta::metadata_keys::OBJECT_LOCK_MODE.to_string(),
17081                            s3s::dto::ObjectLockRetentionMode::COMPLIANCE.to_string(),
17082                        ),
17083                        (
17084                            rustfs_filemeta::metadata_keys::OBJECT_LOCK_RETAIN_UNTIL_DATE.to_string(),
17085                            "2099-01-01T00:00:00Z".to_string(),
17086                        ),
17087                    ]),
17088                    ..Default::default()
17089                },
17090            )
17091            .await
17092            .expect("write second-page locked object");
17093
17094        let first_page = store.pools[0].disk_set[0]
17095            .clone()
17096            .inner_list_object_versions_for_recursive_delete(&bucket, "prefix/", None, None, 2)
17097            .await
17098            .expect("list the first recursive-delete validation page");
17099        assert!(first_page.is_truncated, "the fixture must require a continuation page");
17100        assert!(
17101            first_page.objects.iter().all(|object| object.name != locked_object),
17102            "the locked object must not be visible on the first page"
17103        );
17104        let second_page = store.pools[0].disk_set[0]
17105            .clone()
17106            .inner_list_object_versions_for_recursive_delete(
17107                &bucket,
17108                "prefix/",
17109                first_page.next_marker,
17110                first_page.next_version_idmarker,
17111                2,
17112            )
17113            .await
17114            .expect("list the recursive-delete continuation page");
17115        assert!(
17116            second_page.objects.iter().any(|object| object.name == locked_object),
17117            "the continuation page must contain the locked object"
17118        );
17119
17120        let err = store
17121            .delete_object(
17122                &bucket,
17123                "prefix/",
17124                ObjectOptions {
17125                    delete_prefix: true,
17126                    ..Default::default()
17127                },
17128            )
17129            .await
17130            .expect_err("recursive delete must scan the continuation page and find the locked object");
17131        assert!(
17132            matches!(err, StorageError::PrefixAccessDenied(ref name, ref object) if name == &bucket && object == locked_object),
17133            "unexpected recursive delete error: {err:?}"
17134        );
17135        for object in ["prefix/a.bin", "prefix/b.bin", locked_object] {
17136            store
17137                .get_object_info(&bucket, object, &ObjectOptions::default())
17138                .await
17139                .expect("a rejected recursive delete must leave every page intact");
17140        }
17141    }
17142
17143    #[tokio::test]
17144    #[serial_test::serial(storage_class_env)]
17145    async fn delete_holds_bucket_incarnation_sentinel_through_object_commit() {
17146        let temp = tempfile::tempdir().expect("create temp store dir");
17147        let (_ctx, store, _shutdown) =
17148            without_storage_class_env(build_isolated_test_store(temp.path(), "delete-bucket-incarnation", &[4])).await;
17149        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
17150
17151        let bucket = format!("delete-bucket-incarnation-{}", uuid::Uuid::new_v4());
17152        let object = "same-key.bin";
17153        store
17154            .make_bucket(&bucket, &MakeBucketOptions::default())
17155            .await
17156            .expect("create initial plain bucket");
17157        let old_incarnation = store
17158            .bucket_incarnation_id(&bucket)
17159            .await
17160            .expect("read initial bucket incarnation");
17161        let mut reader = PutObjReader::from_vec(b"old generation".to_vec());
17162        store
17163            .put_object(&bucket, object, &mut reader, &ObjectOptions::default())
17164            .await
17165            .expect("put old-generation object");
17166
17167        let barrier = crate::store::object::DeleteAfterObjectLockSnapshotBarrier::install(&bucket);
17168        let delete_store = Arc::clone(&store);
17169        let delete_bucket = bucket.clone();
17170        let delete = tokio::spawn(async move {
17171            delete_store
17172                .delete_object(&delete_bucket, object, ObjectOptions::default())
17173                .await
17174        });
17175        barrier.wait_until_paused().await;
17176
17177        let recreate_store = Arc::clone(&store);
17178        let recreate_bucket = bucket.clone();
17179        let mut recreate = tokio::spawn(async move {
17180            recreate_store
17181                .delete_bucket(
17182                    &recreate_bucket,
17183                    &crate::storage_api_contracts::bucket::DeleteBucketOptions {
17184                        force: true,
17185                        ..Default::default()
17186                    },
17187                )
17188                .await?;
17189            recreate_store
17190                .make_bucket(
17191                    &recreate_bucket,
17192                    &MakeBucketOptions {
17193                        lock_enabled: true,
17194                        ..Default::default()
17195                    },
17196                )
17197                .await?;
17198            let mut reader = PutObjReader::from_vec(b"new protected generation".to_vec());
17199            recreate_store
17200                .put_object(
17201                    &recreate_bucket,
17202                    object,
17203                    &mut reader,
17204                    &ObjectOptions {
17205                        versioned: true,
17206                        ..Default::default()
17207                    },
17208                )
17209                .await?;
17210            Ok::<_, Error>(())
17211        });
17212
17213        assert!(
17214            tokio::time::timeout(Duration::from_millis(100), &mut recreate).await.is_err(),
17215            "bucket delete and same-name recreation must wait for the in-flight object delete"
17216        );
17217        barrier.release();
17218        delete
17219            .await
17220            .expect("delete task should join")
17221            .expect("old-generation delete should complete");
17222        tokio::time::timeout(Duration::from_secs(10), recreate)
17223            .await
17224            .expect("bucket recreation should finish after delete releases the sentinel")
17225            .expect("bucket recreation task should join")
17226            .expect("bucket recreation should succeed");
17227
17228        let stale_delete_err = store
17229            .delete_object(
17230                &bucket,
17231                object,
17232                ObjectOptions {
17233                    expected_bucket_incarnation_id: Some(old_incarnation),
17234                    versioned: true,
17235                    ..Default::default()
17236                },
17237            )
17238            .await
17239            .expect_err("an authorization snapshot from the old bucket incarnation must be rejected");
17240        assert!(matches!(stale_delete_err, StorageError::BucketNotFound(name) if name == bucket));
17241
17242        let (_deleted, stale_batch_errors) = store
17243            .delete_objects(
17244                &bucket,
17245                vec![crate::storage_api_contracts::object::ObjectToDelete {
17246                    object_name: object.to_string(),
17247                    ..Default::default()
17248                }],
17249                ObjectOptions {
17250                    expected_bucket_incarnation_id: Some(old_incarnation),
17251                    versioned: true,
17252                    ..Default::default()
17253                },
17254            )
17255            .await;
17256        assert!(
17257            matches!(stale_batch_errors.as_slice(), [Some(StorageError::BucketNotFound(name))] if name.as_str() == bucket),
17258            "batch delete must reject an authorization snapshot from the old bucket incarnation"
17259        );
17260
17261        store
17262            .get_object_info(
17263                &bucket,
17264                object,
17265                &ObjectOptions {
17266                    versioned: true,
17267                    ..Default::default()
17268                },
17269            )
17270            .await
17271            .expect("the new locked-bucket object must survive the old-generation delete");
17272    }
17273
17274    #[tokio::test]
17275    #[serial_test::serial(storage_class_env)]
17276    async fn delete_objects_holds_bucket_incarnation_sentinel_through_commit() {
17277        let temp = tempfile::tempdir().expect("create temp store dir");
17278        let (_ctx, store, _shutdown) =
17279            without_storage_class_env(build_isolated_test_store(temp.path(), "batch-delete-bucket-incarnation", &[4])).await;
17280        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
17281
17282        let bucket = format!("batch-incarnation-{}", uuid::Uuid::new_v4());
17283        let object = "same-key.bin";
17284        store
17285            .make_bucket(&bucket, &MakeBucketOptions::default())
17286            .await
17287            .expect("create initial bucket");
17288        let mut reader = PutObjReader::from_vec(b"old generation".to_vec());
17289        store
17290            .put_object(&bucket, object, &mut reader, &ObjectOptions::default())
17291            .await
17292            .expect("put old-generation object");
17293
17294        let barrier = crate::store::object::DeleteAfterObjectLockSnapshotBarrier::install(&bucket);
17295        let delete_store = Arc::clone(&store);
17296        let delete_bucket = bucket.clone();
17297        let delete = tokio::spawn(async move {
17298            delete_store
17299                .delete_objects(
17300                    &delete_bucket,
17301                    vec![crate::storage_api_contracts::object::ObjectToDelete {
17302                        object_name: object.to_string(),
17303                        ..Default::default()
17304                    }],
17305                    ObjectOptions::default(),
17306                )
17307                .await
17308        });
17309        barrier.wait_until_paused().await;
17310
17311        let recreate_store = Arc::clone(&store);
17312        let recreate_bucket = bucket.clone();
17313        let mut recreate = tokio::spawn(async move {
17314            recreate_store
17315                .delete_bucket(
17316                    &recreate_bucket,
17317                    &crate::storage_api_contracts::bucket::DeleteBucketOptions {
17318                        force: true,
17319                        ..Default::default()
17320                    },
17321                )
17322                .await?;
17323            recreate_store
17324                .make_bucket(&recreate_bucket, &MakeBucketOptions::default())
17325                .await?;
17326            let mut reader = PutObjReader::from_vec(b"new generation".to_vec());
17327            recreate_store
17328                .put_object(&recreate_bucket, object, &mut reader, &ObjectOptions::default())
17329                .await?;
17330            Ok::<_, Error>(())
17331        });
17332
17333        assert!(
17334            tokio::time::timeout(Duration::from_millis(100), &mut recreate).await.is_err(),
17335            "bucket recreation must wait for the in-flight batch delete"
17336        );
17337        barrier.release();
17338        let (_deleted, errors) = delete.await.expect("batch delete task should join");
17339        assert!(
17340            errors.iter().all(Option::is_none),
17341            "old-generation batch delete should complete: {errors:?}"
17342        );
17343        tokio::time::timeout(Duration::from_secs(10), recreate)
17344            .await
17345            .expect("bucket recreation should finish after batch delete releases the sentinel")
17346            .expect("bucket recreation task should join")
17347            .expect("bucket recreation should succeed");
17348        store
17349            .get_object_info(&bucket, object, &ObjectOptions::default())
17350            .await
17351            .expect("new-generation object must survive the old batch delete");
17352    }
17353
17354    #[tokio::test]
17355    #[serial_test::serial(storage_class_env)]
17356    async fn prefix_delete_serializes_with_concurrent_object_lock_metadata_updates() {
17357        let temp = tempfile::tempdir().expect("create temp store dir");
17358        let (_ctx, store, _shutdown) =
17359            without_storage_class_env(build_isolated_test_store(temp.path(), "prefix-delete-lock-metadata", &[4])).await;
17360        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
17361
17362        let cases = [
17363            (
17364                "compliance",
17365                HashMap::from([
17366                    ("x-amz-object-lock-mode".to_string(), "COMPLIANCE".to_string()),
17367                    ("x-amz-object-lock-retain-until-date".to_string(), "2099-01-01T00:00:00Z".to_string()),
17368                ]),
17369            ),
17370            (
17371                "legal-hold",
17372                HashMap::from([("x-amz-object-lock-legal-hold".to_string(), "ON".to_string())]),
17373            ),
17374        ];
17375
17376        for (case, eval_metadata) in cases {
17377            let bucket = format!("prefix-lock-{case}-{}", uuid::Uuid::new_v4());
17378            let object = "protected-after-scan.bin";
17379            store
17380                .make_bucket(
17381                    &bucket,
17382                    &MakeBucketOptions {
17383                        lock_enabled: true,
17384                        ..Default::default()
17385                    },
17386                )
17387                .await
17388                .expect("create Object Lock bucket");
17389            let mut reader = PutObjReader::from_vec(b"body".to_vec());
17390            let object_info = store
17391                .put_object(
17392                    &bucket,
17393                    object,
17394                    &mut reader,
17395                    &ObjectOptions {
17396                        versioned: true,
17397                        ..Default::default()
17398                    },
17399                )
17400                .await
17401                .expect("write unprotected version");
17402            let version_id = object_info.version_id.expect("versioned PUT should return an ID").to_string();
17403
17404            let barrier = crate::store::object::DeleteAfterObjectLockSnapshotBarrier::install(&bucket);
17405            let delete_store = Arc::clone(&store);
17406            let delete_bucket = bucket.clone();
17407            let delete = tokio::spawn(async move {
17408                delete_store
17409                    .delete_object(
17410                        &delete_bucket,
17411                        object,
17412                        ObjectOptions {
17413                            delete_prefix: true,
17414                            delete_prefix_object: true,
17415                            versioned: true,
17416                            ..Default::default()
17417                        },
17418                    )
17419                    .await
17420            });
17421            barrier.wait_until_paused().await;
17422
17423            let update_store = Arc::clone(&store);
17424            let update_bucket = bucket.clone();
17425            let mut update = tokio::spawn(async move {
17426                update_store
17427                    .put_object_metadata(
17428                        &update_bucket,
17429                        object,
17430                        &ObjectOptions {
17431                            version_id: Some(version_id),
17432                            versioned: true,
17433                            eval_metadata: Some(eval_metadata),
17434                            ..Default::default()
17435                        },
17436                    )
17437                    .await
17438            });
17439            assert!(
17440                tokio::time::timeout(Duration::from_millis(100), &mut update).await.is_err(),
17441                "{case} metadata update must wait behind the prefix-delete lifecycle write fence"
17442            );
17443
17444            barrier.release();
17445            delete
17446                .await
17447                .expect("prefix delete task should join")
17448                .expect("prefix delete should remove the unprotected version");
17449            update
17450                .await
17451                .expect("metadata update task should join")
17452                .expect_err("metadata update must not succeed after the version was deleted");
17453            assert!(
17454                store
17455                    .get_object_info(
17456                        &bucket,
17457                        object,
17458                        &ObjectOptions {
17459                            versioned: true,
17460                            ..Default::default()
17461                        },
17462                    )
17463                    .await
17464                    .is_err(),
17465                "{case} case must not report a successful protected write followed by deletion"
17466            );
17467        }
17468    }
17469
17470    #[tokio::test]
17471    #[serial_test::serial(storage_class_env)]
17472    async fn put_rejects_same_name_bucket_recreated_after_request_snapshot() {
17473        let temp = tempfile::tempdir().expect("create temp store dir");
17474        let (_ctx, store, _shutdown) =
17475            without_storage_class_env(build_isolated_test_store(temp.path(), "put-bucket-incarnation", &[4])).await;
17476        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
17477
17478        let bucket = format!("put-bucket-incarnation-{}", uuid::Uuid::new_v4());
17479        let object = "object.bin";
17480        store
17481            .make_bucket(&bucket, &MakeBucketOptions::default())
17482            .await
17483            .expect("create initial bucket");
17484        let old_incarnation = store
17485            .bucket_incarnation_id(&bucket)
17486            .await
17487            .expect("read initial bucket incarnation");
17488
17489        let barrier = crate::set_disk::PutObjectCommitBarrier::install(
17490            &bucket,
17491            object,
17492            crate::set_disk::PutObjectCommitPause::BeforeNamespace,
17493        );
17494        let put_store = Arc::clone(&store);
17495        let put_bucket = bucket.clone();
17496        let put = tokio::spawn(async move {
17497            let mut reader = PutObjReader::from_vec(b"old generation".to_vec());
17498            put_store
17499                .put_object(
17500                    &put_bucket,
17501                    object,
17502                    &mut reader,
17503                    &ObjectOptions {
17504                        versioned: true,
17505                        ..Default::default()
17506                    },
17507                )
17508                .await
17509        });
17510        barrier.wait_until_paused().await;
17511
17512        store
17513            .delete_bucket(&bucket, &crate::storage_api_contracts::bucket::DeleteBucketOptions::default())
17514            .await
17515            .expect("delete initial empty bucket while PUT is staged");
17516        store
17517            .make_bucket(&bucket, &MakeBucketOptions::default())
17518            .await
17519            .expect("recreate same-name bucket");
17520        let new_incarnation = store
17521            .bucket_incarnation_id(&bucket)
17522            .await
17523            .expect("read recreated bucket incarnation");
17524        assert_ne!(old_incarnation, new_incarnation);
17525
17526        barrier.release();
17527        let err = put
17528            .await
17529            .expect("PUT task should join")
17530            .expect_err("old-generation PUT must not commit into the recreated bucket");
17531        assert!(
17532            matches!(err, StorageError::BucketNotFound(ref name) if name == &bucket),
17533            "unexpected stale PUT error: {err:?}"
17534        );
17535        assert!(
17536            store
17537                .get_object_info(&bucket, object, &ObjectOptions::default())
17538                .await
17539                .is_err(),
17540            "the recreated bucket must not contain the staged old-generation object"
17541        );
17542    }
17543
17544    #[tokio::test]
17545    #[serial_test::serial(storage_class_env)]
17546    async fn exact_version_put_uses_only_its_own_bucket_object_lock_snapshot() {
17547        let target_temp = tempfile::tempdir().expect("create target store dir");
17548        let foreign_temp = tempfile::tempdir().expect("create foreign store dir");
17549        let (target_ctx, target_store, _target_shutdown) =
17550            without_storage_class_env(build_isolated_test_store(target_temp.path(), "put-object-lock-target", &[4])).await;
17551        let (_foreign_ctx, foreign_store, _foreign_shutdown) =
17552            without_storage_class_env(build_isolated_test_store(foreign_temp.path(), "put-object-lock-foreign", &[4])).await;
17553        crate::bucket::metadata_sys::init_bucket_metadata_sys(target_store.clone(), Vec::new()).await;
17554        crate::bucket::metadata_sys::init_bucket_metadata_sys(foreign_store.clone(), Vec::new()).await;
17555
17556        let bucket = format!("put-object-lock-snapshot-{}", uuid::Uuid::new_v4());
17557        let object = "object.bin";
17558        target_store
17559            .make_bucket(
17560                &bucket,
17561                &MakeBucketOptions {
17562                    lock_enabled: true,
17563                    ..Default::default()
17564                },
17565            )
17566            .await
17567            .expect("create target Object Lock bucket");
17568        foreign_store
17569            .make_bucket(&bucket, &MakeBucketOptions::default())
17570            .await
17571            .expect("create same-name plain bucket in foreign store");
17572        let stale_snapshot = target_store
17573            .object_lock_config_snapshot(&bucket)
17574            .await
17575            .expect("load target snapshot before changing its default retention");
17576
17577        let mut metadata = crate::bucket::metadata_sys::get_in(&target_ctx, &bucket)
17578            .await
17579            .expect("load target bucket metadata")
17580            .as_ref()
17581            .clone();
17582        metadata
17583            .update_config(
17584                crate::bucket::metadata::OBJECT_LOCK_CONFIG,
17585                b"<ObjectLockConfiguration><ObjectLockEnabled>Enabled</ObjectLockEnabled><Rule><DefaultRetention><Mode>COMPLIANCE</Mode><Days>1</Days></DefaultRetention></Rule></ObjectLockConfiguration>".to_vec(),
17586            )
17587            .expect("set default COMPLIANCE metadata");
17588        let update_started = Arc::new(tokio::sync::Barrier::new(2));
17589        let update_ctx = Arc::clone(&target_ctx);
17590        let update_started_task = Arc::clone(&update_started);
17591        let mut update = tokio::spawn(async move {
17592            update_started_task.wait().await;
17593            let _transaction_guard =
17594                crate::bucket::metadata_sys::acquire_bucket_metadata_transaction_lock_in(&update_ctx, &metadata.name).await?;
17595            crate::bucket::metadata_sys::set_bucket_metadata_in(&update_ctx, metadata).await
17596        });
17597        update_started.wait().await;
17598        assert!(
17599            tokio::time::timeout(Duration::from_millis(100), &mut update).await.is_err(),
17600            "an active Object Lock snapshot must hold the metadata revision read fence"
17601        );
17602        drop(stale_snapshot);
17603        update
17604            .await
17605            .expect("Object Lock metadata update task should join")
17606            .expect("persist target Object Lock metadata after snapshot release");
17607
17608        let original_body = b"protected body".to_vec();
17609        let mut original_reader = PutObjReader::from_vec(original_body.clone());
17610        let original = target_store
17611            .put_object(
17612                &bucket,
17613                object,
17614                &mut original_reader,
17615                &ObjectOptions {
17616                    versioned: true,
17617                    ..Default::default()
17618                },
17619            )
17620            .await
17621            .expect("write protected target version");
17622        let version_id = original
17623            .version_id
17624            .expect("versioned PUT should return a version ID")
17625            .to_string();
17626
17627        let mut replacement = PutObjReader::from_vec(b"replacement".to_vec());
17628        let err = target_store
17629            .put_object(
17630                &bucket,
17631                object,
17632                &mut replacement,
17633                &ObjectOptions {
17634                    version_id: Some(version_id.clone()),
17635                    versioned: true,
17636                    ..Default::default()
17637                },
17638            )
17639            .await
17640            .expect_err("ECStore must load default COMPLIANCE before an exact-version overwrite");
17641        assert!(matches!(err, StorageError::PrefixAccessDenied(_, _)));
17642
17643        let foreign_snapshot = foreign_store
17644            .object_lock_config_snapshot(&bucket)
17645            .await
17646            .expect("load same-name foreign bucket snapshot");
17647        let mut foreign_replacement = PutObjReader::from_vec(b"foreign replacement".to_vec());
17648        target_store
17649            .put_object(
17650                &bucket,
17651                object,
17652                &mut foreign_replacement,
17653                &ObjectOptions {
17654                    version_id: Some(version_id.clone()),
17655                    versioned: true,
17656                    object_lock_config_snapshot: Some(foreign_snapshot),
17657                    ..Default::default()
17658                },
17659            )
17660            .await
17661            .expect_err("a snapshot from another store must fail closed");
17662
17663        let mut reader = target_store
17664            .get_object_reader(
17665                &bucket,
17666                object,
17667                None,
17668                HeaderMap::new(),
17669                &ObjectOptions {
17670                    version_id: Some(version_id),
17671                    versioned: true,
17672                    ..Default::default()
17673                },
17674            )
17675            .await
17676            .expect("protected target version should remain readable");
17677        let mut body = Vec::new();
17678        reader
17679            .stream
17680            .read_to_end(&mut body)
17681            .await
17682            .expect("protected body should drain");
17683        assert_eq!(body, original_body);
17684    }
17685
17686    #[cfg(feature = "test-util")]
17687    #[tokio::test]
17688    #[serial_test::serial(storage_class_env)]
17689    async fn object_lock_snapshot_uses_read_quorum_bucket_existence_probe() {
17690        let temp = tempfile::tempdir().expect("create degraded snapshot store dir");
17691        let (ctx, store, _shutdown) = without_storage_class_env(build_isolated_test_store_with_layout(
17692            temp.path(),
17693            "degraded-object-lock-snapshot",
17694            &[(2, 12)],
17695            CancellationToken::new(),
17696            None,
17697        ))
17698        .await;
17699        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
17700
17701        let bucket = format!("degraded-ol-{}", uuid::Uuid::new_v4());
17702        store
17703            .make_bucket(&bucket, &MakeBucketOptions::default())
17704            .await
17705            .expect("create snapshot bucket");
17706        let expected_incarnation = store
17707            .bucket_incarnation_id(&bucket)
17708            .await
17709            .expect("read bucket incarnation before degrading sets");
17710
17711        let mut offline_disks = Vec::new();
17712        for set in store.all_set_disks() {
17713            offline_disks.push(force_set_disk_range_offline_for_test(&set, 6..12).await);
17714        }
17715
17716        let snapshot = store
17717            .object_lock_config_snapshot(&bucket)
17718            .await
17719            .expect("read-quorum bucket existence should admit guarded Object Lock snapshot");
17720        assert!(matches!(
17721            snapshot.state(),
17722            crate::bucket::metadata_sys::ObjectLockConfigState::ConfirmedAbsent
17723        ));
17724        assert!(snapshot.is_valid_for_destructive_put(store.id, &bucket, expected_incarnation));
17725
17726        let current_incarnation = crate::bucket::metadata_sys::get_object_lock_config_and_incarnation_from_disk_in(&ctx, &bucket)
17727            .await
17728            .expect("authoritative metadata read should also survive at read quorum")
17729            .1;
17730        assert_eq!(current_incarnation, expected_incarnation);
17731
17732        drop(offline_disks);
17733    }
17734
17735    #[tokio::test]
17736    #[serial_test::serial(storage_class_env)]
17737    async fn force_create_existing_bucket_preserves_incarnation_and_inflight_request() {
17738        let temp = tempfile::tempdir().expect("create temp store dir");
17739        let (_ctx, store, _shutdown) =
17740            without_storage_class_env(build_isolated_test_store(temp.path(), "force-create-incarnation", &[4])).await;
17741        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
17742
17743        let bucket = format!("force-create-incarnation-{}", uuid::Uuid::new_v4());
17744        store
17745            .make_bucket(&bucket, &MakeBucketOptions::default())
17746            .await
17747            .expect("create initial bucket");
17748        let incarnation = store.bucket_incarnation_id(&bucket).await.expect("read initial incarnation");
17749
17750        store
17751            .make_bucket(
17752                &bucket,
17753                &MakeBucketOptions {
17754                    force_create: true,
17755                    ..Default::default()
17756                },
17757            )
17758            .await
17759            .expect("idempotent force-create should succeed");
17760        assert_eq!(
17761            store.bucket_incarnation_id(&bucket).await.unwrap(),
17762            incarnation,
17763            "idempotent force-create must not rotate the bucket generation"
17764        );
17765
17766        let mut reader = PutObjReader::from_vec(b"authorized before force-create".to_vec());
17767        store
17768            .put_object(
17769                &bucket,
17770                "inflight.bin",
17771                &mut reader,
17772                &ObjectOptions {
17773                    expected_bucket_incarnation_id: Some(incarnation),
17774                    ..Default::default()
17775                },
17776            )
17777            .await
17778            .expect("a request authorized before idempotent force-create must still commit");
17779    }
17780
17781    #[tokio::test]
17782    #[serial_test::serial(storage_class_env)]
17783    async fn lock_enabled_create_retries_metadata_intent_without_unlocking() {
17784        let temp = tempfile::tempdir().expect("create temp store dir");
17785        let (ctx, store, _shutdown) =
17786            without_storage_class_env(build_isolated_test_store(temp.path(), "lock-create-intent-retry", &[4])).await;
17787        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
17788
17789        let bucket = format!("lock-create-intent-retry-{}", uuid::Uuid::new_v4());
17790        let mut intent = crate::bucket::metadata::BucketMetadata::new(&bucket);
17791        intent.lock_enabled = true;
17792        crate::bucket::metadata_sys::set_new_bucket_metadata_in(&ctx, intent)
17793            .await
17794            .expect("persist pre-visibility lock intent");
17795        assert!(
17796            store
17797                .peer_sys
17798                .get_bucket_info(&bucket, &crate::storage_api_contracts::bucket::BucketOptions::default())
17799                .await
17800                .is_err(),
17801            "the intent must not make the physical bucket visible"
17802        );
17803
17804        store
17805            .make_bucket(
17806                &bucket,
17807                &MakeBucketOptions {
17808                    lock_enabled: true,
17809                    ..Default::default()
17810                },
17811            )
17812            .await
17813            .expect("retry lock-enabled creation");
17814        assert!(matches!(
17815            crate::bucket::metadata_sys::get_object_lock_config_state_in(&ctx, &bucket)
17816                .await
17817                .expect("read retried lock state"),
17818            crate::bucket::metadata_sys::ObjectLockConfigState::Configured { .. }
17819        ));
17820    }
17821
17822    #[tokio::test]
17823    #[serial_test::serial(storage_class_env)]
17824    async fn physical_lock_bucket_with_incomplete_metadata_fails_closed() {
17825        let temp = tempfile::tempdir().expect("create temp store dir");
17826        let (ctx, store, _shutdown) =
17827            without_storage_class_env(build_isolated_test_store(temp.path(), "lock-create-incomplete-metadata", &[4])).await;
17828        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
17829
17830        let bucket = format!("lock-create-incomplete-metadata-{}", uuid::Uuid::new_v4());
17831        let mut metadata = crate::bucket::metadata::BucketMetadata::new(&bucket);
17832        metadata.lock_enabled = true;
17833        metadata
17834            .save_with_store(store.clone())
17835            .await
17836            .expect("persist lock metadata before sidecar");
17837        store
17838            .peer_sys
17839            .make_bucket(&bucket, &MakeBucketOptions::default())
17840            .await
17841            .expect("expose physical bucket after metadata phase");
17842
17843        let err = crate::bucket::metadata_sys::get_object_lock_config_state_in(&ctx, &bucket)
17844            .await
17845            .expect_err("missing sidecar must not fabricate an unlocked bucket");
17846        assert!(err.to_string().contains("sidecar is missing"), "unexpected error: {err}");
17847    }
17848
17849    #[tokio::test]
17850    #[serial_test::serial(storage_class_env)]
17851    async fn successful_lock_enabled_create_has_complete_metadata_intent() {
17852        let temp = tempfile::tempdir().expect("create temp store dir");
17853        let (ctx, store, _shutdown) =
17854            without_storage_class_env(build_isolated_test_store(temp.path(), "lock-create-complete-metadata", &[4])).await;
17855        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
17856
17857        let bucket = format!("lock-create-{}", uuid::Uuid::new_v4());
17858        store
17859            .make_bucket(
17860                &bucket,
17861                &MakeBucketOptions {
17862                    lock_enabled: true,
17863                    ..Default::default()
17864                },
17865            )
17866            .await
17867            .expect("create lock-enabled bucket");
17868        let (metadata, persisted) = crate::bucket::metadata_sys::get_config_from_disk_with_presence_in(&ctx, &bucket)
17869            .await
17870            .expect("read completed metadata intent");
17871        assert!(persisted && metadata.bucket_incarnation_sidecar && metadata.lock_enabled);
17872        assert!(matches!(
17873            crate::bucket::metadata_sys::get_object_lock_config_state_in(&ctx, &bucket)
17874                .await
17875                .expect("read completed lock state"),
17876            crate::bucket::metadata_sys::ObjectLockConfigState::Configured { .. }
17877        ));
17878    }
17879
17880    #[tokio::test]
17881    #[serial_test::serial(storage_class_env)]
17882    async fn lifecycle_expiry_rejects_task_from_recreated_bucket_incarnation() {
17883        let temp = tempfile::tempdir().expect("create temp store dir");
17884        let (_ctx, store, _shutdown) =
17885            without_storage_class_env(build_isolated_test_store(temp.path(), "lifecycle-expiry-incarnation", &[4])).await;
17886        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
17887
17888        let bucket = format!("lifecycle-incarnation-{}", uuid::Uuid::new_v4());
17889        let object = "same-key.bin";
17890        store
17891            .make_bucket(&bucket, &MakeBucketOptions::default())
17892            .await
17893            .expect("create initial bucket");
17894        let old_incarnation = store.bucket_incarnation_id(&bucket).await.expect("read old incarnation");
17895        let mut old_reader = PutObjReader::from_vec(b"old-generation".to_vec());
17896        store
17897            .put_object(&bucket, object, &mut old_reader, &ObjectOptions::default())
17898            .await
17899            .expect("put old-generation object");
17900        let old_object = store
17901            .get_object_info(&bucket, object, &ObjectOptions::default())
17902            .await
17903            .expect("snapshot old-generation object");
17904
17905        store
17906            .delete_object(&bucket, object, ObjectOptions::default())
17907            .await
17908            .expect("delete old-generation object");
17909        store
17910            .delete_bucket(&bucket, &crate::storage_api_contracts::bucket::DeleteBucketOptions::default())
17911            .await
17912            .expect("delete old bucket");
17913        store
17914            .make_bucket(&bucket, &MakeBucketOptions::default())
17915            .await
17916            .expect("recreate bucket");
17917        let mut new_reader = PutObjReader::from_vec(b"new-generation".to_vec());
17918        store
17919            .put_object(&bucket, object, &mut new_reader, &ObjectOptions::default())
17920            .await
17921            .expect("put replacement object");
17922
17923        let applied = crate::bucket::lifecycle::bucket_lifecycle_ops::apply_expiry_on_non_transitioned_objects(
17924            store.clone(),
17925            &old_object,
17926            &crate::bucket::lifecycle::lifecycle::Event {
17927                action: crate::bucket::lifecycle::lifecycle::IlmAction::DeleteAction,
17928                ..Default::default()
17929            },
17930            &crate::bucket::lifecycle::bucket_lifecycle_audit::LcEventSrc::Scanner,
17931            old_incarnation,
17932        )
17933        .await;
17934        assert!(!applied, "an expiry task from the old bucket incarnation must fail closed");
17935
17936        let mut reader = store
17937            .get_object_reader(&bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
17938            .await
17939            .expect("new-generation object must survive stale expiry");
17940        let mut body = Vec::new();
17941        reader.stream.read_to_end(&mut body).await.expect("read replacement object");
17942        assert_eq!(body, b"new-generation");
17943    }
17944
17945    #[tokio::test]
17946    #[serial_test::serial(storage_class_env)]
17947    async fn pre_upgrade_multipart_upload_can_complete_and_abort_in_the_same_bucket_lifetime() {
17948        let temp = tempfile::tempdir().expect("create temp store dir");
17949        let (_ctx, store, _shutdown) =
17950            without_storage_class_env(build_isolated_test_store(temp.path(), "multipart-upgrade-compat", &[4])).await;
17951        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
17952
17953        let bucket = format!("multipart-upgrade-compat-{}", uuid::Uuid::new_v4());
17954        let created = OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("fixed timestamp should be valid");
17955        store
17956            .make_bucket(
17957                &bucket,
17958                &MakeBucketOptions {
17959                    created_at: Some(created),
17960                    ..Default::default()
17961                },
17962            )
17963            .await
17964            .expect("create bucket");
17965
17966        let complete_object = "complete.bin";
17967        let complete_upload = store.pools[0]
17968            .new_multipart_upload(
17969                &bucket,
17970                complete_object,
17971                &ObjectOptions {
17972                    mod_time: Some(created + time::Duration::seconds(1)),
17973                    ..Default::default()
17974                },
17975            )
17976            .await
17977            .expect("create a pre-upgrade multipart upload without an incarnation stamp");
17978        let mut part_reader = PutObjReader::from_vec(b"pre-upgrade multipart body".to_vec());
17979        let part = store.pools[0]
17980            .put_object_part(
17981                &bucket,
17982                complete_object,
17983                &complete_upload.upload_id,
17984                1,
17985                &mut part_reader,
17986                &ObjectOptions::default(),
17987            )
17988            .await
17989            .expect("stage a pre-upgrade multipart part");
17990        store
17991            .clone()
17992            .complete_multipart_upload(
17993                &bucket,
17994                complete_object,
17995                &complete_upload.upload_id,
17996                vec![crate::storage_api_contracts::multipart::CompletePart {
17997                    part_num: part.part_num,
17998                    etag: part.etag,
17999                    ..Default::default()
18000                }],
18001                &ObjectOptions::default(),
18002            )
18003            .await
18004            .expect("a pre-upgrade upload from the current bucket lifetime should complete");
18005        store
18006            .get_object_info(&bucket, complete_object, &ObjectOptions::default())
18007            .await
18008            .expect("completed pre-upgrade upload should be visible");
18009
18010        let abort_object = "abort.bin";
18011        let abort_upload = store.pools[0]
18012            .new_multipart_upload(
18013                &bucket,
18014                abort_object,
18015                &ObjectOptions {
18016                    mod_time: Some(created + time::Duration::seconds(2)),
18017                    ..Default::default()
18018                },
18019            )
18020            .await
18021            .expect("create another pre-upgrade multipart upload");
18022        store
18023            .abort_multipart_upload(&bucket, abort_object, &abort_upload.upload_id, &ObjectOptions::default())
18024            .await
18025            .expect("a pre-upgrade upload from the current bucket lifetime should abort");
18026        let err = store
18027            .get_multipart_info(&bucket, abort_object, &abort_upload.upload_id, &ObjectOptions::default())
18028            .await
18029            .expect_err("aborted pre-upgrade upload should be removed");
18030        assert!(matches!(err, StorageError::InvalidUploadID(..)));
18031    }
18032
18033    #[tokio::test]
18034    #[serial_test::serial(storage_class_env)]
18035    async fn complete_multipart_rejects_legacy_upload_from_recreated_bucket_incarnation() {
18036        let temp = tempfile::tempdir().expect("create temp store dir");
18037        let (_ctx, store, _shutdown) =
18038            without_storage_class_env(build_isolated_test_store(temp.path(), "multipart-bucket-incarnation", &[4])).await;
18039        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
18040
18041        let bucket = format!("multipart-incarnation-{}", uuid::Uuid::new_v4());
18042        let object = "object.bin";
18043        let created = OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("fixed timestamp should be valid");
18044        store
18045            .make_bucket(
18046                &bucket,
18047                &MakeBucketOptions {
18048                    created_at: Some(created),
18049                    ..Default::default()
18050                },
18051            )
18052            .await
18053            .expect("create initial bucket");
18054        let old_incarnation = store
18055            .bucket_incarnation_id(&bucket)
18056            .await
18057            .expect("read initial bucket incarnation");
18058        let upload = store.pools[0]
18059            .new_multipart_upload(
18060                &bucket,
18061                object,
18062                &ObjectOptions {
18063                    mod_time: Some(created + time::Duration::seconds(1)),
18064                    ..Default::default()
18065                },
18066            )
18067            .await
18068            .expect("create old-generation legacy multipart upload");
18069
18070        store
18071            .delete_bucket(&bucket, &crate::storage_api_contracts::bucket::DeleteBucketOptions::default())
18072            .await
18073            .expect("delete bucket with an incomplete multipart upload");
18074        store
18075            .make_bucket(
18076                &bucket,
18077                &MakeBucketOptions {
18078                    created_at: Some(created + time::Duration::seconds(2)),
18079                    ..Default::default()
18080                },
18081            )
18082            .await
18083            .expect("recreate same-name bucket");
18084        let new_incarnation = store
18085            .bucket_incarnation_id(&bucket)
18086            .await
18087            .expect("read recreated bucket incarnation");
18088        assert_ne!(old_incarnation, new_incarnation);
18089
18090        let err = store
18091            .clone()
18092            .complete_multipart_upload(&bucket, object, &upload.upload_id, Vec::new(), &ObjectOptions::default())
18093            .await
18094            .expect_err("an upload initiated in the old bucket incarnation must not commit into the replacement");
18095        assert!(
18096            match &err {
18097                StorageError::BucketNotFound(name) => name == &bucket,
18098                StorageError::InvalidUploadID(name, key, _) => name == &bucket && key == object,
18099                _ => false,
18100            },
18101            "unexpected stale multipart completion error: {err:?}"
18102        );
18103        assert!(
18104            store
18105                .get_object_info(&bucket, object, &ObjectOptions::default())
18106                .await
18107                .is_err(),
18108            "the recreated bucket must not contain the old-generation multipart upload"
18109        );
18110    }
18111
18112    #[tokio::test]
18113    #[serial_test::serial(storage_class_env)]
18114    async fn delete_after_bucket_validation_preserves_bucket_not_found() {
18115        let temp = tempfile::tempdir().expect("create temp store dir");
18116        let (_ctx, store, _shutdown) =
18117            without_storage_class_env(build_isolated_test_store(temp.path(), "delete-missing-bucket", &[4])).await;
18118        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
18119
18120        let bucket = format!("delete-missing-bucket-{}", uuid::Uuid::new_v4());
18121        store
18122            .make_bucket(&bucket, &MakeBucketOptions::default())
18123            .await
18124            .expect("create bucket before validation");
18125        store
18126            .get_bucket_info(&bucket, &crate::storage_api_contracts::bucket::BucketOptions::default())
18127            .await
18128            .expect("request-level bucket validation should succeed");
18129        store
18130            .delete_bucket(&bucket, &crate::storage_api_contracts::bucket::DeleteBucketOptions::default())
18131            .await
18132            .expect("delete bucket after validation");
18133
18134        let err = store
18135            .delete_object(&bucket, "missing.bin", ObjectOptions::default())
18136            .await
18137            .expect_err("post-validation bucket deletion must not become an object-not-found error");
18138
18139        assert!(matches!(err, StorageError::BucketNotFound(name) if name == bucket));
18140    }
18141
18142    #[tokio::test]
18143    #[serial_test::serial(storage_class_env)]
18144    async fn object_lock_metadata_read_error_blocks_every_destructive_delete_shape() {
18145        let temp = tempfile::tempdir().expect("create temp store dir");
18146        let (ctx, store, _shutdown) =
18147            without_storage_class_env(build_isolated_test_store(temp.path(), "delete-config-read-error", &[4])).await;
18148        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
18149
18150        let bucket = format!("delete-config-read-error-{}", uuid::Uuid::new_v4());
18151        store
18152            .make_bucket(&bucket, &MakeBucketOptions::default())
18153            .await
18154            .expect("create bucket");
18155        for object in ["single.bin", "batch.bin", "prefix/child.bin", "lifecycle.bin"] {
18156            let mut reader = PutObjReader::from_vec(b"must survive".to_vec());
18157            store
18158                .put_object(&bucket, object, &mut reader, &ObjectOptions::default())
18159                .await
18160                .expect("put object before metadata corruption");
18161        }
18162
18163        crate::bucket::metadata_sys::inject_object_lock_disk_read_error_in(&ctx, &bucket)
18164            .await
18165            .expect("inject single-delete metadata read failure");
18166
18167        let single_err = store
18168            .delete_object(&bucket, "single.bin", ObjectOptions::default())
18169            .await
18170            .expect_err("single delete must propagate the metadata read failure");
18171        assert!(
18172            single_err
18173                .to_string()
18174                .contains("injected Object Lock metadata disk read failure")
18175        );
18176
18177        crate::bucket::metadata_sys::inject_object_lock_disk_read_error_in(&ctx, &bucket)
18178            .await
18179            .expect("inject batch-delete metadata read failure");
18180        let (_deleted, batch_errs) = store
18181            .delete_objects(
18182                &bucket,
18183                vec![crate::storage_api_contracts::object::ObjectToDelete {
18184                    object_name: "batch.bin".to_string(),
18185                    ..Default::default()
18186                }],
18187                ObjectOptions::default(),
18188            )
18189            .await;
18190        assert!(
18191            batch_errs[0]
18192                .as_ref()
18193                .is_some_and(|err| err.to_string().contains("injected Object Lock metadata disk read failure"))
18194        );
18195
18196        crate::bucket::metadata_sys::inject_object_lock_disk_read_error_in(&ctx, &bucket)
18197            .await
18198            .expect("inject prefix-delete metadata read failure");
18199        let prefix_err = store
18200            .delete_object(
18201                &bucket,
18202                "prefix",
18203                ObjectOptions {
18204                    delete_prefix: true,
18205                    ..Default::default()
18206                },
18207            )
18208            .await
18209            .expect_err("prefix delete must propagate the metadata read failure");
18210        assert!(
18211            prefix_err
18212                .to_string()
18213                .contains("injected Object Lock metadata disk read failure")
18214        );
18215
18216        crate::bucket::metadata_sys::inject_object_lock_disk_read_error_in(&ctx, &bucket)
18217            .await
18218            .expect("inject lifecycle-style delete-all metadata read failure");
18219        let lifecycle_err = store
18220            .delete_object(
18221                &bucket,
18222                "lifecycle.bin",
18223                ObjectOptions {
18224                    delete_prefix: true,
18225                    delete_prefix_object: true,
18226                    ..Default::default()
18227                },
18228            )
18229            .await
18230            .expect_err("lifecycle-style delete-all must propagate the metadata read failure");
18231        assert!(
18232            lifecycle_err
18233                .to_string()
18234                .contains("injected Object Lock metadata disk read failure")
18235        );
18236
18237        for object in ["single.bin", "batch.bin", "prefix/child.bin", "lifecycle.bin"] {
18238            store
18239                .get_object_info(&bucket, object, &ObjectOptions::default())
18240                .await
18241                .expect("metadata failure must preserve every object");
18242        }
18243    }
18244
18245    #[cfg(feature = "test-util")]
18246    #[tokio::test]
18247    #[serial_test::serial(storage_class_env)]
18248    async fn legacy_tier_delete_journals_create_redacted_recovery_controls_without_remote_calls() {
18249        let temp_dir = tempfile::tempdir().expect("create legacy journal recovery store dir");
18250        let (ctx, store, _shutdown) =
18251            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "legacy-tier-journal-recovery", &[4])).await;
18252        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
18253
18254        let tier_name = "LEGACY-RECOVERY";
18255        let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
18256        let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
18257            .await
18258            .expect("legacy recovery tier lease should resolve")
18259            .backend_identity();
18260        let fixtures = [
18261            serde_json::json!({
18262                "version": 1,
18263                "obj_name": "legacy/remote-v1",
18264                "version_id": "opaque-v1",
18265                "tier_name": tier_name,
18266            }),
18267            serde_json::json!({
18268                "version": 2,
18269                "obj_name": "legacy/remote-v2",
18270                "version_id": "opaque-v2",
18271                "tier_name": tier_name,
18272                "backend_identity": backend_identity,
18273            }),
18274        ];
18275        let mut journal_paths = Vec::new();
18276        for fixture in &fixtures {
18277            let data = serde_json::to_vec(&fixture).expect("legacy journal fixture should encode");
18278            let entry = crate::bucket::lifecycle::tier_delete_journal::decode_tier_delete_journal_entry(&data)
18279                .expect("legacy journal fixture should decode");
18280            let path = tier_delete_journal_object_name(&entry);
18281            com::save_config(store.clone(), &path, data)
18282                .await
18283                .expect("legacy journal fixture should persist");
18284            journal_paths.push(path);
18285        }
18286        let corrupt_path = format!(
18287            "{TIER_DELETE_JOURNAL_PREFIX}/{}.json",
18288            rustfs_utils::crypto::hex_sha256(b"corrupt legacy tier journal", ToOwned::to_owned)
18289        );
18290        com::save_config(store.clone(), &corrupt_path, b"{corrupt".to_vec())
18291            .await
18292            .expect("corrupt legacy journal fixture should persist");
18293
18294        let (first, concurrent) = tokio::join!(
18295            recover_tier_delete_journal_entries(store.clone(), 100, None),
18296            recover_tier_delete_journal_entries(store.clone(), 100, None),
18297        );
18298        for stats in [first, concurrent] {
18299            let stats = stats.expect("concurrent legacy journal recovery scan should finish");
18300            assert_eq!((stats.scanned, stats.deleted, stats.failed), (3, 0, 0));
18301        }
18302        assert_eq!(tier_delete_journal_count(store.clone()).await, 3);
18303        assert_eq!(backend.remove_count().await, 0, "legacy recovery must not call the remote tier");
18304        assert_eq!(backend.exact_remove_count(), 0, "legacy recovery must not issue exact remote DELETE");
18305        assert!(backend.op_log().await.is_empty(), "legacy recovery must not invoke any backend operation");
18306
18307        let mut first_controls = list_recovery_controls(store.clone(), IlmRecoveryProtocol::TierDeleteJournal, None, 100, None)
18308            .await
18309            .expect("legacy recovery controls should be listable")
18310            .records;
18311        first_controls.sort_by(|left, right| left.control_id.cmp(&right.control_id));
18312        assert_eq!(first_controls.len(), 3);
18313        assert_eq!(
18314            first_controls
18315                .iter()
18316                .filter(|control| control.classification == IlmRecoveryClassification::RetainedAmbiguous)
18317                .count(),
18318            2
18319        );
18320        assert_eq!(
18321            first_controls
18322                .iter()
18323                .filter(|control| control.classification == IlmRecoveryClassification::Corrupt)
18324                .count(),
18325            1
18326        );
18327        for view in &first_controls {
18328            assert_eq!(view.protocol, IlmRecoveryProtocol::TierDeleteJournal);
18329            assert_eq!(view.revision, 1);
18330            assert_eq!(view.attempt_count, 0);
18331            let encoded = serde_json::to_string(view).expect("recovery control view should encode");
18332            for secret in ["legacy/remote-v1", "legacy/remote-v2", "opaque-v1", "opaque-v2", tier_name] {
18333                assert!(!encoded.contains(secret), "recovery control view must redact `{secret}`");
18334            }
18335            let persisted = load_recovery_control(store.clone(), IlmRecoveryProtocol::TierDeleteJournal, &view.control_id)
18336                .await
18337                .expect("legacy recovery control should load");
18338            match view.source_schema.as_str() {
18339                "rustfs-tier-delete-journal-v1" => {
18340                    assert_eq!(persisted.control.identity.record_class, "tier_delete_journal_v1");
18341                    assert_eq!(view.last_error_code, IlmRecoveryErrorCode::RemoteVersionUnknown);
18342                }
18343                "rustfs-tier-delete-journal-v2" => {
18344                    assert_eq!(persisted.control.identity.record_class, "tier_delete_journal_v2");
18345                    assert_eq!(view.last_error_code, IlmRecoveryErrorCode::RemoteVersionUnknown);
18346                }
18347                "rustfs-tier-delete-journal-unknown" => {
18348                    assert_eq!(persisted.control.identity.record_class, "tier_delete_journal_corrupt");
18349                    assert_eq!(view.last_error_code, IlmRecoveryErrorCode::SourceCorrupt);
18350                }
18351                schema => panic!("unexpected legacy recovery source schema: {schema}"),
18352            }
18353        }
18354
18355        let creator_sha256 = rustfs_utils::crypto::hex_sha256(b"legacy-export-actor", ToOwned::to_owned);
18356        let mut created_exports = Vec::new();
18357        for exportable in first_controls
18358            .iter()
18359            .filter(|control| control.classification == IlmRecoveryClassification::RetainedAmbiguous)
18360        {
18361            let observation = inspect_recovery_export_observation(store.clone(), &exportable.control_id)
18362                .await
18363                .expect("fresh legacy recovery observation should be exportable");
18364            let created = create_recovery_export(store.clone(), &observation, &creator_sha256)
18365                .await
18366                .expect("legacy recovery export should be created exactly once");
18367            assert!(!created.replayed);
18368            let loaded = load_recovery_export(store.clone(), &created.export_id)
18369                .await
18370                .expect("created legacy recovery export should load");
18371            assert_eq!(loaded.encoded, created.encoded, "export readback must preserve the exact committed bytes");
18372            let replayed = create_recovery_export(store.clone(), &observation, &creator_sha256)
18373                .await
18374                .expect("the same observed generation should replay its immutable export");
18375            assert!(replayed.replayed);
18376            assert_eq!(replayed.encoded, created.encoded);
18377            created_exports.push(created);
18378        }
18379        assert_eq!(created_exports.len(), 2, "both v1 and v2 legacy journals must have an export path");
18380
18381        let corrupt_export_id = &created_exports[0].export_id;
18382        let export_path = recovery_export_record_object_name(IlmRecoveryProtocol::TierDeleteJournal, corrupt_export_id)
18383            .expect("export path should build");
18384        com::save_config(store.clone(), &export_path, Vec::new())
18385            .await
18386            .expect("zero-byte corruption fixture should persist");
18387        let corrupt_export = load_recovery_export(store.clone(), corrupt_export_id)
18388            .await
18389            .expect_err("an existing zero-byte export must fail closed");
18390        assert!(!matches!(corrupt_export, Error::ConfigNotFound));
18391
18392        com::save_config(
18393            store.clone(),
18394            &journal_paths[0],
18395            serde_json::to_vec_pretty(&fixtures[0]).expect("rewritten legacy journal fixture should encode"),
18396        )
18397        .await
18398        .expect("equivalent legacy journal rewrite should persist");
18399        let second = recover_tier_delete_journal_entries(store.clone(), 100, None)
18400            .await
18401            .expect("repeated legacy journal recovery scan should finish");
18402        assert_eq!((second.scanned, second.deleted, second.failed), (3, 0, 0));
18403        let mut second_controls = list_recovery_controls(store, IlmRecoveryProtocol::TierDeleteJournal, None, 100, None)
18404            .await
18405            .expect("repeated legacy recovery controls should remain listable")
18406            .records;
18407        second_controls.sort_by(|left, right| left.control_id.cmp(&right.control_id));
18408        assert_eq!(second_controls, first_controls, "repeated scans must not reset durable controls");
18409        assert_eq!(backend.remove_count().await, 0, "repeated recovery must remain remote-call free");
18410        assert_eq!(backend.exact_remove_count(), 0);
18411        assert!(
18412            backend.op_log().await.is_empty(),
18413            "repeated recovery must not invoke any backend operation"
18414        );
18415    }
18416
18417    #[cfg(feature = "test-util")]
18418    #[tokio::test]
18419    #[serial_test::serial(storage_class_env)]
18420    async fn legacy_recovery_disposition_removes_only_local_journals_and_replays() {
18421        Box::pin(legacy_recovery_disposition_removes_only_local_journals_and_replays_case()).await;
18422    }
18423
18424    #[cfg(feature = "test-util")]
18425    async fn legacy_recovery_disposition_removes_only_local_journals_and_replays_case() {
18426        let temp_dir = tempfile::tempdir().expect("create legacy disposition store dir");
18427        let (ctx, store, _shutdown) =
18428            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "legacy-recovery-disposition", &[4])).await;
18429        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
18430
18431        let tier_name = "LEGACY-DISPOSITION";
18432        let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
18433        let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
18434            .await
18435            .expect("legacy disposition tier lease should resolve")
18436            .backend_identity();
18437        let fixtures = [
18438            serde_json::json!({
18439                "version": 1,
18440                "obj_name": "legacy/disposition-v1",
18441                "version_id": "opaque-disposition-v1",
18442                "tier_name": tier_name,
18443            }),
18444            serde_json::json!({
18445                "version": 2,
18446                "obj_name": "legacy/disposition-v2",
18447                "version_id": "opaque-disposition-v2",
18448                "tier_name": tier_name,
18449                "backend_identity": backend_identity,
18450            }),
18451        ];
18452        let mut journal_paths = Vec::new();
18453        for fixture in &fixtures {
18454            let data = serde_json::to_vec(fixture).expect("legacy disposition fixture should encode");
18455            let entry = crate::bucket::lifecycle::tier_delete_journal::decode_tier_delete_journal_entry(&data)
18456                .expect("legacy disposition fixture should decode");
18457            let path = tier_delete_journal_object_name(&entry);
18458            com::save_config(store.clone(), &path, data)
18459                .await
18460                .expect("legacy disposition fixture should persist");
18461            journal_paths.push(path);
18462        }
18463
18464        let recovered = recover_tier_delete_journal_entries(store.clone(), 100, None)
18465            .await
18466            .expect("legacy disposition recovery scan should finish");
18467        assert_eq!((recovered.scanned, recovered.deleted, recovered.failed), (2, 0, 0));
18468        assert_eq!(tier_delete_journal_count(store.clone()).await, 2);
18469
18470        let mut controls = list_recovery_controls(
18471            store.clone(),
18472            IlmRecoveryProtocol::TierDeleteJournal,
18473            Some(IlmRecoveryClassification::RetainedAmbiguous),
18474            100,
18475            None,
18476        )
18477        .await
18478        .expect("legacy disposition controls should be listable")
18479        .records;
18480        controls.sort_by(|left, right| left.control_id.cmp(&right.control_id));
18481        assert_eq!(controls.len(), 2, "both legacy schemas must support disposition");
18482
18483        let actor_sha256 = rustfs_utils::crypto::hex_sha256(b"legacy-disposition-actor", ToOwned::to_owned);
18484        let wrong_actor_sha256 = rustfs_utils::crypto::hex_sha256(b"different-disposition-actor", ToOwned::to_owned);
18485        let wrong_export_sha256 = "ff".repeat(32);
18486        for (index, control) in controls.iter().enumerate() {
18487            let observation = inspect_recovery_export_observation(store.clone(), &control.control_id)
18488                .await
18489                .expect("legacy disposition source should be observable");
18490            let export = create_recovery_export(store.clone(), &observation, &actor_sha256)
18491                .await
18492                .expect("legacy disposition export should persist");
18493            let confirmed_at_unix_nanos = i64::try_from(OffsetDateTime::now_utc().unix_timestamp_nanos())
18494                .expect("legacy disposition timestamp should fit i64");
18495
18496            if index == 0 {
18497                let wrong_hash = Box::pin(execute_recovery_disposition(
18498                    store.clone(),
18499                    &observation,
18500                    &export.export_id,
18501                    &wrong_export_sha256,
18502                    &actor_sha256,
18503                    confirmed_at_unix_nanos,
18504                ))
18505                .await
18506                .expect_err("a mismatched export checksum must fail before local deletion");
18507                assert_eq!(wrong_hash, Error::PreconditionFailed);
18508                assert_eq!(tier_delete_journal_count(store.clone()).await, 2);
18509            }
18510
18511            let dry_run = dry_run_recovery_disposition(
18512                store.clone(),
18513                &observation,
18514                &export.export_id,
18515                &export.content_sha256,
18516                &actor_sha256,
18517                confirmed_at_unix_nanos,
18518            )
18519            .await
18520            .expect("legacy disposition dry-run should validate exact local state");
18521            assert_eq!(dry_run.source_copy_count, observation.source_generation.copies.len());
18522            assert_eq!(
18523                tier_delete_journal_count(store.clone()).await,
18524                fixtures.len() - index,
18525                "dry-run must not delete a legacy journal"
18526            );
18527            assert!(
18528                matches!(
18529                    load_recovery_disposition(store.clone(), IlmRecoveryProtocol::TierDeleteJournal, &dry_run.disposition_id,)
18530                        .await,
18531                    Err(Error::ConfigNotFound)
18532                ),
18533                "dry-run must not persist a disposition record"
18534            );
18535
18536            if index == 0 {
18537                inject_recovery_disposition_crash_once(RecoveryDispositionCrashStage::AfterLocalDelete);
18538                Box::pin(execute_recovery_disposition(
18539                    store.clone(),
18540                    &observation,
18541                    &export.export_id,
18542                    &export.content_sha256,
18543                    &actor_sha256,
18544                    confirmed_at_unix_nanos,
18545                ))
18546                .await
18547                .expect_err("the injected crash must stop after local delete commits");
18548                assert_eq!(tier_delete_journal_count(store.clone()).await, 1);
18549                let interrupted =
18550                    load_recovery_disposition(store.clone(), IlmRecoveryProtocol::TierDeleteJournal, &dry_run.disposition_id)
18551                        .await
18552                        .expect("the applying disposition must survive the post-delete crash");
18553                assert_eq!(interrupted.disposition.state, IlmRecoveryDispositionState::Applying);
18554                assert!(
18555                    interrupted.disposition.confirmed_absent.is_empty(),
18556                    "the crash must occur before absence progress is persisted"
18557                );
18558            } else {
18559                inject_recovery_disposition_crash_once(RecoveryDispositionCrashStage::AfterControlAbandon);
18560                Box::pin(execute_recovery_disposition(
18561                    store.clone(),
18562                    &observation,
18563                    &export.export_id,
18564                    &export.content_sha256,
18565                    &actor_sha256,
18566                    confirmed_at_unix_nanos,
18567                ))
18568                .await
18569                .expect_err("the injected crash must stop after control abandonment commits");
18570                let interrupted =
18571                    load_recovery_disposition(store.clone(), IlmRecoveryProtocol::TierDeleteJournal, &dry_run.disposition_id)
18572                        .await
18573                        .expect("the applying disposition must survive the post-control crash");
18574                assert_eq!(interrupted.disposition.state, IlmRecoveryDispositionState::Applying);
18575                assert_eq!(
18576                    interrupted.disposition.confirmed_absent.len(),
18577                    interrupted.disposition.identity.source_generation.copies.len()
18578                );
18579
18580                let abandoned =
18581                    load_recovery_control(store.clone(), IlmRecoveryProtocol::TierDeleteJournal, &observation.control_id)
18582                        .await
18583                        .expect("the abandoned control must survive the injected crash");
18584                let exact_abandoned = abandoned.control.encode().expect("the exact abandoned control should encode");
18585                let mut wrong_history = abandoned.control;
18586                wrong_history.last_error_code = IlmRecoveryErrorCode::CleanupFailed;
18587                let control_path = recovery_control_record_object_name(observation.protocol, &observation.control_id)
18588                    .expect("control path should remain canonical");
18589                com::save_config(
18590                    store.clone(),
18591                    &control_path,
18592                    wrong_history
18593                        .encode()
18594                        .expect("the alternate valid control history should encode"),
18595                )
18596                .await
18597                .expect("the alternate control history fixture should persist");
18598                let wrong_history_err = Box::pin(execute_recovery_disposition(
18599                    store.clone(),
18600                    &observation,
18601                    &export.export_id,
18602                    &export.content_sha256,
18603                    &actor_sha256,
18604                    confirmed_at_unix_nanos + 1,
18605                ))
18606                .await
18607                .expect_err("a different abandoned control history must not bridge to completion");
18608                assert_eq!(wrong_history_err, Error::PreconditionFailed);
18609                com::save_config(store.clone(), &control_path, exact_abandoned)
18610                    .await
18611                    .expect("the exact abandoned control fixture should be restored");
18612            }
18613
18614            let replay_confirmed_at_unix_nanos = confirmed_at_unix_nanos + 2;
18615            let executed = Box::pin(execute_recovery_disposition(
18616                store.clone(),
18617                &observation,
18618                &export.export_id,
18619                &export.content_sha256,
18620                &actor_sha256,
18621                replay_confirmed_at_unix_nanos,
18622            ))
18623            .await
18624            .expect("a later request must resume and complete the interrupted disposition");
18625            assert_eq!(executed.state, IlmRecoveryDispositionState::Completed);
18626            assert_eq!(executed.outcome, IlmRecoveryDispositionExecutionOutcome::Completed);
18627            assert_eq!(executed.confirmed_absent_copy_count, executed.source_copy_count);
18628            assert_eq!(tier_delete_journal_count(store.clone()).await, fixtures.len() - index - 1);
18629            assert!(matches!(
18630                com::read_config(store.clone(), &observation.canonical_source_path).await,
18631                Err(Error::ConfigNotFound)
18632            ));
18633            if index == 0 {
18634                let untouched = journal_paths
18635                    .iter()
18636                    .find(|path| *path != &observation.canonical_source_path)
18637                    .expect("the other legacy journal should remain");
18638                com::read_config(store.clone(), untouched)
18639                    .await
18640                    .expect("disposition must not remove a different legacy journal");
18641            }
18642
18643            let abandoned = load_recovery_control(store.clone(), IlmRecoveryProtocol::TierDeleteJournal, &observation.control_id)
18644                .await
18645                .expect("abandoned recovery control should remain inspectable");
18646            assert_eq!(abandoned.control.classification, IlmRecoveryClassification::Abandoned);
18647            assert_eq!(abandoned.control.revision, observation.control_revision + 1);
18648            assert_eq!(abandoned.control.observed_source_generation, observation.source_generation);
18649
18650            let persisted =
18651                load_recovery_disposition(store.clone(), IlmRecoveryProtocol::TierDeleteJournal, &executed.disposition_id)
18652                    .await
18653                    .expect("completed disposition should remain durable");
18654            assert_eq!(persisted.disposition.state, IlmRecoveryDispositionState::Completed);
18655
18656            let replayed = Box::pin(execute_recovery_disposition(
18657                store.clone(),
18658                &observation,
18659                &export.export_id,
18660                &export.content_sha256,
18661                &actor_sha256,
18662                replay_confirmed_at_unix_nanos + 1,
18663            ))
18664            .await
18665            .expect("same actor should replay the completed disposition");
18666            assert_eq!(replayed.state, IlmRecoveryDispositionState::Completed);
18667            assert_eq!(replayed.outcome, IlmRecoveryDispositionExecutionOutcome::Replayed);
18668
18669            let wrong_actor = Box::pin(execute_recovery_disposition(
18670                store.clone(),
18671                &observation,
18672                &export.export_id,
18673                &export.content_sha256,
18674                &wrong_actor_sha256,
18675                replay_confirmed_at_unix_nanos + 2,
18676            ))
18677            .await
18678            .expect_err("a different actor must not replay a completed disposition");
18679            assert_eq!(wrong_actor, Error::PreconditionFailed);
18680
18681            assert!(
18682                !Box::pin(garbage_collect_completed_recovery_disposition(
18683                    store.clone(),
18684                    &persisted,
18685                    persisted.disposition.retain_until_unix_nanos - 1,
18686                ))
18687                .await
18688                .expect("completed disposition should remain before retention expires")
18689            );
18690            assert!(
18691                Box::pin(garbage_collect_completed_recovery_disposition(
18692                    store.clone(),
18693                    &persisted,
18694                    persisted.disposition.retain_until_unix_nanos,
18695                ))
18696                .await
18697                .expect("expired completed disposition should be garbage collected")
18698            );
18699            assert!(matches!(
18700                load_recovery_disposition(store.clone(), IlmRecoveryProtocol::TierDeleteJournal, &executed.disposition_id).await,
18701                Err(Error::ConfigNotFound)
18702            ));
18703            assert_eq!(backend.remove_count().await, 0, "legacy disposition must not call the remote tier");
18704            assert_eq!(backend.exact_remove_count(), 0, "legacy disposition must not issue exact remote DELETE");
18705            assert!(
18706                backend.op_log().await.is_empty(),
18707                "legacy disposition must not invoke any backend operation"
18708            );
18709        }
18710    }
18711
18712    #[cfg(feature = "test-util")]
18713    #[tokio::test]
18714    #[serial_test::serial(storage_class_env)]
18715    async fn v4_tier_delete_journal_recovery_is_exact_and_store_scoped() {
18716        let temp_a = tempfile::tempdir().expect("create temp store dir a");
18717        let temp_b = tempfile::tempdir().expect("create temp store dir b");
18718        let (ctx_a, store_a, _shutdown_a) =
18719            without_storage_class_env(build_isolated_test_store(temp_a.path(), "tier-journal-recovery-a", &[4])).await;
18720        let (ctx_b, store_b, _shutdown_b) =
18721            without_storage_class_env(build_isolated_test_store(temp_b.path(), "tier-journal-recovery-b", &[4])).await;
18722        crate::bucket::metadata_sys::init_bucket_metadata_sys(store_a.clone(), Vec::new()).await;
18723        crate::bucket::metadata_sys::init_bucket_metadata_sys(store_b.clone(), Vec::new()).await;
18724
18725        let tier_a = "JOURNAL-A";
18726        let tier_b = "JOURNAL-B";
18727        let backend_a = register_mock_tier(&ctx_a.tier_config_mgr(), tier_a).await;
18728        let backend_b = register_mock_tier(&ctx_b.tier_config_mgr(), tier_b).await;
18729        let identity_a = TierConfigMgr::acquire_operation_lease(&ctx_a.tier_config_mgr(), tier_a)
18730            .await
18731            .expect("store A tier lease should resolve")
18732            .backend_identity();
18733        let identity_b = TierConfigMgr::acquire_operation_lease(&ctx_b.tier_config_mgr(), tier_b)
18734            .await
18735            .expect("store B tier lease should resolve")
18736            .backend_identity();
18737        let entry_a = Jentry {
18738            persisted_version: 0,
18739            obj_name: "remote-a".to_string(),
18740            version_id: "version-a".to_string(),
18741            tier_name: tier_a.to_string(),
18742            backend_identity: Some(identity_a),
18743            version_id_exact: true,
18744            version_state: rustfs_filemeta::TransitionVersionState::Exact,
18745            state: crate::bucket::lifecycle::tier_sweeper::TierDeleteJournalState::Committed,
18746            source: None,
18747            dispatch: None,
18748        };
18749        let entry_b = Jentry {
18750            persisted_version: 0,
18751            obj_name: "remote-b".to_string(),
18752            version_id: "version-b".to_string(),
18753            tier_name: tier_b.to_string(),
18754            backend_identity: Some(identity_b),
18755            version_id_exact: true,
18756            version_state: rustfs_filemeta::TransitionVersionState::Exact,
18757            state: crate::bucket::lifecycle::tier_sweeper::TierDeleteJournalState::Committed,
18758            source: None,
18759            dispatch: None,
18760        };
18761        persist_tier_delete_journal_entry(store_a.clone(), &entry_a)
18762            .await
18763            .expect("store A v4 compatibility fixture should persist");
18764        persist_tier_delete_journal_entry(store_b.clone(), &entry_b)
18765            .await
18766            .expect("store B v4 compatibility fixture should persist");
18767        let entry_a_v3 = Jentry {
18768            obj_name: "remote-a-v3".to_string(),
18769            version_id: "version-a-v3".to_string(),
18770            ..entry_a.clone()
18771        };
18772        let mut raw_v3: serde_json::Value = serde_json::from_slice(
18773            &encode_tier_delete_journal_entry(&entry_a_v3).expect("store A v3 fixture should encode from the v4 shape"),
18774        )
18775        .expect("store A v3 fixture should be JSON");
18776        raw_v3["version"] = serde_json::json!(3);
18777        raw_v3
18778            .as_object_mut()
18779            .expect("store A v3 fixture should be an object")
18780            .remove("version_state");
18781        let path_a_v3 = tier_delete_journal_object_name(&entry_a_v3);
18782        com::save_config(
18783            store_a.clone(),
18784            &path_a_v3,
18785            serde_json::to_vec(&raw_v3).expect("store A raw v3 fixture should encode"),
18786        )
18787        .await
18788        .expect("store A raw v3 compatibility fixture should persist");
18789        let path_a = tier_delete_journal_object_name(&entry_a);
18790        let path_b = tier_delete_journal_object_name(&entry_b);
18791        let recovered_a = recover_tier_delete_journal_entries(store_a.clone(), 100, None)
18792            .await
18793            .expect("store A recovery scan should finish");
18794        let recovered_b = recover_tier_delete_journal_entries(store_b.clone(), 100, None)
18795            .await
18796            .expect("store B recovery scan should finish");
18797        assert_eq!((recovered_a.scanned, recovered_a.deleted, recovered_a.failed), (2, 2, 0));
18798        assert_eq!((recovered_b.scanned, recovered_b.deleted, recovered_b.failed), (1, 1, 0));
18799        let mut removed_a = backend_a.remove_versions().await;
18800        removed_a.sort();
18801        assert_eq!(
18802            removed_a,
18803            vec![
18804                ("remote-a".to_string(), "version-a".to_string()),
18805                ("remote-a-v3".to_string(), "version-a-v3".to_string()),
18806            ]
18807        );
18808        assert_eq!(backend_b.remove_versions().await, vec![("remote-b".to_string(), "version-b".to_string())]);
18809        assert!(matches!(com::read_config(store_a.clone(), &path_a).await, Err(Error::ConfigNotFound)));
18810        assert!(matches!(com::read_config(store_a.clone(), &path_a_v3).await, Err(Error::ConfigNotFound)));
18811        assert!(matches!(com::read_config(store_b.clone(), &path_b).await, Err(Error::ConfigNotFound)));
18812        assert_eq!(tier_delete_journal_count(store_a).await, 0);
18813        assert_eq!(tier_delete_journal_count(store_b).await, 0);
18814    }
18815
18816    #[cfg(feature = "test-util")]
18817    #[tokio::test]
18818    #[serial_test::serial(storage_class_env)]
18819    async fn v5_tier_delete_recovery_retains_live_sources_and_aborts_only_prepared() {
18820        let temp_dir = tempfile::tempdir().expect("create v5 live-source store dir");
18821        let (ctx, store, _shutdown) =
18822            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "v5-live-source-recovery", &[4])).await;
18823        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
18824
18825        let tier_name = "V5-LIVE-SOURCE";
18826        let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
18827        let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
18828            .await
18829            .expect("v5 live-source tier lease should resolve")
18830            .backend_identity();
18831        let bucket = "v5-live-source-recovery-bucket";
18832        store
18833            .make_bucket(bucket, &MakeBucketOptions::default())
18834            .await
18835            .expect("v5 live-source bucket should be created");
18836
18837        let mut paths = Vec::new();
18838        for (index, state) in [
18839            TierDeleteJournalState::Prepared,
18840            TierDeleteJournalState::Dispatched,
18841            TierDeleteJournalState::Committed,
18842        ]
18843        .into_iter()
18844        .enumerate()
18845        {
18846            let object = format!("live-{index}.bin");
18847            let remote_version = uuid::Uuid::new_v4().to_string();
18848            backend.set_put_remote_version(Some(remote_version)).await;
18849            let mut reader = PutObjReader::from_vec(format!("v5 live source {index}").into_bytes());
18850            let source = store
18851                .put_object(bucket, &object, &mut reader, &ObjectOptions::default())
18852                .await
18853                .expect("v5 live source should be written");
18854            store
18855                .transition_object(
18856                    bucket,
18857                    &object,
18858                    &ObjectOptions {
18859                        transition: TransitionOptions {
18860                            status: TRANSITION_PENDING.to_string(),
18861                            tier: tier_name.to_string(),
18862                            etag: source.etag.clone().expect("v5 live source should have an ETag"),
18863                            ..Default::default()
18864                        },
18865                        version_id: source.version_id.map(|version| version.to_string()),
18866                        mod_time: source.mod_time,
18867                        ..Default::default()
18868                    },
18869                )
18870                .await
18871                .expect("v5 live source should transition");
18872            let transitioned = store
18873                .get_object_info(
18874                    bucket,
18875                    &object,
18876                    &ObjectOptions {
18877                        no_lock: true,
18878                        metadata_cache_safe: false,
18879                        ..Default::default()
18880                    },
18881                )
18882                .await
18883                .expect("v5 transitioned source should be readable");
18884            let mut entry = transitioned_delete_journal_entry_for_source(None, false, false, bucket, &object, &transitioned)
18885                .expect("v5 transitioned source should produce a stable journal identity");
18886            entry.persisted_version = 5;
18887            entry.backend_identity = Some(backend_identity);
18888            entry.state = state;
18889            entry.dispatch = None;
18890            let path = tier_delete_journal_object_name(&entry);
18891            com::save_config(
18892                store.clone(),
18893                &path,
18894                encode_tier_delete_journal_entry(&entry).expect("raw v5 live-source journal should encode"),
18895            )
18896            .await
18897            .expect("raw v5 live-source journal should persist");
18898            paths.push((state, path));
18899        }
18900
18901        let stats = recover_tier_delete_journal_entries(store.clone(), 100, None)
18902            .await
18903            .expect("v5 live-source recovery should finish");
18904        assert_eq!((stats.scanned, stats.deleted, stats.failed), (3, 1, 2));
18905        assert_eq!(backend.remove_count().await, 0, "a live source must block every remote DELETE");
18906        for (state, path) in paths {
18907            let observed = com::read_config(store.clone(), &path).await;
18908            if state == TierDeleteJournalState::Prepared {
18909                assert!(matches!(observed, Err(Error::ConfigNotFound)), "Prepared v5 evidence should abort");
18910            } else {
18911                assert!(observed.is_ok(), "{state:?} v5 evidence must remain while its exact source exists");
18912            }
18913        }
18914    }
18915
18916    #[cfg(feature = "test-util")]
18917    #[tokio::test]
18918    #[serial_test::serial(storage_class_env)]
18919    async fn v5_tier_delete_recovery_replays_all_crash_states_without_sources() {
18920        let temp_dir = tempfile::tempdir().expect("create v5 crash-replay store dir");
18921        let (ctx, store, _shutdown) =
18922            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "v5-crash-replay", &[4])).await;
18923        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
18924
18925        let tier_name = "V5-CRASH-REPLAY";
18926        let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
18927        let lease = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
18928            .await
18929            .expect("v5 crash-replay tier lease should resolve");
18930        let backend_identity = lease.backend_identity();
18931        let bucket = "v5-crash-replay-bucket";
18932        store
18933            .make_bucket(bucket, &MakeBucketOptions::default())
18934            .await
18935            .expect("v5 crash-replay bucket should be created");
18936
18937        for (index, state, seed_remote) in [
18938            (0, TierDeleteJournalState::Prepared, true),
18939            (1, TierDeleteJournalState::Dispatched, true),
18940            (2, TierDeleteJournalState::Committed, true),
18941            // Models a restart after exact remote DELETE succeeded but before
18942            // the local Committed journal was removed.
18943            (3, TierDeleteJournalState::Committed, false),
18944        ] {
18945            let remote_version = uuid::Uuid::new_v4().to_string();
18946            let mut entry = synthetic_v6_dispatch_entry(
18947                bucket,
18948                &format!("missing-{index}.bin"),
18949                tier_name,
18950                backend_identity,
18951                &remote_version,
18952            );
18953            entry.persisted_version = 5;
18954            entry.state = state;
18955            entry.dispatch = None;
18956            if seed_remote {
18957                backend.set_put_remote_version(Some(remote_version)).await;
18958                let body = bytes::Bytes::from(format!("v5 remote candidate {index}"));
18959                lease
18960                    .put(
18961                        &entry.obj_name,
18962                        ReaderImpl::Body(body.clone()),
18963                        i64::try_from(body.len()).expect("v5 candidate length should fit i64"),
18964                    )
18965                    .await
18966                    .expect("v5 remote candidate should be seeded");
18967            }
18968            let path = tier_delete_journal_object_name(&entry);
18969            com::save_config(
18970                store.clone(),
18971                &path,
18972                encode_tier_delete_journal_entry(&entry).expect("raw v5 crash journal should encode"),
18973            )
18974            .await
18975            .expect("raw v5 crash journal should persist");
18976        }
18977
18978        let stats = recover_tier_delete_journal_entries(store.clone(), 100, None)
18979            .await
18980            .expect("v5 crash-state recovery should finish");
18981        assert_eq!((stats.scanned, stats.deleted, stats.failed), (4, 4, 0));
18982        assert_eq!(tier_delete_journal_count(store).await, 0);
18983        assert_eq!(backend.object_count().await, 0);
18984        assert_eq!(backend.exact_remove_count(), 4, "every v5 state must use exact-version cleanup");
18985    }
18986
18987    #[cfg(feature = "test-util")]
18988    #[tokio::test]
18989    #[serial_test::serial(storage_class_env)]
18990    async fn dispatched_tier_delete_recovery_finds_directory_source_on_encoded_set() {
18991        let temp_dir = tempfile::tempdir().expect("create temp store dir");
18992        let shutdown = CancellationToken::new();
18993        let (ctx, store, _shutdown) = without_storage_class_env(build_isolated_test_store_with_layout(
18994            temp_dir.path(),
18995            "prepared-directory-recovery",
18996            &[(2, 4)],
18997            shutdown,
18998            None,
18999        ))
19000        .await;
19001        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
19002
19003        let pool = &store.pools[0];
19004        let object = (0..10_000)
19005            .map(|index| format!("directory-{index}/"))
19006            .find(|candidate| {
19007                let encoded = rustfs_utils::path::encode_dir_object(candidate);
19008                !Arc::ptr_eq(&pool.get_disks_by_key(candidate), &pool.get_disks_by_key(&encoded))
19009            })
19010            .expect("test topology should have a directory key whose encoded form hashes to another set");
19011        let encoded = rustfs_utils::path::encode_dir_object(&object);
19012        assert!(!Arc::ptr_eq(&pool.get_disks_by_key(&object), &pool.get_disks_by_key(&encoded)));
19013
19014        let tier_name = "PREPAREDDIRECTORY";
19015        let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
19016        let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
19017            .await
19018            .expect("tier lease should resolve")
19019            .backend_identity();
19020        let bucket = "prepared-directory-recovery-bucket";
19021        store
19022            .make_bucket(bucket, &MakeBucketOptions::default())
19023            .await
19024            .expect("bucket should be created");
19025        let mut reader = PutObjReader::from_vec(b"directory source".to_vec());
19026        let original = store
19027            .put_object(bucket, &object, &mut reader, &ObjectOptions::default())
19028            .await
19029            .expect("directory source should be written");
19030        store
19031            .transition_object(
19032                bucket,
19033                &object,
19034                &ObjectOptions {
19035                    transition: TransitionOptions {
19036                        status: TRANSITION_PENDING.to_string(),
19037                        tier: tier_name.to_string(),
19038                        etag: original.etag.clone().expect("source should have an ETag"),
19039                        ..Default::default()
19040                    },
19041                    mod_time: original.mod_time,
19042                    ..Default::default()
19043                },
19044            )
19045            .await
19046            .expect("directory transition should commit");
19047        let committed = store
19048            .get_object_info(
19049                bucket,
19050                &object,
19051                &ObjectOptions {
19052                    no_lock: true,
19053                    metadata_cache_safe: false,
19054                    ..Default::default()
19055                },
19056            )
19057            .await
19058            .expect("transitioned directory source should be readable");
19059        let mut entry = transitioned_delete_journal_entry_for_source(None, false, false, bucket, &object, &committed)
19060            .expect("transitioned source should produce a prepared journal entry");
19061        entry.backend_identity = Some(backend_identity);
19062        let incarnation = store
19063            .bucket_incarnation_id(bucket)
19064            .await
19065            .expect("bucket incarnation should resolve");
19066        install_test_tier_delete_dispatch_fixture(
19067            store.clone(),
19068            bucket,
19069            incarnation,
19070            &object,
19071            vec![(entry, Some(TierDeleteJournalState::Dispatched))],
19072            TierDeleteDispatchManifestState::DispatchAuthorized,
19073        )
19074        .await
19075        .expect("authorized dispatched journal should persist");
19076
19077        let stats = recover_tier_delete_journal_entries(store.clone(), 100, None)
19078            .await
19079            .expect("prepared recovery should complete");
19080
19081        assert_eq!((stats.scanned, stats.deleted, stats.failed), (1, 0, 1));
19082        assert_eq!(tier_delete_journal_count(store).await, 1);
19083        assert_eq!(backend.remove_count().await, 0);
19084        assert_eq!(backend.object_count().await, 1, "live directory source must retain its remote object");
19085    }
19086
19087    #[cfg(feature = "test-util")]
19088    #[tokio::test]
19089    #[serial_test::serial(storage_class_env)]
19090    async fn dispatched_tier_delete_recovery_checks_later_pool_then_commits_after_source_removal() {
19091        let temp_dir = tempfile::tempdir().expect("create cross-pool recovery store dir");
19092        let (ctx, store, _shutdown) =
19093            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "prepared-cross-pool-recovery", &[4, 4])).await;
19094        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
19095        let tier_name = "PREPAREDCROSSPOOL";
19096        let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
19097        let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
19098            .await
19099            .expect("tier lease should resolve")
19100            .backend_identity();
19101        let bucket = "prepared-cross-pool-recovery-bucket";
19102        let object = "object";
19103        store
19104            .make_bucket(bucket, &MakeBucketOptions::default())
19105            .await
19106            .expect("bucket should be created");
19107        let mut reader = PutObjReader::from_vec(b"later pool source".to_vec());
19108        let original = store.pools[1]
19109            .put_object(bucket, object, &mut reader, &ObjectOptions::default())
19110            .await
19111            .expect("source should be written only to the later pool");
19112        store.pools[1]
19113            .transition_object(
19114                bucket,
19115                object,
19116                &ObjectOptions {
19117                    transition: TransitionOptions {
19118                        status: TRANSITION_PENDING.to_string(),
19119                        tier: tier_name.to_string(),
19120                        etag: original.etag.clone().expect("source should have an ETag"),
19121                        ..Default::default()
19122                    },
19123                    mod_time: original.mod_time,
19124                    ..Default::default()
19125                },
19126            )
19127            .await
19128            .expect("later-pool transition should commit");
19129        let committed = store.pools[1]
19130            .get_object_info(bucket, object, &ObjectOptions::default())
19131            .await
19132            .expect("transitioned source should be readable");
19133        let mut entry = transitioned_delete_journal_entry_for_source(None, false, false, bucket, object, &committed)
19134            .expect("transitioned source should produce a prepared journal");
19135        entry.backend_identity = Some(backend_identity);
19136        let incarnation = store
19137            .bucket_incarnation_id(bucket)
19138            .await
19139            .expect("bucket incarnation should resolve");
19140        install_test_tier_delete_dispatch_fixture(
19141            store.clone(),
19142            bucket,
19143            incarnation,
19144            object,
19145            vec![(entry, Some(TierDeleteJournalState::Dispatched))],
19146            TierDeleteDispatchManifestState::DispatchAuthorized,
19147        )
19148        .await
19149        .expect("authorized dispatched journal should persist");
19150
19151        let retained = recover_tier_delete_journal_entries(store.clone(), 100, None)
19152            .await
19153            .expect("recovery should scan the later pool");
19154        assert_eq!((retained.scanned, retained.deleted, retained.failed), (1, 0, 1));
19155        assert_eq!(backend.remove_count().await, 0);
19156        assert_eq!(backend.object_count().await, 1);
19157        assert_eq!(
19158            tier_delete_journal_count(store.clone()).await,
19159            1,
19160            "an authorized journal must remain while its live source is durable"
19161        );
19162
19163        store.pools[1]
19164            .delete_object(
19165                bucket,
19166                object,
19167                ObjectOptions {
19168                    version_id: committed.version_id.map(|version| version.to_string()),
19169                    expiration: crate::storage_api_contracts::lifecycle::ExpirationOptions { expire: true },
19170                    skip_free_version: true,
19171                    ..Default::default()
19172                },
19173            )
19174            .await
19175            .expect("source version should be removed before recovery retry");
19176        let committed_stats = recover_tier_delete_journal_entries(store.clone(), 100, None)
19177            .await
19178            .expect("journal recovery should commit an absent stable source");
19179        assert_eq!((committed_stats.scanned, committed_stats.deleted, committed_stats.failed), (1, 0, 1));
19180        assert_eq!(backend.remove_count().await, 0);
19181
19182        let manifest_stats = recover_tier_delete_dispatch_manifests(store.clone(), 100, None)
19183            .await
19184            .expect("manifest recovery should authorize committed remote cleanup");
19185        assert_eq!((manifest_stats.advanced, manifest_stats.failed), (1, 0));
19186
19187        let deleted = recover_tier_delete_journal_entries(store.clone(), 100, None)
19188            .await
19189            .expect("completed journal should delete the exact remote version");
19190        assert_eq!((deleted.scanned, deleted.deleted, deleted.failed), (1, 1, 0));
19191        assert_eq!(tier_delete_journal_count(store).await, 0);
19192        assert_eq!(backend.remove_count().await, 1);
19193        assert_eq!(backend.object_count().await, 0);
19194    }
19195
19196    #[cfg(feature = "test-util")]
19197    #[tokio::test]
19198    #[serial_test::serial(storage_class_env)]
19199    async fn dispatched_tier_delete_recovery_retains_journal_on_source_metadata_error() {
19200        let temp_dir = tempfile::tempdir().expect("create metadata-error recovery store dir");
19201        let (ctx, store, _shutdown) =
19202            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "prepared-metadata-error-recovery", &[4])).await;
19203        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
19204        let tier_name = "PREPAREDMETADATAERROR";
19205        let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
19206        let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
19207            .await
19208            .expect("tier lease should resolve")
19209            .backend_identity();
19210        let bucket = "prepared-metadata-error-recovery-bucket";
19211        let object = "object";
19212        store
19213            .make_bucket(bucket, &MakeBucketOptions::default())
19214            .await
19215            .expect("bucket should be created");
19216        let mut reader = PutObjReader::from_vec(b"source with unreadable metadata".to_vec());
19217        let original = store
19218            .put_object(bucket, object, &mut reader, &ObjectOptions::default())
19219            .await
19220            .expect("source should be written");
19221        store
19222            .transition_object(
19223                bucket,
19224                object,
19225                &ObjectOptions {
19226                    transition: TransitionOptions {
19227                        status: TRANSITION_PENDING.to_string(),
19228                        tier: tier_name.to_string(),
19229                        etag: original.etag.clone().expect("source should have an ETag"),
19230                        ..Default::default()
19231                    },
19232                    mod_time: original.mod_time,
19233                    ..Default::default()
19234                },
19235            )
19236            .await
19237            .expect("source transition should commit");
19238        let committed = store
19239            .get_object_info(bucket, object, &ObjectOptions::default())
19240            .await
19241            .expect("transitioned source should be readable");
19242        let mut entry = transitioned_delete_journal_entry_for_source(None, false, false, bucket, object, &committed)
19243            .expect("transitioned source should produce a prepared journal");
19244        entry.backend_identity = Some(backend_identity);
19245        let incarnation = store
19246            .bucket_incarnation_id(bucket)
19247            .await
19248            .expect("bucket incarnation should resolve");
19249        install_test_tier_delete_dispatch_fixture(
19250            store.clone(),
19251            bucket,
19252            incarnation,
19253            object,
19254            vec![(entry, Some(TierDeleteJournalState::Dispatched))],
19255            TierDeleteDispatchManifestState::DispatchAuthorized,
19256        )
19257        .await
19258        .expect("authorized dispatched journal should persist");
19259
19260        for disk_index in 0..4 {
19261            let metadata_path = temp_dir
19262                .path()
19263                .join(format!("pool0/set0/disk{disk_index}/{bucket}/{object}/{STORAGE_FORMAT_FILE}"));
19264            tokio::fs::write(metadata_path, b"not-xl-meta")
19265                .await
19266                .expect("source metadata should be corrupted");
19267        }
19268
19269        let stats = recover_tier_delete_journal_entries(store.clone(), 100, None)
19270            .await
19271            .expect("recovery scan should complete despite the entry failure");
19272
19273        assert_eq!((stats.scanned, stats.deleted, stats.failed), (1, 0, 1));
19274        assert_eq!(tier_delete_journal_count(store).await, 1, "journal must remain dispatched for retry");
19275        assert_eq!(backend.remove_count().await, 0, "unreadable source metadata must block remote deletion");
19276        assert_eq!(backend.object_count().await, 1, "remote source must remain intact");
19277    }
19278
19279    #[cfg(feature = "test-util")]
19280    #[tokio::test]
19281    #[serial_test::serial(storage_class_env)]
19282    async fn tier_delete_recovery_retains_content_that_does_not_match_its_object_name() {
19283        let temp_dir = tempfile::tempdir().expect("create mismatched journal store dir");
19284        let (ctx, store, _shutdown) =
19285            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "mismatched-tier-journal", &[4])).await;
19286        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
19287        let tier_name = "MISMATCHEDJOURNAL";
19288        let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
19289        let lease = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
19290            .await
19291            .expect("tier lease should resolve");
19292        let remote_version = uuid::Uuid::new_v4().to_string();
19293        backend.set_put_remote_version(Some(remote_version.clone())).await;
19294        lease
19295            .put("remote/original", ReaderImpl::Body(bytes::Bytes::from_static(b"remote body")), 11)
19296            .await
19297            .expect("remote body should be seeded");
19298        let entry = Jentry {
19299            persisted_version: 0,
19300            obj_name: "remote/original".to_string(),
19301            version_id: remote_version,
19302            tier_name: tier_name.to_string(),
19303            backend_identity: Some(lease.backend_identity()),
19304            version_id_exact: true,
19305            version_state: rustfs_filemeta::TransitionVersionState::Exact,
19306            state: TierDeleteJournalState::Prepared,
19307            source: Some(TierDeleteSourceIdentity {
19308                bucket: "absent-source-bucket".to_string(),
19309                object: "absent-source-object".to_string(),
19310                version_id: Some(uuid::Uuid::new_v4().to_string()),
19311                versioned: true,
19312                version_suspended: false,
19313                data_dir: Some(uuid::Uuid::new_v4().to_string()),
19314                etag: Some("etag".to_string()),
19315                mod_time: Some(OffsetDateTime::UNIX_EPOCH.to_string()),
19316            }),
19317            dispatch: None,
19318        };
19319        let (_manifest_name, bound) = install_test_tier_delete_dispatch_fixture(
19320            store.clone(),
19321            "absent-source-bucket",
19322            uuid::Uuid::new_v4(),
19323            "absent-source-object",
19324            vec![(entry, Some(TierDeleteJournalState::Dispatched))],
19325            TierDeleteDispatchManifestState::DispatchAuthorized,
19326        )
19327        .await
19328        .expect("authorized dispatched journal should persist");
19329        let journal_name = tier_delete_journal_object_name(&bound[0]);
19330        let data = com::read_config(store.clone(), &journal_name)
19331            .await
19332            .expect("prepared journal should be readable");
19333        let mut value: serde_json::Value = serde_json::from_slice(&data).expect("journal should contain JSON");
19334        value["obj_name"] = serde_json::json!("remote/replaced");
19335        com::save_config(
19336            store.clone(),
19337            &journal_name,
19338            serde_json::to_vec(&value).expect("mismatched journal should encode"),
19339        )
19340        .await
19341        .expect("mismatched journal content should be written under the original name");
19342
19343        let stats = recover_tier_delete_journal_entries(store.clone(), 100, None)
19344            .await
19345            .expect("recovery scan should complete");
19346
19347        assert_eq!((stats.scanned, stats.deleted, stats.failed), (1, 0, 1));
19348        assert_eq!(tier_delete_journal_count(store).await, 1, "mismatched journal must be retained");
19349        assert_eq!(backend.remove_count().await, 0);
19350        assert_eq!(backend.object_count().await, 1);
19351    }
19352
19353    #[cfg(feature = "test-util")]
19354    #[test]
19355    #[serial_test::serial(storage_class_env)]
19356    fn transitioned_history_expiry_uses_journal_or_legacy_free_version() {
19357        std::thread::Builder::new()
19358            .name("transitioned-delete-all-test".to_string())
19359            .stack_size(32 * 1024 * 1024)
19360            .spawn(|| {
19361                let runtime = tokio::runtime::Builder::new_multi_thread()
19362                    .enable_all()
19363                    .worker_threads(2)
19364                    .build()
19365                    .expect("test runtime should build");
19366                runtime.block_on(async {
19367                    let temp_dir = tempfile::tempdir().expect("create transitioned delete-all store dir");
19368                    let (ctx, store, _shutdown) =
19369                        without_storage_class_env(build_isolated_test_store(temp_dir.path(), "transitioned-delete-all", &[4]))
19370                            .await;
19371                    crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
19372                    let tier_name = "DELETEALLTRANSITIONED";
19373                    let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
19374                    let bucket = "transitioned-delete-all-bucket";
19375                    let object = "object";
19376                    store
19377                        .make_bucket(bucket, &MakeBucketOptions::default())
19378                        .await
19379                        .expect("bucket should be created");
19380                    crate::bucket::metadata_sys::update_in(
19381                        &ctx,
19382                        bucket,
19383                        BUCKET_VERSIONING_CONFIG,
19384                        b"<VersioningConfiguration><Status>Enabled</Status></VersioningConfiguration>".to_vec(),
19385                    )
19386                    .await
19387                    .expect("bucket versioning should be enabled");
19388                    crate::bucket::metadata_sys::update_in(
19389                        &ctx,
19390                        bucket,
19391                        BUCKET_LIFECYCLE_CONFIG,
19392                        br#"<LifecycleConfiguration>
19393  <Rule>
19394    <ID>delete-all-versions</ID>
19395    <Status>Enabled</Status>
19396    <Filter><Prefix></Prefix></Filter>
19397    <Expiration><Days>1</Days><ExpiredObjectAllVersions>true</ExpiredObjectAllVersions></Expiration>
19398  </Rule>
19399</LifecycleConfiguration>"#
19400                            .to_vec(),
19401                    )
19402                    .await
19403                    .expect("delete-all lifecycle should be configured");
19404
19405                    let old_time = OffsetDateTime::now_utc() - time::Duration::days(3);
19406                    let mut history_reader = PutObjReader::from_vec(b"transitioned history".to_vec());
19407                    let history = store
19408                        .put_object(
19409                            bucket,
19410                            object,
19411                            &mut history_reader,
19412                            &ObjectOptions {
19413                                versioned: true,
19414                                mod_time: Some(old_time - time::Duration::hours(1)),
19415                                ..Default::default()
19416                            },
19417                        )
19418                        .await
19419                        .expect("historical version should be written");
19420                    let mut current_reader = PutObjReader::from_vec(b"current version".to_vec());
19421                    let current = store
19422                        .put_object(
19423                            bucket,
19424                            object,
19425                            &mut current_reader,
19426                            &ObjectOptions {
19427                                versioned: true,
19428                                mod_time: Some(old_time),
19429                                ..Default::default()
19430                            },
19431                        )
19432                        .await
19433                        .expect("current version should be written");
19434                    store
19435                        .transition_object(
19436                            bucket,
19437                            object,
19438                            &ObjectOptions {
19439                                versioned: true,
19440                                version_id: history.version_id.map(|version_id| version_id.to_string()),
19441                                transition: TransitionOptions {
19442                                    status: TRANSITION_PENDING.to_string(),
19443                                    tier: tier_name.to_string(),
19444                                    etag: history.etag.clone().expect("history should have an ETag"),
19445                                    ..Default::default()
19446                                },
19447                                mod_time: history.mod_time,
19448                                ..Default::default()
19449                            },
19450                        )
19451                        .await
19452                        .expect("historical version should transition");
19453                    assert_eq!(backend.object_count().await, 1);
19454                    assert_eq!(backend.put_versions().await.len(), 1);
19455                    backend.set_remove_failure(true);
19456
19457                    let incarnation = store
19458                        .bucket_incarnation_id_from_disk(bucket)
19459                        .await
19460                        .expect("bucket incarnation should be available");
19461                    for disk_index in 0..4 {
19462                        let metadata_path = temp_dir
19463                            .path()
19464                            .join(format!("pool0/set0/disk{disk_index}/{bucket}/{object}/{STORAGE_FORMAT_FILE}"));
19465                        let encoded = tokio::fs::read(&metadata_path)
19466                            .await
19467                            .expect("transition metadata should be readable");
19468                        let mut metadata = FileMeta::load(&encoded).expect("transition metadata should decode");
19469                        let (version_index, mut transitioned) = metadata
19470                            .find_version(history.version_id)
19471                            .expect("transitioned history should exist");
19472                        // Rewrite the serialized record to model legacy metadata;
19473                        // ordinary writes preserve an already reconciled state.
19474                        rustfs_utils::http::metadata_compat::remove_bytes(
19475                            &mut transitioned.object.as_mut().expect("history should be an object").meta_sys,
19476                            rustfs_utils::http::metadata_compat::SUFFIX_TRANSITIONED_VERSION_STATE,
19477                        );
19478                        metadata.versions[version_index] = rustfs_filemeta::FileMetaShallowVersion::try_from(transitioned)
19479                            .expect("legacy history should re-encode");
19480                        tokio::fs::write(
19481                            &metadata_path,
19482                            metadata.marshal_msg().expect("unknown transition metadata should encode"),
19483                        )
19484                        .await
19485                        .expect("unknown transition metadata should be written");
19486                        let encoded = tokio::fs::read(&metadata_path)
19487                            .await
19488                            .expect("legacy transition metadata should be readable");
19489                        let legacy = FileMeta::load(&encoded)
19490                            .expect("legacy transition metadata should decode")
19491                            .find_version(history.version_id)
19492                            .expect("legacy history should exist")
19493                            .1
19494                            .into_fileinfo(bucket, object, true)
19495                            .expect("legacy history should decode");
19496                        assert_eq!(legacy.transition_version_state, rustfs_filemeta::TransitionVersionState::Unknown);
19497                    }
19498                    let lifecycle_event = crate::bucket::lifecycle::lifecycle::Event {
19499                        action: rustfs_scanner_metrics::metrics::IlmAction::DeleteAllVersionsAction,
19500                        rule_id: "delete-all-versions".to_string(),
19501                        ..Default::default()
19502                    };
19503                    let legacy_applied =
19504                        crate::bucket::lifecycle::bucket_lifecycle_ops::apply_expiry_on_non_transitioned_objects(
19505                            store.clone(),
19506                            &current,
19507                            &lifecycle_event,
19508                            &crate::bucket::lifecycle::bucket_lifecycle_audit::LcEventSrc::Scanner,
19509                            incarnation,
19510                        )
19511                        .await;
19512                    assert!(legacy_applied, "legacy Unknown transition metadata should use the free-version fallback");
19513                    assert_eq!(tier_delete_journal_count(store.clone()).await, 0);
19514                    assert_eq!(backend.remove_count().await, 0);
19515                    let retained = store.pools[0].disk_set[0]
19516                        .load_file_info_versions_exact(bucket, object)
19517                        .await
19518                        .expect("fallback delete-all metadata should remain readable")
19519                        .expect("fallback delete-all should retain a hidden cleanup owner");
19520                    assert_eq!(
19521                        retained
19522                            .versions
19523                            .iter()
19524                            .filter(|version| !version.tier_free_version())
19525                            .count(),
19526                        0
19527                    );
19528                    assert_eq!(
19529                        retained.versions.iter().filter(|version| version.tier_free_version()).count(),
19530                        1,
19531                        "legacy cleanup must retain exactly one hidden free-version"
19532                    );
19533                    assert_eq!(backend.object_count().await, 1, "legacy remote cleanup must remain pending");
19534
19535                    // Use a distinct tier/backend so the legacy worker can be
19536                    // held in a deterministic remote-failure state while the
19537                    // exact journal path independently proves convergence.
19538                    let exact_tier_name = "DELETEALLTRANSITIONEDEXACT";
19539                    let exact_backend = register_mock_tier(&ctx.tier_config_mgr(), exact_tier_name).await;
19540                    let exact_object = "exact-object";
19541                    let mut exact_history_reader = PutObjReader::from_vec(b"exact transitioned history".to_vec());
19542                    let exact_history = store
19543                        .put_object(
19544                            bucket,
19545                            exact_object,
19546                            &mut exact_history_reader,
19547                            &ObjectOptions {
19548                                versioned: true,
19549                                mod_time: Some(old_time - time::Duration::hours(1)),
19550                                ..Default::default()
19551                            },
19552                        )
19553                        .await
19554                        .expect("second historical version should be written");
19555                    let mut exact_current_reader = PutObjReader::from_vec(b"exact current version".to_vec());
19556                    let exact_current = store
19557                        .put_object(
19558                            bucket,
19559                            exact_object,
19560                            &mut exact_current_reader,
19561                            &ObjectOptions {
19562                                versioned: true,
19563                                mod_time: Some(old_time),
19564                                ..Default::default()
19565                            },
19566                        )
19567                        .await
19568                        .expect("second current version should be written");
19569                    store
19570                        .transition_object(
19571                            bucket,
19572                            exact_object,
19573                            &ObjectOptions {
19574                                versioned: true,
19575                                version_id: exact_history.version_id.map(|version_id| version_id.to_string()),
19576                                transition: TransitionOptions {
19577                                    status: TRANSITION_PENDING.to_string(),
19578                                    tier: exact_tier_name.to_string(),
19579                                    etag: exact_history.etag.clone().expect("second history should have an ETag"),
19580                                    ..Default::default()
19581                                },
19582                                mod_time: exact_history.mod_time,
19583                                ..Default::default()
19584                            },
19585                        )
19586                        .await
19587                        .expect("second historical version should transition exactly");
19588                    let exact_remote_version = exact_backend
19589                        .put_versions()
19590                        .await
19591                        .last()
19592                        .cloned()
19593                        .expect("second transition should publish a remote version");
19594                    exact_backend.set_remove_failure(true);
19595                    let applied = crate::bucket::lifecycle::bucket_lifecycle_ops::apply_expiry_on_non_transitioned_objects(
19596                        store.clone(),
19597                        &exact_current,
19598                        &lifecycle_event,
19599                        &crate::bucket::lifecycle::bucket_lifecycle_audit::LcEventSrc::Scanner,
19600                        incarnation,
19601                    )
19602                    .await;
19603
19604                    assert!(applied, "delete-all should remove current and transitioned history");
19605                    assert!(
19606                        store.pools[0].disk_set[0]
19607                            .load_file_info_versions_exact(bucket, exact_object)
19608                            .await
19609                            .expect("exact journal-owned metadata lookup should succeed")
19610                            .is_none(),
19611                        "exact journal ownership must remove the source key without leaving a free-version"
19612                    );
19613                    let versions = store.pools[0].disk_set[0]
19614                        .load_file_info_versions_exact(bucket, object)
19615                        .await
19616                        .expect("remaining fallback metadata should be readable")
19617                        .expect("the prior legacy cleanup owner should remain");
19618                    assert_eq!(
19619                        versions
19620                            .versions
19621                            .iter()
19622                            .filter(|version| !version.tier_free_version())
19623                            .count(),
19624                        0,
19625                        "exact delete-all must remove every visible version"
19626                    );
19627                    assert_eq!(
19628                        versions.versions.iter().filter(|version| version.tier_free_version()).count(),
19629                        1,
19630                        "the exact v6 journal must not add another free-version"
19631                    );
19632                    assert_eq!(tier_delete_journal_count(store.clone()).await, 1);
19633                    assert_eq!(backend.object_count().await, 1, "legacy remote cleanup must remain pending");
19634                    assert_eq!(exact_backend.object_count().await, 1, "exact remote cleanup must remain journal-owned");
19635
19636                    exact_backend.set_remove_failure(false);
19637                    let stats = recover_tier_delete_journal_entries(store.clone(), 100, None)
19638                        .await
19639                        .expect("committed journal should recover");
19640                    assert_eq!(stats.failed, 0);
19641                    assert!(stats.scanned <= 1 && stats.deleted <= 1);
19642                    assert_eq!(tier_delete_journal_count(store).await, 0);
19643                    assert_eq!(
19644                        backend.object_count().await,
19645                        1,
19646                        "journal recovery must leave the legacy free-version remote"
19647                    );
19648                    assert_eq!(exact_backend.object_count().await, 0);
19649                    assert!(exact_backend.exact_remove_count() >= 1);
19650                    assert_eq!(exact_backend.remove_versions().await, vec![exact_remote_version]);
19651                });
19652            })
19653            .expect("test thread should spawn")
19654            .join()
19655            .expect("test thread should complete");
19656    }
19657
19658    #[cfg(feature = "test-util")]
19659    #[tokio::test]
19660    #[serial_test::serial(storage_class_env)]
19661    async fn odm_write_back_requires_one_set_and_enabled_namespace_locking() {
19662        for (layout, locking, supported) in [
19663            (&[(1, 4)][..], true, true),
19664            (&[(1, 4), (1, 4)][..], true, false),
19665            (&[(2, 4)][..], true, false),
19666            (&[(1, 4)][..], false, false),
19667        ] {
19668            temp_env::async_with_vars([("RUSTFS_LOCK_ENABLED", Some(if locking { "true" } else { "false" }))], async {
19669                let dir = tempfile::tempdir().expect("isolated topology");
19670                let shutdown = CancellationToken::new();
19671                let (_ctx, store, _) = without_storage_class_env(build_isolated_test_store_with_layout(
19672                    dir.path(),
19673                    "odm-topology",
19674                    layout,
19675                    shutdown.clone(),
19676                    None,
19677                ))
19678                .await;
19679                assert_eq!(
19680                    store.supports_atomic_create_only_write_back(),
19681                    supported,
19682                    "layout={layout:?}, locking={locking}"
19683                );
19684                shutdown.cancel();
19685            })
19686            .await;
19687        }
19688    }
19689
19690    #[cfg(feature = "test-util")]
19691    #[tokio::test]
19692    #[serial_test::serial(storage_class_env)]
19693    async fn dispatched_tier_delete_recovery_requires_namespace_locking() {
19694        temp_env::async_with_vars([("RUSTFS_LOCK_ENABLED", Some("false"))], async {
19695            let temp_dir = tempfile::tempdir().expect("create lock-disabled store dir");
19696            let (ctx, store, _shutdown) =
19697                without_storage_class_env(build_isolated_test_store(temp_dir.path(), "prepared-recovery-lock-disabled", &[4]))
19698                    .await;
19699            crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
19700            assert!(ctx.lock_manager().is_disabled());
19701
19702            let tier_name = "PREPAREDLOCKDISABLED";
19703            let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
19704            let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
19705                .await
19706                .expect("tier lease should resolve")
19707                .backend_identity();
19708            let entry = Jentry {
19709                persisted_version: 0,
19710                obj_name: "remote/lock-disabled".to_string(),
19711                version_id: uuid::Uuid::new_v4().to_string(),
19712                tier_name: tier_name.to_string(),
19713                backend_identity: Some(backend_identity),
19714                version_id_exact: true,
19715                version_state: rustfs_filemeta::TransitionVersionState::Exact,
19716                state: TierDeleteJournalState::Prepared,
19717                source: Some(TierDeleteSourceIdentity {
19718                    bucket: "absent-source-bucket".to_string(),
19719                    object: "absent-source-object".to_string(),
19720                    version_id: Some(uuid::Uuid::new_v4().to_string()),
19721                    versioned: true,
19722                    version_suspended: false,
19723                    data_dir: Some(uuid::Uuid::new_v4().to_string()),
19724                    etag: Some("etag".to_string()),
19725                    mod_time: Some(OffsetDateTime::UNIX_EPOCH.to_string()),
19726                }),
19727                dispatch: None,
19728            };
19729            install_test_tier_delete_dispatch_fixture(
19730                store.clone(),
19731                "absent-source-bucket",
19732                uuid::Uuid::new_v4(),
19733                "absent-source-object",
19734                vec![(entry, Some(TierDeleteJournalState::Dispatched))],
19735                TierDeleteDispatchManifestState::DispatchAuthorized,
19736            )
19737            .await
19738            .expect("authorized dispatched journal should persist");
19739
19740            let stats = recover_tier_delete_journal_entries(store.clone(), 100, None)
19741                .await
19742                .expect("recovery scan should complete");
19743
19744            assert_eq!((stats.scanned, stats.deleted, stats.failed), (1, 0, 1));
19745            assert_eq!(tier_delete_journal_count(store).await, 1, "journal must remain dispatched for retry");
19746            assert_eq!(backend.remove_count().await, 0, "lock-disabled recovery must not delete remotely");
19747        })
19748        .await;
19749    }
19750
19751    #[cfg(feature = "test-util")]
19752    #[tokio::test]
19753    #[serial_test::serial(storage_class_env)]
19754    async fn lifecycle_delete_all_requires_namespace_locking_before_mutation() {
19755        temp_env::async_with_vars([("RUSTFS_LOCK_ENABLED", Some("false"))], async {
19756            let temp_dir = tempfile::tempdir().expect("create lock-disabled delete-all store dir");
19757            let (ctx, store, _shutdown) =
19758                without_storage_class_env(build_isolated_test_store(temp_dir.path(), "delete-all-lock-disabled", &[4])).await;
19759            crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
19760            assert!(ctx.lock_manager().is_disabled());
19761            let bucket = "delete-all-lock-disabled-bucket";
19762            let object = "object";
19763            store
19764                .make_bucket(bucket, &MakeBucketOptions::default())
19765                .await
19766                .expect("bucket should be created");
19767            let mut reader = PutObjReader::from_vec(b"must survive".to_vec());
19768            let no_lock_opts = ObjectOptions {
19769                no_lock: true,
19770                ..Default::default()
19771            };
19772            let original = store.pools[0]
19773                .put_object(bucket, object, &mut reader, &no_lock_opts)
19774                .await
19775                .expect("source should be written");
19776
19777            let mut delete_opts = ObjectOptions {
19778                delete_prefix: true,
19779                delete_prefix_object: true,
19780                lifecycle_delete_all: Some(crate::object_api::LifecycleDeleteAllRequest {
19781                    version_id: original.version_id,
19782                    delete_marker: false,
19783                    action: rustfs_scanner_metrics::metrics::IlmAction::DeleteAllVersionsAction,
19784                    rule_id: "rule".to_string(),
19785                    phase: crate::object_api::LifecycleDeleteAllPhase::Preflight,
19786                }),
19787                delete_replication_config_snapshot: Some(Arc::new(
19788                    crate::bucket::replication::DeleteReplicationConfigSnapshot::default(),
19789                )),
19790                ..Default::default()
19791            };
19792            delete_opts.ensure_lifecycle_delete_all_journal();
19793
19794            let err = store
19795                .delete_object_with_tier_delete_journal(bucket, object, delete_opts)
19796                .await
19797                .expect_err("delete-all must reject disabled namespace locking");
19798
19799            assert!(err.to_string().contains("requires namespace locking"));
19800            let retained = store.pools[0]
19801                .get_object_info(bucket, object, &no_lock_opts)
19802                .await
19803                .expect("rejected delete-all must retain the source");
19804            assert_eq!(retained.etag, original.etag);
19805            assert_eq!(tier_delete_journal_count(store).await, 0, "rejected delete-all must not prepare journals");
19806        })
19807        .await;
19808    }
19809
19810    #[cfg(feature = "test-util")]
19811    #[tokio::test]
19812    #[serial_test::serial(storage_class_env)]
19813    async fn tier_mutation_intent_record_round_trips_through_config_store() {
19814        let temp_dir = tempfile::tempdir().expect("create temp store dir");
19815        let (_ctx, store, _shutdown) =
19816            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "tier-mutation-intent-record", &[4])).await;
19817        let mutation_id = uuid::Uuid::new_v4();
19818        let intent = TierMutationIntent {
19819            mutation_id,
19820            revision: 1,
19821            kind: TierMutationIntentKind::Edit,
19822            state: TierMutationIntentState::Prepared,
19823            old_config_etag: Some("old-etag".to_string()),
19824            committed_config_etag: None,
19825            candidate_digest: [3; 32],
19826            affected_targets: vec![TierMutationIntentTarget {
19827                tier_name: "COLD-A".to_string(),
19828                old_backend_identity: Some([1; 32]),
19829                new_backend_identity: Some([2; 32]),
19830            }],
19831            expires_at_unix_nanos: 1_780_000_000_000_000_000,
19832        };
19833
19834        save_tier_mutation_intent_record(store.clone(), &intent)
19835            .await
19836            .expect("tier mutation intent record should persist");
19837        let loaded = load_tier_mutation_intent_record(store.clone(), mutation_id)
19838            .await
19839            .expect("tier mutation intent record should load");
19840
19841        assert_eq!(loaded, intent);
19842
19843        delete_tier_mutation_intent_record(store.clone(), mutation_id)
19844            .await
19845            .expect("tier mutation intent record delete should be idempotent");
19846        delete_tier_mutation_intent_record(store.clone(), mutation_id)
19847            .await
19848            .expect("tier mutation intent record delete should tolerate missing records");
19849        let err = load_tier_mutation_intent_record(store, mutation_id)
19850            .await
19851            .expect_err("deleted tier mutation intent record should not load");
19852        assert!(matches!(err, Error::ConfigNotFound));
19853    }
19854
19855    #[cfg(feature = "test-util")]
19856    #[tokio::test]
19857    #[serial_test::serial(storage_class_env)]
19858    async fn tier_probe_intent_store_enforces_create_cas_and_terminal_delete_preconditions() {
19859        let temp_dir = tempfile::tempdir().expect("create temp store dir");
19860        let (_ctx, store, _shutdown) =
19861            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "tier-probe-intent-cas", &[4])).await;
19862        let probe_id = uuid::Uuid::new_v4();
19863        let creator_epoch = uuid::Uuid::new_v4();
19864        let initial = TierProbeIntent {
19865            probe_id,
19866            revision: 1,
19867            state: TierProbeIntentState::UploadOutcomeUnknown,
19868            operation: TierProbeOperationIdentity::Verify {
19869                config_etag: "config-etag".to_string(),
19870                backend_identity: [1; 32],
19871            },
19872            tier_name: "COLD-A".to_string(),
19873            destination_id: [1; 32],
19874            probe_object: format!("rustfs-tier-probe-{probe_id}"),
19875            creator_id: "node-a".to_string(),
19876            creator_epoch,
19877            created_at_unix_nanos: 1_780_000_000_000_000_000,
19878            owner: TierProbeOwnerFence {
19879                owner_id: "node-a".to_string(),
19880                owner_epoch: creator_epoch,
19881                not_after_unix_nanos: 1_780_000_900_000_000_000,
19882            },
19883            remote_version: TierProbeRemoteVersion::default(),
19884        };
19885
19886        save_tier_probe_intent_record_if_absent(store.clone(), &initial)
19887            .await
19888            .expect("initial probe intent should persist with create-only semantics");
19889        let duplicate = save_tier_probe_intent_record_if_absent(store.clone(), &initial)
19890            .await
19891            .expect_err("duplicate create must fail closed");
19892        assert!(matches!(duplicate, Error::PreconditionFailed));
19893
19894        let observed_initial = load_tier_probe_intent_record(store.clone(), probe_id)
19895            .await
19896            .expect("initial probe intent should load with an ETag");
19897        assert_eq!(observed_initial.intent(), &initial);
19898
19899        let nonterminal_delete = delete_tier_probe_intent_record_if_current(store.clone(), &observed_initial)
19900            .await
19901            .expect_err("nonterminal evidence must not be deleted");
19902        assert!(nonterminal_delete.to_string().contains("must be terminal"));
19903
19904        let mut fabricated_current_intent = initial.clone();
19905        fabricated_current_intent.tier_name = "COLD-B".to_string();
19906        let mut fabricated_successor = fabricated_current_intent.clone();
19907        fabricated_successor
19908            .advance(
19909                TierProbeIntentState::Uploaded,
19910                TierProbeRemoteVersion::versioned(uuid::Uuid::new_v4().to_string()),
19911            )
19912            .expect("fabricated successor should be internally valid");
19913        let fabricated_current = observed_initial.with_intent_for_test(fabricated_current_intent.clone());
19914        let crossed_cas = save_tier_probe_intent_record_if_current(store.clone(), &fabricated_current, &fabricated_successor)
19915            .await
19916            .expect_err("a live ETag must not authorize a different caller record");
19917        assert!(matches!(crossed_cas, Error::PreconditionFailed));
19918        assert_eq!(
19919            load_tier_probe_intent_record(store.clone(), probe_id)
19920                .await
19921                .expect("crossed CAS must retain the authoritative record")
19922                .intent(),
19923            &initial
19924        );
19925
19926        let mut fabricated_terminal_intent = fabricated_current_intent;
19927        fabricated_terminal_intent
19928            .advance(TierProbeIntentState::AbortedNoRemote, TierProbeRemoteVersion::default())
19929            .expect("fabricated terminal should be internally valid");
19930        let fabricated_terminal = observed_initial.with_intent_for_test(fabricated_terminal_intent);
19931        let crossed_delete = delete_tier_probe_intent_record_if_current(store.clone(), &fabricated_terminal)
19932            .await
19933            .expect_err("a live ETag must not delete for a different caller record");
19934        assert!(matches!(crossed_delete, Error::PreconditionFailed));
19935        assert_eq!(
19936            load_tier_probe_intent_record(store.clone(), probe_id)
19937                .await
19938                .expect("crossed delete must retain the authoritative record")
19939                .intent(),
19940            &initial
19941        );
19942
19943        let remote_version = TierProbeRemoteVersion::versioned(uuid::Uuid::new_v4().to_string());
19944        let mut uploaded = observed_initial.intent().clone();
19945        uploaded
19946            .advance(TierProbeIntentState::Uploaded, remote_version.clone())
19947            .expect("known PUT result should advance");
19948        save_tier_probe_intent_record_if_current(store.clone(), &observed_initial, &uploaded)
19949            .await
19950            .expect("the matching initial ETag should admit one successor");
19951
19952        let stale_cas = save_tier_probe_intent_record_if_current(store.clone(), &observed_initial, &uploaded)
19953            .await
19954            .expect_err("a consumed ETag must not overwrite the current generation");
19955        assert!(matches!(stale_cas, Error::PreconditionFailed));
19956
19957        let observed_uploaded = load_tier_probe_intent_record(store.clone(), probe_id)
19958            .await
19959            .expect("uploaded generation should load");
19960        assert_eq!(observed_uploaded.intent(), &uploaded);
19961        let mut cleanup = observed_uploaded.intent().clone();
19962        cleanup
19963            .advance(TierProbeIntentState::CleanupPending, remote_version.clone())
19964            .expect("known candidate should become cleanup-pending");
19965        save_tier_probe_intent_record_if_current(store.clone(), &observed_uploaded, &cleanup)
19966            .await
19967            .expect("cleanup generation should persist by exact ETag");
19968
19969        let observed_cleanup = load_tier_probe_intent_record(store.clone(), probe_id)
19970            .await
19971            .expect("cleanup generation should load");
19972        let mut completed = observed_cleanup.intent().clone();
19973        completed
19974            .advance(TierProbeIntentState::Completed, remote_version)
19975            .expect("exact cleanup should become terminal");
19976        save_tier_probe_intent_record_if_current(store.clone(), &observed_cleanup, &completed)
19977            .await
19978            .expect("terminal generation should persist by exact ETag");
19979
19980        let stale_terminal = observed_cleanup.with_intent_for_test(completed.clone());
19981        let stale_delete = delete_tier_probe_intent_record_if_current(store.clone(), &stale_terminal)
19982            .await
19983            .expect_err("a stale ETag must not delete terminal evidence");
19984        assert!(matches!(stale_delete, Error::PreconditionFailed));
19985
19986        let observed_completed = load_tier_probe_intent_record(store.clone(), probe_id)
19987            .await
19988            .expect("terminal generation should remain after stale delete");
19989        assert_eq!(observed_completed.intent(), &completed);
19990        delete_tier_probe_intent_record_if_current(store.clone(), &observed_completed)
19991            .await
19992            .expect("the exact terminal ETag should delete the record");
19993        assert!(matches!(load_tier_probe_intent_record(store, probe_id).await, Err(Error::ConfigNotFound)));
19994    }
19995
19996    #[cfg(feature = "test-util")]
19997    #[tokio::test]
19998    #[serial_test::serial(storage_class_env)]
19999    async fn tier_mutation_intent_record_scan_retains_good_records_and_counts_bad_records() {
20000        let temp_dir = tempfile::tempdir().expect("create temp store dir");
20001        let (_ctx, store, _shutdown) =
20002            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "tier-mutation-intent-scan", &[4])).await;
20003        let build_intent = |mutation_id: uuid::Uuid, tier_name: &str| TierMutationIntent {
20004            mutation_id,
20005            revision: 1,
20006            kind: TierMutationIntentKind::Edit,
20007            state: TierMutationIntentState::Prepared,
20008            old_config_etag: Some("old-etag".to_string()),
20009            committed_config_etag: None,
20010            candidate_digest: [3; 32],
20011            affected_targets: vec![TierMutationIntentTarget {
20012                tier_name: tier_name.to_string(),
20013                old_backend_identity: Some([1; 32]),
20014                new_backend_identity: Some([2; 32]),
20015            }],
20016            expires_at_unix_nanos: 1_780_000_000_000_000_000,
20017        };
20018        let first_id = uuid::Uuid::parse_str("12345678-1234-5678-9abc-def012345678").expect("first uuid should parse");
20019        let second_id = uuid::Uuid::parse_str("22345678-1234-5678-9abc-def012345678").expect("second uuid should parse");
20020        let first = build_intent(first_id, "COLD-A");
20021        let second = build_intent(second_id, "COLD-B");
20022        save_tier_mutation_intent_record(store.clone(), &first)
20023            .await
20024            .expect("first tier mutation intent record should persist");
20025        save_tier_mutation_intent_record(store.clone(), &second)
20026            .await
20027            .expect("second tier mutation intent record should persist");
20028        com::save_config(
20029            store.clone(),
20030            &format!("{TIER_MUTATION_INTENT_RECORD_PREFIX}/00/00/33345678123456789abcdef012345678.json"),
20031            b"{}".to_vec(),
20032        )
20033        .await
20034        .expect("malformed-shard intent record should persist");
20035        com::save_config(
20036            store.clone(),
20037            &format!("{TIER_MUTATION_INTENT_RECORD_PREFIX}/44/44/44445678123456789abcdef012345678.json"),
20038            b"{".to_vec(),
20039        )
20040        .await
20041        .expect("corrupt-json intent record should persist");
20042
20043        let scan = list_tier_mutation_intent_records(store, 100, None)
20044            .await
20045            .expect("tier mutation intent records should scan");
20046        let mut loaded_ids: Vec<_> = scan.intents.into_iter().map(|intent| intent.mutation_id).collect();
20047        loaded_ids.sort();
20048
20049        assert_eq!(scan.scanned, 4);
20050        assert_eq!(scan.failed, 2);
20051        assert_eq!(loaded_ids, vec![first_id, second_id]);
20052        assert!(!scan.truncated);
20053        assert_eq!(scan.next_marker, None);
20054    }
20055
20056    #[cfg(feature = "test-util")]
20057    #[tokio::test]
20058    #[serial_test::serial(storage_class_env)]
20059    async fn tier_mutation_intent_record_advance_is_idempotent_in_config_store() {
20060        let temp_dir = tempfile::tempdir().expect("create temp store dir");
20061        let (_ctx, store, _shutdown) =
20062            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "tier-mutation-intent-advance", &[4])).await;
20063        let mutation_id = uuid::Uuid::new_v4();
20064        let intent = TierMutationIntent {
20065            mutation_id,
20066            revision: 1,
20067            kind: TierMutationIntentKind::Edit,
20068            state: TierMutationIntentState::Prepared,
20069            old_config_etag: Some("old-etag".to_string()),
20070            committed_config_etag: None,
20071            candidate_digest: [3; 32],
20072            affected_targets: vec![TierMutationIntentTarget {
20073                tier_name: "COLD-A".to_string(),
20074                old_backend_identity: Some([1; 32]),
20075                new_backend_identity: Some([2; 32]),
20076            }],
20077            expires_at_unix_nanos: 1_780_000_000_000_000_000,
20078        };
20079        save_tier_mutation_intent_record(store.clone(), &intent)
20080            .await
20081            .expect("prepared tier mutation intent record should persist");
20082        let (loaded_before_advance, stale_etag) = load_tier_mutation_intent_record_with_etag(store.clone(), mutation_id)
20083            .await
20084            .expect("prepared tier mutation intent record should load with etag");
20085        assert_eq!(loaded_before_advance, intent);
20086        assert!(!stale_etag.is_empty());
20087
20088        let (committed, first_advanced) = advance_tier_mutation_intent_record_idempotent(
20089            store.clone(),
20090            mutation_id,
20091            TierMutationIntentState::Committed,
20092            Some("new-etag".to_string()),
20093        )
20094        .await
20095        .expect("first commit should advance the record");
20096        assert!(first_advanced);
20097        assert_eq!(committed.state, TierMutationIntentState::Committed);
20098        assert_eq!(committed.revision, 2);
20099        assert_eq!(committed.committed_config_etag.as_deref(), Some("new-etag"));
20100
20101        let (retried, retry_advanced) = advance_tier_mutation_intent_record_idempotent(
20102            store.clone(),
20103            mutation_id,
20104            TierMutationIntentState::Committed,
20105            Some("new-etag".to_string()),
20106        )
20107        .await
20108        .expect("same commit retry should be idempotent");
20109        assert!(!retry_advanced);
20110        assert_eq!(retried, committed);
20111
20112        let conflict = advance_tier_mutation_intent_record_idempotent(
20113            store.clone(),
20114            mutation_id,
20115            TierMutationIntentState::Committed,
20116            Some("other-etag".to_string()),
20117        )
20118        .await
20119        .expect_err("conflicting commit retry should fail closed");
20120        assert!(matches!(conflict, Error::Io(_)));
20121        assert!(conflict.to_string().contains("committed config etag does not match"));
20122
20123        let mut stale_conflict = intent;
20124        stale_conflict
20125            .advance_idempotent(TierMutationIntentState::Committed, Some("other-etag".to_string()))
20126            .expect("stale conflicting intent should advance locally");
20127        let stale_save = save_tier_mutation_intent_record_if_current(store.clone(), &stale_conflict, &stale_etag)
20128            .await
20129            .expect_err("stale etag must fail closed instead of overwriting the committed record");
20130        assert!(matches!(stale_save, Error::PreconditionFailed));
20131
20132        let loaded = load_tier_mutation_intent_record(store.clone(), mutation_id)
20133            .await
20134            .expect("conflicting retry must not overwrite the durable record");
20135        assert_eq!(loaded, committed);
20136
20137        let abort_id = uuid::Uuid::new_v4();
20138        let abort_intent = TierMutationIntent {
20139            mutation_id: abort_id,
20140            revision: 1,
20141            kind: TierMutationIntentKind::Edit,
20142            state: TierMutationIntentState::Prepared,
20143            old_config_etag: Some("old-etag".to_string()),
20144            committed_config_etag: None,
20145            candidate_digest: [4; 32],
20146            affected_targets: vec![TierMutationIntentTarget {
20147                tier_name: "COLD-B".to_string(),
20148                old_backend_identity: Some([1; 32]),
20149                new_backend_identity: Some([2; 32]),
20150            }],
20151            expires_at_unix_nanos: 1_780_000_000_000_000_000,
20152        };
20153        save_tier_mutation_intent_record(store.clone(), &abort_intent)
20154            .await
20155            .expect("prepared abort intent record should persist");
20156
20157        let (aborted, first_abort_advanced) =
20158            advance_tier_mutation_intent_record_idempotent(store.clone(), abort_id, TierMutationIntentState::Aborted, None)
20159                .await
20160                .expect("first abort should advance the record");
20161        assert!(first_abort_advanced);
20162        assert_eq!(aborted.state, TierMutationIntentState::Aborted);
20163        assert_eq!(aborted.revision, 2);
20164        assert_eq!(aborted.committed_config_etag, None);
20165
20166        let (aborted_retry, retry_abort_advanced) =
20167            advance_tier_mutation_intent_record_idempotent(store, abort_id, TierMutationIntentState::Aborted, None)
20168                .await
20169                .expect("same abort retry should be idempotent");
20170        assert!(!retry_abort_advanced);
20171        assert_eq!(aborted_retry, aborted);
20172    }
20173
20174    #[cfg(feature = "test-util")]
20175    fn tier_mutation_peer_test_intent(
20176        mutation_id: uuid::Uuid,
20177        tier_name: &str,
20178        candidate_digest: [u8; 32],
20179    ) -> TierMutationIntent {
20180        TierMutationIntent {
20181            mutation_id,
20182            revision: 1,
20183            kind: TierMutationIntentKind::Edit,
20184            state: TierMutationIntentState::Prepared,
20185            old_config_etag: Some("old-etag".to_string()),
20186            committed_config_etag: None,
20187            candidate_digest,
20188            affected_targets: vec![TierMutationIntentTarget {
20189                tier_name: tier_name.to_string(),
20190                old_backend_identity: Some([1; 32]),
20191                new_backend_identity: Some([2; 32]),
20192            }],
20193            expires_at_unix_nanos: i64::MAX,
20194        }
20195    }
20196
20197    #[cfg(feature = "test-util")]
20198    #[tokio::test]
20199    #[serial_test::serial(storage_class_env)]
20200    async fn tier_mutation_abort_tombstone_rejects_delayed_prepare() {
20201        let temp_dir = tempfile::tempdir().expect("create temp store dir");
20202        let (_ctx, store, _shutdown) =
20203            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "tier-mutation-abort-tombstone", &[4])).await;
20204        register_mock_tier(&store.tier_config_mgr(), "COLD-A").await;
20205        store
20206            .tier_config_mgr()
20207            .read()
20208            .await
20209            .save_tiering_config(store.clone())
20210            .await
20211            .expect("base tier config should persist");
20212        let config_info = store
20213            .get_object_info(
20214                RUSTFS_META_BUCKET,
20215                &format!("{}/{}", com::CONFIG_PREFIX, TIER_CONFIG_FILE),
20216                &ObjectOptions::default(),
20217            )
20218            .await
20219            .expect("base tier config metadata should load");
20220        let mutation_id = uuid::Uuid::new_v4();
20221        let mut intent = tier_mutation_peer_test_intent(mutation_id, "COLD-A", [7; 32]);
20222        intent.old_config_etag = config_info.etag;
20223        intent.expires_at_unix_nanos =
20224            i64::try_from((OffsetDateTime::now_utc() + time::Duration::hours(1)).unix_timestamp_nanos())
20225                .expect("future tombstone expiry should fit i64");
20226        let payload = intent.encode().expect("prepare identity should encode");
20227
20228        let mismatch_id = uuid::Uuid::new_v4();
20229        let mut mismatch = intent.clone();
20230        mismatch.mutation_id = mismatch_id;
20231        mismatch.old_config_etag = Some("not-the-current-config-etag".to_string());
20232        let mismatch_error = handle_tier_mutation_peer_request(
20233            store.clone(),
20234            TIER_MUTATION_RPC_PROTOCOL_VERSION,
20235            TierMutationRpcPhase::Abort,
20236            mismatch_id,
20237            &mismatch.encode().expect("proof-mismatch Abort should encode"),
20238        )
20239        .await
20240        .expect_err("a missing-record Abort with the wrong config proof must fail closed");
20241        assert!(matches!(mismatch_error, TierMutationPeerError::AbortProofMismatch));
20242        assert!(matches!(
20243            load_tier_mutation_intent_record(store.clone(), mismatch_id).await,
20244            Err(Error::ConfigNotFound)
20245        ));
20246
20247        let aborted = handle_tier_mutation_peer_request(
20248            store.clone(),
20249            TIER_MUTATION_RPC_PROTOCOL_VERSION,
20250            TierMutationRpcPhase::Abort,
20251            mutation_id,
20252            &payload,
20253        )
20254        .await
20255        .expect("missing-record abort should persist a tombstone");
20256        assert!(aborted.applied);
20257        assert_eq!(aborted.state, TierMutationPeerState::Aborted);
20258        let tombstone = load_tier_mutation_intent_record(store.clone(), mutation_id)
20259            .await
20260            .expect("abort tombstone should be durable");
20261        assert_eq!(tombstone.state, TierMutationIntentState::Aborted);
20262        assert!(tombstone.same_identity_as(&intent));
20263
20264        let manager = store.tier_config_mgr();
20265        TierConfigMgr::reload_handle(&manager, store.clone())
20266            .await
20267            .expect("reload should retain an unexpired abort tombstone");
20268        let tombstone_after_reload = load_tier_mutation_intent_record(store.clone(), mutation_id)
20269            .await
20270            .expect("reload must not clean an unexpired abort tombstone");
20271        assert_eq!(tombstone_after_reload.state, TierMutationIntentState::Aborted);
20272        assert!(tombstone_after_reload.same_identity_as(&intent));
20273
20274        register_mock_tier(&store.tier_config_mgr(), "COLD-B").await;
20275        store
20276            .tier_config_mgr()
20277            .read()
20278            .await
20279            .save_tiering_config(store.clone())
20280            .await
20281            .expect("later config change should persist before idempotent Abort replay");
20282        let retried_abort = handle_tier_mutation_peer_request(
20283            store.clone(),
20284            TIER_MUTATION_RPC_PROTOCOL_VERSION,
20285            TierMutationRpcPhase::Abort,
20286            mutation_id,
20287            &payload,
20288        )
20289        .await
20290        .expect("terminal Abort retry must not depend on the mutable current config proof");
20291        assert!(!retried_abort.applied);
20292        assert_eq!(retried_abort.state, TierMutationPeerState::Aborted);
20293
20294        let delayed = handle_tier_mutation_peer_request(
20295            store.clone(),
20296            TIER_MUTATION_RPC_PROTOCOL_VERSION,
20297            TierMutationRpcPhase::Prepare,
20298            mutation_id,
20299            &payload,
20300        )
20301        .await
20302        .expect("delayed matching prepare should converge on the tombstone");
20303        assert!(!delayed.applied);
20304        assert_eq!(delayed.state, TierMutationPeerState::Aborted);
20305        TierConfigMgr::acquire_operation_lease(&store.tier_config_mgr(), "COLD-A")
20306            .await
20307            .expect("delayed prepare must not reinstall the runtime block");
20308
20309        let mut conflicting = intent.clone();
20310        conflicting.candidate_digest = [8; 32];
20311        let conflict = handle_tier_mutation_peer_request(
20312            store,
20313            TIER_MUTATION_RPC_PROTOCOL_VERSION,
20314            TierMutationRpcPhase::Prepare,
20315            mutation_id,
20316            &conflicting.encode().expect("conflicting prepare should encode"),
20317        )
20318        .await
20319        .expect_err("different identity must not reuse an abort tombstone");
20320        assert!(matches!(conflict, TierMutationPeerError::ConflictingIntent));
20321    }
20322
20323    #[cfg(feature = "test-util")]
20324    #[tokio::test]
20325    #[serial_test::serial(storage_class_env)]
20326    async fn tier_mutation_expired_prepare_cannot_recreate_a_cleaned_tombstone() {
20327        let temp_dir = tempfile::tempdir().expect("create temp store dir");
20328        let (_ctx, store, _shutdown) =
20329            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "tier-mutation-expired-replay", &[4])).await;
20330        register_mock_tier(&store.tier_config_mgr(), "COLD-A").await;
20331        let mutation_id = uuid::Uuid::new_v4();
20332        let mut intent = tier_mutation_peer_test_intent(mutation_id, "COLD-A", [11; 32]);
20333        intent.expires_at_unix_nanos =
20334            i64::try_from((OffsetDateTime::now_utc() - time::Duration::seconds(1)).unix_timestamp_nanos())
20335                .expect("expired timestamp should fit i64");
20336        let payload = intent.encode().expect("expired signed Prepare should still encode");
20337
20338        let error = handle_tier_mutation_peer_request(
20339            store.clone(),
20340            TIER_MUTATION_RPC_PROTOCOL_VERSION,
20341            TierMutationRpcPhase::Prepare,
20342            mutation_id,
20343            &payload,
20344        )
20345        .await
20346        .expect_err("a missing-record expired Prepare must fail closed");
20347        assert!(matches!(error, TierMutationPeerError::ExpiredIntent));
20348        assert!(matches!(
20349            load_tier_mutation_intent_record(store.clone(), mutation_id).await,
20350            Err(Error::ConfigNotFound)
20351        ));
20352        TierConfigMgr::acquire_operation_lease(&store.tier_config_mgr(), "COLD-A")
20353            .await
20354            .expect("expired replay must not install a peer-only runtime fence");
20355    }
20356
20357    #[cfg(feature = "test-util")]
20358    #[tokio::test]
20359    #[serial_test::serial(storage_class_env)]
20360    async fn tier_mutation_peer_prepare_waits_for_inflight_reference_lease() {
20361        let temp_dir = tempfile::tempdir().expect("create temp store dir");
20362        let (_ctx, store, _shutdown) =
20363            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "tier-mutation-peer-drain", &[4])).await;
20364        register_mock_tier(&store.tier_config_mgr(), "COLD-A").await;
20365        let old_lease = TierConfigMgr::acquire_operation_lease(&store.tier_config_mgr(), "COLD-A")
20366            .await
20367            .expect("old tier operation lease should be available");
20368        let mutation_id = uuid::Uuid::new_v4();
20369        let mut intent = tier_mutation_peer_test_intent(mutation_id, "COLD-A", [9; 32]);
20370        intent.affected_targets[0].old_backend_identity = Some(old_lease.backend_identity());
20371        let payload = intent.encode().expect("prepare intent should encode");
20372        let prepare_store = store.clone();
20373        let prepare = tokio::spawn(async move {
20374            handle_tier_mutation_peer_request(
20375                prepare_store,
20376                TIER_MUTATION_RPC_PROTOCOL_VERSION,
20377                TierMutationRpcPhase::Prepare,
20378                mutation_id,
20379                &payload,
20380            )
20381            .await
20382        });
20383
20384        tokio::time::timeout(Duration::from_secs(5), async {
20385            loop {
20386                if TierConfigMgr::acquire_operation_lease(&store.tier_config_mgr(), "COLD-A")
20387                    .await
20388                    .is_err()
20389                {
20390                    return;
20391                }
20392                tokio::task::yield_now().await;
20393            }
20394        })
20395        .await
20396        .expect("peer Prepare should install its runtime block");
20397        assert!(
20398            !prepare.is_finished(),
20399            "peer Prepare must not acknowledge while a pre-existing reference creator still holds a lease"
20400        );
20401
20402        drop(old_lease);
20403        let outcome = prepare
20404            .await
20405            .expect("peer Prepare task should join")
20406            .expect("peer Prepare should finish after the lease drains");
20407        assert_eq!(outcome.state, TierMutationPeerState::Prepared);
20408        TierConfigMgr::clear_prepared_mutation_intent_block(&store.tier_config_mgr(), mutation_id)
20409            .await
20410            .expect("test should clear the prepared runtime block");
20411    }
20412
20413    #[cfg(feature = "test-util")]
20414    #[test]
20415    #[serial_test::serial(storage_class_env)]
20416    fn tier_mutation_peer_handler_applies_prepare_commit_and_abort_idempotently() {
20417        // This end-to-end state-machine scenario intentionally keeps many
20418        // durable fixtures live at once. Give its debug future the same
20419        // dedicated stack used by the other large ecstore scenarios so the
20420        // assertions run under the default test command without RUST_MIN_STACK.
20421        std::thread::Builder::new()
20422            .name("tier-mutation-peer-handler-test".to_string())
20423            .stack_size(32 * 1024 * 1024)
20424            .spawn(|| {
20425                let runtime = tokio::runtime::Builder::new_multi_thread()
20426                    .enable_all()
20427                    .worker_threads(2)
20428                    .build()
20429                    .expect("tier mutation peer handler test runtime should build");
20430                runtime.block_on(tier_mutation_peer_handler_case());
20431            })
20432            .expect("tier mutation peer handler test thread should spawn")
20433            .join()
20434            .expect("tier mutation peer handler test thread should complete");
20435    }
20436
20437    #[cfg(feature = "test-util")]
20438    async fn tier_mutation_peer_handler_case() {
20439        let temp_dir = tempfile::tempdir().expect("create temp store dir");
20440        let (_ctx, store, _shutdown) =
20441            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "tier-mutation-peer-handler", &[4])).await;
20442        register_mock_tier(&store.tier_config_mgr(), "COLD-A").await;
20443        let (candidate_digest, config_etag) = {
20444            let tier_config_mgr = store.tier_config_mgr();
20445            let manager = tier_config_mgr.read().await;
20446            let candidate_digest = tier_config_candidate_digest(&manager).expect("peer commit candidate digest should build");
20447            manager
20448                .save_tiering_config(store.clone())
20449                .await
20450                .expect("peer commit config fixture should persist");
20451            let config_info = store
20452                .get_object_info(
20453                    RUSTFS_META_BUCKET,
20454                    &format!("{}/{}", com::CONFIG_PREFIX, TIER_CONFIG_FILE),
20455                    &ObjectOptions::default(),
20456                )
20457                .await
20458                .expect("peer commit config fixture should load");
20459            (candidate_digest, config_info.etag.expect("peer commit config should carry an ETag"))
20460        };
20461        let mutation_id = uuid::Uuid::new_v4();
20462        let mut intent = tier_mutation_peer_test_intent(mutation_id, "COLD-A", candidate_digest);
20463        intent.old_config_etag = Some(config_etag.clone());
20464        let prepare_payload = intent.encode().expect("prepare intent should encode");
20465
20466        let prepared = handle_tier_mutation_peer_request(
20467            store.clone(),
20468            TIER_MUTATION_RPC_PROTOCOL_VERSION,
20469            TierMutationRpcPhase::Prepare,
20470            mutation_id,
20471            &prepare_payload,
20472        )
20473        .await
20474        .expect("first prepare should create the peer intent");
20475        assert!(prepared.applied);
20476        assert_eq!(prepared.state, TierMutationPeerState::Prepared);
20477        let blocked = match TierConfigMgr::acquire_operation_lease(&store.tier_config_mgr(), "COLD-A").await {
20478            Ok(_) => panic!("prepared peer mutation should block new tier operation leases"),
20479            Err(err) => err,
20480        };
20481        assert!(
20482            blocked.message.contains("being replaced"),
20483            "prepared peer mutation should reuse the existing blocked-tier error: {blocked}"
20484        );
20485
20486        let retried_prepare = handle_tier_mutation_peer_request(
20487            store.clone(),
20488            TIER_MUTATION_RPC_PROTOCOL_VERSION,
20489            TierMutationRpcPhase::Prepare,
20490            mutation_id,
20491            &prepare_payload,
20492        )
20493        .await
20494        .expect("same prepare retry should be idempotent");
20495        assert!(!retried_prepare.applied);
20496        assert_eq!(retried_prepare.state, TierMutationPeerState::Prepared);
20497        let retried_blocked = match TierConfigMgr::acquire_operation_lease(&store.tier_config_mgr(), "COLD-A").await {
20498            Ok(_) => panic!("prepared retry should keep blocking new tier operation leases"),
20499            Err(err) => err,
20500        };
20501        assert!(
20502            retried_blocked.message.contains("being replaced"),
20503            "prepared retry should keep the existing blocked-tier error: {retried_blocked}"
20504        );
20505
20506        let mismatched_commit = handle_tier_mutation_peer_request(
20507            store.clone(),
20508            TIER_MUTATION_RPC_PROTOCOL_VERSION,
20509            TierMutationRpcPhase::Commit,
20510            mutation_id,
20511            b"not-the-current-etag",
20512        )
20513        .await
20514        .expect_err("commit with a mismatched config proof must fail closed");
20515        assert!(matches!(mismatched_commit, TierMutationPeerError::CommitProofMismatch));
20516
20517        register_mock_tier(&store.tier_config_mgr(), "COLD-C").await;
20518        let bad_digest_id = uuid::Uuid::new_v4();
20519        let mut bad_digest_intent = tier_mutation_peer_test_intent(bad_digest_id, "COLD-C", [9; 32]);
20520        bad_digest_intent.old_config_etag = Some(config_etag.clone());
20521        handle_tier_mutation_peer_request(
20522            store.clone(),
20523            TIER_MUTATION_RPC_PROTOCOL_VERSION,
20524            TierMutationRpcPhase::Prepare,
20525            bad_digest_id,
20526            &bad_digest_intent.encode().expect("bad digest prepare intent should encode"),
20527        )
20528        .await
20529        .expect("bad digest prepare should install a prepared intent");
20530        let mismatched_digest_commit = handle_tier_mutation_peer_request(
20531            store.clone(),
20532            TIER_MUTATION_RPC_PROTOCOL_VERSION,
20533            TierMutationRpcPhase::Commit,
20534            bad_digest_id,
20535            config_etag.as_bytes(),
20536        )
20537        .await
20538        .expect_err("a correct ETag with a mismatched candidate digest must fail closed");
20539        assert!(matches!(mismatched_digest_commit, TierMutationPeerError::CommitProofMismatch));
20540        let bad_digest_loaded = load_tier_mutation_intent_record(store.clone(), bad_digest_id)
20541            .await
20542            .expect("mismatched digest must leave the prepared intent durable");
20543        assert_eq!(bad_digest_loaded.state, TierMutationIntentState::Prepared);
20544        let bad_digest_blocked = match TierConfigMgr::acquire_operation_lease(&store.tier_config_mgr(), "COLD-C").await {
20545            Ok(_) => panic!("mismatched digest must retain the prepared runtime fence"),
20546            Err(err) => err,
20547        };
20548        assert!(bad_digest_blocked.message.contains("being replaced"), "{bad_digest_blocked}");
20549        handle_tier_mutation_peer_request(
20550            store.clone(),
20551            TIER_MUTATION_RPC_PROTOCOL_VERSION,
20552            TierMutationRpcPhase::Abort,
20553            bad_digest_id,
20554            &bad_digest_intent.encode().expect("bad digest abort identity should encode"),
20555        )
20556        .await
20557        .expect("the negative digest proof fixture should clean up through abort");
20558
20559        let committed = handle_tier_mutation_peer_request(
20560            store.clone(),
20561            TIER_MUTATION_RPC_PROTOCOL_VERSION,
20562            TierMutationRpcPhase::Commit,
20563            mutation_id,
20564            config_etag.as_bytes(),
20565        )
20566        .await
20567        .expect("commit should advance the prepared peer intent");
20568        assert!(committed.applied);
20569        assert_eq!(committed.state, TierMutationPeerState::Committed);
20570        let committed_blocked = match TierConfigMgr::acquire_operation_lease(&store.tier_config_mgr(), "COLD-A").await {
20571            Ok(_) => panic!("committed peer mutation must remain blocked until local reload publishes the config"),
20572            Err(err) => err,
20573        };
20574        assert!(
20575            committed_blocked.message.contains("being replaced"),
20576            "committed peer mutation should keep the existing blocked-tier error: {committed_blocked}"
20577        );
20578
20579        let retried_commit = handle_tier_mutation_peer_request(
20580            store.clone(),
20581            TIER_MUTATION_RPC_PROTOCOL_VERSION,
20582            TierMutationRpcPhase::Commit,
20583            mutation_id,
20584            config_etag.as_bytes(),
20585        )
20586        .await
20587        .expect("same commit retry should be idempotent");
20588        assert!(!retried_commit.applied);
20589        assert_eq!(retried_commit.state, TierMutationPeerState::Committed);
20590        let retried_commit_blocked = match TierConfigMgr::acquire_operation_lease(&store.tier_config_mgr(), "COLD-A").await {
20591            Ok(_) => panic!("committed retry must keep the tier blocked until local reload"),
20592            Err(err) => err,
20593        };
20594        assert!(
20595            retried_commit_blocked.message.contains("being replaced"),
20596            "committed retry should keep the existing blocked-tier error: {retried_commit_blocked}"
20597        );
20598
20599        let delayed_prepare_retry = handle_tier_mutation_peer_request(
20600            store.clone(),
20601            TIER_MUTATION_RPC_PROTOCOL_VERSION,
20602            TierMutationRpcPhase::Prepare,
20603            mutation_id,
20604            &prepare_payload,
20605        )
20606        .await
20607        .expect("delayed duplicate prepare should report the durable committed state");
20608        assert!(!delayed_prepare_retry.applied);
20609        assert_eq!(delayed_prepare_retry.state, TierMutationPeerState::Committed);
20610        let delayed_prepare_blocked = match TierConfigMgr::acquire_operation_lease(&store.tier_config_mgr(), "COLD-A").await {
20611            Ok(_) => panic!("delayed committed prepare retry must preserve the committed runtime block"),
20612            Err(err) => err,
20613        };
20614        assert!(
20615            delayed_prepare_blocked.message.contains("being replaced"),
20616            "delayed committed prepare retry should keep the existing blocked-tier error: {delayed_prepare_blocked}"
20617        );
20618
20619        let loaded = load_tier_mutation_intent_record(store.clone(), mutation_id)
20620            .await
20621            .expect("committed peer intent should remain durable");
20622        assert_eq!(loaded.state, TierMutationIntentState::Committed);
20623        assert_eq!(loaded.committed_config_etag.as_deref(), Some(config_etag.as_str()));
20624
20625        store
20626            .tier_config_mgr()
20627            .read()
20628            .await
20629            .save_tiering_config(store.clone())
20630            .await
20631            .expect("tier config should persist for cleaned intent commit proof");
20632        let tier_config_info = store
20633            .get_object_info(
20634                RUSTFS_META_BUCKET,
20635                &format!("{}/{}", com::CONFIG_PREFIX, TIER_CONFIG_FILE),
20636                &ObjectOptions::default(),
20637            )
20638            .await
20639            .expect("tier config object info should load");
20640        let tier_config_etag = tier_config_info.etag.expect("tier config should carry an ETag");
20641        delete_tier_mutation_intent_record(store.clone(), mutation_id)
20642            .await
20643            .expect("simulate another node cleaning the shared committed peer intent");
20644        let cleaned_commit_retry = handle_tier_mutation_peer_request(
20645            store.clone(),
20646            TIER_MUTATION_RPC_PROTOCOL_VERSION,
20647            TierMutationRpcPhase::Commit,
20648            mutation_id,
20649            tier_config_etag.as_bytes(),
20650        )
20651        .await
20652        .expect("commit retry after durable cleanup should be idempotently terminal");
20653        assert!(!cleaned_commit_retry.applied);
20654        assert_eq!(cleaned_commit_retry.state, TierMutationPeerState::Committed);
20655        let cleaned_commit_blocked = match TierConfigMgr::acquire_operation_lease(&store.tier_config_mgr(), "COLD-A").await {
20656            Ok(_) => panic!("shared intent cleanup must not clear this node's committed runtime block"),
20657            Err(err) => err,
20658        };
20659        assert!(
20660            cleaned_commit_blocked.message.contains("being replaced"),
20661            "commit retry after shared cleanup should keep the existing blocked-tier error: {cleaned_commit_blocked}"
20662        );
20663        let mismatched_cleaned_commit = handle_tier_mutation_peer_request(
20664            store.clone(),
20665            TIER_MUTATION_RPC_PROTOCOL_VERSION,
20666            TierMutationRpcPhase::Commit,
20667            mutation_id,
20668            b"not-the-current-etag",
20669        )
20670        .await
20671        .expect_err("missing intent without a matching committed config ETag must fail closed");
20672        assert!(matches!(mismatched_cleaned_commit, TierMutationPeerError::Store(Error::ConfigNotFound)));
20673        let refresh_store = store.clone();
20674        let refresh_manager = store.tier_config_mgr();
20675        let refresh_worker = tokio::spawn(async move {
20676            TierConfigMgr::refresh_tier_config_handle_with(refresh_manager, refresh_store).await;
20677        });
20678        tokio::time::timeout(Duration::from_secs(5), async {
20679            loop {
20680                if let Ok(lease) = TierConfigMgr::acquire_operation_lease(&store.tier_config_mgr(), "COLD-A").await {
20681                    drop(lease);
20682                    break;
20683                }
20684                tokio::task::yield_now().await;
20685            }
20686        })
20687        .await
20688        .expect("matching cleaned commit should wake the refresh worker and clear the committed fence");
20689        refresh_worker.abort();
20690        let _ = refresh_worker.await;
20691
20692        let abort_id = uuid::Uuid::new_v4();
20693        register_mock_tier(&store.tier_config_mgr(), "COLD-B").await;
20694        store
20695            .tier_config_mgr()
20696            .read()
20697            .await
20698            .save_tiering_config(store.clone())
20699            .await
20700            .expect("abort target tier config should persist");
20701        let abort_config_info = store
20702            .get_object_info(
20703                RUSTFS_META_BUCKET,
20704                &format!("{}/{}", com::CONFIG_PREFIX, TIER_CONFIG_FILE),
20705                &ObjectOptions::default(),
20706            )
20707            .await
20708            .expect("abort target config metadata should load");
20709        let abort_config_etag = abort_config_info.etag.expect("abort target config should carry an ETag");
20710        let mut abort_intent = tier_mutation_peer_test_intent(abort_id, "COLD-B", [4; 32]);
20711        abort_intent.old_config_etag = Some(abort_config_etag);
20712        let abort_prepare_payload = abort_intent.encode().expect("abort prepare intent should encode");
20713        handle_tier_mutation_peer_request(
20714            store.clone(),
20715            TIER_MUTATION_RPC_PROTOCOL_VERSION,
20716            TierMutationRpcPhase::Prepare,
20717            abort_id,
20718            &abort_prepare_payload,
20719        )
20720        .await
20721        .expect("abort target prepare should create the peer intent");
20722        let abort_blocked = match TierConfigMgr::acquire_operation_lease(&store.tier_config_mgr(), "COLD-B").await {
20723            Ok(_) => panic!("abort target prepare should block new tier operation leases"),
20724            Err(err) => err,
20725        };
20726        assert!(
20727            abort_blocked.message.contains("being replaced"),
20728            "abort target prepare should reuse the existing blocked-tier error: {abort_blocked}"
20729        );
20730
20731        let aborted = handle_tier_mutation_peer_request(
20732            store.clone(),
20733            TIER_MUTATION_RPC_PROTOCOL_VERSION,
20734            TierMutationRpcPhase::Abort,
20735            abort_id,
20736            &abort_prepare_payload,
20737        )
20738        .await
20739        .expect("abort should advance the prepared peer intent");
20740        assert!(aborted.applied);
20741        assert_eq!(aborted.state, TierMutationPeerState::Aborted);
20742        TierConfigMgr::acquire_operation_lease(&store.tier_config_mgr(), "COLD-B")
20743            .await
20744            .expect("durable abort must clear the peer runtime fence immediately");
20745
20746        let retried_abort = handle_tier_mutation_peer_request(
20747            store.clone(),
20748            TIER_MUTATION_RPC_PROTOCOL_VERSION,
20749            TierMutationRpcPhase::Abort,
20750            abort_id,
20751            &abort_prepare_payload,
20752        )
20753        .await
20754        .expect("same abort retry should be idempotent before recovery cleanup");
20755        assert!(!retried_abort.applied);
20756        assert_eq!(retried_abort.state, TierMutationPeerState::Aborted);
20757    }
20758
20759    #[cfg(feature = "test-util")]
20760    #[tokio::test]
20761    #[serial_test::serial(storage_class_env)]
20762    async fn late_abort_after_config_commit_keeps_fence_until_commit_recovery() {
20763        let temp_dir = tempfile::tempdir().expect("create temp store dir");
20764        let (_ctx, store, _shutdown) =
20765            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "tier-mutation-late-abort", &[4])).await;
20766        register_mock_tier(&store.tier_config_mgr(), "COLD-A").await;
20767        store
20768            .tier_config_mgr()
20769            .read()
20770            .await
20771            .save_tiering_config(store.clone())
20772            .await
20773            .expect("base tier config should persist");
20774        let base_config_info = store
20775            .get_object_info(
20776                RUSTFS_META_BUCKET,
20777                &format!("{}/{}", com::CONFIG_PREFIX, TIER_CONFIG_FILE),
20778                &ObjectOptions::default(),
20779            )
20780            .await
20781            .expect("base tier config metadata should load");
20782        let base_etag = base_config_info.etag.expect("base tier config should carry an ETag");
20783
20784        register_mock_tier(&store.tier_config_mgr(), "COLD-B").await;
20785        let candidate_digest = {
20786            let manager = store.tier_config_mgr();
20787            let manager = manager.read().await;
20788            tier_config_candidate_digest(&manager).expect("candidate digest should build")
20789        };
20790        let mutation_id = uuid::Uuid::new_v4();
20791        let mut intent = tier_mutation_peer_test_intent(mutation_id, "COLD-B", candidate_digest);
20792        intent.old_config_etag = Some(base_etag.clone());
20793        let prepare_payload = intent.encode().expect("late abort prepare intent should encode");
20794        handle_tier_mutation_peer_request(
20795            store.clone(),
20796            TIER_MUTATION_RPC_PROTOCOL_VERSION,
20797            TierMutationRpcPhase::Prepare,
20798            mutation_id,
20799            &prepare_payload,
20800        )
20801        .await
20802        .expect("prepare should install the runtime fence");
20803
20804        store
20805            .tier_config_mgr()
20806            .read()
20807            .await
20808            .save_tiering_config(store.clone())
20809            .await
20810            .expect("candidate tier config should persist before the late abort");
20811        let committed_config_info = store
20812            .get_object_info(
20813                RUSTFS_META_BUCKET,
20814                &format!("{}/{}", com::CONFIG_PREFIX, TIER_CONFIG_FILE),
20815                &ObjectOptions::default(),
20816            )
20817            .await
20818            .expect("committed tier config metadata should load");
20819        let committed_etag = committed_config_info
20820            .etag
20821            .expect("committed tier config should carry an ETag");
20822        assert_ne!(committed_etag, base_etag);
20823
20824        let late_abort = handle_tier_mutation_peer_request(
20825            store.clone(),
20826            TIER_MUTATION_RPC_PROTOCOL_VERSION,
20827            TierMutationRpcPhase::Abort,
20828            mutation_id,
20829            &prepare_payload,
20830        )
20831        .await
20832        .expect_err("an abort after the candidate config commit must fail closed");
20833        assert!(matches!(late_abort, TierMutationPeerError::AbortProofMismatch));
20834        let prepared = load_tier_mutation_intent_record(store.clone(), mutation_id)
20835            .await
20836            .expect("rejected late abort must retain the prepared intent");
20837        assert_eq!(prepared.state, TierMutationIntentState::Prepared);
20838        let blocked = match TierConfigMgr::acquire_operation_lease(&store.tier_config_mgr(), "COLD-B").await {
20839            Ok(_) => panic!("rejected late abort must retain the runtime fence"),
20840            Err(err) => err,
20841        };
20842        assert!(blocked.message.contains("being replaced"), "{blocked}");
20843
20844        let committed = handle_tier_mutation_peer_request(
20845            store.clone(),
20846            TIER_MUTATION_RPC_PROTOCOL_VERSION,
20847            TierMutationRpcPhase::Commit,
20848            mutation_id,
20849            committed_etag.as_bytes(),
20850        )
20851        .await
20852        .expect("matching commit should converge the rejected late abort fixture");
20853        assert!(committed.applied);
20854        assert_eq!(committed.state, TierMutationPeerState::Committed);
20855
20856        let refresh_store = store.clone();
20857        let refresh_manager = store.tier_config_mgr();
20858        let refresh_worker = tokio::spawn(async move {
20859            TierConfigMgr::refresh_tier_config_handle_with(refresh_manager, refresh_store).await;
20860        });
20861        tokio::time::timeout(Duration::from_secs(5), async {
20862            loop {
20863                if let Ok(lease) = TierConfigMgr::acquire_operation_lease(&store.tier_config_mgr(), "COLD-B").await {
20864                    drop(lease);
20865                    break;
20866                }
20867                tokio::task::yield_now().await;
20868            }
20869        })
20870        .await
20871        .expect("commit recovery should publish before clearing the late-abort fence");
20872        refresh_worker.abort();
20873        let _ = refresh_worker.await;
20874    }
20875
20876    #[cfg(feature = "test-util")]
20877    #[tokio::test]
20878    #[serial_test::serial(storage_class_env)]
20879    async fn missing_record_commit_promotes_prepared_fence_until_worker_publish() {
20880        let temp_dir = tempfile::tempdir().expect("create temp store dir");
20881        let (_ctx, store, _shutdown) =
20882            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "tier-mutation-missing-record-commit", &[4]))
20883                .await;
20884        register_mock_tier(&store.tier_config_mgr(), "COLD-A").await;
20885        let tier_config_mgr = store.tier_config_mgr();
20886        let candidate_digest = {
20887            let manager = tier_config_mgr.read().await;
20888            let digest = tier_config_candidate_digest(&manager).expect("candidate digest should build");
20889            manager
20890                .save_tiering_config(store.clone())
20891                .await
20892                .expect("candidate config should persist");
20893            digest
20894        };
20895        let config_info = store
20896            .get_object_info(
20897                RUSTFS_META_BUCKET,
20898                &format!("{}/{}", com::CONFIG_PREFIX, TIER_CONFIG_FILE),
20899                &ObjectOptions::default(),
20900            )
20901            .await
20902            .expect("candidate config metadata should load");
20903        let config_etag = config_info.etag.expect("candidate config should carry an ETag");
20904        let mutation_id = uuid::Uuid::new_v4();
20905        let intent = tier_mutation_peer_test_intent(mutation_id, "COLD-A", candidate_digest);
20906        handle_tier_mutation_peer_request(
20907            store.clone(),
20908            TIER_MUTATION_RPC_PROTOCOL_VERSION,
20909            TierMutationRpcPhase::Prepare,
20910            mutation_id,
20911            &intent.encode().expect("prepare intent should encode"),
20912        )
20913        .await
20914        .expect("prepare should install the runtime fence");
20915        delete_tier_mutation_intent_record(store.clone(), mutation_id)
20916            .await
20917            .expect("simulate shared intent cleanup before the local commit arrives");
20918
20919        let committed = handle_tier_mutation_peer_request(
20920            store.clone(),
20921            TIER_MUTATION_RPC_PROTOCOL_VERSION,
20922            TierMutationRpcPhase::Commit,
20923            mutation_id,
20924            config_etag.as_bytes(),
20925        )
20926        .await
20927        .expect("matching commit after shared cleanup should be terminal");
20928        assert!(!committed.applied);
20929        assert_eq!(committed.state, TierMutationPeerState::Committed);
20930        let blocked = match TierConfigMgr::acquire_operation_lease(&tier_config_mgr, "COLD-A").await {
20931            Ok(_) => panic!("the promoted committed fence must block old-generation leases"),
20932            Err(err) => err,
20933        };
20934        assert!(blocked.message.contains("being replaced"), "{blocked}");
20935
20936        let refresh_store = store.clone();
20937        let refresh_manager = tier_config_mgr.clone();
20938        let refresh_worker = tokio::spawn(async move {
20939            TierConfigMgr::refresh_tier_config_handle_with(refresh_manager, refresh_store).await;
20940        });
20941        tokio::time::timeout(Duration::from_secs(5), async {
20942            loop {
20943                if let Ok(lease) = TierConfigMgr::acquire_operation_lease(&tier_config_mgr, "COLD-A").await {
20944                    drop(lease);
20945                    break;
20946                }
20947                tokio::task::yield_now().await;
20948            }
20949        })
20950        .await
20951        .expect("the commit notification should drive publish and clear the promoted fence");
20952        refresh_worker.abort();
20953        let _ = refresh_worker.await;
20954    }
20955
20956    #[cfg(feature = "test-util")]
20957    #[tokio::test]
20958    #[serial_test::serial(storage_class_env)]
20959    async fn tier_mutation_peer_handler_rejects_conflicting_prepare_without_overwrite() {
20960        let temp_dir = tempfile::tempdir().expect("create temp store dir");
20961        let (_ctx, store, _shutdown) =
20962            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "tier-mutation-peer-conflict", &[4])).await;
20963        let mutation_id = uuid::Uuid::new_v4();
20964        let intent = tier_mutation_peer_test_intent(mutation_id, "COLD-A", [3; 32]);
20965        let prepare_payload = intent.encode().expect("prepare intent should encode");
20966        handle_tier_mutation_peer_request(
20967            store.clone(),
20968            TIER_MUTATION_RPC_PROTOCOL_VERSION,
20969            TierMutationRpcPhase::Prepare,
20970            mutation_id,
20971            &prepare_payload,
20972        )
20973        .await
20974        .expect("first prepare should create the peer intent");
20975
20976        let conflicting = tier_mutation_peer_test_intent(mutation_id, "COLD-A", [4; 32]);
20977        let conflicting_payload = conflicting.encode().expect("conflicting intent should encode");
20978        let conflict = handle_tier_mutation_peer_request(
20979            store.clone(),
20980            TIER_MUTATION_RPC_PROTOCOL_VERSION,
20981            TierMutationRpcPhase::Prepare,
20982            mutation_id,
20983            &conflicting_payload,
20984        )
20985        .await
20986        .expect_err("conflicting prepare must fail closed");
20987        assert!(matches!(conflict, TierMutationPeerError::ConflictingIntent));
20988
20989        let loaded = load_tier_mutation_intent_record(store, mutation_id)
20990            .await
20991            .expect("conflicting prepare must not overwrite the first record");
20992        assert_eq!(loaded.candidate_digest, [3; 32]);
20993        assert_eq!(loaded.state, TierMutationIntentState::Prepared);
20994    }
20995
20996    #[cfg(feature = "test-util")]
20997    #[tokio::test]
20998    #[serial_test::serial(storage_class_env)]
20999    async fn tier_mutation_intent_record_scan_paginates_exact_limit() {
21000        let temp_dir = tempfile::tempdir().expect("create temp store dir");
21001        let (_ctx, store, _shutdown) =
21002            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "tier-mutation-intent-scan-page", &[4])).await;
21003        let build_intent = |mutation_id: uuid::Uuid, tier_name: &str| TierMutationIntent {
21004            mutation_id,
21005            revision: 1,
21006            kind: TierMutationIntentKind::Edit,
21007            state: TierMutationIntentState::Prepared,
21008            old_config_etag: Some("old-etag".to_string()),
21009            committed_config_etag: None,
21010            candidate_digest: [3; 32],
21011            affected_targets: vec![TierMutationIntentTarget {
21012                tier_name: tier_name.to_string(),
21013                old_backend_identity: Some([1; 32]),
21014                new_backend_identity: Some([2; 32]),
21015            }],
21016            expires_at_unix_nanos: 1_780_000_000_000_000_000,
21017        };
21018        let intent_ids = vec![
21019            uuid::Uuid::parse_str("11345678-1234-5678-9abc-def012345678").expect("first uuid should parse"),
21020            uuid::Uuid::parse_str("22345678-1234-5678-9abc-def012345678").expect("second uuid should parse"),
21021            uuid::Uuid::parse_str("33345678-1234-5678-9abc-def012345678").expect("third uuid should parse"),
21022        ];
21023        for (index, mutation_id) in intent_ids.iter().copied().enumerate() {
21024            let intent = build_intent(mutation_id, &format!("COLD-{index}"));
21025            save_tier_mutation_intent_record(store.clone(), &intent)
21026                .await
21027                .expect("tier mutation intent record should persist");
21028        }
21029
21030        let exact_page = list_tier_mutation_intent_records(store.clone(), 3, None)
21031            .await
21032            .expect("exact full page should scan");
21033        assert_eq!(exact_page.scanned, 3);
21034        assert_eq!(exact_page.intents.len(), 3);
21035        assert_eq!(exact_page.failed, 0);
21036        assert!(!exact_page.truncated);
21037        assert_eq!(exact_page.next_marker, None);
21038
21039        let first_page = list_tier_mutation_intent_records(store.clone(), 2, None)
21040            .await
21041            .expect("first page should scan");
21042        assert_eq!(first_page.scanned, 2);
21043        assert_eq!(first_page.intents.len(), 2);
21044        assert_eq!(first_page.failed, 0);
21045        assert!(first_page.truncated);
21046        assert!(first_page.next_marker.is_some());
21047
21048        let second_page = list_tier_mutation_intent_records(store, 2, first_page.next_marker)
21049            .await
21050            .expect("second page should scan");
21051        let mut loaded_ids: Vec<_> = first_page
21052            .intents
21053            .into_iter()
21054            .chain(second_page.intents.into_iter())
21055            .map(|intent| intent.mutation_id)
21056            .collect();
21057        loaded_ids.sort();
21058
21059        assert_eq!(second_page.scanned, 1);
21060        assert_eq!(second_page.failed, 0);
21061        assert!(!second_page.truncated);
21062        assert_eq!(second_page.next_marker, None);
21063        assert_eq!(loaded_ids, intent_ids);
21064    }
21065
21066    #[cfg(feature = "test-util")]
21067    fn transition_mutation_measurement(observations: &[TransitionTransactionMutationObservation]) -> (usize, usize, u128) {
21068        (
21069            observations.len(),
21070            observations.iter().map(|observation| observation.encoded_bytes).sum(),
21071            observations.iter().map(|observation| observation.elapsed.as_micros()).sum(),
21072        )
21073    }
21074
21075    #[cfg(feature = "test-util")]
21076    #[tokio::test]
21077    #[serial_test::serial(storage_class_env)]
21078    async fn compact_transition_transactions_halve_success_path_quorum_mutations() {
21079        let temp_dir = tempfile::tempdir().expect("create transition mutation measurement dir");
21080        let (ctx, store, shutdown) =
21081            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "transition-mutation-measurement", &[4])).await;
21082        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
21083        let tier_name = "MUTATION-MEASURE";
21084        register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
21085        let bucket = "transition-mutation-measurement";
21086        store
21087            .make_bucket(bucket, &MakeBucketOptions::default())
21088            .await
21089            .expect("measurement bucket should be created");
21090
21091        let run_case = |profile: &'static str, size: usize| {
21092            let store = store.clone();
21093            async move {
21094                let object = format!("{profile}-{size}.bin");
21095                let mut reader = PutObjReader::from_vec(vec![b'm'; size]);
21096                let original = store
21097                    .put_object(bucket, &object, &mut reader, &ObjectOptions::default())
21098                    .await
21099                    .expect("measurement source should be written");
21100                let probe = TransitionTransactionMutationProbe::install(bucket, &object);
21101                store
21102                    .transition_object(
21103                        bucket,
21104                        &object,
21105                        &ObjectOptions {
21106                            transition: TransitionOptions {
21107                                status: TRANSITION_PENDING.to_string(),
21108                                tier: tier_name.to_string(),
21109                                etag: original.etag.clone().expect("measurement source should have an ETag"),
21110                                ..Default::default()
21111                            },
21112                            version_id: original.version_id.map(|version| version.to_string()),
21113                            mod_time: original.mod_time,
21114                            ..Default::default()
21115                        },
21116                    )
21117                    .await
21118                    .expect("measurement transition should commit");
21119                probe.observations()
21120            }
21121        };
21122
21123        let sizes = [4 * 1024, 1024 * 1024];
21124        let mut legacy = Vec::with_capacity(sizes.len());
21125        for size in sizes {
21126            legacy.push((size, run_case("legacy", size).await));
21127        }
21128
21129        let compaction_proof = install_transition_transaction_compaction_fleet_proof_for_test("object-transaction-fencing-test");
21130        for ((size, legacy_observations), compact_size) in legacy.into_iter().zip(sizes) {
21131            assert_eq!(size, compact_size);
21132            let compact_observations = run_case("compact", size).await;
21133            let legacy_measurement = transition_mutation_measurement(&legacy_observations);
21134            let compact_measurement = transition_mutation_measurement(&compact_observations);
21135            assert_eq!(legacy_measurement.0, 6, "legacy success should use five saves and one delete");
21136            assert_eq!(compact_measurement.0, 3, "compact success should use two saves and one delete");
21137            assert!(
21138                compact_measurement.1 < legacy_measurement.1,
21139                "compact transaction bodies should write fewer aggregate bytes"
21140            );
21141            assert_eq!(
21142                legacy_observations
21143                    .iter()
21144                    .map(|observation| (observation.kind, observation.previous_state, observation.state))
21145                    .collect::<Vec<_>>(),
21146                vec![
21147                    (TransitionTransactionMutationKind::Create, None, TransitionTransactionState::UploadStarted),
21148                    (
21149                        TransitionTransactionMutationKind::CompareAndSave,
21150                        Some(TransitionTransactionState::UploadStarted),
21151                        TransitionTransactionState::UploadOutcomeUnknown,
21152                    ),
21153                    (
21154                        TransitionTransactionMutationKind::CompareAndSave,
21155                        Some(TransitionTransactionState::UploadOutcomeUnknown),
21156                        TransitionTransactionState::Uploaded,
21157                    ),
21158                    (
21159                        TransitionTransactionMutationKind::CompareAndSave,
21160                        Some(TransitionTransactionState::Uploaded),
21161                        TransitionTransactionState::LocalCommitStarted,
21162                    ),
21163                    (
21164                        TransitionTransactionMutationKind::CompareAndSave,
21165                        Some(TransitionTransactionState::LocalCommitStarted),
21166                        TransitionTransactionState::Committed,
21167                    ),
21168                    (
21169                        TransitionTransactionMutationKind::Delete,
21170                        Some(TransitionTransactionState::Committed),
21171                        TransitionTransactionState::Committed,
21172                    ),
21173                ]
21174            );
21175            assert_eq!(
21176                compact_observations
21177                    .iter()
21178                    .map(|observation| (observation.kind, observation.previous_state, observation.state))
21179                    .collect::<Vec<_>>(),
21180                vec![
21181                    (
21182                        TransitionTransactionMutationKind::Create,
21183                        None,
21184                        TransitionTransactionState::UploadOutcomeUnknown,
21185                    ),
21186                    (
21187                        TransitionTransactionMutationKind::CompareAndSave,
21188                        Some(TransitionTransactionState::UploadOutcomeUnknown),
21189                        TransitionTransactionState::LocalCommitStarted,
21190                    ),
21191                    (
21192                        TransitionTransactionMutationKind::Delete,
21193                        Some(TransitionTransactionState::LocalCommitStarted),
21194                        TransitionTransactionState::LocalCommitStarted,
21195                    ),
21196                ]
21197            );
21198            assert!(legacy_observations.iter().all(|observation| observation.succeeded));
21199            assert!(compact_observations.iter().all(|observation| observation.succeeded));
21200            println!(
21201                "transition_mutation_measurement,profile=legacy,size={size},mutations={},encoded_bytes={},latency_us={},quorum_operations={}",
21202                legacy_measurement.0, legacy_measurement.1, legacy_measurement.2, legacy_measurement.0
21203            );
21204            println!(
21205                "transition_mutation_measurement,profile=compact,size={size},mutations={},encoded_bytes={},latency_us={},quorum_operations={}",
21206                compact_measurement.0, compact_measurement.1, compact_measurement.2, compact_measurement.0
21207            );
21208        }
21209        assert_eq!(transition_transaction_record_count(store.clone()).await, 0);
21210        let admitted = acquire_transition_transaction_compaction_fleet_proof()
21211            .expect("published homogeneous proof should admit one compact writer");
21212        assert!(transition_transaction_compaction_fleet_proof_matches(&admitted));
21213        drop(compaction_proof);
21214        assert!(
21215            !transition_transaction_compaction_fleet_proof_matches(&admitted),
21216            "revocation must fence a writer admitted by the previous process-epoch snapshot"
21217        );
21218        drop(admitted);
21219        assert!(
21220            acquire_transition_transaction_compaction_fleet_proof().is_none(),
21221            "revoking the homogeneous proof must restore the legacy writer profile"
21222        );
21223        shutdown.cancel();
21224    }
21225
21226    #[cfg(feature = "test-util")]
21227    #[tokio::test]
21228    #[serial_test::serial(storage_class_env)]
21229    async fn compact_transition_kill_points_preserve_the_only_remote_owner() {
21230        let temp_dir = tempfile::tempdir().expect("create compact transition kill-point dir");
21231        let (ctx, store, shutdown) =
21232            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "compact-transition-kill-points", &[4])).await;
21233        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
21234        let tier_name = "COMPACT-KILL";
21235        let backend = register_transition_reconcile_test_tier(&ctx.tier_config_mgr(), tier_name).await;
21236        let _tier_lease = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
21237            .await
21238            .expect("mock tier lease should remain available during recovery");
21239        let bucket = "compact-transition-kill-points";
21240        store
21241            .make_bucket(bucket, &MakeBucketOptions::default())
21242            .await
21243            .expect("kill-point bucket should be created");
21244        let _compaction_proof = install_transition_transaction_compaction_fleet_proof_for_test("object-transaction-fencing-test");
21245
21246        for (index, point, expected_state, expected_revision, expected_committed, expected_recovered) in [
21247            (
21248                0,
21249                TransitionTransactionKillPoint::PrePutFence,
21250                TransitionTransactionState::UploadOutcomeUnknown,
21251                1,
21252                false,
21253                true,
21254            ),
21255            (
21256                1,
21257                TransitionTransactionKillPoint::UploadBeforeCommitFence,
21258                TransitionTransactionState::UploadOutcomeUnknown,
21259                1,
21260                false,
21261                true,
21262            ),
21263            (
21264                2,
21265                TransitionTransactionKillPoint::LocalCommitBeforeDelete,
21266                TransitionTransactionState::LocalCommitStarted,
21267                2,
21268                true,
21269                true,
21270            ),
21271            (
21272                3,
21273                TransitionTransactionKillPoint::CommitFenceBeforeLocalCommit,
21274                TransitionTransactionState::LocalCommitStarted,
21275                2,
21276                false,
21277                false,
21278            ),
21279        ] {
21280            let object = format!("kill-point-{index}.bin");
21281            let payload = vec![b'k' + u8::try_from(index).expect("small case index should fit u8"); 64 * 1024];
21282            let mut reader = PutObjReader::from_vec(payload.clone());
21283            let source = store
21284                .put_object(bucket, &object, &mut reader, &ObjectOptions::default())
21285                .await
21286                .expect("kill-point source should be written");
21287            let remote_before = backend.object_count().await;
21288            let barrier = TransitionTransactionKillPointBarrier::install(bucket, &object, point);
21289            let transition_store = store.clone();
21290            let transition_object = object.clone();
21291            let transition = tokio::spawn(async move {
21292                transition_store
21293                    .transition_object(
21294                        bucket,
21295                        &transition_object,
21296                        &ObjectOptions {
21297                            transition: TransitionOptions {
21298                                status: TRANSITION_PENDING.to_string(),
21299                                tier: tier_name.to_string(),
21300                                etag: source.etag.clone().expect("kill-point source should have an ETag"),
21301                                ..Default::default()
21302                            },
21303                            version_id: source.version_id.map(|version| version.to_string()),
21304                            mod_time: source.mod_time,
21305                            ..Default::default()
21306                        },
21307                    )
21308                    .await
21309            });
21310            barrier.wait_until_paused().await;
21311            transition.abort();
21312            assert!(
21313                transition
21314                    .await
21315                    .expect_err("kill-point transition should be cancelled")
21316                    .is_cancelled(),
21317                "kill-point transition should stop without unwinding"
21318            );
21319            drop(barrier);
21320
21321            let transaction = only_transition_transaction(store.clone()).await;
21322            assert_eq!((transaction.state, transaction.revision), (expected_state, expected_revision));
21323            let paused_source = store
21324                .get_object_info(
21325                    bucket,
21326                    &object,
21327                    &ObjectOptions {
21328                        metadata_cache_safe: false,
21329                        ..Default::default()
21330                    },
21331                )
21332                .await
21333                .expect("kill-point source metadata should remain readable");
21334            assert_eq!(
21335                paused_source.transitioned_object.status == rustfs_filemeta::TRANSITION_COMPLETE,
21336                expected_committed
21337            );
21338            let expected_remote_at_pause = remote_before + usize::from(point != TransitionTransactionKillPoint::PrePutFence);
21339            assert_eq!(backend.object_count().await, expected_remote_at_pause);
21340
21341            let stats = recover_transition_transaction_records_at(
21342                store.clone(),
21343                100,
21344                None,
21345                i128::from(transaction.not_after_unix_nanos) + 1,
21346            )
21347            .await
21348            .expect("kill-point transaction recovery should complete");
21349            if expected_recovered {
21350                assert_eq!((stats.scanned, stats.recovered, stats.retained, stats.failed), (1, 1, 0, 0));
21351                assert_eq!(transition_transaction_record_count(store.clone()).await, 0);
21352            } else {
21353                assert_eq!((stats.scanned, stats.recovered, stats.retained, stats.failed), (1, 0, 1, 0));
21354                assert_eq!(
21355                    transition_transaction_record_count(store.clone()).await,
21356                    1,
21357                    "an uncommitted local-commit fence must retain its exact remote owner"
21358                );
21359            }
21360            let expected_remote_after_recovery = if matches!(
21361                point,
21362                TransitionTransactionKillPoint::LocalCommitBeforeDelete
21363                    | TransitionTransactionKillPoint::CommitFenceBeforeLocalCommit
21364            ) {
21365                remote_before + 1
21366            } else {
21367                remote_before
21368            };
21369            assert_eq!(backend.object_count().await, expected_remote_after_recovery);
21370            assert_eq!(backend.remove_count().await, usize::from(index >= 1));
21371
21372            let mut restored = Vec::new();
21373            store
21374                .get_object_reader(bucket, &object, None, HeaderMap::new(), &ObjectOptions::default())
21375                .await
21376                .expect("kill-point source should remain readable through its authoritative location")
21377                .stream
21378                .read_to_end(&mut restored)
21379                .await
21380                .expect("kill-point source body should drain");
21381            assert_eq!(restored, payload);
21382
21383            if !expected_recovered {
21384                break;
21385            }
21386        }
21387        shutdown.cancel();
21388    }
21389
21390    #[cfg(feature = "test-util")]
21391    #[tokio::test]
21392    #[serial_test::serial(storage_class_env)]
21393    async fn cancelled_transition_cleanup_recovers_from_its_own_instance_transaction() {
21394        struct ResolverReset(Arc<std::sync::Mutex<Option<std::sync::Weak<crate::store::ECStore>>>>);
21395
21396        impl Drop for ResolverReset {
21397            fn drop(&mut self) {
21398                *self.0.lock().unwrap_or_else(std::sync::PoisonError::into_inner) = None;
21399            }
21400        }
21401
21402        let temp_a = tempfile::tempdir().expect("create transition store dir a");
21403        let temp_b = tempfile::tempdir().expect("create transition store dir b");
21404        let shutdown_a = CancellationToken::new();
21405        let shutdown_b = CancellationToken::new();
21406        shutdown_a.cancel();
21407        shutdown_b.cancel();
21408        let (ctx_a, store_a, shutdown_a) = without_storage_class_env(build_isolated_test_store_with_shutdown(
21409            temp_a.path(),
21410            "transition-cleanup-context-a",
21411            &[4],
21412            shutdown_a,
21413        ))
21414        .await;
21415        let (ctx_b, store_b, shutdown_b) = without_storage_class_env(build_isolated_test_store_with_shutdown(
21416            temp_b.path(),
21417            "transition-cleanup-context-b",
21418            &[4],
21419            shutdown_b,
21420        ))
21421        .await;
21422        assert!(shutdown_a.is_cancelled());
21423        assert!(shutdown_b.is_cancelled());
21424        crate::bucket::metadata_sys::init_bucket_metadata_sys(store_a.clone(), Vec::new()).await;
21425
21426        let resolver_target = Arc::new(std::sync::Mutex::new(Some(Arc::downgrade(&store_b))));
21427        let resolver_store = resolver_target.clone();
21428        assert!(
21429            set_object_store_resolver(Arc::new(move || {
21430                resolver_store
21431                    .lock()
21432                    .unwrap_or_else(std::sync::PoisonError::into_inner)
21433                    .as_ref()
21434                    .and_then(std::sync::Weak::upgrade)
21435            })),
21436            "the cross-context regression test must install the only process object-store resolver"
21437        );
21438        let _resolver_reset = ResolverReset(resolver_target);
21439        assert!(
21440            runtime_sources::object_store_handle().is_some_and(|store| Arc::ptr_eq(&store, &store_b)),
21441            "the process resolver must deliberately point at store B"
21442        );
21443
21444        let tier_name = "CROSSCTXA";
21445        let backend = register_transition_reconcile_test_tier(&ctx_a.tier_config_mgr(), tier_name).await;
21446        backend.set_put_remote_version(Some(uuid::Uuid::new_v4().to_string())).await;
21447        backend.reject_next_non_empty_remote_version_validation();
21448        let remove_barrier = backend.arm_failing_remove_barrier().await;
21449
21450        let bucket = "transition-cleanup-context-a";
21451        let object = "rejected-candidate.bin";
21452        store_a
21453            .make_bucket(bucket, &MakeBucketOptions::default())
21454            .await
21455            .expect("store A bucket should be created");
21456        let mut reader = PutObjReader::from_vec(b"cross-context rejected transition cleanup".repeat(1024));
21457        let original = store_a
21458            .put_object(bucket, object, &mut reader, &ObjectOptions::default())
21459            .await
21460            .expect("store A source object should be written");
21461        let opts = ObjectOptions {
21462            no_lock: true,
21463            transition: TransitionOptions {
21464                status: TRANSITION_PENDING.to_string(),
21465                tier: tier_name.to_string(),
21466                etag: original.etag.clone().expect("the source object should have an ETag"),
21467                ..Default::default()
21468            },
21469            version_id: original.version_id.map(|version| version.to_string()),
21470            mod_time: original.mod_time,
21471            ..Default::default()
21472        };
21473
21474        let cleanup_store_barrier = TransitionCleanupStoreBarrier::install();
21475        let transition_store = store_a.clone();
21476        let transition = tokio::spawn(async move { transition_store.transition_object(bucket, object, &opts).await });
21477        cleanup_store_barrier.wait_until_paused().await;
21478        transition.abort();
21479        assert!(
21480            transition
21481                .await
21482                .expect_err("the transition task should observe cancellation")
21483                .is_cancelled()
21484        );
21485
21486        remove_barrier.wait_until_paused().await;
21487        let transaction_counts = (
21488            transition_transaction_record_count(store_a.clone()).await,
21489            transition_transaction_record_count(store_b.clone()).await,
21490        );
21491        assert_eq!(
21492            transaction_counts,
21493            (1, 0),
21494            "the transition transaction must remain only on store A even while the process resolver points at store B"
21495        );
21496        let journal_counts = (
21497            tier_delete_journal_count(store_a.clone()).await,
21498            tier_delete_journal_count(store_b.clone()).await,
21499        );
21500        assert_eq!(
21501            journal_counts,
21502            (0, 0),
21503            "rejected transition uploads must not create a tier-delete journal"
21504        );
21505        assert_eq!(backend.object_count().await, 1, "failed cleanup should retain the remote candidate");
21506        remove_barrier.release();
21507        remove_barrier.wait_until_operation_dropped().await;
21508
21509        let recovered = recover_transition_transaction_records(store_a.clone(), 100, None)
21510            .await
21511            .expect("store A should recover its own cancelled-transition transaction");
21512        assert_eq!(
21513            (recovered.scanned, recovered.recovered, recovered.retained, recovered.failed),
21514            (1, 1, 0, 0)
21515        );
21516        assert_eq!(transition_transaction_record_count(store_a.clone()).await, 0);
21517        assert_eq!(transition_transaction_record_count(store_b.clone()).await, 0);
21518        assert_eq!(tier_delete_journal_count(store_a.clone()).await, 0);
21519        assert_eq!(tier_delete_journal_count(store_b.clone()).await, 0);
21520        assert_eq!(
21521            backend.object_count().await,
21522            0,
21523            "store A recovery should delete the exact remote candidate"
21524        );
21525        assert!(!Arc::ptr_eq(&ctx_a, &ctx_b), "the regression requires two distinct instance contexts");
21526    }
21527
21528    #[cfg(feature = "test-util")]
21529    #[tokio::test]
21530    #[serial_test::serial(storage_class_env)]
21531    async fn transition_transaction_recovery_expires_abandoned_attempt_at_budget_bound() {
21532        let temp_dir = tempfile::tempdir().expect("create temp store dir");
21533        let (ctx, store, _shutdown) = without_storage_class_env(build_isolated_test_store(
21534            temp_dir.path(),
21535            "transition-transaction-expired-attempt-budget",
21536            &[4],
21537        ))
21538        .await;
21539        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
21540
21541        let transaction = TransitionTransaction::new(TransitionTransactionInit {
21542            deployment_id: ctx.deployment_id().expect("test store should initialize deployment id"),
21543            transaction_id: uuid::Uuid::new_v4(),
21544            owner_epoch: uuid::Uuid::new_v4(),
21545            write_id: uuid::Uuid::new_v4(),
21546            source: TransitionSourceIdentity {
21547                bucket: "source-bucket".to_string(),
21548                object: "source-object".to_string(),
21549                version_id: Some(uuid::Uuid::new_v4()),
21550                data_dir: uuid::Uuid::new_v4(),
21551                mod_time_unix_nanos: 1_770_000_000_000_000_000,
21552                size: 42,
21553                etag: "source-etag".to_string(),
21554                version_mode: TransitionSourceVersionMode::Versioned,
21555            },
21556            tier_name: "UNUSEDABANDONEDTIER".to_string(),
21557            backend_fingerprint: [7; 32],
21558            not_after_unix_nanos: 1,
21559        })
21560        .expect("transaction should build");
21561        save_transition_transaction_record(store.clone(), &transaction)
21562            .await
21563            .expect("transaction record should persist");
21564
21565        let record_name =
21566            transition_transaction_record_object_name(transaction.transaction_id).expect("transaction record name should derive");
21567        let source = observe_recovery_source(
21568            store.clone(),
21569            &record_name,
21570            crate::bucket::lifecycle::transition_transaction::TRANSITION_TRANSACTION_SCHEMA,
21571        )
21572        .await
21573        .expect("transaction source generation should be observable");
21574        let mut control = IlmRecoveryControl::new(
21575            IlmRecoveryControlIdentity {
21576                protocol: IlmRecoveryProtocol::TransitionTransaction,
21577                canonical_source_path: record_name,
21578                stable_operation_identity: transaction.transaction_id.to_string(),
21579                record_class: "transition_transaction_v1".to_string(),
21580            },
21581            source.generation,
21582            IlmRecoveryClassification::Retrying,
21583            2_000_000_000,
21584            IlmRecoveryErrorCode::None,
21585        )
21586        .expect("recovery control should build");
21587        let mut now = 3_000_000_000;
21588        for _ in 1..MAX_RECOVERY_ATTEMPTS {
21589            now = now.max(control.next_attempt_at_unix_nanos.unwrap_or(now));
21590            control
21591                .claim("cancelled-or-timed-out-owner", uuid::Uuid::new_v4(), now, 1)
21592                .expect("abandoned attempt should claim");
21593            control
21594                .record_expired_attempt(now + 1)
21595                .expect("expired attempt should consume retry budget");
21596            now += 2;
21597        }
21598        now = now.max(control.next_attempt_at_unix_nanos.expect("last retry should have a backoff"));
21599        control
21600            .claim("cancelled-or-timed-out-owner", uuid::Uuid::new_v4(), now, 1)
21601            .expect("final abandoned attempt should claim");
21602        save_recovery_control_if_absent(store.clone(), &control)
21603            .await
21604            .expect("claimed recovery control should persist");
21605
21606        let stats = recover_transition_transaction_records_at(store.clone(), 100, None, i128::from(now + 1))
21607            .await
21608            .expect("recovery should account for the expired attempt");
21609        assert_eq!((stats.scanned, stats.recovered, stats.retained, stats.failed), (1, 0, 1, 0));
21610
21611        let control_id = transition_recovery_control_id(&transaction).expect("control id should derive");
21612        let persisted = load_recovery_control(store.clone(), IlmRecoveryProtocol::TransitionTransaction, &control_id)
21613            .await
21614            .expect("expired recovery control should remain inspectable");
21615        assert_eq!(persisted.control.classification, IlmRecoveryClassification::OperatorRequired);
21616        assert_eq!(persisted.control.attempt_count, u64::from(MAX_RECOVERY_ATTEMPTS));
21617        assert_eq!(persisted.control.consecutive_failure_count, MAX_RECOVERY_ATTEMPTS);
21618        assert_eq!(persisted.control.last_error_code, IlmRecoveryErrorCode::AttemptLeaseExpired);
21619        assert!(persisted.control.owner.is_none());
21620        assert_eq!(
21621            transition_transaction_record_count(store).await,
21622            1,
21623            "budget exhaustion must retain the source record"
21624        );
21625    }
21626
21627    #[cfg(feature = "test-util")]
21628    #[tokio::test]
21629    #[serial_test::serial(storage_class_env)]
21630    async fn transition_transaction_recovery_retains_active_uploaded_candidate_until_commit() {
21631        let temp_dir = tempfile::tempdir().expect("create temp store dir");
21632        let (ctx, store, _shutdown) =
21633            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "transition-transaction-active-uploaded", &[4]))
21634                .await;
21635        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
21636
21637        let tier_name = "TXACTIVEUPLOADED";
21638        let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
21639        let bucket = "transition-transaction-active-uploaded-bucket";
21640        let object = "object.bin";
21641        let payload = b"active Uploaded ownership must survive recovery until local commit".repeat(1024);
21642        store
21643            .make_bucket(bucket, &MakeBucketOptions::default())
21644            .await
21645            .expect("source bucket should be created");
21646        let mut reader = PutObjReader::from_vec(payload.clone());
21647        let source = store
21648            .put_object(bucket, object, &mut reader, &ObjectOptions::default())
21649            .await
21650            .expect("source object should be written");
21651        let barrier = TransitionUploadedCommitBarrier::install(bucket, object);
21652        let transition_store = store.clone();
21653        let transition = tokio::spawn(async move {
21654            transition_store
21655                .transition_object(
21656                    bucket,
21657                    object,
21658                    &ObjectOptions {
21659                        transition: TransitionOptions {
21660                            status: TRANSITION_PENDING.to_string(),
21661                            tier: tier_name.to_string(),
21662                            etag: source.etag.clone().expect("source object should have an ETag"),
21663                            ..Default::default()
21664                        },
21665                        version_id: source.version_id.map(|version_id| version_id.to_string()),
21666                        mod_time: source.mod_time,
21667                        ..Default::default()
21668                    },
21669                )
21670                .await
21671        });
21672        barrier.wait_until_paused().await;
21673
21674        let stats = recover_transition_transaction_records(store.clone(), 100, None)
21675            .await
21676            .expect("recovery should inspect the active Uploaded transaction");
21677        assert_eq!((stats.scanned, stats.recovered, stats.retained, stats.failed), (1, 0, 1, 0));
21678        assert_eq!(backend.object_count().await, 1, "active ownership must retain the remote candidate");
21679        assert_eq!(backend.remove_count().await, 0, "active ownership must fence remote DELETE");
21680        assert_eq!(transition_transaction_record_count(store.clone()).await, 1);
21681
21682        barrier.release();
21683        transition
21684            .await
21685            .expect("transition task should join")
21686            .expect("transition should commit after the barrier is released");
21687        assert_eq!(transition_transaction_record_count(store.clone()).await, 0);
21688        assert_eq!(backend.remove_count().await, 0);
21689
21690        let mut body = Vec::new();
21691        store
21692            .get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
21693            .await
21694            .expect("committed tier object should remain readable")
21695            .stream
21696            .read_to_end(&mut body)
21697            .await
21698            .expect("tier object body should drain");
21699        assert_eq!(body, payload);
21700    }
21701
21702    #[cfg(feature = "test-util")]
21703    #[tokio::test]
21704    #[serial_test::serial(storage_class_env)]
21705    async fn transition_transaction_recovery_deletes_expired_uploaded_remote_candidate() {
21706        let temp_dir = tempfile::tempdir().expect("create temp store dir");
21707        let (ctx, store, _shutdown) =
21708            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "transition-transaction-recovery", &[4])).await;
21709        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
21710
21711        let tier_name = "TXRECOVERY";
21712        let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
21713        let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
21714            .await
21715            .expect("tier lease should resolve")
21716            .backend_identity();
21717        let exact_version = uuid::Uuid::new_v4().to_string();
21718        let cases = [
21719            (
21720                "exact",
21721                exact_version.clone(),
21722                TransitionRemoteVersion::versioned(exact_version),
21723                TransitionSourceVersionMode::Versioned,
21724            ),
21725            (
21726                "suspended-null",
21727                "null".to_string(),
21728                TransitionRemoteVersion::versioned("null"),
21729                TransitionSourceVersionMode::VersionSuspended,
21730            ),
21731            (
21732                "known-unversioned",
21733                String::new(),
21734                TransitionRemoteVersion::unversioned(),
21735                TransitionSourceVersionMode::Unversioned,
21736            ),
21737        ];
21738        let mut expected_removes = Vec::new();
21739        let mut recovery_control_ids = Vec::new();
21740        for (case, put_version, remote_version, source_mode) in cases {
21741            let mut transaction = TransitionTransaction::new(TransitionTransactionInit {
21742                deployment_id: ctx.deployment_id().expect("test store should initialize deployment id"),
21743                transaction_id: uuid::Uuid::new_v4(),
21744                owner_epoch: uuid::Uuid::new_v4(),
21745                write_id: uuid::Uuid::new_v4(),
21746                source: TransitionSourceIdentity {
21747                    bucket: "source-bucket".to_string(),
21748                    object: format!("source-{case}"),
21749                    version_id: (source_mode == TransitionSourceVersionMode::Versioned).then(uuid::Uuid::new_v4),
21750                    data_dir: uuid::Uuid::new_v4(),
21751                    mod_time_unix_nanos: 1_770_000_000_000_000_000,
21752                    size: 42,
21753                    etag: "source-etag".to_string(),
21754                    version_mode: source_mode,
21755                },
21756                tier_name: tier_name.to_string(),
21757                backend_fingerprint: backend_identity,
21758                not_after_unix_nanos: 1,
21759            })
21760            .expect("transaction should build");
21761            transaction
21762                .advance(transaction.fence(), TransitionTransactionState::Uploaded, Some(remote_version))
21763                .expect("transaction should enter uploaded state");
21764            backend.set_put_remote_version(Some(put_version.clone())).await;
21765            let candidate = bytes::Bytes::from_static(b"orphan candidate");
21766            backend
21767                .put(
21768                    &transaction.remote_object,
21769                    ReaderImpl::Body(candidate.clone()),
21770                    i64::try_from(candidate.len()).expect("test candidate length should fit i64"),
21771                )
21772                .await
21773                .expect("mock backend should accept candidate");
21774            save_transition_transaction_record(store.clone(), &transaction)
21775                .await
21776                .expect("transaction record should persist");
21777            recovery_control_ids
21778                .push(transition_recovery_control_id(&transaction).expect("transition recovery control id should derive"));
21779            expected_removes.push((transaction.remote_object, put_version));
21780        }
21781
21782        let stats = recover_transition_transaction_records(store.clone(), 100, None)
21783            .await
21784            .expect("transition transaction recovery should run");
21785
21786        assert_eq!((stats.scanned, stats.recovered, stats.retained, stats.failed), (3, 3, 0, 0));
21787        assert_eq!(transition_transaction_record_count(store.clone()).await, 0);
21788        let mut actual_removes = backend.remove_versions().await;
21789        actual_removes.sort();
21790        expected_removes.sort();
21791        assert_eq!(actual_removes, expected_removes, "recovery must preserve each remote version shape");
21792        assert_eq!(backend.exact_remove_count(), 2);
21793        assert_eq!(backend.object_count().await, 0);
21794        for control_id in recovery_control_ids {
21795            let control = load_recovery_control(store.clone(), IlmRecoveryProtocol::TransitionTransaction, &control_id)
21796                .await
21797                .expect("completed recovery control should remain inspectable");
21798            assert_eq!(control.control.classification, IlmRecoveryClassification::Terminal);
21799            assert_eq!(control.control.attempt_count, 1);
21800            assert!(control.control.owner.is_none());
21801        }
21802
21803        let replay = recover_transition_transaction_records(store, 100, None)
21804            .await
21805            .expect("replayed recovery should remain idempotent");
21806        assert_eq!((replay.scanned, replay.recovered, replay.retained, replay.failed), (0, 0, 0, 0));
21807        assert_eq!(
21808            backend.remove_versions().await.len(),
21809            3,
21810            "each expired candidate must be deleted only once"
21811        );
21812    }
21813
21814    #[cfg(feature = "test-util")]
21815    #[tokio::test]
21816    #[serial_test::serial(storage_class_env)]
21817    async fn transition_transaction_recovery_resumes_source_cleanup_after_terminal_crash() {
21818        let temp_dir = tempfile::tempdir().expect("create temp store dir");
21819        let (ctx, store, _shutdown) =
21820            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "transition-transaction-terminal-crash", &[4]))
21821                .await;
21822        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
21823
21824        let tier_name = "TXTERMINALCRASH";
21825        let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
21826        let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
21827            .await
21828            .expect("tier lease should resolve")
21829            .backend_identity();
21830        let remote_version = uuid::Uuid::new_v4().to_string();
21831        let mut transaction = TransitionTransaction::new(TransitionTransactionInit {
21832            deployment_id: ctx.deployment_id().expect("test store should initialize deployment id"),
21833            transaction_id: uuid::Uuid::new_v4(),
21834            owner_epoch: uuid::Uuid::new_v4(),
21835            write_id: uuid::Uuid::new_v4(),
21836            source: TransitionSourceIdentity {
21837                bucket: "source-bucket".to_string(),
21838                object: "source-object".to_string(),
21839                version_id: Some(uuid::Uuid::new_v4()),
21840                data_dir: uuid::Uuid::new_v4(),
21841                mod_time_unix_nanos: 1_770_000_000_000_000_000,
21842                size: 42,
21843                etag: "source-etag".to_string(),
21844                version_mode: TransitionSourceVersionMode::Versioned,
21845            },
21846            tier_name: tier_name.to_string(),
21847            backend_fingerprint: backend_identity,
21848            not_after_unix_nanos: 1,
21849        })
21850        .expect("transaction should build");
21851        transaction
21852            .advance(
21853                transaction.fence(),
21854                TransitionTransactionState::Uploaded,
21855                Some(TransitionRemoteVersion::versioned(remote_version.clone())),
21856            )
21857            .expect("transaction should enter uploaded state");
21858        backend.set_put_remote_version(Some(remote_version)).await;
21859        let candidate = bytes::Bytes::from_static(b"terminal crash candidate");
21860        backend
21861            .put(
21862                &transaction.remote_object,
21863                ReaderImpl::Body(candidate.clone()),
21864                i64::try_from(candidate.len()).expect("test candidate length should fit i64"),
21865            )
21866            .await
21867            .expect("mock backend should accept candidate");
21868        save_transition_transaction_record(store.clone(), &transaction)
21869            .await
21870            .expect("transaction record should persist");
21871        let control_id = transition_recovery_control_id(&transaction).expect("control id should derive");
21872
21873        let barrier = TransitionRecoveryTerminalBarrier::install(transaction.transaction_id);
21874        let recovery_store = store.clone();
21875        let recovery = tokio::spawn(async move { recover_transition_transaction_records(recovery_store, 100, None).await });
21876        barrier.wait_until_paused().await;
21877        let terminal = load_recovery_control(store.clone(), IlmRecoveryProtocol::TransitionTransaction, &control_id)
21878            .await
21879            .expect("terminal control should persist before source cleanup");
21880        assert_eq!(terminal.control.classification, IlmRecoveryClassification::Terminal);
21881        assert_eq!(transition_transaction_record_count(store.clone()).await, 1);
21882        assert_eq!(backend.object_count().await, 0);
21883        assert_eq!(backend.exact_remove_count(), 1);
21884
21885        recovery.abort();
21886        assert!(
21887            recovery
21888                .await
21889                .expect_err("recovery should be cancelled at the crash boundary")
21890                .is_cancelled()
21891        );
21892        drop(barrier);
21893
21894        let replay = recover_transition_transaction_records(store.clone(), 100, None)
21895            .await
21896            .expect("terminal control should resume source cleanup without another remote delete");
21897        assert_eq!((replay.scanned, replay.recovered, replay.retained, replay.failed), (1, 1, 0, 0));
21898        assert_eq!(transition_transaction_record_count(store).await, 0);
21899        assert_eq!(backend.exact_remove_count(), 1, "terminal replay must not repeat the remote delete");
21900    }
21901
21902    #[cfg(feature = "test-util")]
21903    #[tokio::test]
21904    #[serial_test::serial(storage_class_env)]
21905    async fn transition_transaction_recovery_cas_loses_to_active_state_advance() {
21906        let temp_dir = tempfile::tempdir().expect("create temp store dir");
21907        let (ctx, store, _shutdown) =
21908            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "transition-transaction-recovery-cas", &[4]))
21909                .await;
21910        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
21911
21912        let tier_name = "TXRECOVERYCAS";
21913        let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
21914        let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
21915            .await
21916            .expect("tier lease should resolve")
21917            .backend_identity();
21918        let remote_version = uuid::Uuid::new_v4().to_string();
21919        let mut uploaded = TransitionTransaction::new(TransitionTransactionInit {
21920            deployment_id: ctx.deployment_id().expect("test store should initialize deployment id"),
21921            transaction_id: uuid::Uuid::new_v4(),
21922            owner_epoch: uuid::Uuid::new_v4(),
21923            write_id: uuid::Uuid::new_v4(),
21924            source: TransitionSourceIdentity {
21925                bucket: "source-bucket".to_string(),
21926                object: "source-object".to_string(),
21927                version_id: Some(uuid::Uuid::new_v4()),
21928                data_dir: uuid::Uuid::new_v4(),
21929                mod_time_unix_nanos: 1_770_000_000_000_000_000,
21930                size: 42,
21931                etag: "source-etag".to_string(),
21932                version_mode: TransitionSourceVersionMode::Versioned,
21933            },
21934            tier_name: tier_name.to_string(),
21935            backend_fingerprint: backend_identity,
21936            not_after_unix_nanos: 1,
21937        })
21938        .expect("transaction should build");
21939        uploaded
21940            .advance(
21941                uploaded.fence(),
21942                TransitionTransactionState::Uploaded,
21943                Some(TransitionRemoteVersion::versioned(remote_version.clone())),
21944            )
21945            .expect("transaction should enter uploaded state");
21946        backend.set_put_remote_version(Some(remote_version)).await;
21947        let candidate = bytes::Bytes::from_static(b"CAS-owned transition remote candidate");
21948        backend
21949            .put(
21950                &uploaded.remote_object,
21951                ReaderImpl::Body(candidate.clone()),
21952                i64::try_from(candidate.len()).expect("test candidate length should fit i64"),
21953            )
21954            .await
21955            .expect("mock backend should accept candidate");
21956        save_transition_transaction_record(store.clone(), &uploaded)
21957            .await
21958            .expect("transaction record should persist");
21959        let recovery_control_id =
21960            transition_recovery_control_id(&uploaded).expect("transition recovery control id should derive");
21961
21962        let barrier = TransitionRecoveryClaimBarrier::install(uploaded.transaction_id);
21963        let recovery_store = store.clone();
21964        let recovery = tokio::spawn(async move { recover_transition_transaction_records(recovery_store, 100, None).await });
21965        barrier.wait_until_paused().await;
21966
21967        let mut active = uploaded.clone();
21968        active
21969            .advance(active.fence(), TransitionTransactionState::LocalCommitStarted, None)
21970            .expect("active owner should advance to local commit");
21971        save_transition_transaction_record_if_current(store.clone(), &uploaded, &active)
21972            .await
21973            .expect("active owner should win the persisted CAS");
21974        barrier.release();
21975
21976        let stats = recovery
21977            .await
21978            .expect("recovery task should join")
21979            .expect("recovery should treat the lost CAS as a retained transaction");
21980        assert_eq!((stats.scanned, stats.recovered, stats.retained, stats.failed), (1, 0, 1, 0));
21981        assert_eq!(
21982            load_transition_transaction_record(store.clone(), uploaded.transaction_id)
21983                .await
21984                .expect("newer transaction revision must remain"),
21985            active
21986        );
21987        let control = load_recovery_control(store, IlmRecoveryProtocol::TransitionTransaction, &recovery_control_id)
21988            .await
21989            .expect("lost source CAS should retain a retryable recovery control");
21990        assert_eq!(control.control.classification, IlmRecoveryClassification::Retrying);
21991        assert_eq!(control.control.consecutive_failure_count, 1);
21992        assert_eq!(control.control.last_error_code, IlmRecoveryErrorCode::SourceGenerationChanged);
21993        assert_eq!(backend.object_count().await, 1, "a stale recovery must not delete the candidate");
21994        assert_eq!(backend.remove_count().await, 0);
21995    }
21996
21997    #[cfg(feature = "test-util")]
21998    #[tokio::test]
21999    #[serial_test::serial(storage_class_env)]
22000    async fn transition_transaction_recovery_quarantines_missing_required_fields() {
22001        let temp_dir = tempfile::tempdir().expect("create temp store dir");
22002        let (ctx, store, _shutdown) = without_storage_class_env(build_isolated_test_store(
22003            temp_dir.path(),
22004            "transition-transaction-recovery-corrupt",
22005            &[4],
22006        ))
22007        .await;
22008        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
22009
22010        let tier_name = "TXRECOVERYCORRUPT";
22011        let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
22012        let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
22013            .await
22014            .expect("tier lease should resolve")
22015            .backend_identity();
22016        let remote_version = uuid::Uuid::new_v4().to_string();
22017        let mut transaction = TransitionTransaction::new(TransitionTransactionInit {
22018            deployment_id: ctx.deployment_id().expect("test store should initialize deployment id"),
22019            transaction_id: uuid::Uuid::new_v4(),
22020            owner_epoch: uuid::Uuid::new_v4(),
22021            write_id: uuid::Uuid::new_v4(),
22022            source: TransitionSourceIdentity {
22023                bucket: "source-bucket".to_string(),
22024                object: "source-object".to_string(),
22025                version_id: None,
22026                data_dir: uuid::Uuid::new_v4(),
22027                mod_time_unix_nanos: 1_770_000_000_000_000_000,
22028                size: 42,
22029                etag: "source-etag".to_string(),
22030                version_mode: TransitionSourceVersionMode::Unversioned,
22031            },
22032            tier_name: tier_name.to_string(),
22033            backend_fingerprint: backend_identity,
22034            not_after_unix_nanos: 1,
22035        })
22036        .expect("transaction should build");
22037        transaction
22038            .advance(
22039                transaction.fence(),
22040                TransitionTransactionState::Uploaded,
22041                Some(TransitionRemoteVersion::versioned(remote_version.clone())),
22042            )
22043            .expect("transaction should enter uploaded state");
22044        backend.set_put_remote_version(Some(remote_version)).await;
22045        let candidate = bytes::Bytes::from_static(b"corrupt journal candidate");
22046        backend
22047            .put(
22048                &transaction.remote_object,
22049                ReaderImpl::Body(candidate.clone()),
22050                i64::try_from(candidate.len()).expect("test candidate length should fit i64"),
22051            )
22052            .await
22053            .expect("mock backend should accept candidate");
22054
22055        let mut persisted: serde_json::Value = serde_json::from_slice(&transaction.encode().expect("transaction should encode"))
22056            .expect("transaction should be JSON");
22057        persisted["transaction"]
22058            .as_object_mut()
22059            .expect("transaction payload should be an object")
22060            .remove("not_after_unix_nanos");
22061        let path =
22062            transition_transaction_record_object_name(transaction.transaction_id).expect("transaction path should be canonical");
22063        com::save_config(
22064            store.clone(),
22065            &path,
22066            serde_json::to_vec(&persisted).expect("corrupt fixture should encode"),
22067        )
22068        .await
22069        .expect("corrupt transaction fixture should persist");
22070
22071        let stats = recover_transition_transaction_records(store.clone(), 100, None)
22072            .await
22073            .expect("recovery scan should isolate a corrupt record");
22074        assert_eq!((stats.scanned, stats.recovered, stats.retained, stats.failed), (1, 0, 0, 1));
22075        assert_eq!(
22076            transition_transaction_record_count(store).await,
22077            1,
22078            "corrupt evidence must remain quarantined"
22079        );
22080        assert_eq!(backend.object_count().await, 1, "corrupt evidence must never authorize remote DELETE");
22081        assert_eq!(backend.remove_count().await, 0);
22082    }
22083
22084    #[cfg(feature = "test-util")]
22085    #[tokio::test]
22086    #[serial_test::serial(storage_class_env)]
22087    async fn transition_transaction_recovery_retries_cleanup_pending_candidate() {
22088        let temp_dir = tempfile::tempdir().expect("create temp store dir");
22089        let (ctx, store, _shutdown) =
22090            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "transition-transaction-cleanup", &[4])).await;
22091        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
22092
22093        let tier_name = "TXCLEANUP";
22094        let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
22095        let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
22096            .await
22097            .expect("tier lease should resolve")
22098            .backend_identity();
22099        let remote_version = uuid::Uuid::new_v4().to_string();
22100        let mut transaction = TransitionTransaction::new(TransitionTransactionInit {
22101            deployment_id: ctx.deployment_id().expect("test store should initialize deployment id"),
22102            transaction_id: uuid::Uuid::new_v4(),
22103            owner_epoch: uuid::Uuid::new_v4(),
22104            write_id: uuid::Uuid::new_v4(),
22105            source: TransitionSourceIdentity {
22106                bucket: "source-bucket".to_string(),
22107                object: "source-object".to_string(),
22108                version_id: Some(uuid::Uuid::new_v4()),
22109                data_dir: uuid::Uuid::new_v4(),
22110                mod_time_unix_nanos: 1_770_000_000_000_000_000,
22111                size: 42,
22112                etag: "source-etag".to_string(),
22113                version_mode: TransitionSourceVersionMode::Versioned,
22114            },
22115            tier_name: tier_name.to_string(),
22116            backend_fingerprint: backend_identity,
22117            not_after_unix_nanos: 1_780_000_000_000_000_000,
22118        })
22119        .expect("transaction should build");
22120        let uploaded_fence = transaction
22121            .advance(
22122                transaction.fence(),
22123                TransitionTransactionState::Uploaded,
22124                Some(TransitionRemoteVersion::versioned(remote_version.clone())),
22125            )
22126            .expect("transaction should enter uploaded state");
22127        transaction
22128            .mark_cleanup_pending(
22129                uploaded_fence,
22130                TransitionCleanupProof {
22131                    transaction_id: transaction.transaction_id,
22132                    write_id: transaction.write_id,
22133                    remote_object: transaction.remote_object.clone(),
22134                    remote_version: transaction.remote_version.clone(),
22135                    backend_fingerprint: transaction.backend_fingerprint,
22136                    decision: TransitionCleanupDecision::UploadAbortedBeforeLocalCommit,
22137                },
22138            )
22139            .expect("transaction should enter cleanup pending state");
22140        backend.set_put_remote_version(Some(remote_version.clone())).await;
22141        let candidate = bytes::Bytes::from_static(b"cleanup pending candidate");
22142        backend
22143            .put(
22144                &transaction.remote_object,
22145                ReaderImpl::Body(candidate.clone()),
22146                i64::try_from(candidate.len()).expect("test candidate length should fit i64"),
22147            )
22148            .await
22149            .expect("mock backend should accept candidate");
22150        save_transition_transaction_record(store.clone(), &transaction)
22151            .await
22152            .expect("transaction record should persist");
22153
22154        let stats = recover_transition_transaction_records(store.clone(), 100, None)
22155            .await
22156            .expect("transition transaction recovery should run");
22157
22158        assert_eq!((stats.scanned, stats.recovered, stats.retained, stats.failed), (1, 1, 0, 0));
22159        assert_eq!(transition_transaction_record_count(store.clone()).await, 0);
22160        assert_eq!(
22161            backend.remove_versions().await,
22162            vec![(transaction.remote_object.clone(), remote_version)],
22163            "cleanup pending recovery must retry the exact remote candidate delete"
22164        );
22165        assert_eq!(backend.exact_remove_count(), 1);
22166        assert_eq!(backend.object_count().await, 0);
22167    }
22168
22169    #[cfg(feature = "test-util")]
22170    #[tokio::test]
22171    #[serial_test::serial(storage_class_env)]
22172    async fn transition_transaction_recovery_retries_cleanup_pending_unversioned_candidate() {
22173        let temp_dir = tempfile::tempdir().expect("create temp store dir");
22174        let (ctx, store, _shutdown) = without_storage_class_env(build_isolated_test_store(
22175            temp_dir.path(),
22176            "transition-transaction-cleanup-unversioned",
22177            &[4],
22178        ))
22179        .await;
22180        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
22181
22182        let tier_name = "TXCLEANUPUNVERSIONED";
22183        let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
22184        let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
22185            .await
22186            .expect("tier lease should resolve")
22187            .backend_identity();
22188        let mut transaction = TransitionTransaction::new(TransitionTransactionInit {
22189            deployment_id: ctx.deployment_id().expect("test store should initialize deployment id"),
22190            transaction_id: uuid::Uuid::new_v4(),
22191            owner_epoch: uuid::Uuid::new_v4(),
22192            write_id: uuid::Uuid::new_v4(),
22193            source: TransitionSourceIdentity {
22194                bucket: "source-bucket".to_string(),
22195                object: "source-object".to_string(),
22196                version_id: None,
22197                data_dir: uuid::Uuid::new_v4(),
22198                mod_time_unix_nanos: 1_770_000_000_000_000_000,
22199                size: 42,
22200                etag: "source-etag".to_string(),
22201                version_mode: TransitionSourceVersionMode::Unversioned,
22202            },
22203            tier_name: tier_name.to_string(),
22204            backend_fingerprint: backend_identity,
22205            not_after_unix_nanos: 1_780_000_000_000_000_000,
22206        })
22207        .expect("transaction should build");
22208        let uploaded_fence = transaction
22209            .advance(
22210                transaction.fence(),
22211                TransitionTransactionState::Uploaded,
22212                Some(TransitionRemoteVersion::unversioned()),
22213            )
22214            .expect("transaction should enter uploaded state");
22215        transaction
22216            .mark_cleanup_pending(
22217                uploaded_fence,
22218                TransitionCleanupProof {
22219                    transaction_id: transaction.transaction_id,
22220                    write_id: transaction.write_id,
22221                    remote_object: transaction.remote_object.clone(),
22222                    remote_version: transaction.remote_version.clone(),
22223                    backend_fingerprint: transaction.backend_fingerprint,
22224                    decision: TransitionCleanupDecision::UploadAbortedBeforeLocalCommit,
22225                },
22226            )
22227            .expect("transaction should enter cleanup pending state");
22228        backend.set_put_remote_version(Some(String::new())).await;
22229        let candidate = bytes::Bytes::from_static(b"cleanup pending unversioned candidate");
22230        backend
22231            .put(
22232                &transaction.remote_object,
22233                ReaderImpl::Body(candidate.clone()),
22234                i64::try_from(candidate.len()).expect("test candidate length should fit i64"),
22235            )
22236            .await
22237            .expect("mock backend should accept candidate");
22238        save_transition_transaction_record(store.clone(), &transaction)
22239            .await
22240            .expect("transaction record should persist");
22241
22242        let stats = recover_transition_transaction_records(store.clone(), 100, None)
22243            .await
22244            .expect("transition transaction recovery should run");
22245
22246        assert_eq!((stats.scanned, stats.recovered, stats.retained, stats.failed), (1, 1, 0, 0));
22247        assert_eq!(transition_transaction_record_count(store.clone()).await, 0);
22248        assert_eq!(
22249            backend.remove_versions().await,
22250            vec![(transaction.remote_object.clone(), String::new())],
22251            "unversioned cleanup must not send a synthetic remote version"
22252        );
22253        assert_eq!(backend.exact_remove_count(), 0);
22254        assert_eq!(backend.object_count().await, 0);
22255    }
22256
22257    #[cfg(feature = "test-util")]
22258    #[tokio::test]
22259    #[serial_test::serial(storage_class_env)]
22260    async fn transition_transaction_recovery_keeps_cleanup_pending_record_when_delete_fails() {
22261        let temp_dir = tempfile::tempdir().expect("create temp store dir");
22262        let (ctx, store, _shutdown) =
22263            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "transition-transaction-cleanup-fail", &[4]))
22264                .await;
22265        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
22266
22267        let tier_name = "TXCLEANUPFAIL";
22268        let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
22269        let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
22270            .await
22271            .expect("tier lease should resolve")
22272            .backend_identity();
22273        let remote_version = uuid::Uuid::new_v4().to_string();
22274        let mut transaction = TransitionTransaction::new(TransitionTransactionInit {
22275            deployment_id: ctx.deployment_id().expect("test store should initialize deployment id"),
22276            transaction_id: uuid::Uuid::new_v4(),
22277            owner_epoch: uuid::Uuid::new_v4(),
22278            write_id: uuid::Uuid::new_v4(),
22279            source: TransitionSourceIdentity {
22280                bucket: "source-bucket".to_string(),
22281                object: "source-object".to_string(),
22282                version_id: Some(uuid::Uuid::new_v4()),
22283                data_dir: uuid::Uuid::new_v4(),
22284                mod_time_unix_nanos: 1_770_000_000_000_000_000,
22285                size: 42,
22286                etag: "source-etag".to_string(),
22287                version_mode: TransitionSourceVersionMode::Versioned,
22288            },
22289            tier_name: tier_name.to_string(),
22290            backend_fingerprint: backend_identity,
22291            not_after_unix_nanos: 1_780_000_000_000_000_000,
22292        })
22293        .expect("transaction should build");
22294        transaction
22295            .advance(
22296                transaction.fence(),
22297                TransitionTransactionState::Uploaded,
22298                Some(TransitionRemoteVersion::versioned(remote_version.clone())),
22299            )
22300            .expect("transaction should enter uploaded state");
22301        let candidate = bytes::Bytes::from_static(b"cleanup pending candidate retained after failure");
22302        backend.set_put_remote_version(Some(remote_version)).await;
22303        backend
22304            .put(
22305                &transaction.remote_object,
22306                ReaderImpl::Body(candidate.clone()),
22307                i64::try_from(candidate.len()).expect("test candidate length should fit i64"),
22308            )
22309            .await
22310            .expect("mock backend should accept candidate");
22311        save_transition_transaction_record(store.clone(), &transaction)
22312            .await
22313            .expect("transaction record should persist");
22314        let recovery_control_id =
22315            transition_recovery_control_id(&transaction).expect("transition recovery control id should derive");
22316
22317        backend.set_remove_failure(true);
22318        let stats = recover_transition_transaction_records(store.clone(), 100, None)
22319            .await
22320            .expect("transition transaction recovery should keep scanning after cleanup failure");
22321
22322        assert_eq!((stats.scanned, stats.recovered, stats.retained, stats.failed), (1, 0, 0, 1));
22323        assert_eq!(
22324            transition_transaction_record_count(store.clone()).await,
22325            1,
22326            "failed cleanup must keep the cleanup-pending transaction for retry"
22327        );
22328        assert_eq!(backend.remove_versions().await, Vec::<(String, String)>::new());
22329        assert_eq!(backend.exact_remove_count(), 1);
22330        assert_eq!(backend.object_count().await, 1);
22331        let control = load_recovery_control(store.clone(), IlmRecoveryProtocol::TransitionTransaction, &recovery_control_id)
22332            .await
22333            .expect("failed recovery control should persist");
22334        assert_eq!(control.control.classification, IlmRecoveryClassification::Retrying);
22335        assert_eq!(control.control.attempt_count, 1);
22336        assert_eq!(control.control.consecutive_failure_count, 1);
22337        assert!(
22338            control
22339                .control
22340                .next_attempt_at_unix_nanos
22341                .is_some_and(|next| next > OffsetDateTime::now_utc().unix_timestamp_nanos() as i64)
22342        );
22343
22344        backend.set_remove_failure(false);
22345        let replay = recover_transition_transaction_records(store.clone(), 100, None)
22346            .await
22347            .expect("recovery before the persisted deadline should be skipped");
22348        assert_eq!((replay.scanned, replay.recovered, replay.retained, replay.failed), (1, 0, 1, 0));
22349        assert_eq!(backend.exact_remove_count(), 1, "persisted backoff must prevent an immediate retry");
22350
22351        let retry_at = control
22352            .control
22353            .next_attempt_at_unix_nanos
22354            .expect("retry deadline should persist");
22355        let retried = recover_transition_transaction_records_at(store.clone(), 100, None, i128::from(retry_at) + 1)
22356            .await
22357            .expect("recovery at the persisted deadline should retry the advanced source generation");
22358        assert_eq!((retried.scanned, retried.recovered, retried.retained, retried.failed), (1, 1, 0, 0));
22359        assert_eq!(backend.exact_remove_count(), 2);
22360        assert_eq!(backend.object_count().await, 0);
22361        assert_eq!(transition_transaction_record_count(store.clone()).await, 0);
22362        let terminal = load_recovery_control(store, IlmRecoveryProtocol::TransitionTransaction, &recovery_control_id)
22363            .await
22364            .expect("completed retry control should remain inspectable");
22365        assert_eq!(terminal.control.classification, IlmRecoveryClassification::Terminal);
22366        assert_eq!(terminal.control.attempt_count, 2);
22367    }
22368
22369    #[cfg(feature = "test-util")]
22370    #[tokio::test]
22371    #[serial_test::serial(storage_class_env)]
22372    async fn transition_transaction_recovery_keeps_cleanup_pending_local_commit() {
22373        let temp_dir = tempfile::tempdir().expect("create temp store dir");
22374        let (ctx, store, _shutdown) = without_storage_class_env(build_isolated_test_store(
22375            temp_dir.path(),
22376            "transition-transaction-cleanup-committed",
22377            &[4],
22378        ))
22379        .await;
22380        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
22381
22382        let tier_name = "TXCLEANUPCOMMITTED";
22383        let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
22384        let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
22385            .await
22386            .expect("tier lease should resolve")
22387            .backend_identity();
22388        let bucket = "transition-transaction-cleanup-committed-bucket";
22389        let object = "object.bin";
22390        store
22391            .make_bucket(bucket, &MakeBucketOptions::default())
22392            .await
22393            .expect("bucket should be created");
22394        let mut reader = PutObjReader::from_vec(b"cleanup pending local commit record cleanup".repeat(1024));
22395        let original = store
22396            .put_object(bucket, object, &mut reader, &ObjectOptions::default())
22397            .await
22398            .expect("source object should be written");
22399        let opts = ObjectOptions {
22400            no_lock: true,
22401            transition: TransitionOptions {
22402                status: TRANSITION_PENDING.to_string(),
22403                tier: tier_name.to_string(),
22404                etag: original.etag.clone().expect("source object should have etag"),
22405                ..Default::default()
22406            },
22407            mod_time: original.mod_time,
22408            ..Default::default()
22409        };
22410        store
22411            .transition_object(bucket, object, &opts)
22412            .await
22413            .expect("transition should commit");
22414        let committed = store
22415            .get_object_info(
22416                bucket,
22417                object,
22418                &ObjectOptions {
22419                    no_lock: true,
22420                    metadata_cache_safe: false,
22421                    ..Default::default()
22422                },
22423            )
22424            .await
22425            .expect("committed object info should be readable");
22426        let mut remote_parts = committed.transitioned_object.name.rsplit('/');
22427        let write_id = uuid::Uuid::parse_str(remote_parts.next().expect("remote object should contain write id"))
22428            .expect("write id should parse");
22429        let transaction_id = uuid::Uuid::parse_str(remote_parts.next().expect("remote object should contain transaction id"))
22430            .expect("transaction id should parse");
22431        let source = TransitionSourceIdentity {
22432            bucket: bucket.to_string(),
22433            object: object.to_string(),
22434            version_id: None,
22435            data_dir: original.data_dir.expect("source object should have data_dir"),
22436            mod_time_unix_nanos: original
22437                .mod_time
22438                .expect("source object should have mod_time")
22439                .unix_timestamp_nanos()
22440                .try_into()
22441                .expect("test timestamp should fit i64"),
22442            size: original.size,
22443            etag: original.etag.expect("source object should have etag"),
22444            version_mode: TransitionSourceVersionMode::Unversioned,
22445        };
22446        let mut transaction = TransitionTransaction::new(TransitionTransactionInit {
22447            deployment_id: ctx.deployment_id().expect("test store should initialize deployment id"),
22448            transaction_id,
22449            owner_epoch: uuid::Uuid::new_v4(),
22450            write_id,
22451            source,
22452            tier_name: tier_name.to_string(),
22453            backend_fingerprint: backend_identity,
22454            not_after_unix_nanos: 1_780_000_000_000_000_000,
22455        })
22456        .expect("transaction should build");
22457        transaction
22458            .advance(
22459                transaction.fence(),
22460                TransitionTransactionState::Uploaded,
22461                Some(TransitionRemoteVersion::known_from_put_response(
22462                    committed.transitioned_object.version_id.clone(),
22463                )),
22464            )
22465            .expect("transaction should enter uploaded state");
22466        let local_commit_fence = transaction
22467            .advance(transaction.fence(), TransitionTransactionState::LocalCommitStarted, None)
22468            .expect("transaction should enter local commit state");
22469        transaction
22470            .mark_cleanup_pending(
22471                local_commit_fence,
22472                TransitionCleanupProof {
22473                    transaction_id: transaction.transaction_id,
22474                    write_id: transaction.write_id,
22475                    remote_object: transaction.remote_object.clone(),
22476                    remote_version: transaction.remote_version.clone(),
22477                    backend_fingerprint: transaction.backend_fingerprint,
22478                    decision: TransitionCleanupDecision::SourceReconciledUnchanged {
22479                        observed_source: transaction.source.clone(),
22480                    },
22481                },
22482            )
22483            .expect("transaction should enter cleanup pending state");
22484        save_transition_transaction_record(store.clone(), &transaction)
22485            .await
22486            .expect("transaction record should persist");
22487
22488        let stats = recover_transition_transaction_records(store.clone(), 100, None)
22489            .await
22490            .expect("transition transaction recovery should run");
22491
22492        assert_eq!((stats.scanned, stats.recovered, stats.retained, stats.failed), (1, 1, 0, 0));
22493        assert_eq!(transition_transaction_record_count(store.clone()).await, 0);
22494        assert_eq!(
22495            backend.object_count().await,
22496            1,
22497            "cleanup pending recovery must keep the remote body once local metadata references it"
22498        );
22499        assert_eq!(backend.remove_count().await, 0);
22500    }
22501
22502    #[cfg(feature = "test-util")]
22503    #[tokio::test]
22504    #[serial_test::serial(storage_class_env)]
22505    async fn transition_transaction_recovery_drops_record_after_confirmed_local_commit() {
22506        let temp_dir = tempfile::tempdir().expect("create temp store dir");
22507        let (ctx, store, _shutdown) =
22508            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "transition-transaction-committed", &[4])).await;
22509        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
22510
22511        let tier_name = "TXCOMMITTED";
22512        let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
22513        let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
22514            .await
22515            .expect("tier lease should resolve")
22516            .backend_identity();
22517        let bucket = "transition-transaction-committed-bucket";
22518        let object = "object.bin";
22519        store
22520            .make_bucket(bucket, &MakeBucketOptions::default())
22521            .await
22522            .expect("bucket should be created");
22523        let mut reader = PutObjReader::from_vec(b"confirmed local commit record cleanup".repeat(1024));
22524        let original = store
22525            .put_object(bucket, object, &mut reader, &ObjectOptions::default())
22526            .await
22527            .expect("source object should be written");
22528        let opts = ObjectOptions {
22529            no_lock: true,
22530            transition: TransitionOptions {
22531                status: TRANSITION_PENDING.to_string(),
22532                tier: tier_name.to_string(),
22533                etag: original.etag.clone().expect("source object should have etag"),
22534                ..Default::default()
22535            },
22536            mod_time: original.mod_time,
22537            ..Default::default()
22538        };
22539        store
22540            .transition_object(bucket, object, &opts)
22541            .await
22542            .expect("transition should commit");
22543        let committed = store
22544            .get_object_info(
22545                bucket,
22546                object,
22547                &ObjectOptions {
22548                    no_lock: true,
22549                    metadata_cache_safe: false,
22550                    ..Default::default()
22551                },
22552            )
22553            .await
22554            .expect("committed object info should be readable");
22555        let mut remote_parts = committed.transitioned_object.name.rsplit('/');
22556        let write_id = uuid::Uuid::parse_str(remote_parts.next().expect("remote object should contain write id"))
22557            .expect("write id should parse");
22558        let transaction_id = uuid::Uuid::parse_str(remote_parts.next().expect("remote object should contain transaction id"))
22559            .expect("transaction id should parse");
22560        let mut transaction = TransitionTransaction::new(TransitionTransactionInit {
22561            deployment_id: ctx.deployment_id().expect("test store should initialize deployment id"),
22562            transaction_id,
22563            owner_epoch: uuid::Uuid::new_v4(),
22564            write_id,
22565            source: TransitionSourceIdentity {
22566                bucket: bucket.to_string(),
22567                object: object.to_string(),
22568                version_id: None,
22569                data_dir: original.data_dir.expect("source object should have data_dir"),
22570                mod_time_unix_nanos: original
22571                    .mod_time
22572                    .expect("source object should have mod_time")
22573                    .unix_timestamp_nanos()
22574                    .try_into()
22575                    .expect("test timestamp should fit i64"),
22576                size: original.size,
22577                etag: original.etag.expect("source object should have etag"),
22578                version_mode: TransitionSourceVersionMode::Unversioned,
22579            },
22580            tier_name: tier_name.to_string(),
22581            backend_fingerprint: backend_identity,
22582            not_after_unix_nanos: 1_780_000_000_000_000_000,
22583        })
22584        .expect("transaction should build");
22585        transaction
22586            .advance(
22587                transaction.fence(),
22588                TransitionTransactionState::Uploaded,
22589                Some(TransitionRemoteVersion::known_from_put_response(
22590                    committed.transitioned_object.version_id.clone(),
22591                )),
22592            )
22593            .expect("transaction should enter uploaded state");
22594        transaction
22595            .advance(transaction.fence(), TransitionTransactionState::LocalCommitStarted, None)
22596            .expect("transaction should enter local commit state");
22597        save_transition_transaction_record(store.clone(), &transaction)
22598            .await
22599            .expect("transaction record should persist");
22600        assert_eq!(transition_transaction_record_count(store.clone()).await, 1);
22601
22602        let stats = recover_transition_transaction_records(store.clone(), 100, None)
22603            .await
22604            .expect("transition transaction recovery should run");
22605
22606        assert_eq!((stats.scanned, stats.recovered, stats.retained, stats.failed), (1, 1, 0, 0));
22607        assert_eq!(transition_transaction_record_count(store.clone()).await, 0);
22608        assert_eq!(
22609            backend.object_count().await,
22610            1,
22611            "confirmed local commit recovery must not delete the committed remote body"
22612        );
22613        assert_eq!(backend.remove_count().await, 0);
22614    }
22615
22616    #[cfg(feature = "test-util")]
22617    #[tokio::test]
22618    #[serial_test::serial(storage_class_env)]
22619    async fn transition_transaction_recovery_retains_unproven_remote_candidates() {
22620        let temp_dir = tempfile::tempdir().expect("create temp store dir");
22621        let (ctx, store, _shutdown) =
22622            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "transition-transaction-unproven", &[4])).await;
22623        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
22624
22625        let tier_name = "TXUNPROVEN";
22626        let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
22627        let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
22628            .await
22629            .expect("tier lease should resolve")
22630            .backend_identity();
22631        let remote_version = uuid::Uuid::new_v4().to_string();
22632        let new_transaction = || {
22633            TransitionTransaction::new(TransitionTransactionInit {
22634                deployment_id: ctx.deployment_id().expect("test store should initialize deployment id"),
22635                transaction_id: uuid::Uuid::new_v4(),
22636                owner_epoch: uuid::Uuid::new_v4(),
22637                write_id: uuid::Uuid::new_v4(),
22638                source: TransitionSourceIdentity {
22639                    bucket: "absent-source-bucket".to_string(),
22640                    object: "source-object".to_string(),
22641                    version_id: None,
22642                    data_dir: uuid::Uuid::new_v4(),
22643                    mod_time_unix_nanos: 1_770_000_000_000_000_000,
22644                    size: 42,
22645                    etag: "source-etag".to_string(),
22646                    version_mode: TransitionSourceVersionMode::Unversioned,
22647                },
22648                tier_name: tier_name.to_string(),
22649                backend_fingerprint: backend_identity,
22650                not_after_unix_nanos: 1_780_000_000_000_000_000,
22651            })
22652            .expect("transaction should build")
22653        };
22654
22655        let upload_started = new_transaction();
22656        let mut local_commit_started = new_transaction();
22657        local_commit_started
22658            .advance(
22659                local_commit_started.fence(),
22660                TransitionTransactionState::Uploaded,
22661                Some(TransitionRemoteVersion::versioned(remote_version.clone())),
22662            )
22663            .expect("transaction should enter uploaded state");
22664        local_commit_started
22665            .advance(local_commit_started.fence(), TransitionTransactionState::LocalCommitStarted, None)
22666            .expect("transaction should enter local commit state");
22667        let upload_started_control_id =
22668            transition_recovery_control_id(&upload_started).expect("upload-started control id should derive");
22669        let local_commit_control_id =
22670            transition_recovery_control_id(&local_commit_started).expect("local-commit control id should derive");
22671
22672        backend.set_put_remote_version(Some(remote_version)).await;
22673        for transaction in [&upload_started, &local_commit_started] {
22674            let candidate = bytes::Bytes::from_static(b"unproven transition remote candidate");
22675            backend
22676                .put(
22677                    &transaction.remote_object,
22678                    ReaderImpl::Body(candidate.clone()),
22679                    i64::try_from(candidate.len()).expect("test candidate length should fit i64"),
22680                )
22681                .await
22682                .expect("mock backend should accept candidate");
22683            save_transition_transaction_record(store.clone(), transaction)
22684                .await
22685                .expect("transaction record should persist");
22686        }
22687
22688        let stats = recover_transition_transaction_records(store.clone(), 100, None)
22689            .await
22690            .expect("transition transaction recovery should run");
22691
22692        assert_eq!((stats.scanned, stats.recovered, stats.retained, stats.failed), (2, 0, 2, 0));
22693        assert_eq!(
22694            transition_transaction_record_count(store.clone()).await,
22695            2,
22696            "an upload without completion proof or unproven local commit must remain for authoritative reconcile"
22697        );
22698        assert_eq!(backend.object_count().await, 2, "recovery must not delete an unproven remote candidate");
22699        assert_eq!(backend.remove_count().await, 0);
22700        assert_eq!(backend.exact_remove_count(), 0);
22701        let upload_started_control =
22702            load_recovery_control(store.clone(), IlmRecoveryProtocol::TransitionTransaction, &upload_started_control_id)
22703                .await
22704                .expect("upload-started control should persist");
22705        assert_eq!(
22706            upload_started_control.control.classification,
22707            IlmRecoveryClassification::RetainedAmbiguous
22708        );
22709        let local_commit_control =
22710            load_recovery_control(store.clone(), IlmRecoveryProtocol::TransitionTransaction, &local_commit_control_id)
22711                .await
22712                .expect("local-commit control should persist");
22713        assert_eq!(local_commit_control.control.classification, IlmRecoveryClassification::OperatorRequired);
22714
22715        let upload_status = inspect_transition_recovery_retry_for_operator(store.clone(), &upload_started_control_id)
22716            .await
22717            .expect("retained upload should be inspectable for a bounded retry");
22718        let local_status = inspect_transition_recovery_retry_for_operator(store.clone(), &local_commit_control_id)
22719            .await
22720            .expect("operator-required local commit should be inspectable for a bounded retry");
22721        assert!(upload_status.retry_ready);
22722        assert!(local_status.retry_ready);
22723        assert!(matches!(
22724            retry_transition_recovery_for_operator(
22725                store.clone(),
22726                &upload_started_control_id,
22727                upload_status.control_revision + 1,
22728                &upload_status.source_generation_sha256,
22729            )
22730            .await,
22731            Err(TransitionOperatorError::StaleRecoveryControl)
22732        ));
22733
22734        let put_count_before_retry = backend.put_count().await;
22735        let get_count_before_retry = backend.get_count().await;
22736        let remove_count_before_retry = backend.remove_count().await;
22737        let upload_retry = retry_transition_recovery_for_operator(
22738            store.clone(),
22739            &upload_started_control_id,
22740            upload_status.control_revision,
22741            &upload_status.source_generation_sha256,
22742        )
22743        .await
22744        .expect("exact retained upload generation should be rearmed");
22745        let local_retry = retry_transition_recovery_for_operator(
22746            store.clone(),
22747            &local_commit_control_id,
22748            local_status.control_revision,
22749            &local_status.source_generation_sha256,
22750        )
22751        .await
22752        .expect("exact operator-required local commit generation should be rearmed");
22753        assert_eq!(upload_retry.classification, IlmRecoveryClassification::Retrying);
22754        assert_eq!(local_retry.classification, IlmRecoveryClassification::Retrying);
22755        assert_eq!(upload_retry.attempt_count, upload_status.attempt_count);
22756        assert_eq!(local_retry.attempt_count, local_status.attempt_count);
22757        assert_eq!(backend.put_count().await, put_count_before_retry);
22758        assert_eq!(backend.get_count().await, get_count_before_retry);
22759        assert_eq!(backend.remove_count().await, remove_count_before_retry);
22760        assert_eq!(backend.exact_remove_count(), 0, "operator retry must not directly issue remote DELETE");
22761
22762        let retried = recover_transition_transaction_records(store.clone(), 100, None)
22763            .await
22764            .expect("rearmed records should be re-evaluated through normal recovery");
22765        assert_eq!((retried.scanned, retried.recovered, retried.retained, retried.failed), (2, 0, 2, 0));
22766        let upload_retained =
22767            load_recovery_control(store.clone(), IlmRecoveryProtocol::TransitionTransaction, &upload_started_control_id)
22768                .await
22769                .expect("upload retry result should persist");
22770        let local_retained = load_recovery_control(store, IlmRecoveryProtocol::TransitionTransaction, &local_commit_control_id)
22771            .await
22772            .expect("local commit retry result should persist");
22773        assert_eq!(upload_retained.control.classification, IlmRecoveryClassification::RetainedAmbiguous);
22774        assert_eq!(local_retained.control.classification, IlmRecoveryClassification::OperatorRequired);
22775        assert_eq!(upload_retained.control.attempt_count, upload_status.attempt_count + 1);
22776        assert_eq!(local_retained.control.attempt_count, local_status.attempt_count + 1);
22777        assert_eq!(backend.put_count().await, put_count_before_retry);
22778        assert_eq!(backend.get_count().await, get_count_before_retry);
22779        assert_eq!(backend.remove_count().await, remove_count_before_retry);
22780        assert_eq!(backend.exact_remove_count(), 0);
22781    }
22782
22783    #[cfg(feature = "test-util")]
22784    #[tokio::test]
22785    #[serial_test::serial(storage_class_env)]
22786    async fn transition_transaction_recovery_handles_provider_recovered_unknown_upload() {
22787        let versioned_remote = uuid::Uuid::new_v4().to_string();
22788        let nil_remote = uuid::Uuid::nil().to_string();
22789        for (case, tier_name, remote_version, should_recover) in [
22790            ("missing", "TXPROBEMISSING", None, true),
22791            ("unversioned", "TXPROBEUNVERSIONED", Some(String::new()), true),
22792            ("versioned", "TXPROBEVERSIONED", Some(versioned_remote), true),
22793            ("nil-version", "TXPROBENILVERSION", Some(nil_remote), false),
22794        ] {
22795            let temp_dir = tempfile::tempdir().expect("create temp store dir");
22796            let (ctx, store, _shutdown) = without_storage_class_env(build_isolated_test_store(
22797                temp_dir.path(),
22798                &format!("transition-transaction-probe-{case}"),
22799                &[4],
22800            ))
22801            .await;
22802            crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
22803
22804            let backend = register_transition_reconcile_test_tier(&ctx.tier_config_mgr(), tier_name).await;
22805            let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
22806                .await
22807                .expect("tier lease should resolve")
22808                .backend_identity();
22809            let mut transaction = TransitionTransaction::new(TransitionTransactionInit {
22810                deployment_id: ctx.deployment_id().expect("test store should initialize deployment id"),
22811                transaction_id: uuid::Uuid::new_v4(),
22812                owner_epoch: uuid::Uuid::new_v4(),
22813                write_id: uuid::Uuid::new_v4(),
22814                source: TransitionSourceIdentity {
22815                    bucket: "source-bucket".to_string(),
22816                    object: "source-object".to_string(),
22817                    version_id: None,
22818                    data_dir: uuid::Uuid::new_v4(),
22819                    mod_time_unix_nanos: 1_770_000_000_000_000_000,
22820                    size: 42,
22821                    etag: "source-etag".to_string(),
22822                    version_mode: TransitionSourceVersionMode::Unversioned,
22823                },
22824                tier_name: tier_name.to_string(),
22825                backend_fingerprint: backend_identity,
22826                not_after_unix_nanos: 1,
22827            })
22828            .expect("transaction should build");
22829            transaction
22830                .advance(transaction.fence(), TransitionTransactionState::UploadOutcomeUnknown, None)
22831                .expect("transaction should enter unknown upload outcome state");
22832
22833            if let Some(version) = &remote_version {
22834                backend.set_put_remote_version(Some(version.clone())).await;
22835                let candidate = bytes::Bytes::from_static(b"provider-recovered transition remote candidate");
22836                backend
22837                    .put(
22838                        &transaction.remote_object,
22839                        ReaderImpl::Body(candidate.clone()),
22840                        i64::try_from(candidate.len()).expect("test candidate length should fit i64"),
22841                    )
22842                    .await
22843                    .expect("mock backend should accept candidate");
22844            }
22845            save_transition_transaction_record(store.clone(), &transaction)
22846                .await
22847                .expect("transaction record should persist");
22848
22849            let stats = recover_transition_transaction_records(store.clone(), 100, None)
22850                .await
22851                .expect("transition transaction recovery should run");
22852
22853            assert_eq!(
22854                (stats.scanned, stats.recovered, stats.retained, stats.failed),
22855                if should_recover { (1, 1, 0, 0) } else { (1, 0, 1, 0) }
22856            );
22857            assert_eq!(transition_transaction_record_count(store.clone()).await, usize::from(!should_recover));
22858            assert_eq!(
22859                backend.object_count().await,
22860                usize::from(!should_recover),
22861                "case {case}: only a valid provider version state may be recovered destructively"
22862            );
22863            let removed = remote_version
22864                .filter(|_| should_recover)
22865                .map(|version| vec![(transaction.remote_object.clone(), version)])
22866                .unwrap_or_default();
22867            assert_eq!(
22868                backend.remove_versions().await,
22869                removed,
22870                "case {case}: recovery must delete only provider-recovered candidates"
22871            );
22872            assert_eq!(
22873                backend.exact_remove_count(),
22874                usize::from(removed.first().is_some_and(|(_, version)| !version.is_empty()))
22875            );
22876        }
22877    }
22878
22879    #[cfg(feature = "test-util")]
22880    #[tokio::test]
22881    #[serial_test::serial(storage_class_env)]
22882    async fn operator_reconcile_deletes_exact_candidate_before_finalizing_record() {
22883        let temp_dir = tempfile::tempdir().expect("create temp store dir");
22884        let (ctx, store, _shutdown) =
22885            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "transition-operator-reconcile", &[4])).await;
22886        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
22887
22888        let tier_name = "TXOPERATOR";
22889        let backend = register_transition_reconcile_test_tier(&ctx.tier_config_mgr(), tier_name).await;
22890        let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
22891            .await
22892            .expect("tier lease should resolve")
22893            .backend_identity();
22894        let mut transaction = TransitionTransaction::new(TransitionTransactionInit {
22895            deployment_id: ctx.deployment_id().expect("test store should initialize deployment id"),
22896            transaction_id: uuid::Uuid::new_v4(),
22897            owner_epoch: uuid::Uuid::new_v4(),
22898            write_id: uuid::Uuid::new_v4(),
22899            source: TransitionSourceIdentity {
22900                bucket: "source-bucket".to_string(),
22901                object: "source-object".to_string(),
22902                version_id: None,
22903                data_dir: uuid::Uuid::new_v4(),
22904                mod_time_unix_nanos: 1_770_000_000_000_000_000,
22905                size: 42,
22906                etag: "source-etag".to_string(),
22907                version_mode: TransitionSourceVersionMode::Unversioned,
22908            },
22909            tier_name: tier_name.to_string(),
22910            backend_fingerprint: backend_identity,
22911            not_after_unix_nanos: 1,
22912        })
22913        .expect("transaction should build");
22914        transaction
22915            .advance(transaction.fence(), TransitionTransactionState::UploadOutcomeUnknown, None)
22916            .expect("transaction should enter unknown upload outcome state");
22917
22918        let remote_version = uuid::Uuid::new_v4().to_string();
22919        backend.set_put_remote_version(Some(remote_version.clone())).await;
22920        let candidate = bytes::Bytes::from_static(b"operator-confirmed transition candidate");
22921        backend
22922            .put(
22923                &transaction.remote_object,
22924                ReaderImpl::Body(candidate.clone()),
22925                i64::try_from(candidate.len()).expect("test candidate length should fit i64"),
22926            )
22927            .await
22928            .expect("mock backend should accept candidate");
22929        save_transition_transaction_record(store.clone(), &transaction)
22930            .await
22931            .expect("transaction record should persist");
22932
22933        let status = inspect_transition_transaction_for_operator(store.clone(), transaction.transaction_id)
22934            .await
22935            .expect("operator inspection should probe the candidate");
22936        assert_eq!(status.probe, TransitionOperatorProbe::VersionedPresent(remote_version.clone()));
22937
22938        let wrong_version = uuid::Uuid::new_v4().to_string();
22939        let err = delete_transition_candidate_for_operator(store.clone(), transaction.transaction_id, &wrong_version)
22940            .await
22941            .expect_err("a mismatched exact version must fail before deleting a candidate");
22942        assert!(matches!(
22943            err,
22944            TransitionOperatorError::CandidateVersionMismatch {
22945                expected,
22946                actual: TransitionOperatorProbe::VersionedPresent(ref observed),
22947            } if expected == wrong_version && observed == &remote_version
22948        ));
22949        assert!(backend.contains(&transaction.remote_object).await);
22950        assert_eq!(backend.exact_remove_count(), 0);
22951        load_transition_transaction_record(store.clone(), transaction.transaction_id)
22952            .await
22953            .expect("an incorrect exact version must retain the transaction journal");
22954
22955        let result = delete_transition_candidate_for_operator(store.clone(), transaction.transaction_id, &remote_version)
22956            .await
22957            .expect("operator-confirmed exact candidate should be deleted");
22958        assert_eq!(result.status.probe, TransitionOperatorProbe::Missing);
22959        assert!(result.journal_observed_after_delete);
22960        assert_eq!(backend.exact_remove_count(), 1);
22961        assert_eq!(backend.remove_versions().await, vec![(transaction.remote_object.clone(), remote_version)]);
22962        load_transition_transaction_record(store.clone(), transaction.transaction_id)
22963            .await
22964            .expect("candidate deletion must retain the transaction journal");
22965
22966        finalize_missing_transition_transaction_for_operator(store.clone(), transaction.transaction_id)
22967            .await
22968            .expect("a separately confirmed missing candidate should permit finalization");
22969        assert!(matches!(
22970            load_transition_transaction_record(store, transaction.transaction_id).await,
22971            Err(Error::ConfigNotFound)
22972        ));
22973    }
22974
22975    #[cfg(feature = "test-util")]
22976    #[tokio::test]
22977    #[serial_test::serial(storage_class_env)]
22978    async fn operator_finalize_retains_record_without_missing_proof() {
22979        let temp_dir = tempfile::tempdir().expect("create temp store dir");
22980        let (ctx, store, _shutdown) =
22981            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "transition-operator-fail-closed", &[4])).await;
22982        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
22983
22984        let tier_name = "TXOPERATORFAIL";
22985        let backend = register_transition_reconcile_test_tier(&ctx.tier_config_mgr(), tier_name).await;
22986        let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
22987            .await
22988            .expect("tier lease should resolve")
22989            .backend_identity();
22990        let mut transaction = TransitionTransaction::new(TransitionTransactionInit {
22991            deployment_id: ctx.deployment_id().expect("test store should initialize deployment id"),
22992            transaction_id: uuid::Uuid::new_v4(),
22993            owner_epoch: uuid::Uuid::new_v4(),
22994            write_id: uuid::Uuid::new_v4(),
22995            source: TransitionSourceIdentity {
22996                bucket: "source-bucket".to_string(),
22997                object: "source-object".to_string(),
22998                version_id: None,
22999                data_dir: uuid::Uuid::new_v4(),
23000                mod_time_unix_nanos: 1_770_000_000_000_000_000,
23001                size: 42,
23002                etag: "source-etag".to_string(),
23003                version_mode: TransitionSourceVersionMode::Unversioned,
23004            },
23005            tier_name: tier_name.to_string(),
23006            backend_fingerprint: backend_identity,
23007            not_after_unix_nanos: 1,
23008        })
23009        .expect("transaction should build");
23010        transaction
23011            .advance(transaction.fence(), TransitionTransactionState::UploadOutcomeUnknown, None)
23012            .expect("transaction should enter unknown upload outcome state");
23013        save_transition_transaction_record(store.clone(), &transaction)
23014            .await
23015            .expect("transaction record should persist");
23016        backend
23017            .set_transition_candidate_probe_override(Some(TransitionCandidateProbe::Unsupported))
23018            .await;
23019
23020        assert!(matches!(
23021            finalize_missing_transition_transaction_for_operator(store.clone(), transaction.transaction_id).await,
23022            Err(TransitionOperatorError::CandidateNotMissing(TransitionOperatorProbe::Unsupported))
23023        ));
23024        load_transition_transaction_record(store, transaction.transaction_id)
23025            .await
23026            .expect("an unsupported probe must retain the transaction journal");
23027        assert_eq!(backend.remove_count().await, 0);
23028    }
23029
23030    #[cfg(feature = "test-util")]
23031    #[tokio::test]
23032    #[serial_test::serial(storage_class_env)]
23033    async fn transition_response_loss_persists_unknown_outcome_for_provider_recovery() {
23034        let temp_dir = tempfile::tempdir().expect("create temp store dir");
23035        let (ctx, store, _shutdown) =
23036            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "transition-response-loss", &[4])).await;
23037        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
23038
23039        let tier_name = "TXRESPONSELOSS";
23040        let backend = register_transition_reconcile_test_tier(&ctx.tier_config_mgr(), tier_name).await;
23041        let _compaction_proof = install_transition_transaction_compaction_fleet_proof_for_test("object-transaction-fencing-test");
23042        let bucket = "transition-response-loss-bucket";
23043        let object = "source.bin";
23044        store
23045            .make_bucket(bucket, &MakeBucketOptions::default())
23046            .await
23047            .expect("source bucket should be created");
23048        let payload = b"a response-lost tier PUT must remain recoverable".repeat(1024);
23049        let mut reader = PutObjReader::from_vec(payload.clone());
23050        let source = store
23051            .put_object(bucket, object, &mut reader, &ObjectOptions::default())
23052            .await
23053            .expect("source object should be written");
23054        backend.lose_next_put_response();
23055
23056        let error = store
23057            .transition_object(
23058                bucket,
23059                object,
23060                &ObjectOptions {
23061                    no_lock: true,
23062                    transition: TransitionOptions {
23063                        status: TRANSITION_PENDING.to_string(),
23064                        tier: tier_name.to_string(),
23065                        etag: source.etag.clone().expect("source object should have an ETag"),
23066                        ..Default::default()
23067                    },
23068                    version_id: source.version_id.map(|version| version.to_string()),
23069                    mod_time: source.mod_time,
23070                    ..Default::default()
23071                },
23072            )
23073            .await
23074            .expect_err("a lost tier PUT response must fail the transition request");
23075        assert!(
23076            matches!(error, StorageError::Io(ref err) if err.kind() == std::io::ErrorKind::ConnectionReset),
23077            "the response-loss error must remain visible to the caller: {error:?}"
23078        );
23079
23080        let records = store
23081            .clone()
23082            .list_objects_v2(
23083                RUSTFS_META_BUCKET,
23084                TRANSITION_TRANSACTION_RECORD_PREFIX,
23085                None,
23086                None,
23087                10,
23088                false,
23089                None,
23090                false,
23091            )
23092            .await
23093            .expect("transition transaction records should be listable");
23094        assert_eq!(records.objects.len(), 1, "response loss must leave one durable transaction record");
23095        let transaction_id = records.objects[0]
23096            .name
23097            .rsplit('/')
23098            .next()
23099            .and_then(|name| name.strip_suffix(".json"))
23100            .and_then(|name| uuid::Uuid::parse_str(name).ok())
23101            .expect("transaction record name should contain a UUID");
23102        let transaction = load_transition_transaction_record(store.clone(), transaction_id)
23103            .await
23104            .expect("response loss transaction record should load");
23105        assert_eq!(
23106            transaction.state,
23107            TransitionTransactionState::UploadOutcomeUnknown,
23108            "a response-lost PUT must not remain in UploadStarted"
23109        );
23110        assert_eq!(transaction.revision, 1, "compact response loss must retain the pre-PUT fence generation");
23111        assert!(
23112            backend.contains(&transaction.remote_object).await,
23113            "the test backend must retain the remote candidate"
23114        );
23115
23116        backend
23117            .set_transition_candidate_probe_override(Some(TransitionCandidateProbe::Unsupported))
23118            .await;
23119        let unsupported_stats = recover_transition_transaction_records(store.clone(), 100, None)
23120            .await
23121            .expect("active unknown ownership should remain fenced before provider recovery");
23122        assert_eq!(
23123            (
23124                unsupported_stats.scanned,
23125                unsupported_stats.recovered,
23126                unsupported_stats.retained,
23127                unsupported_stats.failed
23128            ),
23129            (1, 0, 1, 0),
23130            "active unknown ownership must retain the upload before the recovery deadline"
23131        );
23132        assert_eq!(transition_transaction_record_count(store.clone()).await, 1);
23133        let recovery_control_id =
23134            transition_recovery_control_id(&transaction).expect("transition recovery control id should derive");
23135        assert!(matches!(
23136            load_recovery_control(store.clone(), IlmRecoveryProtocol::TransitionTransaction, &recovery_control_id).await,
23137            Err(Error::ConfigNotFound)
23138        ));
23139        assert!(
23140            backend.contains(&transaction.remote_object).await,
23141            "active ownership must not delete the candidate"
23142        );
23143        assert_eq!(backend.remove_count().await, 0, "active ownership must not attempt cleanup");
23144        assert!(
23145            !backend
23146                .op_log()
23147                .await
23148                .iter()
23149                .any(|operation| matches!(operation, MockWarmOp::Probe { .. })),
23150            "active ownership must not probe the provider"
23151        );
23152
23153        backend.set_transition_candidate_probe_override(None).await;
23154        let stats =
23155            recover_transition_transaction_records_at(store.clone(), 100, None, i128::from(transaction.not_after_unix_nanos) + 1)
23156                .await
23157                .expect("provider-authoritative recovery should run");
23158        assert_eq!((stats.scanned, stats.recovered, stats.retained, stats.failed), (1, 1, 0, 0));
23159        assert_eq!(transition_transaction_record_count(store.clone()).await, 0);
23160        assert_eq!(backend.object_count().await, 0, "recovery must delete the provider-confirmed candidate");
23161        let op_log = backend.op_log().await;
23162        assert!(
23163            op_log.iter().any(|operation| matches!(operation, MockWarmOp::Probe { .. })),
23164            "response-loss recovery must enter the provider probe branch"
23165        );
23166        assert!(
23167            op_log.iter().any(|operation| matches!(operation, MockWarmOp::Put { .. })),
23168            "response-loss fixture must record that the remote PUT reached the backend"
23169        );
23170        let source_after = store
23171            .get_object_info(bucket, object, &ObjectOptions::default())
23172            .await
23173            .expect("recovery must preserve the local source object");
23174        assert_eq!(source_after.size, i64::try_from(payload.len()).expect("payload length should fit i64"));
23175    }
23176}