Skip to main content

rustfs_ecstore/store/
rebalance.rs

1// Copyright 2024 RustFS Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use super::*;
16use crate::config::storageclass;
17use crate::core::pools::merge_pool_status_refresh;
18use crate::layout::pool_space::{ServerPoolsAvailableSpace, build_server_pools_available_space};
19use crate::runtime::sources as runtime_sources;
20use crate::storage_api_contracts::{admin::StorageAdminApi, namespace::NamespaceLocking as _, object::ObjectOperations as _};
21use futures::stream::{FuturesUnordered, StreamExt};
22pub(in crate::store) mod support;
23
24const LOG_COMPONENT_ECSTORE: &str = "ecstore";
25const LOG_SUBSYSTEM_POOLS: &str = "pools";
26const EVENT_POOL_META_RELOAD: &str = "pool_meta_reload";
27
28#[cfg(test)]
29struct PreparedPoolReadFallbackBarrierState {
30    object: String,
31    pause_before_refetch: bool,
32    fanout_arrived: tokio::sync::Notify,
33    fanout_release: tokio::sync::Notify,
34    refetch_arrived: tokio::sync::Notify,
35    refetch_release: tokio::sync::Notify,
36}
37
38#[cfg(test)]
39pub(in crate::store) struct PreparedPoolReadFallbackBarrier {
40    state: Arc<PreparedPoolReadFallbackBarrierState>,
41}
42
43#[cfg(test)]
44static PREPARED_POOL_READ_FALLBACK_BARRIER: std::sync::OnceLock<
45    std::sync::Mutex<Option<Arc<PreparedPoolReadFallbackBarrierState>>>,
46> = std::sync::OnceLock::new();
47
48#[cfg(test)]
49impl PreparedPoolReadFallbackBarrier {
50    pub(in crate::store) fn install(object: &str, pause_before_refetch: bool) -> Self {
51        let state = Arc::new(PreparedPoolReadFallbackBarrierState {
52            object: object.to_string(),
53            pause_before_refetch,
54            fanout_arrived: tokio::sync::Notify::new(),
55            fanout_release: tokio::sync::Notify::new(),
56            refetch_arrived: tokio::sync::Notify::new(),
57            refetch_release: tokio::sync::Notify::new(),
58        });
59        *PREPARED_POOL_READ_FALLBACK_BARRIER
60            .get_or_init(|| std::sync::Mutex::new(None))
61            .lock()
62            .expect("prepared pool read fallback barrier must not be poisoned") = Some(Arc::clone(&state));
63        Self { state }
64    }
65
66    pub(in crate::store) async fn wait_after_fanout(&self) {
67        self.state.fanout_arrived.notified().await;
68    }
69
70    pub(in crate::store) fn release_after_fanout(&self) {
71        self.state.fanout_release.notify_one();
72    }
73
74    pub(in crate::store) async fn wait_before_refetch(&self) {
75        self.state.refetch_arrived.notified().await;
76    }
77
78    pub(in crate::store) fn release_before_refetch(&self) {
79        self.state.refetch_release.notify_one();
80    }
81}
82
83#[cfg(test)]
84impl Drop for PreparedPoolReadFallbackBarrier {
85    fn drop(&mut self) {
86        self.state.fanout_release.notify_waiters();
87        self.state.refetch_release.notify_waiters();
88        if let Some(barrier) = PREPARED_POOL_READ_FALLBACK_BARRIER.get() {
89            *barrier
90                .lock()
91                .expect("prepared pool read fallback barrier must not be poisoned") = None;
92        }
93    }
94}
95
96#[cfg(test)]
97async fn pause_prepared_pool_read_after_fanout(object: &str) {
98    let state = PREPARED_POOL_READ_FALLBACK_BARRIER
99        .get_or_init(|| std::sync::Mutex::new(None))
100        .lock()
101        .expect("prepared pool read fallback barrier must not be poisoned")
102        .as_ref()
103        .filter(|state| state.object == object)
104        .cloned();
105    if let Some(state) = state {
106        state.fanout_arrived.notify_one();
107        state.fanout_release.notified().await;
108    }
109}
110
111#[cfg(test)]
112async fn pause_prepared_pool_read_before_refetch(object: &str) {
113    let state = PREPARED_POOL_READ_FALLBACK_BARRIER
114        .get_or_init(|| std::sync::Mutex::new(None))
115        .lock()
116        .expect("prepared pool read fallback barrier must not be poisoned")
117        .as_ref()
118        .filter(|state| state.object == object && state.pause_before_refetch)
119        .cloned();
120    if let Some(state) = state {
121        state.refetch_arrived.notify_one();
122        state.refetch_release.notified().await;
123    }
124}
125#[cfg(test)]
126use support::resolve_latest_object_info_candidates;
127use support::{
128    LatestObjectInfoCandidate, PoolErr, PoolObjInfo, RebalanceDeletePoolResult, pool_lookup_not_found_error,
129    rebalance_disk_set_lookup_error, resolve_latest_object_info_candidates_with_pool_state,
130    resolve_rebalance_delete_from_all_pools_result, resolve_rebalance_delete_from_all_pools_results,
131    resolve_store_rebalance_pool_meta_reload_result, validate_prepared_pool_refetch_identity,
132};
133
134#[derive(Debug, Default, Eq, PartialEq)]
135struct BackendStorageClassInfo {
136    standard_sc_data: Vec<usize>,
137    standard_sc_parities: Vec<usize>,
138    standard_sc_parity: Option<usize>,
139    rr_sc_data: Vec<usize>,
140    rr_sc_parities: Vec<usize>,
141    rr_sc_parity: Option<usize>,
142}
143
144fn resolve_pool_layout(
145    drives_per_set: &[usize],
146    mut parity_for_pool: impl FnMut(usize, usize) -> Option<usize>,
147) -> Option<(Vec<usize>, Vec<usize>)> {
148    let mut parities = Vec::with_capacity(drives_per_set.len());
149    let mut data = Vec::with_capacity(drives_per_set.len());
150
151    for (pool_index, &drives) in drives_per_set.iter().enumerate() {
152        if drives == 0 {
153            return None;
154        }
155
156        let parity = parity_for_pool(pool_index, drives)?;
157        let data_drives = drives.checked_sub(parity)?;
158        if data_drives == 0 || parity > data_drives {
159            return None;
160        }
161
162        data.push(data_drives);
163        parities.push(parity);
164    }
165
166    Some((parities, data))
167}
168
169fn homogeneous_parity(parities: &[usize]) -> Option<usize> {
170    let first = *parities.first()?;
171    parities.iter().all(|&parity| parity == first).then_some(first)
172}
173
174fn resolve_complete_pool_layouts(
175    drives_per_set: &[usize],
176    standard_parity_for_pool: impl FnMut(usize, usize) -> Option<usize>,
177    rr_parity_for_pool: impl FnMut(usize, usize) -> Option<usize>,
178) -> Option<BackendStorageClassInfo> {
179    let (standard_sc_parities, standard_sc_data) = resolve_pool_layout(drives_per_set, standard_parity_for_pool)?;
180    let (rr_sc_parities, rr_sc_data) = resolve_pool_layout(drives_per_set, rr_parity_for_pool)?;
181
182    for ((&standard_parity, &rr_parity), &drives) in standard_sc_parities.iter().zip(&rr_sc_parities).zip(drives_per_set) {
183        storageclass::validate_parity_inner(standard_parity, rr_parity, drives).ok()?;
184    }
185
186    Some(BackendStorageClassInfo {
187        standard_sc_parity: homogeneous_parity(&standard_sc_parities),
188        standard_sc_data,
189        standard_sc_parities,
190        rr_sc_parity: homogeneous_parity(&rr_sc_parities),
191        rr_sc_data,
192        rr_sc_parities,
193    })
194}
195
196fn resolve_backend_storage_class_info(
197    config: &storageclass::Config,
198    drives_per_set: &[usize],
199    default_standard_parities: &[usize],
200) -> BackendStorageClassInfo {
201    if drives_per_set.len() != default_standard_parities.len() {
202        return BackendStorageClassInfo::default();
203    }
204
205    if !config.is_initialized() {
206        let Some((standard_sc_parities, standard_sc_data)) =
207            resolve_pool_layout(drives_per_set, |pool_index, _| default_standard_parities.get(pool_index).copied())
208        else {
209            return BackendStorageClassInfo::default();
210        };
211
212        return BackendStorageClassInfo {
213            standard_sc_parity: homogeneous_parity(&standard_sc_parities),
214            standard_sc_data,
215            standard_sc_parities,
216            ..Default::default()
217        };
218    }
219
220    match (config.parities_for_sc(storageclass::STANDARD), config.parities_for_sc(storageclass::RRS)) {
221        (Some(standard), Some(rr)) if standard.len() == drives_per_set.len() && rr.len() == drives_per_set.len() => {
222            resolve_complete_pool_layouts(
223                drives_per_set,
224                |pool_index, drives| config.parity_for_pool(storageclass::STANDARD, pool_index, drives),
225                |pool_index, drives| config.parity_for_pool(storageclass::RRS, pool_index, drives),
226            )
227            .unwrap_or_default()
228        }
229        (None, None) => {
230            let Some(standard) = config.get_parity_for_sc(storageclass::STANDARD) else {
231                return BackendStorageClassInfo::default();
232            };
233            let Some(rr) = config.get_parity_for_sc(storageclass::RRS) else {
234                return BackendStorageClassInfo::default();
235            };
236
237            resolve_complete_pool_layouts(drives_per_set, |_, _| Some(standard), |_, _| Some(rr)).unwrap_or_default()
238        }
239        _ => BackendStorageClassInfo::default(),
240    }
241}
242
243fn build_backend_info(
244    config: &storageclass::Config,
245    drives_per_set: &[usize],
246    default_standard_parities: &[usize],
247    total_sets: &[usize],
248) -> rustfs_madmin::BackendInfo {
249    let storage_class_info = if total_sets.len() == drives_per_set.len() {
250        resolve_backend_storage_class_info(config, drives_per_set, default_standard_parities)
251    } else {
252        BackendStorageClassInfo::default()
253    };
254
255    rustfs_madmin::BackendInfo {
256        backend_type: rustfs_madmin::BackendByte::Erasure,
257        online_disks: rustfs_madmin::BackendDisks::new(),
258        offline_disks: rustfs_madmin::BackendDisks::new(),
259        standard_sc_data: storage_class_info.standard_sc_data,
260        standard_sc_parities: storage_class_info.standard_sc_parities,
261        standard_sc_parity: storage_class_info.standard_sc_parity,
262        rr_sc_data: storage_class_info.rr_sc_data,
263        rr_sc_parities: storage_class_info.rr_sc_parities,
264        rr_sc_parity: storage_class_info.rr_sc_parity,
265        total_sets: total_sets.to_vec(),
266        drives_per_set: drives_per_set.to_vec(),
267    }
268}
269
270impl ECStore {
271    #[instrument(level = "debug", skip(self))]
272    pub(super) async fn delete_all(&self, bucket: &str, prefix: &str) -> Result<()> {
273        let mut futures = Vec::new();
274        for sets in self.pools.iter() {
275            for set in sets.disk_set.iter() {
276                futures.push(set.delete_all(bucket, prefix));
277                // let disks = set.disks.read().await;
278                // let dd = disks.clone();
279                // for disk in dd {
280                //     if disk.is_none() {
281                //         continue;
282                //     }
283                //     // let disk = disk.as_ref().expect("operation should succeed").clone();
284                //     // futures.push(disk.delete(
285                //     //     bucket,
286                //     //     prefix,
287                //     //     DeleteOptions {
288                //     //         recursive: true,
289                //     //         immediate: false,
290                //     //     },
291                //     // ));
292                // }
293            }
294        }
295        let results = join_all(futures).await;
296
297        let mut errs = Vec::new();
298
299        for res in results {
300            match res {
301                Ok(_) => errs.push(None),
302                Err(e) => errs.push(Some(e)),
303            }
304        }
305
306        debug!("store delete_all errs {:?}", errs);
307
308        Ok(())
309    }
310
311    pub(super) async fn delete_prefix(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<()> {
312        let dispatch_scope = if opts.tier_delete_journal_api.is_some() {
313            let incarnation = opts.expected_bucket_incarnation_id.ok_or(StorageError::PreconditionFailed)?;
314            let authorization = opts
315                .tier_delete_dispatch_authorization
316                .as_ref()
317                .ok_or_else(|| Error::other("prefix mutation is missing its dispatched-journal authorization"))?;
318            authorization.ensure_current(bucket, incarnation, object)?;
319            Some((authorization, incarnation))
320        } else {
321            None
322        };
323        if opts.lifecycle_delete_all.is_some() {
324            let mut preflight_opts = opts.clone();
325            preflight_opts
326                .lifecycle_delete_all
327                .as_mut()
328                .ok_or(StorageError::PreconditionFailed)?
329                .phase = crate::object_api::LifecycleDeleteAllPhase::Preflight;
330            for pool in &self.pools {
331                if let Some((authorization, incarnation)) = dispatch_scope {
332                    authorization.ensure_current(bucket, incarnation, object)?;
333                }
334                #[cfg(test)]
335                lifecycle_delete_all_test_failure(crate::object_api::LifecycleDeleteAllPhase::Preflight, pool.pool_idx)?;
336                pool.delete_object(bucket, object, preflight_opts.clone()).await?;
337            }
338
339            opts.lifecycle_delete_all_journal()
340                .ok_or(StorageError::PreconditionFailed)?
341                .lock()
342                .mark_mutation_started();
343            if let Some((authorization, incarnation)) = dispatch_scope {
344                authorization.mark_mutation_started(bucket, incarnation, object)?;
345            }
346            let mut non_trigger_opts = opts.clone();
347            non_trigger_opts
348                .lifecycle_delete_all
349                .as_mut()
350                .ok_or(StorageError::PreconditionFailed)?
351                .phase = crate::object_api::LifecycleDeleteAllPhase::History;
352            for pool in &self.pools {
353                if let Some((authorization, incarnation)) = dispatch_scope {
354                    authorization.ensure_current(bucket, incarnation, object)?;
355                }
356                #[cfg(test)]
357                lifecycle_delete_all_test_failure(crate::object_api::LifecycleDeleteAllPhase::History, pool.pool_idx)?;
358                let mut pool_opts = non_trigger_opts.clone();
359                pool_opts.delete_prefix = true;
360                pool.delete_object(bucket, object, pool_opts).await?;
361            }
362
363            let mut final_preflight_opts = opts.clone();
364            final_preflight_opts
365                .lifecycle_delete_all
366                .as_mut()
367                .ok_or(StorageError::PreconditionFailed)?
368                .phase = crate::object_api::LifecycleDeleteAllPhase::FinalPreflight;
369            let mut trigger_pools = Vec::new();
370            for (pool_index, pool) in self.pools.iter().enumerate() {
371                if let Some((authorization, incarnation)) = dispatch_scope {
372                    authorization.ensure_current(bucket, incarnation, object)?;
373                }
374                #[cfg(test)]
375                lifecycle_delete_all_test_failure(crate::object_api::LifecycleDeleteAllPhase::FinalPreflight, pool.pool_idx)?;
376                let result = pool.delete_object(bucket, object, final_preflight_opts.clone()).await?;
377                if !result.name.is_empty() {
378                    trigger_pools.push(pool_index);
379                }
380            }
381            if trigger_pools.is_empty() {
382                return Err(StorageError::PreconditionFailed);
383            }
384
385            let mut trigger_opts = opts.clone();
386            trigger_opts
387                .lifecycle_delete_all
388                .as_mut()
389                .ok_or(StorageError::PreconditionFailed)?
390                .phase = crate::object_api::LifecycleDeleteAllPhase::Trigger;
391            for pool_index in trigger_pools {
392                if let Some((authorization, incarnation)) = dispatch_scope {
393                    authorization.ensure_current(bucket, incarnation, object)?;
394                }
395                #[cfg(test)]
396                lifecycle_delete_all_test_failure(crate::object_api::LifecycleDeleteAllPhase::Trigger, pool_index)?;
397                let mut pool_opts = trigger_opts.clone();
398                pool_opts.delete_prefix = true;
399                self.pools[pool_index].delete_object(bucket, object, pool_opts).await?;
400            }
401            return Ok(());
402        }
403
404        let mut first_error = None;
405        let mut first_volume_error = None;
406        let mut has_success = false;
407        if let Some((authorization, incarnation)) = dispatch_scope {
408            authorization.mark_mutation_started(bucket, incarnation, object)?;
409        }
410        for pool in &self.pools {
411            if let Some((authorization, incarnation)) = dispatch_scope {
412                authorization.ensure_current(bucket, incarnation, object)?;
413            }
414            let mut opts = opts.clone();
415            opts.delete_prefix = true;
416            match pool.delete_object(bucket, object, opts).await {
417                Ok(_) => has_success = true,
418                Err(err) if is_err_strict_volume_not_found(&err) => {
419                    if first_volume_error.is_none() {
420                        first_volume_error = Some(err);
421                    }
422                }
423                Err(err) => {
424                    if first_error.is_none() {
425                        first_error = Some(err);
426                    }
427                }
428            }
429        }
430
431        match first_error {
432            Some(err) => Err(err),
433            None if has_success => Ok(()),
434            None => match first_volume_error {
435                Some(err) => Err(err),
436                None => Ok(()),
437            },
438        }
439    }
440
441    pub(super) async fn get_available_pool_idx(&self, bucket: &str, object: &str, size: i64) -> Option<usize> {
442        // // Return a random one first
443
444        let mut server_pools = self.get_server_pools_available_space(bucket, object, size).await;
445        server_pools.filter_max_used(100 - (100_f64 * DISK_RESERVE_FRACTION) as u64);
446        let total = server_pools.total_available();
447
448        if total == 0 {
449            return None;
450        }
451
452        let mut rng = rand::rng();
453        let random_u64: u64 = rng.random_range(0..total);
454
455        let choose = random_u64 % total;
456        let mut at_total = 0;
457
458        for pool in server_pools.iter() {
459            at_total += pool.available;
460            if at_total > choose && pool.available > 0 {
461                return Some(pool.index);
462            }
463        }
464
465        None
466    }
467
468    pub(super) async fn get_available_pool_idx_excluding(
469        &self,
470        bucket: &str,
471        object: &str,
472        size: i64,
473        excluded_pool_idx: usize,
474    ) -> Option<usize> {
475        let mut server_pools = self.get_server_pools_available_space(bucket, object, size).await;
476        server_pools.filter_max_used(100 - (100_f64 * DISK_RESERVE_FRACTION) as u64);
477
478        if let Some(pool) = server_pools.0.get_mut(excluded_pool_idx) {
479            pool.available = 0;
480        }
481
482        let total = server_pools.total_available();
483        if total == 0 {
484            return None;
485        }
486
487        let mut rng = rand::rng();
488        let random_u64: u64 = rng.random_range(0..total);
489
490        let choose = random_u64 % total;
491        let mut at_total = 0;
492
493        for pool in server_pools.iter() {
494            at_total += pool.available;
495            if at_total > choose && pool.available > 0 {
496                return Some(pool.index);
497            }
498        }
499
500        None
501    }
502
503    async fn get_server_pools_available_space(&self, bucket: &str, object: &str, size: i64) -> ServerPoolsAvailableSpace {
504        let mut n_sets = vec![0; self.pools.len()];
505        let mut infos = vec![Vec::new(); self.pools.len()];
506        let pool_inputs = join_all(self.pools.iter().enumerate().map(|(idx, pool)| async move {
507            if self.is_suspended(idx).await || self.is_pool_rebalancing(idx).await {
508                return (idx, 0, Vec::new());
509            }
510
511            let disks = pool.get_disks_by_key(object).disk_inventory().await;
512            let disk_infos = get_disk_infos(&disks).await;
513
514            (idx, pool.set_count, disk_infos)
515        }))
516        .await;
517
518        for (idx, set_count, disk_infos) in pool_inputs {
519            n_sets[idx] = set_count;
520            infos[idx] = disk_infos;
521        }
522
523        build_server_pools_available_space(bucket, size, &n_sets, &infos).await
524    }
525
526    pub(super) async fn is_suspended(&self, idx: usize) -> bool {
527        // TODO(backlog): acquire pool metadata lock for consistent suspension check
528
529        let pool_meta = self.pool_meta.read().await;
530
531        pool_meta.is_suspended(idx)
532    }
533
534    pub(super) async fn get_pool_idx(&self, bucket: &str, object: &str, size: i64) -> Result<usize> {
535        let idx = match self
536            .get_pool_idx_existing_with_opts(
537                bucket,
538                object,
539                &ObjectOptions {
540                    skip_decommissioned: true,
541                    skip_rebalancing: true,
542                    ..Default::default()
543                },
544            )
545            .await
546        {
547            Ok(res) => res,
548            Err(err) => {
549                if !is_err_object_not_found(&err) {
550                    return Err(err);
551                }
552
553                if let Some(hit_idx) = self.get_available_pool_idx(bucket, object, size).await {
554                    hit_idx
555                } else {
556                    return Err(Error::DiskFull);
557                }
558            }
559        };
560
561        Ok(idx)
562    }
563
564    pub(super) async fn get_pool_idx_no_lock(&self, bucket: &str, object: &str, size: i64) -> Result<usize> {
565        let idx = match self.get_pool_idx_existing_no_lock(bucket, object).await {
566            Ok(res) => res,
567            Err(err) => {
568                if !is_err_object_not_found(&err) {
569                    return Err(err);
570                }
571
572                if let Some(idx) = self.get_available_pool_idx(bucket, object, size).await {
573                    idx
574                } else {
575                    warn!("get_pool_idx_no_lock: disk full {}/{}", bucket, object);
576                    return Err(Error::DiskFull);
577                }
578            }
579        };
580
581        Ok(idx)
582    }
583
584    async fn get_pool_idx_existing_no_lock(&self, bucket: &str, object: &str) -> Result<usize> {
585        self.get_pool_idx_existing_with_opts(
586            bucket,
587            object,
588            &ObjectOptions {
589                no_lock: true,
590                skip_decommissioned: true,
591                skip_rebalancing: true,
592                ..Default::default()
593            },
594        )
595        .await
596    }
597
598    pub(super) async fn get_pool_idx_existing_with_opts(
599        &self,
600        bucket: &str,
601        object: &str,
602        opts: &ObjectOptions,
603    ) -> Result<usize> {
604        let (pinfo, _) = self.get_pool_info_existing_with_opts(bucket, object, opts).await?;
605        Ok(pinfo.index)
606    }
607
608    pub(super) async fn get_pool_info_existing_with_opts(
609        &self,
610        bucket: &str,
611        object: &str,
612        opts: &ObjectOptions,
613    ) -> Result<(PoolObjInfo, Vec<PoolErr>)> {
614        self.internal_get_pool_info_existing_with_opts(bucket, object, opts, false)
615            .await
616    }
617
618    pub(super) async fn get_pool_info_for_delete_marker(
619        &self,
620        bucket: &str,
621        object: &str,
622        opts: &ObjectOptions,
623    ) -> Result<(PoolObjInfo, Vec<PoolErr>)> {
624        self.internal_get_pool_info_existing_with_opts(bucket, object, opts, true)
625            .await
626    }
627
628    async fn internal_get_pool_info_existing_with_opts(
629        &self,
630        bucket: &str,
631        object: &str,
632        opts: &ObjectOptions,
633        require_all_pool_reads: bool,
634    ) -> Result<(PoolObjInfo, Vec<PoolErr>)> {
635        let mut futures = Vec::new();
636        for pool in self.pools.iter() {
637            let mut pool_opts = opts.clone();
638            if !pool_opts.metadata_chg {
639                pool_opts.version_id = None;
640            }
641
642            futures.push(async move { pool.get_object_info(bucket, object, &pool_opts).await });
643        }
644
645        let results = join_all(futures).await;
646
647        let mut ress = Vec::new();
648
649        // join_all preserves the input order
650        for (i, res) in results.into_iter().enumerate() {
651            let index = i;
652
653            match res {
654                Ok(r) => {
655                    ress.push(PoolObjInfo {
656                        index,
657                        object_info: r,
658                        err: None,
659                    });
660                }
661                Err(e) => {
662                    // A readable older pool cannot prove ownership of the
663                    // current version while another pool is unreadable. Check
664                    // both raw and object-scoped quorum errors before sorting.
665                    if require_all_pool_reads && !is_err_object_not_found(&e) && !is_err_version_not_found(&e) {
666                        return Err(match e {
667                            Error::ErasureReadQuorum | Error::InsufficientReadQuorum(_, _) => Error::ErasureWriteQuorum,
668                            err => err,
669                        });
670                    }
671                    ress.push(PoolObjInfo {
672                        index,
673                        err: Some(e),
674                        ..Default::default()
675                    });
676                }
677            }
678        }
679
680        if require_all_pool_reads {
681            let suspended_pools = {
682                let pool_meta = self.pool_meta.read().await;
683                (0..self.pools.len())
684                    .map(|idx| pool_meta.is_suspended(idx))
685                    .collect::<Vec<_>>()
686            };
687            let candidates = ress
688                .iter()
689                .map(|pinfo| LatestObjectInfoCandidate {
690                    info: pinfo.err.is_none().then(|| pinfo.object_info.clone()),
691                    idx: pinfo.index,
692                    err: pinfo.err.clone(),
693                })
694                .collect();
695            let (object_info, index) =
696                resolve_latest_object_info_candidates_with_pool_state(candidates, &suspended_pools, bucket, object, opts)?;
697            let pools_with_object = self.pools_with_object(&ress, opts).await;
698            return Ok((
699                PoolObjInfo {
700                    index,
701                    object_info,
702                    err: None,
703                },
704                pools_with_object,
705            ));
706        }
707
708        ress.sort_by(|a, b| {
709            let at = a.object_info.mod_time.unwrap_or(OffsetDateTime::UNIX_EPOCH);
710            let bt = b.object_info.mod_time.unwrap_or(OffsetDateTime::UNIX_EPOCH);
711
712            bt.cmp(&at)
713        });
714
715        let mut def_pool = PoolObjInfo::default();
716        let mut has_def_pool = false;
717
718        for pinfo in ress.iter() {
719            if opts.skip_decommissioned && self.is_suspended(pinfo.index).await {
720                continue;
721            }
722
723            if opts.skip_rebalancing && self.is_pool_rebalancing(pinfo.index).await {
724                continue;
725            }
726
727            if pinfo.err.is_none() {
728                return Ok((pinfo.clone(), self.pools_with_object(&ress, opts).await));
729            }
730
731            let err = pinfo.err.as_ref().expect("operation should succeed");
732
733            if err == &Error::ErasureReadQuorum && !opts.metadata_chg {
734                return Ok((pinfo.clone(), self.pools_with_object(&ress, opts).await));
735            }
736
737            def_pool = pinfo.clone();
738            has_def_pool = true;
739            // https://docs.aws.amazon.com/AmazonS3/latest/userguide/conditional-deletes.html
740            if is_err_object_not_found(err)
741                && let Err(err) = opts.precondition_check(&pinfo.object_info)
742            {
743                return Err(err);
744            }
745
746            if !is_err_object_not_found(err) && !is_err_version_not_found(err) {
747                return Err(err.clone());
748            }
749
750            if pinfo.object_info.delete_marker && !pinfo.object_info.name.is_empty() {
751                return Ok((pinfo.clone(), Vec::new()));
752            }
753        }
754
755        if opts.replication_request && opts.delete_marker && has_def_pool {
756            return Ok((def_pool, Vec::new()));
757        }
758
759        Err(pool_lookup_not_found_error(bucket, object, opts))
760    }
761
762    async fn pools_with_object(&self, pools: &[PoolObjInfo], opts: &ObjectOptions) -> Vec<PoolErr> {
763        let mut errs = Vec::new();
764
765        for pool in pools.iter() {
766            if opts.skip_decommissioned && self.is_suspended(pool.index).await {
767                continue;
768            }
769
770            if opts.skip_rebalancing && self.is_pool_rebalancing(pool.index).await {
771                continue;
772            }
773
774            if let Some(err) = &pool.err {
775                if err == &Error::ErasureReadQuorum {
776                    errs.push(PoolErr {
777                        index: Some(pool.index),
778                        err: Some(Error::ErasureReadQuorum),
779                    });
780                }
781            } else {
782                errs.push(PoolErr {
783                    index: Some(pool.index),
784                    err: None,
785                });
786            }
787        }
788        errs
789    }
790
791    pub(super) async fn get_latest_object_info_with_idx(
792        &self,
793        bucket: &str,
794        object: &str,
795        opts: &ObjectOptions,
796    ) -> Result<(ObjectInfo, usize)> {
797        let suspended_pools = if opts.skip_decommissioned {
798            let pool_meta = self.pool_meta.read().await;
799            Some(
800                (0..self.pools.len())
801                    .map(|idx| pool_meta.is_suspended(idx))
802                    .collect::<Vec<_>>(),
803            )
804        } else {
805            None
806        };
807        let mut futures = Vec::with_capacity(self.pools.len());
808        for (idx, pool) in self.pools.iter().enumerate() {
809            if suspended_pools.as_ref().is_some_and(|pools| pools[idx]) {
810                continue;
811            }
812
813            if opts.skip_rebalancing && self.is_pool_rebalancing(idx).await {
814                continue;
815            }
816
817            futures.push(async move { (idx, pool.get_object_info(bucket, object, opts).await) });
818        }
819
820        let results = join_all(futures).await;
821        let mut candidates = Vec::with_capacity(self.pools.len());
822
823        for (idx, result) in results {
824            match result {
825                Ok(res) => {
826                    candidates.push(LatestObjectInfoCandidate {
827                        info: Some(res),
828                        idx,
829                        err: None,
830                    });
831                }
832                Err(e) => {
833                    candidates.push(LatestObjectInfoCandidate {
834                        info: None,
835                        idx,
836                        err: Some(e),
837                    });
838                }
839            }
840        }
841
842        let suspended_pools = match suspended_pools {
843            Some(pools) => pools,
844            None => {
845                let pool_meta = self.pool_meta.read().await;
846                (0..self.pools.len())
847                    .map(|idx| pool_meta.is_suspended(idx))
848                    .collect::<Vec<_>>()
849            }
850        };
851
852        // Delete markers are returned as latest object infos here. Higher-level
853        // access paths are responsible for translating them into read/write
854        // semantics such as object-not-found or method-not-allowed.
855        resolve_latest_object_info_candidates_with_pool_state(candidates, &suspended_pools, bucket, object, opts)
856    }
857
858    pub(super) async fn prepare_latest_object_metadata_with_idx(
859        &self,
860        bucket: &str,
861        object: &str,
862        opts: &ObjectOptions,
863    ) -> Result<(crate::set_disk::PreparedGetObjectMetadata, usize)> {
864        let suspended_pools = if opts.skip_decommissioned {
865            let pool_meta = self.pool_meta.read().await;
866            Some(
867                (0..self.pools.len())
868                    .map(|idx| pool_meta.is_suspended(idx))
869                    .collect::<Vec<_>>(),
870            )
871        } else {
872            None
873        };
874        let mut futures = FuturesUnordered::new();
875        for (idx, pool) in self.pools.iter().enumerate() {
876            if suspended_pools.as_ref().is_some_and(|pools| pools[idx]) {
877                continue;
878            }
879
880            if opts.skip_rebalancing && self.is_pool_rebalancing(idx).await {
881                continue;
882            }
883
884            futures.push(async move {
885                let result = pool
886                    .prepare_get_object_reader_metadata(bucket, object, opts)
887                    .await
888                    .map_err(|err| to_object_err(err, vec![bucket, object]));
889                (idx, result)
890            });
891        }
892
893        let mut candidates = (0..self.pools.len()).map(|_| None).collect::<Vec<_>>();
894        // Retain one provisional winner. Other pools only need their lightweight
895        // identity for final conflict checks; if pool state changes while the
896        // fanout runs, the final winner is refetched and revalidated below.
897        let mut latest_prepared = None;
898        let mut latest_mod_time = None;
899        let mut provisional_dynamic_pool_state = None;
900        while let Some((idx, result)) = futures.next().await {
901            match result {
902                Ok(metadata) => {
903                    let mod_time = metadata.object_info().mod_time.unwrap_or(OffsetDateTime::UNIX_EPOCH);
904                    let info = metadata.object_info().clone();
905                    let retain = match (latest_mod_time, latest_prepared.as_ref()) {
906                        (None, _) => true,
907                        (Some(current), _) if mod_time > current => true,
908                        (Some(current), _) if mod_time < current => false,
909                        (Some(_), Some((current_idx, _))) => {
910                            if suspended_pools.is_none() && provisional_dynamic_pool_state.is_none() {
911                                let pool_meta = self.pool_meta.read().await;
912                                provisional_dynamic_pool_state = Some(
913                                    (0..self.pools.len())
914                                        .map(|pool_idx| pool_meta.is_suspended(pool_idx))
915                                        .collect::<Vec<_>>(),
916                                );
917                            }
918                            let provisional_pool_state = suspended_pools
919                                .as_ref()
920                                .or(provisional_dynamic_pool_state.as_ref())
921                                .ok_or_else(|| Error::other("GET pool state snapshot is unavailable"))?;
922                            let new_key = (provisional_pool_state.get(idx).copied().unwrap_or(false), std::cmp::Reverse(idx));
923                            let current_key = (
924                                provisional_pool_state.get(*current_idx).copied().unwrap_or(false),
925                                std::cmp::Reverse(*current_idx),
926                            );
927                            new_key < current_key
928                        }
929                        (Some(_), None) => true,
930                    };
931                    if retain {
932                        if latest_mod_time.is_none_or(|current| mod_time > current) {
933                            latest_mod_time = Some(mod_time);
934                        }
935                        latest_prepared = Some((idx, metadata));
936                    }
937                    candidates[idx] = Some(LatestObjectInfoCandidate {
938                        info: Some(info),
939                        idx,
940                        err: None,
941                    });
942                }
943                Err(err) => {
944                    candidates[idx] = Some(LatestObjectInfoCandidate {
945                        info: None,
946                        idx,
947                        err: Some(err),
948                    });
949                }
950            }
951        }
952
953        #[cfg(test)]
954        pause_prepared_pool_read_after_fanout(object).await;
955
956        let suspended_pools = match suspended_pools {
957            Some(pools) => pools,
958            None => {
959                let pool_meta = self.pool_meta.read().await;
960                (0..self.pools.len())
961                    .map(|idx| pool_meta.is_suspended(idx))
962                    .collect::<Vec<_>>()
963            }
964        };
965
966        let candidates = candidates.into_iter().flatten().collect();
967        let (winner_info, winner_idx) =
968            resolve_latest_object_info_candidates_with_pool_state(candidates, &suspended_pools, bucket, object, opts)?;
969        if let Some((prepared_idx, metadata)) = latest_prepared
970            && prepared_idx == winner_idx
971        {
972            return Ok((metadata, winner_idx));
973        }
974
975        let pool = self.pools.get(winner_idx).ok_or(Error::ErasureReadQuorum)?;
976        #[cfg(test)]
977        pause_prepared_pool_read_before_refetch(object).await;
978        let metadata = pool
979            .prepare_get_object_reader_metadata(bucket, object, opts)
980            .await
981            .map_err(|err| to_object_err(err, vec![bucket, object]))?;
982        validate_prepared_pool_refetch_identity(&winner_info, metadata.object_info())?;
983        Ok((metadata, winner_idx))
984    }
985
986    pub(super) async fn delete_object_from_all_pools(
987        &self,
988        bucket: &str,
989        object: &str,
990        opts: &ObjectOptions,
991        exact: &ObjectInfo,
992        errs: Vec<PoolErr>,
993    ) -> Result<ObjectInfo> {
994        self.reconcile_decommission_capacity_before_exact_delete(bucket, object, opts, exact)
995            .await?;
996
997        let mut results = Vec::with_capacity(errs.len());
998
999        for pe in errs.iter() {
1000            if let Some(err) = &pe.err
1001                && err == &StorageError::ErasureWriteQuorum
1002            {
1003                if let Some(idx) = pe.index {
1004                    results.push(RebalanceDeletePoolResult {
1005                        pool_idx: idx,
1006                        result: Err(StorageError::ErasureWriteQuorum),
1007                    });
1008                }
1009                continue;
1010            }
1011
1012            if let Some(idx) = pe.index {
1013                let pool = self.pools[idx].clone();
1014                results.push(RebalanceDeletePoolResult {
1015                    pool_idx: idx,
1016                    result: self
1017                        .run_external_decommission_capacity_object_delete(
1018                            idx,
1019                            bucket,
1020                            object,
1021                            object,
1022                            opts.clone(),
1023                            |opts| async move { pool.delete_object(bucket, object, opts).await },
1024                        )
1025                        .await,
1026                });
1027            }
1028        }
1029
1030        resolve_rebalance_delete_from_all_pools_result(
1031            resolve_rebalance_delete_from_all_pools_results(results, bucket, object),
1032            bucket,
1033            object,
1034        )
1035    }
1036
1037    /// Peer reload entry: refreshes in-memory pool metadata from the shared
1038    /// persisted snapshot. Returns whether newer state was actually merged so
1039    /// callers only trigger missing-worker recovery after a real state change;
1040    /// delayed snapshots are merged monotonically and never blind-assigned.
1041    pub async fn reload_pool_meta(&self) -> Result<bool> {
1042        // Serialize the durable reload with local movement transitions. Loading
1043        // before acquiring this gate would allow a stale disk snapshot to
1044        // overwrite a newer local transition after the writer commits.
1045        let movement_gate = self.ctx.data_movement_operation_gate();
1046        let _movement_guard = movement_gate.write().await;
1047        let reloaded = resolve_store_rebalance_pool_meta_reload_result(
1048            self.load_runtime_pool_meta("store rebalance pool meta reload failed").await,
1049            "reload_pool_meta",
1050        )?;
1051
1052        // Lock order: release the decommission_cancelers guard before taking
1053        // the pool_meta write guard; neither is held without the movement gate.
1054        let active_workers = {
1055            let cancelers = self.decommission_cancelers.read().await;
1056            cancelers
1057                .iter()
1058                .map(|canceler| canceler.as_ref().is_some_and(DecommissionCanceler::is_active))
1059                .collect::<Vec<_>>()
1060        };
1061
1062        let incoming_has_pools = !reloaded.pools.is_empty();
1063        let mut pool_meta = self.pool_meta.write().await;
1064        let movement_before = pool_meta.clone();
1065        let merged_newer = merge_pool_status_refresh(&mut pool_meta, reloaded, &active_workers);
1066        if crate::core::pools::pool_meta_movement_snapshot_changed(&movement_before, &pool_meta) {
1067            self.ctx.advance_data_movement_operation_epoch();
1068        }
1069
1070        if !merged_newer && !incoming_has_pools {
1071            warn!(
1072                event = EVENT_POOL_META_RELOAD,
1073                component = LOG_COMPONENT_ECSTORE,
1074                subsystem = LOG_SUBSYSTEM_POOLS,
1075                result = "ignored",
1076                reason = "missing_metadata",
1077                "Peer pool meta reload ignored because persisted metadata is missing"
1078            );
1079        } else if !merged_newer {
1080            debug!(
1081                event = EVENT_POOL_META_RELOAD,
1082                component = LOG_COMPONENT_ECSTORE,
1083                subsystem = LOG_SUBSYSTEM_POOLS,
1084                result = "ignored",
1085                reason = "stale_snapshot",
1086                "Peer pool meta reload ignored as a stale snapshot"
1087            );
1088        }
1089
1090        Ok(merged_newer)
1091    }
1092
1093    /// Disk information deduplication function
1094    ///
1095    /// Use multiple field combinations to ensure uniqueness:
1096    /// - endpoint (node address)
1097    /// - drive_path (mount path)
1098    /// - pool_index (pool index)
1099    /// - set_index (Collection Index)
1100    /// - disk_index (disk index)
1101    pub(crate) fn deduplicate_disks(disks: Vec<rustfs_madmin::Disk>) -> Vec<rustfs_madmin::Disk> {
1102        use std::collections::HashMap;
1103        use std::collections::hash_map::Entry;
1104
1105        let mut unique_disks: HashMap<String, rustfs_madmin::Disk> = HashMap::new();
1106        let mut duplicate_count = 0;
1107
1108        for disk in disks {
1109            let key = format!(
1110                "{}|{}|p{}s{}d{}",
1111                disk.endpoint, disk.drive_path, disk.pool_index, disk.set_index, disk.disk_index
1112            );
1113
1114            match unique_disks.entry(key) {
1115                Entry::Vacant(entry) => {
1116                    entry.insert(disk);
1117                }
1118                Entry::Occupied(_) => {
1119                    duplicate_count += 1;
1120                }
1121            }
1122        }
1123
1124        if duplicate_count > 0 {
1125            debug!("Deduplicated {} duplicate disk entries", duplicate_count);
1126        }
1127
1128        unique_disks.into_values().collect()
1129    }
1130
1131    #[instrument(level = "trace", skip(self))]
1132    pub(super) async fn handle_new_ns_lock(&self, bucket: &str, object: &str) -> Result<NamespaceLockWrapper> {
1133        self.pools[0].new_ns_lock(bucket, object).await
1134    }
1135
1136    #[instrument(skip(self))]
1137    pub(super) async fn handle_backend_info(&self) -> rustfs_madmin::BackendInfo {
1138        let drives_per_set = StorageAdminApi::set_drive_counts(self);
1139        let default_standard_parities = self.pools.iter().map(|pool| pool.default_parity_count).collect::<Vec<_>>();
1140        let storage_class = runtime_sources::storage_class_config_snapshot();
1141        let total_sets = self.pools.iter().map(|pool| pool.set_count).collect::<Vec<_>>();
1142
1143        build_backend_info(&storage_class, &drives_per_set, &default_standard_parities, &total_sets)
1144    }
1145
1146    #[instrument(skip(self))]
1147    pub(super) async fn handle_storage_info(&self) -> rustfs_madmin::StorageInfo {
1148        let Some(notification_sy) = runtime_sources::notification_sys() else {
1149            return rustfs_madmin::StorageInfo::default();
1150        };
1151
1152        let mut info = notification_sy.storage_info(self).await;
1153
1154        // 🔧 Defensive deduplication: This protection mechanism is retained even if the upstream is fixed
1155        let original_count = info.disks.len();
1156        info.disks = Self::deduplicate_disks(info.disks);
1157        let final_count = info.disks.len();
1158
1159        if original_count != final_count {
1160            warn!(
1161                "Storage info deduplication: removed {} duplicate disk entries ({} -> {})",
1162                original_count - final_count,
1163                original_count,
1164                final_count
1165            );
1166        }
1167
1168        info
1169    }
1170
1171    #[instrument(skip(self))]
1172    pub(super) async fn handle_local_storage_info(&self) -> rustfs_madmin::StorageInfo {
1173        let mut futures = Vec::with_capacity(self.pools.len());
1174
1175        for pool in self.pools.iter() {
1176            futures.push(pool.local_storage_info_snapshot())
1177        }
1178
1179        let results = join_all(futures).await;
1180
1181        let mut disks = Vec::new();
1182
1183        for res in results.into_iter() {
1184            disks.extend_from_slice(&res.disks);
1185        }
1186
1187        // 🔧 Defensive deduplication: when aggregating disks from all pools, drop duplicate
1188        //  entries that may be reported multiple times by backends; this extra layer is kept
1189        //  even if the upstream reporting is later fixed.
1190        let original_count = disks.len();
1191        disks = Self::deduplicate_disks(disks);
1192
1193        if original_count != disks.len() {
1194            warn!("Local storage info deduplication: {} -> {}", original_count, disks.len());
1195        }
1196
1197        let backend = StorageAdminApi::backend_info(self).await;
1198        rustfs_madmin::StorageInfo {
1199            backend,
1200            disks,
1201            ..Default::default()
1202        }
1203    }
1204
1205    #[instrument(skip(self))]
1206    pub(super) async fn handle_get_disks(&self, pool_idx: usize, set_idx: usize) -> Result<Vec<Option<DiskStore>>> {
1207        if pool_idx < self.pools.len() && set_idx < self.pools[pool_idx].disk_set.len() {
1208            Ok(self.pools[pool_idx].disk_set[set_idx].disk_inventory().await)
1209        } else {
1210            Err(rebalance_disk_set_lookup_error(pool_idx, set_idx, self.pools.len()))
1211        }
1212    }
1213
1214    #[instrument(skip(self))]
1215    pub(super) fn handle_set_drive_counts(&self) -> Vec<usize> {
1216        let mut counts = vec![0; self.pools.len()];
1217
1218        for (i, pool) in self.pools.iter().enumerate() {
1219            counts[i] = pool.set_drive_count();
1220        }
1221        counts
1222    }
1223
1224    #[instrument(skip(self))]
1225    pub(super) async fn handle_get_pool_and_set(&self, id: &str) -> Result<(Option<usize>, Option<usize>, Option<usize>)> {
1226        for (pool_idx, pool) in self.pools.iter().enumerate() {
1227            for (set_idx, set) in pool.format.erasure.sets.iter().enumerate() {
1228                for (disk_idx, disk_id) in set.iter().enumerate() {
1229                    if disk_id.to_string() == id {
1230                        return Ok((Some(pool_idx), Some(set_idx), Some(disk_idx)));
1231                    }
1232                }
1233            }
1234        }
1235
1236        Err(Error::DiskNotFound)
1237    }
1238}
1239
1240#[cfg(test)]
1241static LIFECYCLE_DELETE_ALL_TEST_FAILURE: std::sync::Mutex<Option<(crate::object_api::LifecycleDeleteAllPhase, usize)>> =
1242    std::sync::Mutex::new(None);
1243
1244#[cfg(test)]
1245fn lifecycle_delete_all_test_failure(phase: crate::object_api::LifecycleDeleteAllPhase, pool_index: usize) -> Result<()> {
1246    if LIFECYCLE_DELETE_ALL_TEST_FAILURE
1247        .lock()
1248        .expect("lifecycle delete-all failure hook should not poison")
1249        .is_some_and(|failure| failure == (phase, pool_index))
1250    {
1251        return Err(StorageError::PreconditionFailed);
1252    }
1253    Ok(())
1254}
1255
1256#[cfg(test)]
1257mod tests {
1258    use super::*;
1259    use crate::bucket::replication::{ReplicationStatusType, VersionPurgeStatusType};
1260    use crate::config::com::{read_config_no_lock_preserve_empty_with_metadata, save_config};
1261    use crate::config::storageclass::{CLASS_RRS, CLASS_STANDARD, lookup_config_for_pools_without_env};
1262    use crate::core::pools::{
1263        POOL_META_IDENTITY_NAME, POOL_META_NAME, POOL_META_VERSION, PoolDecommissionInfo, PoolStatus,
1264        initialized_pool_meta_identity_for_test, pool_meta_v3_commit_state_for_test,
1265    };
1266    use crate::core::sets::Sets;
1267    use crate::disk::error::DiskError;
1268    use crate::layout::endpoint::Endpoint;
1269    use crate::layout::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints};
1270    use crate::object_api::ObjectLockConfigSnapshot;
1271    use crate::set_disk::get_lock_acquire_timeout;
1272    use crate::storage_api_contracts::bucket::MakeBucketOptions;
1273    use crate::storage_api_contracts::object::ObjectIO as _;
1274    use arc_swap::ArcSwap;
1275    use rustfs_config::server_config::KVS;
1276    use rustfs_filemeta::FileInfo;
1277    use std::sync::Arc;
1278    use time::{Duration as TimeDuration, OffsetDateTime};
1279    use tokio_util::sync::CancellationToken;
1280
1281    async fn setup_multi_pool_test_store(
1282        name: &str,
1283        drives_per_pool: &[usize],
1284    ) -> (tempfile::TempDir, Arc<ECStore>, CancellationToken) {
1285        let temp_dir = tempfile::tempdir().expect("multi-pool test directory should be created");
1286        let mut pools = Vec::with_capacity(drives_per_pool.len());
1287        for (pool_index, drives_per_set) in drives_per_pool.iter().copied().enumerate() {
1288            let mut endpoints = Vec::with_capacity(drives_per_set);
1289            for disk_index in 0..drives_per_set {
1290                let disk_path = temp_dir.path().join(format!("pool{pool_index}-disk{disk_index}"));
1291                tokio::fs::create_dir_all(&disk_path)
1292                    .await
1293                    .expect("multi-pool test disk should be created");
1294                let mut endpoint =
1295                    Endpoint::try_from(disk_path.to_str().expect("disk path should be utf8")).expect("endpoint should parse");
1296                endpoint.set_pool_index(pool_index);
1297                endpoint.set_set_index(0);
1298                endpoint.set_disk_index(disk_index);
1299                endpoints.push(endpoint);
1300            }
1301            pools.push(PoolEndpoints {
1302                legacy: false,
1303                set_count: 1,
1304                drives_per_set,
1305                endpoints: Endpoints::from(endpoints),
1306                cmd_line: format!("{name}-pool-{pool_index}"),
1307                platform: "test".to_string(),
1308            });
1309        }
1310
1311        let endpoint_pools = EndpointServerPools(pools);
1312        let instance_ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
1313        crate::store::init_local_disks_with_instance_ctx(&instance_ctx, endpoint_pools.clone())
1314            .await
1315            .expect("multi-pool local disks should initialize");
1316        let shutdown = CancellationToken::new();
1317        let store = ECStore::new_with_instance_ctx(
1318            "127.0.0.1:0".parse().expect("test address should parse"),
1319            endpoint_pools,
1320            shutdown.clone(),
1321            instance_ctx,
1322        )
1323        .await
1324        .expect("multi-pool store should initialize");
1325        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
1326        (temp_dir, store, shutdown)
1327    }
1328
1329    struct LifecycleDeleteAllFailureGuard;
1330
1331    impl Drop for LifecycleDeleteAllFailureGuard {
1332        fn drop(&mut self) {
1333            *LIFECYCLE_DELETE_ALL_TEST_FAILURE
1334                .lock()
1335                .expect("lifecycle delete-all failure hook should not poison") = None;
1336        }
1337    }
1338
1339    async fn seed_multi_pool_delete_all(store: &Arc<ECStore>, bucket: &str, object: &str) -> ObjectOptions {
1340        let trigger_id = Uuid::new_v4();
1341        for (pool_index, pool) in store.pools.iter().enumerate() {
1342            let mut history_reader = PutObjReader::from_vec(format!("{object}-history-{pool_index}").into_bytes());
1343            pool.put_object(
1344                bucket,
1345                object,
1346                &mut history_reader,
1347                &ObjectOptions {
1348                    versioned: true,
1349                    version_id: Some(Uuid::new_v4().to_string()),
1350                    mod_time: Some(OffsetDateTime::UNIX_EPOCH + time::Duration::seconds(1)),
1351                    ..Default::default()
1352                },
1353            )
1354            .await
1355            .expect("history should be stored");
1356            let mut trigger_reader = PutObjReader::from_vec(format!("{object}-trigger-{pool_index}").into_bytes());
1357            pool.put_object(
1358                bucket,
1359                object,
1360                &mut trigger_reader,
1361                &ObjectOptions {
1362                    versioned: true,
1363                    version_id: Some(trigger_id.to_string()),
1364                    mod_time: Some(OffsetDateTime::UNIX_EPOCH + time::Duration::seconds(2)),
1365                    ..Default::default()
1366                },
1367            )
1368            .await
1369            .expect("shared trigger should be stored");
1370        }
1371        let mut opts = ObjectOptions {
1372            delete_prefix: true,
1373            delete_prefix_object: true,
1374            versioned: true,
1375            lifecycle_delete_all: Some(crate::object_api::LifecycleDeleteAllRequest {
1376                version_id: Some(trigger_id),
1377                delete_marker: false,
1378                action: rustfs_scanner_metrics::metrics::IlmAction::DeleteAllVersionsAction,
1379                rule_id: "rule".to_string(),
1380                phase: crate::object_api::LifecycleDeleteAllPhase::Preflight,
1381            }),
1382            object_lock_config_snapshot: Some(Arc::new(ObjectLockConfigSnapshot::new(
1383                crate::bucket::metadata_sys::ObjectLockConfigState::ConfirmedAbsent,
1384            ))),
1385            delete_replication_config_snapshot: Some(Arc::new(
1386                crate::bucket::replication::DeleteReplicationConfigSnapshot::default(),
1387            )),
1388            ..Default::default()
1389        };
1390        opts.ensure_lifecycle_delete_all_journal();
1391        opts
1392    }
1393
1394    async fn ordinary_version_count(store: &ECStore, pool_index: usize, bucket: &str, object: &str) -> usize {
1395        store.pools[pool_index].disk_set[0]
1396            .load_file_info_versions_exact(bucket, object)
1397            .await
1398            .expect("pool metadata should load")
1399            .map(|versions| {
1400                versions
1401                    .versions
1402                    .iter()
1403                    .filter(|version| !version.tier_free_version())
1404                    .count()
1405            })
1406            .unwrap_or_default()
1407    }
1408
1409    #[tokio::test]
1410    async fn delete_prefix_attempts_later_pools_after_an_earlier_pool_error() {
1411        let (_temp_dir, store, shutdown) = setup_multi_pool_test_store("delete-prefix", &[2, 4]).await;
1412        let bucket = format!("delete-prefix-{}", Uuid::new_v4().simple());
1413        store
1414            .make_bucket(&bucket, &MakeBucketOptions::default())
1415            .await
1416            .expect("bucket should be created in both pools");
1417
1418        let first_pool_disks = store.pools[0].disk_set[0].disks.read().await.clone();
1419        for disk in first_pool_disks.iter().flatten() {
1420            disk.write_all(&bucket, "blocked", bytes::Bytes::from_static(b"not-a-directory"))
1421                .await
1422                .expect("first pool should contain a blocking parent file");
1423        }
1424        let later_pool_disks = store.pools[1].disk_set[0].disks.read().await.clone();
1425        let later_data_disk = later_pool_disks[0].clone().expect("later pool should have its first disk");
1426        later_data_disk
1427            .write_all(&bucket, "blocked/prefix/object", bytes::Bytes::from_static(b"data"))
1428            .await
1429            .expect("later pool should contain the prefix on its available disk");
1430        *store.pools[1].disk_set[0].disks.write().await = vec![Some(later_data_disk.clone()), None, None, None];
1431
1432        let err = store
1433            .delete_prefix(&bucket, "blocked/prefix", &ObjectOptions::default())
1434            .await
1435            .expect_err("the first pool's hard error must be returned");
1436
1437        assert!(
1438            matches!(err, StorageError::PrefixAccessDenied(ref error_bucket, ref error_prefix)
1439                if error_bucket == &bucket && error_prefix == "blocked/prefix"),
1440            "unexpected multi-pool delete error: {err:?}"
1441        );
1442        assert!(matches!(
1443            later_data_disk.read_all(&bucket, "blocked/prefix/object").await,
1444            Err(DiskError::FileNotFound)
1445        ));
1446
1447        *store.pools[1].disk_set[0].disks.write().await = later_pool_disks.clone();
1448        for disk in first_pool_disks.iter().flatten() {
1449            disk.write_all(&bucket, "second-blocked", bytes::Bytes::from_static(b"not-a-directory"))
1450                .await
1451                .expect("first pool should contain a second blocking parent file");
1452        }
1453        for disk in later_pool_disks.iter().flatten() {
1454            disk.write_all(&bucket, "second-blocked/prefix/object", bytes::Bytes::from_static(b"data"))
1455                .await
1456                .expect("later pool should contain the second prefix");
1457        }
1458        let err = store
1459            .delete_prefix(&bucket, "second-blocked/prefix", &ObjectOptions::default())
1460            .await
1461            .expect_err("a successful later pool must not override the first pool's hard error");
1462        assert!(
1463            matches!(err, StorageError::PrefixAccessDenied(ref error_bucket, ref error_prefix)
1464                if error_bucket == &bucket && error_prefix == "second-blocked/prefix"),
1465            "unexpected hard-error plus success result: {err:?}"
1466        );
1467        for disk in later_pool_disks.iter().flatten() {
1468            assert!(matches!(
1469                disk.read_all(&bucket, "second-blocked/prefix/object").await,
1470                Err(DiskError::FileNotFound)
1471            ));
1472        }
1473
1474        for disk in later_pool_disks.iter().flatten() {
1475            disk.delete_volume(&bucket, true)
1476                .await
1477                .expect("the bucket should be absent from the later pool");
1478        }
1479        let healthy_object = "healthy/prefix/object";
1480        for disk in first_pool_disks.iter().flatten() {
1481            disk.write_all(&bucket, healthy_object, bytes::Bytes::from_static(b"data"))
1482                .await
1483                .expect("the first pool should contain the healthy prefix");
1484        }
1485        store
1486            .delete_prefix(&bucket, "healthy/prefix", &ObjectOptions::default())
1487            .await
1488            .expect("one successful pool should make a partially missing bucket idempotent");
1489        for disk in first_pool_disks.iter().flatten() {
1490            assert!(matches!(disk.read_all(&bucket, healthy_object).await, Err(DiskError::FileNotFound)));
1491        }
1492
1493        let missing_bucket = format!("delete-prefix-missing-{}", Uuid::new_v4().simple());
1494        let err = store
1495            .delete_prefix(&missing_bucket, "missing/prefix", &ObjectOptions::default())
1496            .await
1497            .expect_err("a bucket missing from every pool must remain an error");
1498        assert_eq!(err, StorageError::BucketNotFound(missing_bucket));
1499
1500        shutdown.cancel();
1501    }
1502
1503    #[tokio::test]
1504    #[serial_test::serial]
1505    async fn lifecycle_delete_all_history_failure_preserves_trigger_and_retry_converges() {
1506        let (_temp_dir, store, shutdown) = setup_multi_pool_test_store("lifecycle-delete-all", &[4, 4]).await;
1507        let bucket = format!("lifecycle-delete-all-{}", Uuid::new_v4().simple());
1508        let object = "object";
1509        store
1510            .make_bucket(&bucket, &MakeBucketOptions::default())
1511            .await
1512            .expect("bucket should be created in both pools");
1513
1514        for pool_index in 0..2 {
1515            let mut reader = PutObjReader::from_vec(format!("pool-{pool_index}-history").into_bytes());
1516            store.pools[pool_index]
1517                .put_object(
1518                    &bucket,
1519                    object,
1520                    &mut reader,
1521                    &ObjectOptions {
1522                        versioned: true,
1523                        ..Default::default()
1524                    },
1525                )
1526                .await
1527                .expect("historical version should be stored");
1528        }
1529        let marker = store.pools[0]
1530            .delete_object(
1531                &bucket,
1532                object,
1533                ObjectOptions {
1534                    versioned: true,
1535                    ..Default::default()
1536                },
1537            )
1538            .await
1539            .expect("trigger marker should be stored in the first pool");
1540        let marker_id = marker.version_id.expect("trigger marker should have a version id");
1541        let mut opts = ObjectOptions {
1542            delete_prefix: true,
1543            delete_prefix_object: true,
1544            versioned: true,
1545            lifecycle_delete_all: Some(crate::object_api::LifecycleDeleteAllRequest {
1546                version_id: Some(marker_id),
1547                delete_marker: true,
1548                action: rustfs_scanner_metrics::metrics::IlmAction::DelMarkerDeleteAllVersionsAction,
1549                rule_id: "rule".to_string(),
1550                phase: crate::object_api::LifecycleDeleteAllPhase::Preflight,
1551            }),
1552            object_lock_config_snapshot: Some(Arc::new(ObjectLockConfigSnapshot::new(
1553                crate::bucket::metadata_sys::ObjectLockConfigState::ConfirmedAbsent,
1554            ))),
1555            delete_replication_config_snapshot: Some(Arc::new(
1556                crate::bucket::replication::DeleteReplicationConfigSnapshot::default(),
1557            )),
1558            ..Default::default()
1559        };
1560        opts.ensure_lifecycle_delete_all_journal();
1561
1562        let _failure_guard = LifecycleDeleteAllFailureGuard;
1563        *LIFECYCLE_DELETE_ALL_TEST_FAILURE
1564            .lock()
1565            .expect("lifecycle delete-all failure hook should not poison") =
1566            Some((crate::object_api::LifecycleDeleteAllPhase::History, 1));
1567        let err = store
1568            .delete_prefix(&bucket, object, &opts)
1569            .await
1570            .expect_err("a later pool history failure must stop before trigger deletion");
1571        assert_eq!(err, StorageError::PreconditionFailed);
1572        assert!(
1573            opts.lifecycle_delete_all_journal()
1574                .expect("delete-all journal should be initialized")
1575                .lock()
1576                .mutation_started()
1577        );
1578
1579        let first_pool = store.pools[0].disk_set[0]
1580            .load_file_info_versions_exact(&bucket, object)
1581            .await
1582            .expect("first pool metadata should load")
1583            .expect("the trigger should remain");
1584        let first_pool_ordinary: Vec<&FileInfo> = first_pool
1585            .versions
1586            .iter()
1587            .filter(|version| !version.tier_free_version())
1588            .collect();
1589        assert_eq!(first_pool_ordinary.len(), 1);
1590        assert_eq!(first_pool_ordinary[0].version_id, Some(marker_id));
1591        assert!(first_pool_ordinary[0].deleted);
1592
1593        *LIFECYCLE_DELETE_ALL_TEST_FAILURE
1594            .lock()
1595            .expect("lifecycle delete-all failure hook should not poison") = None;
1596        store
1597            .delete_prefix(&bucket, object, &opts)
1598            .await
1599            .expect("retry should delete remaining history and its trigger owner");
1600        for pool in &store.pools {
1601            assert!(
1602                pool.disk_set[0]
1603                    .load_file_info_versions_exact(&bucket, object)
1604                    .await
1605                    .expect("pool metadata should load after retry")
1606                    .is_none(),
1607                "all ordinary versions should be removed after retry"
1608            );
1609        }
1610
1611        shutdown.cancel();
1612    }
1613
1614    #[tokio::test]
1615    #[serial_test::serial]
1616    async fn lifecycle_delete_all_phase_failures_preserve_barriers_and_retry() {
1617        let (_temp_dir, store, shutdown) = setup_multi_pool_test_store("lifecycle-delete-all-phases", &[4, 4]).await;
1618        let bucket = format!("lifecycle-delete-all-phases-{}", Uuid::new_v4().simple());
1619        store
1620            .make_bucket(&bucket, &MakeBucketOptions::default())
1621            .await
1622            .expect("bucket should be created in both pools");
1623        let _failure_guard = LifecycleDeleteAllFailureGuard;
1624
1625        for (object, phase, expected_counts, mutation_started) in [
1626            ("preflight-failure", crate::object_api::LifecycleDeleteAllPhase::Preflight, [2, 2], false),
1627            (
1628                "final-preflight-failure",
1629                crate::object_api::LifecycleDeleteAllPhase::FinalPreflight,
1630                [1, 1],
1631                true,
1632            ),
1633            ("trigger-failure", crate::object_api::LifecycleDeleteAllPhase::Trigger, [0, 1], true),
1634        ] {
1635            let opts = seed_multi_pool_delete_all(&store, &bucket, object).await;
1636            *LIFECYCLE_DELETE_ALL_TEST_FAILURE
1637                .lock()
1638                .expect("lifecycle delete-all failure hook should not poison") = Some((phase, 1));
1639            let err = store
1640                .delete_prefix(&bucket, object, &opts)
1641                .await
1642                .expect_err("injected phase failure should stop the transaction");
1643            assert_eq!(err, StorageError::PreconditionFailed);
1644            assert_eq!(
1645                opts.lifecycle_delete_all_journal()
1646                    .expect("delete-all journal should be initialized")
1647                    .lock()
1648                    .mutation_started(),
1649                mutation_started
1650            );
1651            assert_eq!(ordinary_version_count(&store, 0, &bucket, object).await, expected_counts[0]);
1652            assert_eq!(ordinary_version_count(&store, 1, &bucket, object).await, expected_counts[1]);
1653
1654            *LIFECYCLE_DELETE_ALL_TEST_FAILURE
1655                .lock()
1656                .expect("lifecycle delete-all failure hook should not poison") = None;
1657            store
1658                .delete_prefix(&bucket, object, &opts)
1659                .await
1660                .expect("retry should converge after the injected failure is removed");
1661            assert_eq!(ordinary_version_count(&store, 0, &bucket, object).await, 0);
1662            assert_eq!(ordinary_version_count(&store, 1, &bucket, object).await, 0);
1663        }
1664
1665        shutdown.cancel();
1666    }
1667
1668    fn assert_backend_layout_empty(info: &rustfs_madmin::BackendInfo) {
1669        assert!(info.standard_sc_parities.is_empty());
1670        assert!(info.standard_sc_data.is_empty());
1671        assert_eq!(info.standard_sc_parity, None);
1672        assert!(info.rr_sc_parities.is_empty());
1673        assert!(info.rr_sc_data.is_empty());
1674        assert_eq!(info.rr_sc_parity, None);
1675    }
1676
1677    #[test]
1678    fn build_backend_info_reports_heterogeneous_automatic_config_in_pool_order() {
1679        let config = lookup_config_for_pools_without_env(&KVS::new(), &[4, 2]).expect("automatic storage class should resolve");
1680
1681        let info = build_backend_info(&config, &[4, 2], &[2, 1], &[7, 3]);
1682
1683        assert!(matches!(info.backend_type, rustfs_madmin::BackendByte::Erasure));
1684        assert_eq!(info.standard_sc_parities, vec![2, 1]);
1685        assert_eq!(info.standard_sc_data, vec![2, 1]);
1686        assert_eq!(info.standard_sc_parity, None);
1687        assert_eq!(info.rr_sc_parities, vec![1, 1]);
1688        assert_eq!(info.rr_sc_data, vec![3, 1]);
1689        assert_eq!(info.rr_sc_parity, Some(1));
1690        assert_eq!(info.drives_per_set, vec![4, 2]);
1691        assert_eq!(info.total_sets, vec![7, 3]);
1692    }
1693
1694    #[test]
1695    fn build_backend_info_keeps_truthful_homogeneous_scalar() {
1696        let mut kvs = KVS::new();
1697        kvs.insert(CLASS_STANDARD.to_string(), "EC:2".to_string());
1698        let config =
1699            lookup_config_for_pools_without_env(&kvs, &[4, 6]).expect("explicit storage class should resolve for every pool");
1700
1701        let info = build_backend_info(&config, &[4, 6], &[2, 3], &[1, 1]);
1702
1703        assert_eq!(info.standard_sc_parities, vec![2, 2]);
1704        assert_eq!(info.standard_sc_data, vec![2, 4]);
1705        assert_eq!(info.standard_sc_parity, Some(2));
1706        assert_eq!(info.rr_sc_parities, vec![1, 1]);
1707        assert_eq!(info.rr_sc_data, vec![3, 5]);
1708        assert_eq!(info.rr_sc_parity, Some(1));
1709    }
1710
1711    #[test]
1712    fn build_backend_info_reports_single_disk_pool_without_inventing_parity() {
1713        let config = lookup_config_for_pools_without_env(&KVS::new(), &[4, 1]).expect("single disk pool should resolve");
1714
1715        let info = build_backend_info(&config, &[4, 1], &[2, 0], &[1, 1]);
1716
1717        assert_eq!(info.standard_sc_parities, vec![2, 0]);
1718        assert_eq!(info.standard_sc_data, vec![2, 1]);
1719        assert_eq!(info.standard_sc_parity, None);
1720        assert_eq!(info.rr_sc_parities, vec![1, 0]);
1721        assert_eq!(info.rr_sc_data, vec![3, 1]);
1722        assert_eq!(info.rr_sc_parity, None);
1723    }
1724
1725    #[test]
1726    fn build_backend_info_uses_pool_defaults_only_when_truly_uninitialized() {
1727        let info = build_backend_info(&Default::default(), &[4, 2], &[1, 0], &[1, 1]);
1728
1729        assert_eq!(info.standard_sc_parities, vec![1, 0]);
1730        assert_eq!(info.standard_sc_data, vec![3, 2]);
1731        assert_eq!(info.standard_sc_parity, None);
1732        assert!(info.rr_sc_parities.is_empty());
1733        assert!(info.rr_sc_data.is_empty());
1734        assert_eq!(info.rr_sc_parity, None);
1735    }
1736
1737    #[test]
1738    fn build_backend_info_fails_closed_on_initialized_snapshot_mismatch() {
1739        let config = lookup_config_for_pools_without_env(&KVS::new(), &[4, 2]).expect("automatic storage class should resolve");
1740
1741        let info = build_backend_info(&config, &[4, 6], &[1, 2], &[1, 1]);
1742
1743        assert_backend_layout_empty(&info);
1744        assert_eq!(info.drives_per_set, vec![4, 6]);
1745        assert_eq!(info.total_sets, vec![1, 1]);
1746    }
1747
1748    #[test]
1749    fn build_backend_info_expands_valid_legacy_scalar_snapshot() {
1750        let mut kvs = KVS::new();
1751        kvs.insert(CLASS_STANDARD.to_string(), "EC:2".to_string());
1752        kvs.insert(CLASS_RRS.to_string(), "EC:1".to_string());
1753        let current =
1754            lookup_config_for_pools_without_env(&kvs, &[4, 6]).expect("distinct legacy scalars should resolve for every pool");
1755        let encoded = serde_json::to_string(&current).expect("config should serialize");
1756        let legacy: storageclass::Config = serde_json::from_str(&encoded).expect("legacy scalar config should deserialize");
1757
1758        let info = build_backend_info(&legacy, &[4, 6], &[2, 3], &[1, 1]);
1759
1760        assert_eq!(info.standard_sc_parities, vec![2, 2]);
1761        assert_eq!(info.standard_sc_data, vec![2, 4]);
1762        assert_eq!(info.standard_sc_parity, Some(2));
1763        assert_eq!(info.rr_sc_parities, vec![1, 1]);
1764        assert_eq!(info.rr_sc_data, vec![3, 5]);
1765        assert_eq!(info.rr_sc_parity, Some(1));
1766    }
1767
1768    #[test]
1769    fn build_backend_info_rejects_legacy_scalar_invalid_for_later_pool() {
1770        let current = lookup_config_for_pools_without_env(&KVS::new(), &[4, 2]).expect("automatic config should resolve");
1771        let encoded = serde_json::to_string(&current).expect("config should serialize");
1772        let legacy: storageclass::Config = serde_json::from_str(&encoded).expect("legacy scalar config should deserialize");
1773
1774        let info = build_backend_info(&legacy, &[4, 2], &[2, 1], &[1, 1]);
1775
1776        assert_backend_layout_empty(&info);
1777    }
1778
1779    #[test]
1780    fn build_backend_info_never_reports_invalid_geometry() {
1781        let config = storageclass::Config::default();
1782
1783        assert_backend_layout_empty(&build_backend_info(&config, &[4, 2], &[5, 1], &[1, 1]));
1784        assert_backend_layout_empty(&build_backend_info(&config, &[0], &[0], &[1]));
1785        assert_backend_layout_empty(&build_backend_info(&config, &[4], &[3], &[1]));
1786    }
1787
1788    #[test]
1789    fn build_backend_info_rejects_mismatched_topology_lengths() {
1790        let config = lookup_config_for_pools_without_env(&KVS::new(), &[4, 2]).expect("automatic storage class should resolve");
1791
1792        assert_backend_layout_empty(&build_backend_info(&config, &[4, 2], &[2], &[1, 1]));
1793        assert_backend_layout_empty(&build_backend_info(&config, &[4, 2], &[2, 1], &[1]));
1794
1795        let empty = build_backend_info(&Default::default(), &[], &[], &[]);
1796        assert_backend_layout_empty(&empty);
1797        assert!(empty.drives_per_set.is_empty());
1798        assert!(empty.total_sets.is_empty());
1799    }
1800
1801    #[test]
1802    fn build_backend_info_uses_one_complete_arc_swap_snapshot() {
1803        let old = lookup_config_for_pools_without_env(&KVS::new(), &[4, 2]).expect("old config should resolve");
1804        let mut new_kvs = KVS::new();
1805        new_kvs.insert(CLASS_STANDARD.to_string(), "EC:1".to_string());
1806        let new = lookup_config_for_pools_without_env(&new_kvs, &[4, 2]).expect("new config should resolve");
1807        let snapshots = ArcSwap::from_pointee(old);
1808        let held_old = snapshots.load_full();
1809        snapshots.store(Arc::new(new));
1810
1811        let old_info = build_backend_info(&held_old, &[4, 2], &[2, 1], &[1, 1]);
1812        let new_info = build_backend_info(&snapshots.load_full(), &[4, 2], &[2, 1], &[1, 1]);
1813
1814        assert_eq!(old_info.standard_sc_parities, vec![2, 1]);
1815        assert_eq!(old_info.standard_sc_data, vec![2, 1]);
1816        assert_eq!(old_info.standard_sc_parity, None);
1817        assert_eq!(old_info.rr_sc_parities, vec![1, 1]);
1818        assert_eq!(new_info.standard_sc_parities, vec![1, 1]);
1819        assert_eq!(new_info.standard_sc_data, vec![3, 1]);
1820        assert_eq!(new_info.standard_sc_parity, Some(1));
1821        assert_eq!(new_info.rr_sc_parities, vec![1, 1]);
1822    }
1823
1824    fn object_info_with_mod_time(unix_ts: i64, delete_marker: bool) -> ObjectInfo {
1825        ObjectInfo {
1826            mod_time: Some(OffsetDateTime::from_unix_timestamp(unix_ts).expect("operation should succeed")),
1827            delete_marker,
1828            ..Default::default()
1829        }
1830    }
1831
1832    fn object_info_with_identity(unix_ts: i64, delete_marker: bool, version_id: Uuid, etag: Option<String>) -> ObjectInfo {
1833        ObjectInfo {
1834            version_id: Some(version_id),
1835            etag,
1836            ..object_info_with_mod_time(unix_ts, delete_marker)
1837        }
1838    }
1839
1840    #[test]
1841    fn resolve_latest_object_info_candidates_returns_latest_delete_marker() {
1842        let candidates = vec![
1843            LatestObjectInfoCandidate {
1844                info: Some(object_info_with_mod_time(10, false)),
1845                idx: 0,
1846                err: None,
1847            },
1848            LatestObjectInfoCandidate {
1849                info: Some(object_info_with_mod_time(20, true)),
1850                idx: 1,
1851                err: None,
1852            },
1853        ];
1854
1855        let (info, idx) = resolve_latest_object_info_candidates(candidates, "bucket", "object", &ObjectOptions::default())
1856            .expect("operation should succeed");
1857
1858        assert_eq!(idx, 1);
1859        assert!(info.delete_marker);
1860    }
1861
1862    #[test]
1863    fn resolve_latest_object_info_candidates_prefers_higher_pool_idx_on_equal_mod_time_for_equivalent_candidates() {
1864        let candidates = vec![
1865            LatestObjectInfoCandidate {
1866                info: Some(object_info_with_mod_time(10, false)),
1867                idx: 0,
1868                err: None,
1869            },
1870            LatestObjectInfoCandidate {
1871                info: Some(object_info_with_mod_time(10, false)),
1872                idx: 1,
1873                err: None,
1874            },
1875        ];
1876
1877        let (_, idx) = resolve_latest_object_info_candidates(candidates, "bucket", "object", &ObjectOptions::default())
1878            .expect("operation should succeed");
1879
1880        assert_eq!(idx, 1);
1881    }
1882
1883    #[test]
1884    fn resolve_latest_object_info_candidates_prefers_active_target_over_higher_suspended_source() {
1885        let source = object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string()));
1886        let target = source.clone();
1887
1888        let (info, idx) = resolve_latest_object_info_candidates_with_pool_state(
1889            vec![
1890                LatestObjectInfoCandidate {
1891                    info: Some(target),
1892                    idx: 0,
1893                    err: None,
1894                },
1895                LatestObjectInfoCandidate {
1896                    info: Some(source),
1897                    idx: 1,
1898                    err: None,
1899                },
1900            ],
1901            &[false, true],
1902            "bucket",
1903            "object",
1904            &ObjectOptions::default(),
1905        )
1906        .expect("an active committed target should fence an equivalent suspended source");
1907
1908        assert_eq!(idx, 0);
1909        assert_eq!(info.version_id, Some(Uuid::from_u128(1)));
1910    }
1911
1912    #[test]
1913    fn resolve_latest_object_info_candidates_keeps_lone_suspended_source_readable() {
1914        let source = object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string()));
1915
1916        let (_, idx) = resolve_latest_object_info_candidates_with_pool_state(
1917            vec![LatestObjectInfoCandidate {
1918                info: Some(source),
1919                idx: 1,
1920                err: None,
1921            }],
1922            &[false, true],
1923            "bucket",
1924            "object",
1925            &ObjectOptions::default(),
1926        )
1927        .expect("a source-only object must remain readable before migration commits");
1928
1929        assert_eq!(idx, 1);
1930    }
1931
1932    #[test]
1933    fn resolve_latest_object_info_candidates_rejects_suspended_source_identity_conflict() {
1934        let target = object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-new".to_string()));
1935        let source = object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-old".to_string()));
1936
1937        let err = resolve_latest_object_info_candidates_with_pool_state(
1938            vec![
1939                LatestObjectInfoCandidate {
1940                    info: Some(target),
1941                    idx: 0,
1942                    err: None,
1943                },
1944                LatestObjectInfoCandidate {
1945                    info: Some(source),
1946                    idx: 1,
1947                    err: None,
1948                },
1949            ],
1950            &[false, true],
1951            "bucket",
1952            "object",
1953            &ObjectOptions::default(),
1954        )
1955        .expect_err("pool state must not mask an equal-time identity conflict");
1956
1957        assert_eq!(err, Error::ErasureReadQuorum);
1958    }
1959
1960    #[test]
1961    fn resolve_latest_object_info_candidates_keeps_index_fallback_for_fully_equivalent_identities() {
1962        let candidates = vec![
1963            LatestObjectInfoCandidate {
1964                info: Some(object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string()))),
1965                idx: 2,
1966                err: None,
1967            },
1968            LatestObjectInfoCandidate {
1969                info: Some(object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string()))),
1970                idx: 7,
1971                err: None,
1972            },
1973        ];
1974
1975        let (info, idx) = resolve_latest_object_info_candidates(candidates, "bucket", "object", &ObjectOptions::default())
1976            .expect("equivalent replicas must resolve deterministically");
1977
1978        assert_eq!(idx, 7);
1979        assert_eq!(info.version_id, Some(Uuid::from_u128(1)));
1980    }
1981
1982    #[test]
1983    fn resolve_latest_object_info_candidates_rejects_equal_time_version_id_conflict() {
1984        let candidates = vec![
1985            LatestObjectInfoCandidate {
1986                info: Some(object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string()))),
1987                idx: 0,
1988                err: None,
1989            },
1990            LatestObjectInfoCandidate {
1991                info: Some(object_info_with_identity(10, false, Uuid::from_u128(2), Some("etag-a".to_string()))),
1992                idx: 1,
1993                err: None,
1994            },
1995        ];
1996
1997        let err = resolve_latest_object_info_candidates(candidates, "bucket", "object", &ObjectOptions::default())
1998            .expect_err("divergent version ids must not silently resolve to the higher pool index");
1999
2000        assert_eq!(err, Error::ErasureReadQuorum);
2001    }
2002
2003    #[test]
2004    fn resolve_latest_object_info_candidates_rejects_equal_time_etag_conflict() {
2005        let candidates = vec![
2006            LatestObjectInfoCandidate {
2007                info: Some(object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-old".to_string()))),
2008                idx: 0,
2009                err: None,
2010            },
2011            LatestObjectInfoCandidate {
2012                info: Some(object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-new".to_string()))),
2013                idx: 1,
2014                err: None,
2015            },
2016        ];
2017
2018        let err = resolve_latest_object_info_candidates(candidates, "bucket", "object", &ObjectOptions::default())
2019            .expect_err("divergent etags must not silently resolve to the higher pool index");
2020
2021        assert_eq!(err, Error::ErasureReadQuorum);
2022    }
2023
2024    #[test]
2025    fn resolve_latest_object_info_candidates_rejects_equal_time_delete_marker_conflict() {
2026        let candidates = vec![
2027            LatestObjectInfoCandidate {
2028                info: Some(object_info_with_identity(10, false, Uuid::from_u128(1), None)),
2029                idx: 0,
2030                err: None,
2031            },
2032            LatestObjectInfoCandidate {
2033                info: Some(object_info_with_identity(10, true, Uuid::from_u128(1), Some("etag-a".to_string()))),
2034                idx: 1,
2035                err: None,
2036            },
2037        ];
2038
2039        let err = resolve_latest_object_info_candidates(candidates, "bucket", "object", &ObjectOptions::default())
2040            .expect_err("a delete marker tied with a live version must not be masked by the pool index");
2041
2042        assert_eq!(err, Error::ErasureReadQuorum);
2043    }
2044
2045    fn assert_equal_time_identity_conflict(left: ObjectInfo, right: ObjectInfo) {
2046        let err = resolve_latest_object_info_candidates(
2047            vec![
2048                LatestObjectInfoCandidate {
2049                    info: Some(left),
2050                    idx: 0,
2051                    err: None,
2052                },
2053                LatestObjectInfoCandidate {
2054                    info: Some(right),
2055                    idx: 1,
2056                    err: None,
2057                },
2058            ],
2059            "bucket",
2060            "object",
2061            &ObjectOptions::default(),
2062        )
2063        .expect_err("equal-time identity divergence must fail closed");
2064
2065        assert_eq!(err, Error::ErasureReadQuorum);
2066    }
2067
2068    #[test]
2069    fn resolve_latest_object_info_candidates_rejects_equal_time_payload_identity_conflicts() {
2070        let base = object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string()));
2071
2072        let mut size = base.clone();
2073        size.size = 1;
2074        assert_equal_time_identity_conflict(base.clone(), size);
2075
2076        let mut actual_size = base.clone();
2077        actual_size.actual_size = 1;
2078        assert_equal_time_identity_conflict(base.clone(), actual_size);
2079
2080        let mut checksum = base.clone();
2081        checksum.checksum = Some(bytes::Bytes::from_static(b"checksum"));
2082        assert_equal_time_identity_conflict(base.clone(), checksum);
2083
2084        let mut parts = base.clone();
2085        parts.parts = std::sync::Arc::new(vec![rustfs_filemeta::ObjectPartInfo {
2086            etag: "part-etag".to_string(),
2087            number: 1,
2088            size: 1,
2089            ..Default::default()
2090        }]);
2091        assert_equal_time_identity_conflict(base.clone(), parts);
2092
2093        let mut transition = base;
2094        transition.transitioned_object.tier = "tier-a".to_string();
2095        assert_equal_time_identity_conflict(
2096            object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string())),
2097            transition,
2098        );
2099    }
2100
2101    #[test]
2102    fn resolve_latest_object_info_candidates_accepts_data_movement_rewrites() {
2103        let mut source = object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string()));
2104        source.data_dir = Some(Uuid::from_u128(1));
2105
2106        let mut target = source.clone();
2107        target.data_dir = Some(Uuid::from_u128(2));
2108        rustfs_utils::http::insert_str(
2109            Arc::make_mut(&mut target.user_defined),
2110            rustfs_utils::http::SUFFIX_DATA_MOVED,
2111            "true".to_string(),
2112        );
2113        rustfs_utils::http::insert_str(
2114            Arc::make_mut(&mut target.user_defined),
2115            rustfs_utils::http::SUFFIX_ACTUAL_SIZE,
2116            "0".to_string(),
2117        );
2118
2119        let (info, idx) = resolve_latest_object_info_candidates(
2120            vec![
2121                LatestObjectInfoCandidate {
2122                    info: Some(source),
2123                    idx: 0,
2124                    err: None,
2125                },
2126                LatestObjectInfoCandidate {
2127                    info: Some(target.clone()),
2128                    idx: 1,
2129                    err: None,
2130                },
2131            ],
2132            "bucket",
2133            "object",
2134            &ObjectOptions::default(),
2135        )
2136        .expect("a committed data-movement target must remain readable before source cleanup");
2137
2138        assert_eq!(idx, 1);
2139        assert_eq!(info.data_dir, target.data_dir);
2140    }
2141
2142    #[test]
2143    fn resolve_latest_object_info_candidates_rejects_unmarked_data_dir_conflict() {
2144        let mut left = object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string()));
2145        left.data_dir = Some(Uuid::from_u128(1));
2146
2147        let mut right = left.clone();
2148        right.data_dir = Some(Uuid::from_u128(2));
2149
2150        assert_equal_time_identity_conflict(left, right);
2151    }
2152
2153    #[test]
2154    fn resolve_latest_object_info_candidates_accepts_internal_metadata_aliases() {
2155        let base = object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string()));
2156        let mut rustfs_alias = base.clone();
2157        rustfs_alias.user_defined = std::sync::Arc::new(std::collections::HashMap::from([(
2158            "x-rustfs-internal-compression".to_string(),
2159            "zstd".to_string(),
2160        )]));
2161        let mut minio_alias = base.clone();
2162        minio_alias.user_defined = std::sync::Arc::new(std::collections::HashMap::from([(
2163            "X-MINIO-INTERNAL-COMPRESSION".to_string(),
2164            "zstd".to_string(),
2165        )]));
2166
2167        let (_, idx) = resolve_latest_object_info_candidates(
2168            vec![
2169                LatestObjectInfoCandidate {
2170                    info: Some(rustfs_alias),
2171                    idx: 0,
2172                    err: None,
2173                },
2174                LatestObjectInfoCandidate {
2175                    info: Some(minio_alias),
2176                    idx: 1,
2177                    err: None,
2178                },
2179            ],
2180            "bucket",
2181            "object",
2182            &ObjectOptions::default(),
2183        )
2184        .expect("same-value internal aliases should resolve");
2185        assert_eq!(idx, 1);
2186
2187        let mut dual_alias = base.clone();
2188        dual_alias.user_defined = std::sync::Arc::new(std::collections::HashMap::from([
2189            ("x-rustfs-internal-compression".to_string(), "zstd".to_string()),
2190            ("x-minio-internal-compression".to_string(), "zstd".to_string()),
2191        ]));
2192        let mut single_alias = base;
2193        single_alias.user_defined = std::sync::Arc::new(std::collections::HashMap::from([(
2194            "x-rustfs-internal-compression".to_string(),
2195            "zstd".to_string(),
2196        )]));
2197
2198        let (_, idx) = resolve_latest_object_info_candidates(
2199            vec![
2200                LatestObjectInfoCandidate {
2201                    info: Some(dual_alias),
2202                    idx: 0,
2203                    err: None,
2204                },
2205                LatestObjectInfoCandidate {
2206                    info: Some(single_alias),
2207                    idx: 1,
2208                    err: None,
2209                },
2210            ],
2211            "bucket",
2212            "object",
2213            &ObjectOptions::default(),
2214        )
2215        .expect("dual-key and single-key internal metadata should resolve");
2216        assert_eq!(idx, 1);
2217    }
2218
2219    #[test]
2220    fn resolve_latest_object_info_candidates_rejects_different_internal_metadata_alias_values() {
2221        let base = object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string()));
2222        let mut rustfs_alias = base.clone();
2223        rustfs_alias.user_defined = std::sync::Arc::new(std::collections::HashMap::from([(
2224            "x-rustfs-internal-compression".to_string(),
2225            "zstd".to_string(),
2226        )]));
2227        let mut minio_alias = base;
2228        minio_alias.user_defined = std::sync::Arc::new(std::collections::HashMap::from([(
2229            "x-minio-internal-compression".to_string(),
2230            "snappy".to_string(),
2231        )]));
2232
2233        assert_equal_time_identity_conflict(rustfs_alias, minio_alias);
2234    }
2235
2236    #[test]
2237    fn resolve_latest_object_info_candidates_preserves_dynamic_internal_metadata_identity_case() {
2238        for suffix_prefix in ["replication-reset-", "replication-delete-marker-version-"] {
2239            let base = object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string()));
2240            let mut rustfs_alias = base.clone();
2241            rustfs_alias.user_defined = std::sync::Arc::new(std::collections::HashMap::from([(
2242                format!(
2243                    "X-RUSTFS-INTERNAL-{}{suffix}",
2244                    suffix_prefix.to_uppercase(),
2245                    suffix = "arn:aws:s3:::Bucket"
2246                ),
2247                "value".to_string(),
2248            )]));
2249            let mut minio_alias = base.clone();
2250            minio_alias.user_defined = std::sync::Arc::new(std::collections::HashMap::from([(
2251                format!("x-minio-internal-{suffix_prefix}arn:aws:s3:::Bucket"),
2252                "value".to_string(),
2253            )]));
2254
2255            let (_, idx) = resolve_latest_object_info_candidates(
2256                vec![
2257                    LatestObjectInfoCandidate {
2258                        info: Some(rustfs_alias.clone()),
2259                        idx: 0,
2260                        err: None,
2261                    },
2262                    LatestObjectInfoCandidate {
2263                        info: Some(minio_alias),
2264                        idx: 1,
2265                        err: None,
2266                    },
2267                ],
2268                "bucket",
2269                "object",
2270                &ObjectOptions::default(),
2271            )
2272            .expect("dynamic internal aliases with the same target should resolve");
2273            assert_eq!(idx, 1);
2274
2275            let mut different_target_case = base;
2276            different_target_case.user_defined = std::sync::Arc::new(std::collections::HashMap::from([(
2277                format!("x-minio-internal-{suffix_prefix}arn:aws:s3:::bucket"),
2278                "value".to_string(),
2279            )]));
2280
2281            assert_equal_time_identity_conflict(rustfs_alias, different_target_case);
2282        }
2283    }
2284
2285    #[test]
2286    fn resolve_latest_object_info_candidates_rejects_conflicting_internal_metadata_aliases_in_one_candidate() {
2287        let base = object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string()));
2288        let mut first = base.clone();
2289        first.user_defined = std::sync::Arc::new(std::collections::HashMap::from([
2290            ("x-rustfs-internal-compression".to_string(), "zstd".to_string()),
2291            ("x-minio-internal-compression".to_string(), "snappy".to_string()),
2292        ]));
2293        let mut second = base;
2294        second.user_defined = first.user_defined.clone();
2295
2296        assert_equal_time_identity_conflict(first, second);
2297    }
2298
2299    #[test]
2300    fn resolve_latest_object_info_candidates_rejects_replication_identity_conflict() {
2301        let base = object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string()));
2302
2303        let mut replication = base.clone();
2304        replication.replication_status_internal = Some("PENDING".to_string());
2305        replication.replication_status = ReplicationStatusType::Pending;
2306        assert_equal_time_identity_conflict(base.clone(), replication);
2307
2308        let mut purge = base.clone();
2309        purge.version_purge_status_internal = Some("PENDING".to_string());
2310        purge.version_purge_status = VersionPurgeStatusType::Pending;
2311        assert_equal_time_identity_conflict(base.clone(), purge);
2312
2313        let mut decision = base;
2314        decision.replication_decision = "replicate".to_string();
2315        assert_equal_time_identity_conflict(
2316            object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string())),
2317            decision,
2318        );
2319    }
2320
2321    #[test]
2322    fn resolve_latest_object_info_candidates_rejects_none_vs_unix_epoch_mod_time() {
2323        let mut without_mod_time = object_info_with_identity(0, false, Uuid::from_u128(1), Some("etag-a".to_string()));
2324        without_mod_time.mod_time = None;
2325        let with_unix_epoch = object_info_with_identity(0, false, Uuid::from_u128(1), Some("etag-a".to_string()));
2326
2327        assert_equal_time_identity_conflict(without_mod_time, with_unix_epoch);
2328    }
2329
2330    #[test]
2331    fn resolve_latest_object_info_candidates_ignores_older_identity_conflicts() {
2332        let latest = object_info_with_identity(20, false, Uuid::from_u128(1), Some("etag-latest".to_string()));
2333        let mut older = object_info_with_identity(10, true, Uuid::from_u128(2), Some("etag-old".to_string()));
2334        older.data_dir = Some(Uuid::from_u128(2));
2335
2336        let (info, idx) = resolve_latest_object_info_candidates(
2337            vec![
2338                LatestObjectInfoCandidate {
2339                    info: Some(latest),
2340                    idx: 0,
2341                    err: None,
2342                },
2343                LatestObjectInfoCandidate {
2344                    info: Some(older),
2345                    idx: 9,
2346                    err: None,
2347                },
2348            ],
2349            "bucket",
2350            "object",
2351            &ObjectOptions::default(),
2352        )
2353        .expect("older identity divergence must not affect the latest candidate");
2354
2355        assert_eq!(idx, 0);
2356        assert_eq!(
2357            info.mod_time,
2358            Some(OffsetDateTime::from_unix_timestamp(20).expect("operation should succeed"))
2359        );
2360    }
2361
2362    #[test]
2363    fn resolve_latest_object_info_candidates_ignores_not_found_pools_when_resolving() {
2364        let candidates = vec![
2365            LatestObjectInfoCandidate {
2366                info: Some(object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string()))),
2367                idx: 0,
2368                err: None,
2369            },
2370            LatestObjectInfoCandidate {
2371                info: None,
2372                idx: 1,
2373                err: Some(Error::ObjectNotFound("bucket".to_string(), "object".to_string())),
2374            },
2375        ];
2376
2377        let (info, idx) = resolve_latest_object_info_candidates(candidates, "bucket", "object", &ObjectOptions::default())
2378            .expect("not-found pools must not block resolution of found candidates");
2379
2380        assert_eq!(idx, 0);
2381        assert_eq!(info.version_id, Some(Uuid::from_u128(1)));
2382    }
2383
2384    #[test]
2385    fn resolve_latest_object_info_candidates_returns_non_not_found_error() {
2386        let err = resolve_latest_object_info_candidates(
2387            vec![LatestObjectInfoCandidate {
2388                info: None,
2389                idx: 0,
2390                err: Some(Error::ErasureReadQuorum),
2391            }],
2392            "bucket",
2393            "object",
2394            &ObjectOptions::default(),
2395        )
2396        .unwrap_err();
2397
2398        assert_eq!(err, Error::ErasureReadQuorum);
2399    }
2400
2401    #[test]
2402    fn resolve_latest_object_info_candidates_returns_version_not_found_for_versioned_lookups() {
2403        let err = resolve_latest_object_info_candidates(
2404            vec![LatestObjectInfoCandidate {
2405                info: None,
2406                idx: 0,
2407                err: Some(Error::ObjectNotFound("bucket".to_string(), "object".to_string())),
2408            }],
2409            "bucket",
2410            "object",
2411            &ObjectOptions {
2412                version_id: Some("vid-1".to_string()),
2413                ..Default::default()
2414            },
2415        )
2416        .unwrap_err();
2417
2418        assert_eq!(
2419            err,
2420            Error::VersionNotFound("bucket".to_string(), "object".to_string(), "vid-1".to_string())
2421        );
2422    }
2423
2424    #[test]
2425    fn pool_lookup_not_found_error_returns_object_not_found_for_latest_lookup() {
2426        let err = pool_lookup_not_found_error("bucket", "object", &ObjectOptions::default());
2427
2428        assert_eq!(err, Error::ObjectNotFound("bucket".to_string(), "object".to_string()));
2429    }
2430
2431    #[test]
2432    fn pool_lookup_not_found_error_returns_version_not_found_for_versioned_lookup() {
2433        let err = pool_lookup_not_found_error(
2434            "bucket",
2435            "object",
2436            &ObjectOptions {
2437                version_id: Some("vid-1".to_string()),
2438                ..Default::default()
2439            },
2440        );
2441
2442        assert_eq!(
2443            err,
2444            Error::VersionNotFound("bucket".to_string(), "object".to_string(), "vid-1".to_string())
2445        );
2446    }
2447
2448    #[test]
2449    fn resolve_store_rebalance_pool_meta_reload_result_passthrough_ok() {
2450        resolve_store_rebalance_pool_meta_reload_result(Ok(()), "reload_pool_meta")
2451            .expect("successful pool meta reload should pass through");
2452    }
2453
2454    #[test]
2455    fn resolve_store_rebalance_pool_meta_reload_result_wraps_error_context() {
2456        let err = resolve_store_rebalance_pool_meta_reload_result::<()>(Err(Error::SlowDown), "reload_pool_meta")
2457            .expect_err("failed pool meta reload should be wrapped");
2458        let err_message = err.to_string();
2459        assert!(err_message.contains("store rebalance pool meta reload failed during reload_pool_meta"));
2460        assert!(err_message.contains(&Error::SlowDown.to_string()));
2461    }
2462
2463    #[test]
2464    fn resolve_rebalance_delete_from_all_pools_result_passthrough_ok() {
2465        let info = ObjectInfo {
2466            bucket: "bucket".to_string(),
2467            name: "object".to_string(),
2468            ..Default::default()
2469        };
2470
2471        let resolved = resolve_rebalance_delete_from_all_pools_result(Ok(info.clone()), "bucket", "object")
2472            .expect("successful rebalance delete should pass through");
2473
2474        assert_eq!(resolved.bucket, info.bucket);
2475        assert_eq!(resolved.name, info.name);
2476    }
2477
2478    #[test]
2479    fn resolve_rebalance_delete_from_all_pools_result_wraps_object_context() {
2480        let err = resolve_rebalance_delete_from_all_pools_result(Err(Error::SlowDown), "bucket", "object")
2481            .expect_err("failed rebalance delete should be wrapped");
2482        let rendered = err.to_string();
2483
2484        assert!(rendered.contains("failed to delete rebalance source object bucket/object"), "{rendered}");
2485        assert!(rendered.contains(&Error::SlowDown.to_string()), "{rendered}");
2486    }
2487
2488    #[test]
2489    fn resolve_rebalance_delete_from_all_pools_results_fails_on_later_pool_error() {
2490        let err = resolve_rebalance_delete_from_all_pools_results(
2491            vec![
2492                RebalanceDeletePoolResult {
2493                    pool_idx: 0,
2494                    result: Ok(ObjectInfo {
2495                        bucket: "bucket".to_string(),
2496                        name: "object".to_string(),
2497                        ..Default::default()
2498                    }),
2499                },
2500                RebalanceDeletePoolResult {
2501                    pool_idx: 1,
2502                    result: Err(Error::SlowDown),
2503                },
2504            ],
2505            "bucket",
2506            "object",
2507        )
2508        .expect_err("non-ignorable errors from later pools must not be hidden");
2509        let rendered = err.to_string();
2510
2511        assert!(rendered.contains("pool 1 delete failed for bucket/object"), "{rendered}");
2512        assert!(rendered.contains(&Error::SlowDown.to_string()), "{rendered}");
2513    }
2514
2515    #[test]
2516    fn resolve_rebalance_delete_from_all_pools_results_ignores_later_not_found_after_success() {
2517        let info = ObjectInfo {
2518            bucket: "bucket".to_string(),
2519            name: "object".to_string(),
2520            ..Default::default()
2521        };
2522
2523        let resolved = resolve_rebalance_delete_from_all_pools_results(
2524            vec![
2525                RebalanceDeletePoolResult {
2526                    pool_idx: 0,
2527                    result: Ok(info.clone()),
2528                },
2529                RebalanceDeletePoolResult {
2530                    pool_idx: 1,
2531                    result: Err(Error::ObjectNotFound("bucket".to_string(), "object".to_string())),
2532                },
2533            ],
2534            "bucket",
2535            "object",
2536        )
2537        .expect("not-found errors from other pools should be ignored when a delete succeeds");
2538
2539        assert_eq!(resolved.bucket, info.bucket);
2540        assert_eq!(resolved.name, info.name);
2541    }
2542
2543    #[test]
2544    fn resolve_rebalance_delete_from_all_pools_results_accepts_success_after_not_found() {
2545        let info = ObjectInfo {
2546            bucket: "bucket".to_string(),
2547            name: "object".to_string(),
2548            ..Default::default()
2549        };
2550
2551        let resolved = resolve_rebalance_delete_from_all_pools_results(
2552            vec![
2553                RebalanceDeletePoolResult {
2554                    pool_idx: 0,
2555                    result: Err(Error::ObjectNotFound("bucket".to_string(), "object".to_string())),
2556                },
2557                RebalanceDeletePoolResult {
2558                    pool_idx: 1,
2559                    result: Ok(info.clone()),
2560                },
2561            ],
2562            "bucket",
2563            "object",
2564        )
2565        .expect("a successful delete should pass even when an earlier pool reports not-found");
2566
2567        assert_eq!(resolved.bucket, info.bucket);
2568        assert_eq!(resolved.name, info.name);
2569    }
2570
2571    #[test]
2572    fn resolve_rebalance_delete_from_all_pools_results_fails_when_all_results_are_ignored_errors() {
2573        let err = resolve_rebalance_delete_from_all_pools_results(
2574            vec![
2575                RebalanceDeletePoolResult {
2576                    pool_idx: 0,
2577                    result: Err(Error::ObjectNotFound("bucket".to_string(), "object".to_string())),
2578                },
2579                RebalanceDeletePoolResult {
2580                    pool_idx: 1,
2581                    result: Err(Error::VersionNotFound("bucket".to_string(), "object".to_string(), "vid-1".to_string())),
2582                },
2583            ],
2584            "bucket",
2585            "object",
2586        )
2587        .expect_err("all ignored errors without any successful delete should still fail");
2588        let rendered = err.to_string();
2589
2590        assert!(rendered.contains("pool 1 delete failed for bucket/object"), "{rendered}");
2591        assert!(rendered.contains("Version not found"), "{rendered}");
2592    }
2593
2594    #[test]
2595    fn resolve_rebalance_delete_from_all_pools_results_fails_on_write_quorum_even_with_success() {
2596        let err = resolve_rebalance_delete_from_all_pools_results(
2597            vec![
2598                RebalanceDeletePoolResult {
2599                    pool_idx: 0,
2600                    result: Ok(ObjectInfo {
2601                        bucket: "bucket".to_string(),
2602                        name: "object".to_string(),
2603                        ..Default::default()
2604                    }),
2605                },
2606                RebalanceDeletePoolResult {
2607                    pool_idx: 1,
2608                    result: Err(Error::ErasureWriteQuorum),
2609                },
2610            ],
2611            "bucket",
2612            "object",
2613        )
2614        .expect_err("write quorum failures must fail the aggregate delete");
2615        let rendered = err.to_string();
2616
2617        assert!(rendered.contains("pool 1 delete failed for bucket/object"), "{rendered}");
2618        assert!(rendered.contains(&Error::ErasureWriteQuorum.to_string()), "{rendered}");
2619    }
2620
2621    #[test]
2622    fn rebalance_disk_set_lookup_error_formats_pool_and_set_context() {
2623        let err = rebalance_disk_set_lookup_error(2, 7, 3);
2624
2625        assert!(
2626            err.to_string()
2627                .contains("failed to resolve rebalance disk set: pool index 2, set index 7, pool count 3")
2628        );
2629    }
2630
2631    fn reload_test_pool_status(decommission: Option<PoolDecommissionInfo>, last_update: time::OffsetDateTime) -> PoolStatus {
2632        PoolStatus {
2633            id: 0,
2634            cmd_line: "pool-0".to_string(),
2635            last_update,
2636            decommission,
2637        }
2638    }
2639
2640    fn reload_test_pool_meta(pool: PoolStatus) -> PoolMeta {
2641        PoolMeta {
2642            version: POOL_META_VERSION,
2643            pools: vec![pool],
2644            dont_save: false,
2645        }
2646    }
2647
2648    async fn persist_reload_snapshot(store: &ECStore, snapshot: &PoolMeta) {
2649        snapshot
2650            .save_for_startup(store.pools.clone())
2651            .await
2652            .expect("pool meta snapshot should persist to every pool");
2653    }
2654
2655    #[derive(Debug)]
2656    struct PoolMetaCommitPauseStorage {
2657        inner: Arc<Sets>,
2658        pause_committed_pool_meta: bool,
2659        paused: Arc<tokio::sync::Notify>,
2660        release: Arc<tokio::sync::Notify>,
2661    }
2662
2663    #[async_trait::async_trait]
2664    impl crate::storage_api_contracts::object::ObjectIO for PoolMetaCommitPauseStorage {
2665        type Error = Error;
2666        type RangeSpec = crate::storage_api_contracts::range::HTTPRangeSpec;
2667        type HeaderMap = http::HeaderMap;
2668        type ObjectOptions = crate::object_api::ObjectOptions;
2669        type ObjectInfo = crate::object_api::ObjectInfo;
2670        type GetObjectReader = crate::object_api::GetObjectReader;
2671        type PutObjectReader = crate::object_api::PutObjReader;
2672
2673        async fn get_object_reader(
2674            &self,
2675            bucket: &str,
2676            object: &str,
2677            range: Option<Self::RangeSpec>,
2678            h: Self::HeaderMap,
2679            opts: &Self::ObjectOptions,
2680        ) -> std::result::Result<Self::GetObjectReader, Error> {
2681            self.inner.get_object_reader(bucket, object, range, h, opts).await
2682        }
2683
2684        async fn put_object(
2685            &self,
2686            bucket: &str,
2687            object: &str,
2688            data: &mut Self::PutObjectReader,
2689            opts: &Self::ObjectOptions,
2690        ) -> std::result::Result<Self::ObjectInfo, Error> {
2691            let result = self.inner.put_object(bucket, object, data, opts).await;
2692            if result.is_ok() && object == POOL_META_NAME && self.pause_committed_pool_meta {
2693                let committed = read_config_no_lock_preserve_empty_with_metadata(self.inner.clone(), POOL_META_NAME)
2694                    .await
2695                    .ok()
2696                    .and_then(|(payload, _)| pool_meta_v3_commit_state_for_test(payload).ok())
2697                    .is_some_and(|(_, committed)| committed);
2698                if committed {
2699                    self.paused.notify_one();
2700                    self.release.notified().await;
2701                }
2702            }
2703            result
2704        }
2705    }
2706
2707    #[tokio::test]
2708    #[serial_test::serial]
2709    async fn pool_meta_runtime_load_waits_for_multi_pool_commit_fence() {
2710        let (_temp_dir, store, shutdown) = setup_multi_pool_test_store("pool-meta-load-fence", &[2, 2]).await;
2711        let old = PoolMeta::new(&store.pools, &PoolMeta::default());
2712        old.save_for_startup(store.pools.clone())
2713            .await
2714            .expect("old V3 pool metadata should persist");
2715        let (old_payload, _) = read_config_no_lock_preserve_empty_with_metadata(store.pools[0].clone(), POOL_META_NAME)
2716            .await
2717            .expect("old V3 pool metadata should be readable");
2718        let (old_generation, old_committed) =
2719            pool_meta_v3_commit_state_for_test(old_payload).expect("old replica should be a V3 envelope");
2720        assert!(old_committed, "old generation should be committed before the mixed-state save");
2721        let next_generation = old_generation.checked_add(1).expect("test generation should not exhaust");
2722        let mut newer = old.clone();
2723        newer.pools[0].last_update += TimeDuration::seconds(1);
2724
2725        let pool_meta_lock = store.pools[0]
2726            .new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME)
2727            .await
2728            .expect("pool metadata lock should be created");
2729        let pool_meta_guard = pool_meta_lock
2730            .get_write_lock(get_lock_acquire_timeout())
2731            .await
2732            .expect("pool metadata write fence should be acquired");
2733        let started = Arc::new(tokio::sync::Notify::new());
2734        let mut load_task = tokio::spawn({
2735            let store = store.clone();
2736            let started = started.clone();
2737            async move {
2738                started.notify_one();
2739                store.load_runtime_pool_meta("test runtime pool metadata load").await
2740            }
2741        });
2742        started.notified().await;
2743        let save_paused = Arc::new(tokio::sync::Notify::new());
2744        let save_release = Arc::new(tokio::sync::Notify::new());
2745        let save_task = tokio::spawn({
2746            let newer = newer.clone();
2747            let pools = vec![
2748                Arc::new(PoolMetaCommitPauseStorage {
2749                    inner: store.pools[0].clone(),
2750                    pause_committed_pool_meta: true,
2751                    paused: save_paused.clone(),
2752                    release: save_release.clone(),
2753                }),
2754                Arc::new(PoolMetaCommitPauseStorage {
2755                    inner: store.pools[1].clone(),
2756                    pause_committed_pool_meta: false,
2757                    paused: save_paused.clone(),
2758                    release: save_release.clone(),
2759                }),
2760            ];
2761            async move { newer.save_for_startup(pools).await }
2762        });
2763        tokio::time::timeout(std::time::Duration::from_secs(5), save_paused.notified())
2764            .await
2765            .expect("the newer V3 generation should pause after the first replica commit");
2766        assert!(
2767            tokio::time::timeout(std::time::Duration::from_millis(100), &mut load_task)
2768                .await
2769                .is_err(),
2770            "runtime load must not observe a partially committed V3 generation while fenced"
2771        );
2772        let (committed_payload, _) = read_config_no_lock_preserve_empty_with_metadata(store.pools[0].clone(), POOL_META_NAME)
2773            .await
2774            .expect("first replica should be readable while the runtime read is fenced");
2775        let (pending_payload, _) = read_config_no_lock_preserve_empty_with_metadata(store.pools[1].clone(), POOL_META_NAME)
2776            .await
2777            .expect("second replica should be readable while the runtime read is fenced");
2778        assert_eq!(
2779            pool_meta_v3_commit_state_for_test(committed_payload).expect("first replica should be a V3 envelope"),
2780            (next_generation, true),
2781            "first replica should hold the committed new generation before the fence releases"
2782        );
2783        assert_eq!(
2784            pool_meta_v3_commit_state_for_test(pending_payload).expect("second replica should be a V3 envelope"),
2785            (next_generation, false),
2786            "second replica should still hold the pending new generation before the fence releases"
2787        );
2788
2789        save_release.notify_one();
2790        save_task
2791            .await
2792            .expect("newer V3 save task should not panic")
2793            .expect("the newer generation should finish while the runtime read is fenced");
2794        drop(pool_meta_guard);
2795
2796        let loaded = tokio::time::timeout(std::time::Duration::from_secs(5), load_task)
2797            .await
2798            .expect("runtime load should finish after commit publication")
2799            .expect("runtime load task should not panic")
2800            .expect("runtime load should select the completed snapshot");
2801        assert_eq!(loaded.pools[0].last_update, newer.pools[0].last_update);
2802        shutdown.cancel();
2803    }
2804
2805    #[tokio::test]
2806    #[serial_test::serial]
2807    async fn runtime_save_current_pool_meta_reaches_v3_cas_and_disarms_transaction() {
2808        let (_temp_dir, store, shutdown) = setup_multi_pool_test_store("pool-meta-runtime-cas", &[2]).await;
2809        let bootstrapped = store
2810            .load_runtime_pool_meta("verify fresh bootstrap V3 CAS save")
2811            .await
2812            .expect("fresh startup should durably publish pool metadata");
2813        assert_eq!(bootstrapped.version, 3);
2814        let (bootstrap_payload, _) = read_config_no_lock_preserve_empty_with_metadata(store.pools[0].clone(), POOL_META_NAME)
2815            .await
2816            .expect("fresh bootstrap V3 replica should be readable");
2817        assert_eq!(
2818            pool_meta_v3_commit_state_for_test(bootstrap_payload).expect("fresh bootstrap V3 envelope should decode"),
2819            (1, true)
2820        );
2821
2822        let saved_at = OffsetDateTime::now_utc() + TimeDuration::seconds(30);
2823        {
2824            let mut pool_meta = store.pool_meta.write().await;
2825            pool_meta.pools[0].last_update = saved_at;
2826        }
2827
2828        store
2829            .save_current_pool_meta_for_test(&[0])
2830            .await
2831            .expect("runtime pool metadata save should reach V3 conditional writes");
2832        store
2833            .ensure_pool_meta_side_effects_safe("runtime save publication")
2834            .await
2835            .expect("successful runtime publication must disarm the transaction guard");
2836        let persisted = store
2837            .load_runtime_pool_meta("verify runtime V3 CAS save")
2838            .await
2839            .expect("runtime load should observe the committed save");
2840        assert_eq!(persisted.version, 3);
2841        assert_eq!(persisted.pools[0].last_update, saved_at);
2842        let (runtime_payload, _) = read_config_no_lock_preserve_empty_with_metadata(store.pools[0].clone(), POOL_META_NAME)
2843            .await
2844            .expect("runtime V3 replica should be readable");
2845        assert_eq!(
2846            pool_meta_v3_commit_state_for_test(runtime_payload).expect("runtime V3 envelope should decode"),
2847            (2, true)
2848        );
2849
2850        shutdown.cancel();
2851    }
2852
2853    #[tokio::test]
2854    #[serial_test::serial]
2855    async fn stale_clear_decommission_cannot_erase_active_or_queued_replacement() {
2856        let (_temp_dir, store, shutdown) = setup_multi_pool_test_store("pool-meta-stale-clear", &[2, 2]).await;
2857        let replacement_time = OffsetDateTime::UNIX_EPOCH + TimeDuration::seconds(100);
2858        let cases = [
2859            (
2860                "failed-active",
2861                PoolDecommissionInfo {
2862                    failed: true,
2863                    ..Default::default()
2864                },
2865                PoolDecommissionInfo {
2866                    start_time: Some(replacement_time),
2867                    ..Default::default()
2868                },
2869            ),
2870            (
2871                "canceled-queued",
2872                PoolDecommissionInfo {
2873                    canceled: true,
2874                    ..Default::default()
2875                },
2876                PoolDecommissionInfo {
2877                    queued: true,
2878                    ..Default::default()
2879                },
2880            ),
2881        ];
2882
2883        for (case, stale_terminal, replacement_info) in cases {
2884            let mut replacement = store
2885                .load_runtime_pool_meta("prepare stale clear replacement")
2886                .await
2887                .expect("the current durable pool metadata should load");
2888            replacement.pools[0].last_update = replacement_time;
2889            replacement.pools[0].decommission = Some(replacement_info.clone());
2890            persist_reload_snapshot(&store, &replacement).await;
2891
2892            let mut stale_local = replacement.clone();
2893            stale_local.pools[0].last_update = OffsetDateTime::UNIX_EPOCH;
2894            stale_local.pools[0].decommission = Some(stale_terminal.clone());
2895            *store.pool_meta.write().await = stale_local;
2896
2897            let err = store
2898                .clear_decommission(0)
2899                .await
2900                .expect_err("a stale terminal clear must not erase a durable replacement");
2901            assert!(
2902                err.to_string()
2903                    .contains("persisted active or queued decommission cannot be cleared"),
2904                "unexpected {case} rejection: {err}"
2905            );
2906
2907            let durable = store
2908                .load_runtime_pool_meta("verify stale clear replacement")
2909                .await
2910                .expect("the durable replacement should remain readable");
2911            let durable_info = durable.pools[0]
2912                .decommission
2913                .as_ref()
2914                .expect("the durable replacement must not be cleared");
2915            assert_eq!(durable.pools[0].last_update, replacement_time, "{case} replacement revision changed");
2916            assert_eq!(
2917                durable_info.start_time, replacement_info.start_time,
2918                "{case} replacement generation changed"
2919            );
2920            assert_eq!(durable_info.queued, replacement_info.queued, "{case} replacement queue state changed");
2921
2922            let local = store.pool_meta.read().await;
2923            let local_info = local.pools[0]
2924                .decommission
2925                .as_ref()
2926                .expect("the failed clear must roll back the stale local terminal state");
2927            assert_eq!(local_info.failed, stale_terminal.failed, "{case} failed state was not rolled back");
2928            assert_eq!(local_info.canceled, stale_terminal.canceled, "{case} canceled state was not rolled back");
2929        }
2930
2931        shutdown.cancel();
2932    }
2933
2934    #[tokio::test]
2935    #[serial_test::serial]
2936    async fn peer_pool_meta_reload_does_not_rollback_newer_local_states() {
2937        let (_temp_dir, store, shutdown) = setup_multi_pool_test_store("pool-meta-reload-stale", &[2]).await;
2938
2939        let stale_time = OffsetDateTime::now_utc();
2940        let newer_time = stale_time + TimeDuration::seconds(30);
2941        let progressed_states: [(&str, PoolDecommissionInfo); 4] = [
2942            (
2943                "queued",
2944                PoolDecommissionInfo {
2945                    queued: true,
2946                    start_time: Some(stale_time),
2947                    ..Default::default()
2948                },
2949            ),
2950            (
2951                "canceled",
2952                PoolDecommissionInfo {
2953                    canceled: true,
2954                    start_time: Some(stale_time),
2955                    ..Default::default()
2956                },
2957            ),
2958            (
2959                "failed",
2960                PoolDecommissionInfo {
2961                    failed: true,
2962                    start_time: Some(stale_time),
2963                    ..Default::default()
2964                },
2965            ),
2966            (
2967                "complete",
2968                PoolDecommissionInfo {
2969                    complete: true,
2970                    start_time: Some(stale_time),
2971                    ..Default::default()
2972                },
2973            ),
2974        ];
2975
2976        for (state_label, local_state) in progressed_states {
2977            {
2978                let mut pool_meta = store.pool_meta.write().await;
2979                *pool_meta = reload_test_pool_meta(reload_test_pool_status(Some(local_state.clone()), newer_time));
2980            }
2981
2982            // A delayed peer message carries a snapshot that predates the local progression.
2983            let stale_snapshot = reload_test_pool_meta(reload_test_pool_status(
2984                Some(PoolDecommissionInfo {
2985                    start_time: Some(stale_time),
2986                    ..Default::default()
2987                }),
2988                stale_time,
2989            ));
2990            persist_reload_snapshot(&store, &stale_snapshot).await;
2991
2992            let merged_newer = store.reload_pool_meta().await.expect("stale reload should succeed");
2993            assert!(
2994                !merged_newer,
2995                "a delayed reload must not report merged newer state for the {state_label} progression"
2996            );
2997
2998            let pool_meta = store.pool_meta.read().await;
2999            let info = pool_meta.pools[0]
3000                .decommission
3001                .as_ref()
3002                .expect("local decommission state should survive a stale reload");
3003            assert_eq!(info.queued, local_state.queued, "{state_label} queued flag must not roll back");
3004            assert_eq!(info.canceled, local_state.canceled, "{state_label} canceled flag must not roll back");
3005            assert_eq!(info.failed, local_state.failed, "{state_label} failed flag must not roll back");
3006            assert_eq!(info.complete, local_state.complete, "{state_label} complete flag must not roll back");
3007            assert_eq!(
3008                pool_meta.pools[0].last_update, newer_time,
3009                "{state_label} progress timestamp must be kept"
3010            );
3011        }
3012
3013        shutdown.cancel();
3014    }
3015
3016    #[tokio::test]
3017    #[serial_test::serial]
3018    async fn peer_pool_meta_reload_merges_newer_state_and_is_idempotent_on_duplicate_delivery() {
3019        let (_temp_dir, store, shutdown) = setup_multi_pool_test_store("pool-meta-reload-duplicate", &[2]).await;
3020
3021        let older_time = OffsetDateTime::now_utc();
3022        let newer_time = older_time + TimeDuration::seconds(30);
3023
3024        {
3025            let mut pool_meta = store.pool_meta.write().await;
3026            *pool_meta = reload_test_pool_meta(reload_test_pool_status(
3027                Some(PoolDecommissionInfo {
3028                    items_decommissioned: 1,
3029                    ..Default::default()
3030                }),
3031                older_time,
3032            ));
3033        }
3034        let newer_snapshot = reload_test_pool_meta(reload_test_pool_status(
3035            Some(PoolDecommissionInfo {
3036                complete: true,
3037                items_decommissioned: 10,
3038                ..Default::default()
3039            }),
3040            newer_time,
3041        ));
3042        persist_reload_snapshot(&store, &newer_snapshot).await;
3043
3044        let merged_newer = store.reload_pool_meta().await.expect("first reload should succeed");
3045        assert!(merged_newer, "a strictly newer persisted snapshot must merge");
3046
3047        {
3048            let pool_meta = store.pool_meta.read().await;
3049            let info = pool_meta.pools[0].decommission.as_ref().expect("merged decommission state");
3050            assert!(info.complete);
3051            assert_eq!(info.items_decommissioned, 10);
3052            assert_eq!(pool_meta.pools[0].last_update, newer_time);
3053        }
3054
3055        // Redelivering the same generation must be a no-op.
3056        let duplicate_merged = store.reload_pool_meta().await.expect("duplicate reload should succeed");
3057        assert!(!duplicate_merged, "a duplicate delivery must not re-apply merged state");
3058
3059        {
3060            let pool_meta = store.pool_meta.read().await;
3061            let info = pool_meta.pools[0].decommission.as_ref().expect("merged decommission state");
3062            assert!(info.complete);
3063            assert_eq!(info.items_decommissioned, 10);
3064            assert_eq!(pool_meta.pools[0].last_update, newer_time);
3065        }
3066
3067        shutdown.cancel();
3068    }
3069
3070    #[tokio::test]
3071    #[serial_test::serial]
3072    async fn peer_pool_meta_reload_keeps_active_worker_progress_over_newer_snapshot() {
3073        let (_temp_dir, store, shutdown) = setup_multi_pool_test_store("pool-meta-reload-worker", &[2]).await;
3074        *store.decommission_cancelers.write().await = vec![Some(crate::core::pools::DecommissionCanceler::new_for_test(
3075            CancellationToken::new(),
3076        ))];
3077
3078        let worker_time = OffsetDateTime::now_utc();
3079        let newer_time = worker_time + TimeDuration::seconds(30);
3080
3081        {
3082            let mut pool_meta = store.pool_meta.write().await;
3083            *pool_meta = reload_test_pool_meta(reload_test_pool_status(
3084                Some(PoolDecommissionInfo {
3085                    start_time: Some(worker_time),
3086                    items_decommissioned: 10,
3087                    bytes_done: 1_024,
3088                    ..Default::default()
3089                }),
3090                worker_time,
3091            ));
3092        }
3093        // Even a strictly newer terminal snapshot must not override a live worker.
3094        let newer_terminal_snapshot = reload_test_pool_meta(reload_test_pool_status(
3095            Some(PoolDecommissionInfo {
3096                complete: true,
3097                ..Default::default()
3098            }),
3099            newer_time,
3100        ));
3101        persist_reload_snapshot(&store, &newer_terminal_snapshot).await;
3102
3103        let merged_newer = store
3104            .reload_pool_meta()
3105            .await
3106            .expect("reload under an active worker should succeed");
3107        assert!(!merged_newer, "an active local worker must block snapshot replacement");
3108
3109        let pool_meta = store.pool_meta.read().await;
3110        let info = pool_meta.pools[0]
3111            .decommission
3112            .as_ref()
3113            .expect("worker progress should remain");
3114        assert!(!info.complete);
3115        assert_eq!(info.items_decommissioned, 10);
3116        assert_eq!(info.bytes_done, 1_024);
3117        assert_eq!(pool_meta.pools[0].last_update, worker_time);
3118
3119        shutdown.cancel();
3120    }
3121
3122    #[tokio::test]
3123    #[serial_test::serial]
3124    async fn peer_pool_meta_reload_latches_recovery_when_initialized_metadata_is_missing() {
3125        let (temp_dir, store, shutdown) = setup_multi_pool_test_store("pool-meta-reload-missing", &[2]).await;
3126
3127        let kept_time = OffsetDateTime::now_utc();
3128        {
3129            let mut pool_meta = store.pool_meta.write().await;
3130            *pool_meta = reload_test_pool_meta(reload_test_pool_status(
3131                Some(PoolDecommissionInfo {
3132                    complete: true,
3133                    ..Default::default()
3134                }),
3135                kept_time,
3136            ));
3137        }
3138        // Persist first so the test controls exactly what exists on disk.
3139        persist_reload_snapshot(
3140            &store,
3141            &reload_test_pool_meta(reload_test_pool_status(
3142                Some(PoolDecommissionInfo {
3143                    complete: true,
3144                    ..Default::default()
3145                }),
3146                kept_time,
3147            )),
3148        )
3149        .await;
3150
3151        let mut deleted_any = false;
3152        for disk_index in 0..2 {
3153            let pool_bin_dir = temp_dir
3154                .path()
3155                .join(format!("pool0-disk{disk_index}"))
3156                .join(crate::disk::RUSTFS_META_BUCKET)
3157                .join(crate::core::pools::POOL_META_NAME);
3158            if pool_bin_dir.exists() {
3159                tokio::fs::remove_dir_all(&pool_bin_dir)
3160                    .await
3161                    .expect("persisted pool metadata object dir should be removable");
3162                deleted_any = true;
3163            }
3164        }
3165        // The meta-bucket layout may nest objects per pool; fall back to removing
3166        // every pool.bin object directory below the temp root.
3167        if !deleted_any {
3168            panic!("no pool.bin found under {:?}", temp_dir.path());
3169        }
3170
3171        let err = store
3172            .reload_pool_meta()
3173            .await
3174            .expect_err("an initialized cluster with every pool.bin missing must require recovery");
3175        assert!(err.to_string().contains("initialized cluster identity exists"));
3176        store
3177            .ensure_pool_meta_side_effects_safe("test reload side effect")
3178            .await
3179            .expect_err("the recovery-required reload must latch the runtime side-effect gate");
3180
3181        let pool_meta = store.pool_meta.read().await;
3182        let info = pool_meta.pools[0]
3183            .decommission
3184            .as_ref()
3185            .expect("missing persisted metadata must not default local state away");
3186        assert!(info.complete);
3187        assert_eq!(pool_meta.pools[0].last_update, kept_time);
3188
3189        shutdown.cancel();
3190    }
3191
3192    #[tokio::test]
3193    #[serial_test::serial]
3194    async fn peer_pool_meta_reload_latches_recovery_after_identity_epoch_conflict() {
3195        let (_temp_dir, store, shutdown) = setup_multi_pool_test_store("pool-meta-reload-identity-conflict", &[2, 2]).await;
3196        let conflicting_identity =
3197            initialized_pool_meta_identity_for_test(store.id, 2).expect("conflicting initialized identity should encode");
3198        save_config(store.pools[1].clone(), POOL_META_IDENTITY_NAME, conflicting_identity)
3199            .await
3200            .expect("second pool identity should be replaced for the conflict scenario");
3201
3202        let err = store
3203            .reload_pool_meta()
3204            .await
3205            .expect_err("divergent identity epochs must reject runtime reload");
3206        assert!(
3207            err.to_string()
3208                .contains("identity replicas disagree on cluster identity or epoch")
3209        );
3210        store
3211            .ensure_pool_meta_side_effects_safe("identity epoch conflict side effect")
3212            .await
3213            .expect_err("identity epoch conflict must latch the runtime side-effect gate");
3214
3215        shutdown.cancel();
3216    }
3217
3218    #[tokio::test]
3219    #[serial_test::serial]
3220    async fn peer_pool_meta_reload_latches_recovery_after_future_identity_version() {
3221        let (_temp_dir, store, shutdown) = setup_multi_pool_test_store("pool-meta-reload-future-identity", &[2, 2]).await;
3222        let (mut future_identity, _) =
3223            read_config_no_lock_preserve_empty_with_metadata(store.pools[1].clone(), POOL_META_IDENTITY_NAME)
3224                .await
3225                .expect("current identity should be readable");
3226        let current_version = u16::from_le_bytes([future_identity[2], future_identity[3]]);
3227        let future_version = current_version
3228            .checked_add(1)
3229            .expect("identity version should have a future value");
3230        future_identity[2..4].copy_from_slice(&future_version.to_le_bytes());
3231        save_config(store.pools[1].clone(), POOL_META_IDENTITY_NAME, future_identity)
3232            .await
3233            .expect("second pool identity should be replaced with a future version");
3234
3235        let err = store
3236            .reload_pool_meta()
3237            .await
3238            .expect_err("a future identity version must reject runtime reload");
3239        assert!(err.to_string().contains("pool metadata incompatible"));
3240        store
3241            .ensure_pool_meta_side_effects_safe("future identity version side effect")
3242            .await
3243            .expect_err("future identity version must latch the runtime side-effect gate");
3244
3245        shutdown.cancel();
3246    }
3247}