Skip to main content

rustfs_ecstore/store/
heal.rs

1// Copyright 2024 RustFS Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use super::*;
16use crate::core::pools::{POOL_META_NAME, load_pool_meta_identity_observing};
17use crate::disk::{self, error::DiskError};
18use crate::services::rebalance::{REBAL_META_NAME, RebalStatus};
19use crate::set_disk::get_lock_acquire_timeout;
20use crate::storage_api_contracts::heal::HealOperations as _;
21use crate::storage_api_contracts::namespace::NamespaceLocking as _;
22use rustfs_common::mrf_channel::MrfDeleteMarkerPurge;
23use rustfs_lock::NamespaceLockGuard;
24use std::collections::BTreeSet;
25use tracing::trace;
26
27const LOG_COMPONENT_ECSTORE: &str = "ecstore";
28const LOG_SUBSYSTEM_HEAL: &str = "heal";
29const EVENT_HEAL_ABANDONED_PARTS: &str = "heal_abandoned_parts";
30const EVENT_HEAL_FORMAT_COMPLETED: &str = "heal_format_completed";
31const EVENT_HEAL_OBJECT_STARTED: &str = "heal_object_started";
32
33/// An explicit bucket-heal admission owns one lifecycle generation through its write tail.
34pub(crate) struct BucketHealScope {
35    pub(crate) bucket: String,
36    pub(crate) incarnation: uuid::Uuid,
37    pub(crate) store: Arc<ECStore>,
38    fence: super::BucketIncarnationFenceGuard,
39}
40
41impl BucketHealScope {
42    pub(crate) fn check(&self) -> disk::error::Result<()> {
43        if self.fence.is_lock_lost() {
44            return Err(DiskError::other("bucket heal incarnation fence was lost"));
45        }
46        Ok(())
47    }
48}
49
50tokio::task_local! {
51    static BUCKET_HEAL_SCOPE: Arc<BucketHealScope>;
52}
53
54pub(crate) fn bucket_heal_scope(bucket: &str) -> Option<Arc<BucketHealScope>> {
55    BUCKET_HEAL_SCOPE
56        .try_with(|scope| (scope.bucket == bucket).then(|| scope.clone()))
57        .ok()
58        .flatten()
59}
60
61/// Storage-owned proof for the exact version and every selected erasure location.
62/// This is an in-process result, never reconstructed from admin drive telemetry.
63#[derive(Debug)]
64pub struct HealObjectAbsenceProof {
65    pub bucket: String,
66    pub object: String,
67    pub version_id: String,
68    pub bucket_incarnation_id: Uuid,
69    pub pool_index: Option<usize>,
70    pub set_index: Option<usize>,
71    pub locations: Vec<(usize, usize)>,
72    pub removed: bool,
73}
74
75#[derive(Debug)]
76pub struct HealObjectStorageResult {
77    pub item: HealResultItem,
78    pub error: Option<Error>,
79    pub absence: Option<HealObjectAbsenceProof>,
80}
81
82fn invalid_heal_pool_index(pool_idx: usize, pool_count: usize) -> Error {
83    StorageError::InvalidArgument(
84        "heal".to_string(),
85        "pool".to_string(),
86        format!("invalid heal pool index {pool_idx} for {pool_count} pools"),
87    )
88}
89
90fn is_pool_meta_object(bucket: &str, object: &str) -> bool {
91    bucket == RUSTFS_META_BUCKET && object == POOL_META_NAME
92}
93
94#[derive(Debug, Clone, Copy)]
95enum HealFormatPoolSkip {
96    Completed,
97    Retryable,
98}
99
100fn classify_heal_format_pool(
101    pool_idx: usize,
102    pool_cmd_line: &str,
103    pool_meta: &PoolMeta,
104    rebalance_meta: Option<&RebalanceMeta>,
105) -> Option<HealFormatPoolSkip> {
106    let Some(pool) = pool_meta.pools.get(pool_idx) else {
107        return Some(HealFormatPoolSkip::Retryable);
108    };
109
110    if pool.id != pool_idx || pool_cmd_line.is_empty() || pool.cmd_line.is_empty() || pool.cmd_line != pool_cmd_line {
111        return Some(HealFormatPoolSkip::Retryable);
112    }
113
114    if let Some(decommission) = pool.decommission.as_ref() {
115        if decommission.complete {
116            return Some(HealFormatPoolSkip::Completed);
117        }
118        if decommission.failed || decommission.canceled || decommission.queued || pool_meta.is_suspended(pool_idx) {
119            return Some(HealFormatPoolSkip::Retryable);
120        }
121    }
122
123    if let Some(meta) = rebalance_meta {
124        let Some(pool_stats) = meta.pool_stats.get(pool_idx) else {
125            return Some(HealFormatPoolSkip::Retryable);
126        };
127        if pool_stats.info.stopping || (pool_stats.participating && pool_stats.info.status == RebalStatus::Started) {
128            return Some(HealFormatPoolSkip::Retryable);
129        }
130    }
131
132    None
133}
134
135fn heal_format_pool_skip_error(skip: HealFormatPoolSkip) -> Error {
136    match skip {
137        HealFormatPoolSkip::Completed => StorageError::NoHealRequired,
138        HealFormatPoolSkip::Retryable => StorageError::SlowDown,
139    }
140}
141
142fn heal_format_fence_lost_error() -> Error {
143    StorageError::SlowDown
144}
145
146impl ECStore {
147    pub async fn purge_delete_marker_with_proof(
148        self: &Arc<Self>,
149        bucket: &str,
150        object: &str,
151        version_id: &str,
152        purge: &MrfDeleteMarkerPurge,
153        opts: &HealOpts,
154    ) -> Result<bool> {
155        if opts.dry_run || opts.no_lock || !purge.is_valid() {
156            return Err(StorageError::PreconditionFailed);
157        }
158        let version = Uuid::parse_str(version_id)
159            .ok()
160            .filter(|version| !version.is_nil())
161            .ok_or(StorageError::PreconditionFailed)?;
162        let (Some(pool_index), Some(set_index)) = (opts.pool, opts.set) else {
163            return Err(StorageError::PreconditionFailed);
164        };
165        let mut marker = rustfs_filemeta::MetaDeleteMarker::default();
166        let consumed = marker.unmarshal_msg(&purge.marker)?;
167        if usize::try_from(consumed).ok() != Some(purge.marker.len())
168            || marker.version_id != Some(version)
169            || marker.stable_identity() != purge.marker_identity
170        {
171            return Err(StorageError::PreconditionFailed);
172        }
173        let marker_info = marker.clone().into_fileinfo(bucket, object, false)?;
174        if !marker_info.is_canonical_delete_marker()
175            || marker_info.delete_marker_incarnation() != Some(purge.marker_incarnation_id)
176        {
177            return Err(StorageError::PreconditionFailed);
178        }
179
180        let current = self.bucket_incarnation_id_from_disk(bucket).await?;
181        if current.is_nil() {
182            return Err(StorageError::BucketNotFound(bucket.to_owned()));
183        }
184        let original = purge.bucket_incarnation_id;
185        let object = object.to_owned();
186        self.run_bucket_heal_at_incarnation(bucket, current, opts, move |store, bucket, _opts| async move {
187            let scope = bucket_heal_scope(&bucket).ok_or(StorageError::PreconditionFailed)?;
188            scope.check()?;
189            if original != current {
190                let retirement_store = crate::bucket::metadata_sys::object_store_if_initialized_in(&store.ctx)
191                    .await
192                    .ok_or(StorageError::PreconditionFailed)?;
193                if !crate::bucket::retirement::is_retired(retirement_store, &bucket, original).await? {
194                    return Err(StorageError::PreconditionFailed);
195                }
196            }
197            let pool = store
198                .pools
199                .get(pool_index)
200                .ok_or_else(|| invalid_heal_pool_index(pool_index, store.pools.len()))?;
201            let set = pool.disk_set.get(set_index).cloned().ok_or_else(|| {
202                StorageError::InvalidArgument(
203                    "heal".to_string(),
204                    "set".to_string(),
205                    format!("invalid heal set index {set_index} for pool {pool_index}"),
206                )
207            })?;
208            set.purge_delete_marker_exact(&bucket, &object, version, marker).await
209        })
210        .await
211    }
212
213    pub(super) async fn run_bucket_heal_at_incarnation<T, F, Fut>(
214        self: &Arc<Self>,
215        bucket: &str,
216        expected: uuid::Uuid,
217        opts: &HealOpts,
218        operation: F,
219    ) -> Result<T>
220    where
221        T: Send + 'static,
222        F: FnOnce(Arc<Self>, String, HealOpts) -> Fut + Send + 'static,
223        Fut: std::future::Future<Output = Result<T>> + Send + 'static,
224    {
225        if expected.is_nil() || self.ctx.lock_manager().is_disabled() {
226            return Err(Error::other(
227                "incarnation-bound bucket heal requires a valid identity and namespace locking",
228            ));
229        }
230        // Lock order: bucket lifecycle, then bucket/object and capacity locks used by healing.
231        let fence = self.acquire_bucket_incarnation_fence(bucket, expected).await?;
232        let scope = Arc::new(BucketHealScope {
233            bucket: bucket.to_owned(),
234            incarnation: expected,
235            store: self.clone(),
236            fence,
237        });
238        let store = self.clone();
239        let bucket = bucket.to_owned();
240        let opts = *opts;
241        // Dropping a caller's cancellation/timeout waiter must not release the lifecycle
242        // owner while a storage operation is still committing.
243        tokio::spawn(BUCKET_HEAL_SCOPE.scope(scope.clone(), async move {
244            scope.check().map_err(Error::from)?;
245            let result = operation(store, bucket, opts).await;
246            scope.check().map_err(Error::from)?;
247            result
248        }))
249        .await
250        .map_err(|error| Error::other_with_context("bucket heal owner task failed", error))?
251    }
252
253    pub async fn heal_bucket_at_incarnation(
254        self: &Arc<Self>,
255        bucket: &str,
256        expected: uuid::Uuid,
257        opts: &HealOpts,
258    ) -> Result<HealResultItem> {
259        self.run_bucket_heal_at_incarnation(bucket, expected, opts, |store, bucket, opts| async move {
260            store.heal_bucket(&bucket, &opts).await
261        })
262        .await
263    }
264
265    pub async fn heal_local_bucket_at_incarnation(
266        self: &Arc<Self>,
267        bucket: &str,
268        expected: uuid::Uuid,
269        opts: &HealOpts,
270        fenced_pools: Vec<usize>,
271        pools: Option<Vec<usize>>,
272    ) -> Result<HealResultItem> {
273        self.run_bucket_heal_at_incarnation(bucket, expected, opts, |store, bucket, opts| async move {
274            let peer =
275                crate::cluster::rpc::peer_s3_client::LocalPeerS3Client::new_with_instance_ctx(None, pools, store.ctx.clone());
276            crate::cluster::rpc::peer_s3_client::PeerS3Client::heal_bucket_with_fence(&peer, &bucket, &opts, &fenced_pools)
277                .await
278                .map_err(Error::from)
279        })
280        .await
281    }
282
283    pub async fn heal_object_at_incarnation(
284        self: &Arc<Self>,
285        bucket: &str,
286        object: &str,
287        version_id: &str,
288        expected: uuid::Uuid,
289        opts: &HealOpts,
290    ) -> Result<HealObjectStorageResult> {
291        let object = object.to_owned();
292        let version_id = version_id.to_owned();
293        self.run_bucket_heal_at_incarnation(bucket, expected, opts, |store, bucket, opts| async move {
294            store.heal_object_with_proof(&bucket, &object, &version_id, &opts).await
295        })
296        .await
297    }
298
299    async fn acquire_heal_format_fence(
300        &self,
301    ) -> Result<(
302        tokio::sync::MutexGuard<'_, PoolMetaWriteState>,
303        NamespaceLockGuard,
304        NamespaceLockGuard,
305        PoolMeta,
306        Option<RebalanceMeta>,
307    )> {
308        let metadata_pool = self
309            .pools
310            .first()
311            .cloned()
312            .ok_or_else(|| Error::other("heal format requires at least one storage pool"))?;
313        let mut write_state = self.pool_meta_save_gate.lock().await;
314        write_state.ensure_write_safe("heal format fence failed")?;
315
316        // Metadata fence order is part of the decommission/rebalance protocol:
317        // pool.bin must always be acquired before rebalance.bin. A read guard is
318        // sufficient to freeze the pool snapshot and lets replacement format
319        // recovery coexist with ordinary Heal's capacity fence. The rebalance
320        // write guard still serializes format repair and excludes transitions.
321        let pool_lock = metadata_pool.new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME).await?;
322        let pool_guard = pool_lock.get_read_lock(get_lock_acquire_timeout()).await?;
323        let rebalance_lock = metadata_pool.new_ns_lock(RUSTFS_META_BUCKET, REBAL_META_NAME).await?;
324        let rebalance_guard = rebalance_lock.get_write_lock(get_lock_acquire_timeout()).await?;
325
326        if pool_guard.is_lock_lost() || rebalance_guard.is_lock_lost() {
327            return Err(heal_format_fence_lost_error());
328        }
329
330        load_pool_meta_identity_observing(self.pools.clone(), &mut write_state).await?;
331        let mut pool_meta = PoolMeta::default();
332        let replica_state = pool_meta
333            .load_no_lock_from_replicas_observing(self.pools.clone(), &mut write_state)
334            .await?;
335        write_state.observe_replicas(replica_state);
336        write_state.ensure_missing_metadata_can_initialize()?;
337        write_state.ensure_write_safe("heal format fence failed")?;
338        if pool_meta.pools.len() != self.pools.len()
339            || pool_meta.pools.iter().enumerate().any(|(pool_idx, pool)| {
340                pool.id != pool_idx || pool.cmd_line.is_empty() || pool.cmd_line != self.pools[pool_idx].endpoints.cmd_line
341            })
342        {
343            return Err(heal_format_fence_lost_error());
344        }
345
346        let mut rebalance_meta = RebalanceMeta::new();
347        let rebalance_meta = match rebalance_meta
348            .load_with_opts(
349                metadata_pool,
350                ObjectOptions {
351                    no_lock: true,
352                    ..Default::default()
353                },
354            )
355            .await
356        {
357            Ok(()) => Some(rebalance_meta),
358            Err(Error::ConfigNotFound) => None,
359            Err(err) => return Err(err),
360        };
361
362        if rebalance_meta
363            .as_ref()
364            .is_some_and(|meta| meta.pool_stats.len() != self.pools.len())
365        {
366            return Err(heal_format_fence_lost_error());
367        }
368
369        write_state.ensure_write_safe("heal format fence failed")?;
370        if pool_guard.is_lock_lost() || rebalance_guard.is_lock_lost() {
371            return Err(heal_format_fence_lost_error());
372        }
373
374        Ok((write_state, pool_guard, rebalance_guard, pool_meta, rebalance_meta))
375    }
376
377    fn get_pools_for_heal_object(&self, opts: &HealOpts) -> Result<Vec<Arc<Sets>>> {
378        match opts.pool {
379            Some(pool_idx) => Ok(vec![
380                self.pools
381                    .get(pool_idx)
382                    .cloned()
383                    .ok_or_else(|| invalid_heal_pool_index(pool_idx, self.pools.len()))?,
384            ]),
385            None => Ok(self.pools.clone()),
386        }
387    }
388
389    /// Whether a replacement set owns this pool's `pool.bin` replica.
390    /// Placement is determined by the same hash as metadata writes, never by
391    /// whether a shard is currently readable on the replacement disk.
392    pub fn replacement_pool_metadata_required(&self, pool_index: usize, set_index: usize) -> Result<bool> {
393        let pool = self
394            .pools
395            .get(pool_index)
396            .ok_or_else(|| invalid_heal_pool_index(pool_index, self.pools.len()))?;
397        let target_set = pool.get_disks_for_heal_object(
398            POOL_META_NAME,
399            &HealOpts {
400                set: Some(set_index),
401                ..Default::default()
402            },
403        )?;
404        Ok(Arc::ptr_eq(&target_set, &pool.get_disks_by_key(POOL_META_NAME)))
405    }
406
407    /// Return every live erasure set selected by an object-heal scope.
408    pub async fn heal_erasure_set_scopes(&self, opts: &HealOpts) -> Result<Vec<(usize, usize)>> {
409        let pools = self.get_pools_for_heal_object(opts)?;
410        let pool_meta = self.pool_meta.read().await;
411        let mut scopes = Vec::new();
412
413        for pool in pools {
414            let suspended_complete = pool_meta.is_suspended(pool.pool_idx).then(|| {
415                pool_meta
416                    .pools
417                    .get(pool.pool_idx)
418                    .and_then(|status| status.decommission.as_ref())
419                    .is_some_and(|decommission| decommission.complete)
420            });
421            if let Some(complete) = suspended_complete {
422                if opts.pool.is_some() {
423                    return Err(if complete {
424                        StorageError::InvalidArgument(
425                            "heal".to_string(),
426                            "pool".to_string(),
427                            format!("heal pool {} has completed decommission", pool.pool_idx),
428                        )
429                    } else {
430                        Error::SlowDown
431                    });
432                }
433                continue;
434            }
435
436            if let Some(set_idx) = opts.set {
437                if set_idx >= pool.disk_set.len() {
438                    return Err(StorageError::InvalidArgument(
439                        "heal".to_string(),
440                        "set".to_string(),
441                        format!(
442                            "invalid heal set index {set_idx} for pool {} with {} sets",
443                            pool.pool_idx,
444                            pool.disk_set.len()
445                        ),
446                    ));
447                }
448                scopes.push((pool.pool_idx, set_idx));
449            } else {
450                scopes.extend((0..pool.disk_set.len()).map(|set_idx| (pool.pool_idx, set_idx)));
451            }
452        }
453
454        if scopes.is_empty() {
455            return Err(Error::SlowDown);
456        }
457
458        Ok(scopes)
459    }
460
461    #[instrument(skip(self))]
462    pub(super) async fn handle_heal_format(&self, dry_run: bool) -> Result<(HealResultItem, Option<Error>)> {
463        let mut r = HealResultItem {
464            heal_item_type: HealItemType::Metadata.to_string(),
465            detail: "disk-format".to_string(),
466            ..Default::default()
467        };
468
469        let mut count_no_heal = 0;
470        let mut count_completed = 0;
471        let mut first_error = None;
472        for (pool_idx, pool) in self.pools.iter().enumerate() {
473            let (mut write_state, pool_guard, rebalance_guard, pool_meta, rebalance_meta) =
474                self.acquire_heal_format_fence().await?;
475            if pool_guard.is_lock_lost() || rebalance_guard.is_lock_lost() {
476                first_error.get_or_insert(heal_format_fence_lost_error());
477                break;
478            }
479            if let Some(skip) = classify_heal_format_pool(pool_idx, &pool.endpoints.cmd_line, &pool_meta, rebalance_meta.as_ref())
480            {
481                if matches!(skip, HealFormatPoolSkip::Completed) {
482                    count_completed += 1;
483                } else {
484                    first_error.get_or_insert(heal_format_pool_skip_error(skip));
485                }
486                continue;
487            }
488
489            let fence_lost = || {
490                let lost = pool_guard.is_lock_lost()
491                    || rebalance_guard.is_lock_lost()
492                    || write_state.ensure_write_safe("heal format write fence failed").is_err();
493                if lost {
494                    write_state.block_writes_after_fence_loss();
495                }
496                lost
497            };
498            let (mut result, err) = pool.heal_format_with_fence(dry_run, fence_lost).await?;
499            if let Some(err) = err {
500                match err {
501                    StorageError::NoHealRequired => {
502                        count_no_heal += 1;
503                    }
504                    err => {
505                        first_error.get_or_insert(err);
506                    }
507                }
508            }
509            r.disk_count += result.disk_count;
510            r.set_count += result.set_count;
511            r.before.drives.append(&mut result.before.drives);
512            r.after.drives.append(&mut result.after.drives);
513
514            // A lease can be lost after the final write; fail closed before
515            // reporting the pool as successfully healed.
516            let fence_lost = pool_guard.is_lock_lost()
517                || rebalance_guard.is_lock_lost()
518                || write_state.ensure_write_safe("heal format publication fence failed").is_err();
519            if fence_lost {
520                write_state.block_writes_after_fence_loss();
521                first_error.get_or_insert(heal_format_fence_lost_error());
522                break;
523            }
524        }
525        if let Some(err) = first_error {
526            return Ok((r, Some(err)));
527        }
528        if count_no_heal + count_completed == self.pools.len() {
529            info!(
530                event = EVENT_HEAL_FORMAT_COMPLETED,
531                component = LOG_COMPONENT_ECSTORE,
532                subsystem = LOG_SUBSYSTEM_HEAL,
533                dry_run,
534                result = "no_heal_required",
535                pool_count = self.pools.len(),
536                "Heal format completed"
537            );
538            return Ok((r, Some(StorageError::NoHealRequired)));
539        }
540        info!(
541            event = EVENT_HEAL_FORMAT_COMPLETED,
542            component = LOG_COMPONENT_ECSTORE,
543            subsystem = LOG_SUBSYSTEM_HEAL,
544            dry_run,
545            result = "healed_or_inspected",
546            disk_count = r.disk_count,
547            set_count = r.set_count,
548            "Heal format completed"
549        );
550        Ok((r, None))
551    }
552
553    #[instrument(skip(self, targets), fields(pool_index, set_index, target_count = targets.len()))]
554    pub async fn heal_replacement_format(
555        &self,
556        dry_run: bool,
557        pool_index: usize,
558        set_index: usize,
559        targets: &[String],
560    ) -> Result<(HealResultItem, Option<Error>)> {
561        let pool = self
562            .pools
563            .get(pool_index)
564            .ok_or_else(|| invalid_heal_pool_index(pool_index, self.pools.len()))?;
565        let set = pool.disk_set.get(set_index).cloned().ok_or_else(|| {
566            StorageError::InvalidArgument(
567                "heal".to_string(),
568                "set".to_string(),
569                format!("invalid heal set index {set_index} for pool {pool_index}"),
570            )
571        })?;
572
573        let (mut write_state, pool_guard, rebalance_guard, pool_meta, rebalance_meta) = self.acquire_heal_format_fence().await?;
574        if let Some(skip) = classify_heal_format_pool(pool_index, &pool.endpoints.cmd_line, &pool_meta, rebalance_meta.as_ref()) {
575            return Ok((HealResultItem::default(), Some(heal_format_pool_skip_error(skip))));
576        }
577
578        let fence_lost = || {
579            let lost = pool_guard.is_lock_lost()
580                || rebalance_guard.is_lock_lost()
581                || write_state
582                    .ensure_write_safe("replacement format write fence failed")
583                    .is_err();
584            if lost {
585                write_state.block_writes_after_fence_loss();
586            }
587            lost
588        };
589        let result = set.heal_replacement_format_with_fence(dry_run, targets, fence_lost).await?;
590        let fence_lost = pool_guard.is_lock_lost()
591            || rebalance_guard.is_lock_lost()
592            || write_state
593                .ensure_write_safe("replacement format publication fence failed")
594                .is_err();
595        if fence_lost {
596            write_state.block_writes_after_fence_loss();
597            return Ok((result.0, Some(heal_format_fence_lost_error())));
598        }
599        Ok(result)
600    }
601
602    #[instrument(skip(self, targets), fields(pool_index, set_index, target_count = targets.len()))]
603    pub async fn replacement_targets_have_version(
604        &self,
605        bucket: &str,
606        object: &str,
607        version_id: &str,
608        pool_index: usize,
609        set_index: usize,
610        targets: &[String],
611    ) -> Result<bool> {
612        let pool = self
613            .pools
614            .get(pool_index)
615            .ok_or_else(|| invalid_heal_pool_index(pool_index, self.pools.len()))?;
616        let set = pool.disk_set.get(set_index).cloned().ok_or_else(|| {
617            StorageError::InvalidArgument(
618                "heal".to_string(),
619                "set".to_string(),
620                format!("invalid heal set index {set_index} for pool {pool_index}"),
621            )
622        })?;
623
624        set.replacement_targets_have_version(bucket, object, version_id, targets)
625            .await
626            .map_err(Into::into)
627    }
628
629    #[instrument(skip(self))]
630    pub(super) async fn handle_heal_bucket(&self, bucket: &str, opts: &HealOpts) -> Result<HealResultItem> {
631        let movement_gate = self.ctx.data_movement_operation_gate();
632        let _movement_guard = movement_gate.read().await;
633        let save_guard = self.pool_meta_save_gate.lock().await;
634        save_guard.ensure_write_safe("bucket heal cannot run while pool metadata requires recovery")?;
635        let mut fenced_pools = BTreeSet::new();
636        {
637            let pool_meta = self.pool_meta.read().await;
638            fenced_pools.extend((0..pool_meta.pools.len()).filter(|pool_idx| pool_meta.is_suspended(*pool_idx)));
639            if let Some(pool_idx) = opts.pool {
640                if pool_idx >= pool_meta.pools.len() {
641                    return Err(invalid_heal_pool_index(pool_idx, pool_meta.pools.len()));
642                }
643                if pool_meta.is_suspended(pool_idx) {
644                    let complete = pool_meta.pools[pool_idx]
645                        .decommission
646                        .as_ref()
647                        .is_some_and(|decommission| decommission.complete);
648                    return Err(if complete {
649                        StorageError::InvalidArgument(
650                            "heal".to_string(),
651                            "pool".to_string(),
652                            format!("heal pool {pool_idx} has completed decommission"),
653                        )
654                    } else {
655                        Error::SlowDown
656                    });
657                }
658            }
659        }
660
661        let dispatch_fenced_pools = fenced_pools.iter().copied().collect::<Vec<_>>();
662        drop(save_guard);
663        let mut res = self
664            .peer_sys
665            .heal_bucket_with_fence_from_movement_guarded_coordinator(bucket, opts, &dispatch_fenced_pools)
666            .await?;
667        {
668            let pool_meta = self.pool_meta.read().await;
669            fenced_pools.extend((0..pool_meta.pools.len()).filter(|pool_idx| pool_meta.is_suspended(*pool_idx)));
670        }
671        if !fenced_pools.is_empty() {
672            let pools = fenced_pools.iter().map(usize::to_string).collect::<Vec<_>>().join(", ");
673            res.detail = format!("skipped: bucket-volume heal fenced on decommission-suspended pool(s): {pools}");
674        }
675
676        Ok(res)
677    }
678
679    #[instrument(level = "trace", skip(self, opts), fields(bucket = %bucket, object = %object, version_id = %version_id))]
680    pub(super) async fn handle_heal_object(
681        &self,
682        bucket: &str,
683        object: &str,
684        version_id: &str,
685        opts: &HealOpts,
686    ) -> Result<(HealResultItem, Option<Error>)> {
687        self.handle_heal_object_with_absence(bucket, object, version_id, opts, &mut None, None)
688            .await
689    }
690
691    pub async fn heal_object_with_proof(
692        &self,
693        bucket: &str,
694        object: &str,
695        version_id: &str,
696        opts: &HealOpts,
697    ) -> Result<HealObjectStorageResult> {
698        if opts.dry_run || opts.no_lock || version_id.is_empty() || super::utils::is_reserved_or_invalid_bucket(bucket, false) {
699            let (item, error) = self.handle_heal_object(bucket, object, version_id, opts).await?;
700            return Ok(HealObjectStorageResult {
701                item,
702                error,
703                absence: None,
704            });
705        }
706
707        // Match object publication: bucket lifecycle before capacity and object
708        // namespace locks. Keep the incarnation pinned through proof delivery.
709        // Reuse the admission's owner: reacquiring a read lock behind a queued
710        // lifecycle writer would deadlock with the read lock we already hold.
711        let scope = bucket_heal_scope(bucket).filter(|scope| std::ptr::eq(scope.store.as_ref(), self));
712        let guard = if let Some(scope) = &scope {
713            scope.check().map_err(Error::from)?;
714            None
715        } else {
716            Some(self.acquire_bucket_lifecycle_read_lock(bucket).await?)
717        };
718        let fence_is_lost = || {
719            scope.as_ref().is_some_and(|scope| scope.fence.is_lock_lost())
720                || guard.as_ref().is_some_and(NamespaceLockGuard::is_lock_lost)
721        };
722        let lifecycle_guard = scope
723            .as_ref()
724            .and_then(|scope| scope.fence.namespace_lock_guard())
725            .or(guard.as_ref())
726            .ok_or_else(|| Error::other("bucket heal requires a held lifecycle guard"))?;
727        let retirement = crate::bucket::retirement::MarkerRetirementContext {
728            store: crate::bucket::metadata_sys::object_store_if_initialized_in(&self.ctx).await,
729            current_incarnation: if let Some(scope) = &scope {
730                Some(scope.incarnation)
731            } else {
732                self.bucket_incarnation_id_from_disk(bucket)
733                    .await
734                    .ok()
735                    .filter(|id| !id.is_nil())
736            },
737            lifecycle_guard,
738        };
739        let mut proofs = None;
740        let (item, mut error) = self
741            .handle_heal_object_with_absence(bucket, object, version_id, opts, &mut proofs, Some(&retirement))
742            .await?;
743        // Read the authoritative incarnation only for an absence candidate.
744        // The lifecycle guard has pinned it throughout the storage operation.
745        let incarnation = if proofs.is_some() && !fence_is_lost() {
746            if let Some(scope) = &scope {
747                Some(scope.incarnation)
748            } else {
749                self.bucket_incarnation_id_from_disk(bucket)
750                    .await
751                    .ok()
752                    .filter(|id| !id.is_nil())
753            }
754        } else {
755            None
756        };
757        let absence = match (incarnation, proofs) {
758            (Some(incarnation), Some(proofs)) if !fence_is_lost() => Some(HealObjectAbsenceProof {
759                bucket: bucket.to_owned(),
760                object: object.to_owned(),
761                version_id: version_id.to_owned(),
762                bucket_incarnation_id: incarnation,
763                pool_index: opts.pool,
764                set_index: opts.set,
765                removed: proofs.iter().any(|proof| proof.removed),
766                locations: proofs.into_iter().map(|proof| (proof.pool_index, proof.set_index)).collect(),
767            }),
768            _ => None,
769        };
770        if absence.is_some() {
771            error = None;
772        }
773        Ok(HealObjectStorageResult { item, error, absence })
774    }
775
776    async fn handle_heal_object_with_absence(
777        &self,
778        bucket: &str,
779        object: &str,
780        version_id: &str,
781        opts: &HealOpts,
782        absence: &mut Option<Vec<crate::set_disk::HealedObjectAbsence>>,
783        retirement: Option<&crate::bucket::retirement::MarkerRetirementContext<'_>>,
784    ) -> Result<(HealResultItem, Option<Error>)> {
785        trace!(
786            event = EVENT_HEAL_OBJECT_STARTED,
787            component = LOG_COMPONENT_ECSTORE,
788            subsystem = LOG_SUBSYSTEM_HEAL,
789            bucket = %bucket,
790            object = %object,
791            version_id = %version_id,
792            remove = opts.remove,
793            scan_mode = ?opts.scan_mode,
794            "Heal object started"
795        );
796        let object = encode_dir_object(object);
797
798        *absence = None;
799        let pools = self.get_pools_for_heal_object(opts)?;
800        let requested_pool_count = pools.len();
801        if let Some(set_idx) = opts.set {
802            for pool in &pools {
803                if set_idx >= pool.disk_set.len() {
804                    let err = StorageError::InvalidArgument(
805                        "heal".to_string(),
806                        "set".to_string(),
807                        format!(
808                            "invalid heal set index {set_idx} for pool {} with {} sets",
809                            pool.pool_idx,
810                            pool.disk_set.len()
811                        ),
812                    );
813                    if opts.pool.is_some() {
814                        return Err(err);
815                    }
816                    return Ok((HealResultItem::default(), Some(err)));
817                }
818            }
819        }
820        #[cfg(test)]
821        let store_id = self.id;
822
823        let mut heal_pools = Vec::with_capacity(pools.len());
824        for pool in pools.iter() {
825            let suspended_complete = {
826                let pool_meta = self.pool_meta.read().await;
827                pool_meta.is_suspended(pool.pool_idx).then(|| {
828                    pool_meta
829                        .pools
830                        .get(pool.pool_idx)
831                        .and_then(|status| status.decommission.as_ref())
832                        .is_some_and(|decommission| decommission.complete)
833                })
834            };
835            if let Some(complete) = suspended_complete {
836                if opts.pool.is_some() {
837                    let _ = pool.get_disks_for_heal_object(&object, opts)?;
838                    let err = if complete {
839                        StorageError::InvalidArgument(
840                            "heal".to_string(),
841                            "pool".to_string(),
842                            format!("heal pool {} has completed decommission", pool.pool_idx),
843                        )
844                    } else {
845                        Error::SlowDown
846                    };
847                    return Ok((HealResultItem::default(), Some(err)));
848                }
849                continue;
850            }
851            heal_pools.push(Arc::clone(pool));
852        }
853        let results = if is_pool_meta_object(bucket, &object) && !opts.no_lock && !heal_pools.is_empty() {
854            let target_pool_indices = heal_pools.iter().map(|pool| pool.pool_idx).collect::<Vec<_>>();
855            match self.acquire_pool_meta_object_heal_fence(&target_pool_indices).await {
856                Ok((pool_meta_guard, admissions)) => {
857                    let fixed_set = self.pools.first().and_then(|pool| pool.disk_set.first()).cloned();
858                    let futures = heal_pools.iter().zip(admissions).map(|(pool, admission)| {
859                        let pool = Arc::clone(pool);
860                        let pool_object = object.clone();
861                        let fixed_set = fixed_set.clone();
862                        let mut opts = *opts;
863                        async move {
864                            admission?;
865                            let fixed_set = fixed_set.ok_or_else(|| Error::other("pool metadata heal requires a fixed set"))?;
866                            let target_set = pool.get_disks_for_heal_object(&pool_object, &opts)?;
867                            opts.no_lock = fixed_set.shares_namespace_lock_domain(&target_set).await;
868                            #[cfg(test)]
869                            if !opts.no_lock {
870                                crate::core::pools::notify_decommission_external_heal_target_lock_attempted();
871                            }
872                            #[cfg(test)]
873                            crate::core::pools::notify_decommission_external_heal_operation_started(store_id);
874                            pool.heal_object_with_absence(bucket, &pool_object, version_id, &opts, retirement)
875                                .await
876                        }
877                    });
878                    let results = join_all(futures).await;
879                    drop(pool_meta_guard);
880                    results
881                }
882                Err(err) => (0..heal_pools.len()).map(|_| Err(err.clone())).collect(),
883            }
884        } else {
885            let mut futures = Vec::with_capacity(heal_pools.len());
886            for pool in heal_pools {
887                let pool_idx = pool.pool_idx;
888                let pool_object = object.clone();
889                let opts = *opts;
890                futures.push(self.run_external_decommission_capacity_heal(
891                    pool_idx,
892                    bucket,
893                    &object,
894                    opts,
895                    move |opts| async move {
896                        #[cfg(test)]
897                        crate::core::pools::notify_decommission_external_heal_operation_started(store_id);
898                        pool.heal_object_with_absence(bucket, &pool_object, version_id, &opts, retirement)
899                            .await
900                    },
901                ));
902            }
903            join_all(futures).await
904        };
905
906        let mut proofs = Vec::with_capacity(requested_pool_count);
907        let mut errs = Vec::with_capacity(self.pools.len());
908        let mut ress = Vec::with_capacity(self.pools.len());
909
910        for res in results.into_iter() {
911            match res {
912                Ok((result, err, proof)) => {
913                    if let Some(proof) = proof
914                        && (err.is_none()
915                            || err
916                                .as_ref()
917                                .is_some_and(|err| is_err_object_not_found(err) || is_err_version_not_found(err)))
918                    {
919                        proofs.push(proof);
920                    }
921                    let mut result = result;
922                    result.object = decode_dir_object(&result.object);
923                    ress.push(result);
924                    errs.push(err);
925                }
926                Err(err) => {
927                    errs.push(Some(err));
928                    ress.push(HealResultItem::default());
929                }
930            }
931        }
932
933        // Absence in one pool cannot discharge a responsibility covering other
934        // pools, including skipped decommission sources or failed lookups.
935        if requested_pool_count > 0 && proofs.len() == requested_pool_count {
936            *absence = Some(proofs);
937        }
938
939        for (idx, err) in errs.iter().enumerate() {
940            if err.is_none() {
941                return Ok((ress.remove(idx), None));
942            }
943        }
944
945        // No pool returned a nil error, return the first non 'not found' error
946        for (index, err) in errs.iter().enumerate() {
947            return match err {
948                Some(err) => {
949                    if is_err_object_not_found(err) || is_err_version_not_found(err) {
950                        continue;
951                    }
952                    Ok((ress.remove(index), Some(err.clone())))
953                }
954                None => Ok((ress.remove(index), None)),
955            };
956        }
957
958        // At this stage, all errors are 'not found'
959        if !version_id.is_empty() {
960            return Ok((HealResultItem::default(), Some(Error::FileVersionNotFound)));
961        }
962
963        Ok((HealResultItem::default(), Some(Error::FileNotFound)))
964    }
965
966    #[instrument(skip(self))]
967    pub(super) async fn handle_check_abandoned_parts(&self, bucket: &str, object: &str, opts: &HealOpts) -> Result<()> {
968        let object = encode_dir_object(object);
969        let pools = self.get_pools_for_heal_object(opts)?;
970
971        let mut futures = Vec::with_capacity(pools.len());
972        for pool in pools.iter() {
973            futures.push(pool.check_abandoned_parts(bucket, &object, opts));
974        }
975
976        let mut first_error = None;
977        for result in join_all(futures).await {
978            if let Err(err) = result
979                && first_error.is_none()
980            {
981                first_error = Some(err);
982            }
983        }
984
985        if let Some(err) = first_error {
986            return Err(err);
987        }
988
989        trace!(
990            event = EVENT_HEAL_ABANDONED_PARTS,
991            component = LOG_COMPONENT_ECSTORE,
992            subsystem = LOG_SUBSYSTEM_HEAL,
993            state = "completed",
994            result = "ok",
995            bucket,
996            object,
997            dry_run = opts.dry_run,
998            "Heal abandoned parts completed"
999        );
1000
1001        Ok(())
1002    }
1003}
1004
1005#[cfg(test)]
1006mod tests {
1007    use super::*;
1008    use crate::bucket::metadata_sys;
1009    use crate::cluster::rpc::PeerS3Client;
1010    use crate::config::com::{delete_config, read_config_no_lock_preserve_empty_with_metadata, save_config};
1011    use crate::core::pools::{
1012        DecommissionCapacityAdmission, DecommissionCapacityLockOrderBarrier, DecommissionErasureLayout,
1013        DecommissionPoolCapacityInfo, POOL_META_IDENTITY_NAME, PoolDecommissionInfo, PoolMetaReplicaState, PoolStatus,
1014        initialized_pool_meta_identity_for_test, set_decommission_capacity_info_overrides_for_test,
1015    };
1016    use crate::core::sets::HealFormatAfterSaveBarrier;
1017    use crate::disk::error::Result as DiskResult;
1018    use crate::disk::{DeleteOptions, DiskOption, DiskStore, FORMAT_CONFIG_FILE, format::FormatV3, new_disk};
1019    use crate::layout::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints};
1020    use crate::runtime::instance::InstanceContext;
1021    use crate::services::rebalance::{
1022        RebalanceInfo, RebalanceStats, test_three_pool_stores_with_isolated_node_contexts, test_two_pool_stores,
1023    };
1024    use crate::storage_api_contracts::bucket::{
1025        BucketInfo, BucketOperations, BucketOptions, DeleteBucketOptions, MakeBucketOptions,
1026    };
1027    use crate::storage_api_contracts::object::{ObjectIO as _, ObjectOperations};
1028    use crate::store::init_format::{load_format_erasure, save_format_file};
1029    use crate::store::init_local_disks_with_instance_ctx;
1030    use rustfs_heal_contracts::heal_channel::DriveState;
1031    use tokio_util::sync::CancellationToken;
1032
1033    #[derive(Debug)]
1034    struct BlockingHealPeer {
1035        started: Arc<tokio::sync::Notify>,
1036        release: Arc<tokio::sync::Notify>,
1037    }
1038
1039    #[async_trait::async_trait]
1040    impl PeerS3Client for BlockingHealPeer {
1041        async fn heal_bucket(&self, _bucket: &str, _opts: &HealOpts) -> DiskResult<HealResultItem> {
1042            self.started.notify_one();
1043            self.release.notified().await;
1044            Ok(HealResultItem::default())
1045        }
1046
1047        async fn make_bucket(&self, _bucket: &str, _opts: &MakeBucketOptions) -> DiskResult<()> {
1048            Ok(())
1049        }
1050
1051        async fn list_bucket(&self, _opts: &BucketOptions) -> DiskResult<Vec<BucketInfo>> {
1052            Ok(Vec::new())
1053        }
1054
1055        async fn delete_bucket(&self, _bucket: &str, _opts: &DeleteBucketOptions) -> DiskResult<()> {
1056            Ok(())
1057        }
1058
1059        async fn get_bucket_info(&self, _bucket: &str, _opts: &BucketOptions) -> DiskResult<BucketInfo> {
1060            Ok(BucketInfo::default())
1061        }
1062
1063        fn get_pools(&self) -> Option<Vec<usize>> {
1064            Some(vec![0, 1])
1065        }
1066    }
1067
1068    #[derive(Debug)]
1069    struct WriterQueuedLocalHealPeer {
1070        movement_gate: Arc<tokio::sync::RwLock<()>>,
1071        writer_queued: Arc<tokio::sync::Notify>,
1072        writer_acquired: Arc<tokio::sync::Notify>,
1073    }
1074
1075    #[async_trait::async_trait]
1076    impl PeerS3Client for WriterQueuedLocalHealPeer {
1077        async fn heal_bucket(&self, _bucket: &str, _opts: &HealOpts) -> DiskResult<HealResultItem> {
1078            let _movement_guard = self
1079                .movement_gate
1080                .try_read()
1081                .map_err(|_| crate::disk::error::DiskError::TooManyOpenFiles)?;
1082            Ok(HealResultItem::default())
1083        }
1084
1085        async fn heal_bucket_with_fence_from_movement_guarded_coordinator(
1086            &self,
1087            _bucket: &str,
1088            _opts: &HealOpts,
1089            _fenced_pools: &[usize],
1090        ) -> DiskResult<HealResultItem> {
1091            Ok(HealResultItem::default())
1092        }
1093
1094        async fn make_bucket(&self, _bucket: &str, _opts: &MakeBucketOptions) -> DiskResult<()> {
1095            Ok(())
1096        }
1097
1098        async fn list_bucket(&self, _opts: &BucketOptions) -> DiskResult<Vec<BucketInfo>> {
1099            Ok(Vec::new())
1100        }
1101
1102        async fn delete_bucket(&self, _bucket: &str, _opts: &DeleteBucketOptions) -> DiskResult<()> {
1103            Ok(())
1104        }
1105
1106        async fn get_bucket_info(&self, _bucket: &str, _opts: &BucketOptions) -> DiskResult<BucketInfo> {
1107            let movement_gate = self.movement_gate.clone();
1108            let writer_acquired = self.writer_acquired.clone();
1109            tokio::spawn(async move {
1110                let _movement_guard = movement_gate.write().await;
1111                writer_acquired.notify_one();
1112            });
1113            while self.movement_gate.try_read().is_ok() {
1114                tokio::task::yield_now().await;
1115            }
1116            self.writer_queued.notify_one();
1117            Ok(BucketInfo::default())
1118        }
1119
1120        fn get_pools(&self) -> Option<Vec<usize>> {
1121            Some(vec![0, 1])
1122        }
1123    }
1124
1125    async fn minimal_heal_pool(pool_idx: usize) -> Arc<Sets> {
1126        let format = FormatV3::new(1, 1);
1127        let endpoint_url = format!("http://127.0.0.1:{}/data", 19000 + pool_idx);
1128        let mut endpoint = Endpoint::try_from(endpoint_url.as_str()).expect("endpoint should parse");
1129        endpoint.set_pool_index(pool_idx);
1130        endpoint.set_set_index(0);
1131        endpoint.set_disk_index(0);
1132
1133        Sets::new(
1134            vec![None],
1135            &PoolEndpoints {
1136                legacy: false,
1137                set_count: 1,
1138                drives_per_set: 1,
1139                endpoints: Endpoints::from(vec![endpoint]),
1140                cmd_line: String::new(),
1141                platform: String::new(),
1142            },
1143            &format,
1144            pool_idx,
1145            0,
1146        )
1147        .await
1148        .expect("minimal pool should build")
1149    }
1150
1151    #[tokio::test]
1152    #[serial_test::serial]
1153    async fn bucket_incarnation_heal_preserves_successor_shards_and_allows_fresh_repair() {
1154        let (_root, store, shutdown) = multi_pool_heal_store().await;
1155        let bucket = format!("heal-incarnation-{}", Uuid::new_v4().simple());
1156        store
1157            .make_bucket(&bucket, &MakeBucketOptions::default())
1158            .await
1159            .expect("original bucket");
1160        let old = store
1161            .bucket_incarnation_id_from_disk(&bucket)
1162            .await
1163            .expect("original identity");
1164        store
1165            .delete_bucket(&bucket, &DeleteBucketOptions::default())
1166            .await
1167            .expect("normal bucket deletion");
1168        store
1169            .make_bucket(&bucket, &MakeBucketOptions::default())
1170            .await
1171            .expect("same-name successor");
1172        let new = store
1173            .bucket_incarnation_id_from_disk(&bucket)
1174            .await
1175            .expect("successor identity");
1176        assert_ne!(old, new);
1177        let object = "successor-version";
1178        let version = Uuid::new_v4().to_string();
1179        let set = store.pools[0].get_disks(0);
1180        let mut reader = PutObjReader::from_vec(vec![17; 512 * 1024]);
1181        set.put_object(
1182            &bucket,
1183            object,
1184            &mut reader,
1185            &ObjectOptions {
1186                versioned: true,
1187                version_id: Some(version.clone()),
1188                // Drain every rename before the fixture removes a physical shard.
1189                write_completion: crate::object_api::WriteCompletion::TailDrained,
1190                ..Default::default()
1191            },
1192        )
1193        .await
1194        .expect("write successor version");
1195        let missing = set.disks.read().await[0].clone().expect("missing target");
1196        let healthy = set.disks.read().await[1].clone().expect("healthy target");
1197        missing
1198            .delete(
1199                &bucket,
1200                object,
1201                DeleteOptions {
1202                    recursive: true,
1203                    immediate: true,
1204                    ..Default::default()
1205                },
1206            )
1207            .await
1208            .expect("remove one successor shard");
1209        let baseline = healthy
1210            .read_all(&bucket, &format!("{object}/xl.meta"))
1211            .await
1212            .expect("healthy baseline");
1213        let opts = HealOpts {
1214            pool: Some(0),
1215            set: Some(0),
1216            scan_mode: rustfs_heal_contracts::heal_channel::HealScanMode::Deep,
1217            ..Default::default()
1218        };
1219        // Positive control for the original bug: name-only healing repairs the successor.
1220        let (_, error) = store
1221            .heal_object(&bucket, object, &version, &opts)
1222            .await
1223            .expect("unbound control");
1224        assert!(error.is_none());
1225        assert!(missing.read_xl(&bucket, object, false).await.is_ok());
1226        missing
1227            .delete(
1228                &bucket,
1229                object,
1230                DeleteOptions {
1231                    recursive: true,
1232                    immediate: true,
1233                    ..Default::default()
1234                },
1235            )
1236            .await
1237            .expect("restore the same missing-shard condition");
1238        let fi = healthy
1239            .read_version("", &bucket, object, &version, &Default::default())
1240            .await
1241            .expect("successor metadata");
1242        let disk_ref = healthy.endpoint().to_string();
1243        assert!(
1244            store
1245                .write_local_metadata_at_incarnation(&disk_ref, (&bucket, object), fi.clone(), old)
1246                .await
1247                .is_err()
1248        );
1249        assert!(
1250            store
1251                .delete_local_path_at_incarnation(
1252                    &disk_ref,
1253                    (&bucket, object),
1254                    DeleteOptions {
1255                        recursive: true,
1256                        immediate: true,
1257                        ..Default::default()
1258                    },
1259                    old
1260                )
1261                .await
1262                .is_err()
1263        );
1264        assert!(
1265            store
1266                .delete_local_version_at_incarnation(
1267                    &disk_ref,
1268                    (&bucket, object),
1269                    fi.clone(),
1270                    false,
1271                    DeleteOptions::default(),
1272                    old
1273                )
1274                .await
1275                .is_err()
1276        );
1277        assert!(
1278            store
1279                .rename_local_data_at_incarnation(&disk_ref, (&bucket, object), &fi, (&bucket, "stale-destination"), old)
1280                .await
1281                .is_err()
1282        );
1283        assert!(store.heal_bucket_at_incarnation(&bucket, old, &opts).await.is_err());
1284        assert!(
1285            store
1286                .heal_object_at_incarnation(&bucket, object, &version, old, &opts)
1287                .await
1288                .is_err()
1289        );
1290        assert!(
1291            missing.read_xl(&bucket, object, false).await.is_err(),
1292            "obsolete admission must not publish a shard"
1293        );
1294        assert_eq!(
1295            healthy
1296                .read_all(&bucket, &format!("{object}/xl.meta"))
1297                .await
1298                .expect("healthy after stale heal"),
1299            baseline
1300        );
1301        store
1302            .heal_bucket_at_incarnation(&bucket, new, &opts)
1303            .await
1304            .expect("fresh bucket metadata repair");
1305        let result = store
1306            .heal_object_at_incarnation(&bucket, object, &version, new, &opts)
1307            .await
1308            .expect("new admission repair");
1309        assert!(result.error.is_none(), "new admission must repair independently: {:?}", result.error);
1310        assert!(
1311            missing.read_xl(&bucket, object, false).await.is_ok(),
1312            "new admission publishes the missing shard"
1313        );
1314        store
1315            .write_local_metadata_at_incarnation(&disk_ref, (&bucket, object), fi.clone(), new)
1316            .await
1317            .expect("fresh target metadata");
1318        store
1319            .delete_local_version_at_incarnation(&disk_ref, (&bucket, object), fi, false, DeleteOptions::default(), new)
1320            .await
1321            .expect("fresh target version cleanup");
1322        healthy
1323            .write_all(&bucket, "fresh-cleanup/data", bytes::Bytes::from_static(b"temporary"))
1324            .await
1325            .expect("cleanup fixture");
1326        store
1327            .delete_local_path_at_incarnation(
1328                &disk_ref,
1329                (&bucket, "fresh-cleanup"),
1330                DeleteOptions {
1331                    recursive: true,
1332                    immediate: true,
1333                    ..Default::default()
1334                },
1335                new,
1336            )
1337            .await
1338            .expect("fresh target path cleanup");
1339        shutdown.cancel();
1340    }
1341
1342    #[tokio::test]
1343    #[serial_test::serial]
1344    async fn bucket_incarnation_heal_owner_survives_cancelled_waiter_and_queued_writer() {
1345        let (_root, store, shutdown) = multi_pool_heal_store().await;
1346        let bucket = format!("heal-owner-{}", Uuid::new_v4().simple());
1347        store
1348            .make_bucket(&bucket, &MakeBucketOptions::default())
1349            .await
1350            .expect("bucket");
1351        let identity = store.bucket_incarnation_id_from_disk(&bucket).await.expect("identity");
1352        let (started_tx, started_rx) = tokio::sync::oneshot::channel();
1353        let (release_tx, release_rx) = tokio::sync::oneshot::channel();
1354        let (drained_tx, drained_rx) = tokio::sync::oneshot::channel();
1355        let worker_store = store.clone();
1356        let worker_bucket = bucket.clone();
1357        let waiter = tokio::spawn(async move {
1358            worker_store
1359                .run_bucket_heal_at_incarnation(
1360                    &worker_bucket,
1361                    identity,
1362                    &HealOpts::default(),
1363                    move |store, bucket, opts| async move {
1364                        started_tx.send(()).expect("announce held generation");
1365                        release_rx.await.expect("release commit");
1366                        let result = store
1367                            .heal_object_with_proof(&bucket, "history.txt", &Uuid::new_v4().to_string(), &opts)
1368                            .await?;
1369                        let proof = result.absence.expect("absence proof must reuse the held lifecycle owner");
1370                        assert_eq!(proof.bucket_incarnation_id, identity);
1371                        assert_eq!(proof.locations.len(), 2);
1372                        drained_tx.send(()).expect("announce drain");
1373                        Ok(())
1374                    },
1375                )
1376                .await
1377        });
1378        started_rx.await.expect("operation acquired generation fence");
1379        waiter.abort();
1380        assert!(waiter.await.expect_err("waiter aborted").is_cancelled());
1381        let writer_store = store.clone();
1382        let writer_bucket = bucket.clone();
1383        let mut writer = Box::pin(writer_store.acquire_bucket_lifecycle_write_lock(&writer_bucket));
1384        assert!(
1385            futures::poll!(&mut writer).is_pending(),
1386            "cancellation must not release the live operation's generation"
1387        );
1388        release_tx.send(()).expect("finish operation");
1389        tokio::time::timeout(std::time::Duration::from_secs(10), drained_rx)
1390            .await
1391            .expect("proof must not reacquire a lifecycle read lock behind the queued writer")
1392            .expect("physical operation drains");
1393        let guard = tokio::time::timeout(std::time::Duration::from_secs(10), writer)
1394            .await
1395            .expect("writer resumes after drain")
1396            .expect("lifecycle writer");
1397        drop(guard);
1398        shutdown.cancel();
1399    }
1400
1401    async fn minimal_heal_store() -> ECStore {
1402        ECStore {
1403            id: Uuid::new_v4(),
1404            disk_map: HashMap::new(),
1405            pools: vec![minimal_heal_pool(0).await, minimal_heal_pool(1).await],
1406            peer_sys: S3PeerSys {
1407                clients: Vec::new(),
1408                pools_count: 2,
1409            },
1410            pool_meta: RwLock::new(PoolMeta::default()),
1411            rebalance_meta: RwLock::new(None),
1412            decommission_cancelers: RwLock::new(Vec::new()),
1413            start_gate: Mutex::new(()),
1414            pool_meta_save_gate: Mutex::default(),
1415            ctx: crate::runtime::instance::bootstrap_ctx(),
1416            bucket_fence_registry: std::sync::Arc::default(),
1417        }
1418    }
1419
1420    async fn remove_pool_meta_shard(store: &ECStore, pool_idx: usize) -> DiskStore {
1421        let target_set = store.pools[pool_idx].get_disks_by_key(POOL_META_NAME);
1422        let missing_disk = target_set.disks.read().await[0]
1423            .clone()
1424            .expect("pool metadata fixture disk should be online");
1425        missing_disk
1426            .delete(
1427                RUSTFS_META_BUCKET,
1428                POOL_META_NAME,
1429                DeleteOptions {
1430                    recursive: true,
1431                    immediate: true,
1432                    ..Default::default()
1433                },
1434            )
1435            .await
1436            .expect("one pool metadata shard should be removable");
1437        assert!(
1438            missing_disk.read_xl(RUSTFS_META_BUCKET, POOL_META_NAME, false).await.is_err(),
1439            "pool metadata fixture must start with one missing shard"
1440        );
1441        missing_disk
1442    }
1443
1444    #[tokio::test]
1445    async fn heal_erasure_set_scopes_follow_requested_pool_and_set() {
1446        let store = minimal_heal_store().await;
1447
1448        assert_eq!(
1449            store
1450                .heal_erasure_set_scopes(&HealOpts::default())
1451                .await
1452                .expect("unscoped heal should enumerate every live set"),
1453            vec![(0, 0), (1, 0)]
1454        );
1455        assert_eq!(
1456            store
1457                .heal_erasure_set_scopes(&HealOpts {
1458                    pool: Some(1),
1459                    set: Some(0),
1460                    ..Default::default()
1461                })
1462                .await
1463                .expect("scoped heal should enumerate only its requested set"),
1464            vec![(1, 0)]
1465        );
1466
1467        let err = store
1468            .heal_erasure_set_scopes(&HealOpts {
1469                pool: Some(0),
1470                set: Some(1),
1471                ..Default::default()
1472            })
1473            .await
1474            .expect_err("an invalid set scope must fail closed");
1475        assert!(matches!(err, Error::InvalidArgument(..)));
1476    }
1477
1478    fn pool_meta_with_decommission(info: PoolDecommissionInfo) -> PoolMeta {
1479        PoolMeta {
1480            pools: vec![PoolStatus {
1481                id: 0,
1482                cmd_line: "pool-0".to_string(),
1483                last_update: OffsetDateTime::UNIX_EPOCH,
1484                decommission: Some(info),
1485            }],
1486            ..Default::default()
1487        }
1488    }
1489
1490    #[test]
1491    fn heal_format_pool_state_barriers_are_classified() {
1492        let active = pool_meta_with_decommission(PoolDecommissionInfo {
1493            start_time: Some(OffsetDateTime::UNIX_EPOCH),
1494            ..Default::default()
1495        });
1496        assert!(matches!(
1497            classify_heal_format_pool(0, "pool-0", &active, None),
1498            Some(HealFormatPoolSkip::Retryable)
1499        ));
1500
1501        for info in [
1502            PoolDecommissionInfo {
1503                failed: true,
1504                ..Default::default()
1505            },
1506            PoolDecommissionInfo {
1507                canceled: true,
1508                ..Default::default()
1509            },
1510        ] {
1511            assert!(matches!(
1512                classify_heal_format_pool(0, "pool-0", &pool_meta_with_decommission(info), None),
1513                Some(HealFormatPoolSkip::Retryable)
1514            ));
1515        }
1516
1517        let completed = pool_meta_with_decommission(PoolDecommissionInfo {
1518            complete: true,
1519            ..Default::default()
1520        });
1521        assert!(matches!(
1522            classify_heal_format_pool(0, "pool-0", &completed, None),
1523            Some(HealFormatPoolSkip::Completed)
1524        ));
1525    }
1526
1527    #[test]
1528    fn heal_format_pool_rebalance_barriers_and_identity_are_fail_closed() {
1529        let identity_meta = pool_meta_with_decommission(PoolDecommissionInfo::default());
1530        let rebalance = RebalanceMeta {
1531            pool_stats: vec![RebalanceStats {
1532                participating: true,
1533                info: RebalanceInfo {
1534                    status: RebalStatus::Started,
1535                    ..Default::default()
1536                },
1537                ..Default::default()
1538            }],
1539            ..Default::default()
1540        };
1541        assert!(matches!(
1542            classify_heal_format_pool(0, "pool-0", &identity_meta, Some(&rebalance)),
1543            Some(HealFormatPoolSkip::Retryable)
1544        ));
1545
1546        let stopping = RebalanceMeta {
1547            pool_stats: vec![RebalanceStats {
1548                info: RebalanceInfo {
1549                    stopping: true,
1550                    ..Default::default()
1551                },
1552                ..Default::default()
1553            }],
1554            ..Default::default()
1555        };
1556        assert!(matches!(
1557            classify_heal_format_pool(0, "pool-0", &identity_meta, Some(&stopping)),
1558            Some(HealFormatPoolSkip::Retryable)
1559        ));
1560
1561        let identity = pool_meta_with_decommission(PoolDecommissionInfo::default());
1562        assert!(matches!(
1563            classify_heal_format_pool(0, "pool-new", &identity, None),
1564            Some(HealFormatPoolSkip::Retryable)
1565        ));
1566
1567        let identity_without_decommission = PoolMeta {
1568            pools: vec![PoolStatus {
1569                id: 0,
1570                cmd_line: "pool-0".to_string(),
1571                last_update: OffsetDateTime::UNIX_EPOCH,
1572                decommission: None,
1573            }],
1574            ..Default::default()
1575        };
1576        assert!(matches!(
1577            classify_heal_format_pool(0, "pool-new", &identity_without_decommission, None),
1578            Some(HealFormatPoolSkip::Retryable)
1579        ));
1580
1581        assert!(matches!(
1582            classify_heal_format_pool(0, "", &identity_meta, None),
1583            Some(HealFormatPoolSkip::Retryable)
1584        ));
1585
1586        assert!(matches!(
1587            classify_heal_format_pool(0, "pool-0", &PoolMeta::default(), None),
1588            Some(HealFormatPoolSkip::Retryable)
1589        ));
1590
1591        let stopped = RebalanceMeta {
1592            stopped_at: Some(OffsetDateTime::UNIX_EPOCH),
1593            pool_stats: vec![RebalanceStats {
1594                participating: true,
1595                info: RebalanceInfo {
1596                    status: RebalStatus::Stopped,
1597                    ..Default::default()
1598                },
1599                ..Default::default()
1600            }],
1601            ..Default::default()
1602        };
1603        assert!(classify_heal_format_pool(0, "pool-0", &identity_meta, Some(&stopped)).is_none());
1604
1605        let stopping_after_stop = RebalanceMeta {
1606            stopped_at: Some(OffsetDateTime::UNIX_EPOCH),
1607            pool_stats: vec![RebalanceStats {
1608                participating: true,
1609                info: RebalanceInfo {
1610                    status: RebalStatus::Started,
1611                    stopping: true,
1612                    ..Default::default()
1613                },
1614                ..Default::default()
1615            }],
1616            ..Default::default()
1617        };
1618        assert!(matches!(
1619            classify_heal_format_pool(0, "pool-0", &identity_meta, Some(&stopping_after_stop)),
1620            Some(HealFormatPoolSkip::Retryable)
1621        ));
1622    }
1623
1624    #[test]
1625    fn skipped_heal_format_pool_is_never_reported_as_success() {
1626        assert!(matches!(
1627            heal_format_pool_skip_error(HealFormatPoolSkip::Retryable),
1628            StorageError::SlowDown
1629        ));
1630        assert!(matches!(
1631            heal_format_pool_skip_error(HealFormatPoolSkip::Completed),
1632            StorageError::NoHealRequired
1633        ));
1634    }
1635
1636    async fn multi_pool_heal_store() -> (tempfile::TempDir, Arc<ECStore>, CancellationToken) {
1637        let temp_dir = tempfile::tempdir().expect("multi-pool heal test directory should be created");
1638        let mut pool_endpoints = Vec::new();
1639        for pool_index in 0..2 {
1640            let mut endpoints = Vec::new();
1641            for disk_index in 0..4 {
1642                let disk_path = temp_dir.path().join(format!("pool{pool_index}-disk{disk_index}"));
1643                tokio::fs::create_dir_all(&disk_path)
1644                    .await
1645                    .expect("multi-pool heal test disk should be created");
1646                let mut endpoint = Endpoint::try_from(disk_path.to_str().expect("disk path should be utf8"))
1647                    .expect("test endpoint should parse");
1648                endpoint.set_pool_index(pool_index);
1649                endpoint.set_set_index(0);
1650                endpoint.set_disk_index(disk_index);
1651                endpoints.push(endpoint);
1652            }
1653            pool_endpoints.push(PoolEndpoints {
1654                legacy: false,
1655                set_count: 1,
1656                drives_per_set: 4,
1657                endpoints: Endpoints::from(endpoints),
1658                cmd_line: format!("heal-owner-pool-{pool_index}"),
1659                platform: "test".to_string(),
1660            });
1661        }
1662
1663        let endpoint_pools = EndpointServerPools::from(pool_endpoints);
1664        let instance_ctx = Arc::new(InstanceContext::new());
1665        init_local_disks_with_instance_ctx(&instance_ctx, endpoint_pools.clone())
1666            .await
1667            .expect("multi-pool local disks should initialize");
1668        let shutdown = CancellationToken::new();
1669        let store = ECStore::new_with_instance_ctx(
1670            "127.0.0.1:0".parse().expect("test address should parse"),
1671            endpoint_pools,
1672            shutdown.clone(),
1673            instance_ctx,
1674        )
1675        .await
1676        .expect("multi-pool test store should initialize");
1677        metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
1678        (temp_dir, store, shutdown)
1679    }
1680
1681    #[tokio::test]
1682    #[serial_test::serial]
1683    async fn absence_proof_requires_every_selected_pool() {
1684        let (_temp_dir, store, shutdown) = multi_pool_heal_store().await;
1685        let bucket = format!("absence-scope-{}", Uuid::new_v4().simple());
1686        store
1687            .make_bucket(&bucket, &MakeBucketOptions::default())
1688            .await
1689            .expect("create bucket in both pools");
1690        let version = Uuid::new_v4().to_string();
1691        let result = store
1692            .heal_object_with_proof(&bucket, "history.txt", &version, &HealOpts::default())
1693            .await
1694            .expect("exact absence lookup should complete across both pools");
1695        assert!(result.error.is_none());
1696        let proof = result.absence.expect("all selected pools proved the exact version absent");
1697        assert_eq!(proof.locations, vec![(0, 0), (1, 0)]);
1698        assert_eq!((proof.pool_index, proof.set_index), (None, None));
1699        assert_eq!(proof.bucket, bucket);
1700        assert_eq!(proof.version_id, version);
1701        assert!(!proof.removed, "already absent versions do not count as another cleanup");
1702
1703        store.pool_meta.write().await.pools[1].decommission = Some(PoolDecommissionInfo {
1704            start_time: Some(OffsetDateTime::now_utc()),
1705            ..Default::default()
1706        });
1707        let partial = store
1708            .heal_object_with_proof(&bucket, "history.txt", &version, &HealOpts::default())
1709            .await
1710            .expect("unscoped heal should retain its legacy suspended-pool behavior");
1711        assert!(partial.absence.is_none(), "an uninspected suspended pool prevents global absence proof");
1712        shutdown.cancel();
1713    }
1714
1715    fn heal_test_format_path(temp_dir: &tempfile::TempDir, pool_index: usize, disk_index: usize) -> std::path::PathBuf {
1716        temp_dir
1717            .path()
1718            .join(format!("pool{pool_index}-disk{disk_index}"))
1719            .join(crate::disk::RUSTFS_META_BUCKET)
1720            .join(FORMAT_CONFIG_FILE)
1721    }
1722
1723    async fn remove_heal_test_format(
1724        temp_dir: &tempfile::TempDir,
1725        store: &ECStore,
1726        pool_index: usize,
1727        disk_index: usize,
1728    ) -> String {
1729        let endpoint = store.pools[pool_index].endpoints.endpoints.as_ref()[disk_index].clone();
1730        let target = endpoint.to_string();
1731        let format_path = heal_test_format_path(temp_dir, pool_index, disk_index);
1732        tokio::fs::remove_file(&format_path)
1733            .await
1734            .expect("replacement target format should be removable");
1735        assert!(
1736            !tokio::fs::try_exists(&format_path)
1737                .await
1738                .expect("replacement target format path should be inspectable")
1739        );
1740        let replacement = new_disk(
1741            &endpoint,
1742            &DiskOption {
1743                cleanup: false,
1744                health_check: false,
1745            },
1746        )
1747        .await
1748        .expect("unformatted replacement disk should open");
1749        let set = &store.pools[pool_index].disk_set[0];
1750        let target_slot = set
1751            .set_endpoints
1752            .iter()
1753            .position(|candidate| candidate == &endpoint)
1754            .expect("replacement endpoint should belong to its set");
1755        set.disks.write().await[target_slot] = Some(replacement);
1756        target
1757    }
1758
1759    async fn assert_heal_test_format_missing(temp_dir: &tempfile::TempDir, pool_index: usize, disk_index: usize) {
1760        assert!(
1761            !tokio::fs::try_exists(heal_test_format_path(temp_dir, pool_index, disk_index))
1762                .await
1763                .expect("replacement target format path should be inspectable"),
1764            "format heal must not write the replacement target"
1765        );
1766    }
1767
1768    #[tokio::test]
1769    #[serial_test::serial]
1770    async fn replacement_format_heal_coexists_with_capacity_read_fence() {
1771        let (temp_dir, store, shutdown) = multi_pool_heal_store().await;
1772        let target = remove_heal_test_format(&temp_dir, &store, 0, 3).await;
1773        let capacity_guard = store
1774            .acquire_external_decommission_capacity_fence(&[0], DecommissionCapacityAdmission::Heal)
1775            .await
1776            .expect("ordinary heal capacity fence should be acquired");
1777
1778        let (_, err) =
1779            tokio::time::timeout(std::time::Duration::from_secs(2), store.heal_replacement_format(false, 0, 0, &[target]))
1780                .await
1781                .expect("replacement format heal must not wait on an existing capacity read fence")
1782                .expect("replacement format heal should complete");
1783
1784        assert!(err.is_none(), "replacement format heal should succeed: {err:?}");
1785        assert!(
1786            tokio::fs::try_exists(heal_test_format_path(&temp_dir, 0, 3))
1787                .await
1788                .expect("replacement target format path should be inspectable"),
1789            "replacement format heal should restore the target while the capacity read fence is held"
1790        );
1791        drop(capacity_guard);
1792        shutdown.cancel();
1793    }
1794
1795    #[tokio::test]
1796    #[serial_test::serial]
1797    async fn replacement_format_heal_remains_blocked_by_pool_metadata_writer() {
1798        let (temp_dir, store, shutdown) = multi_pool_heal_store().await;
1799        let target = remove_heal_test_format(&temp_dir, &store, 0, 3).await;
1800        let pool_lock = store.pools[0]
1801            .new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME)
1802            .await
1803            .expect("pool metadata lock should be created");
1804        let pool_writer = pool_lock
1805            .get_write_lock(get_lock_acquire_timeout())
1806            .await
1807            .expect("pool metadata writer should be acquired");
1808
1809        assert!(
1810            tokio::time::timeout(
1811                std::time::Duration::from_secs(1),
1812                store.heal_replacement_format(false, 0, 0, std::slice::from_ref(&target)),
1813            )
1814            .await
1815            .is_err(),
1816            "replacement format heal must wait for a pool metadata writer"
1817        );
1818        assert_heal_test_format_missing(&temp_dir, 0, 3).await;
1819
1820        drop(pool_writer);
1821        let (_, err) =
1822            tokio::time::timeout(std::time::Duration::from_secs(30), store.heal_replacement_format(false, 0, 0, &[target]))
1823                .await
1824                .expect("replacement format heal should resume after the writer releases")
1825                .expect("replacement format heal should complete after the writer releases");
1826        assert!(err.is_none(), "replacement format heal should succeed: {err:?}");
1827        assert!(
1828            tokio::fs::try_exists(heal_test_format_path(&temp_dir, 0, 3))
1829                .await
1830                .expect("replacement target format path should be inspectable"),
1831            "replacement format heal should restore the target after the writer releases"
1832        );
1833        shutdown.cancel();
1834    }
1835
1836    #[tokio::test]
1837    #[serial_test::serial]
1838    async fn full_format_heal_preblocked_pool_metadata_never_writes_format() {
1839        let (temp_dir, store, shutdown) = multi_pool_heal_store().await;
1840        remove_heal_test_format(&temp_dir, &store, 0, 3).await;
1841        store.pool_meta_save_gate.lock().await.observe_replicas(PoolMetaReplicaState {
1842            needs_repair: true,
1843            repair_write_safe: false,
1844        });
1845
1846        let err = store
1847            .handle_heal_format(false)
1848            .await
1849            .expect_err("a preblocked pool metadata state must reject full format heal");
1850        assert!(
1851            err.to_string()
1852                .contains("restart after all replicas are readable and consistent")
1853        );
1854        assert_heal_test_format_missing(&temp_dir, 0, 3).await;
1855        store
1856            .ensure_pool_meta_side_effects_safe("preblocked format heal side effect")
1857            .await
1858            .expect_err("the preblocked state must remain sticky");
1859        shutdown.cancel();
1860    }
1861
1862    #[tokio::test]
1863    #[serial_test::serial]
1864    async fn full_format_heal_future_identity_never_writes_format_and_latches() {
1865        let (temp_dir, store, shutdown) = multi_pool_heal_store().await;
1866        remove_heal_test_format(&temp_dir, &store, 0, 3).await;
1867        let (mut future_identity, _) =
1868            read_config_no_lock_preserve_empty_with_metadata(store.pools[1].clone(), POOL_META_IDENTITY_NAME)
1869                .await
1870                .expect("current identity should be readable");
1871        let future_version = u16::from_le_bytes([future_identity[2], future_identity[3]])
1872            .checked_add(1)
1873            .expect("identity version should have a future value");
1874        future_identity[2..4].copy_from_slice(&future_version.to_le_bytes());
1875        save_config(store.pools[1].clone(), POOL_META_IDENTITY_NAME, future_identity)
1876            .await
1877            .expect("future identity should be persisted");
1878
1879        let err = store
1880            .handle_heal_format(false)
1881            .await
1882            .expect_err("a future identity must reject full format heal");
1883        assert!(err.to_string().contains("pool metadata incompatible"));
1884        assert_heal_test_format_missing(&temp_dir, 0, 3).await;
1885        store
1886            .ensure_pool_meta_side_effects_safe("future identity format heal side effect")
1887            .await
1888            .expect_err("the future identity rejection must latch the write gate");
1889        shutdown.cancel();
1890    }
1891
1892    #[tokio::test]
1893    #[serial_test::serial]
1894    async fn replacement_format_heal_epoch_conflict_never_writes_format_and_latches() {
1895        let (temp_dir, store, shutdown) = multi_pool_heal_store().await;
1896        let target = remove_heal_test_format(&temp_dir, &store, 0, 3).await;
1897        let conflicting_identity =
1898            initialized_pool_meta_identity_for_test(store.id, 2).expect("conflicting identity should encode");
1899        save_config(store.pools[1].clone(), POOL_META_IDENTITY_NAME, conflicting_identity)
1900            .await
1901            .expect("conflicting identity should be persisted");
1902
1903        let err = store
1904            .heal_replacement_format(false, 0, 0, &[target])
1905            .await
1906            .expect_err("an identity epoch conflict must reject replacement format heal");
1907        assert!(
1908            err.to_string()
1909                .contains("identity replicas disagree on cluster identity or epoch")
1910        );
1911        assert_heal_test_format_missing(&temp_dir, 0, 3).await;
1912        store
1913            .ensure_pool_meta_side_effects_safe("epoch conflict replacement format side effect")
1914            .await
1915            .expect_err("the identity epoch conflict must latch the write gate");
1916        shutdown.cancel();
1917    }
1918
1919    #[tokio::test]
1920    #[serial_test::serial]
1921    async fn replacement_format_heal_initialized_identity_without_pool_meta_never_writes_and_latches() {
1922        let (temp_dir, store, shutdown) = multi_pool_heal_store().await;
1923        let target = remove_heal_test_format(&temp_dir, &store, 0, 3).await;
1924        for pool in &store.pools {
1925            delete_config(pool.clone(), POOL_META_NAME)
1926                .await
1927                .expect("pool metadata replica should be removable");
1928        }
1929
1930        let err = store
1931            .heal_replacement_format(false, 0, 0, &[target])
1932            .await
1933            .expect_err("initialized identity without pool metadata must reject replacement format heal");
1934        assert!(err.to_string().contains("initialized cluster identity exists"));
1935        assert_heal_test_format_missing(&temp_dir, 0, 3).await;
1936        store
1937            .ensure_pool_meta_side_effects_safe("missing pool metadata replacement format side effect")
1938            .await
1939            .expect_err("missing initialized pool metadata must latch the write gate");
1940        shutdown.cancel();
1941    }
1942
1943    #[tokio::test]
1944    #[serial_test::serial]
1945    async fn full_format_heal_lost_after_last_save_does_not_publish_or_renew_and_latches() {
1946        let (temp_dir, store, shutdown) = multi_pool_heal_store().await;
1947        let target = remove_heal_test_format(&temp_dir, &store, 0, 3).await;
1948        store.pools[0].disk_set[0].disks.write().await[3] = None;
1949        let barrier = HealFormatAfterSaveBarrier::install(&store.pools[0], 3);
1950        let recovery_latch = store.pool_meta_save_gate.lock().await.aborted_transaction_latch_for_test();
1951        let mut heal = tokio::spawn({
1952            let store = Arc::clone(&store);
1953            async move { store.handle_heal_format(false).await }
1954        });
1955
1956        barrier.wait_until_paused().await;
1957        let saved = tokio::fs::read(heal_test_format_path(&temp_dir, 0, 3))
1958            .await
1959            .expect("the last replacement format must be durable before fence loss");
1960        let saved = FormatV3::try_from(saved.as_slice()).expect("the durable replacement format should decode");
1961        assert_eq!(saved.erasure.this, store.pools[0].format.erasure.sets[0][3]);
1962        recovery_latch.store(true, std::sync::atomic::Ordering::SeqCst);
1963        barrier.release();
1964
1965        let (result, err) = tokio::time::timeout(std::time::Duration::from_secs(30), &mut heal)
1966            .await
1967            .expect("format heal should stop after the lost fence")
1968            .expect("format heal task should not panic")
1969            .expect("format heal should return its fenced result");
1970        assert!(matches!(err, Some(StorageError::SlowDown)));
1971        assert!(
1972            result
1973                .after
1974                .drives
1975                .iter()
1976                .any(|drive| drive.endpoint == target && drive.state == DriveState::Missing.to_string()),
1977            "the lost fence must prevent the durable format from being published in the heal result"
1978        );
1979        assert!(
1980            store.pools[0].disk_set[0].disks.read().await[3].is_none(),
1981            "the lost fence must prevent renew_disk from attaching the replacement"
1982        );
1983        recovery_latch.store(false, std::sync::atomic::Ordering::SeqCst);
1984        store
1985            .ensure_pool_meta_side_effects_safe("post-save format fence loss side effect")
1986            .await
1987            .expect_err("post-save format fence loss must remain sticky");
1988        shutdown.cancel();
1989    }
1990
1991    #[tokio::test]
1992    async fn heal_object_pool_scope_selects_only_requested_pool() {
1993        let store = minimal_heal_store().await;
1994        let pools = store
1995            .get_pools_for_heal_object(&HealOpts {
1996                pool: Some(1),
1997                ..Default::default()
1998            })
1999            .expect("requested pool should be selected");
2000
2001        assert_eq!(pools.len(), 1);
2002        assert!(Arc::ptr_eq(&pools[0], &store.pools[1]));
2003    }
2004
2005    #[tokio::test]
2006    async fn heal_object_pool_scope_rejects_invalid_pool() {
2007        let store = minimal_heal_store().await;
2008        let err = store
2009            .get_pools_for_heal_object(&HealOpts {
2010                pool: Some(2),
2011                ..Default::default()
2012            })
2013            .expect_err("out-of-range pool scope must fail closed");
2014
2015        assert!(
2016            matches!(err, StorageError::InvalidArgument(_, ref field, ref reason)
2017                if field == "pool" && reason.contains("invalid heal pool index 2 for 2 pools")),
2018            "unexpected invalid pool error: {err:?}"
2019        );
2020    }
2021
2022    #[tokio::test]
2023    #[serial_test::serial]
2024    async fn pool_meta_read_repair_reuses_its_write_fence() {
2025        let (_temp_dirs, store, _other_store) = test_two_pool_stores(None).await;
2026        let missing_disk = remove_pool_meta_shard(&store, 0).await;
2027
2028        let (result, err) = tokio::time::timeout(
2029            std::time::Duration::from_secs(30),
2030            store.handle_heal_object(
2031                RUSTFS_META_BUCKET,
2032                POOL_META_NAME,
2033                "",
2034                &HealOpts {
2035                    read_repair: true,
2036                    pool: Some(0),
2037                    ..Default::default()
2038                },
2039            ),
2040        )
2041        .await
2042        .expect("pool metadata read repair must not wait on its own capacity fence")
2043        .expect("pool metadata read repair should complete");
2044
2045        assert!(err.is_none(), "pool metadata read repair should succeed: {err:?}");
2046        assert_eq!(result.object, POOL_META_NAME);
2047        assert!(
2048            missing_disk.read_xl(RUSTFS_META_BUCKET, POOL_META_NAME, false).await.is_ok(),
2049            "pool metadata read repair should restore the missing shard"
2050        );
2051    }
2052
2053    #[tokio::test]
2054    #[serial_test::serial]
2055    async fn pool_meta_heal_preserves_typed_lock_timeout() {
2056        let (_temp_dirs, store, _other_store) = test_two_pool_stores(None).await;
2057        let lock = store.pools[0]
2058            .new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME)
2059            .await
2060            .expect("pool metadata lock should be created");
2061        let guard = lock
2062            .get_write_lock(get_lock_acquire_timeout())
2063            .await
2064            .expect("pool metadata lock should be acquired");
2065
2066        let (_, err) = temp_env::async_with_vars(
2067            [(rustfs_config::ENV_OBJECT_LOCK_ACQUIRE_TIMEOUT, Some("1"))],
2068            store.handle_heal_object(
2069                RUSTFS_META_BUCKET,
2070                POOL_META_NAME,
2071                "",
2072                &HealOpts {
2073                    pool: Some(0),
2074                    ..Default::default()
2075                },
2076            ),
2077        )
2078        .await
2079        .expect("pool metadata lock timeout should be mapped into the heal result");
2080
2081        assert!(
2082            matches!(err, Some(Error::Lock(rustfs_lock::LockError::Timeout { .. }))),
2083            "pool metadata heal must preserve the recoverable lock timeout: {err:?}"
2084        );
2085        drop(guard);
2086    }
2087
2088    #[tokio::test]
2089    #[serial_test::serial]
2090    async fn pool_meta_neighbor_keeps_ordinary_object_locking() {
2091        let (_temp_dirs, store, _other_store) = test_two_pool_stores(None).await;
2092        let object = "pool.bin.backup";
2093        save_config(store.pools[0].clone(), object, b"neighbor metadata".to_vec())
2094            .await
2095            .expect("neighbor metadata fixture should be written");
2096
2097        let target_set = store.pools[0].get_disks_by_key(object);
2098        let lock = target_set
2099            .new_ns_lock(RUSTFS_META_BUCKET, object)
2100            .await
2101            .expect("neighbor metadata lock should be created");
2102        let guard = lock
2103            .get_write_lock(get_lock_acquire_timeout())
2104            .await
2105            .expect("neighbor metadata lock should be acquired");
2106        let barrier = DecommissionCapacityLockOrderBarrier::install(store.id, store.id);
2107        let heal_store = Arc::clone(&store);
2108        let mut heal = tokio::spawn(async move {
2109            heal_store
2110                .handle_heal_object(
2111                    RUSTFS_META_BUCKET,
2112                    object,
2113                    "",
2114                    &HealOpts {
2115                        pool: Some(0),
2116                        ..Default::default()
2117                    },
2118                )
2119                .await
2120        });
2121
2122        barrier.wait_until_external_heal_target_lock_attempted().await;
2123        tokio::task::yield_now().await;
2124        assert!(!heal.is_finished(), "neighbor metadata heal must retain ordinary object locking");
2125        drop(guard);
2126
2127        let (result, err) = tokio::time::timeout(std::time::Duration::from_secs(30), &mut heal)
2128            .await
2129            .expect("neighbor metadata heal should finish after the object lock is released")
2130            .expect("neighbor metadata heal task should not panic")
2131            .expect("neighbor metadata heal should complete");
2132        assert!(err.is_none(), "neighbor metadata heal should succeed: {err:?}");
2133        assert_eq!(result.object, object);
2134        drop(barrier);
2135    }
2136
2137    #[tokio::test]
2138    #[serial_test::serial]
2139    async fn targeted_heal_is_blocked_by_exact_fit_decommission_reservation() {
2140        let (_temp_dirs, store, _other_store) = test_two_pool_stores(None).await;
2141        let bucket = format!("heal-capacity-{}", Uuid::new_v4().simple());
2142        let object = "targeted-heal.bin";
2143        store
2144            .make_bucket(&bucket, &MakeBucketOptions::default())
2145            .await
2146            .expect("heal capacity bucket should be created");
2147
2148        let target_set = store.pools[1].get_disks(0);
2149        let mut reader = PutObjReader::from_vec(b"targeted heal body".to_vec());
2150        target_set
2151            .put_object(&bucket, object, &mut reader, &ObjectOptions::default())
2152            .await
2153            .expect("targeted heal fixture should be written");
2154        let missing_disk = target_set.disks.read().await[0]
2155            .clone()
2156            .expect("targeted heal fixture disk should be online");
2157        missing_disk
2158            .delete(
2159                &bucket,
2160                object,
2161                DeleteOptions {
2162                    recursive: true,
2163                    immediate: true,
2164                    ..Default::default()
2165                },
2166            )
2167            .await
2168            .expect("targeted heal fixture shard should be removed");
2169        assert!(
2170            missing_disk.read_xl(&bucket, object, false).await.is_err(),
2171            "targeted heal fixture must start with one missing metadata copy"
2172        );
2173
2174        let layout = DecommissionErasureLayout { data: 1, parity: 0 };
2175        set_decommission_capacity_info_overrides_for_test(
2176            store.id,
2177            vec![vec![
2178                DecommissionPoolCapacityInfo::for_test(0, layout, 0, 30, 30),
2179                DecommissionPoolCapacityInfo::for_test(1, layout, 60, 60, 0),
2180            ]],
2181        );
2182        store
2183            .save_current_pool_meta_for_decommission_start(&[0], Vec::new())
2184            .await
2185            .expect("the exact-fit decommission reservation should activate");
2186        {
2187            let pool_meta = store.pool_meta.read().await;
2188            let reservation = pool_meta.pools[0]
2189                .decommission
2190                .as_ref()
2191                .and_then(|info| info.capacity_reservation.as_ref())
2192                .expect("the active decommission reservation should be durable");
2193            assert_eq!(reservation.targets.len(), 1);
2194            assert_eq!(reservation.targets[0].pool_index, 1);
2195            assert_eq!(reservation.targets[0].reserved_physical_bytes, 60);
2196        }
2197
2198        let (_, err) = store
2199            .handle_heal_object(
2200                &bucket,
2201                object,
2202                "",
2203                &HealOpts {
2204                    pool: Some(1),
2205                    set: Some(0),
2206                    ..Default::default()
2207                },
2208            )
2209            .await
2210            .expect("targeted heal should return a mapped capacity result");
2211        assert!(matches!(err, Some(Error::SlowDown)), "targeted heal must be capacity-blocked: {err:?}");
2212        assert!(
2213            missing_disk.read_xl(&bucket, object, false).await.is_err(),
2214            "capacity-blocked targeted heal must not rewrite the missing shard"
2215        );
2216
2217        let (_, err) = tokio::time::timeout(
2218            std::time::Duration::from_secs(30),
2219            store.handle_heal_object(
2220                RUSTFS_META_BUCKET,
2221                POOL_META_NAME,
2222                "",
2223                &HealOpts {
2224                    pool: Some(1),
2225                    ..Default::default()
2226                },
2227            ),
2228        )
2229        .await
2230        .expect("pool metadata admission should not recurse on its namespace lock")
2231        .expect("pool metadata capacity rejection should be mapped");
2232        assert!(
2233            matches!(err, Some(Error::SlowDown)),
2234            "pool metadata heal must preserve target reservation admission: {err:?}"
2235        );
2236    }
2237
2238    #[tokio::test]
2239    #[serial_test::serial]
2240    async fn pool_meta_heal_keeps_per_target_capacity_admission() {
2241        let (_temp_dirs, store, _other_store) = test_three_pool_stores_with_isolated_node_contexts(None).await;
2242        let layout = DecommissionErasureLayout { data: 1, parity: 0 };
2243        set_decommission_capacity_info_overrides_for_test(
2244            store.id,
2245            vec![vec![
2246                DecommissionPoolCapacityInfo::for_test(0, layout, 0, 30, 30),
2247                DecommissionPoolCapacityInfo::for_test(1, layout, 60, 60, 0),
2248                DecommissionPoolCapacityInfo::for_test(2, layout, 60, 60, 0),
2249            ]],
2250        );
2251        store
2252            .save_current_pool_meta_for_decommission_start(&[0], Vec::new())
2253            .await
2254            .expect("pool metadata reservation should activate");
2255        {
2256            let pool_meta = store.pool_meta.read().await;
2257            let reservation = pool_meta.pools[0]
2258                .decommission
2259                .as_ref()
2260                .and_then(|info| info.capacity_reservation.as_ref())
2261                .expect("pool metadata reservation should be durable");
2262            assert_eq!(reservation.targets.len(), 1);
2263            assert_eq!(reservation.targets[0].pool_index, 1);
2264        }
2265
2266        let missing_disk = remove_pool_meta_shard(&store, 2).await;
2267
2268        let (result, err) = tokio::time::timeout(
2269            std::time::Duration::from_secs(30),
2270            store.handle_heal_object(RUSTFS_META_BUCKET, POOL_META_NAME, "", &HealOpts::default()),
2271        )
2272        .await
2273        .expect("unscoped pool metadata heal should complete")
2274        .expect("unscoped pool metadata heal should return a mapped result");
2275
2276        assert!(err.is_none(), "an admitted target should let unscoped metadata heal succeed: {err:?}");
2277        assert_eq!(result.object, POOL_META_NAME);
2278        assert!(
2279            missing_disk.read_xl(RUSTFS_META_BUCKET, POOL_META_NAME, false).await.is_ok(),
2280            "unscoped metadata heal should repair the admitted target while another target is reserved"
2281        );
2282    }
2283
2284    #[tokio::test]
2285    #[serial_test::serial]
2286    async fn targeted_heal_keeps_target_lock_for_different_lock_domain() {
2287        let (_temp_dirs, store, other_store) = test_three_pool_stores_with_isolated_node_contexts(None).await;
2288        let bucket = format!("heal-lock-domain-{}", Uuid::new_v4().simple());
2289        let object = "different-domain-heal.bin";
2290        store
2291            .make_bucket(&bucket, &MakeBucketOptions::default())
2292            .await
2293            .expect("heal lock-domain bucket should be created");
2294
2295        let target_set = other_store.pools[1].get_disks(0);
2296        let mut reader = PutObjReader::from_vec(b"different lock domain body".to_vec());
2297        target_set
2298            .put_object(&bucket, object, &mut reader, &ObjectOptions::default())
2299            .await
2300            .expect("heal lock-domain fixture should be written");
2301        let missing_disk = target_set.disks.read().await[0]
2302            .clone()
2303            .expect("heal lock-domain fixture disk should be online");
2304        missing_disk
2305            .delete(
2306                &bucket,
2307                object,
2308                DeleteOptions {
2309                    recursive: true,
2310                    immediate: true,
2311                    ..Default::default()
2312                },
2313            )
2314            .await
2315            .expect("heal lock-domain fixture shard should be removed");
2316        assert!(
2317            missing_disk.read_xl(&bucket, object, false).await.is_err(),
2318            "heal lock-domain fixture must start with one missing metadata copy"
2319        );
2320
2321        let layout = DecommissionErasureLayout { data: 1, parity: 0 };
2322        set_decommission_capacity_info_overrides_for_test(
2323            store.id,
2324            vec![vec![
2325                DecommissionPoolCapacityInfo::for_test(0, layout, 0, 30, 30),
2326                DecommissionPoolCapacityInfo::for_test(1, layout, 0, 100, 100),
2327                DecommissionPoolCapacityInfo::for_test(2, layout, 60, 60, 0),
2328            ]],
2329        );
2330        store
2331            .save_current_pool_meta_for_decommission_start(&[0], Vec::new())
2332            .await
2333            .expect("heal lock-domain reservation should activate");
2334        {
2335            let pool_meta = store.pool_meta.read().await;
2336            let reservation = pool_meta.pools[0]
2337                .decommission
2338                .as_ref()
2339                .and_then(|info| info.capacity_reservation.as_ref())
2340                .expect("heal lock-domain reservation should be durable");
2341            assert_eq!(reservation.targets.len(), 1);
2342            assert_eq!(reservation.targets[0].pool_index, 2);
2343        }
2344        let pool_meta = store.pool_meta.read().await.clone();
2345        *other_store.pool_meta.write().await = pool_meta;
2346
2347        let fixed_set = other_store.pools[0].disk_set[0].clone();
2348        assert!(
2349            !fixed_set.shares_namespace_lock_domain(&target_set).await,
2350            "heal fixture must use different fixed and target lock domains"
2351        );
2352        let barrier = DecommissionCapacityLockOrderBarrier::install(store.id, other_store.id);
2353        let heal_store = Arc::clone(&other_store);
2354        let heal_bucket = bucket.clone();
2355        let heal_object = object.to_string();
2356        let mut heal = tokio::spawn(async move {
2357            heal_store
2358                .handle_heal_object(
2359                    &heal_bucket,
2360                    &heal_object,
2361                    "",
2362                    &HealOpts {
2363                        pool: Some(1),
2364                        set: Some(0),
2365                        ..Default::default()
2366                    },
2367                )
2368                .await
2369        });
2370
2371        barrier.wait_until_external_capacity_released().await;
2372        barrier.wait_until_external_heal_target_lock_attempted().await;
2373
2374        let (_, err) = tokio::time::timeout(std::time::Duration::from_secs(30), &mut heal)
2375            .await
2376            .expect("targeted heal should finish after target lock release")
2377            .expect("targeted heal task should not panic")
2378            .expect("targeted heal should complete");
2379        assert!(err.is_none(), "targeted heal should repair after target lock release: {err:?}");
2380        assert!(
2381            missing_disk.read_xl(&bucket, object, false).await.is_ok(),
2382            "targeted heal should rewrite the missing shard after the target lock is released"
2383        );
2384        drop(barrier);
2385    }
2386
2387    #[tokio::test]
2388    #[serial_test::serial]
2389    async fn targeted_heal_reuses_shared_target_lock_without_reentrant_lock() {
2390        let (_temp_dirs, store, _other_store) = test_three_pool_stores_with_isolated_node_contexts(None).await;
2391        let bucket = format!("heal-shared-lock-domain-{}", Uuid::new_v4().simple());
2392        let object = "shared-domain-heal.bin";
2393        store
2394            .make_bucket(&bucket, &MakeBucketOptions::default())
2395            .await
2396            .expect("shared heal lock-domain bucket should be created");
2397
2398        let target_set = store.pools[0].get_disks(0);
2399        let mut reader = PutObjReader::from_vec(b"shared lock domain body".to_vec());
2400        target_set
2401            .put_object(&bucket, object, &mut reader, &ObjectOptions::default())
2402            .await
2403            .expect("shared heal lock-domain fixture should be written");
2404        let missing_disk = target_set.disks.read().await[0]
2405            .clone()
2406            .expect("shared heal lock-domain fixture disk should be online");
2407        missing_disk
2408            .delete(
2409                &bucket,
2410                object,
2411                DeleteOptions {
2412                    recursive: true,
2413                    immediate: true,
2414                    ..Default::default()
2415                },
2416            )
2417            .await
2418            .expect("shared heal lock-domain fixture shard should be removed");
2419        assert!(
2420            missing_disk.read_xl(&bucket, object, false).await.is_err(),
2421            "shared heal lock-domain fixture must start with one missing metadata copy"
2422        );
2423
2424        let layout = DecommissionErasureLayout { data: 1, parity: 0 };
2425        set_decommission_capacity_info_overrides_for_test(
2426            store.id,
2427            vec![vec![
2428                DecommissionPoolCapacityInfo::for_test(0, layout, 0, 100, 100),
2429                DecommissionPoolCapacityInfo::for_test(1, layout, 60, 60, 0),
2430                DecommissionPoolCapacityInfo::for_test(2, layout, 0, 30, 30),
2431            ]],
2432        );
2433        store
2434            .save_current_pool_meta_for_decommission_start(&[2], Vec::new())
2435            .await
2436            .expect("shared heal lock-domain reservation should activate");
2437        {
2438            let pool_meta = store.pool_meta.read().await;
2439            let reservation = pool_meta.pools[2]
2440                .decommission
2441                .as_ref()
2442                .and_then(|info| info.capacity_reservation.as_ref())
2443                .expect("shared heal lock-domain reservation should be durable");
2444            assert_eq!(reservation.targets.len(), 1);
2445            assert_eq!(reservation.targets[0].pool_index, 1);
2446            assert_eq!(reservation.targets[0].reserved_physical_bytes, 60);
2447        }
2448
2449        let fixed_set = store.pools[0].disk_set[0].clone();
2450        assert!(
2451            fixed_set.shares_namespace_lock_domain(&target_set).await,
2452            "shared heal fixture must use one fixed and target lock domain"
2453        );
2454
2455        let barrier = DecommissionCapacityLockOrderBarrier::install(store.id, store.id);
2456        let heal_store = Arc::clone(&store);
2457        let heal_bucket = bucket.clone();
2458        let heal_object = object.to_string();
2459        let mut heal = tokio::spawn(async move {
2460            heal_store
2461                .handle_heal_object(
2462                    &heal_bucket,
2463                    &heal_object,
2464                    "",
2465                    &HealOpts {
2466                        pool: Some(0),
2467                        set: Some(0),
2468                        ..Default::default()
2469                    },
2470                )
2471                .await
2472        });
2473
2474        barrier.wait_until_external_capacity_released().await;
2475        barrier.wait_until_external_heal_operation_started().await;
2476        let (_, err) = tokio::time::timeout(std::time::Duration::from_secs(5), &mut heal)
2477            .await
2478            .expect("shared-domain heal must not reenter the fixed namespace lock")
2479            .expect("shared-domain heal task should not panic")
2480            .expect("shared-domain heal should complete");
2481        assert!(err.is_none(), "shared-domain heal should repair after admission: {err:?}");
2482        assert!(
2483            missing_disk.read_xl(&bucket, object, false).await.is_ok(),
2484            "shared-domain heal should rewrite the missing shard"
2485        );
2486        drop(barrier);
2487    }
2488
2489    #[tokio::test]
2490    async fn scoped_heal_object_defers_when_requested_pool_is_suspended() {
2491        let mut store = minimal_heal_store().await;
2492        store.pool_meta = RwLock::new(PoolMeta {
2493            pools: vec![
2494                PoolStatus {
2495                    id: 0,
2496                    cmd_line: "pool-0".to_string(),
2497                    last_update: OffsetDateTime::UNIX_EPOCH,
2498                    decommission: None,
2499                },
2500                PoolStatus {
2501                    id: 1,
2502                    cmd_line: "pool-1".to_string(),
2503                    last_update: OffsetDateTime::UNIX_EPOCH,
2504                    decommission: Some(PoolDecommissionInfo {
2505                        start_time: Some(OffsetDateTime::UNIX_EPOCH),
2506                        ..Default::default()
2507                    }),
2508                },
2509            ],
2510            ..Default::default()
2511        });
2512
2513        let (_, err) = store
2514            .handle_heal_object(
2515                "bucket",
2516                "object",
2517                "",
2518                &HealOpts {
2519                    pool: Some(1),
2520                    set: Some(0),
2521                    ..Default::default()
2522                },
2523            )
2524            .await
2525            .expect("suspended pool should return a deferred heal result");
2526
2527        assert!(matches!(err, Some(StorageError::SlowDown)));
2528
2529        let (_, err) = store
2530            .handle_heal_object(
2531                "bucket",
2532                "object",
2533                "",
2534                &HealOpts {
2535                    set: Some(1),
2536                    ..Default::default()
2537                },
2538            )
2539            .await
2540            .expect("unscoped heal should return the active pool result");
2541
2542        assert!(matches!(err, Some(StorageError::InvalidArgument(_, ref field, _)) if field == "set"));
2543
2544        let err = store
2545            .handle_heal_object(
2546                "bucket",
2547                "object",
2548                "",
2549                &HealOpts {
2550                    pool: Some(1),
2551                    set: Some(1),
2552                    ..Default::default()
2553                },
2554            )
2555            .await
2556            .expect_err("invalid set scope should fail before suspended pool deferral");
2557
2558        assert!(matches!(err, StorageError::InvalidArgument(_, ref field, _) if field == "set"));
2559
2560        {
2561            let mut pool_meta = store.pool_meta.write().await;
2562            let decommission = pool_meta.pools[1]
2563                .decommission
2564                .as_mut()
2565                .expect("test pool should have decommission state");
2566            decommission.complete = true;
2567        }
2568        let (_, err) = store
2569            .handle_heal_object(
2570                "bucket",
2571                "object",
2572                "",
2573                &HealOpts {
2574                    pool: Some(1),
2575                    set: Some(0),
2576                    ..Default::default()
2577                },
2578            )
2579            .await
2580            .expect("completed pool should return a terminal heal result");
2581
2582        assert!(matches!(
2583            err,
2584            Some(StorageError::InvalidArgument(_, ref field, ref reason))
2585                if field == "pool" && reason.contains("completed decommission")
2586        ));
2587
2588        for canceled in [false, true] {
2589            {
2590                let mut pool_meta = store.pool_meta.write().await;
2591                let decommission = pool_meta.pools[1]
2592                    .decommission
2593                    .as_mut()
2594                    .expect("test pool should have decommission state");
2595                decommission.complete = false;
2596                decommission.failed = !canceled;
2597                decommission.canceled = canceled;
2598            }
2599            let (_, err) = store
2600                .handle_heal_object(
2601                    "bucket",
2602                    "object",
2603                    "",
2604                    &HealOpts {
2605                        pool: Some(1),
2606                        set: Some(0),
2607                        ..Default::default()
2608                    },
2609                )
2610                .await
2611                .expect("clearable terminal pool should return a deferred heal result");
2612
2613            assert!(matches!(err, Some(StorageError::SlowDown)));
2614        }
2615    }
2616
2617    #[tokio::test]
2618    async fn scoped_heal_bucket_blocks_before_dispatch_when_pool_is_suspended() {
2619        let mut store = minimal_heal_store().await;
2620        store.pool_meta = RwLock::new(PoolMeta {
2621            pools: vec![
2622                PoolStatus {
2623                    id: 0,
2624                    cmd_line: "pool-0".to_string(),
2625                    last_update: OffsetDateTime::UNIX_EPOCH,
2626                    decommission: None,
2627                },
2628                PoolStatus {
2629                    id: 1,
2630                    cmd_line: "pool-1".to_string(),
2631                    last_update: OffsetDateTime::UNIX_EPOCH,
2632                    decommission: Some(PoolDecommissionInfo {
2633                        start_time: Some(OffsetDateTime::UNIX_EPOCH),
2634                        ..Default::default()
2635                    }),
2636                },
2637            ],
2638            ..Default::default()
2639        });
2640
2641        let err = store
2642            .handle_heal_bucket(
2643                "bucket",
2644                &HealOpts {
2645                    pool: Some(1),
2646                    ..Default::default()
2647                },
2648            )
2649            .await
2650            .expect_err("suspended pool must be blocked before bucket-heal fan-out");
2651        assert_eq!(err, Error::SlowDown);
2652
2653        store.pool_meta.write().await.pools[1]
2654            .decommission
2655            .as_mut()
2656            .expect("decommission state should exist")
2657            .complete = true;
2658        let err = store
2659            .handle_heal_bucket(
2660                "bucket",
2661                &HealOpts {
2662                    pool: Some(1),
2663                    ..Default::default()
2664                },
2665            )
2666            .await
2667            .expect_err("completed pool must remain fenced from bucket heal");
2668        assert!(
2669            matches!(err, StorageError::InvalidArgument(_, ref field, ref reason)
2670                if field == "pool" && reason.contains("completed decommission")),
2671            "unexpected completed-pool error: {err:?}"
2672        );
2673    }
2674
2675    #[tokio::test]
2676    async fn bucket_heal_blocks_before_dispatch_after_unreadable_pool_meta_replica() {
2677        let store = minimal_heal_store().await;
2678        store.pool_meta_save_gate.lock().await.observe_replicas(PoolMetaReplicaState {
2679            needs_repair: true,
2680            repair_write_safe: false,
2681        });
2682
2683        let err = store
2684            .handle_heal_bucket("bucket", &HealOpts::default())
2685            .await
2686            .expect_err("bucket heal must stay blocked until restart after an unreadable replica");
2687
2688        assert!(
2689            err.to_string()
2690                .contains("restart after all replicas are readable and consistent")
2691        );
2692    }
2693
2694    #[tokio::test]
2695    async fn bucket_heal_releases_save_gate_before_peer_dispatch_and_holds_movement_snapshot() {
2696        let mut store = minimal_heal_store().await;
2697        let started = Arc::new(tokio::sync::Notify::new());
2698        let release = Arc::new(tokio::sync::Notify::new());
2699        let peer: Box<dyn PeerS3Client> = Box::new(BlockingHealPeer {
2700            started: started.clone(),
2701            release: release.clone(),
2702        });
2703        store.peer_sys.clients = vec![Arc::new(peer)];
2704        let store = Arc::new(store);
2705        let movement_gate = store.ctx.data_movement_operation_gate();
2706        let mut heal = tokio::spawn({
2707            let store = store.clone();
2708            async move { store.handle_heal_bucket("bucket", &HealOpts::default()).await }
2709        });
2710
2711        tokio::time::timeout(std::time::Duration::from_secs(1), started.notified())
2712            .await
2713            .expect("peer dispatch should start");
2714        assert!(
2715            store.pool_meta_save_gate.try_lock().is_ok(),
2716            "coordinator must release its local save gate before waiting for peers"
2717        );
2718        assert!(
2719            movement_gate.try_write().is_err(),
2720            "bucket heal must hold the movement snapshot through peer dispatch"
2721        );
2722
2723        release.notify_one();
2724        tokio::time::timeout(std::time::Duration::from_secs(1), &mut heal)
2725            .await
2726            .expect("bucket heal should finish after peer release")
2727            .expect("bucket heal task should not panic")
2728            .expect("bucket heal should succeed");
2729    }
2730
2731    #[tokio::test]
2732    async fn bucket_heal_local_fanout_does_not_reenter_movement_read_behind_queued_writer() {
2733        let mut store = minimal_heal_store().await;
2734        let movement_gate = store.ctx.data_movement_operation_gate();
2735        let writer_queued = Arc::new(tokio::sync::Notify::new());
2736        let writer_acquired = Arc::new(tokio::sync::Notify::new());
2737        let peer: Box<dyn PeerS3Client> = Box::new(WriterQueuedLocalHealPeer {
2738            movement_gate: movement_gate.clone(),
2739            writer_queued: writer_queued.clone(),
2740            writer_acquired: writer_acquired.clone(),
2741        });
2742        store.peer_sys.clients = vec![Arc::new(peer)];
2743        let store = Arc::new(store);
2744        let mut heal = tokio::spawn({
2745            let store = store.clone();
2746            async move { store.handle_heal_bucket("bucket", &HealOpts::default()).await }
2747        });
2748
2749        tokio::time::timeout(std::time::Duration::from_secs(1), writer_queued.notified())
2750            .await
2751            .expect("movement writer should queue during local peer lookup");
2752        tokio::time::timeout(std::time::Duration::from_secs(1), &mut heal)
2753            .await
2754            .expect("local fan-out must not reenter movement read behind the queued writer")
2755            .expect("bucket heal task should not panic")
2756            .expect("bucket heal should succeed");
2757        tokio::time::timeout(std::time::Duration::from_secs(1), writer_acquired.notified())
2758            .await
2759            .expect("queued movement writer should proceed after bucket heal releases its read guard");
2760    }
2761
2762    #[tokio::test]
2763    #[serial_test::serial]
2764    async fn unscoped_heal_object_suspended_owner_semantics() {
2765        let (_temp_dir, store, shutdown) = multi_pool_heal_store().await;
2766        let bucket = format!("heal-owner-{}", Uuid::new_v4().simple());
2767        let active_object = "active-owner";
2768        let suspended_only_object = "suspended-only";
2769        let duplicate_object = "duplicate-owner";
2770        let marker_object = "marker-owner";
2771        let quorum_object = "quorum-owner";
2772        store
2773            .make_bucket(&bucket, &MakeBucketOptions::default())
2774            .await
2775            .expect("bucket should be created in all pools");
2776
2777        let mut active_reader = PutObjReader::from_vec(b"active owner".to_vec());
2778        store.pools[0]
2779            .put_object(&bucket, active_object, &mut active_reader, &ObjectOptions::default())
2780            .await
2781            .expect("active owner object should be written");
2782        let active_disks = store.pools[0].disk_set[0].disks.read().await.clone();
2783        let missing_active_disk = active_disks[0].clone().expect("active disk should be online");
2784        missing_active_disk
2785            .delete(
2786                &bucket,
2787                active_object,
2788                DeleteOptions {
2789                    recursive: true,
2790                    immediate: true,
2791                    ..Default::default()
2792                },
2793            )
2794            .await
2795            .expect("active owner shard should be removed for repair");
2796        assert!(
2797            missing_active_disk.read_xl(&bucket, active_object, false).await.is_err(),
2798            "the active owner fixture must start with one missing metadata copy"
2799        );
2800
2801        let mut suspended_reader = PutObjReader::from_vec(b"suspended owner".to_vec());
2802        store.pools[1]
2803            .put_object(&bucket, suspended_only_object, &mut suspended_reader, &ObjectOptions::default())
2804            .await
2805            .expect("suspended owner object should be written");
2806        for (pool_index, mod_time) in [1_i64, 2_i64].into_iter().enumerate() {
2807            let mut duplicate_reader = PutObjReader::from_vec(format!("duplicate-pool-{pool_index}").into_bytes());
2808            store.pools[pool_index]
2809                .put_object(
2810                    &bucket,
2811                    duplicate_object,
2812                    &mut duplicate_reader,
2813                    &ObjectOptions {
2814                        mod_time: Some(OffsetDateTime::UNIX_EPOCH + time::Duration::seconds(mod_time)),
2815                        ..Default::default()
2816                    },
2817                )
2818                .await
2819                .expect("duplicate owner object should be written");
2820        }
2821        let duplicate_missing_disk = store.pools[0].disk_set[0].disks.read().await[0]
2822            .clone()
2823            .expect("duplicate active owner disk should be online");
2824        duplicate_missing_disk
2825            .delete(
2826                &bucket,
2827                duplicate_object,
2828                DeleteOptions {
2829                    recursive: true,
2830                    immediate: true,
2831                    ..Default::default()
2832                },
2833            )
2834            .await
2835            .expect("duplicate active owner shard should be removed for repair");
2836        let history_version = Uuid::new_v4();
2837        let mut history_reader = PutObjReader::from_vec(b"marker history".to_vec());
2838        store.pools[0]
2839            .put_object(
2840                &bucket,
2841                marker_object,
2842                &mut history_reader,
2843                &ObjectOptions {
2844                    versioned: true,
2845                    version_id: Some(history_version.to_string()),
2846                    mod_time: Some(OffsetDateTime::UNIX_EPOCH + time::Duration::seconds(1)),
2847                    ..Default::default()
2848                },
2849            )
2850            .await
2851            .expect("versioned marker history should be written");
2852        store.pools[0]
2853            .delete_object(
2854                &bucket,
2855                marker_object,
2856                ObjectOptions {
2857                    versioned: true,
2858                    mod_time: Some(OffsetDateTime::UNIX_EPOCH + time::Duration::seconds(2)),
2859                    ..Default::default()
2860                },
2861            )
2862            .await
2863            .expect("delete marker should be written");
2864        let mut quorum_reader = PutObjReader::from_vec(b"quorum boundary".to_vec());
2865        store.pools[0]
2866            .put_object(&bucket, quorum_object, &mut quorum_reader, &ObjectOptions::default())
2867            .await
2868            .expect("quorum boundary object should be written");
2869        {
2870            let mut pool_meta = store.pool_meta.write().await;
2871            let mut next = PoolMeta::new(&store.pools, &pool_meta);
2872            next.pools[1].decommission = Some(PoolDecommissionInfo {
2873                start_time: Some(OffsetDateTime::UNIX_EPOCH),
2874                ..Default::default()
2875            });
2876            *pool_meta = next;
2877        }
2878
2879        let (_, duplicate_owner) = store
2880            .get_latest_object_info_with_idx(&bucket, duplicate_object, &ObjectOptions::default())
2881            .await
2882            .expect("duplicate owner should resolve");
2883        assert_eq!(duplicate_owner, 1, "latest duplicate must win when all pools are eligible");
2884        let (_, active_duplicate_owner) = store
2885            .get_latest_object_info_with_idx(
2886                &bucket,
2887                duplicate_object,
2888                &ObjectOptions {
2889                    skip_decommissioned: true,
2890                    ..Default::default()
2891                },
2892            )
2893            .await
2894            .expect("active duplicate owner should resolve");
2895        assert_eq!(
2896            active_duplicate_owner, 0,
2897            "suspended duplicate must be excluded from active owner selection"
2898        );
2899        let (duplicate_result, duplicate_err) = store
2900            .handle_heal_object(&bucket, duplicate_object, "", &HealOpts::default())
2901            .await
2902            .expect("duplicate owner heal should complete through the production path");
2903        assert_eq!(duplicate_result.object, duplicate_object);
2904        assert!(duplicate_err.is_none(), "active duplicate should be repaired: {duplicate_err:?}");
2905        assert!(
2906            duplicate_missing_disk.read_xl(&bucket, duplicate_object, false).await.is_ok(),
2907            "production heal must repair the active duplicate owner rather than the suspended owner"
2908        );
2909        let (marker_info, marker_owner) = store
2910            .get_latest_object_info_with_idx(
2911                &bucket,
2912                marker_object,
2913                &ObjectOptions {
2914                    skip_decommissioned: true,
2915                    versioned: true,
2916                    ..Default::default()
2917                },
2918            )
2919            .await
2920            .expect("latest delete marker should resolve");
2921        assert_eq!(marker_owner, 0);
2922        assert!(marker_info.delete_marker, "latest version must preserve delete-marker semantics");
2923
2924        let (active_result, active_err) = store
2925            .handle_heal_object(&bucket, active_object, "", &HealOpts::default())
2926            .await
2927            .expect("unscoped active-owner heal should complete");
2928        assert_eq!(active_result.object, active_object);
2929        assert!(active_err.is_none(), "active owner must be selected even with a suspended pool");
2930        assert!(
2931            missing_active_disk.read_xl(&bucket, active_object, false).await.is_ok(),
2932            "active owner heal must write the missing disk metadata: result={active_result:?}, err={active_err:?}"
2933        );
2934        assert!(
2935            store.pools[1]
2936                .get_object_info(&bucket, active_object, &ObjectOptions::default())
2937                .await
2938                .is_err(),
2939            "the suspended pool must not be written for an active-owner object"
2940        );
2941
2942        let (suspended_result, suspended_err) = store
2943            .handle_heal_object(&bucket, suspended_only_object, "", &HealOpts::default())
2944            .await
2945            .expect("unscoped suspended-only heal should return a terminal result");
2946        assert!(suspended_result.object.is_empty());
2947        assert!(matches!(suspended_err, Some(Error::FileNotFound)));
2948        assert!(
2949            store.pools[1]
2950                .get_object_info(&bucket, suspended_only_object, &ObjectOptions::default())
2951                .await
2952                .is_ok(),
2953            "suspended-only data must remain untouched when unscoped heal reports absent"
2954        );
2955
2956        let (_, explicit_err) = store
2957            .handle_heal_object(
2958                &bucket,
2959                suspended_only_object,
2960                "",
2961                &HealOpts {
2962                    pool: Some(1),
2963                    ..Default::default()
2964                },
2965            )
2966            .await
2967            .expect("explicit suspended-owner heal should return a mapped error");
2968        assert!(matches!(explicit_err, Some(Error::SlowDown)));
2969
2970        let original_quorum_disks = store.pools[0].disk_set[0].disks.read().await.clone();
2971        let surviving_quorum_disk = original_quorum_disks[3].clone();
2972        *store.pools[0].disk_set[0].disks.write().await = vec![None, None, None, surviving_quorum_disk];
2973        let (_, quorum_err) = store
2974            .handle_heal_object(&bucket, quorum_object, "", &HealOpts::default())
2975            .await
2976            .expect("quorum boundary heal should return a mapped result");
2977        *store.pools[0].disk_set[0].disks.write().await = original_quorum_disks;
2978        let quorum_err = quorum_err
2979            .as_ref()
2980            .expect("heal must fail closed when capacity admission cannot verify pool metadata");
2981        let quorum_failure = quorum_err
2982            .pool_metadata_failure()
2983            .expect("capacity admission failure should preserve typed pool metadata context");
2984        assert_eq!(
2985            quorum_failure.kind,
2986            crate::error::PoolMetadataFailure::ReadUnavailable,
2987            "read-only capacity admission failure must remain retryable"
2988        );
2989        assert_eq!(quorum_failure.operation, "target capacity admission failed");
2990        assert_eq!(quorum_failure.phase, "pool_read");
2991        assert!(
2992            store.pool_meta_writes_ready().await,
2993            "read-only capacity admission failure must not latch the pool metadata writer"
2994        );
2995        shutdown.cancel();
2996    }
2997
2998    #[tokio::test]
2999    async fn handle_heal_format_continues_after_a_pool_error() {
3000        let canonical_format = FormatV3::new(1, 3);
3001        let mut foreign_format = canonical_format.clone();
3002        foreign_format.id = Uuid::new_v4();
3003        let mut temp_dirs = Vec::new();
3004        let mut endpoints = Vec::new();
3005        let mut disks = Vec::new();
3006
3007        for disk_index in 0..3 {
3008            let temp_dir = tempfile::tempdir().expect("temporary disk root should be created");
3009            let mut endpoint = Endpoint::try_from(temp_dir.path().to_str().expect("temporary path should be UTF-8"))
3010                .expect("temporary endpoint should parse");
3011            endpoint.set_pool_index(0);
3012            endpoint.set_set_index(0);
3013            endpoint.set_disk_index(disk_index);
3014            let disk = new_disk(
3015                &endpoint,
3016                &DiskOption {
3017                    cleanup: false,
3018                    health_check: false,
3019                },
3020            )
3021            .await
3022            .expect("temporary disk should open");
3023            let mut disk_format = foreign_format.clone();
3024            disk_format.erasure.this = foreign_format.erasure.sets[0][disk_index];
3025            save_format_file(&Some(disk.clone()), &Some(disk_format))
3026                .await
3027                .expect("foreign format should be written");
3028            temp_dirs.push(temp_dir);
3029            endpoints.push(endpoint);
3030            disks.push(Some(disk));
3031        }
3032
3033        let pool_endpoints = PoolEndpoints {
3034            legacy: false,
3035            set_count: 1,
3036            drives_per_set: 3,
3037            endpoints: Endpoints::from(endpoints),
3038            cmd_line: "foreign-format-majority-test".to_string(),
3039            platform: "test".to_string(),
3040        };
3041        let pool = Sets::new(disks, &pool_endpoints, &canonical_format, 0, 1)
3042            .await
3043            .expect("test pool should build around the cached canonical format");
3044
3045        let mut recoverable_format = FormatV3::new(1, 3);
3046        recoverable_format.id = canonical_format.id;
3047        let mut recoverable_temp_dirs = Vec::new();
3048        let mut recoverable_endpoints = Vec::new();
3049        let mut recoverable_disks = Vec::new();
3050        let mut unformatted_disk = None;
3051        for disk_index in 0..3 {
3052            let temp_dir = tempfile::tempdir().expect("temporary disk root should be created");
3053            let mut endpoint = Endpoint::try_from(temp_dir.path().to_str().expect("temporary path should be UTF-8"))
3054                .expect("temporary endpoint should parse");
3055            endpoint.set_pool_index(1);
3056            endpoint.set_set_index(0);
3057            endpoint.set_disk_index(disk_index);
3058            let disk = new_disk(
3059                &endpoint,
3060                &DiskOption {
3061                    cleanup: false,
3062                    health_check: false,
3063                },
3064            )
3065            .await
3066            .expect("temporary disk should open");
3067            if disk_index < 2 {
3068                let mut disk_format = recoverable_format.clone();
3069                disk_format.erasure.this = recoverable_format.erasure.sets[0][disk_index];
3070                save_format_file(&Some(disk.clone()), &Some(disk_format))
3071                    .await
3072                    .expect("recoverable format should be written");
3073            } else {
3074                unformatted_disk = Some(disk.clone());
3075            }
3076            recoverable_temp_dirs.push(temp_dir);
3077            recoverable_endpoints.push(endpoint);
3078            recoverable_disks.push(Some(disk));
3079        }
3080        let recoverable_pool_endpoints = PoolEndpoints {
3081            legacy: false,
3082            set_count: 1,
3083            drives_per_set: 3,
3084            endpoints: Endpoints::from(recoverable_endpoints),
3085            cmd_line: "recoverable-format-test".to_string(),
3086            platform: "test".to_string(),
3087        };
3088        let recoverable_pool = Sets::new(recoverable_disks, &recoverable_pool_endpoints, &recoverable_format, 1, 1)
3089            .await
3090            .expect("recoverable test pool should build");
3091
3092        let endpoint_pools = EndpointServerPools::from(vec![pool_endpoints.clone(), recoverable_pool_endpoints.clone()]);
3093        let store = ECStore {
3094            id: canonical_format.id,
3095            disk_map: HashMap::new(),
3096            pools: vec![pool, recoverable_pool],
3097            peer_sys: S3PeerSys::new(&endpoint_pools),
3098            pool_meta: RwLock::new(PoolMeta::default()),
3099            rebalance_meta: RwLock::new(None),
3100            decommission_cancelers: RwLock::new(Vec::new()),
3101            start_gate: Mutex::new(()),
3102            pool_meta_save_gate: Mutex::default(),
3103            ctx: crate::runtime::instance::bootstrap_ctx(),
3104            bucket_fence_registry: std::sync::Arc::default(),
3105        };
3106
3107        let err = store
3108            .handle_heal_format(false)
3109            .await
3110            .expect_err("missing pool metadata must fail closed before format writes");
3111        assert!(err.to_string().contains("no durable bootstrap identity or pool.bin replica"));
3112        store
3113            .ensure_pool_meta_side_effects_safe("missing format-heal metadata side effect")
3114            .await
3115            .expect_err("missing metadata must latch the format-heal write gate");
3116
3117        let pool_meta = PoolMeta::new(&store.pools, &PoolMeta::default());
3118        pool_meta
3119            .save_for_startup(store.pools.clone())
3120            .await
3121            .expect("pool metadata should be persisted before format heal");
3122        *store.pool_meta_save_gate.lock().await = PoolMetaWriteState::default();
3123
3124        let (result, err) = store
3125            .handle_heal_format(false)
3126            .await
3127            .expect("format heal should return the typed pool error");
3128        assert!(
3129            matches!(err, Some(StorageError::CorruptedFormat)),
3130            "foreign format majority must not be downgraded to a successful heal: {err:?}"
3131        );
3132        assert_eq!(result.disk_count, 3, "the recoverable pool should still be inspected");
3133        let healed = load_format_erasure(&unformatted_disk.expect("the unformatted disk handle should be retained"), true)
3134            .await
3135            .expect("the later pool should be healed despite the first pool error");
3136        assert_eq!(healed.erasure.this, recoverable_format.erasure.sets[0][2]);
3137
3138        let mut completed_meta = PoolMeta::new(&store.pools, &PoolMeta::default());
3139        for status in &mut completed_meta.pools {
3140            status.decommission = Some(PoolDecommissionInfo {
3141                complete: true,
3142                ..Default::default()
3143            });
3144        }
3145        completed_meta
3146            .save(store.pools.clone())
3147            .await
3148            .expect("completed pool metadata should be persisted");
3149        let (_, err) = store
3150            .handle_heal_format(false)
3151            .await
3152            .expect("completed pools should be reported as a no-op");
3153        assert!(matches!(err, Some(StorageError::NoHealRequired)));
3154    }
3155}