Skip to main content

rustfs_ecstore/store/
heal.rs

1// Copyright 2024 RustFS Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use super::*;
16use crate::core::pools::{POOL_META_NAME, load_pool_meta_identity_observing};
17use crate::disk::{self, error::DiskError};
18use crate::services::rebalance::{REBAL_META_NAME, RebalStatus};
19use crate::set_disk::get_lock_acquire_timeout;
20use crate::storage_api_contracts::heal::HealOperations as _;
21use crate::storage_api_contracts::namespace::NamespaceLocking as _;
22use rustfs_common::mrf_channel::MrfDeleteMarkerPurge;
23use rustfs_lock::NamespaceLockGuard;
24use std::collections::BTreeSet;
25use tracing::trace;
26
27const LOG_COMPONENT_ECSTORE: &str = "ecstore";
28const LOG_SUBSYSTEM_HEAL: &str = "heal";
29const EVENT_HEAL_ABANDONED_PARTS: &str = "heal_abandoned_parts";
30const EVENT_HEAL_FORMAT_COMPLETED: &str = "heal_format_completed";
31const EVENT_HEAL_OBJECT_STARTED: &str = "heal_object_started";
32
33/// An explicit bucket-heal admission owns one lifecycle generation through its write tail.
34pub(crate) struct BucketHealScope {
35    pub(crate) bucket: String,
36    pub(crate) incarnation: uuid::Uuid,
37    pub(crate) store: Arc<ECStore>,
38    fence: super::BucketIncarnationFenceGuard,
39}
40
41impl BucketHealScope {
42    pub(crate) fn check(&self) -> disk::error::Result<()> {
43        if self.fence.is_lock_lost() {
44            return Err(DiskError::other("bucket heal incarnation fence was lost"));
45        }
46        Ok(())
47    }
48}
49
50tokio::task_local! {
51    static BUCKET_HEAL_SCOPE: Arc<BucketHealScope>;
52}
53
54pub(crate) fn bucket_heal_scope(bucket: &str) -> Option<Arc<BucketHealScope>> {
55    BUCKET_HEAL_SCOPE
56        .try_with(|scope| (scope.bucket == bucket).then(|| scope.clone()))
57        .ok()
58        .flatten()
59}
60
61/// Resolve only the two canonical bucket records; other internal paths carry no bucket authority.
62pub(super) fn bucket_metadata_owner(object: &str) -> Option<&str> {
63    use crate::bucket::metadata::{BUCKET_INCARNATION_FILE, BUCKET_METADATA_FILE};
64    let (prefix, rest) = object.split_once('/')?;
65    let (bucket, file) = rest.split_once('/')?;
66    (prefix == disk::BUCKET_META_PREFIX
67        && matches!(file, BUCKET_METADATA_FILE | BUCKET_INCARNATION_FILE)
68        && check_valid_bucket_name_strict(bucket).is_ok())
69    .then_some(bucket)
70}
71
72pub(crate) fn bucket_heal_scope_for_object(bucket: &str, object: &str) -> Option<Arc<BucketHealScope>> {
73    let owner = if bucket == RUSTFS_META_BUCKET {
74        bucket_metadata_owner(object)?
75    } else {
76        bucket
77    };
78    bucket_heal_scope(owner)
79}
80
81/// Storage-owned proof for the exact version and every selected erasure location.
82/// This is an in-process result, never reconstructed from admin drive telemetry.
83#[derive(Debug)]
84pub struct HealObjectAbsenceProof {
85    pub bucket: String,
86    pub object: String,
87    pub version_id: String,
88    pub bucket_incarnation_id: Uuid,
89    pub pool_index: Option<usize>,
90    pub set_index: Option<usize>,
91    pub locations: Vec<(usize, usize)>,
92    pub removed: bool,
93}
94
95#[derive(Debug)]
96pub struct HealObjectStorageResult {
97    pub item: HealResultItem,
98    pub error: Option<Error>,
99    pub absence: Option<HealObjectAbsenceProof>,
100}
101
102fn invalid_heal_pool_index(pool_idx: usize, pool_count: usize) -> Error {
103    StorageError::InvalidArgument(
104        "heal".to_string(),
105        "pool".to_string(),
106        format!("invalid heal pool index {pool_idx} for {pool_count} pools"),
107    )
108}
109
110fn is_pool_meta_object(bucket: &str, object: &str) -> bool {
111    bucket == RUSTFS_META_BUCKET && object == POOL_META_NAME
112}
113
114#[derive(Debug, Clone, Copy)]
115enum HealFormatPoolSkip {
116    Completed,
117    Retryable,
118}
119
120fn classify_heal_format_pool(
121    pool_idx: usize,
122    pool_cmd_line: &str,
123    pool_meta: &PoolMeta,
124    rebalance_meta: Option<&RebalanceMeta>,
125) -> Option<HealFormatPoolSkip> {
126    let Some(pool) = pool_meta.pools.get(pool_idx) else {
127        return Some(HealFormatPoolSkip::Retryable);
128    };
129
130    if pool.id != pool_idx || pool_cmd_line.is_empty() || pool.cmd_line.is_empty() || pool.cmd_line != pool_cmd_line {
131        return Some(HealFormatPoolSkip::Retryable);
132    }
133
134    if let Some(decommission) = pool.decommission.as_ref() {
135        if decommission.complete {
136            return Some(HealFormatPoolSkip::Completed);
137        }
138        if decommission.failed || decommission.canceled || decommission.queued || pool_meta.is_suspended(pool_idx) {
139            return Some(HealFormatPoolSkip::Retryable);
140        }
141    }
142
143    if let Some(meta) = rebalance_meta {
144        let Some(pool_stats) = meta.pool_stats.get(pool_idx) else {
145            return Some(HealFormatPoolSkip::Retryable);
146        };
147        if pool_stats.info.stopping || (pool_stats.participating && pool_stats.info.status == RebalStatus::Started) {
148            return Some(HealFormatPoolSkip::Retryable);
149        }
150    }
151
152    None
153}
154
155fn heal_format_pool_skip_error(skip: HealFormatPoolSkip) -> Error {
156    match skip {
157        HealFormatPoolSkip::Completed => StorageError::NoHealRequired,
158        HealFormatPoolSkip::Retryable => StorageError::SlowDown,
159    }
160}
161
162fn heal_format_fence_lost_error() -> Error {
163    StorageError::SlowDown
164}
165
166impl ECStore {
167    pub async fn purge_delete_marker_with_proof(
168        self: &Arc<Self>,
169        bucket: &str,
170        object: &str,
171        version_id: &str,
172        purge: &MrfDeleteMarkerPurge,
173        opts: &HealOpts,
174    ) -> Result<bool> {
175        if opts.dry_run || opts.no_lock || !purge.is_valid() {
176            return Err(StorageError::PreconditionFailed);
177        }
178        let version = Uuid::parse_str(version_id)
179            .ok()
180            .filter(|version| !version.is_nil())
181            .ok_or(StorageError::PreconditionFailed)?;
182        let (Some(pool_index), Some(set_index)) = (opts.pool, opts.set) else {
183            return Err(StorageError::PreconditionFailed);
184        };
185        let mut marker = rustfs_filemeta::MetaDeleteMarker::default();
186        let consumed = marker.unmarshal_msg(&purge.marker)?;
187        if usize::try_from(consumed).ok() != Some(purge.marker.len())
188            || marker.version_id != Some(version)
189            || marker.stable_identity() != purge.marker_identity
190        {
191            return Err(StorageError::PreconditionFailed);
192        }
193        let marker_info = marker.clone().into_fileinfo(bucket, object, false)?;
194        if !marker_info.is_canonical_delete_marker()
195            || marker_info.delete_marker_incarnation() != Some(purge.marker_incarnation_id)
196        {
197            return Err(StorageError::PreconditionFailed);
198        }
199
200        let current = self.bucket_incarnation_id_from_disk(bucket).await?;
201        if current.is_nil() {
202            return Err(StorageError::BucketNotFound(bucket.to_owned()));
203        }
204        let original = purge.bucket_incarnation_id;
205        let object = object.to_owned();
206        self.run_bucket_heal_at_incarnation(bucket, current, opts, move |store, bucket, _opts| async move {
207            let scope = bucket_heal_scope(&bucket).ok_or(StorageError::PreconditionFailed)?;
208            scope.check()?;
209            if original != current {
210                let retirement_store = crate::bucket::metadata_sys::object_store_if_initialized_in(&store.ctx)
211                    .await
212                    .ok_or(StorageError::PreconditionFailed)?;
213                if !crate::bucket::retirement::is_retired(retirement_store, &bucket, original).await? {
214                    return Err(StorageError::PreconditionFailed);
215                }
216            }
217            let pool = store
218                .pools
219                .get(pool_index)
220                .ok_or_else(|| invalid_heal_pool_index(pool_index, store.pools.len()))?;
221            let set = pool.disk_set.get(set_index).cloned().ok_or_else(|| {
222                StorageError::InvalidArgument(
223                    "heal".to_string(),
224                    "set".to_string(),
225                    format!("invalid heal set index {set_index} for pool {pool_index}"),
226                )
227            })?;
228            set.purge_delete_marker_exact(&bucket, &object, version, marker).await
229        })
230        .await
231    }
232
233    pub(super) async fn run_bucket_heal_at_incarnation<T, F, Fut>(
234        self: &Arc<Self>,
235        bucket: &str,
236        expected: uuid::Uuid,
237        opts: &HealOpts,
238        operation: F,
239    ) -> Result<T>
240    where
241        T: Send + 'static,
242        F: FnOnce(Arc<Self>, String, HealOpts) -> Fut + Send + 'static,
243        Fut: std::future::Future<Output = Result<T>> + Send + 'static,
244    {
245        if expected.is_nil() || self.ctx.lock_manager().is_disabled() {
246            return Err(Error::other(
247                "incarnation-bound bucket heal requires a valid identity and namespace locking",
248            ));
249        }
250        // Lock order: bucket lifecycle, then bucket/object and capacity locks used by healing.
251        let fence = self.acquire_bucket_incarnation_fence(bucket, expected).await?;
252        let scope = Arc::new(BucketHealScope {
253            bucket: bucket.to_owned(),
254            incarnation: expected,
255            store: self.clone(),
256            fence,
257        });
258        let store = self.clone();
259        let bucket = bucket.to_owned();
260        let opts = *opts;
261        // Dropping a caller's cancellation/timeout waiter must not release the lifecycle
262        // owner while a storage operation is still committing.
263        tokio::spawn(BUCKET_HEAL_SCOPE.scope(scope.clone(), async move {
264            scope.check().map_err(Error::from)?;
265            let result = operation(store, bucket, opts).await;
266            scope.check().map_err(Error::from)?;
267            result
268        }))
269        .await
270        .map_err(|error| Error::other_with_context("bucket heal owner task failed", error))?
271    }
272
273    pub async fn heal_bucket_at_incarnation(
274        self: &Arc<Self>,
275        bucket: &str,
276        expected: uuid::Uuid,
277        opts: &HealOpts,
278    ) -> Result<HealResultItem> {
279        self.run_bucket_heal_at_incarnation(bucket, expected, opts, |store, bucket, opts| async move {
280            store.heal_bucket(&bucket, &opts).await
281        })
282        .await
283    }
284
285    pub async fn heal_local_bucket_at_incarnation(
286        self: &Arc<Self>,
287        bucket: &str,
288        expected: uuid::Uuid,
289        opts: &HealOpts,
290        fenced_pools: Vec<usize>,
291        pools: Option<Vec<usize>>,
292    ) -> Result<HealResultItem> {
293        self.run_bucket_heal_at_incarnation(bucket, expected, opts, |store, bucket, opts| async move {
294            let peer =
295                crate::cluster::rpc::peer_s3_client::LocalPeerS3Client::new_with_instance_ctx(None, pools, store.ctx.clone());
296            crate::cluster::rpc::peer_s3_client::PeerS3Client::heal_bucket_with_fence(&peer, &bucket, &opts, &fenced_pools)
297                .await
298                .map_err(Error::from)
299        })
300        .await
301    }
302
303    pub async fn heal_object_at_incarnation(
304        self: &Arc<Self>,
305        bucket: &str,
306        object: &str,
307        version_id: &str,
308        expected: uuid::Uuid,
309        opts: &HealOpts,
310    ) -> Result<HealObjectStorageResult> {
311        let object = object.to_owned();
312        let version_id = version_id.to_owned();
313        self.run_bucket_heal_at_incarnation(bucket, expected, opts, |store, bucket, opts| async move {
314            store.heal_object_with_proof(&bucket, &object, &version_id, &opts).await
315        })
316        .await
317    }
318
319    pub async fn heal_mrf_object_at_incarnation(
320        self: &Arc<Self>,
321        bucket: &str,
322        object: &str,
323        version_id: &str,
324        expected: uuid::Uuid,
325        opts: &HealOpts,
326    ) -> Result<HealObjectStorageResult> {
327        let object = object.to_owned();
328        let version_id = version_id.to_owned();
329        self.run_bucket_heal_at_incarnation(bucket, expected, opts, |store, bucket, opts| async move {
330            store
331                .heal_object_with_authoritative_absence_proof(&bucket, &object, &version_id, &opts)
332                .await
333        })
334        .await
335    }
336
337    async fn acquire_heal_format_fence(
338        &self,
339    ) -> Result<(
340        tokio::sync::MutexGuard<'_, PoolMetaWriteState>,
341        NamespaceLockGuard,
342        NamespaceLockGuard,
343        PoolMeta,
344        Option<RebalanceMeta>,
345    )> {
346        let metadata_pool = self
347            .pools
348            .first()
349            .cloned()
350            .ok_or_else(|| Error::other("heal format requires at least one storage pool"))?;
351        let mut write_state = self.pool_meta_save_gate.lock().await;
352        write_state.ensure_write_safe("heal format fence failed")?;
353
354        // Metadata fence order is part of the decommission/rebalance protocol:
355        // pool.bin must always be acquired before rebalance.bin. A read guard is
356        // sufficient to freeze the pool snapshot and lets replacement format
357        // recovery coexist with ordinary Heal's capacity fence. The rebalance
358        // write guard still serializes format repair and excludes transitions.
359        let pool_lock = metadata_pool.new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME).await?;
360        let pool_guard = pool_lock.get_read_lock(get_lock_acquire_timeout()).await?;
361        let rebalance_lock = metadata_pool.new_ns_lock(RUSTFS_META_BUCKET, REBAL_META_NAME).await?;
362        let rebalance_guard = rebalance_lock.get_write_lock(get_lock_acquire_timeout()).await?;
363
364        if pool_guard.is_lock_lost() || rebalance_guard.is_lock_lost() {
365            return Err(heal_format_fence_lost_error());
366        }
367
368        load_pool_meta_identity_observing(self.pools.clone(), &mut write_state).await?;
369        let mut pool_meta = PoolMeta::default();
370        let replica_state = pool_meta
371            .load_no_lock_from_replicas_observing(self.pools.clone(), &mut write_state)
372            .await?;
373        write_state.observe_replicas(replica_state);
374        write_state.ensure_missing_metadata_can_initialize()?;
375        write_state.ensure_write_safe("heal format fence failed")?;
376        if pool_meta.pools.len() != self.pools.len()
377            || pool_meta.pools.iter().enumerate().any(|(pool_idx, pool)| {
378                pool.id != pool_idx || pool.cmd_line.is_empty() || pool.cmd_line != self.pools[pool_idx].endpoints.cmd_line
379            })
380        {
381            return Err(heal_format_fence_lost_error());
382        }
383
384        let mut rebalance_meta = RebalanceMeta::new();
385        let rebalance_meta = match rebalance_meta
386            .load_with_opts(
387                metadata_pool,
388                ObjectOptions {
389                    no_lock: true,
390                    ..Default::default()
391                },
392            )
393            .await
394        {
395            Ok(()) => Some(rebalance_meta),
396            Err(Error::ConfigNotFound) => None,
397            Err(err) => return Err(err),
398        };
399
400        if rebalance_meta
401            .as_ref()
402            .is_some_and(|meta| meta.pool_stats.len() != self.pools.len())
403        {
404            return Err(heal_format_fence_lost_error());
405        }
406
407        write_state.ensure_write_safe("heal format fence failed")?;
408        if pool_guard.is_lock_lost() || rebalance_guard.is_lock_lost() {
409            return Err(heal_format_fence_lost_error());
410        }
411
412        Ok((write_state, pool_guard, rebalance_guard, pool_meta, rebalance_meta))
413    }
414
415    fn get_pools_for_heal_object(&self, opts: &HealOpts) -> Result<Vec<Arc<Sets>>> {
416        match opts.pool {
417            Some(pool_idx) => Ok(vec![
418                self.pools
419                    .get(pool_idx)
420                    .cloned()
421                    .ok_or_else(|| invalid_heal_pool_index(pool_idx, self.pools.len()))?,
422            ]),
423            None => Ok(self.pools.clone()),
424        }
425    }
426
427    /// Whether a replacement set owns this pool's `pool.bin` replica.
428    /// Placement is determined by the same hash as metadata writes, never by
429    /// whether a shard is currently readable on the replacement disk.
430    pub fn replacement_pool_metadata_required(&self, pool_index: usize, set_index: usize) -> Result<bool> {
431        let pool = self
432            .pools
433            .get(pool_index)
434            .ok_or_else(|| invalid_heal_pool_index(pool_index, self.pools.len()))?;
435        let target_set = pool.get_disks_for_heal_object(
436            POOL_META_NAME,
437            &HealOpts {
438                set: Some(set_index),
439                ..Default::default()
440            },
441        )?;
442        Ok(Arc::ptr_eq(&target_set, &pool.get_disks_by_key(POOL_META_NAME)))
443    }
444
445    /// Restore the bucket configuration and incarnation before retiring a replacement marker.
446    pub async fn heal_replacement_bucket_metadata(
447        self: &Arc<Self>,
448        bucket: &str,
449        opts: &HealOpts,
450        targets: &[String],
451    ) -> Result<()> {
452        use crate::bucket::metadata::{BUCKET_INCARNATION_FILE, BUCKET_METADATA_FILE};
453        use crate::bucket::metadata_sys::acquire_bucket_metadata_transaction_read_lock_in;
454
455        let (Some(pool_index), Some(set_index)) = (opts.pool, opts.set) else {
456            return Err(Error::PreconditionFailed);
457        };
458        if targets.is_empty() || opts.dry_run || opts.no_lock || check_valid_bucket_name_strict(bucket).is_err() {
459            return Err(Error::PreconditionFailed);
460        }
461        let pool = self
462            .pools
463            .get(pool_index)
464            .ok_or_else(|| invalid_heal_pool_index(pool_index, self.pools.len()))?;
465        let target_set = pool.get_disks_for_heal_object(BUCKET_METADATA_FILE, opts)?;
466        if targets.iter().any(|target| {
467            target_set
468                .set_endpoints
469                .iter()
470                .filter(|endpoint| endpoint.to_string() == *target)
471                .count()
472                != 1
473        }) {
474            return Err(Error::PreconditionFailed);
475        }
476        let objects: Vec<_> = [BUCKET_METADATA_FILE, BUCKET_INCARNATION_FILE]
477            .into_iter()
478            .map(|file| format!("{}/{bucket}/{file}", disk::BUCKET_META_PREFIX))
479            .filter(|object| Arc::ptr_eq(&target_set, &pool.get_disks_by_key(object)))
480            .collect();
481        if objects.is_empty() {
482            return Ok(());
483        }
484
485        // Read the persisted identity without lazy migration: a replacement
486        // must never turn lost configuration into newly fabricated defaults.
487        let incarnation = self.bucket_incarnation_id_from_disk(bucket).await?;
488        let targets = targets.to_vec();
489        self.run_bucket_heal_at_incarnation(bucket, incarnation, opts, move |store, bucket, opts| async move {
490            // Match config writers: lifecycle, metadata transaction, object locks.
491            // Keep the transaction stable through target readback, including when
492            // the caller is cancelled and the storage owner finishes its write.
493            let transaction = acquire_bucket_metadata_transaction_read_lock_in(&store.ctx, &bucket).await?;
494            for object in objects {
495                let scope = bucket_heal_scope(&bucket).ok_or(Error::PreconditionFailed)?;
496                scope.check()?;
497                if transaction.is_lock_lost() {
498                    return Err(Error::PreconditionFailed);
499                }
500                // Config objects occupy one pool, unlike pool.bin. Require a
501                // successful all-pool lookup before accepting another pool's ownership.
502                if !store.single_pool() {
503                    let (owner, _) = store
504                        .get_pool_info_for_delete_marker(RUSTFS_META_BUCKET, &object, &ObjectOptions::default())
505                        .await?;
506                    if owner.index != pool_index {
507                        continue;
508                    }
509                }
510                let metadata_opts = HealOpts { remove: false, ..opts };
511                let (result, error) = store.heal_object(RUSTFS_META_BUCKET, &object, "", &metadata_opts).await?;
512                if let Some(error) = error {
513                    return Err(error);
514                }
515                for target in &targets {
516                    let mut drives = result.after.drives.iter().filter(|drive| &drive.endpoint == target);
517                    if !drives.next().is_some_and(|drive| drive.state == "ok") || drives.next().is_some() {
518                        return Err(Error::PreconditionFailed);
519                    }
520                }
521                if !store
522                    .replacement_targets_have_version(RUSTFS_META_BUCKET, &object, "", pool_index, set_index, &targets)
523                    .await?
524                {
525                    return Err(Error::PreconditionFailed);
526                }
527            }
528            if transaction.is_lock_lost() {
529                return Err(Error::PreconditionFailed);
530            }
531            Ok(())
532        })
533        .await
534    }
535
536    /// Return every live erasure set selected by an object-heal scope.
537    pub async fn heal_erasure_set_scopes(&self, opts: &HealOpts) -> Result<Vec<(usize, usize)>> {
538        let pools = self.get_pools_for_heal_object(opts)?;
539        let pool_meta = self.pool_meta.read().await;
540        let mut scopes = Vec::new();
541
542        for pool in pools {
543            let suspended_complete = pool_meta.is_suspended(pool.pool_idx).then(|| {
544                pool_meta
545                    .pools
546                    .get(pool.pool_idx)
547                    .and_then(|status| status.decommission.as_ref())
548                    .is_some_and(|decommission| decommission.complete)
549            });
550            if let Some(complete) = suspended_complete {
551                if opts.pool.is_some() {
552                    return Err(if complete {
553                        StorageError::InvalidArgument(
554                            "heal".to_string(),
555                            "pool".to_string(),
556                            format!("heal pool {} has completed decommission", pool.pool_idx),
557                        )
558                    } else {
559                        Error::SlowDown
560                    });
561                }
562                continue;
563            }
564
565            if let Some(set_idx) = opts.set {
566                if set_idx >= pool.disk_set.len() {
567                    return Err(StorageError::InvalidArgument(
568                        "heal".to_string(),
569                        "set".to_string(),
570                        format!(
571                            "invalid heal set index {set_idx} for pool {} with {} sets",
572                            pool.pool_idx,
573                            pool.disk_set.len()
574                        ),
575                    ));
576                }
577                scopes.push((pool.pool_idx, set_idx));
578            } else {
579                scopes.extend((0..pool.disk_set.len()).map(|set_idx| (pool.pool_idx, set_idx)));
580            }
581        }
582
583        if scopes.is_empty() {
584            return Err(Error::SlowDown);
585        }
586
587        Ok(scopes)
588    }
589
590    #[instrument(skip(self))]
591    pub(super) async fn handle_heal_format(&self, dry_run: bool) -> Result<(HealResultItem, Option<Error>)> {
592        let mut r = HealResultItem {
593            heal_item_type: HealItemType::Metadata.to_string(),
594            detail: "disk-format".to_string(),
595            ..Default::default()
596        };
597
598        let mut count_no_heal = 0;
599        let mut count_completed = 0;
600        let mut first_error = None;
601        for (pool_idx, pool) in self.pools.iter().enumerate() {
602            let (mut write_state, pool_guard, rebalance_guard, pool_meta, rebalance_meta) =
603                self.acquire_heal_format_fence().await?;
604            if pool_guard.is_lock_lost() || rebalance_guard.is_lock_lost() {
605                first_error.get_or_insert(heal_format_fence_lost_error());
606                break;
607            }
608            if let Some(skip) = classify_heal_format_pool(pool_idx, &pool.endpoints.cmd_line, &pool_meta, rebalance_meta.as_ref())
609            {
610                if matches!(skip, HealFormatPoolSkip::Completed) {
611                    count_completed += 1;
612                } else {
613                    first_error.get_or_insert(heal_format_pool_skip_error(skip));
614                }
615                continue;
616            }
617
618            let fence_lost = || {
619                let lost = pool_guard.is_lock_lost()
620                    || rebalance_guard.is_lock_lost()
621                    || write_state.ensure_write_safe("heal format write fence failed").is_err();
622                if lost {
623                    write_state.block_writes_after_fence_loss();
624                }
625                lost
626            };
627            let (mut result, err) = pool.heal_format_with_fence(dry_run, fence_lost).await?;
628            if let Some(err) = err {
629                match err {
630                    StorageError::NoHealRequired => {
631                        count_no_heal += 1;
632                    }
633                    err => {
634                        first_error.get_or_insert(err);
635                    }
636                }
637            }
638            r.disk_count += result.disk_count;
639            r.set_count += result.set_count;
640            r.before.drives.append(&mut result.before.drives);
641            r.after.drives.append(&mut result.after.drives);
642
643            // A lease can be lost after the final write; fail closed before
644            // reporting the pool as successfully healed.
645            let fence_lost = pool_guard.is_lock_lost()
646                || rebalance_guard.is_lock_lost()
647                || write_state.ensure_write_safe("heal format publication fence failed").is_err();
648            if fence_lost {
649                write_state.block_writes_after_fence_loss();
650                first_error.get_or_insert(heal_format_fence_lost_error());
651                break;
652            }
653        }
654        if let Some(err) = first_error {
655            return Ok((r, Some(err)));
656        }
657        if count_no_heal + count_completed == self.pools.len() {
658            info!(
659                event = EVENT_HEAL_FORMAT_COMPLETED,
660                component = LOG_COMPONENT_ECSTORE,
661                subsystem = LOG_SUBSYSTEM_HEAL,
662                dry_run,
663                result = "no_heal_required",
664                pool_count = self.pools.len(),
665                "Heal format completed"
666            );
667            return Ok((r, Some(StorageError::NoHealRequired)));
668        }
669        info!(
670            event = EVENT_HEAL_FORMAT_COMPLETED,
671            component = LOG_COMPONENT_ECSTORE,
672            subsystem = LOG_SUBSYSTEM_HEAL,
673            dry_run,
674            result = "healed_or_inspected",
675            disk_count = r.disk_count,
676            set_count = r.set_count,
677            "Heal format completed"
678        );
679        Ok((r, None))
680    }
681
682    #[instrument(skip(self, targets), fields(pool_index, set_index, target_count = targets.len()))]
683    pub async fn heal_replacement_format(
684        &self,
685        dry_run: bool,
686        pool_index: usize,
687        set_index: usize,
688        targets: &[String],
689    ) -> Result<(HealResultItem, Option<Error>)> {
690        let pool = self
691            .pools
692            .get(pool_index)
693            .ok_or_else(|| invalid_heal_pool_index(pool_index, self.pools.len()))?;
694        let set = pool.disk_set.get(set_index).cloned().ok_or_else(|| {
695            StorageError::InvalidArgument(
696                "heal".to_string(),
697                "set".to_string(),
698                format!("invalid heal set index {set_index} for pool {pool_index}"),
699            )
700        })?;
701
702        let (mut write_state, pool_guard, rebalance_guard, pool_meta, rebalance_meta) = self.acquire_heal_format_fence().await?;
703        if let Some(skip) = classify_heal_format_pool(pool_index, &pool.endpoints.cmd_line, &pool_meta, rebalance_meta.as_ref()) {
704            return Ok((HealResultItem::default(), Some(heal_format_pool_skip_error(skip))));
705        }
706
707        let fence_lost = || {
708            let lost = pool_guard.is_lock_lost()
709                || rebalance_guard.is_lock_lost()
710                || write_state
711                    .ensure_write_safe("replacement format write fence failed")
712                    .is_err();
713            if lost {
714                write_state.block_writes_after_fence_loss();
715            }
716            lost
717        };
718        let result = set.heal_replacement_format_with_fence(dry_run, targets, fence_lost).await?;
719        let fence_lost = pool_guard.is_lock_lost()
720            || rebalance_guard.is_lock_lost()
721            || write_state
722                .ensure_write_safe("replacement format publication fence failed")
723                .is_err();
724        if fence_lost {
725            write_state.block_writes_after_fence_loss();
726            return Ok((result.0, Some(heal_format_fence_lost_error())));
727        }
728        Ok(result)
729    }
730
731    #[instrument(skip(self, targets), fields(pool_index, set_index, target_count = targets.len()))]
732    pub async fn replacement_targets_have_version(
733        &self,
734        bucket: &str,
735        object: &str,
736        version_id: &str,
737        pool_index: usize,
738        set_index: usize,
739        targets: &[String],
740    ) -> Result<bool> {
741        let pool = self
742            .pools
743            .get(pool_index)
744            .ok_or_else(|| invalid_heal_pool_index(pool_index, self.pools.len()))?;
745        let set = pool.disk_set.get(set_index).cloned().ok_or_else(|| {
746            StorageError::InvalidArgument(
747                "heal".to_string(),
748                "set".to_string(),
749                format!("invalid heal set index {set_index} for pool {pool_index}"),
750            )
751        })?;
752
753        set.replacement_targets_have_version(bucket, object, version_id, targets)
754            .await
755            .map_err(Into::into)
756    }
757
758    #[instrument(skip(self))]
759    pub(super) async fn handle_heal_bucket(&self, bucket: &str, opts: &HealOpts) -> Result<HealResultItem> {
760        let movement_gate = self.ctx.data_movement_operation_gate();
761        let _movement_guard = movement_gate.read().await;
762        let save_guard = self.pool_meta_save_gate.lock().await;
763        save_guard.ensure_write_safe("bucket heal cannot run while pool metadata requires recovery")?;
764        let mut fenced_pools = BTreeSet::new();
765        {
766            let pool_meta = self.pool_meta.read().await;
767            fenced_pools.extend((0..pool_meta.pools.len()).filter(|pool_idx| pool_meta.is_suspended(*pool_idx)));
768            if let Some(pool_idx) = opts.pool {
769                if pool_idx >= pool_meta.pools.len() {
770                    return Err(invalid_heal_pool_index(pool_idx, pool_meta.pools.len()));
771                }
772                if pool_meta.is_suspended(pool_idx) {
773                    let complete = pool_meta.pools[pool_idx]
774                        .decommission
775                        .as_ref()
776                        .is_some_and(|decommission| decommission.complete);
777                    return Err(if complete {
778                        StorageError::InvalidArgument(
779                            "heal".to_string(),
780                            "pool".to_string(),
781                            format!("heal pool {pool_idx} has completed decommission"),
782                        )
783                    } else {
784                        Error::SlowDown
785                    });
786                }
787            }
788        }
789
790        let dispatch_fenced_pools = fenced_pools.iter().copied().collect::<Vec<_>>();
791        drop(save_guard);
792        let mut res = self
793            .peer_sys
794            .heal_bucket_with_fence_from_movement_guarded_coordinator(bucket, opts, &dispatch_fenced_pools)
795            .await?;
796        {
797            let pool_meta = self.pool_meta.read().await;
798            fenced_pools.extend((0..pool_meta.pools.len()).filter(|pool_idx| pool_meta.is_suspended(*pool_idx)));
799        }
800        if !fenced_pools.is_empty() {
801            let pools = fenced_pools.iter().map(usize::to_string).collect::<Vec<_>>().join(", ");
802            res.detail = format!("skipped: bucket-volume heal fenced on decommission-suspended pool(s): {pools}");
803        }
804
805        Ok(res)
806    }
807
808    #[instrument(level = "trace", skip(self, opts), fields(bucket = %bucket, object = %object, version_id = %version_id))]
809    pub(super) async fn handle_heal_object(
810        &self,
811        bucket: &str,
812        object: &str,
813        version_id: &str,
814        opts: &HealOpts,
815    ) -> Result<(HealResultItem, Option<Error>)> {
816        self.handle_heal_object_with_absence(
817            bucket,
818            object,
819            version_id,
820            opts,
821            &mut None,
822            crate::set_disk::AbsenceProofRequest {
823                retirement: None,
824                allow_unversioned: false,
825            },
826        )
827        .await
828    }
829
830    pub async fn heal_object_with_proof(
831        &self,
832        bucket: &str,
833        object: &str,
834        version_id: &str,
835        opts: &HealOpts,
836    ) -> Result<HealObjectStorageResult> {
837        self.heal_object_with_absence_proof(bucket, object, version_id, opts, false)
838            .await
839    }
840
841    async fn heal_object_with_authoritative_absence_proof(
842        &self,
843        bucket: &str,
844        object: &str,
845        version_id: &str,
846        opts: &HealOpts,
847    ) -> Result<HealObjectStorageResult> {
848        self.heal_object_with_absence_proof(bucket, object, version_id, opts, true)
849            .await
850    }
851
852    async fn heal_object_with_absence_proof(
853        &self,
854        bucket: &str,
855        object: &str,
856        version_id: &str,
857        opts: &HealOpts,
858        allow_unversioned_absence: bool,
859    ) -> Result<HealObjectStorageResult> {
860        // Object heal may recreate a missing bucket volume before committing a
861        // shard. Keep the current bucket generation fenced for the whole
862        // operation, including the ordinary unversioned path.
863        if opts.dry_run || opts.no_lock || super::utils::is_reserved_or_invalid_bucket(bucket, false) {
864            let (item, error) = self.handle_heal_object(bucket, object, version_id, opts).await?;
865            return Ok(HealObjectStorageResult {
866                item,
867                error,
868                absence: None,
869            });
870        }
871
872        // Match object publication: bucket lifecycle before capacity and object
873        // namespace locks. Keep the incarnation pinned through proof delivery.
874        // Reuse the admission's owner: reacquiring a read lock behind a queued
875        // lifecycle writer would deadlock with the read lock we already hold.
876        let scope = bucket_heal_scope(bucket).filter(|scope| std::ptr::eq(scope.store.as_ref(), self));
877        let guard = if let Some(scope) = &scope {
878            scope.check().map_err(Error::from)?;
879            None
880        } else {
881            Some(self.acquire_bucket_lifecycle_read_lock(bucket).await?)
882        };
883        let fence_is_lost = || {
884            scope.as_ref().is_some_and(|scope| scope.fence.is_lock_lost())
885                || guard.as_ref().is_some_and(NamespaceLockGuard::is_lock_lost)
886        };
887        let lifecycle_guard = scope
888            .as_ref()
889            .and_then(|scope| scope.fence.namespace_lock_guard())
890            .or(guard.as_ref())
891            .ok_or_else(|| Error::other("bucket heal requires a held lifecycle guard"))?;
892        let retirement = crate::bucket::retirement::MarkerRetirementContext {
893            store: crate::bucket::metadata_sys::object_store_if_initialized_in(&self.ctx).await,
894            current_incarnation: if let Some(scope) = &scope {
895                Some(scope.incarnation)
896            } else {
897                self.bucket_incarnation_id_from_disk(bucket)
898                    .await
899                    .ok()
900                    .filter(|id| !id.is_nil())
901            },
902            lifecycle_guard,
903        };
904        let mut proofs = None;
905        let (item, mut error) = self
906            .handle_heal_object_with_absence(
907                bucket,
908                object,
909                version_id,
910                opts,
911                &mut proofs,
912                crate::set_disk::AbsenceProofRequest {
913                    retirement: Some(&retirement),
914                    allow_unversioned: allow_unversioned_absence,
915                },
916            )
917            .await?;
918        // Read the authoritative incarnation only for an absence candidate.
919        // The lifecycle guard has pinned it throughout the storage operation.
920        let incarnation = if proofs.is_some() && !fence_is_lost() {
921            if let Some(scope) = &scope {
922                Some(scope.incarnation)
923            } else {
924                self.bucket_incarnation_id_from_disk(bucket)
925                    .await
926                    .ok()
927                    .filter(|id| !id.is_nil())
928            }
929        } else {
930            None
931        };
932        let absence = match (incarnation, proofs) {
933            (Some(incarnation), Some(proofs)) if !fence_is_lost() => Some(HealObjectAbsenceProof {
934                bucket: bucket.to_owned(),
935                object: object.to_owned(),
936                version_id: version_id.to_owned(),
937                bucket_incarnation_id: incarnation,
938                pool_index: opts.pool,
939                set_index: opts.set,
940                removed: proofs.iter().any(|proof| proof.removed),
941                locations: proofs.into_iter().map(|proof| (proof.pool_index, proof.set_index)).collect(),
942            }),
943            _ => None,
944        };
945        if absence.is_some() {
946            error = None;
947        }
948        Ok(HealObjectStorageResult { item, error, absence })
949    }
950
951    async fn handle_heal_object_with_absence(
952        &self,
953        bucket: &str,
954        object: &str,
955        version_id: &str,
956        opts: &HealOpts,
957        absence: &mut Option<Vec<crate::set_disk::HealedObjectAbsence>>,
958        proof: crate::set_disk::AbsenceProofRequest<'_>,
959    ) -> Result<(HealResultItem, Option<Error>)> {
960        trace!(
961            event = EVENT_HEAL_OBJECT_STARTED,
962            component = LOG_COMPONENT_ECSTORE,
963            subsystem = LOG_SUBSYSTEM_HEAL,
964            bucket = %bucket,
965            object = %object,
966            version_id = %version_id,
967            remove = opts.remove,
968            scan_mode = ?opts.scan_mode,
969            "Heal object started"
970        );
971        let object = encode_dir_object(object);
972
973        *absence = None;
974        let pools = self.get_pools_for_heal_object(opts)?;
975        let requested_pool_count = pools.len();
976        if let Some(set_idx) = opts.set {
977            for pool in &pools {
978                if set_idx >= pool.disk_set.len() {
979                    let err = StorageError::InvalidArgument(
980                        "heal".to_string(),
981                        "set".to_string(),
982                        format!(
983                            "invalid heal set index {set_idx} for pool {} with {} sets",
984                            pool.pool_idx,
985                            pool.disk_set.len()
986                        ),
987                    );
988                    if opts.pool.is_some() {
989                        return Err(err);
990                    }
991                    return Ok((HealResultItem::default(), Some(err)));
992                }
993            }
994        }
995        #[cfg(test)]
996        let store_id = self.id;
997
998        let mut heal_pools = Vec::with_capacity(pools.len());
999        for pool in pools.iter() {
1000            let suspended_complete = {
1001                let pool_meta = self.pool_meta.read().await;
1002                pool_meta.is_suspended(pool.pool_idx).then(|| {
1003                    pool_meta
1004                        .pools
1005                        .get(pool.pool_idx)
1006                        .and_then(|status| status.decommission.as_ref())
1007                        .is_some_and(|decommission| decommission.complete)
1008                })
1009            };
1010            if let Some(complete) = suspended_complete {
1011                if opts.pool.is_some() {
1012                    let _ = pool.get_disks_for_heal_object(&object, opts)?;
1013                    let err = if complete {
1014                        StorageError::InvalidArgument(
1015                            "heal".to_string(),
1016                            "pool".to_string(),
1017                            format!("heal pool {} has completed decommission", pool.pool_idx),
1018                        )
1019                    } else {
1020                        Error::SlowDown
1021                    };
1022                    return Ok((HealResultItem::default(), Some(err)));
1023                }
1024                continue;
1025            }
1026            heal_pools.push(Arc::clone(pool));
1027        }
1028        let results = if is_pool_meta_object(bucket, &object) && !opts.no_lock && !heal_pools.is_empty() {
1029            let target_pool_indices = heal_pools.iter().map(|pool| pool.pool_idx).collect::<Vec<_>>();
1030            match self.acquire_pool_meta_object_heal_fence(&target_pool_indices).await {
1031                Ok((pool_meta_guard, admissions)) => {
1032                    let fixed_set = self.pools.first().and_then(|pool| pool.disk_set.first()).cloned();
1033                    let futures = heal_pools.iter().zip(admissions).map(|(pool, admission)| {
1034                        let pool = Arc::clone(pool);
1035                        let pool_object = object.clone();
1036                        let fixed_set = fixed_set.clone();
1037                        let mut opts = *opts;
1038                        async move {
1039                            admission?;
1040                            let fixed_set = fixed_set.ok_or_else(|| Error::other("pool metadata heal requires a fixed set"))?;
1041                            let target_set = pool.get_disks_for_heal_object(&pool_object, &opts)?;
1042                            opts.no_lock = fixed_set.shares_namespace_lock_domain(&target_set).await;
1043                            #[cfg(test)]
1044                            if !opts.no_lock {
1045                                crate::core::pools::notify_decommission_external_heal_target_lock_attempted();
1046                            }
1047                            #[cfg(test)]
1048                            crate::core::pools::notify_decommission_external_heal_operation_started(store_id);
1049                            pool.heal_object_with_absence(bucket, &pool_object, version_id, &opts, proof)
1050                                .await
1051                        }
1052                    });
1053                    let results = join_all(futures).await;
1054                    drop(pool_meta_guard);
1055                    results
1056                }
1057                Err(err) => (0..heal_pools.len()).map(|_| Err(err.clone())).collect(),
1058            }
1059        } else {
1060            let mut futures = Vec::with_capacity(heal_pools.len());
1061            for pool in heal_pools {
1062                let pool_idx = pool.pool_idx;
1063                let pool_object = object.clone();
1064                let opts = *opts;
1065                futures.push(self.run_external_decommission_capacity_heal(
1066                    pool_idx,
1067                    bucket,
1068                    &object,
1069                    opts,
1070                    move |opts| async move {
1071                        #[cfg(test)]
1072                        crate::core::pools::notify_decommission_external_heal_operation_started(store_id);
1073                        pool.heal_object_with_absence(bucket, &pool_object, version_id, &opts, proof)
1074                            .await
1075                    },
1076                ));
1077            }
1078            join_all(futures).await
1079        };
1080
1081        let mut proofs = Vec::with_capacity(requested_pool_count);
1082        let mut errs = Vec::with_capacity(self.pools.len());
1083        let mut ress = Vec::with_capacity(self.pools.len());
1084
1085        for res in results.into_iter() {
1086            match res {
1087                Ok((result, err, proof)) => {
1088                    if let Some(proof) = proof
1089                        && (err.is_none()
1090                            || err
1091                                .as_ref()
1092                                .is_some_and(|err| is_err_object_not_found(err) || is_err_version_not_found(err)))
1093                    {
1094                        proofs.push(proof);
1095                    }
1096                    let mut result = result;
1097                    result.object = decode_dir_object(&result.object);
1098                    ress.push(result);
1099                    errs.push(err);
1100                }
1101                Err(err) => {
1102                    errs.push(Some(err));
1103                    ress.push(HealResultItem::default());
1104                }
1105            }
1106        }
1107
1108        // Absence in one pool cannot discharge a responsibility covering other
1109        // pools, including skipped decommission sources or failed lookups.
1110        if requested_pool_count > 0 && proofs.len() == requested_pool_count {
1111            *absence = Some(proofs);
1112        }
1113
1114        for (idx, err) in errs.iter().enumerate() {
1115            if err.is_none() {
1116                return Ok((ress.remove(idx), None));
1117            }
1118        }
1119
1120        // No pool returned a nil error, return the first non 'not found' error
1121        for (index, err) in errs.iter().enumerate() {
1122            return match err {
1123                Some(err) => {
1124                    if is_err_object_not_found(err) || is_err_version_not_found(err) {
1125                        continue;
1126                    }
1127                    Ok((ress.remove(index), Some(err.clone())))
1128                }
1129                None => Ok((ress.remove(index), None)),
1130            };
1131        }
1132
1133        // At this stage, all errors are 'not found'
1134        if !version_id.is_empty() {
1135            return Ok((HealResultItem::default(), Some(Error::FileVersionNotFound)));
1136        }
1137
1138        Ok((HealResultItem::default(), Some(Error::FileNotFound)))
1139    }
1140
1141    #[instrument(skip(self))]
1142    pub(super) async fn handle_check_abandoned_parts(&self, bucket: &str, object: &str, opts: &HealOpts) -> Result<()> {
1143        let object = encode_dir_object(object);
1144        let pools = self.get_pools_for_heal_object(opts)?;
1145
1146        let mut futures = Vec::with_capacity(pools.len());
1147        for pool in pools.iter() {
1148            futures.push(pool.check_abandoned_parts(bucket, &object, opts));
1149        }
1150
1151        let mut first_error = None;
1152        for result in join_all(futures).await {
1153            if let Err(err) = result
1154                && first_error.is_none()
1155            {
1156                first_error = Some(err);
1157            }
1158        }
1159
1160        if let Some(err) = first_error {
1161            return Err(err);
1162        }
1163
1164        trace!(
1165            event = EVENT_HEAL_ABANDONED_PARTS,
1166            component = LOG_COMPONENT_ECSTORE,
1167            subsystem = LOG_SUBSYSTEM_HEAL,
1168            state = "completed",
1169            result = "ok",
1170            bucket,
1171            object,
1172            dry_run = opts.dry_run,
1173            "Heal abandoned parts completed"
1174        );
1175
1176        Ok(())
1177    }
1178}
1179
1180#[cfg(test)]
1181mod tests {
1182    use super::*;
1183    use crate::bucket::metadata_sys;
1184    use crate::cluster::rpc::PeerS3Client;
1185    use crate::config::com::{delete_config, read_config_no_lock_preserve_empty_with_metadata, save_config};
1186    use crate::core::pools::{
1187        DecommissionCapacityAdmission, DecommissionCapacityLockOrderBarrier, DecommissionErasureLayout,
1188        DecommissionPoolCapacityInfo, POOL_META_IDENTITY_NAME, PoolDecommissionInfo, PoolMetaReplicaState, PoolStatus,
1189        initialized_pool_meta_identity_for_test, set_decommission_capacity_info_overrides_for_test,
1190    };
1191    use crate::core::sets::HealFormatAfterSaveBarrier;
1192    use crate::disk::error::Result as DiskResult;
1193    use crate::disk::{DeleteOptions, DiskOption, DiskStore, FORMAT_CONFIG_FILE, format::FormatV3, new_disk};
1194    use crate::layout::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints};
1195    use crate::runtime::instance::InstanceContext;
1196    use crate::services::rebalance::{
1197        RebalanceInfo, RebalanceStats, test_three_pool_stores_with_isolated_node_contexts, test_two_pool_stores,
1198    };
1199    use crate::storage_api_contracts::bucket::{
1200        BucketInfo, BucketOperations, BucketOptions, DeleteBucketOptions, MakeBucketOptions,
1201    };
1202    use crate::storage_api_contracts::object::{ObjectIO as _, ObjectOperations};
1203    use crate::store::init_format::{load_format_erasure, save_format_file};
1204    use crate::store::init_local_disks_with_instance_ctx;
1205    use rustfs_heal_contracts::heal_channel::DriveState;
1206    use tokio_util::sync::CancellationToken;
1207
1208    #[derive(Debug)]
1209    struct BlockingHealPeer {
1210        started: Arc<tokio::sync::Notify>,
1211        release: Arc<tokio::sync::Notify>,
1212    }
1213
1214    #[async_trait::async_trait]
1215    impl PeerS3Client for BlockingHealPeer {
1216        async fn heal_bucket(&self, _bucket: &str, _opts: &HealOpts) -> DiskResult<HealResultItem> {
1217            self.started.notify_one();
1218            self.release.notified().await;
1219            Ok(HealResultItem::default())
1220        }
1221
1222        async fn make_bucket(&self, _bucket: &str, _opts: &MakeBucketOptions) -> DiskResult<()> {
1223            Ok(())
1224        }
1225
1226        async fn list_bucket(&self, _opts: &BucketOptions) -> DiskResult<Vec<BucketInfo>> {
1227            Ok(Vec::new())
1228        }
1229
1230        async fn delete_bucket(&self, _bucket: &str, _opts: &DeleteBucketOptions) -> DiskResult<()> {
1231            Ok(())
1232        }
1233
1234        async fn get_bucket_info(&self, _bucket: &str, _opts: &BucketOptions) -> DiskResult<BucketInfo> {
1235            Ok(BucketInfo::default())
1236        }
1237
1238        fn get_pools(&self) -> Option<Vec<usize>> {
1239            Some(vec![0, 1])
1240        }
1241    }
1242
1243    #[derive(Debug)]
1244    struct WriterQueuedLocalHealPeer {
1245        movement_gate: Arc<tokio::sync::RwLock<()>>,
1246        writer_queued: Arc<tokio::sync::Notify>,
1247        writer_acquired: Arc<tokio::sync::Notify>,
1248    }
1249
1250    #[async_trait::async_trait]
1251    impl PeerS3Client for WriterQueuedLocalHealPeer {
1252        async fn heal_bucket(&self, _bucket: &str, _opts: &HealOpts) -> DiskResult<HealResultItem> {
1253            let _movement_guard = self
1254                .movement_gate
1255                .try_read()
1256                .map_err(|_| crate::disk::error::DiskError::TooManyOpenFiles)?;
1257            Ok(HealResultItem::default())
1258        }
1259
1260        async fn heal_bucket_with_fence_from_movement_guarded_coordinator(
1261            &self,
1262            _bucket: &str,
1263            _opts: &HealOpts,
1264            _fenced_pools: &[usize],
1265        ) -> DiskResult<HealResultItem> {
1266            Ok(HealResultItem::default())
1267        }
1268
1269        async fn make_bucket(&self, _bucket: &str, _opts: &MakeBucketOptions) -> DiskResult<()> {
1270            Ok(())
1271        }
1272
1273        async fn list_bucket(&self, _opts: &BucketOptions) -> DiskResult<Vec<BucketInfo>> {
1274            Ok(Vec::new())
1275        }
1276
1277        async fn delete_bucket(&self, _bucket: &str, _opts: &DeleteBucketOptions) -> DiskResult<()> {
1278            Ok(())
1279        }
1280
1281        async fn get_bucket_info(&self, _bucket: &str, _opts: &BucketOptions) -> DiskResult<BucketInfo> {
1282            let movement_gate = self.movement_gate.clone();
1283            let writer_acquired = self.writer_acquired.clone();
1284            tokio::spawn(async move {
1285                let _movement_guard = movement_gate.write().await;
1286                writer_acquired.notify_one();
1287            });
1288            while self.movement_gate.try_read().is_ok() {
1289                tokio::task::yield_now().await;
1290            }
1291            self.writer_queued.notify_one();
1292            Ok(BucketInfo::default())
1293        }
1294
1295        fn get_pools(&self) -> Option<Vec<usize>> {
1296            Some(vec![0, 1])
1297        }
1298    }
1299
1300    async fn minimal_heal_pool(pool_idx: usize) -> Arc<Sets> {
1301        let format = FormatV3::new(1, 1);
1302        let endpoint_url = format!("http://127.0.0.1:{}/data", 19000 + pool_idx);
1303        let mut endpoint = Endpoint::try_from(endpoint_url.as_str()).expect("endpoint should parse");
1304        endpoint.set_pool_index(pool_idx);
1305        endpoint.set_set_index(0);
1306        endpoint.set_disk_index(0);
1307
1308        Sets::new(
1309            vec![None],
1310            &PoolEndpoints {
1311                legacy: false,
1312                set_count: 1,
1313                drives_per_set: 1,
1314                endpoints: Endpoints::from(vec![endpoint]),
1315                cmd_line: String::new(),
1316                platform: String::new(),
1317            },
1318            &format,
1319            pool_idx,
1320            0,
1321        )
1322        .await
1323        .expect("minimal pool should build")
1324    }
1325
1326    #[tokio::test]
1327    #[serial_test::serial]
1328    async fn bucket_incarnation_heal_preserves_successor_shards_and_allows_fresh_repair() {
1329        let (_root, store, shutdown) = multi_pool_heal_store().await;
1330        let bucket = format!("heal-incarnation-{}", Uuid::new_v4().simple());
1331        store
1332            .make_bucket(&bucket, &MakeBucketOptions::default())
1333            .await
1334            .expect("original bucket");
1335        let old = store
1336            .bucket_incarnation_id_from_disk(&bucket)
1337            .await
1338            .expect("original identity");
1339        store
1340            .delete_bucket(&bucket, &DeleteBucketOptions::default())
1341            .await
1342            .expect("normal bucket deletion");
1343        store
1344            .make_bucket(&bucket, &MakeBucketOptions::default())
1345            .await
1346            .expect("same-name successor");
1347        let new = store
1348            .bucket_incarnation_id_from_disk(&bucket)
1349            .await
1350            .expect("successor identity");
1351        assert_ne!(old, new);
1352        let object = "successor-version";
1353        let version = Uuid::new_v4().to_string();
1354        let set = store.pools[0].get_disks(0);
1355        let mut reader = PutObjReader::from_vec(vec![17; 512 * 1024]);
1356        set.put_object(
1357            &bucket,
1358            object,
1359            &mut reader,
1360            &ObjectOptions {
1361                versioned: true,
1362                version_id: Some(version.clone()),
1363                // Drain every rename before the fixture removes a physical shard.
1364                write_completion: crate::object_api::WriteCompletion::TailDrained,
1365                ..Default::default()
1366            },
1367        )
1368        .await
1369        .expect("write successor version");
1370        let missing = set.disks.read().await[0].clone().expect("missing target");
1371        let healthy = set.disks.read().await[1].clone().expect("healthy target");
1372        missing
1373            .delete(
1374                &bucket,
1375                object,
1376                DeleteOptions {
1377                    recursive: true,
1378                    immediate: true,
1379                    ..Default::default()
1380                },
1381            )
1382            .await
1383            .expect("remove one successor shard");
1384        let baseline = healthy
1385            .read_all(&bucket, &format!("{object}/xl.meta"))
1386            .await
1387            .expect("healthy baseline");
1388        let opts = HealOpts {
1389            pool: Some(0),
1390            set: Some(0),
1391            scan_mode: rustfs_heal_contracts::heal_channel::HealScanMode::Deep,
1392            ..Default::default()
1393        };
1394        // Positive control for the original bug: name-only healing repairs the successor.
1395        let (_, error) = store
1396            .heal_object(&bucket, object, &version, &opts)
1397            .await
1398            .expect("unbound control");
1399        assert!(error.is_none());
1400        assert!(missing.read_xl(&bucket, object, false).await.is_ok());
1401        missing
1402            .delete(
1403                &bucket,
1404                object,
1405                DeleteOptions {
1406                    recursive: true,
1407                    immediate: true,
1408                    ..Default::default()
1409                },
1410            )
1411            .await
1412            .expect("restore the same missing-shard condition");
1413        let fi = healthy
1414            .read_version("", &bucket, object, &version, &Default::default())
1415            .await
1416            .expect("successor metadata");
1417        let disk_ref = healthy.endpoint().to_string();
1418        assert!(
1419            store
1420                .write_local_metadata_at_incarnation(&disk_ref, (&bucket, object), fi.clone(), old)
1421                .await
1422                .is_err()
1423        );
1424        assert!(
1425            store
1426                .delete_local_path_at_incarnation(
1427                    &disk_ref,
1428                    (&bucket, object),
1429                    DeleteOptions {
1430                        recursive: true,
1431                        immediate: true,
1432                        ..Default::default()
1433                    },
1434                    old
1435                )
1436                .await
1437                .is_err()
1438        );
1439        assert!(
1440            store
1441                .delete_local_version_at_incarnation(
1442                    &disk_ref,
1443                    (&bucket, object),
1444                    fi.clone(),
1445                    false,
1446                    DeleteOptions::default(),
1447                    old
1448                )
1449                .await
1450                .is_err()
1451        );
1452        assert!(
1453            store
1454                .rename_local_data_at_incarnation(&disk_ref, (&bucket, object), &fi, (&bucket, "stale-destination"), old)
1455                .await
1456                .is_err()
1457        );
1458        for file in [".metadata.bin", ".bucket-incarnation"] {
1459            let metadata = format!("buckets/{bucket}/{file}");
1460            assert!(
1461                store
1462                    .rename_local_data_at_incarnation(&disk_ref, (&bucket, object), &fi, (RUSTFS_META_BUCKET, &metadata), old)
1463                    .await
1464                    .is_err(),
1465                "stale remote heal must not replace the successor's {file}"
1466            );
1467        }
1468        assert!(store.heal_bucket_at_incarnation(&bucket, old, &opts).await.is_err());
1469        assert!(
1470            store
1471                .heal_object_at_incarnation(&bucket, object, &version, old, &opts)
1472                .await
1473                .is_err()
1474        );
1475        assert!(
1476            missing.read_xl(&bucket, object, false).await.is_err(),
1477            "obsolete admission must not publish a shard"
1478        );
1479        assert_eq!(
1480            healthy
1481                .read_all(&bucket, &format!("{object}/xl.meta"))
1482                .await
1483                .expect("healthy after stale heal"),
1484            baseline
1485        );
1486        store
1487            .heal_bucket_at_incarnation(&bucket, new, &opts)
1488            .await
1489            .expect("fresh bucket metadata repair");
1490        let result = store
1491            .heal_object_at_incarnation(&bucket, object, &version, new, &opts)
1492            .await
1493            .expect("new admission repair");
1494        assert!(result.error.is_none(), "new admission must repair independently: {:?}", result.error);
1495        assert!(
1496            missing.read_xl(&bucket, object, false).await.is_ok(),
1497            "new admission publishes the missing shard"
1498        );
1499        store
1500            .write_local_metadata_at_incarnation(&disk_ref, (&bucket, object), fi.clone(), new)
1501            .await
1502            .expect("fresh target metadata");
1503        store
1504            .delete_local_version_at_incarnation(&disk_ref, (&bucket, object), fi, false, DeleteOptions::default(), new)
1505            .await
1506            .expect("fresh target version cleanup");
1507        healthy
1508            .write_all(&bucket, "fresh-cleanup/data", bytes::Bytes::from_static(b"temporary"))
1509            .await
1510            .expect("cleanup fixture");
1511        store
1512            .delete_local_path_at_incarnation(
1513                &disk_ref,
1514                (&bucket, "fresh-cleanup"),
1515                DeleteOptions {
1516                    recursive: true,
1517                    immediate: true,
1518                    ..Default::default()
1519                },
1520                new,
1521            )
1522            .await
1523            .expect("fresh target path cleanup");
1524        shutdown.cancel();
1525    }
1526
1527    #[tokio::test]
1528    #[serial_test::serial]
1529    async fn bucket_incarnation_heal_owner_survives_cancelled_waiter_and_queued_writer() {
1530        let (_root, store, shutdown) = multi_pool_heal_store().await;
1531        let bucket = format!("heal-owner-{}", Uuid::new_v4().simple());
1532        store
1533            .make_bucket(&bucket, &MakeBucketOptions::default())
1534            .await
1535            .expect("bucket");
1536        let identity = store.bucket_incarnation_id_from_disk(&bucket).await.expect("identity");
1537        let (started_tx, started_rx) = tokio::sync::oneshot::channel();
1538        let (release_tx, release_rx) = tokio::sync::oneshot::channel();
1539        let (drained_tx, drained_rx) = tokio::sync::oneshot::channel();
1540        let worker_store = store.clone();
1541        let worker_bucket = bucket.clone();
1542        let waiter = tokio::spawn(async move {
1543            worker_store
1544                .run_bucket_heal_at_incarnation(
1545                    &worker_bucket,
1546                    identity,
1547                    &HealOpts::default(),
1548                    move |store, bucket, opts| async move {
1549                        started_tx.send(()).expect("announce held generation");
1550                        release_rx.await.expect("release commit");
1551                        let result = store
1552                            .heal_object_with_proof(&bucket, "history.txt", &Uuid::new_v4().to_string(), &opts)
1553                            .await?;
1554                        let proof = result.absence.expect("absence proof must reuse the held lifecycle owner");
1555                        assert_eq!(proof.bucket_incarnation_id, identity);
1556                        assert_eq!(proof.locations.len(), 2);
1557                        drained_tx.send(()).expect("announce drain");
1558                        Ok(())
1559                    },
1560                )
1561                .await
1562        });
1563        started_rx.await.expect("operation acquired generation fence");
1564        waiter.abort();
1565        assert!(waiter.await.expect_err("waiter aborted").is_cancelled());
1566        let writer_store = store.clone();
1567        let writer_bucket = bucket.clone();
1568        let mut writer = Box::pin(writer_store.acquire_bucket_lifecycle_write_lock(&writer_bucket));
1569        assert!(
1570            futures::poll!(&mut writer).is_pending(),
1571            "cancellation must not release the live operation's generation"
1572        );
1573        release_tx.send(()).expect("finish operation");
1574        tokio::time::timeout(std::time::Duration::from_secs(10), drained_rx)
1575            .await
1576            .expect("proof must not reacquire a lifecycle read lock behind the queued writer")
1577            .expect("physical operation drains");
1578        let guard = tokio::time::timeout(std::time::Duration::from_secs(10), writer)
1579            .await
1580            .expect("writer resumes after drain")
1581            .expect("lifecycle writer");
1582        drop(guard);
1583        shutdown.cancel();
1584    }
1585
1586    async fn minimal_heal_store() -> ECStore {
1587        ECStore {
1588            id: Uuid::new_v4(),
1589            disk_map: HashMap::new(),
1590            pools: vec![minimal_heal_pool(0).await, minimal_heal_pool(1).await],
1591            peer_sys: S3PeerSys {
1592                clients: Vec::new(),
1593                pools_count: 2,
1594            },
1595            pool_meta: RwLock::new(PoolMeta::default()),
1596            rebalance_meta: RwLock::new(None),
1597            decommission_cancelers: RwLock::new(Vec::new()),
1598            start_gate: Mutex::new(()),
1599            pool_meta_save_gate: Mutex::default(),
1600            ctx: crate::runtime::instance::bootstrap_ctx(),
1601            bucket_fence_registry: std::sync::Arc::default(),
1602        }
1603    }
1604
1605    async fn remove_pool_meta_shard(store: &ECStore, pool_idx: usize) -> DiskStore {
1606        let target_set = store.pools[pool_idx].get_disks_by_key(POOL_META_NAME);
1607        let missing_disk = target_set.disks.read().await[0]
1608            .clone()
1609            .expect("pool metadata fixture disk should be online");
1610        missing_disk
1611            .delete(
1612                RUSTFS_META_BUCKET,
1613                POOL_META_NAME,
1614                DeleteOptions {
1615                    recursive: true,
1616                    immediate: true,
1617                    ..Default::default()
1618                },
1619            )
1620            .await
1621            .expect("one pool metadata shard should be removable");
1622        assert!(
1623            missing_disk.read_xl(RUSTFS_META_BUCKET, POOL_META_NAME, false).await.is_err(),
1624            "pool metadata fixture must start with one missing shard"
1625        );
1626        missing_disk
1627    }
1628
1629    #[tokio::test]
1630    async fn heal_erasure_set_scopes_follow_requested_pool_and_set() {
1631        let store = minimal_heal_store().await;
1632
1633        assert_eq!(
1634            store
1635                .heal_erasure_set_scopes(&HealOpts::default())
1636                .await
1637                .expect("unscoped heal should enumerate every live set"),
1638            vec![(0, 0), (1, 0)]
1639        );
1640        assert_eq!(
1641            store
1642                .heal_erasure_set_scopes(&HealOpts {
1643                    pool: Some(1),
1644                    set: Some(0),
1645                    ..Default::default()
1646                })
1647                .await
1648                .expect("scoped heal should enumerate only its requested set"),
1649            vec![(1, 0)]
1650        );
1651
1652        let err = store
1653            .heal_erasure_set_scopes(&HealOpts {
1654                pool: Some(0),
1655                set: Some(1),
1656                ..Default::default()
1657            })
1658            .await
1659            .expect_err("an invalid set scope must fail closed");
1660        assert!(matches!(err, Error::InvalidArgument(..)));
1661    }
1662
1663    fn pool_meta_with_decommission(info: PoolDecommissionInfo) -> PoolMeta {
1664        PoolMeta {
1665            pools: vec![PoolStatus {
1666                id: 0,
1667                cmd_line: "pool-0".to_string(),
1668                last_update: OffsetDateTime::UNIX_EPOCH,
1669                decommission: Some(info),
1670            }],
1671            ..Default::default()
1672        }
1673    }
1674
1675    #[test]
1676    fn heal_format_pool_state_barriers_are_classified() {
1677        let active = pool_meta_with_decommission(PoolDecommissionInfo {
1678            start_time: Some(OffsetDateTime::UNIX_EPOCH),
1679            ..Default::default()
1680        });
1681        assert!(matches!(
1682            classify_heal_format_pool(0, "pool-0", &active, None),
1683            Some(HealFormatPoolSkip::Retryable)
1684        ));
1685
1686        for info in [
1687            PoolDecommissionInfo {
1688                failed: true,
1689                ..Default::default()
1690            },
1691            PoolDecommissionInfo {
1692                canceled: true,
1693                ..Default::default()
1694            },
1695        ] {
1696            assert!(matches!(
1697                classify_heal_format_pool(0, "pool-0", &pool_meta_with_decommission(info), None),
1698                Some(HealFormatPoolSkip::Retryable)
1699            ));
1700        }
1701
1702        let completed = pool_meta_with_decommission(PoolDecommissionInfo {
1703            complete: true,
1704            ..Default::default()
1705        });
1706        assert!(matches!(
1707            classify_heal_format_pool(0, "pool-0", &completed, None),
1708            Some(HealFormatPoolSkip::Completed)
1709        ));
1710    }
1711
1712    #[test]
1713    fn heal_format_pool_rebalance_barriers_and_identity_are_fail_closed() {
1714        let identity_meta = pool_meta_with_decommission(PoolDecommissionInfo::default());
1715        let rebalance = RebalanceMeta {
1716            pool_stats: vec![RebalanceStats {
1717                participating: true,
1718                info: RebalanceInfo {
1719                    status: RebalStatus::Started,
1720                    ..Default::default()
1721                },
1722                ..Default::default()
1723            }],
1724            ..Default::default()
1725        };
1726        assert!(matches!(
1727            classify_heal_format_pool(0, "pool-0", &identity_meta, Some(&rebalance)),
1728            Some(HealFormatPoolSkip::Retryable)
1729        ));
1730
1731        let stopping = RebalanceMeta {
1732            pool_stats: vec![RebalanceStats {
1733                info: RebalanceInfo {
1734                    stopping: true,
1735                    ..Default::default()
1736                },
1737                ..Default::default()
1738            }],
1739            ..Default::default()
1740        };
1741        assert!(matches!(
1742            classify_heal_format_pool(0, "pool-0", &identity_meta, Some(&stopping)),
1743            Some(HealFormatPoolSkip::Retryable)
1744        ));
1745
1746        let identity = pool_meta_with_decommission(PoolDecommissionInfo::default());
1747        assert!(matches!(
1748            classify_heal_format_pool(0, "pool-new", &identity, None),
1749            Some(HealFormatPoolSkip::Retryable)
1750        ));
1751
1752        let identity_without_decommission = PoolMeta {
1753            pools: vec![PoolStatus {
1754                id: 0,
1755                cmd_line: "pool-0".to_string(),
1756                last_update: OffsetDateTime::UNIX_EPOCH,
1757                decommission: None,
1758            }],
1759            ..Default::default()
1760        };
1761        assert!(matches!(
1762            classify_heal_format_pool(0, "pool-new", &identity_without_decommission, None),
1763            Some(HealFormatPoolSkip::Retryable)
1764        ));
1765
1766        assert!(matches!(
1767            classify_heal_format_pool(0, "", &identity_meta, None),
1768            Some(HealFormatPoolSkip::Retryable)
1769        ));
1770
1771        assert!(matches!(
1772            classify_heal_format_pool(0, "pool-0", &PoolMeta::default(), None),
1773            Some(HealFormatPoolSkip::Retryable)
1774        ));
1775
1776        let stopped = RebalanceMeta {
1777            stopped_at: Some(OffsetDateTime::UNIX_EPOCH),
1778            pool_stats: vec![RebalanceStats {
1779                participating: true,
1780                info: RebalanceInfo {
1781                    status: RebalStatus::Stopped,
1782                    ..Default::default()
1783                },
1784                ..Default::default()
1785            }],
1786            ..Default::default()
1787        };
1788        assert!(classify_heal_format_pool(0, "pool-0", &identity_meta, Some(&stopped)).is_none());
1789
1790        let stopping_after_stop = RebalanceMeta {
1791            stopped_at: Some(OffsetDateTime::UNIX_EPOCH),
1792            pool_stats: vec![RebalanceStats {
1793                participating: true,
1794                info: RebalanceInfo {
1795                    status: RebalStatus::Started,
1796                    stopping: true,
1797                    ..Default::default()
1798                },
1799                ..Default::default()
1800            }],
1801            ..Default::default()
1802        };
1803        assert!(matches!(
1804            classify_heal_format_pool(0, "pool-0", &identity_meta, Some(&stopping_after_stop)),
1805            Some(HealFormatPoolSkip::Retryable)
1806        ));
1807    }
1808
1809    #[test]
1810    fn skipped_heal_format_pool_is_never_reported_as_success() {
1811        assert!(matches!(
1812            heal_format_pool_skip_error(HealFormatPoolSkip::Retryable),
1813            StorageError::SlowDown
1814        ));
1815        assert!(matches!(
1816            heal_format_pool_skip_error(HealFormatPoolSkip::Completed),
1817            StorageError::NoHealRequired
1818        ));
1819    }
1820
1821    async fn multi_pool_heal_store() -> (tempfile::TempDir, Arc<ECStore>, CancellationToken) {
1822        let temp_dir = tempfile::tempdir().expect("multi-pool heal test directory should be created");
1823        let mut pool_endpoints = Vec::new();
1824        for pool_index in 0..2 {
1825            let mut endpoints = Vec::new();
1826            for disk_index in 0..4 {
1827                let disk_path = temp_dir.path().join(format!("pool{pool_index}-disk{disk_index}"));
1828                tokio::fs::create_dir_all(&disk_path)
1829                    .await
1830                    .expect("multi-pool heal test disk should be created");
1831                let mut endpoint = Endpoint::try_from(disk_path.to_str().expect("disk path should be utf8"))
1832                    .expect("test endpoint should parse");
1833                endpoint.set_pool_index(pool_index);
1834                endpoint.set_set_index(0);
1835                endpoint.set_disk_index(disk_index);
1836                endpoints.push(endpoint);
1837            }
1838            pool_endpoints.push(PoolEndpoints {
1839                legacy: false,
1840                set_count: 1,
1841                drives_per_set: 4,
1842                endpoints: Endpoints::from(endpoints),
1843                cmd_line: format!("heal-owner-pool-{pool_index}"),
1844                platform: "test".to_string(),
1845            });
1846        }
1847
1848        let endpoint_pools = EndpointServerPools::from(pool_endpoints);
1849        let instance_ctx = Arc::new(InstanceContext::new());
1850        init_local_disks_with_instance_ctx(&instance_ctx, endpoint_pools.clone())
1851            .await
1852            .expect("multi-pool local disks should initialize");
1853        let shutdown = CancellationToken::new();
1854        let store = ECStore::new_with_instance_ctx(
1855            "127.0.0.1:0".parse().expect("test address should parse"),
1856            endpoint_pools,
1857            shutdown.clone(),
1858            instance_ctx,
1859        )
1860        .await
1861        .expect("multi-pool test store should initialize");
1862        metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
1863        (temp_dir, store, shutdown)
1864    }
1865
1866    #[test]
1867    fn bucket_metadata_heal_owner_rejects_path_aliases_and_unrelated_records() {
1868        for file in [".metadata.bin", ".bucket-incarnation"] {
1869            assert_eq!(bucket_metadata_owner(&format!("buckets/example/{file}")), Some("example"));
1870        }
1871        for path in [
1872            "/buckets/example/.metadata.bin",
1873            "buckets/../.metadata.bin",
1874            "buckets/example/../.metadata.bin",
1875            "buckets/example/.metadata.bin/extra",
1876            "buckets/example//.metadata.bin",
1877            "buckets/example/.metadata.bin.old",
1878            "config/iam/example/.metadata.bin",
1879            "buckets/example/usage.json",
1880        ] {
1881            assert_eq!(bucket_metadata_owner(path), None, "unexpected bucket authority for {path}");
1882        }
1883    }
1884
1885    #[tokio::test]
1886    async fn replacement_bucket_metadata_respects_pool_ownership_and_required_records() {
1887        let (root, store, shutdown) = multi_pool_heal_store().await;
1888        let bucket = "replacement-bucket-metadata";
1889        for invalid_bucket in ["MixedCase", "../example", RUSTFS_META_BUCKET] {
1890            assert!(matches!(
1891                store
1892                    .heal_replacement_bucket_metadata(
1893                        invalid_bucket,
1894                        &HealOpts {
1895                            pool: Some(0),
1896                            set: Some(0),
1897                            ..Default::default()
1898                        },
1899                        &[root.path().join("pool0-disk0").to_string_lossy().into_owned()],
1900                    )
1901                    .await,
1902                Err(Error::PreconditionFailed)
1903            ));
1904        }
1905        store
1906            .make_bucket(bucket, &MakeBucketOptions::default())
1907            .await
1908            .expect("create bucket and its persisted records");
1909        let mut owned = 0;
1910        for pool in 0..2 {
1911            let target = root.path().join(format!("pool{pool}-disk0"));
1912            let records: Vec<_> = [".metadata.bin", ".bucket-incarnation"]
1913                .into_iter()
1914                .map(|file| {
1915                    let path = target.join(RUSTFS_META_BUCKET).join("buckets").join(bucket).join(file);
1916                    let exists = path.join("xl.meta").exists();
1917                    if exists {
1918                        std::fs::remove_dir_all(&path).expect("remove replacement metadata shard");
1919                        owned += 1;
1920                    }
1921                    (path, exists)
1922                })
1923                .collect();
1924            store
1925                .heal_replacement_bucket_metadata(
1926                    bucket,
1927                    &HealOpts {
1928                        pool: Some(pool),
1929                        set: Some(0),
1930                        recreate: true,
1931                        ..Default::default()
1932                    },
1933                    &[target.to_string_lossy().into_owned()],
1934                )
1935                .await
1936                .expect("repair owned records and accept authoritative ownership in the other pool");
1937            for (path, existed) in records {
1938                assert_eq!(path.join("xl.meta").exists(), existed, "wrong placement for {}", path.display());
1939            }
1940        }
1941        assert_eq!(owned, 2, "each required record must have exactly one owning pool");
1942        let object = format!("buckets/{bucket}/.metadata.bin");
1943        let owner = store
1944            .get_pool_idx_existing_with_opts(RUSTFS_META_BUCKET, &object, &ObjectOptions::default())
1945            .await
1946            .expect("persisted metadata owner");
1947        let disk = store.pools[owner].disk_set[0].disks.read().await[0]
1948            .clone()
1949            .expect("metadata target disk");
1950        let fi = disk
1951            .read_version(
1952                "",
1953                RUSTFS_META_BUCKET,
1954                &object,
1955                "",
1956                &disk::ReadOptions {
1957                    read_data: true,
1958                    ..Default::default()
1959                },
1960            )
1961            .await
1962            .expect("inline metadata to repair through target-side admission");
1963        assert!(fi.data.is_some(), "fixture must contain an inline metadata shard");
1964        let target_path = root.path().join(format!("pool{owner}-disk0"));
1965        std::fs::remove_dir_all(target_path.join(RUSTFS_META_BUCKET).join(&object)).expect("remove remote target shard");
1966        let incarnation = store
1967            .bucket_incarnation_id_from_disk(bucket)
1968            .await
1969            .expect("current incarnation");
1970        store
1971            .rename_local_data_at_incarnation(
1972                &disk.endpoint().to_string(),
1973                (disk::RUSTFS_META_TMP_BUCKET, "metadata-heal-control"),
1974                &fi,
1975                (RUSTFS_META_BUCKET, &object),
1976                incarnation,
1977            )
1978            .await
1979            .expect("the target-side RPC admission must accept current bucket metadata repair");
1980        assert!(target_path.join(RUSTFS_META_BUCKET).join(&object).join("xl.meta").exists());
1981        let (configuration, _) = read_config_no_lock_preserve_empty_with_metadata(store.clone(), &object)
1982            .await
1983            .expect("read the current bucket configuration");
1984        save_config(store.clone(), &object, configuration)
1985            .await
1986            .expect("publish a newer config revision in the same bucket incarnation");
1987        let committed =
1988            std::fs::read(target_path.join(RUSTFS_META_BUCKET).join(&object).join("xl.meta")).expect("new target configuration");
1989        assert!(
1990            store
1991                .rename_local_data_at_incarnation(
1992                    &disk.endpoint().to_string(),
1993                    (disk::RUSTFS_META_TMP_BUCKET, "delayed-metadata-heal"),
1994                    &fi,
1995                    (RUSTFS_META_BUCKET, &object),
1996                    incarnation,
1997                )
1998                .await
1999                .is_err(),
2000            "a delayed repair must not overwrite a newer config in the same incarnation"
2001        );
2002        assert_eq!(
2003            std::fs::read(target_path.join(RUSTFS_META_BUCKET).join(&object).join("xl.meta")).expect("retained configuration"),
2004            committed,
2005            "rejected repair must leave the newer configuration unchanged"
2006        );
2007        for pool in 0..2 {
2008            for disk in 0..4 {
2009                let path = root
2010                    .path()
2011                    .join(format!("pool{pool}-disk{disk}"))
2012                    .join(RUSTFS_META_BUCKET)
2013                    .join("buckets")
2014                    .join(bucket)
2015                    .join(".metadata.bin");
2016                if path.exists() {
2017                    std::fs::remove_dir_all(path).expect("remove every copy of required bucket configuration");
2018                }
2019            }
2020        }
2021        metadata_sys::remove_bucket_metadata_in(&store.ctx, bucket)
2022            .await
2023            .expect("remove the cached configuration so repair cannot rely on it");
2024        assert!(
2025            store
2026                .heal_replacement_bucket_metadata(
2027                    bucket,
2028                    &HealOpts {
2029                        pool: Some(0),
2030                        set: Some(0),
2031                        recreate: true,
2032                        ..Default::default()
2033                    },
2034                    &[root.path().join("pool0-disk0").to_string_lossy().into_owned()],
2035                )
2036                .await
2037                .is_err(),
2038            "total metadata loss must not be accepted as ownership in another pool"
2039        );
2040        for pool in 0..2 {
2041            assert!(
2042                !root
2043                    .path()
2044                    .join(format!("pool{pool}-disk0"))
2045                    .join(RUSTFS_META_BUCKET)
2046                    .join(&object)
2047                    .exists(),
2048                "missing required configuration must not be recreated with defaults"
2049            );
2050        }
2051        shutdown.cancel();
2052    }
2053
2054    #[tokio::test]
2055    #[serial_test::serial]
2056    async fn absence_proof_requires_every_selected_pool() {
2057        let (_temp_dir, store, shutdown) = multi_pool_heal_store().await;
2058        let bucket = format!("absence-scope-{}", Uuid::new_v4().simple());
2059        store
2060            .make_bucket(&bucket, &MakeBucketOptions::default())
2061            .await
2062            .expect("create bucket in both pools");
2063        let version = Uuid::new_v4().to_string();
2064        let result = store
2065            .heal_object_with_proof(&bucket, "history.txt", &version, &HealOpts::default())
2066            .await
2067            .expect("exact absence lookup should complete across both pools");
2068        assert!(result.error.is_none());
2069        let proof = result.absence.expect("all selected pools proved the exact version absent");
2070        assert_eq!(proof.locations, vec![(0, 0), (1, 0)]);
2071        assert_eq!((proof.pool_index, proof.set_index), (None, None));
2072        assert_eq!(proof.bucket, bucket);
2073        assert_eq!(proof.version_id, version);
2074        assert!(!proof.removed, "already absent versions do not count as another cleanup");
2075
2076        store.pool_meta.write().await.pools[1].decommission = Some(PoolDecommissionInfo {
2077            start_time: Some(OffsetDateTime::now_utc()),
2078            ..Default::default()
2079        });
2080        let partial = store
2081            .heal_object_with_proof(&bucket, "history.txt", &version, &HealOpts::default())
2082            .await
2083            .expect("unscoped heal should retain its legacy suspended-pool behavior");
2084        assert!(partial.absence.is_none(), "an uninspected suspended pool prevents global absence proof");
2085        shutdown.cancel();
2086    }
2087
2088    fn heal_test_format_path(temp_dir: &tempfile::TempDir, pool_index: usize, disk_index: usize) -> std::path::PathBuf {
2089        temp_dir
2090            .path()
2091            .join(format!("pool{pool_index}-disk{disk_index}"))
2092            .join(crate::disk::RUSTFS_META_BUCKET)
2093            .join(FORMAT_CONFIG_FILE)
2094    }
2095
2096    async fn remove_heal_test_format(
2097        temp_dir: &tempfile::TempDir,
2098        store: &ECStore,
2099        pool_index: usize,
2100        disk_index: usize,
2101    ) -> String {
2102        let endpoint = store.pools[pool_index].endpoints.endpoints.as_ref()[disk_index].clone();
2103        let target = endpoint.to_string();
2104        let format_path = heal_test_format_path(temp_dir, pool_index, disk_index);
2105        tokio::fs::remove_file(&format_path)
2106            .await
2107            .expect("replacement target format should be removable");
2108        assert!(
2109            !tokio::fs::try_exists(&format_path)
2110                .await
2111                .expect("replacement target format path should be inspectable")
2112        );
2113        let replacement = new_disk(
2114            &endpoint,
2115            &DiskOption {
2116                cleanup: false,
2117                health_check: false,
2118            },
2119        )
2120        .await
2121        .expect("unformatted replacement disk should open");
2122        let set = &store.pools[pool_index].disk_set[0];
2123        let target_slot = set
2124            .set_endpoints
2125            .iter()
2126            .position(|candidate| candidate == &endpoint)
2127            .expect("replacement endpoint should belong to its set");
2128        set.disks.write().await[target_slot] = Some(replacement);
2129        target
2130    }
2131
2132    async fn assert_heal_test_format_missing(temp_dir: &tempfile::TempDir, pool_index: usize, disk_index: usize) {
2133        assert!(
2134            !tokio::fs::try_exists(heal_test_format_path(temp_dir, pool_index, disk_index))
2135                .await
2136                .expect("replacement target format path should be inspectable"),
2137            "format heal must not write the replacement target"
2138        );
2139    }
2140
2141    #[tokio::test]
2142    #[serial_test::serial]
2143    async fn replacement_format_heal_coexists_with_capacity_read_fence() {
2144        let (temp_dir, store, shutdown) = multi_pool_heal_store().await;
2145        let target = remove_heal_test_format(&temp_dir, &store, 0, 3).await;
2146        let capacity_guard = store
2147            .acquire_external_decommission_capacity_fence(&[0], DecommissionCapacityAdmission::Heal)
2148            .await
2149            .expect("ordinary heal capacity fence should be acquired");
2150
2151        let (_, err) =
2152            tokio::time::timeout(std::time::Duration::from_secs(2), store.heal_replacement_format(false, 0, 0, &[target]))
2153                .await
2154                .expect("replacement format heal must not wait on an existing capacity read fence")
2155                .expect("replacement format heal should complete");
2156
2157        assert!(err.is_none(), "replacement format heal should succeed: {err:?}");
2158        assert!(
2159            tokio::fs::try_exists(heal_test_format_path(&temp_dir, 0, 3))
2160                .await
2161                .expect("replacement target format path should be inspectable"),
2162            "replacement format heal should restore the target while the capacity read fence is held"
2163        );
2164        drop(capacity_guard);
2165        shutdown.cancel();
2166    }
2167
2168    #[tokio::test]
2169    #[serial_test::serial]
2170    async fn replacement_format_heal_remains_blocked_by_pool_metadata_writer() {
2171        let (temp_dir, store, shutdown) = multi_pool_heal_store().await;
2172        let target = remove_heal_test_format(&temp_dir, &store, 0, 3).await;
2173        let pool_lock = store.pools[0]
2174            .new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME)
2175            .await
2176            .expect("pool metadata lock should be created");
2177        let pool_writer = pool_lock
2178            .get_write_lock(get_lock_acquire_timeout())
2179            .await
2180            .expect("pool metadata writer should be acquired");
2181
2182        assert!(
2183            tokio::time::timeout(
2184                std::time::Duration::from_secs(1),
2185                store.heal_replacement_format(false, 0, 0, std::slice::from_ref(&target)),
2186            )
2187            .await
2188            .is_err(),
2189            "replacement format heal must wait for a pool metadata writer"
2190        );
2191        assert_heal_test_format_missing(&temp_dir, 0, 3).await;
2192
2193        drop(pool_writer);
2194        let (_, err) =
2195            tokio::time::timeout(std::time::Duration::from_secs(30), store.heal_replacement_format(false, 0, 0, &[target]))
2196                .await
2197                .expect("replacement format heal should resume after the writer releases")
2198                .expect("replacement format heal should complete after the writer releases");
2199        assert!(err.is_none(), "replacement format heal should succeed: {err:?}");
2200        assert!(
2201            tokio::fs::try_exists(heal_test_format_path(&temp_dir, 0, 3))
2202                .await
2203                .expect("replacement target format path should be inspectable"),
2204            "replacement format heal should restore the target after the writer releases"
2205        );
2206        shutdown.cancel();
2207    }
2208
2209    #[tokio::test]
2210    #[serial_test::serial]
2211    async fn full_format_heal_preblocked_pool_metadata_never_writes_format() {
2212        let (temp_dir, store, shutdown) = multi_pool_heal_store().await;
2213        remove_heal_test_format(&temp_dir, &store, 0, 3).await;
2214        store.pool_meta_save_gate.lock().await.observe_replicas(PoolMetaReplicaState {
2215            needs_repair: true,
2216            repair_write_safe: false,
2217        });
2218
2219        let err = store
2220            .handle_heal_format(false)
2221            .await
2222            .expect_err("a preblocked pool metadata state must reject full format heal");
2223        assert!(
2224            err.to_string()
2225                .contains("restart after all replicas are readable and consistent")
2226        );
2227        assert_heal_test_format_missing(&temp_dir, 0, 3).await;
2228        store
2229            .ensure_pool_meta_side_effects_safe("preblocked format heal side effect")
2230            .await
2231            .expect_err("the preblocked state must remain sticky");
2232        shutdown.cancel();
2233    }
2234
2235    #[tokio::test]
2236    #[serial_test::serial]
2237    async fn full_format_heal_future_identity_never_writes_format_and_latches() {
2238        let (temp_dir, store, shutdown) = multi_pool_heal_store().await;
2239        remove_heal_test_format(&temp_dir, &store, 0, 3).await;
2240        let (mut future_identity, _) =
2241            read_config_no_lock_preserve_empty_with_metadata(store.pools[1].clone(), POOL_META_IDENTITY_NAME)
2242                .await
2243                .expect("current identity should be readable");
2244        let future_version = u16::from_le_bytes([future_identity[2], future_identity[3]])
2245            .checked_add(1)
2246            .expect("identity version should have a future value");
2247        future_identity[2..4].copy_from_slice(&future_version.to_le_bytes());
2248        save_config(store.pools[1].clone(), POOL_META_IDENTITY_NAME, future_identity)
2249            .await
2250            .expect("future identity should be persisted");
2251
2252        let err = store
2253            .handle_heal_format(false)
2254            .await
2255            .expect_err("a future identity must reject full format heal");
2256        assert!(err.to_string().contains("pool metadata incompatible"));
2257        assert_heal_test_format_missing(&temp_dir, 0, 3).await;
2258        store
2259            .ensure_pool_meta_side_effects_safe("future identity format heal side effect")
2260            .await
2261            .expect_err("the future identity rejection must latch the write gate");
2262        shutdown.cancel();
2263    }
2264
2265    #[tokio::test]
2266    #[serial_test::serial]
2267    async fn replacement_format_heal_epoch_conflict_never_writes_format_and_latches() {
2268        let (temp_dir, store, shutdown) = multi_pool_heal_store().await;
2269        let target = remove_heal_test_format(&temp_dir, &store, 0, 3).await;
2270        let conflicting_identity =
2271            initialized_pool_meta_identity_for_test(store.id, 2).expect("conflicting identity should encode");
2272        save_config(store.pools[1].clone(), POOL_META_IDENTITY_NAME, conflicting_identity)
2273            .await
2274            .expect("conflicting identity should be persisted");
2275
2276        let err = store
2277            .heal_replacement_format(false, 0, 0, &[target])
2278            .await
2279            .expect_err("an identity epoch conflict must reject replacement format heal");
2280        assert!(
2281            err.to_string()
2282                .contains("identity replicas disagree on cluster identity or epoch")
2283        );
2284        assert_heal_test_format_missing(&temp_dir, 0, 3).await;
2285        store
2286            .ensure_pool_meta_side_effects_safe("epoch conflict replacement format side effect")
2287            .await
2288            .expect_err("the identity epoch conflict must latch the write gate");
2289        shutdown.cancel();
2290    }
2291
2292    #[tokio::test]
2293    #[serial_test::serial]
2294    async fn replacement_format_heal_initialized_identity_without_pool_meta_never_writes_and_latches() {
2295        let (temp_dir, store, shutdown) = multi_pool_heal_store().await;
2296        let target = remove_heal_test_format(&temp_dir, &store, 0, 3).await;
2297        for pool in &store.pools {
2298            delete_config(pool.clone(), POOL_META_NAME)
2299                .await
2300                .expect("pool metadata replica should be removable");
2301        }
2302
2303        let err = store
2304            .heal_replacement_format(false, 0, 0, &[target])
2305            .await
2306            .expect_err("initialized identity without pool metadata must reject replacement format heal");
2307        assert!(err.to_string().contains("initialized cluster identity exists"));
2308        assert_heal_test_format_missing(&temp_dir, 0, 3).await;
2309        store
2310            .ensure_pool_meta_side_effects_safe("missing pool metadata replacement format side effect")
2311            .await
2312            .expect_err("missing initialized pool metadata must latch the write gate");
2313        shutdown.cancel();
2314    }
2315
2316    #[tokio::test]
2317    #[serial_test::serial]
2318    async fn full_format_heal_lost_after_last_save_does_not_publish_or_renew_and_latches() {
2319        let (temp_dir, store, shutdown) = multi_pool_heal_store().await;
2320        let target = remove_heal_test_format(&temp_dir, &store, 0, 3).await;
2321        store.pools[0].disk_set[0].disks.write().await[3] = None;
2322        let barrier = HealFormatAfterSaveBarrier::install(&store.pools[0], 3);
2323        let recovery_latch = store.pool_meta_save_gate.lock().await.aborted_transaction_latch_for_test();
2324        let mut heal = tokio::spawn({
2325            let store = Arc::clone(&store);
2326            async move { store.handle_heal_format(false).await }
2327        });
2328
2329        barrier.wait_until_paused().await;
2330        let saved = tokio::fs::read(heal_test_format_path(&temp_dir, 0, 3))
2331            .await
2332            .expect("the last replacement format must be durable before fence loss");
2333        let saved = FormatV3::try_from(saved.as_slice()).expect("the durable replacement format should decode");
2334        assert_eq!(saved.erasure.this, store.pools[0].format.erasure.sets[0][3]);
2335        recovery_latch.store(true, std::sync::atomic::Ordering::SeqCst);
2336        barrier.release();
2337
2338        let (result, err) = tokio::time::timeout(std::time::Duration::from_secs(30), &mut heal)
2339            .await
2340            .expect("format heal should stop after the lost fence")
2341            .expect("format heal task should not panic")
2342            .expect("format heal should return its fenced result");
2343        assert!(matches!(err, Some(StorageError::SlowDown)));
2344        assert!(
2345            result
2346                .after
2347                .drives
2348                .iter()
2349                .any(|drive| drive.endpoint == target && drive.state == DriveState::Missing.to_string()),
2350            "the lost fence must prevent the durable format from being published in the heal result"
2351        );
2352        assert!(
2353            store.pools[0].disk_set[0].disks.read().await[3].is_none(),
2354            "the lost fence must prevent renew_disk from attaching the replacement"
2355        );
2356        recovery_latch.store(false, std::sync::atomic::Ordering::SeqCst);
2357        store
2358            .ensure_pool_meta_side_effects_safe("post-save format fence loss side effect")
2359            .await
2360            .expect_err("post-save format fence loss must remain sticky");
2361        shutdown.cancel();
2362    }
2363
2364    #[tokio::test]
2365    async fn heal_object_pool_scope_selects_only_requested_pool() {
2366        let store = minimal_heal_store().await;
2367        let pools = store
2368            .get_pools_for_heal_object(&HealOpts {
2369                pool: Some(1),
2370                ..Default::default()
2371            })
2372            .expect("requested pool should be selected");
2373
2374        assert_eq!(pools.len(), 1);
2375        assert!(Arc::ptr_eq(&pools[0], &store.pools[1]));
2376    }
2377
2378    #[tokio::test]
2379    async fn heal_object_pool_scope_rejects_invalid_pool() {
2380        let store = minimal_heal_store().await;
2381        let err = store
2382            .get_pools_for_heal_object(&HealOpts {
2383                pool: Some(2),
2384                ..Default::default()
2385            })
2386            .expect_err("out-of-range pool scope must fail closed");
2387
2388        assert!(
2389            matches!(err, StorageError::InvalidArgument(_, ref field, ref reason)
2390                if field == "pool" && reason.contains("invalid heal pool index 2 for 2 pools")),
2391            "unexpected invalid pool error: {err:?}"
2392        );
2393    }
2394
2395    #[tokio::test]
2396    #[serial_test::serial]
2397    async fn pool_meta_read_repair_reuses_its_write_fence() {
2398        let (_temp_dirs, store, _other_store) = test_two_pool_stores(None).await;
2399        let missing_disk = remove_pool_meta_shard(&store, 0).await;
2400
2401        let (result, err) = tokio::time::timeout(
2402            std::time::Duration::from_secs(30),
2403            store.handle_heal_object(
2404                RUSTFS_META_BUCKET,
2405                POOL_META_NAME,
2406                "",
2407                &HealOpts {
2408                    read_repair: true,
2409                    pool: Some(0),
2410                    ..Default::default()
2411                },
2412            ),
2413        )
2414        .await
2415        .expect("pool metadata read repair must not wait on its own capacity fence")
2416        .expect("pool metadata read repair should complete");
2417
2418        assert!(err.is_none(), "pool metadata read repair should succeed: {err:?}");
2419        assert_eq!(result.object, POOL_META_NAME);
2420        assert!(
2421            missing_disk.read_xl(RUSTFS_META_BUCKET, POOL_META_NAME, false).await.is_ok(),
2422            "pool metadata read repair should restore the missing shard"
2423        );
2424    }
2425
2426    #[tokio::test]
2427    #[serial_test::serial]
2428    async fn pool_meta_heal_preserves_typed_lock_timeout() {
2429        let (_temp_dirs, store, _other_store) = test_two_pool_stores(None).await;
2430        let lock = store.pools[0]
2431            .new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME)
2432            .await
2433            .expect("pool metadata lock should be created");
2434        let guard = lock
2435            .get_write_lock(get_lock_acquire_timeout())
2436            .await
2437            .expect("pool metadata lock should be acquired");
2438
2439        let (_, err) = temp_env::async_with_vars(
2440            [(rustfs_config::ENV_OBJECT_LOCK_ACQUIRE_TIMEOUT, Some("1"))],
2441            store.handle_heal_object(
2442                RUSTFS_META_BUCKET,
2443                POOL_META_NAME,
2444                "",
2445                &HealOpts {
2446                    pool: Some(0),
2447                    ..Default::default()
2448                },
2449            ),
2450        )
2451        .await
2452        .expect("pool metadata lock timeout should be mapped into the heal result");
2453
2454        assert!(
2455            matches!(err, Some(Error::Lock(rustfs_lock::LockError::Timeout { .. }))),
2456            "pool metadata heal must preserve the recoverable lock timeout: {err:?}"
2457        );
2458        drop(guard);
2459    }
2460
2461    #[tokio::test]
2462    #[serial_test::serial]
2463    async fn pool_meta_neighbor_keeps_ordinary_object_locking() {
2464        let (_temp_dirs, store, _other_store) = test_two_pool_stores(None).await;
2465        let object = "pool.bin.backup";
2466        save_config(store.pools[0].clone(), object, b"neighbor metadata".to_vec())
2467            .await
2468            .expect("neighbor metadata fixture should be written");
2469
2470        let target_set = store.pools[0].get_disks_by_key(object);
2471        let lock = target_set
2472            .new_ns_lock(RUSTFS_META_BUCKET, object)
2473            .await
2474            .expect("neighbor metadata lock should be created");
2475        let guard = lock
2476            .get_write_lock(get_lock_acquire_timeout())
2477            .await
2478            .expect("neighbor metadata lock should be acquired");
2479        let barrier = DecommissionCapacityLockOrderBarrier::install(store.id, store.id);
2480        let heal_store = Arc::clone(&store);
2481        let mut heal = tokio::spawn(async move {
2482            heal_store
2483                .handle_heal_object(
2484                    RUSTFS_META_BUCKET,
2485                    object,
2486                    "",
2487                    &HealOpts {
2488                        pool: Some(0),
2489                        ..Default::default()
2490                    },
2491                )
2492                .await
2493        });
2494
2495        barrier.wait_until_external_heal_target_lock_attempted().await;
2496        tokio::task::yield_now().await;
2497        assert!(!heal.is_finished(), "neighbor metadata heal must retain ordinary object locking");
2498        drop(guard);
2499
2500        let (result, err) = tokio::time::timeout(std::time::Duration::from_secs(30), &mut heal)
2501            .await
2502            .expect("neighbor metadata heal should finish after the object lock is released")
2503            .expect("neighbor metadata heal task should not panic")
2504            .expect("neighbor metadata heal should complete");
2505        assert!(err.is_none(), "neighbor metadata heal should succeed: {err:?}");
2506        assert_eq!(result.object, object);
2507        drop(barrier);
2508    }
2509
2510    #[tokio::test]
2511    #[serial_test::serial]
2512    async fn targeted_heal_is_blocked_by_exact_fit_decommission_reservation() {
2513        let (_temp_dirs, store, _other_store) = test_two_pool_stores(None).await;
2514        let bucket = format!("heal-capacity-{}", Uuid::new_v4().simple());
2515        let object = "targeted-heal.bin";
2516        store
2517            .make_bucket(&bucket, &MakeBucketOptions::default())
2518            .await
2519            .expect("heal capacity bucket should be created");
2520
2521        let target_set = store.pools[1].get_disks(0);
2522        let mut reader = PutObjReader::from_vec(b"targeted heal body".to_vec());
2523        target_set
2524            .put_object(&bucket, object, &mut reader, &ObjectOptions::default())
2525            .await
2526            .expect("targeted heal fixture should be written");
2527        let missing_disk = target_set.disks.read().await[0]
2528            .clone()
2529            .expect("targeted heal fixture disk should be online");
2530        missing_disk
2531            .delete(
2532                &bucket,
2533                object,
2534                DeleteOptions {
2535                    recursive: true,
2536                    immediate: true,
2537                    ..Default::default()
2538                },
2539            )
2540            .await
2541            .expect("targeted heal fixture shard should be removed");
2542        assert!(
2543            missing_disk.read_xl(&bucket, object, false).await.is_err(),
2544            "targeted heal fixture must start with one missing metadata copy"
2545        );
2546
2547        let layout = DecommissionErasureLayout { data: 1, parity: 0 };
2548        set_decommission_capacity_info_overrides_for_test(
2549            store.id,
2550            vec![vec![
2551                DecommissionPoolCapacityInfo::for_test(0, layout, 0, 30, 30),
2552                DecommissionPoolCapacityInfo::for_test(1, layout, 60, 60, 0),
2553            ]],
2554        );
2555        store
2556            .save_current_pool_meta_for_decommission_start(&[0], Vec::new())
2557            .await
2558            .expect("the exact-fit decommission reservation should activate");
2559        {
2560            let pool_meta = store.pool_meta.read().await;
2561            let reservation = pool_meta.pools[0]
2562                .decommission
2563                .as_ref()
2564                .and_then(|info| info.capacity_reservation.as_ref())
2565                .expect("the active decommission reservation should be durable");
2566            assert_eq!(reservation.targets.len(), 1);
2567            assert_eq!(reservation.targets[0].pool_index, 1);
2568            assert_eq!(reservation.targets[0].reserved_physical_bytes, 60);
2569        }
2570
2571        let (_, err) = store
2572            .handle_heal_object(
2573                &bucket,
2574                object,
2575                "",
2576                &HealOpts {
2577                    pool: Some(1),
2578                    set: Some(0),
2579                    ..Default::default()
2580                },
2581            )
2582            .await
2583            .expect("targeted heal should return a mapped capacity result");
2584        assert!(matches!(err, Some(Error::SlowDown)), "targeted heal must be capacity-blocked: {err:?}");
2585        assert!(
2586            missing_disk.read_xl(&bucket, object, false).await.is_err(),
2587            "capacity-blocked targeted heal must not rewrite the missing shard"
2588        );
2589
2590        let (_, err) = tokio::time::timeout(
2591            std::time::Duration::from_secs(30),
2592            store.handle_heal_object(
2593                RUSTFS_META_BUCKET,
2594                POOL_META_NAME,
2595                "",
2596                &HealOpts {
2597                    pool: Some(1),
2598                    ..Default::default()
2599                },
2600            ),
2601        )
2602        .await
2603        .expect("pool metadata admission should not recurse on its namespace lock")
2604        .expect("pool metadata capacity rejection should be mapped");
2605        assert!(
2606            matches!(err, Some(Error::SlowDown)),
2607            "pool metadata heal must preserve target reservation admission: {err:?}"
2608        );
2609    }
2610
2611    #[tokio::test]
2612    #[serial_test::serial]
2613    async fn pool_meta_heal_keeps_per_target_capacity_admission() {
2614        let (_temp_dirs, store, _other_store) = test_three_pool_stores_with_isolated_node_contexts(None).await;
2615        let layout = DecommissionErasureLayout { data: 1, parity: 0 };
2616        set_decommission_capacity_info_overrides_for_test(
2617            store.id,
2618            vec![vec![
2619                DecommissionPoolCapacityInfo::for_test(0, layout, 0, 30, 30),
2620                DecommissionPoolCapacityInfo::for_test(1, layout, 60, 60, 0),
2621                DecommissionPoolCapacityInfo::for_test(2, layout, 60, 60, 0),
2622            ]],
2623        );
2624        store
2625            .save_current_pool_meta_for_decommission_start(&[0], Vec::new())
2626            .await
2627            .expect("pool metadata reservation should activate");
2628        {
2629            let pool_meta = store.pool_meta.read().await;
2630            let reservation = pool_meta.pools[0]
2631                .decommission
2632                .as_ref()
2633                .and_then(|info| info.capacity_reservation.as_ref())
2634                .expect("pool metadata reservation should be durable");
2635            assert_eq!(reservation.targets.len(), 1);
2636            assert_eq!(reservation.targets[0].pool_index, 1);
2637        }
2638
2639        let missing_disk = remove_pool_meta_shard(&store, 2).await;
2640
2641        let (result, err) = tokio::time::timeout(
2642            std::time::Duration::from_secs(30),
2643            store.handle_heal_object(RUSTFS_META_BUCKET, POOL_META_NAME, "", &HealOpts::default()),
2644        )
2645        .await
2646        .expect("unscoped pool metadata heal should complete")
2647        .expect("unscoped pool metadata heal should return a mapped result");
2648
2649        assert!(err.is_none(), "an admitted target should let unscoped metadata heal succeed: {err:?}");
2650        assert_eq!(result.object, POOL_META_NAME);
2651        assert!(
2652            missing_disk.read_xl(RUSTFS_META_BUCKET, POOL_META_NAME, false).await.is_ok(),
2653            "unscoped metadata heal should repair the admitted target while another target is reserved"
2654        );
2655    }
2656
2657    #[tokio::test]
2658    #[serial_test::serial]
2659    async fn targeted_heal_keeps_target_lock_for_different_lock_domain() {
2660        let (_temp_dirs, store, other_store) = test_three_pool_stores_with_isolated_node_contexts(None).await;
2661        let bucket = format!("heal-lock-domain-{}", Uuid::new_v4().simple());
2662        let object = "different-domain-heal.bin";
2663        store
2664            .make_bucket(&bucket, &MakeBucketOptions::default())
2665            .await
2666            .expect("heal lock-domain bucket should be created");
2667
2668        let target_set = other_store.pools[1].get_disks(0);
2669        let mut reader = PutObjReader::from_vec(b"different lock domain body".to_vec());
2670        target_set
2671            .put_object(&bucket, object, &mut reader, &ObjectOptions::default())
2672            .await
2673            .expect("heal lock-domain fixture should be written");
2674        let missing_disk = target_set.disks.read().await[0]
2675            .clone()
2676            .expect("heal lock-domain fixture disk should be online");
2677        missing_disk
2678            .delete(
2679                &bucket,
2680                object,
2681                DeleteOptions {
2682                    recursive: true,
2683                    immediate: true,
2684                    ..Default::default()
2685                },
2686            )
2687            .await
2688            .expect("heal lock-domain fixture shard should be removed");
2689        assert!(
2690            missing_disk.read_xl(&bucket, object, false).await.is_err(),
2691            "heal lock-domain fixture must start with one missing metadata copy"
2692        );
2693
2694        let layout = DecommissionErasureLayout { data: 1, parity: 0 };
2695        set_decommission_capacity_info_overrides_for_test(
2696            store.id,
2697            vec![vec![
2698                DecommissionPoolCapacityInfo::for_test(0, layout, 0, 30, 30),
2699                DecommissionPoolCapacityInfo::for_test(1, layout, 0, 100, 100),
2700                DecommissionPoolCapacityInfo::for_test(2, layout, 60, 60, 0),
2701            ]],
2702        );
2703        store
2704            .save_current_pool_meta_for_decommission_start(&[0], Vec::new())
2705            .await
2706            .expect("heal lock-domain reservation should activate");
2707        {
2708            let pool_meta = store.pool_meta.read().await;
2709            let reservation = pool_meta.pools[0]
2710                .decommission
2711                .as_ref()
2712                .and_then(|info| info.capacity_reservation.as_ref())
2713                .expect("heal lock-domain reservation should be durable");
2714            assert_eq!(reservation.targets.len(), 1);
2715            assert_eq!(reservation.targets[0].pool_index, 2);
2716        }
2717        let pool_meta = store.pool_meta.read().await.clone();
2718        *other_store.pool_meta.write().await = pool_meta;
2719
2720        let fixed_set = other_store.pools[0].disk_set[0].clone();
2721        assert!(
2722            !fixed_set.shares_namespace_lock_domain(&target_set).await,
2723            "heal fixture must use different fixed and target lock domains"
2724        );
2725        let barrier = DecommissionCapacityLockOrderBarrier::install(store.id, other_store.id);
2726        let heal_store = Arc::clone(&other_store);
2727        let heal_bucket = bucket.clone();
2728        let heal_object = object.to_string();
2729        let mut heal = tokio::spawn(async move {
2730            heal_store
2731                .handle_heal_object(
2732                    &heal_bucket,
2733                    &heal_object,
2734                    "",
2735                    &HealOpts {
2736                        pool: Some(1),
2737                        set: Some(0),
2738                        ..Default::default()
2739                    },
2740                )
2741                .await
2742        });
2743
2744        barrier.wait_until_external_capacity_released().await;
2745        barrier.wait_until_external_heal_target_lock_attempted().await;
2746
2747        let (_, err) = tokio::time::timeout(std::time::Duration::from_secs(30), &mut heal)
2748            .await
2749            .expect("targeted heal should finish after target lock release")
2750            .expect("targeted heal task should not panic")
2751            .expect("targeted heal should complete");
2752        assert!(err.is_none(), "targeted heal should repair after target lock release: {err:?}");
2753        assert!(
2754            missing_disk.read_xl(&bucket, object, false).await.is_ok(),
2755            "targeted heal should rewrite the missing shard after the target lock is released"
2756        );
2757        drop(barrier);
2758    }
2759
2760    #[tokio::test]
2761    #[serial_test::serial]
2762    async fn targeted_heal_reuses_shared_target_lock_without_reentrant_lock() {
2763        let (_temp_dirs, store, _other_store) = test_three_pool_stores_with_isolated_node_contexts(None).await;
2764        let bucket = format!("heal-shared-lock-domain-{}", Uuid::new_v4().simple());
2765        let object = "shared-domain-heal.bin";
2766        store
2767            .make_bucket(&bucket, &MakeBucketOptions::default())
2768            .await
2769            .expect("shared heal lock-domain bucket should be created");
2770
2771        let target_set = store.pools[0].get_disks(0);
2772        let mut reader = PutObjReader::from_vec(b"shared lock domain body".to_vec());
2773        target_set
2774            .put_object(&bucket, object, &mut reader, &ObjectOptions::default())
2775            .await
2776            .expect("shared heal lock-domain fixture should be written");
2777        let missing_disk = target_set.disks.read().await[0]
2778            .clone()
2779            .expect("shared heal lock-domain fixture disk should be online");
2780        missing_disk
2781            .delete(
2782                &bucket,
2783                object,
2784                DeleteOptions {
2785                    recursive: true,
2786                    immediate: true,
2787                    ..Default::default()
2788                },
2789            )
2790            .await
2791            .expect("shared heal lock-domain fixture shard should be removed");
2792        assert!(
2793            missing_disk.read_xl(&bucket, object, false).await.is_err(),
2794            "shared heal lock-domain fixture must start with one missing metadata copy"
2795        );
2796
2797        let layout = DecommissionErasureLayout { data: 1, parity: 0 };
2798        set_decommission_capacity_info_overrides_for_test(
2799            store.id,
2800            vec![vec![
2801                DecommissionPoolCapacityInfo::for_test(0, layout, 0, 100, 100),
2802                DecommissionPoolCapacityInfo::for_test(1, layout, 60, 60, 0),
2803                DecommissionPoolCapacityInfo::for_test(2, layout, 0, 30, 30),
2804            ]],
2805        );
2806        store
2807            .save_current_pool_meta_for_decommission_start(&[2], Vec::new())
2808            .await
2809            .expect("shared heal lock-domain reservation should activate");
2810        {
2811            let pool_meta = store.pool_meta.read().await;
2812            let reservation = pool_meta.pools[2]
2813                .decommission
2814                .as_ref()
2815                .and_then(|info| info.capacity_reservation.as_ref())
2816                .expect("shared heal lock-domain reservation should be durable");
2817            assert_eq!(reservation.targets.len(), 1);
2818            assert_eq!(reservation.targets[0].pool_index, 1);
2819            assert_eq!(reservation.targets[0].reserved_physical_bytes, 60);
2820        }
2821
2822        let fixed_set = store.pools[0].disk_set[0].clone();
2823        assert!(
2824            fixed_set.shares_namespace_lock_domain(&target_set).await,
2825            "shared heal fixture must use one fixed and target lock domain"
2826        );
2827
2828        let barrier = DecommissionCapacityLockOrderBarrier::install(store.id, store.id);
2829        let heal_store = Arc::clone(&store);
2830        let heal_bucket = bucket.clone();
2831        let heal_object = object.to_string();
2832        let mut heal = tokio::spawn(async move {
2833            heal_store
2834                .handle_heal_object(
2835                    &heal_bucket,
2836                    &heal_object,
2837                    "",
2838                    &HealOpts {
2839                        pool: Some(0),
2840                        set: Some(0),
2841                        ..Default::default()
2842                    },
2843                )
2844                .await
2845        });
2846
2847        barrier.wait_until_external_capacity_released().await;
2848        barrier.wait_until_external_heal_operation_started().await;
2849        let (_, err) = tokio::time::timeout(std::time::Duration::from_secs(5), &mut heal)
2850            .await
2851            .expect("shared-domain heal must not reenter the fixed namespace lock")
2852            .expect("shared-domain heal task should not panic")
2853            .expect("shared-domain heal should complete");
2854        assert!(err.is_none(), "shared-domain heal should repair after admission: {err:?}");
2855        assert!(
2856            missing_disk.read_xl(&bucket, object, false).await.is_ok(),
2857            "shared-domain heal should rewrite the missing shard"
2858        );
2859        drop(barrier);
2860    }
2861
2862    #[tokio::test]
2863    async fn scoped_heal_object_defers_when_requested_pool_is_suspended() {
2864        let mut store = minimal_heal_store().await;
2865        store.pool_meta = RwLock::new(PoolMeta {
2866            pools: vec![
2867                PoolStatus {
2868                    id: 0,
2869                    cmd_line: "pool-0".to_string(),
2870                    last_update: OffsetDateTime::UNIX_EPOCH,
2871                    decommission: None,
2872                },
2873                PoolStatus {
2874                    id: 1,
2875                    cmd_line: "pool-1".to_string(),
2876                    last_update: OffsetDateTime::UNIX_EPOCH,
2877                    decommission: Some(PoolDecommissionInfo {
2878                        start_time: Some(OffsetDateTime::UNIX_EPOCH),
2879                        ..Default::default()
2880                    }),
2881                },
2882            ],
2883            ..Default::default()
2884        });
2885
2886        let (_, err) = store
2887            .handle_heal_object(
2888                "bucket",
2889                "object",
2890                "",
2891                &HealOpts {
2892                    pool: Some(1),
2893                    set: Some(0),
2894                    ..Default::default()
2895                },
2896            )
2897            .await
2898            .expect("suspended pool should return a deferred heal result");
2899
2900        assert!(matches!(err, Some(StorageError::SlowDown)));
2901
2902        let (_, err) = store
2903            .handle_heal_object(
2904                "bucket",
2905                "object",
2906                "",
2907                &HealOpts {
2908                    set: Some(1),
2909                    ..Default::default()
2910                },
2911            )
2912            .await
2913            .expect("unscoped heal should return the active pool result");
2914
2915        assert!(matches!(err, Some(StorageError::InvalidArgument(_, ref field, _)) if field == "set"));
2916
2917        let err = store
2918            .handle_heal_object(
2919                "bucket",
2920                "object",
2921                "",
2922                &HealOpts {
2923                    pool: Some(1),
2924                    set: Some(1),
2925                    ..Default::default()
2926                },
2927            )
2928            .await
2929            .expect_err("invalid set scope should fail before suspended pool deferral");
2930
2931        assert!(matches!(err, StorageError::InvalidArgument(_, ref field, _) if field == "set"));
2932
2933        {
2934            let mut pool_meta = store.pool_meta.write().await;
2935            let decommission = pool_meta.pools[1]
2936                .decommission
2937                .as_mut()
2938                .expect("test pool should have decommission state");
2939            decommission.complete = true;
2940        }
2941        let (_, err) = store
2942            .handle_heal_object(
2943                "bucket",
2944                "object",
2945                "",
2946                &HealOpts {
2947                    pool: Some(1),
2948                    set: Some(0),
2949                    ..Default::default()
2950                },
2951            )
2952            .await
2953            .expect("completed pool should return a terminal heal result");
2954
2955        assert!(matches!(
2956            err,
2957            Some(StorageError::InvalidArgument(_, ref field, ref reason))
2958                if field == "pool" && reason.contains("completed decommission")
2959        ));
2960
2961        for canceled in [false, true] {
2962            {
2963                let mut pool_meta = store.pool_meta.write().await;
2964                let decommission = pool_meta.pools[1]
2965                    .decommission
2966                    .as_mut()
2967                    .expect("test pool should have decommission state");
2968                decommission.complete = false;
2969                decommission.failed = !canceled;
2970                decommission.canceled = canceled;
2971            }
2972            let (_, err) = store
2973                .handle_heal_object(
2974                    "bucket",
2975                    "object",
2976                    "",
2977                    &HealOpts {
2978                        pool: Some(1),
2979                        set: Some(0),
2980                        ..Default::default()
2981                    },
2982                )
2983                .await
2984                .expect("clearable terminal pool should return a deferred heal result");
2985
2986            assert!(matches!(err, Some(StorageError::SlowDown)));
2987        }
2988    }
2989
2990    #[tokio::test]
2991    async fn scoped_heal_bucket_blocks_before_dispatch_when_pool_is_suspended() {
2992        let mut store = minimal_heal_store().await;
2993        store.pool_meta = RwLock::new(PoolMeta {
2994            pools: vec![
2995                PoolStatus {
2996                    id: 0,
2997                    cmd_line: "pool-0".to_string(),
2998                    last_update: OffsetDateTime::UNIX_EPOCH,
2999                    decommission: None,
3000                },
3001                PoolStatus {
3002                    id: 1,
3003                    cmd_line: "pool-1".to_string(),
3004                    last_update: OffsetDateTime::UNIX_EPOCH,
3005                    decommission: Some(PoolDecommissionInfo {
3006                        start_time: Some(OffsetDateTime::UNIX_EPOCH),
3007                        ..Default::default()
3008                    }),
3009                },
3010            ],
3011            ..Default::default()
3012        });
3013
3014        let err = store
3015            .handle_heal_bucket(
3016                "bucket",
3017                &HealOpts {
3018                    pool: Some(1),
3019                    ..Default::default()
3020                },
3021            )
3022            .await
3023            .expect_err("suspended pool must be blocked before bucket-heal fan-out");
3024        assert_eq!(err, Error::SlowDown);
3025
3026        store.pool_meta.write().await.pools[1]
3027            .decommission
3028            .as_mut()
3029            .expect("decommission state should exist")
3030            .complete = true;
3031        let err = store
3032            .handle_heal_bucket(
3033                "bucket",
3034                &HealOpts {
3035                    pool: Some(1),
3036                    ..Default::default()
3037                },
3038            )
3039            .await
3040            .expect_err("completed pool must remain fenced from bucket heal");
3041        assert!(
3042            matches!(err, StorageError::InvalidArgument(_, ref field, ref reason)
3043                if field == "pool" && reason.contains("completed decommission")),
3044            "unexpected completed-pool error: {err:?}"
3045        );
3046    }
3047
3048    #[tokio::test]
3049    async fn bucket_heal_blocks_before_dispatch_after_unreadable_pool_meta_replica() {
3050        let store = minimal_heal_store().await;
3051        store.pool_meta_save_gate.lock().await.observe_replicas(PoolMetaReplicaState {
3052            needs_repair: true,
3053            repair_write_safe: false,
3054        });
3055
3056        let err = store
3057            .handle_heal_bucket("bucket", &HealOpts::default())
3058            .await
3059            .expect_err("bucket heal must stay blocked until restart after an unreadable replica");
3060
3061        assert!(
3062            err.to_string()
3063                .contains("restart after all replicas are readable and consistent")
3064        );
3065    }
3066
3067    #[tokio::test]
3068    async fn bucket_heal_releases_save_gate_before_peer_dispatch_and_holds_movement_snapshot() {
3069        let mut store = minimal_heal_store().await;
3070        let started = Arc::new(tokio::sync::Notify::new());
3071        let release = Arc::new(tokio::sync::Notify::new());
3072        let peer: Box<dyn PeerS3Client> = Box::new(BlockingHealPeer {
3073            started: started.clone(),
3074            release: release.clone(),
3075        });
3076        store.peer_sys.clients = vec![Arc::new(peer)];
3077        let store = Arc::new(store);
3078        let movement_gate = store.ctx.data_movement_operation_gate();
3079        let mut heal = tokio::spawn({
3080            let store = store.clone();
3081            async move { store.handle_heal_bucket("bucket", &HealOpts::default()).await }
3082        });
3083
3084        tokio::time::timeout(std::time::Duration::from_secs(1), started.notified())
3085            .await
3086            .expect("peer dispatch should start");
3087        assert!(
3088            store.pool_meta_save_gate.try_lock().is_ok(),
3089            "coordinator must release its local save gate before waiting for peers"
3090        );
3091        assert!(
3092            movement_gate.try_write().is_err(),
3093            "bucket heal must hold the movement snapshot through peer dispatch"
3094        );
3095
3096        release.notify_one();
3097        tokio::time::timeout(std::time::Duration::from_secs(1), &mut heal)
3098            .await
3099            .expect("bucket heal should finish after peer release")
3100            .expect("bucket heal task should not panic")
3101            .expect("bucket heal should succeed");
3102    }
3103
3104    #[tokio::test]
3105    async fn bucket_heal_local_fanout_does_not_reenter_movement_read_behind_queued_writer() {
3106        let mut store = minimal_heal_store().await;
3107        let movement_gate = store.ctx.data_movement_operation_gate();
3108        let writer_queued = Arc::new(tokio::sync::Notify::new());
3109        let writer_acquired = Arc::new(tokio::sync::Notify::new());
3110        let peer: Box<dyn PeerS3Client> = Box::new(WriterQueuedLocalHealPeer {
3111            movement_gate: movement_gate.clone(),
3112            writer_queued: writer_queued.clone(),
3113            writer_acquired: writer_acquired.clone(),
3114        });
3115        store.peer_sys.clients = vec![Arc::new(peer)];
3116        let store = Arc::new(store);
3117        let mut heal = tokio::spawn({
3118            let store = store.clone();
3119            async move { store.handle_heal_bucket("bucket", &HealOpts::default()).await }
3120        });
3121
3122        tokio::time::timeout(std::time::Duration::from_secs(1), writer_queued.notified())
3123            .await
3124            .expect("movement writer should queue during local peer lookup");
3125        tokio::time::timeout(std::time::Duration::from_secs(1), &mut heal)
3126            .await
3127            .expect("local fan-out must not reenter movement read behind the queued writer")
3128            .expect("bucket heal task should not panic")
3129            .expect("bucket heal should succeed");
3130        tokio::time::timeout(std::time::Duration::from_secs(1), writer_acquired.notified())
3131            .await
3132            .expect("queued movement writer should proceed after bucket heal releases its read guard");
3133    }
3134
3135    #[tokio::test]
3136    #[serial_test::serial]
3137    async fn unscoped_heal_object_suspended_owner_semantics() {
3138        let (_temp_dir, store, shutdown) = multi_pool_heal_store().await;
3139        let bucket = format!("heal-owner-{}", Uuid::new_v4().simple());
3140        let active_object = "active-owner";
3141        let suspended_only_object = "suspended-only";
3142        let duplicate_object = "duplicate-owner";
3143        let marker_object = "marker-owner";
3144        let quorum_object = "quorum-owner";
3145        store
3146            .make_bucket(&bucket, &MakeBucketOptions::default())
3147            .await
3148            .expect("bucket should be created in all pools");
3149
3150        let mut active_reader = PutObjReader::from_vec(b"active owner".to_vec());
3151        store.pools[0]
3152            .put_object(
3153                &bucket,
3154                active_object,
3155                &mut active_reader,
3156                &ObjectOptions {
3157                    // Drain the rename tail before removing a physical shard.
3158                    write_completion: crate::object_api::WriteCompletion::TailDrained,
3159                    ..Default::default()
3160                },
3161            )
3162            .await
3163            .expect("active owner object should be written");
3164        let active_disks = store.pools[0].disk_set[0].disks.read().await.clone();
3165        let missing_active_disk = active_disks[0].clone().expect("active disk should be online");
3166        missing_active_disk
3167            .delete(
3168                &bucket,
3169                active_object,
3170                DeleteOptions {
3171                    recursive: true,
3172                    immediate: true,
3173                    ..Default::default()
3174                },
3175            )
3176            .await
3177            .expect("active owner shard should be removed for repair");
3178        assert!(
3179            missing_active_disk.read_xl(&bucket, active_object, false).await.is_err(),
3180            "the active owner fixture must start with one missing metadata copy"
3181        );
3182
3183        let mut suspended_reader = PutObjReader::from_vec(b"suspended owner".to_vec());
3184        store.pools[1]
3185            .put_object(&bucket, suspended_only_object, &mut suspended_reader, &ObjectOptions::default())
3186            .await
3187            .expect("suspended owner object should be written");
3188        for (pool_index, mod_time) in [1_i64, 2_i64].into_iter().enumerate() {
3189            let mut duplicate_reader = PutObjReader::from_vec(format!("duplicate-pool-{pool_index}").into_bytes());
3190            store.pools[pool_index]
3191                .put_object(
3192                    &bucket,
3193                    duplicate_object,
3194                    &mut duplicate_reader,
3195                    &ObjectOptions {
3196                        mod_time: Some(OffsetDateTime::UNIX_EPOCH + time::Duration::seconds(mod_time)),
3197                        write_completion: crate::object_api::WriteCompletion::TailDrained,
3198                        ..Default::default()
3199                    },
3200                )
3201                .await
3202                .expect("duplicate owner object should be written");
3203        }
3204        let duplicate_missing_disk = store.pools[0].disk_set[0].disks.read().await[0]
3205            .clone()
3206            .expect("duplicate active owner disk should be online");
3207        duplicate_missing_disk
3208            .delete(
3209                &bucket,
3210                duplicate_object,
3211                DeleteOptions {
3212                    recursive: true,
3213                    immediate: true,
3214                    ..Default::default()
3215                },
3216            )
3217            .await
3218            .expect("duplicate active owner shard should be removed for repair");
3219        let history_version = Uuid::new_v4();
3220        let mut history_reader = PutObjReader::from_vec(b"marker history".to_vec());
3221        store.pools[0]
3222            .put_object(
3223                &bucket,
3224                marker_object,
3225                &mut history_reader,
3226                &ObjectOptions {
3227                    versioned: true,
3228                    version_id: Some(history_version.to_string()),
3229                    mod_time: Some(OffsetDateTime::UNIX_EPOCH + time::Duration::seconds(1)),
3230                    ..Default::default()
3231                },
3232            )
3233            .await
3234            .expect("versioned marker history should be written");
3235        store.pools[0]
3236            .delete_object(
3237                &bucket,
3238                marker_object,
3239                ObjectOptions {
3240                    versioned: true,
3241                    mod_time: Some(OffsetDateTime::UNIX_EPOCH + time::Duration::seconds(2)),
3242                    ..Default::default()
3243                },
3244            )
3245            .await
3246            .expect("delete marker should be written");
3247        let mut quorum_reader = PutObjReader::from_vec(b"quorum boundary".to_vec());
3248        store.pools[0]
3249            .put_object(&bucket, quorum_object, &mut quorum_reader, &ObjectOptions::default())
3250            .await
3251            .expect("quorum boundary object should be written");
3252        {
3253            let mut pool_meta = store.pool_meta.write().await;
3254            let mut next = PoolMeta::new(&store.pools, &pool_meta);
3255            next.pools[1].decommission = Some(PoolDecommissionInfo {
3256                start_time: Some(OffsetDateTime::UNIX_EPOCH),
3257                ..Default::default()
3258            });
3259            *pool_meta = next;
3260        }
3261
3262        let (_, duplicate_owner) = store
3263            .get_latest_object_info_with_idx(&bucket, duplicate_object, &ObjectOptions::default())
3264            .await
3265            .expect("duplicate owner should resolve");
3266        assert_eq!(duplicate_owner, 1, "latest duplicate must win when all pools are eligible");
3267        let (_, active_duplicate_owner) = store
3268            .get_latest_object_info_with_idx(
3269                &bucket,
3270                duplicate_object,
3271                &ObjectOptions {
3272                    skip_decommissioned: true,
3273                    ..Default::default()
3274                },
3275            )
3276            .await
3277            .expect("active duplicate owner should resolve");
3278        assert_eq!(
3279            active_duplicate_owner, 0,
3280            "suspended duplicate must be excluded from active owner selection"
3281        );
3282        let (duplicate_result, duplicate_err) = store
3283            .handle_heal_object(&bucket, duplicate_object, "", &HealOpts::default())
3284            .await
3285            .expect("duplicate owner heal should complete through the production path");
3286        assert_eq!(duplicate_result.object, duplicate_object);
3287        assert!(duplicate_err.is_none(), "active duplicate should be repaired: {duplicate_err:?}");
3288        assert!(
3289            duplicate_missing_disk.read_xl(&bucket, duplicate_object, false).await.is_ok(),
3290            "production heal must repair the active duplicate owner rather than the suspended owner"
3291        );
3292        let (marker_info, marker_owner) = store
3293            .get_latest_object_info_with_idx(
3294                &bucket,
3295                marker_object,
3296                &ObjectOptions {
3297                    skip_decommissioned: true,
3298                    versioned: true,
3299                    ..Default::default()
3300                },
3301            )
3302            .await
3303            .expect("latest delete marker should resolve");
3304        assert_eq!(marker_owner, 0);
3305        assert!(marker_info.delete_marker, "latest version must preserve delete-marker semantics");
3306
3307        let (active_result, active_err) = store
3308            .handle_heal_object(&bucket, active_object, "", &HealOpts::default())
3309            .await
3310            .expect("unscoped active-owner heal should complete");
3311        assert_eq!(active_result.object, active_object);
3312        assert!(active_err.is_none(), "active owner must be selected even with a suspended pool");
3313        assert!(
3314            missing_active_disk.read_xl(&bucket, active_object, false).await.is_ok(),
3315            "active owner heal must write the missing disk metadata: result={active_result:?}, err={active_err:?}"
3316        );
3317        assert!(
3318            store.pools[1]
3319                .get_object_info(&bucket, active_object, &ObjectOptions::default())
3320                .await
3321                .is_err(),
3322            "the suspended pool must not be written for an active-owner object"
3323        );
3324
3325        let (suspended_result, suspended_err) = store
3326            .handle_heal_object(&bucket, suspended_only_object, "", &HealOpts::default())
3327            .await
3328            .expect("unscoped suspended-only heal should return a terminal result");
3329        assert!(suspended_result.object.is_empty());
3330        assert!(matches!(suspended_err, Some(Error::FileNotFound)));
3331        assert!(
3332            store.pools[1]
3333                .get_object_info(&bucket, suspended_only_object, &ObjectOptions::default())
3334                .await
3335                .is_ok(),
3336            "suspended-only data must remain untouched when unscoped heal reports absent"
3337        );
3338
3339        let (_, explicit_err) = store
3340            .handle_heal_object(
3341                &bucket,
3342                suspended_only_object,
3343                "",
3344                &HealOpts {
3345                    pool: Some(1),
3346                    ..Default::default()
3347                },
3348            )
3349            .await
3350            .expect("explicit suspended-owner heal should return a mapped error");
3351        assert!(matches!(explicit_err, Some(Error::SlowDown)));
3352
3353        let original_quorum_disks = store.pools[0].disk_set[0].disks.read().await.clone();
3354        let surviving_quorum_disk = original_quorum_disks[3].clone();
3355        *store.pools[0].disk_set[0].disks.write().await = vec![None, None, None, surviving_quorum_disk];
3356        let (_, quorum_err) = store
3357            .handle_heal_object(&bucket, quorum_object, "", &HealOpts::default())
3358            .await
3359            .expect("quorum boundary heal should return a mapped result");
3360        *store.pools[0].disk_set[0].disks.write().await = original_quorum_disks;
3361        let quorum_err = quorum_err
3362            .as_ref()
3363            .expect("heal must fail closed when capacity admission cannot verify pool metadata");
3364        let quorum_failure = quorum_err
3365            .pool_metadata_failure()
3366            .expect("capacity admission failure should preserve typed pool metadata context");
3367        assert_eq!(
3368            quorum_failure.kind,
3369            crate::error::PoolMetadataFailure::ReadUnavailable,
3370            "read-only capacity admission failure must remain retryable"
3371        );
3372        assert_eq!(quorum_failure.operation, "target capacity admission failed");
3373        assert_eq!(quorum_failure.phase, "pool_read");
3374        assert!(
3375            store.pool_meta_writes_ready().await,
3376            "read-only capacity admission failure must not latch the pool metadata writer"
3377        );
3378        shutdown.cancel();
3379    }
3380
3381    #[tokio::test]
3382    async fn handle_heal_format_continues_after_a_pool_error() {
3383        let canonical_format = FormatV3::new(1, 3);
3384        let mut foreign_format = canonical_format.clone();
3385        foreign_format.id = Uuid::new_v4();
3386        let mut temp_dirs = Vec::new();
3387        let mut endpoints = Vec::new();
3388        let mut disks = Vec::new();
3389
3390        for disk_index in 0..3 {
3391            let temp_dir = tempfile::tempdir().expect("temporary disk root should be created");
3392            let mut endpoint = Endpoint::try_from(temp_dir.path().to_str().expect("temporary path should be UTF-8"))
3393                .expect("temporary endpoint should parse");
3394            endpoint.set_pool_index(0);
3395            endpoint.set_set_index(0);
3396            endpoint.set_disk_index(disk_index);
3397            let disk = new_disk(
3398                &endpoint,
3399                &DiskOption {
3400                    cleanup: false,
3401                    health_check: false,
3402                },
3403            )
3404            .await
3405            .expect("temporary disk should open");
3406            let mut disk_format = foreign_format.clone();
3407            disk_format.erasure.this = foreign_format.erasure.sets[0][disk_index];
3408            save_format_file(&Some(disk.clone()), &Some(disk_format))
3409                .await
3410                .expect("foreign format should be written");
3411            temp_dirs.push(temp_dir);
3412            endpoints.push(endpoint);
3413            disks.push(Some(disk));
3414        }
3415
3416        let pool_endpoints = PoolEndpoints {
3417            legacy: false,
3418            set_count: 1,
3419            drives_per_set: 3,
3420            endpoints: Endpoints::from(endpoints),
3421            cmd_line: "foreign-format-majority-test".to_string(),
3422            platform: "test".to_string(),
3423        };
3424        let pool = Sets::new(disks, &pool_endpoints, &canonical_format, 0, 1)
3425            .await
3426            .expect("test pool should build around the cached canonical format");
3427
3428        let mut recoverable_format = FormatV3::new(1, 3);
3429        recoverable_format.id = canonical_format.id;
3430        let mut recoverable_temp_dirs = Vec::new();
3431        let mut recoverable_endpoints = Vec::new();
3432        let mut recoverable_disks = Vec::new();
3433        let mut unformatted_disk = None;
3434        for disk_index in 0..3 {
3435            let temp_dir = tempfile::tempdir().expect("temporary disk root should be created");
3436            let mut endpoint = Endpoint::try_from(temp_dir.path().to_str().expect("temporary path should be UTF-8"))
3437                .expect("temporary endpoint should parse");
3438            endpoint.set_pool_index(1);
3439            endpoint.set_set_index(0);
3440            endpoint.set_disk_index(disk_index);
3441            let disk = new_disk(
3442                &endpoint,
3443                &DiskOption {
3444                    cleanup: false,
3445                    health_check: false,
3446                },
3447            )
3448            .await
3449            .expect("temporary disk should open");
3450            if disk_index < 2 {
3451                let mut disk_format = recoverable_format.clone();
3452                disk_format.erasure.this = recoverable_format.erasure.sets[0][disk_index];
3453                save_format_file(&Some(disk.clone()), &Some(disk_format))
3454                    .await
3455                    .expect("recoverable format should be written");
3456            } else {
3457                unformatted_disk = Some(disk.clone());
3458            }
3459            recoverable_temp_dirs.push(temp_dir);
3460            recoverable_endpoints.push(endpoint);
3461            recoverable_disks.push(Some(disk));
3462        }
3463        let recoverable_pool_endpoints = PoolEndpoints {
3464            legacy: false,
3465            set_count: 1,
3466            drives_per_set: 3,
3467            endpoints: Endpoints::from(recoverable_endpoints),
3468            cmd_line: "recoverable-format-test".to_string(),
3469            platform: "test".to_string(),
3470        };
3471        let recoverable_pool = Sets::new(recoverable_disks, &recoverable_pool_endpoints, &recoverable_format, 1, 1)
3472            .await
3473            .expect("recoverable test pool should build");
3474
3475        let endpoint_pools = EndpointServerPools::from(vec![pool_endpoints.clone(), recoverable_pool_endpoints.clone()]);
3476        let store = ECStore {
3477            id: canonical_format.id,
3478            disk_map: HashMap::new(),
3479            pools: vec![pool, recoverable_pool],
3480            peer_sys: S3PeerSys::new(&endpoint_pools),
3481            pool_meta: RwLock::new(PoolMeta::default()),
3482            rebalance_meta: RwLock::new(None),
3483            decommission_cancelers: RwLock::new(Vec::new()),
3484            start_gate: Mutex::new(()),
3485            pool_meta_save_gate: Mutex::default(),
3486            ctx: crate::runtime::instance::bootstrap_ctx(),
3487            bucket_fence_registry: std::sync::Arc::default(),
3488        };
3489
3490        let err = store
3491            .handle_heal_format(false)
3492            .await
3493            .expect_err("missing pool metadata must fail closed before format writes");
3494        assert!(err.to_string().contains("no durable bootstrap identity or pool.bin replica"));
3495        store
3496            .ensure_pool_meta_side_effects_safe("missing format-heal metadata side effect")
3497            .await
3498            .expect_err("missing metadata must latch the format-heal write gate");
3499
3500        let pool_meta = PoolMeta::new(&store.pools, &PoolMeta::default());
3501        pool_meta
3502            .save_for_startup(store.pools.clone())
3503            .await
3504            .expect("pool metadata should be persisted before format heal");
3505        *store.pool_meta_save_gate.lock().await = PoolMetaWriteState::default();
3506
3507        let (result, err) = store
3508            .handle_heal_format(false)
3509            .await
3510            .expect("format heal should return the typed pool error");
3511        assert!(
3512            matches!(err, Some(StorageError::CorruptedFormat)),
3513            "foreign format majority must not be downgraded to a successful heal: {err:?}"
3514        );
3515        assert_eq!(result.disk_count, 3, "the recoverable pool should still be inspected");
3516        let healed = load_format_erasure(&unformatted_disk.expect("the unformatted disk handle should be retained"), true)
3517            .await
3518            .expect("the later pool should be healed despite the first pool error");
3519        assert_eq!(healed.erasure.this, recoverable_format.erasure.sets[0][2]);
3520
3521        let mut completed_meta = PoolMeta::new(&store.pools, &PoolMeta::default());
3522        for status in &mut completed_meta.pools {
3523            status.decommission = Some(PoolDecommissionInfo {
3524                complete: true,
3525                ..Default::default()
3526            });
3527        }
3528        completed_meta
3529            .save(store.pools.clone())
3530            .await
3531            .expect("completed pool metadata should be persisted");
3532        let (_, err) = store
3533            .handle_heal_format(false)
3534            .await
3535            .expect("completed pools should be reported as a no-op");
3536        assert!(matches!(err, Some(StorageError::NoHealRequired)));
3537    }
3538}